Compare commits
18
Commits
prod
...
d88948a1c3
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d88948a1c3 | ||
|
|
7d360babed | ||
|
|
803a94be37 | ||
|
|
43ed8d4834 | ||
|
|
1c721ede41 | ||
|
|
bfdbc937c6 | ||
|
|
1cafaeb953 | ||
|
|
515c359e56 | ||
|
|
30e65dce53 | ||
|
|
1ac4fe0be4 | ||
|
|
a56f1fe0c8 | ||
|
|
9d29baaa2d | ||
|
|
507980a347 | ||
|
|
b79e4649a1 | ||
|
|
1428154bbd | ||
|
|
8ad2a86e4b | ||
|
|
a891b726e5 | ||
|
|
ab63b5f777 |
+2
-4
@@ -3,10 +3,8 @@
|
|||||||
# stage of the Dockerfile.
|
# stage of the Dockerfile.
|
||||||
.git/
|
.git/
|
||||||
bin/
|
bin/
|
||||||
# Third-party browser assets are fetched and hash-verified inside the build by
|
# Extracted from 3p/ by `make assets` inside the build; a host copy is not
|
||||||
# script/fetch-assets. Excluding any host copy keeps a developer's working tree
|
# needed. The tarball in 3p/ must stay in the context.
|
||||||
# from supplying the bytes that get shipped. The script and its
|
|
||||||
# static/vendor.sha256 manifest stay in the context.
|
|
||||||
static/js/alpine.min.js
|
static/js/alpine.min.js
|
||||||
*.md
|
*.md
|
||||||
LICENSE
|
LICENSE
|
||||||
|
|||||||
+2
-3
@@ -46,7 +46,6 @@ temp/
|
|||||||
# CI cache barrier, written into the build context by the check workflow
|
# CI cache barrier, written into the build context by the check workflow
|
||||||
.ci-fingerprint
|
.ci-fingerprint
|
||||||
|
|
||||||
# Third-party browser assets, fetched and hash-verified by
|
# Alpine.js, extracted by `make assets` from its tarball in 3p/, which is
|
||||||
# script/fetch-assets against static/vendor.sha256. Not committed:
|
# what is committed.
|
||||||
# REPO_POLICIES.md forbids minified bundles in version control.
|
|
||||||
/static/js/alpine.min.js
|
/static/js/alpine.min.js
|
||||||
Binary file not shown.
+13
-13
@@ -51,15 +51,8 @@ RUN go mod download
|
|||||||
# the lint stage above.
|
# the lint stage above.
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
# Fetch the third-party browser assets the UI serves. They are not committed
|
# Run tests and build. Both first run script/assets, which extracts Alpine.js
|
||||||
# (REPO_POLICIES.md forbids minified bundles in version control) and
|
# from its tarball in 3p/.
|
||||||
# .dockerignore keeps any host copy out of the build context, so this step is
|
|
||||||
# the only way they enter the image. Each download is checked against a
|
|
||||||
# hardcoded sha256 and the build fails on mismatch; make test re-checks the
|
|
||||||
# hashes against the bytes go:embed actually put in the binary.
|
|
||||||
RUN script/fetch-assets
|
|
||||||
|
|
||||||
# Run tests and build
|
|
||||||
RUN make test
|
RUN make test
|
||||||
|
|
||||||
# Version stamped into the binary. .dockerignore excludes .git/, so
|
# Version stamped into the binary. .dockerignore excludes .git/, so
|
||||||
@@ -67,8 +60,8 @@ RUN make test
|
|||||||
# host and passes it in. The default is what a bare `docker build .`
|
# host and passes it in. The default is what a bare `docker build .`
|
||||||
# with no --build-arg gets, and it names no tag the tree may not be at.
|
# with no --build-arg gets, and it names no tag the tree may not be at.
|
||||||
#
|
#
|
||||||
# Declared here, below the test and asset steps, so a changed version
|
# Declared here, below the test step, so a changed version does not
|
||||||
# does not invalidate their cached layers.
|
# invalidate its cached layer.
|
||||||
ARG VERSION=unknown
|
ARG VERSION=unknown
|
||||||
|
|
||||||
RUN make build VERSION="$VERSION"
|
RUN make build VERSION="$VERSION"
|
||||||
@@ -88,7 +81,9 @@ RUN CGO_ENABLED=1 make build VERSION="$VERSION" GO_LDFLAGS='-extldflags "-static
|
|||||||
# alpine:3.21, 2026-03-17
|
# alpine:3.21, 2026-03-17
|
||||||
FROM alpine:3.21@sha256:c3f8e73fdb79deaebaa2037150150191b9dcbfba68b4a46d70103204c53f4709
|
FROM alpine:3.21@sha256:c3f8e73fdb79deaebaa2037150150191b9dcbfba68b4a46d70103204c53f4709
|
||||||
|
|
||||||
RUN apk --no-cache add ca-certificates
|
# su-exec 0.2-r3 (Alpine 3.21), 2026-09-29: the entrypoint runs the app
|
||||||
|
# as webhooker with it.
|
||||||
|
RUN apk --no-cache add ca-certificates su-exec=0.2-r3
|
||||||
|
|
||||||
# Create non-root user
|
# Create non-root user
|
||||||
RUN addgroup -g 1000 -S webhooker && \
|
RUN addgroup -g 1000 -S webhooker && \
|
||||||
@@ -99,13 +94,17 @@ WORKDIR /app
|
|||||||
# Copy binary from builder
|
# Copy binary from builder
|
||||||
COPY --from=builder /build/bin/webhooker /app/webhooker
|
COPY --from=builder /build/bin/webhooker /app/webhooker
|
||||||
|
|
||||||
|
# Not under /app, which belongs to webhooker: this script runs as root.
|
||||||
|
COPY deploy/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
|
||||||
|
|
||||||
# Create data directory for all SQLite databases (main app DB +
|
# Create data directory for all SQLite databases (main app DB +
|
||||||
# per-webhook event DBs). DATA_DIR defaults to /var/lib/webhooker.
|
# per-webhook event DBs). DATA_DIR defaults to /var/lib/webhooker.
|
||||||
RUN mkdir -p /var/lib/webhooker
|
RUN mkdir -p /var/lib/webhooker
|
||||||
|
|
||||||
RUN chown -R webhooker:webhooker /app /var/lib/webhooker
|
RUN chown -R webhooker:webhooker /app /var/lib/webhooker
|
||||||
|
|
||||||
USER webhooker
|
# No USER: the entrypoint starts as root to make the data directory
|
||||||
|
# webhooker's, then runs the app as webhooker.
|
||||||
|
|
||||||
EXPOSE 8080
|
EXPOSE 8080
|
||||||
|
|
||||||
@@ -124,4 +123,5 @@ ENV BIND_ADDRESS=0.0.0.0
|
|||||||
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
|
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
|
||||||
CMD wget --no-verbose --tries=1 --spider http://localhost:8080/.well-known/healthcheck || exit 1
|
CMD wget --no-verbose --tries=1 --spider http://localhost:8080/.well-known/healthcheck || exit 1
|
||||||
|
|
||||||
|
ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
|
||||||
CMD ["/app/webhooker"]
|
CMD ["/app/webhooker"]
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ setup:
|
|||||||
@script/setup
|
@script/setup
|
||||||
|
|
||||||
assets:
|
assets:
|
||||||
@script/fetch-assets
|
@script/assets
|
||||||
|
|
||||||
test:
|
test:
|
||||||
@script/test
|
@script/test
|
||||||
@@ -45,13 +45,13 @@ fmt-check:
|
|||||||
check:
|
check:
|
||||||
@script/check
|
@script/check
|
||||||
|
|
||||||
build:
|
build: assets
|
||||||
go build -ldflags '$(strip -X main.version=$(VERSION) $(GO_LDFLAGS))' -o bin/webhooker ./cmd/webhooker
|
go build -ldflags '$(strip -X main.version=$(VERSION) $(GO_LDFLAGS))' -o bin/webhooker ./cmd/webhooker
|
||||||
|
|
||||||
run: build
|
run: build
|
||||||
./bin/webhooker
|
./bin/webhooker
|
||||||
|
|
||||||
dev:
|
dev: assets
|
||||||
go run ./cmd/webhooker
|
go run ./cmd/webhooker
|
||||||
|
|
||||||
deps:
|
deps:
|
||||||
|
|||||||
@@ -387,7 +387,7 @@ point of the branch.
|
|||||||
- 2026-03-05 security headers middleware, session regeneration on
|
- 2026-03-05 security headers middleware, session regeneration on
|
||||||
login, request body size limits (#41)
|
login, request body size limits (#41)
|
||||||
- 2026-03-04 tests for delivery, middleware, and session packages
|
- 2026-03-04 tests for delivery, middleware, and session packages
|
||||||
(#32); removed globals.Buildarch (#31)
|
(#32); removed the build-architecture global (#31)
|
||||||
- 2026-03-04 1.0 MVP merge: Webhook/Entrypoint/Target rename, core
|
- 2026-03-04 1.0 MVP merge: Webhook/Entrypoint/Target rename, core
|
||||||
delivery engine with bounded worker pool and circuit breaker,
|
delivery engine with bounded worker pool and circuit breaker,
|
||||||
parallel fan-out, per-webhook event databases, management UI (#16)
|
parallel fan-out, per-webhook event databases, management UI (#16)
|
||||||
|
|||||||
@@ -187,11 +187,10 @@ func newApp() *fx.App {
|
|||||||
// Wire *delivery.Engine as delivery.Notifier so the
|
// Wire *delivery.Engine as delivery.Notifier so the
|
||||||
// webhook handler can notify the engine of new deliveries.
|
// webhook handler can notify the engine of new deliveries.
|
||||||
func(e *delivery.Engine) delivery.Notifier { return e },
|
func(e *delivery.Engine) delivery.Notifier { return e },
|
||||||
// Wire *delivery.Engine as delivery.WebhookEvictor so
|
// Wire *delivery.Engine as delivery.Archives so deleting
|
||||||
// deleting a webhook releases its archive writer.
|
// or renaming a webhook or target reaches its archive
|
||||||
func(e *delivery.Engine) delivery.WebhookEvictor {
|
// files.
|
||||||
return e
|
func(e *delivery.Engine) delivery.Archives { return e },
|
||||||
},
|
|
||||||
server.New,
|
server.New,
|
||||||
),
|
),
|
||||||
fx.Invoke(
|
fx.Invoke(
|
||||||
|
|||||||
Executable
+22
@@ -0,0 +1,22 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# deploy/docker-entrypoint.sh: the image's ENTRYPOINT. A bind-mounted
|
||||||
|
# data directory keeps its owner from the host, often root, and the app
|
||||||
|
# could not write to it. Started as root, this creates DATA_DIR if
|
||||||
|
# needed, gives it and everything in it to webhooker, sets its mode, and
|
||||||
|
# runs the command as webhooker, so the app never runs as root. Started
|
||||||
|
# as another user, it only runs the command.
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
main() {
|
||||||
|
if [ "$(id -u)" != 0 ]; then
|
||||||
|
exec "$@"
|
||||||
|
fi
|
||||||
|
|
||||||
|
dir="${DATA_DIR:-/var/lib/webhooker}"
|
||||||
|
mkdir -p "$dir"
|
||||||
|
find "$dir" ! -user webhooker -exec chown -h webhooker:webhooker {} +
|
||||||
|
chmod 750 "$dir"
|
||||||
|
exec su-exec webhooker "$@"
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
@@ -4,6 +4,7 @@ go 1.26.1
|
|||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/99designs/basicauth-go v0.0.0-20230316000542-bf6f9cbbf0f8
|
github.com/99designs/basicauth-go v0.0.0-20230316000542-bf6f9cbbf0f8
|
||||||
|
github.com/dustin/go-humanize v1.0.1
|
||||||
github.com/getsentry/sentry-go v0.25.0
|
github.com/getsentry/sentry-go v0.25.0
|
||||||
github.com/go-chi/chi v1.5.5
|
github.com/go-chi/chi v1.5.5
|
||||||
github.com/go-chi/cors v1.2.1
|
github.com/go-chi/cors v1.2.1
|
||||||
@@ -29,7 +30,6 @@ require (
|
|||||||
github.com/beorn7/perks v1.0.1 // indirect
|
github.com/beorn7/perks v1.0.1 // indirect
|
||||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
|
||||||
github.com/gorilla/securecookie v1.1.2 // indirect
|
github.com/gorilla/securecookie v1.1.2 // indirect
|
||||||
github.com/jinzhu/inflection v1.0.0 // indirect
|
github.com/jinzhu/inflection v1.0.0 // indirect
|
||||||
github.com/jinzhu/now v1.1.5 // indirect
|
github.com/jinzhu/now v1.1.5 // indirect
|
||||||
|
|||||||
+22
-60
@@ -75,6 +75,11 @@ const (
|
|||||||
// internet-exposed endpoint.
|
// internet-exposed endpoint.
|
||||||
defaultReceiverRateLimit = 120
|
defaultReceiverRateLimit = 120
|
||||||
|
|
||||||
|
// defaultTrustedProxies is TRUSTED_PROXIES when it is unset: the
|
||||||
|
// RFC 1918 private ranges, which a reverse proxy reaching the
|
||||||
|
// process over a Docker network or a private LAN connects from.
|
||||||
|
defaultTrustedProxies = "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16"
|
||||||
|
|
||||||
// maxPort is the highest valid TCP port number. The lower
|
// maxPort is the highest valid TCP port number. The lower
|
||||||
// bound (at least 1) is enforced by envPositiveInt.
|
// bound (at least 1) is enforced by envPositiveInt.
|
||||||
maxPort = 65535
|
maxPort = 65535
|
||||||
@@ -172,13 +177,14 @@ type Config struct {
|
|||||||
|
|
||||||
// TrustedProxies is the set of networks whose members are
|
// TrustedProxies is the set of networks whose members are
|
||||||
// allowed to speak for the client with X-Forwarded-For, the
|
// allowed to speak for the client with X-Forwarded-For, the
|
||||||
// only forwarded header read. It is empty unless
|
// only forwarded header read. Unless TRUSTED_PROXIES is set it
|
||||||
// TRUSTED_PROXIES is set, and empty means no peer is
|
// is the RFC 1918 private ranges (defaultTrustedProxies); a set
|
||||||
// trusted: forwarded headers are then ignored entirely and
|
// value replaces them. If any client can reach the process, or
|
||||||
// clients are identified by the connection's own address.
|
// the proxy in front of it, from an RFC 1918 source address
|
||||||
// Members can choose their own rate-limit key, so this must
|
// (directly, or through anything that can rewrite source
|
||||||
// name proxy hosts only, never a block that also covers
|
// addresses, such as NAT or a published container port), it
|
||||||
// clients.
|
// must be set to the proxy's address alone, or every rate limit
|
||||||
|
// can be bypassed by those clients.
|
||||||
TrustedProxies []netip.Prefix
|
TrustedProxies []netip.Prefix
|
||||||
|
|
||||||
// AllowedEgressCIDRs is the set of networks a delivery target
|
// AllowedEgressCIDRs is the set of networks a delivery target
|
||||||
@@ -460,14 +466,15 @@ func parseCIDR(entry string) (netip.Prefix, error) {
|
|||||||
|
|
||||||
// envPrefixList returns the value of the named environment variable
|
// envPrefixList returns the value of the named environment variable
|
||||||
// parsed as a comma-separated list of CIDR blocks (bare addresses
|
// parsed as a comma-separated list of CIDR blocks (bare addresses
|
||||||
// allowed). An unset, empty, or blank value yields an empty list. A
|
// allowed). An unset, empty, or blank value is read as defaultValue
|
||||||
// set value containing an unparseable entry is a hard error naming
|
// instead. A set value containing an unparseable entry is a hard
|
||||||
// the key and the bad entry, so startup fails loudly rather than
|
// error naming the key and the bad entry, so startup fails loudly
|
||||||
// silently running with a list the operator did not intend.
|
// rather than silently running with a list the operator did not
|
||||||
func envPrefixList(key string) ([]netip.Prefix, error) {
|
// intend.
|
||||||
|
func envPrefixList(key, defaultValue string) ([]netip.Prefix, error) {
|
||||||
v := strings.TrimSpace(os.Getenv(key))
|
v := strings.TrimSpace(os.Getenv(key))
|
||||||
if v == "" {
|
if v == "" {
|
||||||
return nil, nil
|
v = defaultValue
|
||||||
}
|
}
|
||||||
|
|
||||||
var prefixes []netip.Prefix
|
var prefixes []netip.Prefix
|
||||||
@@ -681,12 +688,12 @@ func loadFromEnv() (*Config, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
trustedProxies, err := envPrefixList("TRUSTED_PROXIES")
|
trustedProxies, err := envPrefixList("TRUSTED_PROXIES", defaultTrustedProxies)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
allowedEgressCIDRs, err := envPrefixList("ALLOWED_EGRESS_CIDRS")
|
allowedEgressCIDRs, err := envPrefixList("ALLOWED_EGRESS_CIDRS", "")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -760,50 +767,6 @@ func (c *Config) warnEgressAllowlist(log *slog.Logger) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// warnSharedRateLimitBucket logs a startup warning whenever
|
|
||||||
// TRUSTED_PROXIES is empty, in any environment.
|
|
||||||
//
|
|
||||||
// With no trusted proxies every rate limiter keys on the connecting
|
|
||||||
// peer's address. Whether that is harmless or dangerous depends on
|
|
||||||
// what is in front of the process, which this code cannot observe:
|
|
||||||
// with nothing in front, the peer is the client and the limits are
|
|
||||||
// per-client as intended; behind a reverse proxy the peer is the proxy
|
|
||||||
// for every request, so all clients share one bucket per limiter.
|
|
||||||
//
|
|
||||||
// The login endpoint no longer spends budget on arrival — it verifies
|
|
||||||
// credentials first and charges only failures — so a shared bucket
|
|
||||||
// cannot deny the operator a correct password. What it does collapse
|
|
||||||
// is the failure counting: one client's wrong passwords throttle
|
|
||||||
// everyone else's wrong passwords, and the receiver's limits become
|
|
||||||
// service-wide ceilings.
|
|
||||||
//
|
|
||||||
// The warning is deliberately not gated on WEBHOOKER_ENVIRONMENT:
|
|
||||||
// behind a proxy every client shares one bucket in dev and prod alike.
|
|
||||||
//
|
|
||||||
// The default of trusting nobody is deliberate — trusting forwarded
|
|
||||||
// headers from arbitrary peers lets any client choose its own bucket —
|
|
||||||
// so this warns rather than failing startup or changing the key.
|
|
||||||
func (c *Config) warnSharedRateLimitBucket(log *slog.Logger) {
|
|
||||||
if len(c.TrustedProxies) > 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Warn(
|
|
||||||
"TRUSTED_PROXIES is empty: every rate limit keys on the "+
|
|
||||||
"connecting peer's address. With nothing proxying to "+
|
|
||||||
"this process that is the client itself and the limits "+
|
|
||||||
"are per-client as intended. Behind a reverse proxy the "+
|
|
||||||
"peer is the proxy on every request, so all clients "+
|
|
||||||
"share one bucket per limit: the receiver limits become "+
|
|
||||||
"service-wide ceilings, and one client's failed logins "+
|
|
||||||
"throttle every other client's failed logins — a "+
|
|
||||||
"correct password still gets in. If anything proxies to "+
|
|
||||||
"this process, set TRUSTED_PROXIES to its address.",
|
|
||||||
"environment", c.Environment,
|
|
||||||
"trustedProxies", len(c.TrustedProxies),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// New creates a Config by reading environment variables.
|
// New creates a Config by reading environment variables.
|
||||||
//
|
//
|
||||||
//nolint:revive // lc parameter is required by fx even if unused.
|
//nolint:revive // lc parameter is required by fx even if unused.
|
||||||
@@ -849,7 +812,6 @@ func New(lc fx.Lifecycle, params ConfigParams) (*Config, error) {
|
|||||||
"hasMetricsAuth", s.MetricsAuthEnabled(),
|
"hasMetricsAuth", s.MetricsAuthEnabled(),
|
||||||
)
|
)
|
||||||
|
|
||||||
s.warnSharedRateLimitBucket(log)
|
|
||||||
s.warnEgressAllowlist(log)
|
s.warnEgressAllowlist(log)
|
||||||
|
|
||||||
return s, nil
|
return s, nil
|
||||||
|
|||||||
+14
-101
@@ -551,6 +551,11 @@ func testReceiverRateLimitSuccess(
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestTrustedProxies(t *testing.T) {
|
func TestTrustedProxies(t *testing.T) {
|
||||||
|
// Unset, the RFC 1918 private ranges are trusted, so a reverse
|
||||||
|
// proxy on a Docker network or a private LAN is covered without
|
||||||
|
// configuration.
|
||||||
|
defaultProxies := []string{cidrPrivateV4, "172.16.0.0/12", "192.168.0.0/16"}
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
set bool
|
set bool
|
||||||
@@ -559,18 +564,21 @@ func TestTrustedProxies(t *testing.T) {
|
|||||||
expected []string
|
expected []string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
// The default must be "trust nobody": an empty list
|
|
||||||
// means forwarded headers are ignored, never that
|
|
||||||
// every peer may speak for the client.
|
|
||||||
name: caseUnsetUsesDefault,
|
name: caseUnsetUsesDefault,
|
||||||
set: false,
|
set: false,
|
||||||
expected: []string{},
|
expected: defaultProxies,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "blank value trusts nothing",
|
name: "blank value uses default",
|
||||||
set: true,
|
set: true,
|
||||||
value: " ",
|
value: " ",
|
||||||
expected: []string{},
|
expected: defaultProxies,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "set value replaces the default entirely",
|
||||||
|
set: true,
|
||||||
|
value: "203.0.113.7",
|
||||||
|
expected: []string{"203.0.113.7/32"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: caseValidValueParsed,
|
name: caseValidValueParsed,
|
||||||
@@ -845,101 +853,6 @@ func TestEgressAllowlistWarning(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestSharedRateLimitBucketWarning covers the startup warning that
|
|
||||||
// tells an operator a deployment behind a reverse proxy shares one
|
|
||||||
// rate-limit bucket between every client, which turns the receiver
|
|
||||||
// limits into service-wide ceilings and collapses login failure
|
|
||||||
// counting. It must fire whenever TRUSTED_PROXIES is empty, in any
|
|
||||||
// environment, because behind a proxy every client shares one bucket
|
|
||||||
// in dev and prod alike. It stays quiet once proxies are named.
|
|
||||||
func TestSharedRateLimitBucketWarning(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
environment string
|
|
||||||
trustedProxies string
|
|
||||||
expectWarning bool
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "prod without trusted proxies warns",
|
|
||||||
environment: config.EnvironmentProd,
|
|
||||||
expectWarning: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "prod with trusted proxies is quiet",
|
|
||||||
environment: config.EnvironmentProd,
|
|
||||||
trustedProxies: cidrPrivateV4,
|
|
||||||
expectWarning: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "dev without trusted proxies warns",
|
|
||||||
environment: config.EnvironmentDev,
|
|
||||||
expectWarning: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "dev with trusted proxies is quiet",
|
|
||||||
environment: config.EnvironmentDev,
|
|
||||||
trustedProxies: cidrPrivateV4,
|
|
||||||
expectWarning: false,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
|
||||||
// is incompatible with parallel subtests.
|
|
||||||
t.Setenv("WEBHOOKER_ENVIRONMENT", tt.environment)
|
|
||||||
|
|
||||||
if tt.trustedProxies == "" {
|
|
||||||
require.NoError(
|
|
||||||
t, os.Unsetenv("TRUSTED_PROXIES"),
|
|
||||||
)
|
|
||||||
} else {
|
|
||||||
t.Setenv("TRUSTED_PROXIES", tt.trustedProxies)
|
|
||||||
}
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
|
|
||||||
log := slog.New(slog.NewJSONHandler(
|
|
||||||
&buf, &slog.HandlerOptions{
|
|
||||||
Level: slog.LevelDebug,
|
|
||||||
},
|
|
||||||
))
|
|
||||||
|
|
||||||
require.NoError(
|
|
||||||
t,
|
|
||||||
config.WarnSharedRateLimitBucketForTest(log),
|
|
||||||
)
|
|
||||||
|
|
||||||
if !tt.expectWarning {
|
|
||||||
assert.Empty(t, buf.String())
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
logged := buf.String()
|
|
||||||
|
|
||||||
assert.Contains(t, logged, `"level":"WARN"`)
|
|
||||||
assert.Contains(t, logged, "TRUSTED_PROXIES")
|
|
||||||
assert.Contains(t, logged, "share one bucket")
|
|
||||||
assert.Contains(
|
|
||||||
t, logged, "throttle every other client's failed logins",
|
|
||||||
)
|
|
||||||
// The warning must not claim a lockout the login
|
|
||||||
// endpoint no longer permits: credentials are verified
|
|
||||||
// before any budget is spent.
|
|
||||||
assert.Contains(
|
|
||||||
t, logged, "a correct password still gets in",
|
|
||||||
)
|
|
||||||
// The text must stay accurate for a developer with
|
|
||||||
// nothing in front of the process, where an empty
|
|
||||||
// list costs nothing.
|
|
||||||
assert.Contains(
|
|
||||||
t, logged, "nothing proxying to this process",
|
|
||||||
)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// metricsEnv describes what one subtest below puts in the
|
// metricsEnv describes what one subtest below puts in the
|
||||||
// environment for a single METRICS_ variable. A variable that is
|
// environment for a single METRICS_ variable. A variable that is
|
||||||
// set to the empty string and one that is not set at all are
|
// set to the empty string and one that is not set at all are
|
||||||
|
|||||||
@@ -6,21 +6,6 @@ import "log/slog"
|
|||||||
// the external config_test package so each helper can be covered by
|
// the external config_test package so each helper can be covered by
|
||||||
// its own table-driven test without weakening the package API.
|
// its own table-driven test without weakening the package API.
|
||||||
|
|
||||||
// WarnSharedRateLimitBucketForTest loads a Config from the current
|
|
||||||
// environment and emits its startup warnings to log. The real logger
|
|
||||||
// writes to stdout, so this lets the warning's firing condition be
|
|
||||||
// asserted against a handler the test controls.
|
|
||||||
func WarnSharedRateLimitBucketForTest(log *slog.Logger) error {
|
|
||||||
c, err := loadFromEnv()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
c.warnSharedRateLimitBucket(log)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// WarnEgressAllowlistForTest loads a Config from the current
|
// WarnEgressAllowlistForTest loads a Config from the current
|
||||||
// environment and emits its egress-allowlist startup warning to
|
// environment and emits its egress-allowlist startup warning to
|
||||||
// log, so a test can assert both that the warning fires only when
|
// log, so a test can assert both that the warning fires only when
|
||||||
|
|||||||
@@ -3,6 +3,8 @@ package database_test
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
"log/slog"
|
||||||
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
@@ -83,3 +85,37 @@ func TestFirstBoot_PrintsTheAdminPasswordAsABanner(t *testing.T) {
|
|||||||
t, ok, "the printed password must open the seeded account",
|
t, ok, "the printed password must open the seeded account",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestNewDatabase_IsLoggedWithItsPath is the log half of
|
||||||
|
// https://git.eeqj.de/sneak/webhooker/issues/359. A DATA_DIR that is
|
||||||
|
// unexpectedly empty boots exactly like a first start, so the start
|
||||||
|
// that creates the database must say so, and where. Opening that
|
||||||
|
// database again must not.
|
||||||
|
func TestNewDatabase_IsLoggedWithItsPath(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
|
||||||
|
open := func() string {
|
||||||
|
var out bytes.Buffer
|
||||||
|
|
||||||
|
db, err := database.Open(dir, slog.New(slog.NewTextHandler(&out, nil)))
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, db.Close())
|
||||||
|
|
||||||
|
return out.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
const created = `level=WARN msg="created a new, empty database"`
|
||||||
|
|
||||||
|
first := open()
|
||||||
|
second := open()
|
||||||
|
|
||||||
|
assert.Contains(
|
||||||
|
t, first,
|
||||||
|
created+" path="+filepath.Join(dir, database.MainDBFileName),
|
||||||
|
)
|
||||||
|
assert.NotContains(
|
||||||
|
t, second, created, "an existing database is not new",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"io/fs"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -199,6 +200,12 @@ func (d *Database) connectTo(dataDir string) error {
|
|||||||
// Construct the main application database path inside DATA_DIR.
|
// Construct the main application database path inside DATA_DIR.
|
||||||
dbPath := filepath.Join(dataDir, MainDBFileName)
|
dbPath := filepath.Join(dataDir, MainDBFileName)
|
||||||
|
|
||||||
|
// Checked before opening, which creates the file. A DATA_DIR that
|
||||||
|
// is unexpectedly empty -- its volume not mounted, say -- looks
|
||||||
|
// exactly like a first start, so a new database is a warning.
|
||||||
|
_, statErr := os.Stat(dbPath)
|
||||||
|
created := errors.Is(statErr, fs.ErrNotExist)
|
||||||
|
|
||||||
// Opened through OpenSQLite so this handle carries the same WAL
|
// Opened through OpenSQLite so this handle carries the same WAL
|
||||||
// journaling, busy timeout, immediate-transaction locking, and pool
|
// journaling, busy timeout, immediate-transaction locking, and pool
|
||||||
// bounds as every other database file. See sqlite_open.go.
|
// bounds as every other database file. See sqlite_open.go.
|
||||||
@@ -229,7 +236,12 @@ func (d *Database) connectTo(dataDir string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
d.db = db
|
d.db = db
|
||||||
d.log.Info("connected to database", "path", dbPath)
|
|
||||||
|
if created {
|
||||||
|
d.log.Warn("created a new, empty database", "path", dbPath)
|
||||||
|
} else {
|
||||||
|
d.log.Info("connected to database", "path", dbPath)
|
||||||
|
}
|
||||||
|
|
||||||
// Run migrations
|
// Run migrations
|
||||||
return d.migrate()
|
return d.migrate()
|
||||||
|
|||||||
@@ -93,11 +93,11 @@ func TestEventTierQueriesUseTheirIndexes(t *testing.T) {
|
|||||||
deliveries []database.Delivery
|
deliveries []database.Delivery
|
||||||
results []database.DeliveryResult
|
results []database.DeliveryResult
|
||||||
depths []struct{ Depth int }
|
depths []struct{ Depth int }
|
||||||
|
removed []database.TargetTotals
|
||||||
)
|
)
|
||||||
|
|
||||||
byStatus := "idx_deliveries_status (status=? AND deleted_at=?)"
|
byStatus := "idx_deliveries_status (status=? AND deleted_at=?)"
|
||||||
byEvent := "idx_deliveries_event_id (event_id=? AND deleted_at=?)"
|
byEvent := "idx_deliveries_event_id (event_id=? AND deleted_at=?)"
|
||||||
byAge := "idx_events_deleted_at_created_at (deleted_at=? AND created_at<?)"
|
|
||||||
|
|
||||||
// The delivery engine: recovery and the retry sweep, the sweep for
|
// The delivery engine: recovery and the retry sweep, the sweep for
|
||||||
// stranded pending deliveries, and the queue depth count.
|
// stranded pending deliveries, and the queue depth count.
|
||||||
@@ -123,25 +123,80 @@ func TestEventTierQueriesUseTheirIndexes(t *testing.T) {
|
|||||||
Order("attempt_num ASC").Find(&results),
|
Order("attempt_num ASC").Find(&results),
|
||||||
"idx_delivery_results_delivery_id (delivery_id=? AND deleted_at=?)")
|
"idx_delivery_results_delivery_id (delivery_id=? AND deleted_at=?)")
|
||||||
|
|
||||||
// Retention's three deletes (reapExpired), whose subqueries are built
|
// Retention (reapExpired, deleteEvents): one batch of expired
|
||||||
// afresh for each statement as it builds them.
|
// events, then their attempts, deliveries and the events.
|
||||||
expiredEventIDs := func() *gorm.DB {
|
var expired []string
|
||||||
return dry.Model(&database.Event{}).Select("id").
|
|
||||||
Where("created_at < ?", cutoff)
|
|
||||||
}
|
|
||||||
|
|
||||||
|
assertPlanUses(t, db, dry.Unscoped().Model(&database.Event{}).
|
||||||
|
Where("created_at < ?", cutoff).
|
||||||
|
Limit(database.ExportReapBatchSize).Pluck("id", &expired),
|
||||||
|
"idx_events_created_at (created_at<?)")
|
||||||
assertPlanUses(t, db, dry.Unscoped().Where(
|
assertPlanUses(t, db, dry.Unscoped().Where(
|
||||||
"delivery_id IN (?)", dry.Model(&database.Delivery{}).
|
"delivery_id IN (?)", dry.Unscoped().Model(&database.Delivery{}).
|
||||||
Select("id").Where("event_id IN (?)", expiredEventIDs()),
|
Select("id").Where("event_id IN ?", ids),
|
||||||
).Delete(&database.DeliveryResult{}),
|
).Delete(&database.DeliveryResult{}),
|
||||||
"idx_delivery_results_delivery_id (delivery_id=?)", byEvent, byAge)
|
"idx_delivery_results_delivery_id (delivery_id=?)",
|
||||||
assertPlanUses(t, db, dry.Unscoped().Where(
|
"idx_deliveries_event_id (event_id=?)")
|
||||||
"event_id IN (?)", expiredEventIDs(),
|
assertPlanUses(t, db, dry.Unscoped().Model(&database.Delivery{}).
|
||||||
).Delete(&database.Delivery{}),
|
Select("target_id, count(*) AS deliveries_removed, "+
|
||||||
"idx_deliveries_event_id (event_id=?)", byAge)
|
"count(CASE WHEN status = ? THEN 1 END) AS failed_removed",
|
||||||
assertPlanUses(t, db, dry.Unscoped().Where(
|
database.DeliveryStatusFailed).
|
||||||
"created_at < ?", cutoff,
|
Where("event_id IN ?", ids).Group("target_id").Find(&removed),
|
||||||
).Delete(&database.Event{}), "idx_events_created_at (created_at<?)")
|
"idx_deliveries_event_id (event_id=?)")
|
||||||
|
assertPlanUses(t, db, dry.Unscoped().Where("event_id IN ?", ids).
|
||||||
|
Delete(&database.Delivery{}), "idx_deliveries_event_id (event_id=?)")
|
||||||
|
assertPlanUses(t, db, dry.Unscoped().Where("id IN ?", ids).
|
||||||
|
Delete(&database.Event{}), "sqlite_autoindex_events_1 (id=?)")
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestStatisticsQueriesUseTheirIndexes does the same for the webhook
|
||||||
|
// page's statistics (readEventStats in the handlers): deliveries in
|
||||||
|
// progress, each target's deliveries finished since a time, which must
|
||||||
|
// come from the index alone, and events received since a time.
|
||||||
|
func TestStatisticsQueriesUseTheirIndexes(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
mgr, lc := setupTestWebhookDBManager(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
require.NoError(t, lc.Start(ctx))
|
||||||
|
|
||||||
|
defer func() { require.NoError(t, lc.Stop(ctx)) }()
|
||||||
|
|
||||||
|
db, err := mgr.GetDB(uuid.New().String())
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
dry := db.Session(&gorm.Session{DryRun: true})
|
||||||
|
since := time.Now()
|
||||||
|
|
||||||
|
var (
|
||||||
|
count int64
|
||||||
|
byTarget []struct{ TargetID string }
|
||||||
|
)
|
||||||
|
|
||||||
|
assertPlanUses(t, db, dry.Model(&database.Delivery{}).
|
||||||
|
Where("status IN ?", []database.DeliveryStatus{
|
||||||
|
database.DeliveryStatusPending,
|
||||||
|
database.DeliveryStatusRetrying,
|
||||||
|
}).Count(&count),
|
||||||
|
"idx_deliveries_status (status=? AND deleted_at=?)")
|
||||||
|
assertPlanUses(t, db, dry.Model(&database.Delivery{}).
|
||||||
|
Select("target_id, "+
|
||||||
|
"count(CASE WHEN status = ? THEN 1 END) AS delivered, "+
|
||||||
|
"count(CASE WHEN status = ? THEN 1 END) AS failed",
|
||||||
|
database.DeliveryStatusDelivered,
|
||||||
|
database.DeliveryStatusFailed).
|
||||||
|
Where("status IN ? AND finished_at >= ?",
|
||||||
|
[]database.DeliveryStatus{
|
||||||
|
database.DeliveryStatusDelivered,
|
||||||
|
database.DeliveryStatusFailed,
|
||||||
|
}, since).
|
||||||
|
Group("target_id").Find(&byTarget),
|
||||||
|
"COVERING INDEX idx_deliveries_status "+
|
||||||
|
"(status=? AND deleted_at=? AND finished_at>?)")
|
||||||
|
assertPlanUses(t, db, dry.Model(&database.Event{}).
|
||||||
|
Where("created_at >= ?", since).Count(&count),
|
||||||
|
"idx_events_deleted_at_created_at "+
|
||||||
|
"(deleted_at=? AND created_at>?)")
|
||||||
}
|
}
|
||||||
|
|
||||||
// assertPlanUses asserts that SQLite's plan for a statement GORM built
|
// assertPlanUses asserts that SQLite's plan for a statement GORM built
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"os"
|
"os"
|
||||||
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
@@ -28,6 +29,10 @@ func NewTestRetentionReaper(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ExportReapBatchSize exposes how many expired events one retention
|
||||||
|
// transaction deletes.
|
||||||
|
const ExportReapBatchSize = reapBatchSize
|
||||||
|
|
||||||
// ExportSweep runs a single retention sweep synchronously for tests.
|
// ExportSweep runs a single retention sweep synchronously for tests.
|
||||||
func (r *RetentionReaper) ExportSweep(ctx context.Context) {
|
func (r *RetentionReaper) ExportSweep(ctx context.Context) {
|
||||||
r.sweep(ctx)
|
r.sweep(ctx)
|
||||||
@@ -79,3 +84,14 @@ func (d *Database) ExportSetBannerOut(w io.Writer) {
|
|||||||
func DummyPasswordHashForTest() string {
|
func DummyPasswordHashForTest() string {
|
||||||
return dummyPasswordHash()
|
return dummyPasswordHash()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// HashAtShippedCostForTest makes HashPassword hash at the shipped
|
||||||
|
// memory cost until t ends. t must not run in parallel with other
|
||||||
|
// tests, which would hash at that cost alongside it.
|
||||||
|
func HashAtShippedCostForTest(t *testing.T) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
hashAtShippedCostInTest = true
|
||||||
|
|
||||||
|
t.Cleanup(func() { hashAtShippedCostInTest = false })
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,6 +1,10 @@
|
|||||||
package database
|
package database
|
||||||
|
|
||||||
import "gorm.io/gorm"
|
import (
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gorm.io/gorm"
|
||||||
|
)
|
||||||
|
|
||||||
// DeliveryStatus represents the status of a delivery
|
// DeliveryStatus represents the status of a delivery
|
||||||
type DeliveryStatus string
|
type DeliveryStatus string
|
||||||
@@ -37,7 +41,7 @@ type Delivery struct {
|
|||||||
BaseModel
|
BaseModel
|
||||||
|
|
||||||
EventID string `gorm:"type:uuid;not null;index:idx_deliveries_event_id,priority:1" json:"eventId"`
|
EventID string `gorm:"type:uuid;not null;index:idx_deliveries_event_id,priority:1" json:"eventId"`
|
||||||
TargetID string `gorm:"type:uuid;not null" json:"targetId"`
|
TargetID string `gorm:"type:uuid;not null;index:idx_deliveries_status,priority:4" json:"targetId"`
|
||||||
Status DeliveryStatus `gorm:"not null;default:'pending';index:idx_deliveries_status,priority:1" json:"status"`
|
Status DeliveryStatus `gorm:"not null;default:'pending';index:idx_deliveries_status,priority:1" json:"status"`
|
||||||
|
|
||||||
// DeletedAt repeats the BaseModel field only to be the second column
|
// DeletedAt repeats the BaseModel field only to be the second column
|
||||||
@@ -45,6 +49,13 @@ type Delivery struct {
|
|||||||
// gives.
|
// gives.
|
||||||
DeletedAt gorm.DeletedAt `gorm:"index:idx_deliveries_event_id,priority:2;index:idx_deliveries_status,priority:2" json:"deletedAt,omitzero"`
|
DeletedAt gorm.DeletedAt `gorm:"index:idx_deliveries_event_id,priority:2;index:idx_deliveries_status,priority:2" json:"deletedAt,omitzero"`
|
||||||
|
|
||||||
|
// FinishedAt is when the delivery became delivered or failed, and
|
||||||
|
// nil while it is pending or retrying. It and then TargetID end the
|
||||||
|
// status index, so the webhook page counts each target's deliveries
|
||||||
|
// that finished in a recent window by reading just that window from
|
||||||
|
// the index.
|
||||||
|
FinishedAt *time.Time `gorm:"index:idx_deliveries_status,priority:3" json:"finishedAt,omitempty"`
|
||||||
|
|
||||||
// Relations
|
// Relations
|
||||||
Event Event `json:"event,omitzero"`
|
Event Event `json:"event,omitzero"`
|
||||||
Target Target `json:"target,omitzero"`
|
Target Target `json:"target,omitzero"`
|
||||||
|
|||||||
@@ -31,6 +31,11 @@ type Event struct {
|
|||||||
Body string `gorm:"type:text" json:"body"`
|
Body string `gorm:"type:text" json:"body"`
|
||||||
ContentType string `json:"contentType"`
|
ContentType string `json:"contentType"`
|
||||||
|
|
||||||
|
// BodyBytes is the size of Body in bytes, recorded when the event
|
||||||
|
// is stored so the recent events list can show it without reading
|
||||||
|
// the body.
|
||||||
|
BodyBytes int64 `gorm:"not null" json:"bodyBytes"`
|
||||||
|
|
||||||
// ResubmittedFromID names the event this one was copied from by
|
// ResubmittedFromID names the event this one was copied from by
|
||||||
// an operator resubmit. It is nil for an event that arrived on
|
// an operator resubmit. It is nil for an event that arrived on
|
||||||
// the receiver, which is every event created before the column
|
// the receiver, which is every event created before the column
|
||||||
|
|||||||
@@ -0,0 +1,99 @@
|
|||||||
|
package database
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gorm.io/gorm"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The running totals in a webhook's event database keep the webhook
|
||||||
|
// page's lifetime figures right after retention has removed the rows
|
||||||
|
// they count, and let the page show them without counting every row.
|
||||||
|
// Each total changes in the transaction that writes or deletes the
|
||||||
|
// rows it counts.
|
||||||
|
|
||||||
|
// EventTotals is the single row counting a webhook's events: every
|
||||||
|
// event ever stored, how many of them retention has deleted, and when
|
||||||
|
// the newest arrived, which retention leaves as it is.
|
||||||
|
type EventTotals struct {
|
||||||
|
ID int64 `gorm:"primaryKey"`
|
||||||
|
|
||||||
|
Events int64 `gorm:"not null"`
|
||||||
|
EventsRemoved int64 `gorm:"not null"`
|
||||||
|
|
||||||
|
// LastEventAt is when the newest event arrived, or nil before the
|
||||||
|
// first.
|
||||||
|
LastEventAt *time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// TableName names the table AddEventTotals updates.
|
||||||
|
func (EventTotals) TableName() string {
|
||||||
|
return "event_totals"
|
||||||
|
}
|
||||||
|
|
||||||
|
// TargetTotals is one row per target counting its deliveries: every
|
||||||
|
// delivery ever created, how many became delivered and how many
|
||||||
|
// failed, and how many deliveries and failed deliveries retention has
|
||||||
|
// deleted. The webhook's delivery figures are these rows summed.
|
||||||
|
type TargetTotals struct {
|
||||||
|
TargetID string `gorm:"type:uuid;primaryKey"`
|
||||||
|
|
||||||
|
Deliveries int64 `gorm:"not null"`
|
||||||
|
Delivered int64 `gorm:"not null"`
|
||||||
|
Failed int64 `gorm:"not null"`
|
||||||
|
|
||||||
|
DeliveriesRemoved int64 `gorm:"not null"`
|
||||||
|
FailedRemoved int64 `gorm:"not null"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// TableName names the table AddTargetTotals updates.
|
||||||
|
func (TargetTotals) TableName() string {
|
||||||
|
return "target_totals"
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddEventTotals adds each count in add to the webhook's event totals,
|
||||||
|
// and records add.LastEventAt as when the newest event arrived if it is
|
||||||
|
// set. Call it on the transaction that writes or deletes the events it
|
||||||
|
// counts.
|
||||||
|
func AddEventTotals(tx *gorm.DB, add EventTotals) error {
|
||||||
|
err := tx.Exec(
|
||||||
|
`UPDATE event_totals SET
|
||||||
|
events = events + ?,
|
||||||
|
events_removed = events_removed + ?,
|
||||||
|
last_event_at = coalesce(?, last_event_at)`,
|
||||||
|
add.Events, add.EventsRemoved, add.LastEventAt,
|
||||||
|
).Error
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("adding to event totals: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddTargetTotals adds each count in add to the totals of the target
|
||||||
|
// add.TargetID names, creating its row the first time. Call it on the
|
||||||
|
// transaction that writes or deletes the deliveries it counts.
|
||||||
|
func AddTargetTotals(tx *gorm.DB, add TargetTotals) error {
|
||||||
|
err := tx.Exec(
|
||||||
|
`INSERT INTO target_totals (target_id, deliveries, delivered,
|
||||||
|
failed, deliveries_removed, failed_removed)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?)
|
||||||
|
ON CONFLICT (target_id) DO UPDATE SET
|
||||||
|
deliveries = deliveries + excluded.deliveries,
|
||||||
|
delivered = delivered + excluded.delivered,
|
||||||
|
failed = failed + excluded.failed,
|
||||||
|
deliveries_removed =
|
||||||
|
deliveries_removed + excluded.deliveries_removed,
|
||||||
|
failed_removed = failed_removed + excluded.failed_removed`,
|
||||||
|
add.TargetID, add.Deliveries, add.Delivered,
|
||||||
|
add.Failed, add.DeliveriesRemoved, add.FailedRemoved,
|
||||||
|
).Error
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"adding to totals of target %s: %w", add.TargetID, err,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -1,13 +1,58 @@
|
|||||||
package database
|
package database
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"gorm.io/gorm"
|
||||||
|
)
|
||||||
|
|
||||||
|
// MaxUsernameBytes is the longest username, in bytes, that a user may
|
||||||
|
// have. The same number appears in the check constraint on
|
||||||
|
// User.Username, because a struct tag cannot reference a constant.
|
||||||
|
//
|
||||||
|
// A login stores the username in the session cookie, and both
|
||||||
|
// securecookie and browsers refuse a cookie value past about 4096
|
||||||
|
// bytes. That value is the session base64-encoded twice, so it holds
|
||||||
|
// 4096 × 3/4 × 3/4 = 2304 bytes of session, and the signature,
|
||||||
|
// timestamp and the session's other values take about 270 of those: a
|
||||||
|
// username longer than about 2030 bytes can never log in. The limit is
|
||||||
|
// about half that, so the session can carry more values later without
|
||||||
|
// locking out an account whose username is already at the limit.
|
||||||
|
const MaxUsernameBytes = 1024
|
||||||
|
|
||||||
|
// ErrUsernameTooLong is returned when a user is saved with a username
|
||||||
|
// longer than MaxUsernameBytes.
|
||||||
|
var ErrUsernameTooLong = errors.New("username is too long")
|
||||||
|
|
||||||
// User represents a user of the webhooker service
|
// User represents a user of the webhooker service
|
||||||
|
//
|
||||||
|
//nolint:lll // a struct tag cannot wrap
|
||||||
type User struct {
|
type User struct {
|
||||||
BaseModel
|
BaseModel
|
||||||
|
|
||||||
Username string `gorm:"uniqueIndex;not null" json:"username"`
|
Username string `gorm:"uniqueIndex;not null;check:length(CAST(username AS BLOB)) <= 1024" json:"username"`
|
||||||
Password string `gorm:"not null" json:"-"` // Argon2 hashed
|
Password string `gorm:"not null" json:"-"` // Argon2 hashed
|
||||||
|
|
||||||
// Relations
|
// Relations
|
||||||
Webhooks []Webhook `json:"webhooks,omitempty"`
|
Webhooks []Webhook `json:"webhooks,omitempty"`
|
||||||
APIKeys []APIKey `json:"apiKeys,omitempty"`
|
APIKeys []APIKey `json:"apiKeys,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// BeforeSave rejects a username longer than MaxUsernameBytes when a whole
|
||||||
|
// User is created or saved, so those calls get ErrUsernameTooLong rather
|
||||||
|
// than the database's constraint error. A column update such as
|
||||||
|
// Update("username", ...) is caught only by the check constraint, as is
|
||||||
|
// any path that writes the table without this model.
|
||||||
|
func (u *User) BeforeSave(_ *gorm.DB) error {
|
||||||
|
if len(u.Username) > MaxUsernameBytes {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"%w: %d bytes, limit is %d",
|
||||||
|
ErrUsernameTooLong,
|
||||||
|
len(u.Username),
|
||||||
|
MaxUsernameBytes,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,65 @@
|
|||||||
|
package database_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/google/uuid"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
|
)
|
||||||
|
|
||||||
|
// usernameAtLimit is exactly MaxUsernameBytes long, built from a
|
||||||
|
// two-byte character. A check that counted characters rather than bytes
|
||||||
|
// would see half the length and let the one-byte-longer name through.
|
||||||
|
func usernameAtLimit() string {
|
||||||
|
return strings.Repeat("é", database.MaxUsernameBytes/2)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUserCreate_RejectsOverlongUsername(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
db := startedTestDB(t)
|
||||||
|
|
||||||
|
err := db.Create(&database.User{
|
||||||
|
Username: usernameAtLimit() + "x",
|
||||||
|
Password: "hash",
|
||||||
|
}).Error
|
||||||
|
|
||||||
|
require.ErrorIs(t, err, database.ErrUsernameTooLong)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUserCreate_AcceptsUsernameAtLimit(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
db := startedTestDB(t)
|
||||||
|
|
||||||
|
require.NoError(t, db.Create(&database.User{
|
||||||
|
Username: usernameAtLimit(),
|
||||||
|
Password: "hash",
|
||||||
|
}).Error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestUsersTable_EnforcesUsernameLimitWithoutTheModel inserts with raw
|
||||||
|
// SQL, as a path that bypassed User.BeforeSave would, so only the
|
||||||
|
// table's check constraint stands between it and an over-long
|
||||||
|
// username. Accepting the name at the limit and refusing the next byte
|
||||||
|
// also pins the constraint's number to MaxUsernameBytes.
|
||||||
|
func TestUsersTable_EnforcesUsernameLimitWithoutTheModel(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
db := startedTestDB(t)
|
||||||
|
|
||||||
|
insert := "INSERT INTO users (id, username, password) VALUES (?, ?, ?)"
|
||||||
|
|
||||||
|
require.NoError(t, db.Exec(
|
||||||
|
insert, uuid.New().String(), usernameAtLimit(), "hash",
|
||||||
|
).Error)
|
||||||
|
|
||||||
|
err := db.Exec(
|
||||||
|
insert, uuid.New().String(), usernameAtLimit()+"x", "hash",
|
||||||
|
).Error
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Contains(t, err.Error(), "CHECK constraint failed")
|
||||||
|
}
|
||||||
@@ -2,7 +2,8 @@ package database
|
|||||||
|
|
||||||
// Migrate runs database migrations for the main application database.
|
// Migrate runs database migrations for the main application database.
|
||||||
// Only configuration-tier models are stored in the main database.
|
// Only configuration-tier models are stored in the main database.
|
||||||
// Event-tier models (Event, Delivery, DeliveryResult) live in
|
// Event-tier models (Event, Delivery, DeliveryResult, EventTotals,
|
||||||
|
// TargetTotals) live in
|
||||||
// per-webhook dedicated databases managed by WebhookDBManager.
|
// per-webhook dedicated databases managed by WebhookDBManager.
|
||||||
func (d *Database) Migrate() error {
|
func (d *Database) Migrate() error {
|
||||||
return d.db.AutoMigrate(
|
return d.db.AutoMigrate(
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
"math/big"
|
"math/big"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
"golang.org/x/crypto/argon2"
|
"golang.org/x/crypto/argon2"
|
||||||
)
|
)
|
||||||
@@ -63,10 +64,30 @@ func DefaultPasswordConfig() *PasswordConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// HashPassword generates an Argon2id hash of the password
|
// testArgon2Memory is the Argon2id memory cost, in KiB, that a test
|
||||||
|
// binary hashes with: 1 MB instead of the shipped 64 MB. Every test
|
||||||
|
// that starts a database hashes the bootstrap admin password, dozens
|
||||||
|
// of them run in parallel, and under the race detector each 64 MB hash
|
||||||
|
// holds about 150 MB. VerifyPassword reads the cost from the hash it
|
||||||
|
// checks, so verification follows.
|
||||||
|
const testArgon2Memory = 1024
|
||||||
|
|
||||||
|
// hashAtShippedCostInTest makes a test binary hash at the shipped
|
||||||
|
// memory cost. Only TestHashPassword_ShippedParameters sets it.
|
||||||
|
//
|
||||||
|
//nolint:gochecknoglobals // set by one test, see above
|
||||||
|
var hashAtShippedCostInTest bool
|
||||||
|
|
||||||
|
// HashPassword generates an Argon2id hash of the password. A binary
|
||||||
|
// built by go test hashes at testArgon2Memory; one built by go build
|
||||||
|
// always hashes at the defaults.
|
||||||
func HashPassword(password string) (string, error) {
|
func HashPassword(password string) (string, error) {
|
||||||
config := DefaultPasswordConfig()
|
config := DefaultPasswordConfig()
|
||||||
|
|
||||||
|
if testing.Testing() && !hashAtShippedCostInTest {
|
||||||
|
config.Memory = testArgon2Memory
|
||||||
|
}
|
||||||
|
|
||||||
// Generate a salt
|
// Generate a salt
|
||||||
salt := make([]byte, config.SaltLen)
|
salt := make([]byte, config.SaltLen)
|
||||||
|
|
||||||
|
|||||||
@@ -192,6 +192,39 @@ func TestHashPasswordUniqueness(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestHashPassword_ShippedParameters hashes and verifies through
|
||||||
|
// HashPassword at the shipped Argon2id parameters. Every other test
|
||||||
|
// hashes at the lower memory cost a test binary uses, so this is the
|
||||||
|
// one that keeps production hashing covered. One hash and one
|
||||||
|
// verification: each costs 64 MB.
|
||||||
|
//
|
||||||
|
//nolint:paralleltest // changes the hashing cost for the whole binary
|
||||||
|
func TestHashPassword_ShippedParameters(t *testing.T) {
|
||||||
|
database.HashAtShippedCostForTest(t)
|
||||||
|
|
||||||
|
password := "correct horse battery staple"
|
||||||
|
|
||||||
|
hash, err := database.HashPassword(password)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("hashing with the shipped parameters: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
const shipped = "$argon2id$v=19$m=65536,t=1,p=4$"
|
||||||
|
|
||||||
|
if !strings.HasPrefix(hash, shipped) {
|
||||||
|
t.Errorf("hash = %q, want prefix %q", hash, shipped)
|
||||||
|
}
|
||||||
|
|
||||||
|
valid, err := database.VerifyPassword(password, hash)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("VerifyPassword() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !valid {
|
||||||
|
t.Error("VerifyPassword() returned false for correct password")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestVerifyDummyPassword_DoesRealWork covers the anti-enumeration
|
// TestVerifyDummyPassword_DoesRealWork covers the anti-enumeration
|
||||||
// path. Login charges an unknown username a verification against a
|
// path. Login charges an unknown username a verification against a
|
||||||
// dummy hash so that a nonexistent account is not answered in
|
// dummy hash so that a nonexistent account is not answered in
|
||||||
|
|||||||
+117
-52
@@ -18,6 +18,19 @@ import (
|
|||||||
// computation.
|
// computation.
|
||||||
const hoursPerDay = 24
|
const hoursPerDay = 24
|
||||||
|
|
||||||
|
// reapBatchSize is how many expired events one retention transaction
|
||||||
|
// deletes. A transaction holds the event database's write lock, which
|
||||||
|
// the receiver and the delivery workers wait for, so a large prune is
|
||||||
|
// split into transactions each short enough to finish well inside the
|
||||||
|
// busy timeout.
|
||||||
|
const reapBatchSize = 1000
|
||||||
|
|
||||||
|
// reapBatchPause is how long retention waits after one batch before
|
||||||
|
// starting the next. A writer waiting for the write lock checks for it
|
||||||
|
// again after at most 100 ms, so a longer pause lets it in between two
|
||||||
|
// batches instead of only after the whole prune.
|
||||||
|
const reapBatchPause = 200 * time.Millisecond
|
||||||
|
|
||||||
// RetentionReaperParams holds the fx dependencies for the
|
// RetentionReaperParams holds the fx dependencies for the
|
||||||
// RetentionReaper.
|
// RetentionReaper.
|
||||||
type RetentionReaperParams struct {
|
type RetentionReaperParams struct {
|
||||||
@@ -187,13 +200,15 @@ func (r *RetentionReaper) sweep(ctx context.Context) {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
r.reapWebhook(wh.ID, wh.RetentionDays)
|
r.reapWebhook(ctx, wh.ID, wh.RetentionDays)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// reapWebhook removes every expired event (and its dependents) from a
|
// reapWebhook removes every expired event (and its dependents) from a
|
||||||
// single webhook's database.
|
// single webhook's database, or as many as it reaches before ctx is
|
||||||
|
// cancelled.
|
||||||
func (r *RetentionReaper) reapWebhook(
|
func (r *RetentionReaper) reapWebhook(
|
||||||
|
ctx context.Context,
|
||||||
webhookID string,
|
webhookID string,
|
||||||
retentionDays int,
|
retentionDays int,
|
||||||
) {
|
) {
|
||||||
@@ -213,7 +228,7 @@ func (r *RetentionReaper) reapWebhook(
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
deleted, err := reapExpired(db, cutoff)
|
deleted, err := reapExpired(ctx, db, cutoff)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
r.log.Error(
|
r.log.Error(
|
||||||
"retention sweep: failed to reap expired events",
|
"retention sweep: failed to reap expired events",
|
||||||
@@ -265,57 +280,107 @@ func retentionCutoff(
|
|||||||
), true
|
), true
|
||||||
}
|
}
|
||||||
|
|
||||||
// reapExpired hard-deletes, in foreign-key-safe order, the delivery
|
// reapExpired hard-deletes the events older than cutoff, with their
|
||||||
// results, deliveries, and events associated with events older than
|
// deliveries and delivery results, reapBatchSize events per
|
||||||
// cutoff. Deletes are unscoped so rows are physically removed rather
|
// transaction with reapBatchPause between transactions, until none is
|
||||||
// than soft-deleted, reclaiming disk. It returns the number of events
|
// left. Once ctx is cancelled it returns after the batch in hand,
|
||||||
// deleted.
|
// leaving the rest to the next sweep, so stopping the app does not
|
||||||
func reapExpired(db *gorm.DB, cutoff time.Time) (int64, error) {
|
// wait for a long prune. It returns the number of events deleted.
|
||||||
// Fresh subqueries are built per statement to avoid reusing a
|
func reapExpired(
|
||||||
// mutated builder across executions.
|
ctx context.Context, db *gorm.DB, cutoff time.Time,
|
||||||
expiredEventIDs := func() *gorm.DB {
|
) (int64, error) {
|
||||||
return db.Model(&Event{}).
|
var total int64
|
||||||
Select("id").
|
|
||||||
Where("created_at < ?", cutoff)
|
|
||||||
}
|
|
||||||
expiredDeliveryIDs := func() *gorm.DB {
|
|
||||||
return db.Model(&Delivery{}).
|
|
||||||
Select("id").
|
|
||||||
Where("event_id IN (?)", expiredEventIDs())
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1. Delivery results whose delivery belongs to an expired event.
|
for {
|
||||||
res := db.Unscoped().
|
var eventIDs []string
|
||||||
Where("delivery_id IN (?)", expiredDeliveryIDs()).
|
|
||||||
Delete(&DeliveryResult{})
|
|
||||||
if res.Error != nil {
|
|
||||||
return 0, fmt.Errorf(
|
|
||||||
"deleting expired delivery results: %w",
|
|
||||||
res.Error,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// 2. Deliveries belonging to an expired event.
|
err := db.Transaction(func(tx *gorm.DB) error {
|
||||||
del := db.Unscoped().
|
err := tx.Unscoped().Model(&Event{}).
|
||||||
Where("event_id IN (?)", expiredEventIDs()).
|
Where("created_at < ?", cutoff).
|
||||||
Delete(&Delivery{})
|
Limit(reapBatchSize).
|
||||||
if del.Error != nil {
|
Pluck("id", &eventIDs).Error
|
||||||
return 0, fmt.Errorf(
|
if err != nil {
|
||||||
"deleting expired deliveries: %w",
|
return fmt.Errorf("selecting expired events: %w", err)
|
||||||
del.Error,
|
}
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// 3. The expired events themselves.
|
if len(eventIDs) == 0 {
|
||||||
ev := db.Unscoped().
|
return nil
|
||||||
Where("created_at < ?", cutoff).
|
}
|
||||||
Delete(&Event{})
|
|
||||||
if ev.Error != nil {
|
|
||||||
return 0, fmt.Errorf(
|
|
||||||
"deleting expired events: %w",
|
|
||||||
ev.Error,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return ev.RowsAffected, nil
|
return deleteEvents(tx, eventIDs)
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return total, err
|
||||||
|
}
|
||||||
|
|
||||||
|
total += int64(len(eventIDs))
|
||||||
|
|
||||||
|
if len(eventIDs) < reapBatchSize {
|
||||||
|
return total, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return total, nil
|
||||||
|
case <-time.After(reapBatchPause):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// deleteEvents hard-deletes the given events and, in foreign-key-safe
|
||||||
|
// order before them, their delivery results and deliveries, then adds
|
||||||
|
// what it deleted to the running totals. It runs on reapExpired's
|
||||||
|
// transaction, so the totals change exactly when the rows do. Deletes
|
||||||
|
// are unscoped so rows are physically removed rather than
|
||||||
|
// soft-deleted, reclaiming disk.
|
||||||
|
func deleteEvents(tx *gorm.DB, eventIDs []string) error {
|
||||||
|
// 1. The delivery results of the events' deliveries.
|
||||||
|
err := tx.Unscoped().
|
||||||
|
Where("delivery_id IN (?)", tx.Unscoped().Model(&Delivery{}).
|
||||||
|
Select("id").
|
||||||
|
Where("event_id IN ?", eventIDs)).
|
||||||
|
Delete(&DeliveryResult{}).Error
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("deleting expired delivery results: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. The events' deliveries, after counting them, and the failed
|
||||||
|
// ones among them, per target. The status is tested in the select
|
||||||
|
// list rather than the WHERE clause: there, SQLite would read every
|
||||||
|
// failed delivery the webhook has through the status index,
|
||||||
|
// instead of only these through the event_id index.
|
||||||
|
var removed []TargetTotals
|
||||||
|
|
||||||
|
err = tx.Unscoped().Model(&Delivery{}).
|
||||||
|
Select("target_id, count(*) AS deliveries_removed, "+
|
||||||
|
"count(CASE WHEN status = ? THEN 1 END) AS failed_removed",
|
||||||
|
DeliveryStatusFailed).
|
||||||
|
Where("event_id IN ?", eventIDs).
|
||||||
|
Group("target_id").
|
||||||
|
Find(&removed).Error
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("counting expired deliveries: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = tx.Unscoped().
|
||||||
|
Where("event_id IN ?", eventIDs).
|
||||||
|
Delete(&Delivery{}).Error
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("deleting expired deliveries: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. The events themselves.
|
||||||
|
ev := tx.Unscoped().Where("id IN ?", eventIDs).Delete(&Event{})
|
||||||
|
if ev.Error != nil {
|
||||||
|
return fmt.Errorf("deleting expired events: %w", ev.Error)
|
||||||
|
}
|
||||||
|
|
||||||
|
for i := range removed {
|
||||||
|
err = AddTargetTotals(tx, removed[i])
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return AddEventTotals(tx, EventTotals{EventsRemoved: ev.RowsAffected})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,410 @@
|
|||||||
|
package database_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/google/uuid"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"gorm.io/gorm"
|
||||||
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
|
)
|
||||||
|
|
||||||
|
// readEventTotals reads a webhook database's row of event totals,
|
||||||
|
// asserting that it has exactly one.
|
||||||
|
func readEventTotals(t *testing.T, db *gorm.DB) database.EventTotals {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
var rows []database.EventTotals
|
||||||
|
|
||||||
|
require.NoError(t, db.Find(&rows).Error)
|
||||||
|
require.Len(t, rows, 1)
|
||||||
|
|
||||||
|
return rows[0]
|
||||||
|
}
|
||||||
|
|
||||||
|
// readTargetTotals reads a webhook database's target totals, keyed by
|
||||||
|
// target.
|
||||||
|
func readTargetTotals(
|
||||||
|
t *testing.T, db *gorm.DB,
|
||||||
|
) map[string]database.TargetTotals {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
var rows []database.TargetTotals
|
||||||
|
|
||||||
|
require.NoError(t, db.Find(&rows).Error)
|
||||||
|
|
||||||
|
byTarget := make(map[string]database.TargetTotals, len(rows))
|
||||||
|
for _, row := range rows {
|
||||||
|
byTarget[row.TargetID] = row
|
||||||
|
}
|
||||||
|
|
||||||
|
return byTarget
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestWebhookDBManager_TotalsSurviveReopen verifies that a new event
|
||||||
|
// database starts with one row of zero event totals and no target
|
||||||
|
// totals, that adding to a target twice adds to the one row, and that
|
||||||
|
// opening the database again keeps everything added.
|
||||||
|
func TestWebhookDBManager_TotalsSurviveReopen(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
mgr, lc := setupTestWebhookDBManager(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
require.NoError(t, lc.Start(ctx))
|
||||||
|
|
||||||
|
defer func() { require.NoError(t, lc.Stop(ctx)) }()
|
||||||
|
|
||||||
|
webhookID := uuid.New().String()
|
||||||
|
|
||||||
|
db, err := mgr.GetDB(webhookID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
fresh := readEventTotals(t, db)
|
||||||
|
assert.Equal(t, database.EventTotals{ID: fresh.ID}, fresh)
|
||||||
|
assert.Empty(t, readTargetTotals(t, db))
|
||||||
|
|
||||||
|
first, second := uuid.New().String(), uuid.New().String()
|
||||||
|
|
||||||
|
require.NoError(t, database.AddEventTotals(db, database.EventTotals{
|
||||||
|
Events: 2,
|
||||||
|
}))
|
||||||
|
require.NoError(t, database.AddTargetTotals(db, database.TargetTotals{
|
||||||
|
TargetID: first, Deliveries: 2, Delivered: 1,
|
||||||
|
}))
|
||||||
|
require.NoError(t, database.AddTargetTotals(db, database.TargetTotals{
|
||||||
|
TargetID: first, Failed: 1,
|
||||||
|
}))
|
||||||
|
require.NoError(t, database.AddTargetTotals(db, database.TargetTotals{
|
||||||
|
TargetID: second, Deliveries: 1,
|
||||||
|
}))
|
||||||
|
|
||||||
|
// Drop the cached connection so the next open reopens the file,
|
||||||
|
// as a restart would.
|
||||||
|
require.NoError(t, mgr.CloseAll())
|
||||||
|
|
||||||
|
db, err = mgr.GetDB(webhookID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
assert.Equal(t, database.EventTotals{ID: fresh.ID, Events: 2},
|
||||||
|
readEventTotals(t, db))
|
||||||
|
assert.Equal(t, map[string]database.TargetTotals{
|
||||||
|
first: {
|
||||||
|
TargetID: first, Deliveries: 2, Delivered: 1, Failed: 1,
|
||||||
|
},
|
||||||
|
second: {TargetID: second, Deliveries: 1},
|
||||||
|
}, readTargetTotals(t, db))
|
||||||
|
}
|
||||||
|
|
||||||
|
// seedExpiredEvents stores count events created at the given time,
|
||||||
|
// each with a delivered delivery to one target and a failed delivery
|
||||||
|
// to the other, and one attempt for each delivery.
|
||||||
|
func seedExpiredEvents(
|
||||||
|
t *testing.T,
|
||||||
|
db *gorm.DB,
|
||||||
|
webhookID string,
|
||||||
|
count int,
|
||||||
|
createdAt time.Time,
|
||||||
|
delivered, failed string,
|
||||||
|
) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
events := make([]database.Event, count)
|
||||||
|
deliveries := make([]database.Delivery, 0, 2*count)
|
||||||
|
|
||||||
|
for i := range events {
|
||||||
|
events[i] = database.Event{
|
||||||
|
WebhookID: webhookID,
|
||||||
|
EntrypointID: uuid.New().String(),
|
||||||
|
Method: http.MethodPost,
|
||||||
|
}
|
||||||
|
events[i].ID = uuid.New().String()
|
||||||
|
events[i].CreatedAt = createdAt
|
||||||
|
|
||||||
|
deliveries = append(deliveries,
|
||||||
|
database.Delivery{
|
||||||
|
EventID: events[i].ID,
|
||||||
|
TargetID: delivered,
|
||||||
|
Status: database.DeliveryStatusDelivered,
|
||||||
|
},
|
||||||
|
database.Delivery{
|
||||||
|
EventID: events[i].ID,
|
||||||
|
TargetID: failed,
|
||||||
|
Status: database.DeliveryStatusFailed,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
require.NoError(t, db.CreateInBatches(events, 500).Error)
|
||||||
|
require.NoError(t, db.CreateInBatches(deliveries, 500).Error)
|
||||||
|
|
||||||
|
results := make([]database.DeliveryResult, len(deliveries))
|
||||||
|
for i := range deliveries {
|
||||||
|
results[i] = database.DeliveryResult{
|
||||||
|
DeliveryID: deliveries[i].ID, AttemptNum: 1,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
require.NoError(t, db.CreateInBatches(results, 500).Error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// seedBareEvents stores count events created at the given time, with
|
||||||
|
// no deliveries.
|
||||||
|
func seedBareEvents(
|
||||||
|
t *testing.T,
|
||||||
|
db *gorm.DB,
|
||||||
|
webhookID string,
|
||||||
|
count int,
|
||||||
|
createdAt time.Time,
|
||||||
|
) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
events := make([]database.Event, count)
|
||||||
|
for i := range events {
|
||||||
|
events[i] = database.Event{
|
||||||
|
WebhookID: webhookID,
|
||||||
|
EntrypointID: uuid.New().String(),
|
||||||
|
Method: http.MethodPost,
|
||||||
|
}
|
||||||
|
events[i].CreatedAt = createdAt
|
||||||
|
}
|
||||||
|
|
||||||
|
require.NoError(t, db.CreateInBatches(events, 500).Error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRetentionReaper_PrunesMoreThanOneBatch verifies that a prune
|
||||||
|
// larger than one transaction's batch removes every expired event with
|
||||||
|
// its deliveries and delivery results, keeps the recent event, and
|
||||||
|
// adds what it removed to the event and target totals, so the totals
|
||||||
|
// within retention match the rows still stored.
|
||||||
|
func TestRetentionReaper_PrunesMoreThanOneBatch(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
env := setupRetentionTest(t)
|
||||||
|
|
||||||
|
webhookID := createWebhook(t, env.mainDB.DB(), 30)
|
||||||
|
|
||||||
|
db, err := env.mgr.GetDB(webhookID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
expired := database.ExportReapBatchSize + 1
|
||||||
|
delivered, failed := uuid.New().String(), uuid.New().String()
|
||||||
|
seedExpiredEvents(t, db, webhookID, expired,
|
||||||
|
time.Now().Add(-40*24*time.Hour), delivered, failed)
|
||||||
|
|
||||||
|
// One recent event, delivered to the first target.
|
||||||
|
recent := seedEventChain(t, db, webhookID, time.Now())
|
||||||
|
require.NoError(t, db.Model(&database.Delivery{}).
|
||||||
|
Where("id = ?", recent.deliveryID).
|
||||||
|
Update("target_id", delivered).Error)
|
||||||
|
|
||||||
|
// The totals storing those rows would have left.
|
||||||
|
n := int64(expired)
|
||||||
|
require.NoError(t, database.AddEventTotals(db, database.EventTotals{
|
||||||
|
Events: n + 1,
|
||||||
|
}))
|
||||||
|
require.NoError(t, database.AddTargetTotals(db, database.TargetTotals{
|
||||||
|
TargetID: delivered, Deliveries: n + 1, Delivered: n + 1,
|
||||||
|
}))
|
||||||
|
require.NoError(t, database.AddTargetTotals(db, database.TargetTotals{
|
||||||
|
TargetID: failed, Deliveries: n, Failed: n,
|
||||||
|
}))
|
||||||
|
|
||||||
|
env.reaper.ExportSweep(context.Background())
|
||||||
|
|
||||||
|
// Only the recent event's rows are left.
|
||||||
|
for _, model := range []any{
|
||||||
|
&database.Event{}, &database.Delivery{}, &database.DeliveryResult{},
|
||||||
|
} {
|
||||||
|
var count int64
|
||||||
|
|
||||||
|
require.NoError(t, db.Model(model).Count(&count).Error)
|
||||||
|
assert.Equal(t, int64(1), count, "%T rows left", model)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertChainPresent(t, db, recent)
|
||||||
|
|
||||||
|
eventTotals := readEventTotals(t, db)
|
||||||
|
assert.Equal(t, database.EventTotals{
|
||||||
|
ID: eventTotals.ID, Events: n + 1, EventsRemoved: n,
|
||||||
|
}, eventTotals)
|
||||||
|
|
||||||
|
targetTotals := readTargetTotals(t, db)
|
||||||
|
assert.Equal(t, map[string]database.TargetTotals{
|
||||||
|
delivered: {
|
||||||
|
TargetID: delivered, Deliveries: n + 1, Delivered: n + 1,
|
||||||
|
DeliveriesRemoved: n,
|
||||||
|
},
|
||||||
|
failed: {
|
||||||
|
TargetID: failed, Deliveries: n, Failed: n,
|
||||||
|
DeliveriesRemoved: n, FailedRemoved: n,
|
||||||
|
},
|
||||||
|
}, targetTotals)
|
||||||
|
|
||||||
|
// A sweep with nothing left to remove changes nothing.
|
||||||
|
env.reaper.ExportSweep(context.Background())
|
||||||
|
|
||||||
|
assert.Equal(t, eventTotals, readEventTotals(t, db))
|
||||||
|
assert.Equal(t, targetTotals, readTargetTotals(t, db))
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRetentionReaper_WriteDuringPruneSucceeds verifies that a prune
|
||||||
|
// of several batches lets other writers in between its batches: an
|
||||||
|
// event stored once the first batch is deleted is stored while expired
|
||||||
|
// events are still left, not only after the prune has finished.
|
||||||
|
func TestRetentionReaper_WriteDuringPruneSucceeds(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
env := setupRetentionTest(t)
|
||||||
|
|
||||||
|
webhookID := createWebhook(t, env.mainDB.DB(), 30)
|
||||||
|
|
||||||
|
db, err := env.mgr.GetDB(webhookID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// Three batches of expired events, with nothing else stored: only
|
||||||
|
// the number of batches matters here.
|
||||||
|
expired := 3 * database.ExportReapBatchSize
|
||||||
|
seedBareEvents(t, db, webhookID, expired,
|
||||||
|
time.Now().Add(-40*24*time.Hour))
|
||||||
|
|
||||||
|
cutoff := time.Now().Add(-30 * 24 * time.Hour)
|
||||||
|
countExpired := func() int64 {
|
||||||
|
var count int64
|
||||||
|
|
||||||
|
require.NoError(t, db.Model(&database.Event{}).
|
||||||
|
Where("created_at < ?", cutoff).
|
||||||
|
Count(&count).Error)
|
||||||
|
|
||||||
|
return count
|
||||||
|
}
|
||||||
|
|
||||||
|
pruned := make(chan struct{})
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
defer close(pruned)
|
||||||
|
|
||||||
|
env.reaper.ExportSweep(context.Background())
|
||||||
|
}()
|
||||||
|
|
||||||
|
t.Cleanup(func() { <-pruned })
|
||||||
|
|
||||||
|
// Every stored event is expired until the write below.
|
||||||
|
require.Eventually(t, func() bool {
|
||||||
|
var count int64
|
||||||
|
|
||||||
|
err := db.Model(&database.Event{}).Count(&count).Error
|
||||||
|
|
||||||
|
return err == nil && count < int64(expired)
|
||||||
|
}, 10*time.Second, 10*time.Millisecond)
|
||||||
|
|
||||||
|
event := &database.Event{
|
||||||
|
WebhookID: webhookID,
|
||||||
|
EntrypointID: uuid.New().String(),
|
||||||
|
Method: http.MethodPost,
|
||||||
|
}
|
||||||
|
require.NoError(t, db.Create(event).Error)
|
||||||
|
|
||||||
|
assert.Positive(t, countExpired(),
|
||||||
|
"the event was stored only after the whole prune")
|
||||||
|
|
||||||
|
<-pruned
|
||||||
|
|
||||||
|
assert.Zero(t, countExpired())
|
||||||
|
|
||||||
|
var stored database.Event
|
||||||
|
|
||||||
|
require.NoError(t, db.First(&stored, "id = ?", event.ID).Error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRetentionReaper_StopDuringPruneLeavesTheRest verifies that
|
||||||
|
// stopping the reaper during a prune of several batches returns
|
||||||
|
// between two batches, well inside the stop timeout, leaving the
|
||||||
|
// remaining expired events for the next sweep, and that the totals
|
||||||
|
// match the rows left.
|
||||||
|
func TestRetentionReaper_StopDuringPruneLeavesTheRest(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
env := setupRetentionTest(t)
|
||||||
|
|
||||||
|
webhookID := createWebhook(t, env.mainDB.DB(), 30)
|
||||||
|
|
||||||
|
db, err := env.mgr.GetDB(webhookID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// Two batches and one more of expired events, a few of them with a
|
||||||
|
// delivered and a failed delivery for the target totals to count.
|
||||||
|
// Most carry nothing else, to keep the test quick.
|
||||||
|
const withDeliveries = 10
|
||||||
|
|
||||||
|
expiredAt := time.Now().Add(-40 * 24 * time.Hour)
|
||||||
|
delivered, failed := uuid.New().String(), uuid.New().String()
|
||||||
|
seedExpiredEvents(t, db, webhookID, withDeliveries, expiredAt,
|
||||||
|
delivered, failed)
|
||||||
|
seedBareEvents(t, db, webhookID,
|
||||||
|
2*database.ExportReapBatchSize+1-withDeliveries, expiredAt)
|
||||||
|
|
||||||
|
n := int64(2*database.ExportReapBatchSize + 1)
|
||||||
|
require.NoError(t, database.AddEventTotals(db, database.EventTotals{
|
||||||
|
Events: n,
|
||||||
|
}))
|
||||||
|
require.NoError(t, database.AddTargetTotals(db, database.TargetTotals{
|
||||||
|
TargetID: delivered, Deliveries: withDeliveries,
|
||||||
|
Delivered: withDeliveries,
|
||||||
|
}))
|
||||||
|
require.NoError(t, database.AddTargetTotals(db, database.TargetTotals{
|
||||||
|
TargetID: failed, Deliveries: withDeliveries,
|
||||||
|
Failed: withDeliveries,
|
||||||
|
}))
|
||||||
|
|
||||||
|
env.reaper.ExportSetInterval(time.Millisecond)
|
||||||
|
env.reaper.ExportStart()
|
||||||
|
|
||||||
|
// Stop once the first batch is deleted. The stop lands in the pause
|
||||||
|
// after it, or at worst during the second batch, so at least the
|
||||||
|
// last event is left.
|
||||||
|
require.Eventually(t, func() bool {
|
||||||
|
var count int64
|
||||||
|
|
||||||
|
err := db.Model(&database.Event{}).Count(&count).Error
|
||||||
|
|
||||||
|
return err == nil && count < n
|
||||||
|
}, 10*time.Second, 10*time.Millisecond)
|
||||||
|
|
||||||
|
// The app's stop timeout.
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
require.NoError(t, env.reaper.ExportStop(ctx))
|
||||||
|
|
||||||
|
var events int64
|
||||||
|
|
||||||
|
require.NoError(t, db.Model(&database.Event{}).Count(&events).Error)
|
||||||
|
assert.Positive(t, events, "the stop waited for the whole prune")
|
||||||
|
|
||||||
|
eventTotals := readEventTotals(t, db)
|
||||||
|
assert.Equal(t, events, eventTotals.Events-eventTotals.EventsRemoved)
|
||||||
|
|
||||||
|
targetTotals := readTargetTotals(t, db)
|
||||||
|
require.Len(t, targetTotals, 2)
|
||||||
|
|
||||||
|
for target, totals := range targetTotals {
|
||||||
|
var deliveries, failures int64
|
||||||
|
|
||||||
|
require.NoError(t, db.Model(&database.Delivery{}).
|
||||||
|
Where("target_id = ?", target).
|
||||||
|
Count(&deliveries).Error)
|
||||||
|
require.NoError(t, db.Model(&database.Delivery{}).
|
||||||
|
Where("target_id = ? AND status = ?",
|
||||||
|
target, database.DeliveryStatusFailed).
|
||||||
|
Count(&failures).Error)
|
||||||
|
|
||||||
|
assert.Equal(t, deliveries,
|
||||||
|
totals.Deliveries-totals.DeliveriesRemoved, target)
|
||||||
|
assert.Equal(t, failures, totals.Failed-totals.FailedRemoved,
|
||||||
|
target)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -35,7 +35,8 @@ var errInvalidCachedDBType = errors.New(
|
|||||||
|
|
||||||
// WebhookDBManager manages per-webhook SQLite database files
|
// WebhookDBManager manages per-webhook SQLite database files
|
||||||
// for event storage. Each webhook gets its own dedicated
|
// for event storage. Each webhook gets its own dedicated
|
||||||
// database containing Events, Deliveries, and DeliveryResults.
|
// database containing Events, Deliveries, DeliveryResults and the
|
||||||
|
// running totals of them (EventTotals, TargetTotals).
|
||||||
// Database connections are opened lazily and cached.
|
// Database connections are opened lazily and cached.
|
||||||
type WebhookDBManager struct {
|
type WebhookDBManager struct {
|
||||||
dataDir string
|
dataDir string
|
||||||
@@ -295,6 +296,7 @@ func (m *WebhookDBManager) openDB(
|
|||||||
// Run migrations for event-tier models only
|
// Run migrations for event-tier models only
|
||||||
err = db.AutoMigrate(
|
err = db.AutoMigrate(
|
||||||
&Event{}, &Delivery{}, &DeliveryResult{},
|
&Event{}, &Delivery{}, &DeliveryResult{},
|
||||||
|
&EventTotals{}, &TargetTotals{},
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = sqlDB.Close()
|
_ = sqlDB.Close()
|
||||||
@@ -305,6 +307,18 @@ func (m *WebhookDBManager) openDB(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A new database gets its row of event totals, all zero. Target
|
||||||
|
// totals rows are created by the first delivery to each target.
|
||||||
|
err = db.FirstOrCreate(&EventTotals{}).Error
|
||||||
|
if err != nil {
|
||||||
|
_ = sqlDB.Close()
|
||||||
|
|
||||||
|
return nil, fmt.Errorf(
|
||||||
|
"creating event totals for webhook database %s: %w",
|
||||||
|
webhookID, err,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
m.log.Info(
|
m.log.Info(
|
||||||
"opened per-webhook database",
|
"opened per-webhook database",
|
||||||
"webhook_id", webhookID,
|
"webhook_id", webhookID,
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
|
"gorm.io/gorm"
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/lifecycle"
|
"sneak.berlin/go/webhooker/internal/lifecycle"
|
||||||
@@ -25,14 +26,14 @@ type ArchiveSweeperParams struct {
|
|||||||
Logger *logger.Logger
|
Logger *logger.Logger
|
||||||
}
|
}
|
||||||
|
|
||||||
// ArchiveSweeper periodically prunes expired rows from
|
// ArchiveSweeper periodically prunes expired rows from the
|
||||||
// per-webhook archive databases whose database target carries a
|
// archive databases of database targets that carry a positive
|
||||||
// positive expiry.
|
// expiry.
|
||||||
//
|
//
|
||||||
// Without it, pruning happens only when an archive is
|
// Without it, pruning happens only when an archive is
|
||||||
// (re)opened, and archives are only ever reopened by writes: an
|
// (re)opened, and archives are only ever reopened by writes: an
|
||||||
// archive belonging to a webhook that has stopped receiving
|
// archive whose target has stopped receiving events would keep
|
||||||
// events would keep its expired rows forever. The sweep closes
|
// its expired rows forever. The sweep closes
|
||||||
// that gap without changing anything for archives whose expiry
|
// that gap without changing anything for archives whose expiry
|
||||||
// is unset or "never".
|
// is unset or "never".
|
||||||
//
|
//
|
||||||
@@ -155,7 +156,7 @@ func (s *ArchiveSweeper) run(ctx context.Context) {
|
|||||||
// soft-deleted along with it, so GORM's default scope already
|
// soft-deleted along with it, so GORM's default scope already
|
||||||
// excludes them.
|
// excludes them.
|
||||||
//
|
//
|
||||||
// A failure for one webhook is logged and the sweep continues,
|
// A failure for one target is logged and the sweep continues,
|
||||||
// matching how the write path already treats a prune error as
|
// matching how the write path already treats a prune error as
|
||||||
// non-fatal.
|
// non-fatal.
|
||||||
func (s *ArchiveSweeper) sweep(ctx context.Context) {
|
func (s *ArchiveSweeper) sweep(ctx context.Context) {
|
||||||
@@ -210,19 +211,20 @@ func (s *ArchiveSweeper) sweepTarget(target *database.Target) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
err = s.eng.dbTarget.sweepWebhook(target.WebhookID, expiry)
|
err = s.eng.dbTarget.sweepArchive(target.ID, expiry)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// A writer evicted underneath the sweep means the operator
|
// A writer evicted, or a target row gone, underneath the sweep
|
||||||
// deleted the webhook (or its last database target) while the
|
// means the operator deleted the target or its webhook while
|
||||||
// sweep was walking the target list. That is an ordinary
|
// the sweep was walking the target list. That is an ordinary
|
||||||
// interleaving, not a failure, so it must not produce an
|
// interleaving, not a failure, so it must not produce an
|
||||||
// error line.
|
// error line.
|
||||||
if errors.Is(err, errArchiveWriterEvicted) {
|
if errors.Is(err, errArchiveWriterEvicted) ||
|
||||||
|
errors.Is(err, gorm.ErrRecordNotFound) {
|
||||||
s.log.Debug(
|
s.log.Debug(
|
||||||
"archive sweep: writer evicted mid-sweep",
|
"archive sweep: target deleted mid-sweep",
|
||||||
"webhook_id", target.WebhookID,
|
"webhook_id", target.WebhookID,
|
||||||
"target_id", target.ID,
|
"target_id", target.ID,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -34,18 +34,23 @@ const (
|
|||||||
sweepConcurrentWrites = 20
|
sweepConcurrentWrites = 20
|
||||||
)
|
)
|
||||||
|
|
||||||
// sweeperEnv bundles the pieces an archive sweep test drives:
|
// archiveTestWebhookName is the name of every webhook
|
||||||
// a main configuration database holding webhooks and targets, a
|
// seedDatabaseTarget creates. It is not safe in a file name as it
|
||||||
// delivery engine owning the archive writer registry, and the
|
// stands, so every archive test goes through archiveNamePart.
|
||||||
// data directory the archive files live in.
|
const archiveTestWebhookName = "Sweep Test!"
|
||||||
type sweeperEnv struct {
|
|
||||||
|
// archiveEnv bundles the pieces an archive test drives: a main
|
||||||
|
// configuration database holding webhooks and targets, a delivery
|
||||||
|
// engine owning the archive writer registry, the archive sweeper,
|
||||||
|
// and the data directory the archive files live in.
|
||||||
|
type archiveEnv struct {
|
||||||
sweeper *delivery.ArchiveSweeper
|
sweeper *delivery.ArchiveSweeper
|
||||||
eng *delivery.Engine
|
eng *delivery.Engine
|
||||||
mainDB *database.Database
|
mainDB *database.Database
|
||||||
dataDir string
|
dataDir string
|
||||||
}
|
}
|
||||||
|
|
||||||
func setupSweeperTest(t *testing.T) *sweeperEnv {
|
func setupArchiveTest(t *testing.T) *archiveEnv {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
dataDir := t.TempDir()
|
dataDir := t.TempDir()
|
||||||
@@ -78,7 +83,7 @@ func setupSweeperTest(t *testing.T) *sweeperEnv {
|
|||||||
1,
|
1,
|
||||||
)
|
)
|
||||||
|
|
||||||
return &sweeperEnv{
|
return &archiveEnv{
|
||||||
sweeper: delivery.NewTestArchiveSweeper(
|
sweeper: delivery.NewTestArchiveSweeper(
|
||||||
mainDB, eng, log,
|
mainDB, eng, log,
|
||||||
),
|
),
|
||||||
@@ -88,25 +93,27 @@ func setupSweeperTest(t *testing.T) *sweeperEnv {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// archivePath returns where the engine keeps a webhook's
|
// archivePath returns where the engine keeps a database target's
|
||||||
// archive file.
|
// archive file, for the names seedDatabaseTarget gave it.
|
||||||
func (env *sweeperEnv) archivePath(webhookID string) string {
|
func (env *archiveEnv) archivePath(tgt *database.Target) string {
|
||||||
return filepath.Join(
|
return filepath.Join(
|
||||||
env.dataDir, fmt.Sprintf("archive-%s.db", webhookID),
|
env.dataDir,
|
||||||
|
delivery.ArchiveFileName(
|
||||||
|
archiveTestWebhookName, tgt.Name, tgt.ID,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// seedDatabaseTarget creates a webhook with one database target
|
// seedDatabaseTarget creates a webhook with one database target
|
||||||
// carrying the given target config JSON, and returns the
|
// carrying the given target config JSON, and returns the target.
|
||||||
// webhook id.
|
func (env *archiveEnv) seedDatabaseTarget(
|
||||||
func (env *sweeperEnv) seedDatabaseTarget(
|
|
||||||
t *testing.T, configJSON string,
|
t *testing.T, configJSON string,
|
||||||
) string {
|
) *database.Target {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
wh := &database.Webhook{
|
wh := &database.Webhook{
|
||||||
UserID: uuid.New().String(),
|
UserID: uuid.New().String(),
|
||||||
Name: "sweep-test",
|
Name: archiveTestWebhookName,
|
||||||
}
|
}
|
||||||
require.NoError(
|
require.NoError(
|
||||||
t,
|
t,
|
||||||
@@ -115,9 +122,19 @@ func (env *sweeperEnv) seedDatabaseTarget(
|
|||||||
Create(wh).Error,
|
Create(wh).Error,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
return env.addDatabaseTarget(t, wh.ID, configJSON)
|
||||||
|
}
|
||||||
|
|
||||||
|
// addDatabaseTarget creates one more database target on an
|
||||||
|
// existing webhook and returns it.
|
||||||
|
func (env *archiveEnv) addDatabaseTarget(
|
||||||
|
t *testing.T, webhookID, configJSON string,
|
||||||
|
) *database.Target {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
tgt := &database.Target{
|
tgt := &database.Target{
|
||||||
WebhookID: wh.ID,
|
WebhookID: webhookID,
|
||||||
Name: "archive",
|
Name: "Archive",
|
||||||
Type: database.TargetTypeDatabase,
|
Type: database.TargetTypeDatabase,
|
||||||
Active: true,
|
Active: true,
|
||||||
Config: configJSON,
|
Config: configJSON,
|
||||||
@@ -129,19 +146,19 @@ func (env *sweeperEnv) seedDatabaseTarget(
|
|||||||
Create(tgt).Error,
|
Create(tgt).Error,
|
||||||
)
|
)
|
||||||
|
|
||||||
return wh.ID
|
return tgt
|
||||||
}
|
}
|
||||||
|
|
||||||
// seedArchiveRows creates the archive file for a webhook and
|
// seedArchiveRows creates the archive file for a target and
|
||||||
// inserts one row per supplied archived-at timestamp, returning
|
// inserts one row per supplied archived-at timestamp, returning
|
||||||
// the archive path. The handle is closed before returning, so
|
// the archive path. The handle is closed before returning, so
|
||||||
// the archive is idle exactly as it would be with no traffic.
|
// the archive is idle exactly as it would be with no traffic.
|
||||||
func (env *sweeperEnv) seedArchiveRows(
|
func (env *archiveEnv) seedArchiveRows(
|
||||||
t *testing.T, webhookID string, archivedAt ...time.Time,
|
t *testing.T, tgt *database.Target, archivedAt ...time.Time,
|
||||||
) string {
|
) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
path := env.archivePath(webhookID)
|
path := env.archivePath(tgt)
|
||||||
|
|
||||||
sqlDB, err := sql.Open(
|
sqlDB, err := sql.Open(
|
||||||
"sqlite", fmt.Sprintf("file:%s?mode=rwc", path),
|
"sqlite", fmt.Sprintf("file:%s?mode=rwc", path),
|
||||||
@@ -160,7 +177,7 @@ func (env *sweeperEnv) seedArchiveRows(
|
|||||||
for i, at := range archivedAt {
|
for i, at := range archivedAt {
|
||||||
row := delivery.ExportArchivedEvent{
|
row := delivery.ExportArchivedEvent{
|
||||||
EventID: fmt.Sprintf("ev-%d", i),
|
EventID: fmt.Sprintf("ev-%d", i),
|
||||||
WebhookID: webhookID,
|
WebhookID: tgt.WebhookID,
|
||||||
Method: http.MethodPost,
|
Method: http.MethodPost,
|
||||||
Body: `{"seeded":true}`,
|
Body: `{"seeded":true}`,
|
||||||
ArchivedAt: at,
|
ArchivedAt: at,
|
||||||
@@ -243,13 +260,13 @@ func TestArchiveSweeper_LoopOutlivesStartHookContext(
|
|||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupSweeperTest(t)
|
env := setupArchiveTest(t)
|
||||||
|
|
||||||
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
|
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
path := env.seedArchiveRows(
|
path := env.seedArchiveRows(
|
||||||
t, webhookID,
|
t, tgt,
|
||||||
now.Add(-48*time.Hour),
|
now.Add(-48*time.Hour),
|
||||||
now.Add(-time.Minute),
|
now.Add(-time.Minute),
|
||||||
)
|
)
|
||||||
@@ -287,60 +304,60 @@ func TestArchiveSweeper_LoopOutlivesStartHookContext(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TestArchiveSweep_DoesNotResurrectEvictedWriter covers the
|
// TestArchiveSweep_DoesNotResurrectEvictedWriter covers the
|
||||||
// interleaving where a sweep tick has already listed a webhook's
|
// interleaving where a sweep tick has already listed a target
|
||||||
// target when the webhook is deleted and its writer evicted. The
|
// when the target is deleted and its writer evicted. The sweep
|
||||||
// sweep must not put a writer back into the registry: nothing
|
// must not put a writer back into the registry: nothing would
|
||||||
// would ever evict it again, which is precisely the leak this
|
// ever evict it again, which is precisely the leak this change
|
||||||
// change exists to close.
|
// exists to close.
|
||||||
func TestArchiveSweep_DoesNotResurrectEvictedWriter(
|
func TestArchiveSweep_DoesNotResurrectEvictedWriter(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupSweeperTest(t)
|
env := setupArchiveTest(t)
|
||||||
|
|
||||||
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
env.seedArchiveRows(
|
env.seedArchiveRows(
|
||||||
t, webhookID, time.Now().Add(-48*time.Hour),
|
t, tgt, time.Now().Add(-48*time.Hour),
|
||||||
)
|
)
|
||||||
|
|
||||||
// Prime the registry the way a delivery would, then evict as
|
// Prime the registry the way a delivery would, then evict as
|
||||||
// the deletion path does. The target row is deliberately left
|
// the deletion path does. The target row is deliberately left
|
||||||
// in place: this is the tick that listed the webhook before
|
// in place: this is the tick that listed the target before
|
||||||
// the deletion committed.
|
// the deletion committed.
|
||||||
_, err := env.eng.ExportEnsureArchiveWriter(webhookID)
|
_, err := env.eng.ExportEnsureArchiveWriter(tgt.ID)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
env.eng.EvictWebhook(webhookID)
|
env.eng.EvictTarget(tgt.ID)
|
||||||
require.False(t, env.eng.ExportHasArchiveWriter(webhookID))
|
require.False(t, env.eng.ExportHasArchiveWriter(tgt.ID))
|
||||||
|
|
||||||
env.sweeper.ExportSweep(context.Background())
|
env.sweeper.ExportSweep(context.Background())
|
||||||
|
|
||||||
assert.False(
|
assert.False(
|
||||||
t, env.eng.ExportHasArchiveWriter(webhookID),
|
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||||
"a sweep must never re-register a writer for a webhook "+
|
"a sweep must never re-register a writer for a target "+
|
||||||
"whose registry entry has already been released",
|
"whose registry entry has already been released",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestArchiveSweep_LeavesNoRegistryEntry states the same
|
// TestArchiveSweep_LeavesNoRegistryEntry states the same
|
||||||
// invariant in its general form: sweeping an archive whose
|
// invariant in its general form: sweeping an archive whose
|
||||||
// webhook has no cached writer must not leave one behind, so the
|
// target has no cached writer must not leave one behind, so the
|
||||||
// registry keeps holding only writers a delivery created and an
|
// registry keeps holding only writers a delivery created and an
|
||||||
// eviction can reach.
|
// eviction can reach.
|
||||||
func TestArchiveSweep_LeavesNoRegistryEntry(t *testing.T) {
|
func TestArchiveSweep_LeavesNoRegistryEntry(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupSweeperTest(t)
|
env := setupArchiveTest(t)
|
||||||
|
|
||||||
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
path := env.seedArchiveRows(
|
path := env.seedArchiveRows(
|
||||||
t, webhookID,
|
t, tgt,
|
||||||
time.Now().Add(-48*time.Hour),
|
time.Now().Add(-48*time.Hour),
|
||||||
time.Now().Add(-time.Minute),
|
time.Now().Add(-time.Minute),
|
||||||
)
|
)
|
||||||
|
|
||||||
require.False(t, env.eng.ExportHasArchiveWriter(webhookID))
|
require.False(t, env.eng.ExportHasArchiveWriter(tgt.ID))
|
||||||
|
|
||||||
env.sweeper.ExportSweep(context.Background())
|
env.sweeper.ExportSweep(context.Background())
|
||||||
|
|
||||||
@@ -349,7 +366,7 @@ func TestArchiveSweep_LeavesNoRegistryEntry(t *testing.T) {
|
|||||||
"the sweep must still prune an idle archive",
|
"the sweep must still prune an idle archive",
|
||||||
)
|
)
|
||||||
assert.False(
|
assert.False(
|
||||||
t, env.eng.ExportHasArchiveWriter(webhookID),
|
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||||
"the sweep must release the registry entry it created",
|
"the sweep must release the registry entry it created",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -364,34 +381,31 @@ func TestArchiveSweep_KeepsWriterAdoptedByDelivery(
|
|||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupSweeperTest(t)
|
env := setupArchiveTest(t)
|
||||||
|
|
||||||
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
env.seedArchiveRows(
|
env.seedArchiveRows(
|
||||||
t, webhookID, time.Now().Add(-48*time.Hour),
|
t, tgt, time.Now().Add(-48*time.Hour),
|
||||||
)
|
)
|
||||||
|
|
||||||
webhookDB := testWebhookDB(t)
|
webhookDB := testWebhookDB(t)
|
||||||
event := seedEvent(t, webhookDB, `{"n":1}`)
|
event := seedEvent(t, webhookDB, `{"n":1}`)
|
||||||
event.WebhookID = webhookID
|
d := seedDatabaseTargetDelivery(t, webhookDB, event, tgt)
|
||||||
d := seedDatabaseTargetDelivery(
|
|
||||||
t, webhookDB, event, `{"expiry":"1h"}`,
|
|
||||||
)
|
|
||||||
|
|
||||||
env.sweeper.ExportSweep(context.Background())
|
env.sweeper.ExportSweep(context.Background())
|
||||||
require.False(t, env.eng.ExportHasArchiveWriter(webhookID))
|
require.False(t, env.eng.ExportHasArchiveWriter(tgt.ID))
|
||||||
|
|
||||||
env.eng.ExportDeliverDatabase(webhookDB, d)
|
env.eng.ExportDeliverDatabase(webhookDB, d)
|
||||||
|
|
||||||
assert.True(
|
assert.True(
|
||||||
t, env.eng.ExportHasArchiveWriter(webhookID),
|
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||||
"a delivery's writer must stay registered",
|
"a delivery's writer must stay registered",
|
||||||
)
|
)
|
||||||
|
|
||||||
env.sweeper.ExportSweep(context.Background())
|
env.sweeper.ExportSweep(context.Background())
|
||||||
|
|
||||||
assert.True(
|
assert.True(
|
||||||
t, env.eng.ExportHasArchiveWriter(webhookID),
|
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||||
"a sweep must not drop a writer a delivery owns",
|
"a sweep must not drop a writer a delivery owns",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -423,15 +437,15 @@ func TestArchiveSweep_KeepsWriterAdoptedDuringSweep(
|
|||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupSweeperTest(t)
|
env := setupArchiveTest(t)
|
||||||
|
|
||||||
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
env.seedArchiveRows(
|
env.seedArchiveRows(
|
||||||
t, webhookID, time.Now().Add(-48*time.Hour),
|
t, tgt, time.Now().Add(-48*time.Hour),
|
||||||
)
|
)
|
||||||
|
|
||||||
sweepWriter, created, err := env.eng.ExportSweepWriterFor(
|
sweepWriter, created, err := env.eng.ExportSweepWriterFor(
|
||||||
webhookID,
|
tgt.ID,
|
||||||
)
|
)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.True(
|
require.True(
|
||||||
@@ -442,37 +456,34 @@ func TestArchiveSweep_KeepsWriterAdoptedDuringSweep(
|
|||||||
// The delivery lands mid-sweep and adopts the entry.
|
// The delivery lands mid-sweep and adopts the entry.
|
||||||
webhookDB := testWebhookDB(t)
|
webhookDB := testWebhookDB(t)
|
||||||
event := seedEvent(t, webhookDB, `{"n":1}`)
|
event := seedEvent(t, webhookDB, `{"n":1}`)
|
||||||
event.WebhookID = webhookID
|
d := seedDatabaseTargetDelivery(t, webhookDB, event, tgt)
|
||||||
d := seedDatabaseTargetDelivery(
|
|
||||||
t, webhookDB, event, `{"expiry":"1h"}`,
|
|
||||||
)
|
|
||||||
|
|
||||||
env.eng.ExportDeliverDatabase(webhookDB, d)
|
env.eng.ExportDeliverDatabase(webhookDB, d)
|
||||||
|
|
||||||
adopted := env.eng.ExportArchiveWriterFor(webhookID)
|
adopted := env.eng.ExportArchiveWriterFor(tgt.ID)
|
||||||
require.NotNil(t, adopted)
|
require.NotNil(t, adopted)
|
||||||
require.True(
|
require.True(
|
||||||
t, sweepWriter.Same(adopted),
|
t, sweepWriter.Same(adopted),
|
||||||
"the delivery must have adopted the sweep's writer",
|
"the delivery must have adopted the sweep's writer",
|
||||||
)
|
)
|
||||||
require.True(
|
require.True(
|
||||||
t, env.eng.ExportArchiveHandleOpen(webhookID),
|
t, env.eng.ExportArchiveHandleOpen(tgt.ID),
|
||||||
"the delivery leaves the archive handle open",
|
"the delivery leaves the archive handle open",
|
||||||
)
|
)
|
||||||
|
|
||||||
// The sweep finishes.
|
// The sweep finishes.
|
||||||
env.eng.ExportReleaseSweepWriter(webhookID, sweepWriter)
|
env.eng.ExportReleaseSweepWriter(tgt.ID, sweepWriter)
|
||||||
|
|
||||||
require.True(
|
require.True(
|
||||||
t, env.eng.ExportHasArchiveWriter(webhookID),
|
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||||
"a writer adopted by a delivery during a sweep must "+
|
"a writer adopted by a delivery during a sweep must "+
|
||||||
"stay registered, or its open handle is unreachable",
|
"stay registered, or its open handle is unreachable",
|
||||||
)
|
)
|
||||||
|
|
||||||
env.eng.EvictWebhook(webhookID)
|
env.eng.EvictTarget(tgt.ID)
|
||||||
|
|
||||||
assert.False(
|
assert.False(
|
||||||
t, env.eng.ExportHasArchiveWriter(webhookID),
|
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||||
"the adopted writer must still be evictable",
|
"the adopted writer must still be evictable",
|
||||||
)
|
)
|
||||||
assert.False(
|
assert.False(
|
||||||
@@ -481,34 +492,34 @@ func TestArchiveSweep_KeepsWriterAdoptedDuringSweep(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestArchiveSweep_ContinuesAfterPerWebhookFailure proves a
|
// TestArchiveSweep_ContinuesAfterPerTargetFailure proves a
|
||||||
// failure for one webhook does not abort the sweep for the
|
// failure for one target does not abort the sweep for the
|
||||||
// others: an unparseable expiry and an unreadable archive both
|
// others: an unparseable expiry and an unreadable archive both
|
||||||
// have to be logged and stepped over.
|
// have to be logged and stepped over.
|
||||||
func TestArchiveSweep_ContinuesAfterPerWebhookFailure(
|
func TestArchiveSweep_ContinuesAfterPerTargetFailure(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupSweeperTest(t)
|
env := setupArchiveTest(t)
|
||||||
|
|
||||||
// Seeded first so the sweep reaches them before the healthy
|
// Seeded first so the sweep reaches them before the healthy
|
||||||
// webhook: targets come back in insertion order.
|
// target: targets come back in insertion order.
|
||||||
badConfigID := env.seedDatabaseTarget(t, `{"expiry":"!!!"}`)
|
badConfig := env.seedDatabaseTarget(t, `{"expiry":"!!!"}`)
|
||||||
env.seedArchiveRows(
|
env.seedArchiveRows(
|
||||||
t, badConfigID, time.Now().Add(-48*time.Hour),
|
t, badConfig, time.Now().Add(-48*time.Hour),
|
||||||
)
|
)
|
||||||
|
|
||||||
corruptID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
corrupt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
require.NoError(t, os.WriteFile(
|
require.NoError(t, os.WriteFile(
|
||||||
env.archivePath(corruptID),
|
env.archivePath(corrupt),
|
||||||
[]byte("this is not a sqlite database"),
|
[]byte("this is not a sqlite database"),
|
||||||
0o600,
|
0o600,
|
||||||
))
|
))
|
||||||
|
|
||||||
healthyID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
healthy := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
healthyPath := env.seedArchiveRows(
|
healthyPath := env.seedArchiveRows(
|
||||||
t, healthyID,
|
t, healthy,
|
||||||
time.Now().Add(-48*time.Hour),
|
time.Now().Add(-48*time.Hour),
|
||||||
time.Now().Add(-time.Minute),
|
time.Now().Add(-time.Minute),
|
||||||
)
|
)
|
||||||
@@ -518,14 +529,14 @@ func TestArchiveSweep_ContinuesAfterPerWebhookFailure(
|
|||||||
assert.Equal(
|
assert.Equal(
|
||||||
t, []string{sweepRowNew},
|
t, []string{sweepRowNew},
|
||||||
archivedEventIDs(t, healthyPath),
|
archivedEventIDs(t, healthyPath),
|
||||||
"a failure for an earlier webhook must not stop the "+
|
"a failure for an earlier target must not stop the "+
|
||||||
"sweep from pruning the ones after it",
|
"sweep from pruning the ones after it",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestArchiveSweep_OpenExistingDoesNotCreateFile pins the second
|
// TestArchiveSweep_OpenExistingDoesNotCreateFile pins the second
|
||||||
// of the two no-create guards. The first is the stat in
|
// of the two no-create guards. The first is the stat in
|
||||||
// sweepWebhook; this one is the SQLite open mode, which is what
|
// sweepExpired; this one is the SQLite open mode, which is what
|
||||||
// protects the window between that stat and the open. Flipping
|
// protects the window between that stat and the open. Flipping
|
||||||
// the sweep's mode to create-if-missing makes this fail.
|
// the sweep's mode to create-if-missing makes this fail.
|
||||||
func TestArchiveSweep_OpenExistingDoesNotCreateFile(
|
func TestArchiveSweep_OpenExistingDoesNotCreateFile(
|
||||||
@@ -561,13 +572,13 @@ func TestArchiveSweep_OpenExistingDoesNotCreateFile(
|
|||||||
func TestArchiveSweep_PrunesIdleArchive(t *testing.T) {
|
func TestArchiveSweep_PrunesIdleArchive(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupSweeperTest(t)
|
env := setupArchiveTest(t)
|
||||||
|
|
||||||
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
|
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
path := env.seedArchiveRows(
|
path := env.seedArchiveRows(
|
||||||
t, webhookID,
|
t, tgt,
|
||||||
now.Add(-48*time.Hour),
|
now.Add(-48*time.Hour),
|
||||||
now.Add(-time.Minute),
|
now.Add(-time.Minute),
|
||||||
)
|
)
|
||||||
@@ -600,11 +611,11 @@ func TestArchiveSweep_PrunesIdleArchive(t *testing.T) {
|
|||||||
func TestArchiveSweep_LeavesArchiveClosed(t *testing.T) {
|
func TestArchiveSweep_LeavesArchiveClosed(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupSweeperTest(t)
|
env := setupArchiveTest(t)
|
||||||
|
|
||||||
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
path := env.seedArchiveRows(
|
path := env.seedArchiveRows(
|
||||||
t, webhookID, time.Now().Add(-48*time.Hour),
|
t, tgt, time.Now().Add(-48*time.Hour),
|
||||||
)
|
)
|
||||||
|
|
||||||
w := delivery.NewExportArchiveWriter(
|
w := delivery.NewExportArchiveWriter(
|
||||||
@@ -640,35 +651,32 @@ func TestArchiveSweep_ClosesHandleOfRegisteredWriter(
|
|||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupSweeperTest(t)
|
env := setupArchiveTest(t)
|
||||||
|
|
||||||
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
env.seedArchiveRows(
|
env.seedArchiveRows(
|
||||||
t, webhookID, time.Now().Add(-48*time.Hour),
|
t, tgt, time.Now().Add(-48*time.Hour),
|
||||||
)
|
)
|
||||||
|
|
||||||
webhookDB := testWebhookDB(t)
|
webhookDB := testWebhookDB(t)
|
||||||
event := seedEvent(t, webhookDB, `{"n":1}`)
|
event := seedEvent(t, webhookDB, `{"n":1}`)
|
||||||
event.WebhookID = webhookID
|
d := seedDatabaseTargetDelivery(t, webhookDB, event, tgt)
|
||||||
d := seedDatabaseTargetDelivery(
|
|
||||||
t, webhookDB, event, `{"expiry":"1h"}`,
|
|
||||||
)
|
|
||||||
|
|
||||||
env.eng.ExportDeliverDatabase(webhookDB, d)
|
env.eng.ExportDeliverDatabase(webhookDB, d)
|
||||||
|
|
||||||
require.True(
|
require.True(
|
||||||
t, env.eng.ExportArchiveHandleOpen(webhookID),
|
t, env.eng.ExportArchiveHandleOpen(tgt.ID),
|
||||||
"the delivery must leave the archive handle open",
|
"the delivery must leave the archive handle open",
|
||||||
)
|
)
|
||||||
|
|
||||||
env.sweeper.ExportSweep(context.Background())
|
env.sweeper.ExportSweep(context.Background())
|
||||||
|
|
||||||
require.True(
|
require.True(
|
||||||
t, env.eng.ExportHasArchiveWriter(webhookID),
|
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||||
"the delivery's registry entry must survive the sweep",
|
"the delivery's registry entry must survive the sweep",
|
||||||
)
|
)
|
||||||
assert.False(
|
assert.False(
|
||||||
t, env.eng.ExportArchiveHandleOpen(webhookID),
|
t, env.eng.ExportArchiveHandleOpen(tgt.ID),
|
||||||
"the sweep must leave the archive closed",
|
"the sweep must leave the archive closed",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -684,11 +692,11 @@ func TestArchiveSweep_NeverExpiryUntouched(t *testing.T) {
|
|||||||
`{"expiry":""}`,
|
`{"expiry":""}`,
|
||||||
"",
|
"",
|
||||||
} {
|
} {
|
||||||
env := setupSweeperTest(t)
|
env := setupArchiveTest(t)
|
||||||
|
|
||||||
webhookID := env.seedDatabaseTarget(t, configJSON)
|
tgt := env.seedDatabaseTarget(t, configJSON)
|
||||||
path := env.seedArchiveRows(
|
path := env.seedArchiveRows(
|
||||||
t, webhookID,
|
t, tgt,
|
||||||
time.Now().Add(-10000*time.Hour),
|
time.Now().Add(-10000*time.Hour),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -699,7 +707,7 @@ func TestArchiveSweep_NeverExpiryUntouched(t *testing.T) {
|
|||||||
"config %q must keep rows forever", configJSON,
|
"config %q must keep rows forever", configJSON,
|
||||||
)
|
)
|
||||||
assert.False(
|
assert.False(
|
||||||
t, env.eng.ExportHasArchiveWriter(webhookID),
|
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||||
"config %q must leave no registry entry behind",
|
"config %q must leave no registry entry behind",
|
||||||
configJSON,
|
configJSON,
|
||||||
)
|
)
|
||||||
@@ -722,10 +730,10 @@ func TestArchiveSweep_NeverExpirySkipsBeforeOpening(
|
|||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupSweeperTest(t)
|
env := setupArchiveTest(t)
|
||||||
|
|
||||||
webhookID := env.seedDatabaseTarget(t, `{"expiry":"never"}`)
|
tgt := env.seedDatabaseTarget(t, `{"expiry":"never"}`)
|
||||||
path := env.archivePath(webhookID)
|
path := env.archivePath(tgt)
|
||||||
|
|
||||||
seedUnmigratedArchive(t, path)
|
seedUnmigratedArchive(t, path)
|
||||||
require.False(t, archiveTableExists(t, path))
|
require.False(t, archiveTableExists(t, path))
|
||||||
@@ -768,16 +776,16 @@ func archiveTableExists(t *testing.T, path string) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TestArchiveSweep_DoesNotCreateArchiveFile proves the sweep
|
// TestArchiveSweep_DoesNotCreateArchiveFile proves the sweep
|
||||||
// never conjures an archive: a webhook with a database target
|
// never conjures an archive: a database target that has never
|
||||||
// that has never received an event must still have no archive
|
// received an event must still have no archive file (nor SQLite
|
||||||
// file (nor SQLite sidecar) after a sweep.
|
// sidecar) after a sweep, and no registry entry either.
|
||||||
func TestArchiveSweep_DoesNotCreateArchiveFile(t *testing.T) {
|
func TestArchiveSweep_DoesNotCreateArchiveFile(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupSweeperTest(t)
|
env := setupArchiveTest(t)
|
||||||
|
|
||||||
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
path := env.archivePath(webhookID)
|
path := env.archivePath(tgt)
|
||||||
|
|
||||||
require.NoFileExists(t, path)
|
require.NoFileExists(t, path)
|
||||||
|
|
||||||
@@ -789,6 +797,11 @@ func TestArchiveSweep_DoesNotCreateArchiveFile(t *testing.T) {
|
|||||||
"the sweep must not create an archive file",
|
"the sweep must not create an archive file",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
assert.False(
|
||||||
|
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||||
|
"the sweep must leave no registry entry behind",
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestArchiveSweep_DoesNotCreateAfterWriterExists covers the
|
// TestArchiveSweep_DoesNotCreateAfterWriterExists covers the
|
||||||
@@ -800,11 +813,11 @@ func TestArchiveSweep_DoesNotCreateAfterWriterExists(
|
|||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupSweeperTest(t)
|
env := setupArchiveTest(t)
|
||||||
|
|
||||||
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
|
|
||||||
path, err := env.eng.ExportEnsureArchiveWriter(webhookID)
|
path, err := env.eng.ExportEnsureArchiveWriter(tgt.ID)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.NoFileExists(t, path)
|
require.NoFileExists(t, path)
|
||||||
|
|
||||||
@@ -819,17 +832,17 @@ func TestArchiveSweep_DoesNotCreateAfterWriterExists(
|
|||||||
func TestArchiveSweep_SkipsDeletedWebhookTargets(t *testing.T) {
|
func TestArchiveSweep_SkipsDeletedWebhookTargets(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupSweeperTest(t)
|
env := setupArchiveTest(t)
|
||||||
|
|
||||||
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
path := env.seedArchiveRows(
|
path := env.seedArchiveRows(
|
||||||
t, webhookID, time.Now().Add(-48*time.Hour),
|
t, tgt, time.Now().Add(-48*time.Hour),
|
||||||
)
|
)
|
||||||
|
|
||||||
require.NoError(
|
require.NoError(
|
||||||
t,
|
t,
|
||||||
env.mainDB.DB().
|
env.mainDB.DB().
|
||||||
Where("webhook_id = ?", webhookID).
|
Where("webhook_id = ?", tgt.WebhookID).
|
||||||
Delete(&database.Target{}).Error,
|
Delete(&database.Target{}).Error,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -842,14 +855,14 @@ func TestArchiveSweep_SkipsDeletedWebhookTargets(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TestArchiveSweep_ConcurrentWrites proves the sweep serialises
|
// TestArchiveSweep_ConcurrentWrites proves the sweep serialises
|
||||||
// against writes through the per-webhook writer mutex. Run
|
// against writes through the target's writer mutex. Run under
|
||||||
// under -race, an unsynchronised sweep would be caught here.
|
// -race, an unsynchronised sweep would be caught here.
|
||||||
func TestArchiveSweep_ConcurrentWrites(t *testing.T) {
|
func TestArchiveSweep_ConcurrentWrites(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupSweeperTest(t)
|
env := setupArchiveTest(t)
|
||||||
|
|
||||||
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
|
|
||||||
webhookDB := testWebhookDB(t)
|
webhookDB := testWebhookDB(t)
|
||||||
|
|
||||||
@@ -862,13 +875,10 @@ func TestArchiveSweep_ConcurrentWrites(t *testing.T) {
|
|||||||
|
|
||||||
for range sweepConcurrentWrites {
|
for range sweepConcurrentWrites {
|
||||||
event := seedEvent(t, webhookDB, `{"n":1}`)
|
event := seedEvent(t, webhookDB, `{"n":1}`)
|
||||||
event.WebhookID = webhookID
|
|
||||||
|
|
||||||
deliveries = append(
|
deliveries = append(
|
||||||
deliveries,
|
deliveries,
|
||||||
seedDatabaseTargetDelivery(
|
seedDatabaseTargetDelivery(t, webhookDB, event, tgt),
|
||||||
t, webhookDB, event, `{"expiry":"1h"}`,
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -894,7 +904,7 @@ func TestArchiveSweep_ConcurrentWrites(t *testing.T) {
|
|||||||
|
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
|
|
||||||
assert.FileExists(t, env.archivePath(webhookID))
|
assert.FileExists(t, env.archivePath(tgt))
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestArchiveSweeper_StopsCleanly proves the background loop
|
// TestArchiveSweeper_StopsCleanly proves the background loop
|
||||||
@@ -902,11 +912,11 @@ func TestArchiveSweep_ConcurrentWrites(t *testing.T) {
|
|||||||
func TestArchiveSweeper_StopsCleanly(t *testing.T) {
|
func TestArchiveSweeper_StopsCleanly(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupSweeperTest(t)
|
env := setupArchiveTest(t)
|
||||||
|
|
||||||
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
env.seedArchiveRows(
|
env.seedArchiveRows(
|
||||||
t, webhookID, time.Now().Add(-48*time.Hour),
|
t, tgt, time.Now().Add(-48*time.Hour),
|
||||||
)
|
)
|
||||||
|
|
||||||
env.sweeper.ExportSetInterval(time.Millisecond)
|
env.sweeper.ExportSetInterval(time.Millisecond)
|
||||||
@@ -930,7 +940,7 @@ func TestArchiveSweeper_StopHookHonoursStopTimeout(
|
|||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupSweeperTest(t)
|
env := setupArchiveTest(t)
|
||||||
|
|
||||||
lc := &recordingLifecycle{}
|
lc := &recordingLifecycle{}
|
||||||
env.sweeper.ExportRegisterHooks(lc)
|
env.sweeper.ExportRegisterHooks(lc)
|
||||||
|
|||||||
@@ -0,0 +1,177 @@
|
|||||||
|
package delivery_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/google/uuid"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"gorm.io/gorm"
|
||||||
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
|
)
|
||||||
|
|
||||||
|
// targetTotals reads one target's totals from a webhook database, all
|
||||||
|
// zero when it has no row.
|
||||||
|
func targetTotals(
|
||||||
|
t *testing.T, db *gorm.DB, targetID string,
|
||||||
|
) database.TargetTotals {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
var rows []database.TargetTotals
|
||||||
|
|
||||||
|
require.NoError(t, db.Where("target_id = ?", targetID).
|
||||||
|
Find(&rows).Error)
|
||||||
|
|
||||||
|
if len(rows) == 0 {
|
||||||
|
return database.TargetTotals{TargetID: targetID}
|
||||||
|
}
|
||||||
|
|
||||||
|
return rows[0]
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestUpdateDeliveryStatus_FinishTimeAndTargetTotals pins what a status
|
||||||
|
// write records for the webhook page's statistics: the time a delivery
|
||||||
|
// finished, set only when it becomes delivered or failed, and one more
|
||||||
|
// on its target's delivered or failed total.
|
||||||
|
func TestUpdateDeliveryStatus_FinishTimeAndTargetTotals(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
status database.DeliveryStatus
|
||||||
|
finished bool
|
||||||
|
delivered int64
|
||||||
|
failed int64
|
||||||
|
}{
|
||||||
|
{database.DeliveryStatusRetrying, false, 0, 0},
|
||||||
|
{database.DeliveryStatusDelivered, true, 1, 0},
|
||||||
|
{database.DeliveryStatusFailed, true, 0, 1},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(string(tt.status), func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
db := testWebhookDB(t)
|
||||||
|
e := testEngine(t, 1)
|
||||||
|
event := seedEvent(t, db, `{}`)
|
||||||
|
targetID := uuid.New().String()
|
||||||
|
d := seedDelivery(
|
||||||
|
t, db, event.ID, targetID,
|
||||||
|
database.DeliveryStatusPending,
|
||||||
|
)
|
||||||
|
|
||||||
|
before := time.Now()
|
||||||
|
|
||||||
|
require.NoError(t, e.ExportUpdateDeliveryStatus(
|
||||||
|
db, &d, tt.status,
|
||||||
|
))
|
||||||
|
|
||||||
|
var stored database.Delivery
|
||||||
|
|
||||||
|
require.NoError(t, db.First(&stored, "id = ?", d.ID).Error)
|
||||||
|
assert.Equal(t, tt.status, stored.Status)
|
||||||
|
|
||||||
|
if tt.finished {
|
||||||
|
require.NotNil(t, stored.FinishedAt)
|
||||||
|
assert.False(t, stored.FinishedAt.Before(before))
|
||||||
|
} else {
|
||||||
|
assert.Nil(t, stored.FinishedAt)
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.Equal(t, database.TargetTotals{
|
||||||
|
TargetID: targetID,
|
||||||
|
Delivered: tt.delivered,
|
||||||
|
Failed: tt.failed,
|
||||||
|
}, targetTotals(t, db, targetID))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestUpdateDeliveryStatus_DeletedDeliveryIsNotCounted covers a
|
||||||
|
// delivery retention deleted while the engine still held it. Failing
|
||||||
|
// it afterwards writes no row, so it adds no failure either: retention
|
||||||
|
// has already counted what it removed.
|
||||||
|
func TestUpdateDeliveryStatus_DeletedDeliveryIsNotCounted(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
db := testWebhookDB(t)
|
||||||
|
e := testEngine(t, 1)
|
||||||
|
event := seedEvent(t, db, `{}`)
|
||||||
|
targetID := uuid.New().String()
|
||||||
|
d := seedDelivery(
|
||||||
|
t, db, event.ID, targetID,
|
||||||
|
database.DeliveryStatusRetrying,
|
||||||
|
)
|
||||||
|
|
||||||
|
require.NoError(t, db.Unscoped().
|
||||||
|
Delete(&database.Delivery{}, "id = ?", d.ID).Error)
|
||||||
|
|
||||||
|
require.NoError(t, e.ExportUpdateDeliveryStatus(
|
||||||
|
db, &d, database.DeliveryStatusFailed,
|
||||||
|
))
|
||||||
|
|
||||||
|
assert.Equal(t, database.TargetTotals{TargetID: targetID},
|
||||||
|
targetTotals(t, db, targetID))
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestUpdateDeliveryStatus_FinishedDeliveryIsNotSettledAgain covers a
|
||||||
|
// delivery settled a second time, as recovery can do when a worker has
|
||||||
|
// settled it since recovery read it. Neither status writes over the
|
||||||
|
// first, and the totals do not move.
|
||||||
|
func TestUpdateDeliveryStatus_FinishedDeliveryIsNotSettledAgain(
|
||||||
|
t *testing.T,
|
||||||
|
) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
finished := []database.DeliveryStatus{
|
||||||
|
database.DeliveryStatusDelivered,
|
||||||
|
database.DeliveryStatusFailed,
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, first := range finished {
|
||||||
|
t.Run(string(first), func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
db := testWebhookDB(t)
|
||||||
|
e := testEngine(t, 1)
|
||||||
|
event := seedEvent(t, db, `{}`)
|
||||||
|
targetID := uuid.New().String()
|
||||||
|
d := seedDelivery(
|
||||||
|
t, db, event.ID, targetID,
|
||||||
|
database.DeliveryStatusRetrying,
|
||||||
|
)
|
||||||
|
|
||||||
|
// The delivery as recovery read it, before the worker
|
||||||
|
// settled it.
|
||||||
|
readBefore := d
|
||||||
|
|
||||||
|
require.NoError(t, e.ExportUpdateDeliveryStatus(
|
||||||
|
db, &d, first,
|
||||||
|
))
|
||||||
|
|
||||||
|
var settled database.Delivery
|
||||||
|
|
||||||
|
require.NoError(t, db.First(&settled, "id = ?", d.ID).Error)
|
||||||
|
require.NotNil(t, settled.FinishedAt)
|
||||||
|
|
||||||
|
totals := targetTotals(t, db, targetID)
|
||||||
|
|
||||||
|
for _, again := range finished {
|
||||||
|
stale := readBefore
|
||||||
|
|
||||||
|
require.NoError(t, e.ExportUpdateDeliveryStatus(
|
||||||
|
db, &stale, again,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
var stored database.Delivery
|
||||||
|
|
||||||
|
require.NoError(t, db.First(&stored, "id = ?", d.ID).Error)
|
||||||
|
assert.Equal(t, first, stored.Status)
|
||||||
|
require.NotNil(t, stored.FinishedAt)
|
||||||
|
assert.True(t, settled.FinishedAt.Equal(*stored.FinishedAt))
|
||||||
|
assert.Equal(t, totals, targetTotals(t, db, targetID))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
+95
-24
@@ -122,21 +122,24 @@ type Notifier interface {
|
|||||||
Notify(tasks []Task)
|
Notify(tasks []Task)
|
||||||
}
|
}
|
||||||
|
|
||||||
// WebhookEvictor releases the delivery engine's per-webhook
|
// Archives is how the handlers keep the database targets' archive
|
||||||
// state for a webhook that no longer needs it — currently the
|
// files in step with the configuration. Deleting a webhook or a
|
||||||
// cached archive writer of the database target, whose open
|
// target releases the cached archive writers, whose open file
|
||||||
// file handle would otherwise outlive the webhook.
|
// handles would otherwise outlive them; renaming one renames the
|
||||||
|
// archive files, which are named for the webhook and the target
|
||||||
|
// (see ArchiveFileName).
|
||||||
//
|
//
|
||||||
// It is deliberately separate from Notifier and deliberately
|
// It is deliberately separate from Notifier: archiving lifecycle
|
||||||
// one method wide: archiving lifecycle is not notification, and
|
// is not notification, and a small interface keeps the handlers
|
||||||
// a single-method interface keeps the handlers package free of
|
// package free of any dependency on the engine's internals while
|
||||||
// any dependency on the engine's internals while staying
|
// staying trivially fakeable in tests.
|
||||||
// trivially fakeable in tests.
|
|
||||||
//
|
//
|
||||||
// EvictWebhook never deletes an archive file. It is idempotent
|
// Neither eviction deletes an archive file. Both are idempotent
|
||||||
// and is a no-op for a webhook with no engine state.
|
// and are no-ops for a webhook or target with no engine state.
|
||||||
type WebhookEvictor interface {
|
type Archives interface {
|
||||||
EvictWebhook(webhookID string)
|
EvictWebhook(webhookID string)
|
||||||
|
EvictTarget(targetID string)
|
||||||
|
RenameArchive(targetID, webhookName, targetName string) error
|
||||||
}
|
}
|
||||||
|
|
||||||
// EngineParams are the fx dependencies for the delivery
|
// EngineParams are the fx dependencies for the delivery
|
||||||
@@ -181,7 +184,7 @@ type Engine struct {
|
|||||||
httpTarget *httpTarget
|
httpTarget *httpTarget
|
||||||
|
|
||||||
// dbTarget is retained so the engine can reach the archive
|
// dbTarget is retained so the engine can reach the archive
|
||||||
// writer registry for webhook eviction and the idle sweep.
|
// writer registry for eviction, renames and the idle sweep.
|
||||||
dbTarget *databaseTarget
|
dbTarget *databaseTarget
|
||||||
|
|
||||||
// inflight is the set of deliveries this engine currently owns.
|
// inflight is the set of deliveries this engine currently owns.
|
||||||
@@ -249,17 +252,43 @@ func (e *Engine) Notify(tasks []Task) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// EvictWebhook implements WebhookEvictor. It releases the
|
// EvictWebhook implements Archives. The cached archive writer of
|
||||||
// engine's per-webhook archiving state: the database target's
|
// every database target of the webhook is dropped from the
|
||||||
// cached archive writer is dropped from the registry and its
|
// registry and its file handle closed. The archive files
|
||||||
// file handle closed. The archive file itself is left on disk
|
// themselves are left on disk — they are long-term storage the
|
||||||
// — it is long-term storage the operator owns.
|
// operator owns.
|
||||||
func (e *Engine) EvictWebhook(webhookID string) {
|
func (e *Engine) EvictWebhook(webhookID string) {
|
||||||
if e.dbTarget == nil {
|
if e.dbTarget == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
e.dbTarget.evict(webhookID)
|
e.dbTarget.evictWebhook(webhookID)
|
||||||
|
}
|
||||||
|
|
||||||
|
// EvictTarget implements Archives. It is EvictWebhook for a single
|
||||||
|
// database target, and leaves the archive file on disk the same
|
||||||
|
// way.
|
||||||
|
func (e *Engine) EvictTarget(targetID string) {
|
||||||
|
if e.dbTarget == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
e.dbTarget.evict(targetID)
|
||||||
|
}
|
||||||
|
|
||||||
|
// RenameArchive implements Archives. It renames a database target's
|
||||||
|
// archive file to ArchiveFileName(webhookName, targetName,
|
||||||
|
// targetID), under the lock the target's archive writes and the
|
||||||
|
// idle sweep take. The caller renames before it saves the new name:
|
||||||
|
// see databaseTarget.rename.
|
||||||
|
func (e *Engine) RenameArchive(
|
||||||
|
targetID, webhookName, targetName string,
|
||||||
|
) error {
|
||||||
|
if e.dbTarget == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return e.dbTarget.rename(targetID, webhookName, targetName)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ScheduleRetry schedules a task to be re-enqueued onto the
|
// ScheduleRetry schedules a task to be re-enqueued onto the
|
||||||
@@ -366,7 +395,8 @@ func (e *Engine) start() {
|
|||||||
// Once the pool has drained it closes the archive writers, so a
|
// Once the pool has drained it closes the archive writers, so a
|
||||||
// clean stop leaves no archive -wal behind. Nothing else holds a
|
// clean stop leaves no archive -wal behind. Nothing else holds a
|
||||||
// writer for long by then: the archive sweeper stops before the
|
// writer for long by then: the archive sweeper stops before the
|
||||||
// engine, and deleting a webhook only closes one. If the pool did
|
// engine, and deleting or renaming a webhook or target only closes
|
||||||
|
// or moves one. If the pool did
|
||||||
// not drain in time, the writers are left open, as a kill would
|
// not drain in time, the writers are left open, as a kill would
|
||||||
// leave them. Closing them would wait for any write in progress,
|
// leave them. Closing them would wait for any write in progress,
|
||||||
// and a worker still running would then open new writers that
|
// and a worker still running would then open new writers that
|
||||||
@@ -531,6 +561,11 @@ func (e *Engine) processRetryTask(
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Set before anything below can fail the delivery: the failure is
|
||||||
|
// added to this target's totals.
|
||||||
|
d.EventID = task.EventID
|
||||||
|
d.TargetID = task.TargetID
|
||||||
|
|
||||||
if d.Status != database.DeliveryStatusRetrying {
|
if d.Status != database.DeliveryStatusRetrying {
|
||||||
e.log.Debug(
|
e.log.Debug(
|
||||||
"skipping retry for delivery "+
|
"skipping retry for delivery "+
|
||||||
@@ -562,8 +597,6 @@ func (e *Engine) processRetryTask(
|
|||||||
}
|
}
|
||||||
|
|
||||||
target := buildTargetFromTask(task)
|
target := buildTargetFromTask(task)
|
||||||
d.EventID = task.EventID
|
|
||||||
d.TargetID = task.TargetID
|
|
||||||
d.Event = event
|
d.Event = event
|
||||||
d.Target = target
|
d.Target = target
|
||||||
|
|
||||||
@@ -1554,8 +1587,9 @@ func (e *Engine) updateDeliveryStatus(
|
|||||||
targetType database.TargetType,
|
targetType database.TargetType,
|
||||||
status database.DeliveryStatus,
|
status database.DeliveryStatus,
|
||||||
) error {
|
) error {
|
||||||
err := webhookDB.Model(d).
|
err := webhookDB.Transaction(func(tx *gorm.DB) error {
|
||||||
Update("status", status).Error
|
return writeDeliveryStatus(tx, d, status)
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf(
|
return fmt.Errorf(
|
||||||
"updating delivery %s to status %s: %w",
|
"updating delivery %s to status %s: %w",
|
||||||
@@ -1574,6 +1608,43 @@ func (e *Engine) updateDeliveryStatus(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// writeDeliveryStatus writes a delivery's new status. A delivery that
|
||||||
|
// becomes delivered or failed also gets the time it finished, and is
|
||||||
|
// added to its target's delivered or failed total. That write changes
|
||||||
|
// only a delivery not yet delivered or failed, and the total moves
|
||||||
|
// only when it changed a row: retention may have deleted the delivery
|
||||||
|
// while the engine was working on it, and a recovery path may settle
|
||||||
|
// a delivery that a worker has already settled.
|
||||||
|
func writeDeliveryStatus(
|
||||||
|
tx *gorm.DB,
|
||||||
|
d *database.Delivery,
|
||||||
|
status database.DeliveryStatus,
|
||||||
|
) error {
|
||||||
|
if !status.Terminal() {
|
||||||
|
return tx.Model(d).Update("status", status).Error
|
||||||
|
}
|
||||||
|
|
||||||
|
res := tx.Model(d).
|
||||||
|
Where("status NOT IN ?", []database.DeliveryStatus{
|
||||||
|
database.DeliveryStatusDelivered,
|
||||||
|
database.DeliveryStatusFailed,
|
||||||
|
}).
|
||||||
|
Updates(map[string]any{
|
||||||
|
"status": status,
|
||||||
|
"finished_at": time.Now(),
|
||||||
|
})
|
||||||
|
if res.Error != nil || res.RowsAffected == 0 {
|
||||||
|
return res.Error
|
||||||
|
}
|
||||||
|
|
||||||
|
add := database.TargetTotals{TargetID: d.TargetID, Delivered: 1}
|
||||||
|
if status == database.DeliveryStatusFailed {
|
||||||
|
add = database.TargetTotals{TargetID: d.TargetID, Failed: 1}
|
||||||
|
}
|
||||||
|
|
||||||
|
return database.AddTargetTotals(tx, add)
|
||||||
|
}
|
||||||
|
|
||||||
// settleStatus moves a delivery to its outcome status and reports a
|
// settleStatus moves a delivery to its outcome status and reports a
|
||||||
// failed write through bookkeepingFailed, which leaves the row
|
// failed write through bookkeepingFailed, which leaves the row
|
||||||
// recoverable. It exists so the target call sites read as one
|
// recoverable. It exists so the target call sites read as one
|
||||||
|
|||||||
@@ -2,7 +2,6 @@ package delivery_test
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -10,6 +9,7 @@ import (
|
|||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
|
"gorm.io/gorm/clause"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
)
|
)
|
||||||
@@ -272,22 +272,35 @@ func TestEngine_StopHookHonoursStopTimeout(t *testing.T) {
|
|||||||
requireStopHookExpires(t, lc.hooks[0], "delivery engine")
|
requireStopHookExpires(t, lc.hooks[0], "delivery engine")
|
||||||
}
|
}
|
||||||
|
|
||||||
// deliverToArchive runs one delivery to a database target through
|
// deliverToArchive gives the setup's webhook a database target,
|
||||||
// the running engine and returns the webhook's archive file path.
|
// runs one delivery to it through the running engine, and returns
|
||||||
// The archive writer holds the file open afterwards.
|
// the target's ID and archive file path. The archive writer holds
|
||||||
func deliverToArchive(t *testing.T, s iSetup) string {
|
// the file open afterwards.
|
||||||
|
func deliverToArchive(t *testing.T, s iSetup) (string, string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
|
iCreateWebhook(t, s.MainDB, s.WebhookID, "hook")
|
||||||
|
|
||||||
|
tgt := &database.Target{
|
||||||
|
WebhookID: s.WebhookID,
|
||||||
|
Name: "archive",
|
||||||
|
Type: database.TargetTypeDatabase,
|
||||||
|
}
|
||||||
|
require.NoError(
|
||||||
|
t, s.MainDB.Omit(clause.Associations).Create(tgt).Error,
|
||||||
|
)
|
||||||
|
|
||||||
deliveryID, task := seedLogTask(t, s)
|
deliveryID, task := seedLogTask(t, s)
|
||||||
|
task.TargetID = tgt.ID
|
||||||
task.TargetType = database.TargetTypeDatabase
|
task.TargetType = database.TargetTypeDatabase
|
||||||
|
|
||||||
s.Engine.Notify([]delivery.Task{task})
|
s.Engine.Notify([]delivery.Task{task})
|
||||||
|
|
||||||
iWaitForDelivered(t, s.WebhookDB, deliveryID)
|
iWaitForDelivered(t, s.WebhookDB, deliveryID)
|
||||||
|
|
||||||
return filepath.Join(
|
return tgt.ID, filepath.Join(
|
||||||
filepath.Dir(s.DBMgr.DBPath(s.WebhookID)),
|
filepath.Dir(s.DBMgr.DBPath(s.WebhookID)),
|
||||||
fmt.Sprintf("archive-%s.db", s.WebhookID),
|
"archive-hook-archive-"+tgt.ID+".db",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -304,7 +317,7 @@ func TestEngine_StopHookClosesArchives(t *testing.T) {
|
|||||||
|
|
||||||
lc := startEngineViaHook(t, s.Engine)
|
lc := startEngineViaHook(t, s.Engine)
|
||||||
|
|
||||||
path := deliverToArchive(t, s)
|
_, path := deliverToArchive(t, s)
|
||||||
require.FileExists(
|
require.FileExists(
|
||||||
t, path+"-wal",
|
t, path+"-wal",
|
||||||
"an open archive should have a -wal for the stop to remove",
|
"an open archive should have a -wal for the stop to remove",
|
||||||
@@ -338,7 +351,7 @@ func TestEngine_StopHookTimeoutLeavesArchivesOpen(t *testing.T) {
|
|||||||
|
|
||||||
lc := startEngineViaHook(t, s.Engine)
|
lc := startEngineViaHook(t, s.Engine)
|
||||||
|
|
||||||
deliverToArchive(t, s)
|
targetID, _ := deliverToArchive(t, s)
|
||||||
|
|
||||||
release := make(chan struct{})
|
release := make(chan struct{})
|
||||||
|
|
||||||
@@ -352,7 +365,7 @@ func TestEngine_StopHookTimeoutLeavesArchivesOpen(t *testing.T) {
|
|||||||
requireStopHookExpires(t, lc.hooks[0], "delivery engine")
|
requireStopHookExpires(t, lc.hooks[0], "delivery engine")
|
||||||
|
|
||||||
require.True(
|
require.True(
|
||||||
t, s.Engine.ExportArchiveHandleOpen(s.WebhookID),
|
t, s.Engine.ExportArchiveHandleOpen(targetID),
|
||||||
"a stop that timed out must not close archive writers",
|
"a stop that timed out must not close archive writers",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -57,7 +57,10 @@ func testWebhookDB(t *testing.T) *gorm.DB {
|
|||||||
&database.Event{},
|
&database.Event{},
|
||||||
&database.Delivery{},
|
&database.Delivery{},
|
||||||
&database.DeliveryResult{},
|
&database.DeliveryResult{},
|
||||||
|
&database.EventTotals{},
|
||||||
|
&database.TargetTotals{},
|
||||||
))
|
))
|
||||||
|
require.NoError(t, db.Create(&database.EventTotals{}).Error)
|
||||||
|
|
||||||
return db
|
return db
|
||||||
}
|
}
|
||||||
@@ -348,23 +351,15 @@ func TestDeliverDatabase_ImmediateSuccess(
|
|||||||
|
|
||||||
db := testWebhookDB(t)
|
db := testWebhookDB(t)
|
||||||
|
|
||||||
// The database target archives for real now, so the engine
|
// The database target archives for real, so the engine needs
|
||||||
// needs a webhook DB manager to locate the data directory.
|
// the target in the main database and a data directory.
|
||||||
e := delivery.NewTestEngineWithDB(
|
env := setupArchiveTest(t)
|
||||||
nil,
|
tgt := env.seedDatabaseTarget(t, "")
|
||||||
database.NewTestWebhookDBManager(t.TempDir()),
|
|
||||||
slog.New(slog.NewTextHandler(
|
|
||||||
os.Stderr,
|
|
||||||
&slog.HandlerOptions{Level: slog.LevelDebug},
|
|
||||||
)),
|
|
||||||
&http.Client{Timeout: 5 * time.Second},
|
|
||||||
1,
|
|
||||||
)
|
|
||||||
|
|
||||||
event := seedEvent(t, db, `{"db":"target"}`)
|
event := seedEvent(t, db, `{"db":"target"}`)
|
||||||
d := seedDatabaseTargetDelivery(t, db, event, "")
|
d := seedDatabaseTargetDelivery(t, db, event, tgt)
|
||||||
|
|
||||||
e.ExportDeliverDatabase(db, d)
|
env.eng.ExportDeliverDatabase(db, d)
|
||||||
|
|
||||||
var updated database.Delivery
|
var updated database.Delivery
|
||||||
|
|
||||||
@@ -1333,32 +1328,27 @@ func TestProcessDelivery_RoutesToCorrectHandler(
|
|||||||
|
|
||||||
db := testWebhookDB(t)
|
db := testWebhookDB(t)
|
||||||
|
|
||||||
// The database target archives for real now, so the engine
|
// The database target archives for real, so the engine needs
|
||||||
// needs a webhook DB manager to locate the data directory.
|
// the target in the main database and a data directory.
|
||||||
e := delivery.NewTestEngineWithDB(
|
env := setupArchiveTest(t)
|
||||||
nil,
|
archive := env.seedDatabaseTarget(t, "")
|
||||||
database.NewTestWebhookDBManager(t.TempDir()),
|
|
||||||
slog.New(slog.NewTextHandler(
|
|
||||||
os.Stderr,
|
|
||||||
&slog.HandlerOptions{Level: slog.LevelDebug},
|
|
||||||
)),
|
|
||||||
&http.Client{Timeout: 5 * time.Second},
|
|
||||||
1,
|
|
||||||
)
|
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
targetType database.TargetType
|
targetType database.TargetType
|
||||||
|
targetID string
|
||||||
wantStatus database.DeliveryStatus
|
wantStatus database.DeliveryStatus
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
"database target",
|
"database target",
|
||||||
database.TargetTypeDatabase,
|
database.TargetTypeDatabase,
|
||||||
|
archive.ID,
|
||||||
database.DeliveryStatusDelivered,
|
database.DeliveryStatusDelivered,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"log target",
|
"log target",
|
||||||
database.TargetTypeLog,
|
database.TargetTypeLog,
|
||||||
|
uuid.New().String(),
|
||||||
database.DeliveryStatusDelivered,
|
database.DeliveryStatusDelivered,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -1368,7 +1358,7 @@ func TestProcessDelivery_RoutesToCorrectHandler(
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
runRoutingSubtest(
|
runRoutingSubtest(
|
||||||
t, db, e, tt.targetType,
|
t, db, env.eng, tt.targetType, tt.targetID,
|
||||||
tt.wantStatus,
|
tt.wantStatus,
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
@@ -1380,6 +1370,7 @@ func runRoutingSubtest(
|
|||||||
db *gorm.DB,
|
db *gorm.DB,
|
||||||
e *delivery.Engine,
|
e *delivery.Engine,
|
||||||
targetType database.TargetType,
|
targetType database.TargetType,
|
||||||
|
targetID string,
|
||||||
wantStatus database.DeliveryStatus,
|
wantStatus database.DeliveryStatus,
|
||||||
) {
|
) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
@@ -1387,8 +1378,7 @@ func runRoutingSubtest(
|
|||||||
event := seedEvent(t, db, `{"routing":"test"}`)
|
event := seedEvent(t, db, `{"routing":"test"}`)
|
||||||
|
|
||||||
dlv := seedDelivery(
|
dlv := seedDelivery(
|
||||||
t, db, event.ID,
|
t, db, event.ID, targetID,
|
||||||
uuid.New().String(),
|
|
||||||
database.DeliveryStatusPending,
|
database.DeliveryStatusPending,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -150,6 +150,16 @@ func (e *Engine) ExportDeliverSlack(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ExportUpdateDeliveryStatus exposes updateDeliveryStatus. It passes no
|
||||||
|
// target type, so no metric moves.
|
||||||
|
func (e *Engine) ExportUpdateDeliveryStatus(
|
||||||
|
webhookDB *gorm.DB,
|
||||||
|
d *database.Delivery,
|
||||||
|
status database.DeliveryStatus,
|
||||||
|
) error {
|
||||||
|
return e.updateDeliveryStatus(webhookDB, d, "", status)
|
||||||
|
}
|
||||||
|
|
||||||
// ExportProcessNewTask exposes processNewTask.
|
// ExportProcessNewTask exposes processNewTask.
|
||||||
func (e *Engine) ExportProcessNewTask(
|
func (e *Engine) ExportProcessNewTask(
|
||||||
ctx context.Context, task *Task,
|
ctx context.Context, task *Task,
|
||||||
@@ -463,7 +473,7 @@ func NewTestCircuitBreaker(
|
|||||||
type ExportArchivedEvent = archivedEvent
|
type ExportArchivedEvent = archivedEvent
|
||||||
|
|
||||||
// ExportArchiveWriter wraps an archiveWriter so black-box tests
|
// ExportArchiveWriter wraps an archiveWriter so black-box tests
|
||||||
// can exercise the per-webhook archive file mechanics.
|
// can exercise the archive file mechanics.
|
||||||
type ExportArchiveWriter struct {
|
type ExportArchiveWriter struct {
|
||||||
w *archiveWriter
|
w *archiveWriter
|
||||||
}
|
}
|
||||||
@@ -538,6 +548,12 @@ func (e *ExportArchiveWriter) Evict() {
|
|||||||
e.w.evict()
|
e.w.evict()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Rename gives the archive file a new name in the same directory,
|
||||||
|
// as a rename of the webhook or target does.
|
||||||
|
func (e *ExportArchiveWriter) Rename(name string) error {
|
||||||
|
return e.w.rename(name)
|
||||||
|
}
|
||||||
|
|
||||||
// HandleOpen reports whether the writer currently holds an open
|
// HandleOpen reports whether the writer currently holds an open
|
||||||
// archive handle.
|
// archive handle.
|
||||||
func (e *ExportArchiveWriter) HandleOpen() bool {
|
func (e *ExportArchiveWriter) HandleOpen() bool {
|
||||||
@@ -557,16 +573,16 @@ func (e *ExportArchiveWriter) Same(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ExportArchiveWriterFor returns the archive writer the registry
|
// ExportArchiveWriterFor returns the archive writer the registry
|
||||||
// currently caches for a webhook, or nil when none is cached. It
|
// currently caches for a database target, or nil when none is
|
||||||
// never creates one, so a test can hold a reference to the very
|
// cached. It never creates one, so a test can hold a reference to
|
||||||
// writer an eviction is about to detach.
|
// the very writer an eviction is about to detach.
|
||||||
func (e *Engine) ExportArchiveWriterFor(
|
func (e *Engine) ExportArchiveWriterFor(
|
||||||
webhookID string,
|
targetID string,
|
||||||
) *ExportArchiveWriter {
|
) *ExportArchiveWriter {
|
||||||
e.dbTarget.mu.Lock()
|
e.dbTarget.mu.Lock()
|
||||||
defer e.dbTarget.mu.Unlock()
|
defer e.dbTarget.mu.Unlock()
|
||||||
|
|
||||||
w, ok := e.dbTarget.writers[webhookID]
|
w, ok := e.dbTarget.writers[targetID]
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -575,26 +591,26 @@ func (e *Engine) ExportArchiveWriterFor(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ExportHasArchiveWriter reports whether the database target
|
// ExportHasArchiveWriter reports whether the database target
|
||||||
// currently caches an archive writer for a webhook.
|
// type currently caches an archive writer for a target.
|
||||||
func (e *Engine) ExportHasArchiveWriter(
|
func (e *Engine) ExportHasArchiveWriter(
|
||||||
webhookID string,
|
targetID string,
|
||||||
) bool {
|
) bool {
|
||||||
e.dbTarget.mu.Lock()
|
e.dbTarget.mu.Lock()
|
||||||
defer e.dbTarget.mu.Unlock()
|
defer e.dbTarget.mu.Unlock()
|
||||||
|
|
||||||
_, ok := e.dbTarget.writers[webhookID]
|
_, ok := e.dbTarget.writers[targetID]
|
||||||
|
|
||||||
return ok
|
return ok
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExportArchiveHandleOpen reports whether the cached archive
|
// ExportArchiveHandleOpen reports whether the cached archive
|
||||||
// writer for a webhook holds an open database handle. It
|
// writer for a target holds an open database handle. It
|
||||||
// returns false when no writer is cached.
|
// returns false when no writer is cached.
|
||||||
func (e *Engine) ExportArchiveHandleOpen(
|
func (e *Engine) ExportArchiveHandleOpen(
|
||||||
webhookID string,
|
targetID string,
|
||||||
) bool {
|
) bool {
|
||||||
e.dbTarget.mu.Lock()
|
e.dbTarget.mu.Lock()
|
||||||
w, ok := e.dbTarget.writers[webhookID]
|
w, ok := e.dbTarget.writers[targetID]
|
||||||
e.dbTarget.mu.Unlock()
|
e.dbTarget.mu.Unlock()
|
||||||
|
|
||||||
if !ok {
|
if !ok {
|
||||||
@@ -608,12 +624,12 @@ func (e *Engine) ExportArchiveHandleOpen(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ExportEnsureArchiveWriter creates (if needed) and returns the
|
// ExportEnsureArchiveWriter creates (if needed) and returns the
|
||||||
// archive file path of the cached writer for a webhook, so a
|
// archive file path of the cached writer for a target, so a
|
||||||
// test can prime the registry the way a delivery would.
|
// test can prime the registry the way a delivery would.
|
||||||
func (e *Engine) ExportEnsureArchiveWriter(
|
func (e *Engine) ExportEnsureArchiveWriter(
|
||||||
webhookID string,
|
targetID string,
|
||||||
) (string, error) {
|
) (string, error) {
|
||||||
w, err := e.dbTarget.writerFor(webhookID)
|
w, err := e.dbTarget.writerFor(targetID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
@@ -621,14 +637,14 @@ func (e *Engine) ExportEnsureArchiveWriter(
|
|||||||
return w.path, nil
|
return w.path, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExportSweepWriterFor takes a webhook's registry writer exactly
|
// ExportSweepWriterFor takes a target's registry writer exactly
|
||||||
// as the idle sweep does, reporting whether the sweep had to
|
// as the idle sweep does, reporting whether the sweep had to
|
||||||
// create the entry. It lets a test drive the registry through the
|
// create the entry. It lets a test drive the registry through the
|
||||||
// sweep's own entry point instead of choreographing goroutines.
|
// sweep's own entry point instead of choreographing goroutines.
|
||||||
func (e *Engine) ExportSweepWriterFor(
|
func (e *Engine) ExportSweepWriterFor(
|
||||||
webhookID string,
|
targetID string,
|
||||||
) (*ExportArchiveWriter, bool, error) {
|
) (*ExportArchiveWriter, bool, error) {
|
||||||
w, created, err := e.dbTarget.sweepWriterFor(webhookID)
|
w, created, err := e.dbTarget.sweepWriterFor(targetID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, false, err
|
return nil, false, err
|
||||||
}
|
}
|
||||||
@@ -639,9 +655,9 @@ func (e *Engine) ExportSweepWriterFor(
|
|||||||
// ExportReleaseSweepWriter releases a sweep-created registry entry
|
// ExportReleaseSweepWriter releases a sweep-created registry entry
|
||||||
// exactly as a finished sweep does.
|
// exactly as a finished sweep does.
|
||||||
func (e *Engine) ExportReleaseSweepWriter(
|
func (e *Engine) ExportReleaseSweepWriter(
|
||||||
webhookID string, w *ExportArchiveWriter,
|
targetID string, w *ExportArchiveWriter,
|
||||||
) {
|
) {
|
||||||
e.dbTarget.releaseSweepWriter(webhookID, w.w)
|
e.dbTarget.releaseSweepWriter(targetID, w.w)
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewTestArchiveSweeper builds an ArchiveSweeper backed by the
|
// NewTestArchiveSweeper builds an ArchiveSweeper backed by the
|
||||||
|
|||||||
@@ -43,6 +43,13 @@ var (
|
|||||||
// permit specific blocks out of this set with
|
// permit specific blocks out of this set with
|
||||||
// ALLOWED_EGRESS_CIDRS; see Guard.
|
// ALLOWED_EGRESS_CIDRS; see Guard.
|
||||||
//
|
//
|
||||||
|
// A public address belongs on the default blocklist only if it
|
||||||
|
// hands credentials, user data or bootstrap material to whatever
|
||||||
|
// can reach it, without the caller presenting anything. A
|
||||||
|
// provider's other public addresses are not refused, since
|
||||||
|
// reaching them can be legitimate and no list of them could be
|
||||||
|
// complete.
|
||||||
|
//
|
||||||
//nolint:gochecknoglobals // package-level network list is appropriate here
|
//nolint:gochecknoglobals // package-level network list is appropriate here
|
||||||
var blockedNetworks []*net.IPNet
|
var blockedNetworks []*net.IPNet
|
||||||
|
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -11,22 +12,75 @@ import (
|
|||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
)
|
)
|
||||||
|
|
||||||
// databaseTarget is a no-retry target that archives the
|
// archiveNameMaxLen is how many characters of a webhook or target
|
||||||
// full inbound event into a per-webhook archive SQLite file,
|
// name an archive file name keeps.
|
||||||
// separate from the per-webhook event database. The event is
|
const archiveNameMaxLen = 40
|
||||||
// already persisted in the per-webhook event DB by the time
|
|
||||||
// delivery runs; the database target additionally writes a
|
// databaseTarget is a no-retry target that archives the full
|
||||||
// durable long-term copy into archive-{webhookID}.db and then
|
// inbound event into the target's own archive SQLite file, separate
|
||||||
// records a single attempt whose outcome reflects whether the
|
// from the per-webhook event database. The event is already
|
||||||
// archive write succeeded. See archiveWriter for the
|
// persisted in the per-webhook event DB by the time delivery runs;
|
||||||
// close/reopen, auto-recreate, and expiry semantics.
|
// the database target additionally writes a durable long-term copy
|
||||||
|
// into the file ArchiveFileName names and then records a single
|
||||||
|
// attempt whose outcome reflects whether the archive write
|
||||||
|
// succeeded. See archiveWriter for the close/reopen, auto-recreate,
|
||||||
|
// and expiry semantics.
|
||||||
type databaseTarget struct {
|
type databaseTarget struct {
|
||||||
eng *Engine
|
eng *Engine
|
||||||
|
|
||||||
|
// writers holds one archive writer per database target, keyed
|
||||||
|
// by target ID.
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
writers map[string]*archiveWriter
|
writers map[string]*archiveWriter
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ArchiveFileName returns the file name of a database target's
|
||||||
|
// archive: archive-WEBHOOKNAME-TARGETNAME-TARGETID.db, with both
|
||||||
|
// names passed through archiveNamePart. The target ID keeps the
|
||||||
|
// name unique when two targets' names come out the same.
|
||||||
|
func ArchiveFileName(webhookName, targetName, targetID string) string {
|
||||||
|
return "archive-" + archiveNamePart(webhookName) + "-" +
|
||||||
|
archiveNamePart(targetName) + "-" + targetID + ".db"
|
||||||
|
}
|
||||||
|
|
||||||
|
// archiveNamePart makes a webhook or target name safe to put in a
|
||||||
|
// file name. It is lowercased; ASCII letters and digits are kept,
|
||||||
|
// every other run of characters becomes a single "-", and no "-" is
|
||||||
|
// left at either end. It is cut to archiveNameMaxLen characters, and
|
||||||
|
// a name with nothing left is "unnamed".
|
||||||
|
func archiveNamePart(name string) string {
|
||||||
|
var b strings.Builder
|
||||||
|
|
||||||
|
dash := false
|
||||||
|
|
||||||
|
for _, r := range strings.ToLower(name) {
|
||||||
|
if (r < 'a' || r > 'z') && (r < '0' || r > '9') {
|
||||||
|
dash = b.Len() > 0
|
||||||
|
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if dash {
|
||||||
|
b.WriteByte('-')
|
||||||
|
|
||||||
|
dash = false
|
||||||
|
}
|
||||||
|
|
||||||
|
b.WriteRune(r)
|
||||||
|
}
|
||||||
|
|
||||||
|
part := b.String()
|
||||||
|
if len(part) > archiveNameMaxLen {
|
||||||
|
part = strings.TrimRight(part[:archiveNameMaxLen], "-")
|
||||||
|
}
|
||||||
|
|
||||||
|
if part == "" {
|
||||||
|
return "unnamed"
|
||||||
|
}
|
||||||
|
|
||||||
|
return part
|
||||||
|
}
|
||||||
|
|
||||||
// Deliver implements Target. It archives the event, then
|
// Deliver implements Target. It archives the event, then
|
||||||
// records one successful attempt and marks the delivery
|
// records one successful attempt and marks the delivery
|
||||||
// delivered. An archiving error fails the delivery: the
|
// delivered. An archiving error fails the delivery: the
|
||||||
@@ -92,7 +146,7 @@ func (t *databaseTarget) Deliver(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// archive writes the full event as a row into the webhook's
|
// archive writes the full event as a row into the target's
|
||||||
// archive database, honouring the optional per-target expiry
|
// archive database, honouring the optional per-target expiry
|
||||||
// parsed from the target config JSON.
|
// parsed from the target config JSON.
|
||||||
func (t *databaseTarget) archive(d *database.Delivery) error {
|
func (t *databaseTarget) archive(d *database.Delivery) error {
|
||||||
@@ -106,7 +160,7 @@ func (t *databaseTarget) archive(d *database.Delivery) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
w, err := t.writerFor(webhookID)
|
w, err := t.writerFor(d.TargetID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -124,30 +178,31 @@ func (t *databaseTarget) archive(d *database.Delivery) error {
|
|||||||
return w.write(row, expiry)
|
return w.write(row, expiry)
|
||||||
}
|
}
|
||||||
|
|
||||||
// writerFor returns the archiveWriter for a webhook, creating
|
// writerFor returns the archive writer for a database target,
|
||||||
// and caching it on first use. Each webhook has one writer so
|
// creating and caching it on first use. Each target has one writer
|
||||||
// its close/reopen debounce state is shared across concurrent
|
// so its close/reopen debounce state is shared across concurrent
|
||||||
// deliveries. The archive file lives beside the per-webhook
|
// deliveries, and so a rename and the idle sweep take the same lock
|
||||||
// event database in the data directory.
|
// as its writes.
|
||||||
func (t *databaseTarget) writerFor(
|
func (t *databaseTarget) writerFor(
|
||||||
webhookID string,
|
targetID string,
|
||||||
) (*archiveWriter, error) {
|
) (*archiveWriter, error) {
|
||||||
path, err := t.archivePath(webhookID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
t.mu.Lock()
|
t.mu.Lock()
|
||||||
defer t.mu.Unlock()
|
defer t.mu.Unlock()
|
||||||
|
|
||||||
if t.writers == nil {
|
w, ok := t.writers[targetID]
|
||||||
t.writers = make(map[string]*archiveWriter)
|
|
||||||
}
|
|
||||||
|
|
||||||
w, ok := t.writers[webhookID]
|
|
||||||
if !ok {
|
if !ok {
|
||||||
w = newArchiveWriter(path, t.eng.log)
|
var err error
|
||||||
t.writers[webhookID] = w
|
|
||||||
|
w, err = t.newWriter(targetID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if t.writers == nil {
|
||||||
|
t.writers = make(map[string]*archiveWriter)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.writers[targetID] = w
|
||||||
}
|
}
|
||||||
|
|
||||||
// A delivery claims the entry: even if the idle sweep created
|
// A delivery claims the entry: even if the idle sweep created
|
||||||
@@ -159,40 +214,39 @@ func (t *databaseTarget) writerFor(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// sweepWriterFor returns the archive writer the idle sweep should
|
// sweepWriterFor returns the archive writer the idle sweep should
|
||||||
// prune a webhook through, together with whether the sweep itself
|
// prune a target's archive through, together with whether the sweep
|
||||||
// created the registry entry.
|
// itself created the registry entry.
|
||||||
//
|
//
|
||||||
// The sweep must route its prune through the registered writer so
|
// The sweep must route its prune through the registered writer so
|
||||||
// the writer's mutex orders it against concurrent writes, but it
|
// the writer's mutex orders it against concurrent writes, but it
|
||||||
// must never leave a registry entry behind: a sweep that ran
|
// must never leave a registry entry behind: a sweep that ran
|
||||||
// concurrently with the webhook's deletion would otherwise
|
// concurrently with the target's deletion would otherwise
|
||||||
// re-create an entry that nothing will ever evict again, which is
|
// re-create an entry that nothing will ever evict again, which is
|
||||||
// exactly the leak eviction exists to prevent. An entry the sweep
|
// exactly the leak eviction exists to prevent. An entry the sweep
|
||||||
// creates is therefore marked sweep-owned and handed back to
|
// creates is therefore marked sweep-owned and handed back to
|
||||||
// releaseSweepWriter when the sweep is done.
|
// releaseSweepWriter when the sweep is done.
|
||||||
func (t *databaseTarget) sweepWriterFor(
|
func (t *databaseTarget) sweepWriterFor(
|
||||||
webhookID string,
|
targetID string,
|
||||||
) (*archiveWriter, bool, error) {
|
) (*archiveWriter, bool, error) {
|
||||||
path, err := t.archivePath(webhookID)
|
t.mu.Lock()
|
||||||
|
defer t.mu.Unlock()
|
||||||
|
|
||||||
|
w, ok := t.writers[targetID]
|
||||||
|
if ok {
|
||||||
|
return w, false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
w, err := t.newWriter(targetID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, false, err
|
return nil, false, err
|
||||||
}
|
}
|
||||||
|
|
||||||
t.mu.Lock()
|
|
||||||
defer t.mu.Unlock()
|
|
||||||
|
|
||||||
if t.writers == nil {
|
if t.writers == nil {
|
||||||
t.writers = make(map[string]*archiveWriter)
|
t.writers = make(map[string]*archiveWriter)
|
||||||
}
|
}
|
||||||
|
|
||||||
w, ok := t.writers[webhookID]
|
|
||||||
if ok {
|
|
||||||
return w, false, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
w = newArchiveWriter(path, t.eng.log)
|
|
||||||
w.sweepOwned = true
|
w.sweepOwned = true
|
||||||
t.writers[webhookID] = w
|
t.writers[targetID] = w
|
||||||
|
|
||||||
return w, true, nil
|
return w, true, nil
|
||||||
}
|
}
|
||||||
@@ -209,57 +263,95 @@ func (t *databaseTarget) sweepWriterFor(
|
|||||||
// delivery that adopted the writer keeps a registered, evictable
|
// delivery that adopted the writer keeps a registered, evictable
|
||||||
// one.
|
// one.
|
||||||
func (t *databaseTarget) releaseSweepWriter(
|
func (t *databaseTarget) releaseSweepWriter(
|
||||||
webhookID string, w *archiveWriter,
|
targetID string, w *archiveWriter,
|
||||||
) {
|
) {
|
||||||
t.mu.Lock()
|
t.mu.Lock()
|
||||||
defer t.mu.Unlock()
|
defer t.mu.Unlock()
|
||||||
|
|
||||||
cur, ok := t.writers[webhookID]
|
cur, ok := t.writers[targetID]
|
||||||
if !ok || cur != w || !cur.sweepOwned {
|
if !ok || cur != w || !cur.sweepOwned {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
delete(t.writers, webhookID)
|
delete(t.writers, targetID)
|
||||||
}
|
}
|
||||||
|
|
||||||
// archivePath returns the archive file path for a webhook: it
|
// newWriter builds the writer for a database target's archive. The
|
||||||
// lives beside the per-webhook event database in the data
|
// file lives beside the webhook's event database in the data
|
||||||
// directory. It does not touch the filesystem.
|
// directory and is named for the webhook and the target as the main
|
||||||
func (t *databaseTarget) archivePath(
|
// database has them now; from then on only rename changes the name
|
||||||
webhookID string,
|
// the writer uses. It does not touch the archive file.
|
||||||
) (string, error) {
|
func (t *databaseTarget) newWriter(
|
||||||
|
targetID string,
|
||||||
|
) (*archiveWriter, error) {
|
||||||
if t.eng.dbManager == nil {
|
if t.eng.dbManager == nil {
|
||||||
return "", errArchiveNoDataDir
|
return nil, errArchiveNoDataDir
|
||||||
}
|
}
|
||||||
|
|
||||||
dir := filepath.Dir(t.eng.dbManager.DBPath(webhookID))
|
var target database.Target
|
||||||
|
|
||||||
return filepath.Join(
|
err := t.eng.database.DB().
|
||||||
dir, fmt.Sprintf("archive-%s.db", webhookID),
|
Preload("Webhook").
|
||||||
), nil
|
First(&target, "id = ?", targetID).Error
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf(
|
||||||
|
"loading database target %s: %w", targetID, err,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
dir := filepath.Dir(t.eng.dbManager.DBPath(target.WebhookID))
|
||||||
|
name := ArchiveFileName(
|
||||||
|
target.Webhook.Name, target.Name, target.ID,
|
||||||
|
)
|
||||||
|
|
||||||
|
w := newArchiveWriter(filepath.Join(dir, name), t.eng.log)
|
||||||
|
w.webhookID = target.WebhookID
|
||||||
|
|
||||||
|
return w, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// evict drops a webhook's archive writer from the registry and
|
// rename moves a database target's archive file to the name for
|
||||||
// closes its handle, so a deleted webhook does not leave a
|
// webhookName and targetName. It goes through the target's writer,
|
||||||
// writer (and an open archive handle within its debounce
|
// so the move holds the lock that writes and the idle sweep take,
|
||||||
// window) alive for the process lifetime.
|
// and later writes use the new name.
|
||||||
|
//
|
||||||
|
// The writer is created if there is none, and it stays cached. The
|
||||||
|
// handlers rename before they save the new name, so until the save
|
||||||
|
// the main database still has the old one; a delivery in that window
|
||||||
|
// must find this writer rather than build one from the old name.
|
||||||
|
func (t *databaseTarget) rename(
|
||||||
|
targetID, webhookName, targetName string,
|
||||||
|
) error {
|
||||||
|
w, err := t.writerFor(targetID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return w.rename(ArchiveFileName(webhookName, targetName, targetID))
|
||||||
|
}
|
||||||
|
|
||||||
|
// evict drops a database target's archive writer from the registry
|
||||||
|
// and closes its handle, so a deleted target does not leave a
|
||||||
|
// writer (and an open archive handle within its debounce window)
|
||||||
|
// alive for the process lifetime.
|
||||||
//
|
//
|
||||||
// The map entry is removed under the registry lock, which is
|
// The map entry is removed under the registry lock, which is
|
||||||
// then released before the handle is closed under the writer's
|
// then released before the handle is closed under the writer's
|
||||||
// own lock: that ordering keeps the registry available to other
|
// own lock: that ordering keeps the registry available to other
|
||||||
// webhooks while an in-flight write on this one drains, and
|
// targets while an in-flight write on this one drains, and
|
||||||
// closing under the writer's lock means eviction can never race
|
// closing under the writer's lock means eviction can never race
|
||||||
// a write.
|
// a write.
|
||||||
//
|
//
|
||||||
// Eviction is idempotent and silent for a webhook with no
|
// Eviction is idempotent and silent for a target with no writer,
|
||||||
// writer, which is the common case: a webhook with no database
|
// which is the common case: only a database target that has
|
||||||
// target never creates one. It never deletes the archive file.
|
// received an event or been renamed has one. It never deletes the
|
||||||
func (t *databaseTarget) evict(webhookID string) {
|
// archive file.
|
||||||
|
func (t *databaseTarget) evict(targetID string) {
|
||||||
t.mu.Lock()
|
t.mu.Lock()
|
||||||
|
|
||||||
w, ok := t.writers[webhookID]
|
w, ok := t.writers[targetID]
|
||||||
if ok {
|
if ok {
|
||||||
delete(t.writers, webhookID)
|
delete(t.writers, targetID)
|
||||||
}
|
}
|
||||||
|
|
||||||
t.mu.Unlock()
|
t.mu.Unlock()
|
||||||
@@ -272,13 +364,41 @@ func (t *databaseTarget) evict(webhookID string) {
|
|||||||
|
|
||||||
t.eng.log.Info(
|
t.eng.log.Info(
|
||||||
"evicted archive writer",
|
"evicted archive writer",
|
||||||
"webhook_id", webhookID,
|
"target_id", targetID,
|
||||||
"path", w.path,
|
"path", w.path,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// evictWebhook evicts, exactly as evict does, the writer of every
|
||||||
|
// database target of a webhook.
|
||||||
|
func (t *databaseTarget) evictWebhook(webhookID string) {
|
||||||
|
t.mu.Lock()
|
||||||
|
|
||||||
|
var gone []*archiveWriter
|
||||||
|
|
||||||
|
for targetID, w := range t.writers {
|
||||||
|
if w.webhookID == webhookID {
|
||||||
|
delete(t.writers, targetID)
|
||||||
|
|
||||||
|
gone = append(gone, w)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
t.mu.Unlock()
|
||||||
|
|
||||||
|
for _, w := range gone {
|
||||||
|
w.evict()
|
||||||
|
|
||||||
|
t.eng.log.Info(
|
||||||
|
"evicted archive writer",
|
||||||
|
"webhook_id", webhookID,
|
||||||
|
"path", w.path,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// evictAll evicts every cached archive writer, exactly as evict
|
// evictAll evicts every cached archive writer, exactly as evict
|
||||||
// does for one webhook. The engine calls it at shutdown, once its
|
// does for one target. The engine calls it at shutdown, once its
|
||||||
// workers have returned. Closing the last handle on an archive
|
// workers have returned. Closing the last handle on an archive
|
||||||
// moves the contents of its -wal into the .db and removes the
|
// moves the contents of its -wal into the .db and removes the
|
||||||
// -wal, so a clean stop leaves each archive as a single file.
|
// -wal, so a clean stop leaves each archive as a single file.
|
||||||
@@ -295,38 +415,25 @@ func (t *databaseTarget) evictAll() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// sweepWebhook prunes one webhook's archive of rows older than
|
// sweepArchive prunes one database target's archive of rows older
|
||||||
// expiry, without requiring a write. It returns nil (nothing to
|
// than expiry, without requiring a write. A missing archive file is
|
||||||
// do) when the archive file does not exist, so a sweep never
|
// left missing (see sweepExpired), so a sweep never creates an
|
||||||
// creates an archive for a webhook that has a database target
|
// archive for a target that has never received an event.
|
||||||
// but has never received an event.
|
|
||||||
//
|
//
|
||||||
// It also never leaves a registry entry behind: an entry it had
|
// It also never leaves a registry entry behind: an entry it had
|
||||||
// to create to reach the writer's mutex is released again once
|
// to create to reach the writer's mutex is released again once
|
||||||
// the prune is done, so a sweep racing a webhook deletion cannot
|
// the prune is done, so a sweep racing a target deletion cannot
|
||||||
// resurrect the writer the eviction just dropped.
|
// resurrect the writer the eviction just dropped.
|
||||||
func (t *databaseTarget) sweepWebhook(
|
func (t *databaseTarget) sweepArchive(
|
||||||
webhookID string, expiry time.Duration,
|
targetID string, expiry time.Duration,
|
||||||
) error {
|
) error {
|
||||||
path, err := t.archivePath(webhookID)
|
w, created, err := t.sweepWriterFor(targetID)
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check before taking a writer at all: a webhook whose
|
|
||||||
// archive has never been created gets no writer, no handle,
|
|
||||||
// and no file.
|
|
||||||
if !fileExists(path) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
w, created, err := t.sweepWriterFor(webhookID)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if created {
|
if created {
|
||||||
defer t.releaseSweepWriter(webhookID, w)
|
defer t.releaseSweepWriter(targetID, w)
|
||||||
}
|
}
|
||||||
|
|
||||||
return w.sweepExpired(expiry)
|
return w.sweepExpired(expiry)
|
||||||
|
|||||||
@@ -4,8 +4,10 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -41,7 +43,7 @@ const (
|
|||||||
|
|
||||||
var (
|
var (
|
||||||
// errArchiveMissingWebhookID is returned when an event to
|
// errArchiveMissingWebhookID is returned when an event to
|
||||||
// archive has no webhook id to key its archive file on.
|
// archive has no webhook id to record in its archive row.
|
||||||
errArchiveMissingWebhookID = errors.New(
|
errArchiveMissingWebhookID = errors.New(
|
||||||
"cannot archive event without a webhook id",
|
"cannot archive event without a webhook id",
|
||||||
)
|
)
|
||||||
@@ -61,10 +63,9 @@ var (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// errArchiveWriterEvicted is returned when a writer that has
|
// errArchiveWriterEvicted is returned when a writer that has
|
||||||
// been evicted (its webhook was deleted, or its last database
|
// been evicted (its target or its webhook was deleted) is used
|
||||||
// target was removed) is used again. An evicted writer is no
|
// again. An evicted writer is no longer in the registry, so
|
||||||
// longer in the registry, so reopening its file would leak a
|
// reopening its file would leak a handle nothing owns.
|
||||||
// handle nothing owns.
|
|
||||||
errArchiveWriterEvicted = errors.New(
|
errArchiveWriterEvicted = errors.New(
|
||||||
"archive writer has been evicted",
|
"archive writer has been evicted",
|
||||||
)
|
)
|
||||||
@@ -80,7 +81,7 @@ type databaseTargetConfig struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// archivedEvent is one fully captured webhook event stored in a
|
// archivedEvent is one fully captured webhook event stored in a
|
||||||
// per-webhook archive database for long-term retention. It is a
|
// database target's archive for long-term retention. It is a
|
||||||
// self-contained copy — independent of the per-webhook event
|
// self-contained copy — independent of the per-webhook event
|
||||||
// database, which may prune events under its own retention.
|
// database, which may prune events under its own retention.
|
||||||
type archivedEvent struct {
|
type archivedEvent struct {
|
||||||
@@ -170,8 +171,8 @@ func ValidateArchiveExpiry(expiry string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// archiveWriter owns one per-webhook archive SQLite file. It
|
// archiveWriter owns one database target's archive SQLite file.
|
||||||
// serialises writes, and after each write closes and reopens
|
// It serialises writes, and after each write closes and reopens
|
||||||
// the file (debounced to at most once per debounce window) so
|
// the file (debounced to at most once per debounce window) so
|
||||||
// an operator can move the file away for offline archiving. The
|
// an operator can move the file away for offline archiving. The
|
||||||
// next write recreates a moved or removed file, because the
|
// next write recreates a moved or removed file, because the
|
||||||
@@ -187,16 +188,21 @@ type archiveWriter struct {
|
|||||||
reopens int
|
reopens int
|
||||||
|
|
||||||
// evicted marks a writer that has been removed from the
|
// evicted marks a writer that has been removed from the
|
||||||
// per-webhook registry. Its handle is closed and it must
|
// registry. Its handle is closed and it must never open the
|
||||||
// never open the file again: nothing holds it any more, so a
|
// file again: nothing holds it any more, so a reopen would
|
||||||
// reopen would leak the handle for the process lifetime.
|
// leak the handle for the process lifetime.
|
||||||
evicted bool
|
evicted bool
|
||||||
|
|
||||||
|
// webhookID is the webhook the archive's target belongs to,
|
||||||
|
// so deleting the webhook can find its writers. It is set
|
||||||
|
// when the writer is created and never changes.
|
||||||
|
webhookID string
|
||||||
|
|
||||||
// sweepOwned marks a registry entry that the idle sweep
|
// sweepOwned marks a registry entry that the idle sweep
|
||||||
// created because no writer was cached for the webhook. The
|
// created because no writer was cached for the target. The
|
||||||
// sweep removes such an entry again when it is done, so a
|
// sweep removes such an entry again when it is done, so a
|
||||||
// sweep can never leave — or resurrect — a registry entry
|
// sweep can never leave — or resurrect — a registry entry
|
||||||
// for a webhook that has been deleted. A delivery that adopts
|
// for a target that has been deleted. A delivery that adopts
|
||||||
// the writer clears the flag, handing the entry to the
|
// the writer clears the flag, handing the entry to the
|
||||||
// registry proper.
|
// registry proper.
|
||||||
//
|
//
|
||||||
@@ -385,11 +391,49 @@ func (w *archiveWriter) sweepExpired(expiry time.Duration) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// rename gives the archive file a new name in the same directory,
|
||||||
|
// and the writer uses the file under that name from now on. The
|
||||||
|
// handle is closed first, which folds the -wal into the .db; any
|
||||||
|
// -wal or -shm still beside the file (left by a crash) is moved with
|
||||||
|
// it, because SQLite finds them by name. A missing file is not an
|
||||||
|
// error: the operator may have moved it away, and the next write
|
||||||
|
// creates it under the new name.
|
||||||
|
func (w *archiveWriter) rename(name string) error {
|
||||||
|
w.mu.Lock()
|
||||||
|
defer w.mu.Unlock()
|
||||||
|
|
||||||
|
if w.evicted {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"%w: %s", errArchiveWriterEvicted, w.path,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
path := filepath.Join(filepath.Dir(w.path), name)
|
||||||
|
if path == w.path {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
w.close()
|
||||||
|
|
||||||
|
for _, suffix := range []string{"", "-wal", "-shm"} {
|
||||||
|
err := os.Rename(w.path+suffix, path+suffix)
|
||||||
|
if err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"renaming archive %s to %s: %w", w.path, path, err,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
w.path = path
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// evict closes the writer's handle and marks it unusable. It is
|
// evict closes the writer's handle and marks it unusable. It is
|
||||||
// called when the writer leaves the registry, either because the
|
// called when the writer leaves the registry, because its target
|
||||||
// webhook was deleted or because its last database target was
|
// or its webhook was deleted, or at shutdown. The archive FILE is
|
||||||
// removed. The archive FILE is deliberately left on disk: it is
|
// deliberately left on disk: it is long-term storage an operator
|
||||||
// long-term storage an operator may still want.
|
// may still want.
|
||||||
func (w *archiveWriter) evict() {
|
func (w *archiveWriter) evict() {
|
||||||
w.mu.Lock()
|
w.mu.Lock()
|
||||||
defer w.mu.Unlock()
|
defer w.mu.Unlock()
|
||||||
|
|||||||
@@ -17,85 +17,109 @@ import (
|
|||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
)
|
)
|
||||||
|
|
||||||
// evictTestEngine builds an engine backed by a temporary data
|
// deliverTo archives one event to a database target, leaving the
|
||||||
// directory and returns it along with that directory.
|
// target's writer cached with its handle open.
|
||||||
func evictTestEngine(t *testing.T) (*delivery.Engine, string) {
|
func deliverTo(
|
||||||
|
t *testing.T, env *archiveEnv, tgt *database.Target,
|
||||||
|
) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
dataDir := t.TempDir()
|
|
||||||
|
|
||||||
eng := delivery.NewTestEngineWithDB(
|
|
||||||
nil,
|
|
||||||
database.NewTestWebhookDBManager(dataDir),
|
|
||||||
archiveTestLogger(),
|
|
||||||
&http.Client{Timeout: 5 * time.Second},
|
|
||||||
1,
|
|
||||||
)
|
|
||||||
|
|
||||||
return eng, dataDir
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEvictWebhook_ClosesAndRemovesWriter proves that evicting
|
|
||||||
// a webhook drops its archive writer from the registry and
|
|
||||||
// closes the open archive handle, rather than leaving both
|
|
||||||
// alive for the process lifetime.
|
|
||||||
func TestEvictWebhook_ClosesAndRemovesWriter(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
eng, dataDir := evictTestEngine(t)
|
|
||||||
|
|
||||||
webhookDB := testWebhookDB(t)
|
webhookDB := testWebhookDB(t)
|
||||||
event := seedEvent(t, webhookDB, `{"archived":true}`)
|
event := seedEvent(t, webhookDB, `{"archived":true}`)
|
||||||
d := seedDatabaseTargetDelivery(t, webhookDB, event, "")
|
|
||||||
|
|
||||||
eng.ExportDeliverDatabase(webhookDB, d)
|
env.eng.ExportDeliverDatabase(
|
||||||
|
webhookDB, seedDatabaseTargetDelivery(t, webhookDB, event, tgt),
|
||||||
webhookID := event.WebhookID
|
|
||||||
|
|
||||||
require.True(
|
|
||||||
t, eng.ExportHasArchiveWriter(webhookID),
|
|
||||||
"a delivery should have cached an archive writer",
|
|
||||||
)
|
|
||||||
require.True(
|
|
||||||
t, eng.ExportArchiveHandleOpen(webhookID),
|
|
||||||
"the writer should hold an open handle after a write",
|
|
||||||
)
|
)
|
||||||
|
}
|
||||||
|
|
||||||
eng.EvictWebhook(webhookID)
|
// TestEvictWebhook_ClosesAndRemovesWriter proves that evicting
|
||||||
|
// a webhook drops the archive writers of its database targets
|
||||||
|
// from the registry and closes their open handles, rather than
|
||||||
|
// leaving them alive for the process lifetime, and leaves another
|
||||||
|
// webhook's writer alone.
|
||||||
|
func TestEvictWebhook_ClosesAndRemovesWriter(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
assert.False(
|
env := setupArchiveTest(t)
|
||||||
t, eng.ExportHasArchiveWriter(webhookID),
|
first := env.seedDatabaseTarget(t, "")
|
||||||
"eviction should remove the registry entry",
|
second := env.addDatabaseTarget(t, first.WebhookID, "")
|
||||||
)
|
other := env.seedDatabaseTarget(t, "")
|
||||||
assert.False(
|
|
||||||
t, eng.ExportArchiveHandleOpen(webhookID),
|
|
||||||
"eviction should close the archive handle",
|
|
||||||
)
|
|
||||||
|
|
||||||
archivePath := filepath.Join(
|
for _, tgt := range []*database.Target{first, second, other} {
|
||||||
dataDir, fmt.Sprintf("archive-%s.db", webhookID),
|
deliverTo(t, env, tgt)
|
||||||
|
|
||||||
|
require.True(
|
||||||
|
t, env.eng.ExportArchiveHandleOpen(tgt.ID),
|
||||||
|
"the writer should hold an open handle after a write",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
env.eng.EvictWebhook(first.WebhookID)
|
||||||
|
|
||||||
|
for _, tgt := range []*database.Target{first, second} {
|
||||||
|
assert.False(
|
||||||
|
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||||
|
"eviction should remove the registry entry",
|
||||||
|
)
|
||||||
|
assert.False(
|
||||||
|
t, env.eng.ExportArchiveHandleOpen(tgt.ID),
|
||||||
|
"eviction should close the archive handle",
|
||||||
|
)
|
||||||
|
assert.FileExists(
|
||||||
|
t, env.archivePath(tgt),
|
||||||
|
"eviction must not delete the archive file",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.True(
|
||||||
|
t, env.eng.ExportArchiveHandleOpen(other.ID),
|
||||||
|
"another webhook's writer must be left alone",
|
||||||
)
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestEvictTarget_LeavesOtherTargets proves that evicting one
|
||||||
|
// database target leaves the writer of another target of the same
|
||||||
|
// webhook in place.
|
||||||
|
func TestEvictTarget_LeavesOtherTargets(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
env := setupArchiveTest(t)
|
||||||
|
doomed := env.seedDatabaseTarget(t, "")
|
||||||
|
kept := env.addDatabaseTarget(t, doomed.WebhookID, "")
|
||||||
|
|
||||||
|
deliverTo(t, env, doomed)
|
||||||
|
deliverTo(t, env, kept)
|
||||||
|
|
||||||
|
env.eng.EvictTarget(doomed.ID)
|
||||||
|
|
||||||
|
assert.False(t, env.eng.ExportHasArchiveWriter(doomed.ID))
|
||||||
assert.FileExists(
|
assert.FileExists(
|
||||||
t, archivePath,
|
t, env.archivePath(doomed),
|
||||||
"eviction must not delete the archive file",
|
"eviction must not delete the archive file",
|
||||||
)
|
)
|
||||||
|
assert.True(
|
||||||
|
t, env.eng.ExportArchiveHandleOpen(kept.ID),
|
||||||
|
"the other target's writer must be left alone",
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestEvictWebhook_UnknownWebhookIsNoOp proves eviction is safe
|
// TestEvictWebhook_UnknownWebhookIsNoOp proves eviction is safe
|
||||||
// for the common case of a webhook that never had a database
|
// for the common case of a webhook or target that never had an
|
||||||
// target, and that repeating it does not panic.
|
// archive writer, and that repeating it does not panic.
|
||||||
func TestEvictWebhook_UnknownWebhookIsNoOp(t *testing.T) {
|
func TestEvictWebhook_UnknownWebhookIsNoOp(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
eng, _ := evictTestEngine(t)
|
env := setupArchiveTest(t)
|
||||||
|
|
||||||
assert.NotPanics(t, func() {
|
assert.NotPanics(t, func() {
|
||||||
eng.EvictWebhook("no-such-webhook")
|
env.eng.EvictWebhook("no-such-webhook")
|
||||||
eng.EvictWebhook("no-such-webhook")
|
env.eng.EvictWebhook("no-such-webhook")
|
||||||
|
env.eng.EvictTarget("no-such-target")
|
||||||
|
env.eng.EvictTarget("no-such-target")
|
||||||
})
|
})
|
||||||
|
|
||||||
assert.False(
|
assert.False(
|
||||||
t, eng.ExportHasArchiveWriter("no-such-webhook"),
|
t, env.eng.ExportHasArchiveWriter("no-such-target"),
|
||||||
"eviction must not create a writer",
|
"eviction must not create a writer",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -289,17 +313,14 @@ func TestEvictWebhook_RacingWriteDoesNotReopenHandle(
|
|||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
eng, _ := evictTestEngine(t)
|
env := setupArchiveTest(t)
|
||||||
|
tgt := env.seedDatabaseTarget(t, "")
|
||||||
webhookDB := testWebhookDB(t)
|
|
||||||
event := seedEvent(t, webhookDB, `{"archived":true}`)
|
|
||||||
d := seedDatabaseTargetDelivery(t, webhookDB, event, "")
|
|
||||||
|
|
||||||
// Prime the registry so the test can hold the very writer the
|
// Prime the registry so the test can hold the very writer the
|
||||||
// eviction is about to detach.
|
// eviction is about to detach.
|
||||||
eng.ExportDeliverDatabase(webhookDB, d)
|
deliverTo(t, env, tgt)
|
||||||
|
|
||||||
w := eng.ExportArchiveWriterFor(event.WebhookID)
|
w := env.eng.ExportArchiveWriterFor(tgt.ID)
|
||||||
require.NotNil(t, w)
|
require.NotNil(t, w)
|
||||||
require.True(t, w.HandleOpen())
|
require.True(t, w.HandleOpen())
|
||||||
|
|
||||||
@@ -309,7 +330,7 @@ func TestEvictWebhook_RacingWriteDoesNotReopenHandle(
|
|||||||
// eviction has to contend for the writer's mutex.
|
// eviction has to contend for the writer's mutex.
|
||||||
race.awaitFirstWrite()
|
race.awaitFirstWrite()
|
||||||
|
|
||||||
eng.EvictWebhook(event.WebhookID)
|
env.eng.EvictWebhook(tgt.WebhookID)
|
||||||
|
|
||||||
sawEvicted, otherErr := race.wait()
|
sawEvicted, otherErr := race.wait()
|
||||||
|
|
||||||
@@ -324,41 +345,33 @@ func TestEvictWebhook_RacingWriteDoesNotReopenHandle(
|
|||||||
"been evicted",
|
"been evicted",
|
||||||
)
|
)
|
||||||
assert.False(
|
assert.False(
|
||||||
t, eng.ExportHasArchiveWriter(event.WebhookID),
|
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||||
"the registry entry must stay gone",
|
"the registry entry must stay gone",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestEvictWebhook_LaterDeliveryRecreatesWriter proves eviction
|
// TestEvictWebhook_LaterDeliveryRecreatesWriter proves eviction
|
||||||
// does not break archiving for a webhook that is still alive: a
|
// does not break archiving for a target that is still alive: a
|
||||||
// subsequent delivery gets a brand new writer from the registry.
|
// subsequent delivery gets a brand new writer from the registry.
|
||||||
// It says nothing about the evicted writer itself — that is what
|
// It says nothing about the evicted writer itself — that is what
|
||||||
// TestEvictedWriter_WriteDoesNotReopenFile covers.
|
// TestEvictedWriter_WriteDoesNotReopenFile covers.
|
||||||
func TestEvictWebhook_LaterDeliveryRecreatesWriter(t *testing.T) {
|
func TestEvictWebhook_LaterDeliveryRecreatesWriter(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
eng, _ := evictTestEngine(t)
|
env := setupArchiveTest(t)
|
||||||
|
tgt := env.seedDatabaseTarget(t, "")
|
||||||
|
|
||||||
webhookDB := testWebhookDB(t)
|
deliverTo(t, env, tgt)
|
||||||
event := seedEvent(t, webhookDB, `{"archived":true}`)
|
require.True(t, env.eng.ExportHasArchiveWriter(tgt.ID))
|
||||||
d := seedDatabaseTargetDelivery(t, webhookDB, event, "")
|
|
||||||
|
|
||||||
eng.ExportDeliverDatabase(webhookDB, d)
|
env.eng.EvictWebhook(tgt.WebhookID)
|
||||||
require.True(
|
|
||||||
t, eng.ExportHasArchiveWriter(event.WebhookID),
|
|
||||||
)
|
|
||||||
|
|
||||||
eng.EvictWebhook(event.WebhookID)
|
// A fresh delivery for the same target gets a brand new
|
||||||
|
|
||||||
// A fresh delivery for the same webhook gets a brand new
|
|
||||||
// writer from the registry, so archiving keeps working.
|
// writer from the registry, so archiving keeps working.
|
||||||
second := seedDatabaseTargetDelivery(
|
deliverTo(t, env, tgt)
|
||||||
t, webhookDB, event, "",
|
|
||||||
)
|
|
||||||
eng.ExportDeliverDatabase(webhookDB, second)
|
|
||||||
|
|
||||||
assert.True(
|
assert.True(
|
||||||
t, eng.ExportHasArchiveWriter(event.WebhookID),
|
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||||
"a later delivery should recreate the writer",
|
"a later delivery should recreate the writer",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -370,19 +383,16 @@ func TestEvictWebhook_LaterDeliveryRecreatesWriter(t *testing.T) {
|
|||||||
func TestEngineStop_WriteAfterStopIsRefused(t *testing.T) {
|
func TestEngineStop_WriteAfterStopIsRefused(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
eng, _ := evictTestEngine(t)
|
env := setupArchiveTest(t)
|
||||||
|
tgt := env.seedDatabaseTarget(t, "")
|
||||||
|
|
||||||
webhookDB := testWebhookDB(t)
|
deliverTo(t, env, tgt)
|
||||||
event := seedEvent(t, webhookDB, `{"archived":true}`)
|
|
||||||
d := seedDatabaseTargetDelivery(t, webhookDB, event, "")
|
|
||||||
|
|
||||||
eng.ExportDeliverDatabase(webhookDB, d)
|
w := env.eng.ExportArchiveWriterFor(tgt.ID)
|
||||||
|
|
||||||
w := eng.ExportArchiveWriterFor(event.WebhookID)
|
|
||||||
require.NotNil(t, w)
|
require.NotNil(t, w)
|
||||||
require.True(t, w.HandleOpen())
|
require.True(t, w.HandleOpen())
|
||||||
|
|
||||||
require.NoError(t, eng.ExportStop(context.Background()))
|
require.NoError(t, env.eng.ExportStop(context.Background()))
|
||||||
|
|
||||||
err := w.Write(evictTestRow("ev-after-stop"), 0)
|
err := w.Write(evictTestRow("ev-after-stop"), 0)
|
||||||
|
|
||||||
@@ -395,7 +405,7 @@ func TestEngineStop_WriteAfterStopIsRefused(t *testing.T) {
|
|||||||
"a refused write must not reopen the archive",
|
"a refused write must not reopen the archive",
|
||||||
)
|
)
|
||||||
assert.False(
|
assert.False(
|
||||||
t, eng.ExportHasArchiveWriter(event.WebhookID),
|
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||||
"the stop should empty the registry",
|
"the stop should empty the registry",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -4,13 +4,12 @@ import (
|
|||||||
"database/sql"
|
"database/sql"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/http"
|
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/google/uuid"
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"gorm.io/driver/sqlite"
|
"gorm.io/driver/sqlite"
|
||||||
@@ -74,25 +73,18 @@ func removeArchiveFiles(t *testing.T, path string) {
|
|||||||
|
|
||||||
// TestDeliverDatabase_ArchivesEvent verifies that delivering to
|
// TestDeliverDatabase_ArchivesEvent verifies that delivering to
|
||||||
// a database target marks the delivery delivered and archives
|
// a database target marks the delivery delivered and archives
|
||||||
// the full event into a separate per-webhook archive file.
|
// the full event into the target's own archive file.
|
||||||
func TestDeliverDatabase_ArchivesEvent(t *testing.T) {
|
func TestDeliverDatabase_ArchivesEvent(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
dataDir := t.TempDir()
|
env := setupArchiveTest(t)
|
||||||
dbMgr := database.NewTestWebhookDBManager(dataDir)
|
tgt := env.seedDatabaseTarget(t, "")
|
||||||
|
|
||||||
e := delivery.NewTestEngineWithDB(
|
|
||||||
nil, dbMgr,
|
|
||||||
archiveTestLogger(),
|
|
||||||
&http.Client{Timeout: 5 * time.Second},
|
|
||||||
1,
|
|
||||||
)
|
|
||||||
|
|
||||||
webhookDB := testWebhookDB(t)
|
webhookDB := testWebhookDB(t)
|
||||||
event := seedEvent(t, webhookDB, `{"archived":true}`)
|
event := seedEvent(t, webhookDB, `{"archived":true}`)
|
||||||
d := seedDatabaseTargetDelivery(t, webhookDB, event, "")
|
d := seedDatabaseTargetDelivery(t, webhookDB, event, tgt)
|
||||||
|
|
||||||
e.ExportDeliverDatabase(webhookDB, d)
|
env.eng.ExportDeliverDatabase(webhookDB, d)
|
||||||
|
|
||||||
var updated database.Delivery
|
var updated database.Delivery
|
||||||
|
|
||||||
@@ -105,8 +97,7 @@ func TestDeliverDatabase_ArchivesEvent(t *testing.T) {
|
|||||||
)
|
)
|
||||||
|
|
||||||
archivePath := filepath.Join(
|
archivePath := filepath.Join(
|
||||||
dataDir,
|
env.dataDir, "archive-sweep-test-archive-"+tgt.ID+".db",
|
||||||
fmt.Sprintf("archive-%s.db", event.WebhookID),
|
|
||||||
)
|
)
|
||||||
assert.FileExists(t, archivePath)
|
assert.FileExists(t, archivePath)
|
||||||
|
|
||||||
@@ -288,31 +279,31 @@ func TestParseArchiveExpiry(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// seedDatabaseTargetDelivery seeds a pending delivery for a
|
// seedDatabaseTargetDelivery seeds a pending delivery of an event
|
||||||
// database target with the given config JSON and returns the
|
// to a database target and returns the in-memory delivery the
|
||||||
// in-memory delivery the target handler is invoked with.
|
// target handler is invoked with.
|
||||||
func seedDatabaseTargetDelivery(
|
func seedDatabaseTargetDelivery(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
webhookDB *gorm.DB,
|
webhookDB *gorm.DB,
|
||||||
event database.Event,
|
event database.Event,
|
||||||
config string,
|
tgt *database.Target,
|
||||||
) *database.Delivery {
|
) *database.Delivery {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
dlv := seedDelivery(
|
dlv := seedDelivery(
|
||||||
t, webhookDB, event.ID, uuid.New().String(),
|
t, webhookDB, event.ID, tgt.ID,
|
||||||
database.DeliveryStatusPending,
|
database.DeliveryStatusPending,
|
||||||
)
|
)
|
||||||
|
|
||||||
d := &database.Delivery{
|
d := &database.Delivery{
|
||||||
EventID: event.ID,
|
EventID: event.ID,
|
||||||
TargetID: dlv.TargetID,
|
TargetID: tgt.ID,
|
||||||
Status: database.DeliveryStatusPending,
|
Status: database.DeliveryStatusPending,
|
||||||
Event: event,
|
Event: event,
|
||||||
Target: database.Target{
|
Target: database.Target{
|
||||||
Name: "test-db",
|
Name: tgt.Name,
|
||||||
Type: database.TargetTypeDatabase,
|
Type: database.TargetTypeDatabase,
|
||||||
Config: config,
|
Config: tgt.Config,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
d.ID = dlv.ID
|
d.ID = dlv.ID
|
||||||
@@ -330,22 +321,14 @@ func TestDeliverDatabase_ArchiveFailureFailsDelivery(
|
|||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
dataDir := t.TempDir()
|
env := setupArchiveTest(t)
|
||||||
|
tgt := env.seedDatabaseTarget(t, `{"expiry":"nonsense"}`)
|
||||||
e := delivery.NewTestEngineWithDB(
|
|
||||||
nil, database.NewTestWebhookDBManager(dataDir),
|
|
||||||
archiveTestLogger(),
|
|
||||||
&http.Client{Timeout: 5 * time.Second},
|
|
||||||
1,
|
|
||||||
)
|
|
||||||
|
|
||||||
webhookDB := testWebhookDB(t)
|
webhookDB := testWebhookDB(t)
|
||||||
event := seedEvent(t, webhookDB, `{"archived":false}`)
|
event := seedEvent(t, webhookDB, `{"archived":false}`)
|
||||||
d := seedDatabaseTargetDelivery(
|
d := seedDatabaseTargetDelivery(t, webhookDB, event, tgt)
|
||||||
t, webhookDB, event, `{"expiry":"nonsense"}`,
|
|
||||||
)
|
|
||||||
|
|
||||||
e.ExportDeliverDatabase(webhookDB, d)
|
env.eng.ExportDeliverDatabase(webhookDB, d)
|
||||||
|
|
||||||
var updated database.Delivery
|
var updated database.Delivery
|
||||||
|
|
||||||
@@ -373,10 +356,7 @@ func TestDeliverDatabase_ArchiveFailureFailsDelivery(
|
|||||||
)
|
)
|
||||||
|
|
||||||
assert.NoFileExists(t,
|
assert.NoFileExists(t,
|
||||||
filepath.Join(
|
env.archivePath(tgt),
|
||||||
dataDir,
|
|
||||||
fmt.Sprintf("archive-%s.db", event.WebhookID),
|
|
||||||
),
|
|
||||||
"no archive file should exist for a failed config",
|
"no archive file should exist for a failed config",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -400,3 +380,204 @@ func TestValidateArchiveExpiry(t *testing.T) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestArchiveFileName pins the archive file name and the rules
|
||||||
|
// that make a webhook or target name safe to put in it.
|
||||||
|
func TestArchiveFileName(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const id = "3f2a1c9e-8d4b-4c1a-9e2f-0a1b2c3d4e5f"
|
||||||
|
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
webhook string
|
||||||
|
target string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
"plain names", "orders", "archive",
|
||||||
|
"archive-orders-archive-" + id + ".db",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"lowercased", "Orders", "Main Archive",
|
||||||
|
"archive-orders-main-archive-" + id + ".db",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"a run of other characters is one dash",
|
||||||
|
`a /\..b`, "c__--d",
|
||||||
|
"archive-a-b-c-d-" + id + ".db",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"no dash at either end", " --orders!! ", "(archive)",
|
||||||
|
"archive-orders-archive-" + id + ".db",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"path separators", "../../etc/passwd", "a/b",
|
||||||
|
"archive-etc-passwd-a-b-" + id + ".db",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"letters outside ASCII are dropped",
|
||||||
|
"Bestellungen Größe", "café",
|
||||||
|
"archive-bestellungen-gr-e-caf-" + id + ".db",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"nothing left is unnamed", "", "!!!",
|
||||||
|
"archive-unnamed-unnamed-" + id + ".db",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cut to 40 characters", strings.Repeat("a", 50), "x",
|
||||||
|
"archive-" + strings.Repeat("a", 40) + "-x-" + id + ".db",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"no dash left by the cut",
|
||||||
|
strings.Repeat("a", 39) + " b", "x",
|
||||||
|
"archive-" + strings.Repeat("a", 39) + "-x-" + id + ".db",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
assert.Equal(
|
||||||
|
t, tc.want,
|
||||||
|
delivery.ArchiveFileName(tc.webhook, tc.target, id),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDeliverDatabase_EachTargetHasItsOwnArchive proves two
|
||||||
|
// database targets of one webhook archive into separate files.
|
||||||
|
func TestDeliverDatabase_EachTargetHasItsOwnArchive(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
env := setupArchiveTest(t)
|
||||||
|
first := env.seedDatabaseTarget(t, "")
|
||||||
|
second := env.addDatabaseTarget(t, first.WebhookID, "")
|
||||||
|
|
||||||
|
webhookDB := testWebhookDB(t)
|
||||||
|
event := seedEvent(t, webhookDB, `{"n":1}`)
|
||||||
|
|
||||||
|
for _, tgt := range []*database.Target{first, second} {
|
||||||
|
env.eng.ExportDeliverDatabase(
|
||||||
|
webhookDB,
|
||||||
|
seedDatabaseTargetDelivery(t, webhookDB, event, tgt),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
require.NotEqual(
|
||||||
|
t, env.archivePath(first), env.archivePath(second),
|
||||||
|
)
|
||||||
|
assert.Equal(
|
||||||
|
t, []string{event.ID},
|
||||||
|
archivedEventIDs(t, env.archivePath(first)),
|
||||||
|
)
|
||||||
|
assert.Equal(
|
||||||
|
t, []string{event.ID},
|
||||||
|
archivedEventIDs(t, env.archivePath(second)),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRenameArchive_MovesTheFile proves a rename moves the archive,
|
||||||
|
// rows and all, and that later writes go to the new name.
|
||||||
|
func TestRenameArchive_MovesTheFile(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
env := setupArchiveTest(t)
|
||||||
|
tgt := env.seedDatabaseTarget(t, "")
|
||||||
|
oldPath := env.archivePath(tgt)
|
||||||
|
|
||||||
|
webhookDB := testWebhookDB(t)
|
||||||
|
first := seedEvent(t, webhookDB, `{"n":1}`)
|
||||||
|
env.eng.ExportDeliverDatabase(
|
||||||
|
webhookDB, seedDatabaseTargetDelivery(t, webhookDB, first, tgt),
|
||||||
|
)
|
||||||
|
require.FileExists(t, oldPath)
|
||||||
|
|
||||||
|
require.NoError(
|
||||||
|
t, env.eng.RenameArchive(tgt.ID, "Orders", "Long Term"),
|
||||||
|
)
|
||||||
|
|
||||||
|
newPath := filepath.Join(
|
||||||
|
env.dataDir, "archive-orders-long-term-"+tgt.ID+".db",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert.NoFileExists(t, oldPath)
|
||||||
|
assert.Equal(t, []string{first.ID}, archivedEventIDs(t, newPath))
|
||||||
|
|
||||||
|
second := seedEvent(t, webhookDB, `{"n":2}`)
|
||||||
|
env.eng.ExportDeliverDatabase(
|
||||||
|
webhookDB,
|
||||||
|
seedDatabaseTargetDelivery(t, webhookDB, second, tgt),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert.ElementsMatch(
|
||||||
|
t, []string{first.ID, second.ID},
|
||||||
|
archivedEventIDs(t, newPath),
|
||||||
|
)
|
||||||
|
assert.NoFileExists(
|
||||||
|
t, oldPath, "a write after the rename must use the new name",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRenameArchive_BeforeTheNameIsSaved covers the order the
|
||||||
|
// handlers use: they rename before they save the new name, so a
|
||||||
|
// delivery in between must write under the new name although the
|
||||||
|
// main database still has the old one. It also shows that renaming
|
||||||
|
// an archive that does not exist yet is not an error.
|
||||||
|
func TestRenameArchive_BeforeTheNameIsSaved(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
env := setupArchiveTest(t)
|
||||||
|
tgt := env.seedDatabaseTarget(t, "")
|
||||||
|
|
||||||
|
require.NoError(
|
||||||
|
t, env.eng.RenameArchive(tgt.ID, "Orders", "Archive"),
|
||||||
|
)
|
||||||
|
|
||||||
|
webhookDB := testWebhookDB(t)
|
||||||
|
event := seedEvent(t, webhookDB, `{"n":1}`)
|
||||||
|
env.eng.ExportDeliverDatabase(
|
||||||
|
webhookDB, seedDatabaseTargetDelivery(t, webhookDB, event, tgt),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert.FileExists(
|
||||||
|
t,
|
||||||
|
filepath.Join(
|
||||||
|
env.dataDir, "archive-orders-archive-"+tgt.ID+".db",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
assert.NoFileExists(t, env.archivePath(tgt))
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestArchiveWriter_RenameMovesSidecars proves a rename carries
|
||||||
|
// the -wal and -shm a crash can leave beside an archive no handle
|
||||||
|
// has opened since. SQLite finds them by name, so a -wal left
|
||||||
|
// behind would lose the transactions it holds.
|
||||||
|
func TestArchiveWriter_RenameMovesSidecars(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
oldPath := filepath.Join(dir, "archive-old.db")
|
||||||
|
newPath := filepath.Join(dir, "archive-new.db")
|
||||||
|
|
||||||
|
for _, suffix := range archiveFileSuffixes() {
|
||||||
|
require.NoError(
|
||||||
|
t, os.WriteFile(oldPath+suffix, []byte(suffix), 0o600),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
w := delivery.NewExportArchiveWriter(
|
||||||
|
oldPath, archiveTestLogger(), 0,
|
||||||
|
)
|
||||||
|
|
||||||
|
require.NoError(t, w.Rename("archive-new.db"))
|
||||||
|
|
||||||
|
for _, suffix := range archiveFileSuffixes() {
|
||||||
|
assert.NoFileExists(t, oldPath+suffix)
|
||||||
|
assert.FileExists(t, newPath+suffix)
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.Equal(t, newPath, w.Path())
|
||||||
|
}
|
||||||
|
|||||||
@@ -418,6 +418,38 @@ func TestProcessRetryTask_TargetDeleted_MakesNoAttempt(
|
|||||||
assert.Zero(t, s.Engine.ExportInflightHeld())
|
assert.Zero(t, s.Engine.ExportInflightHeld())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestProcessRetryTask_TargetDeleted_CountsFailureOnTarget verifies
|
||||||
|
// that the failure of a retry abandoned because its target is gone is
|
||||||
|
// added to that target's own totals, not to a row with no target.
|
||||||
|
func TestProcessRetryTask_TargetDeleted_CountsFailureOnTarget(
|
||||||
|
t *testing.T,
|
||||||
|
) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
s := newISetup(t)
|
||||||
|
|
||||||
|
var hits atomic.Int64
|
||||||
|
|
||||||
|
task, targetID := tRetryChainSetup(
|
||||||
|
t, s, "gone-counted", &hits,
|
||||||
|
)
|
||||||
|
|
||||||
|
require.NoError(t, s.MainDB.Delete(
|
||||||
|
&database.Target{}, "id = ?", targetID,
|
||||||
|
).Error)
|
||||||
|
|
||||||
|
s.Engine.ExportProcessRetryTask(
|
||||||
|
context.Background(), &task,
|
||||||
|
)
|
||||||
|
|
||||||
|
var rows []database.TargetTotals
|
||||||
|
|
||||||
|
require.NoError(t, s.WebhookDB.Find(&rows).Error)
|
||||||
|
assert.Equal(t, []database.TargetTotals{
|
||||||
|
{TargetID: targetID, Failed: 1},
|
||||||
|
}, rows)
|
||||||
|
}
|
||||||
|
|
||||||
// TestProcessRetryTask_TargetPresent_StillDelivers is the guard's
|
// TestProcessRetryTask_TargetPresent_StillDelivers is the guard's
|
||||||
// mutation check: a liveness check that refused every retry would pass
|
// mutation check: a liveness check that refused every retry would pass
|
||||||
// the test above and break every retry there is.
|
// the test above and break every retry there is.
|
||||||
|
|||||||
@@ -7,7 +7,7 @@
|
|||||||
// SQL — parameters and all — for every statement that returns an
|
// SQL — parameters and all — for every statement that returns an
|
||||||
// error, including gorm.ErrRecordNotFound. Two of this service's
|
// error, including gorm.ErrRecordNotFound. Two of this service's
|
||||||
// lookups miss by design on unauthenticated routes: the entrypoint
|
// lookups miss by design on unauthenticated routes: the entrypoint
|
||||||
// lookup on /webhook/{uuid}, whose path segment the client picks
|
// lookup on /h/{uuid}, whose path segment the client picks
|
||||||
// outright, and the user lookup behind the login form, whose username
|
// outright, and the user lookup behind the login form, whose username
|
||||||
// the client picks outright. Under the default logger each of those
|
// the client picks outright. Under the default logger each of those
|
||||||
// misses printed an unbounded, attacker-chosen string, at no level the
|
// misses printed an unbounded, attacker-chosen string, at no level the
|
||||||
|
|||||||
@@ -2,19 +2,56 @@ package handlers
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"net/url"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"unicode"
|
||||||
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/logfield"
|
"sneak.berlin/go/webhooker/internal/logfield"
|
||||||
|
"sneak.berlin/go/webhooker/internal/middleware"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// loginDestination returns where a successful login sends the
|
||||||
|
// browser: next when it is a path on this site, otherwise "/", which
|
||||||
|
// leads to the webhook list.
|
||||||
|
//
|
||||||
|
// A browser reads "//host" as another site, reads "\" as "/", and
|
||||||
|
// drops tabs and newlines before reading at all. So the value must
|
||||||
|
// start with exactly one "/" and hold no "\" or control character
|
||||||
|
// anywhere: http.Redirect cleans "/a/../\host" down to "/\host". It
|
||||||
|
// is checked after percent-decoding, so an encoded form of any of
|
||||||
|
// these is refused too.
|
||||||
|
func loginDestination(next string) string {
|
||||||
|
if len(next) > middleware.MaxNextBytes {
|
||||||
|
return "/"
|
||||||
|
}
|
||||||
|
|
||||||
|
decoded, err := url.PathUnescape(next)
|
||||||
|
if err != nil ||
|
||||||
|
!strings.HasPrefix(decoded, "/") ||
|
||||||
|
strings.HasPrefix(decoded, "//") ||
|
||||||
|
strings.Contains(decoded, `\`) ||
|
||||||
|
strings.ContainsFunc(decoded, unicode.IsControl) {
|
||||||
|
return "/"
|
||||||
|
}
|
||||||
|
|
||||||
|
return next
|
||||||
|
}
|
||||||
|
|
||||||
// HandleLoginPage returns a handler for the login page (GET)
|
// HandleLoginPage returns a handler for the login page (GET)
|
||||||
func (h *Handlers) HandleLoginPage() http.HandlerFunc {
|
func (h *Handlers) HandleLoginPage() http.HandlerFunc {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
next := loginDestination(
|
||||||
|
r.URL.Query().Get(middleware.NextParam),
|
||||||
|
)
|
||||||
|
|
||||||
// Check if already logged in
|
// Check if already logged in
|
||||||
sess, err := h.session.Get(r)
|
sess, err := h.session.Get(r)
|
||||||
if err == nil && h.session.IsAuthenticated(sess) {
|
if err == nil && h.session.IsAuthenticated(sess) {
|
||||||
http.Redirect(w, r, "/", http.StatusSeeOther)
|
http.Redirect( //nolint:gosec // checked by loginDestination
|
||||||
|
w, r, next, http.StatusSeeOther,
|
||||||
|
)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -22,6 +59,7 @@ func (h *Handlers) HandleLoginPage() http.HandlerFunc {
|
|||||||
// Render login page
|
// Render login page
|
||||||
data := map[string]any{
|
data := map[string]any{
|
||||||
tmplKeyError: "",
|
tmplKeyError: "",
|
||||||
|
tmplKeyNext: next,
|
||||||
}
|
}
|
||||||
|
|
||||||
h.renderTemplate(w, r, "login.html", data)
|
h.renderTemplate(w, r, "login.html", data)
|
||||||
@@ -77,8 +115,13 @@ func (h *Handlers) HandleLoginSubmit() http.HandlerFunc {
|
|||||||
"user_id", user.ID,
|
"user_id", user.ID,
|
||||||
)
|
)
|
||||||
|
|
||||||
// Redirect to home page
|
// The form value is the client's to set, so it is checked
|
||||||
http.Redirect(w, r, "/", http.StatusSeeOther)
|
// again here rather than trusted from the rendered page.
|
||||||
|
http.Redirect( //nolint:gosec // checked by loginDestination
|
||||||
|
w, r,
|
||||||
|
loginDestination(r.PostFormValue(middleware.NextParam)),
|
||||||
|
http.StatusSeeOther,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -91,6 +134,9 @@ func (h *Handlers) renderLoginError(
|
|||||||
) {
|
) {
|
||||||
data := map[string]any{
|
data := map[string]any{
|
||||||
tmplKeyError: msg,
|
tmplKeyError: msg,
|
||||||
|
tmplKeyNext: loginDestination(
|
||||||
|
r.PostFormValue(middleware.NextParam),
|
||||||
|
),
|
||||||
}
|
}
|
||||||
|
|
||||||
w.WriteHeader(status)
|
w.WriteHeader(status)
|
||||||
@@ -103,9 +149,10 @@ func (h *Handlers) renderLoginError(
|
|||||||
// The credential check runs BEFORE any rate-limit budget is
|
// The credential check runs BEFORE any rate-limit budget is
|
||||||
// consulted, and only a failed check spends budget. That is what
|
// consulted, and only a failed check spends budget. That is what
|
||||||
// keeps the single administrative path reachable: behind the reverse
|
// keeps the single administrative path reachable: behind the reverse
|
||||||
// proxy this deployment requires, with TRUSTED_PROXIES unset, every
|
// proxy this deployment requires, when TRUSTED_PROXIES does not cover
|
||||||
// client shares one bucket, so a limiter spent on arrival lets any
|
// it, every client shares one bucket, so a limiter spent on arrival
|
||||||
// stranger deny the operator's own correct password indefinitely.
|
// lets any stranger deny the operator's own correct password
|
||||||
|
// indefinitely.
|
||||||
//
|
//
|
||||||
// Verifying first means every login POST costs an Argon2id hash, so
|
// Verifying first means every login POST costs an Argon2id hash, so
|
||||||
// the work is taken under a bounded number of verification slots.
|
// the work is taken under a bounded number of verification slots.
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ const (
|
|||||||
|
|
||||||
// sharedProxyPeer is the whole point of this file. Production is
|
// sharedProxyPeer is the whole point of this file. Production is
|
||||||
// required to run behind a TLS-terminating reverse proxy, and
|
// required to run behind a TLS-terminating reverse proxy, and
|
||||||
// TRUSTED_PROXIES defaults to empty, so every client — attacker
|
// when TRUSTED_PROXIES does not cover it every client — attacker
|
||||||
// and operator alike — reaches the process from the proxy's
|
// and operator alike — reaches the process from the proxy's
|
||||||
// address and shares one rate-limit bucket. Both parties in
|
// address and shares one rate-limit bucket. Both parties in
|
||||||
// these tests therefore use the same RemoteAddr.
|
// these tests therefore use the same RemoteAddr.
|
||||||
@@ -115,11 +115,11 @@ func floodFailures(
|
|||||||
// done-criterion of https://git.eeqj.de/sneak/webhooker/issues/150.
|
// done-criterion of https://git.eeqj.de/sneak/webhooker/issues/150.
|
||||||
//
|
//
|
||||||
// The attacker and the operator share one rate-limit bucket, because
|
// The attacker and the operator share one rate-limit bucket, because
|
||||||
// behind the mandated reverse proxy with TRUSTED_PROXIES unset every
|
// behind the mandated reverse proxy, when TRUSTED_PROXIES does not
|
||||||
// client keys on the proxy's address. The attacker floods the
|
// cover it, every client keys on the proxy's address. The attacker
|
||||||
// operator's own username — a single-admin product has a predictable
|
// floods the operator's own username — a single-admin product has a
|
||||||
// one — far past the failure limit. The operator must still be able
|
// predictable one — far past the failure limit. The operator must
|
||||||
// to log in with the correct password.
|
// still be able to log in with the correct password.
|
||||||
//
|
//
|
||||||
// This fails if credentials stop being verified ahead of the limiter.
|
// This fails if credentials stop being verified ahead of the limiter.
|
||||||
func TestLogin_StrangersFloodCannotLockOutTheOperator(t *testing.T) {
|
func TestLogin_StrangersFloodCannotLockOutTheOperator(t *testing.T) {
|
||||||
@@ -453,3 +453,229 @@ func TestLogin_SuccessCreatesSession(t *testing.T) {
|
|||||||
"the issued cookie must carry an authenticated session",
|
"the issued cookie must carry an authenticated session",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestLogin_ReturnsOnlyToAPathOnThisSite is the security half of
|
||||||
|
// https://git.eeqj.de/sneak/webhooker/issues/384: the page a login
|
||||||
|
// returns to is client-chosen, so anything that is not a path on this
|
||||||
|
// site, plain or percent-encoded, must land on "/", the webhook list.
|
||||||
|
func TestLogin_ReturnsOnlyToAPathOnThisSite(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var (
|
||||||
|
h *handlers.Handlers
|
||||||
|
db *database.Database
|
||||||
|
)
|
||||||
|
|
||||||
|
app := newTestApp(t, &h, &db)
|
||||||
|
app.RequireStart()
|
||||||
|
|
||||||
|
t.Cleanup(app.RequireStop)
|
||||||
|
|
||||||
|
seedOperator(t, db)
|
||||||
|
|
||||||
|
cases := []struct{ next, want string }{
|
||||||
|
{"/hook/abc/events?page=2", "/hook/abc/events?page=2"},
|
||||||
|
{"", "/"},
|
||||||
|
{"https://evil.example/", "/"},
|
||||||
|
{"https%3A%2F%2Fevil.example%2F", "/"},
|
||||||
|
{"//evil.example/", "/"},
|
||||||
|
{"%2F%2Fevil.example/", "/"},
|
||||||
|
{"/%2Fevil.example/", "/"},
|
||||||
|
{`/\evil.example/`, "/"},
|
||||||
|
{"%2F%5Cevil.example/", "/"},
|
||||||
|
{"/%5Cevil.example/", "/"},
|
||||||
|
{`/a/../\evil.example/`, "/"},
|
||||||
|
{"/\t/evil.example/", "/"},
|
||||||
|
{"/%09/evil.example/", "/"},
|
||||||
|
{"/\n/evil.example/", "/"},
|
||||||
|
{"/%0A/evil.example/", "/"},
|
||||||
|
{"/\r/evil.example/", "/"},
|
||||||
|
{"/%0D/evil.example/", "/"},
|
||||||
|
{"/%00/evil.example/", "/"},
|
||||||
|
{"/%7F/evil.example/", "/"},
|
||||||
|
{"%252F%252Fevil.example/", "/"},
|
||||||
|
{"https%253A%252F%252Fevil.example%252F", "/"},
|
||||||
|
{"/" + strings.Repeat("a", 4096), "/"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, c := range cases {
|
||||||
|
form := url.Values{}
|
||||||
|
form.Set("username", operatorUser)
|
||||||
|
form.Set("password", operatorPassword)
|
||||||
|
form.Set("next", c.next)
|
||||||
|
|
||||||
|
req := httptest.NewRequestWithContext(
|
||||||
|
context.Background(),
|
||||||
|
http.MethodPost,
|
||||||
|
"/pages/login",
|
||||||
|
strings.NewReader(form.Encode()),
|
||||||
|
)
|
||||||
|
req.Header.Set(
|
||||||
|
"Content-Type", "application/x-www-form-urlencoded",
|
||||||
|
)
|
||||||
|
req.RemoteAddr = sharedProxyPeer
|
||||||
|
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
h.HandleLoginSubmit().ServeHTTP(w, req)
|
||||||
|
|
||||||
|
assert.Equal(t, http.StatusSeeOther, w.Code, "next %q", c.next)
|
||||||
|
assert.Equal(
|
||||||
|
t, c.want, w.Header().Get("Location"), "next %q", c.next,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestLogin_WrongPasswordKeepsTheRequestedPage: after a wrong
|
||||||
|
// password the login page is shown again with the same next, so the
|
||||||
|
// next attempt still returns to the page that was asked for.
|
||||||
|
func TestLogin_WrongPasswordKeepsTheRequestedPage(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var (
|
||||||
|
h *handlers.Handlers
|
||||||
|
db *database.Database
|
||||||
|
)
|
||||||
|
|
||||||
|
app := newTestApp(t, &h, &db)
|
||||||
|
app.RequireStart()
|
||||||
|
|
||||||
|
t.Cleanup(app.RequireStop)
|
||||||
|
|
||||||
|
seedOperator(t, db)
|
||||||
|
|
||||||
|
form := url.Values{}
|
||||||
|
form.Set("username", operatorUser)
|
||||||
|
form.Set("password", "wrong")
|
||||||
|
form.Set("next", "/hook/abc")
|
||||||
|
|
||||||
|
req := httptest.NewRequestWithContext(
|
||||||
|
context.Background(),
|
||||||
|
http.MethodPost,
|
||||||
|
"/pages/login",
|
||||||
|
strings.NewReader(form.Encode()),
|
||||||
|
)
|
||||||
|
req.Header.Set(
|
||||||
|
"Content-Type", "application/x-www-form-urlencoded",
|
||||||
|
)
|
||||||
|
req.RemoteAddr = sharedProxyPeer
|
||||||
|
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
h.HandleLoginSubmit().ServeHTTP(w, req)
|
||||||
|
|
||||||
|
assert.Equal(t, http.StatusUnauthorized, w.Code)
|
||||||
|
assert.Contains(
|
||||||
|
t, w.Body.String(), `name="next" value="/hook/abc"`,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// loginPageGet renders the login page as a GET with the given next
|
||||||
|
// value and cookies.
|
||||||
|
func loginPageGet(
|
||||||
|
h *handlers.Handlers, next string, cookies []*http.Cookie,
|
||||||
|
) *httptest.ResponseRecorder {
|
||||||
|
req := httptest.NewRequestWithContext(
|
||||||
|
context.Background(), http.MethodGet,
|
||||||
|
"/pages/login?"+url.Values{"next": {next}}.Encode(), nil,
|
||||||
|
)
|
||||||
|
|
||||||
|
for _, c := range cookies {
|
||||||
|
req.AddCookie(c)
|
||||||
|
}
|
||||||
|
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
h.HandleLoginPage().ServeHTTP(w, req)
|
||||||
|
|
||||||
|
return w
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestLoginPage_CarriesOnlyAPathOnThisSite covers the login page
|
||||||
|
// itself: its form carries the requested page only when it is a path
|
||||||
|
// on this site, and a browser already logged in goes straight there,
|
||||||
|
// or to "/" when it is not.
|
||||||
|
func TestLoginPage_CarriesOnlyAPathOnThisSite(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var (
|
||||||
|
h *handlers.Handlers
|
||||||
|
sess *session.Session
|
||||||
|
)
|
||||||
|
|
||||||
|
app := newTestApp(t, &h, &sess)
|
||||||
|
app.RequireStart()
|
||||||
|
|
||||||
|
t.Cleanup(app.RequireStop)
|
||||||
|
|
||||||
|
assert.Contains(
|
||||||
|
t, loginPageGet(h, "/hook/abc", nil).Body.String(),
|
||||||
|
`name="next" value="/hook/abc"`,
|
||||||
|
)
|
||||||
|
assert.Contains(
|
||||||
|
t, loginPageGet(h, "//evil.example/", nil).Body.String(),
|
||||||
|
`name="next" value="/"`,
|
||||||
|
)
|
||||||
|
|
||||||
|
cookies := authenticatedCookies(t, sess, "test-user-id", "testuser")
|
||||||
|
|
||||||
|
cases := []struct{ next, want string }{
|
||||||
|
{"/hook/abc", "/hook/abc"},
|
||||||
|
{"//evil.example/", "/"},
|
||||||
|
{`/\evil.example/`, "/"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, c := range cases {
|
||||||
|
w := loginPageGet(h, c.next, cookies)
|
||||||
|
|
||||||
|
assert.Equal(t, http.StatusSeeOther, w.Code, "next %q", c.next)
|
||||||
|
assert.Equal(
|
||||||
|
t, c.want, w.Header().Get("Location"), "next %q", c.next,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestLoginPage_HasNoLinkToItself: the navigation bar on the login
|
||||||
|
// page offers no link to the login page.
|
||||||
|
func TestLoginPage_HasNoLinkToItself(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var h *handlers.Handlers
|
||||||
|
|
||||||
|
app := newTestApp(t, &h)
|
||||||
|
app.RequireStart()
|
||||||
|
|
||||||
|
t.Cleanup(app.RequireStop)
|
||||||
|
|
||||||
|
w := loginPageGet(h, "", nil)
|
||||||
|
|
||||||
|
require.Equal(t, http.StatusOK, w.Code)
|
||||||
|
assert.NotContains(t, w.Body.String(), `href="/pages/login"`)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestLogin_UsernameAtLimitCanLogIn shows that a username of exactly
|
||||||
|
// database.MaxUsernameBytes still fits in the session cookie. Past
|
||||||
|
// what the cookie can carry, a correct login answers 500.
|
||||||
|
func TestLogin_UsernameAtLimitCanLogIn(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var (
|
||||||
|
h *handlers.Handlers
|
||||||
|
db *database.Database
|
||||||
|
)
|
||||||
|
|
||||||
|
app := newTestApp(t, &h, &db)
|
||||||
|
app.RequireStart()
|
||||||
|
|
||||||
|
t.Cleanup(app.RequireStop)
|
||||||
|
|
||||||
|
username := strings.Repeat("a", database.MaxUsernameBytes)
|
||||||
|
|
||||||
|
hash, err := database.HashPassword(operatorPassword)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, db.DB().Create(&database.User{
|
||||||
|
Username: username,
|
||||||
|
Password: hash,
|
||||||
|
}).Error)
|
||||||
|
|
||||||
|
w := submitLogin(h, sharedProxyPeer, username, operatorPassword)
|
||||||
|
|
||||||
|
assert.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
|
}
|
||||||
|
|||||||
@@ -299,8 +299,9 @@ func countInFlightDeliveries(
|
|||||||
return count, err
|
return count, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// createReplayDelivery writes the new pending delivery row and returns
|
// createReplayDelivery writes the new pending delivery row, adds it to
|
||||||
// the task that carries it to the delivery engine.
|
// its target's totals in the same transaction, and returns the task
|
||||||
|
// that carries it to the delivery engine.
|
||||||
//
|
//
|
||||||
// The row is written with associations omitted, and neither Event nor
|
// The row is written with associations omitted, and neither Event nor
|
||||||
// Target is populated on it: GORM's SaveBeforeAssociations would
|
// Target is populated on it: GORM's SaveBeforeAssociations would
|
||||||
@@ -319,7 +320,16 @@ func createReplayDelivery(
|
|||||||
Status: database.DeliveryStatusPending,
|
Status: database.DeliveryStatusPending,
|
||||||
}
|
}
|
||||||
|
|
||||||
err := webhookDB.Omit(clause.Associations).Create(dlv).Error
|
err := webhookDB.Transaction(func(tx *gorm.DB) error {
|
||||||
|
err := tx.Omit(clause.Associations).Create(dlv).Error
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return database.AddTargetTotals(tx, database.TargetTotals{
|
||||||
|
TargetID: dlv.TargetID, Deliveries: 1,
|
||||||
|
})
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return delivery.Task{}, err
|
return delivery.Task{}, err
|
||||||
}
|
}
|
||||||
@@ -362,7 +372,7 @@ func (h *Handlers) finishReplay(
|
|||||||
webhook database.Webhook,
|
webhook database.Webhook,
|
||||||
code replayOutcomeCode,
|
code replayOutcomeCode,
|
||||||
) {
|
) {
|
||||||
dest := "/source/" + webhook.ID + "/logs?" +
|
dest := "/hook/" + webhook.ID + "/events?" +
|
||||||
replayOutcomeParam + "=" + string(code)
|
replayOutcomeParam + "=" + string(code)
|
||||||
|
|
||||||
// The page is read from the form rather than the query string:
|
// The page is read from the form rather than the query string:
|
||||||
|
|||||||
@@ -138,7 +138,7 @@ func postReplay(
|
|||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
req := postRequest(
|
req := postRequest(
|
||||||
"/source/"+webhookID+"/deliveries/"+
|
"/hook/"+webhookID+"/deliveries/"+
|
||||||
deliveryID+"/replay",
|
deliveryID+"/replay",
|
||||||
authenticatedCookies(
|
authenticatedCookies(
|
||||||
t, sess, deleteTestUserID, deleteTestUsername,
|
t, sess, deleteTestUserID, deleteTestUsername,
|
||||||
@@ -212,7 +212,7 @@ func TestHandleDeliveryReplay_AppendsDeliveryAndLeavesOriginal(
|
|||||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t,
|
t,
|
||||||
"/source/"+wh.ID+"/logs?replay=queued",
|
"/hook/"+wh.ID+"/events?replay=queued",
|
||||||
w.Header().Get("Location"),
|
w.Header().Get("Location"),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -362,7 +362,7 @@ func TestHandleDeliveryReplay_RefusesDeletedTarget(t *testing.T) {
|
|||||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t,
|
t,
|
||||||
"/source/"+wh.ID+"/logs?replay=target-deleted",
|
"/hook/"+wh.ID+"/events?replay=target-deleted",
|
||||||
w.Header().Get("Location"),
|
w.Header().Get("Location"),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -390,7 +390,7 @@ func TestHandleDeliveryReplay_RefusesDeletedTarget(t *testing.T) {
|
|||||||
require.Equal(t, http.StatusSeeOther, missing.Code)
|
require.Equal(t, http.StatusSeeOther, missing.Code)
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t,
|
t,
|
||||||
"/source/"+wh.ID+"/logs?replay=target-missing",
|
"/hook/"+wh.ID+"/events?replay=target-missing",
|
||||||
missing.Header().Get("Location"),
|
missing.Header().Get("Location"),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -431,7 +431,7 @@ func TestHandleDeliveryReplay_RefusesWhileEarlierReplayInFlight(
|
|||||||
require.Equal(t, http.StatusSeeOther, first.Code)
|
require.Equal(t, http.StatusSeeOther, first.Code)
|
||||||
require.Equal(
|
require.Equal(
|
||||||
t,
|
t,
|
||||||
"/source/"+wh.ID+"/logs?replay=queued",
|
"/hook/"+wh.ID+"/events?replay=queued",
|
||||||
first.Header().Get("Location"),
|
first.Header().Get("Location"),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -439,7 +439,7 @@ func TestHandleDeliveryReplay_RefusesWhileEarlierReplayInFlight(
|
|||||||
require.Equal(t, http.StatusSeeOther, second.Code)
|
require.Equal(t, http.StatusSeeOther, second.Code)
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t,
|
t,
|
||||||
"/source/"+wh.ID+"/logs?replay=in-flight",
|
"/hook/"+wh.ID+"/events?replay=in-flight",
|
||||||
second.Header().Get("Location"),
|
second.Header().Get("Location"),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -465,7 +465,7 @@ func TestHandleDeliveryReplay_RefusesWhileEarlierReplayInFlight(
|
|||||||
require.Equal(t, http.StatusSeeOther, pending.Code)
|
require.Equal(t, http.StatusSeeOther, pending.Code)
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t,
|
t,
|
||||||
"/source/"+wh.ID+"/logs?replay=not-terminal",
|
"/hook/"+wh.ID+"/events?replay=not-terminal",
|
||||||
pending.Header().Get("Location"),
|
pending.Header().Get("Location"),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -501,7 +501,7 @@ func TestHandleSourceLogs_RendersReplayControlAndBanner(t *testing.T) {
|
|||||||
|
|
||||||
assert.Contains(
|
assert.Contains(
|
||||||
t, body,
|
t, body,
|
||||||
`action="/source/`+wh.ID+`/deliveries/`+
|
`action="/hook/`+wh.ID+`/deliveries/`+
|
||||||
original.ID+`/replay"`,
|
original.ID+`/replay"`,
|
||||||
)
|
)
|
||||||
assert.Contains(t, body, `method="POST"`)
|
assert.Contains(t, body, `method="POST"`)
|
||||||
|
|||||||
@@ -64,8 +64,8 @@ func fetchEventBody(
|
|||||||
req := httptest.NewRequestWithContext(
|
req := httptest.NewRequestWithContext(
|
||||||
context.Background(),
|
context.Background(),
|
||||||
http.MethodGet,
|
http.MethodGet,
|
||||||
"/source/"+url.PathEscape(sourceID)+
|
"/hook/"+url.PathEscape(sourceID)+
|
||||||
"/logs/"+url.PathEscape(eventID)+"/body",
|
"/events/"+url.PathEscape(eventID)+"/body",
|
||||||
nil,
|
nil,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -490,7 +490,7 @@ func TestHandleSourceLogs_TruncationMarkerLinksToDownload(
|
|||||||
page := renderSourceLogsPage(t, h, sess, big.ID)
|
page := renderSourceLogsPage(t, h, sess, big.ID)
|
||||||
assert.Contains(
|
assert.Contains(
|
||||||
t, page,
|
t, page,
|
||||||
"/source/"+big.ID+"/logs/"+bigEvt.ID+"/body",
|
"/hook/"+big.ID+"/events/"+bigEvt.ID+"/body",
|
||||||
)
|
)
|
||||||
|
|
||||||
small := seedWebhook(t, db)
|
small := seedWebhook(t, db)
|
||||||
@@ -501,6 +501,6 @@ func TestHandleSourceLogs_TruncationMarkerLinksToDownload(
|
|||||||
page = renderSourceLogsPage(t, h, sess, small.ID)
|
page = renderSourceLogsPage(t, h, sess, small.ID)
|
||||||
assert.NotContains(
|
assert.NotContains(
|
||||||
t, page,
|
t, page,
|
||||||
"/source/"+small.ID+"/logs/"+smallEvt.ID+"/body",
|
"/hook/"+small.ID+"/events/"+smallEvt.ID+"/body",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -257,7 +257,7 @@ func (h *Handlers) finishResubmit(
|
|||||||
webhook database.Webhook,
|
webhook database.Webhook,
|
||||||
code resubmitOutcomeCode,
|
code resubmitOutcomeCode,
|
||||||
) {
|
) {
|
||||||
dest := "/source/" + webhook.ID + "/logs?" +
|
dest := "/hook/" + webhook.ID + "/events?" +
|
||||||
resubmitOutcomeParam + "=" + string(code)
|
resubmitOutcomeParam + "=" + string(code)
|
||||||
|
|
||||||
// The page is read from the form rather than the query string:
|
// The page is read from the form rather than the query string:
|
||||||
|
|||||||
@@ -65,7 +65,7 @@ func postResubmit(
|
|||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
req := postRequest(
|
req := postRequest(
|
||||||
"/source/"+webhookID+"/events/"+eventID+"/resubmit",
|
"/hook/"+webhookID+"/events/"+eventID+"/resubmit",
|
||||||
authenticatedCookies(
|
authenticatedCookies(
|
||||||
t, sess, deleteTestUserID, deleteTestUsername,
|
t, sess, deleteTestUserID, deleteTestUsername,
|
||||||
),
|
),
|
||||||
@@ -154,7 +154,7 @@ func TestHandleEventResubmit_DeliversToTargetCreatedAfterTheEvent(
|
|||||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t,
|
t,
|
||||||
"/source/"+wh.ID+"/logs?resubmit=queued",
|
"/hook/"+wh.ID+"/events?resubmit=queued",
|
||||||
w.Header().Get("Location"),
|
w.Header().Get("Location"),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -204,6 +204,7 @@ func assertEventCopy(
|
|||||||
assert.Equal(t, original.Method, fresh.Method)
|
assert.Equal(t, original.Method, fresh.Method)
|
||||||
assert.Equal(t, original.Headers, fresh.Headers)
|
assert.Equal(t, original.Headers, fresh.Headers)
|
||||||
assert.Equal(t, original.Body, fresh.Body)
|
assert.Equal(t, original.Body, fresh.Body)
|
||||||
|
assert.Equal(t, int64(len(original.Body)), fresh.BodyBytes)
|
||||||
assert.Equal(t, original.ContentType, fresh.ContentType)
|
assert.Equal(t, original.ContentType, fresh.ContentType)
|
||||||
assert.Equal(t, original.EntrypointID, fresh.EntrypointID)
|
assert.Equal(t, original.EntrypointID, fresh.EntrypointID)
|
||||||
assert.Equal(t, original.WebhookID, fresh.WebhookID)
|
assert.Equal(t, original.WebhookID, fresh.WebhookID)
|
||||||
@@ -281,7 +282,7 @@ func TestHandleEventResubmit_IsRepeatable(t *testing.T) {
|
|||||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t,
|
t,
|
||||||
"/source/"+wh.ID+"/logs?resubmit=queued",
|
"/hook/"+wh.ID+"/events?resubmit=queued",
|
||||||
w.Header().Get("Location"),
|
w.Header().Get("Location"),
|
||||||
"a resubmit must not be refused while an earlier "+
|
"a resubmit must not be refused while an earlier "+
|
||||||
"one is in flight",
|
"one is in flight",
|
||||||
@@ -435,7 +436,7 @@ func TestHandleEventResubmit_SkipsInactiveTarget(t *testing.T) {
|
|||||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t,
|
t,
|
||||||
"/source/"+wh.ID+"/logs?resubmit=queued",
|
"/hook/"+wh.ID+"/events?resubmit=queued",
|
||||||
w.Header().Get("Location"),
|
w.Header().Get("Location"),
|
||||||
"an inactive target is skipped, not an error",
|
"an inactive target is skipped, not an error",
|
||||||
)
|
)
|
||||||
@@ -481,7 +482,7 @@ func TestHandleEventResubmit_NoActiveTargetsStillStoresEvent(
|
|||||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t,
|
t,
|
||||||
"/source/"+wh.ID+"/logs?resubmit=no-targets",
|
"/hook/"+wh.ID+"/events?resubmit=no-targets",
|
||||||
w.Header().Get("Location"),
|
w.Header().Get("Location"),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -597,7 +598,7 @@ func TestHandleSourceLogs_ShowsResubmitProvenance(t *testing.T) {
|
|||||||
)
|
)
|
||||||
assert.Contains(
|
assert.Contains(
|
||||||
t, body,
|
t, body,
|
||||||
"/source/"+wh.ID+"/events/"+original.ID+"/resubmit",
|
"/hook/"+wh.ID+"/events/"+original.ID+"/resubmit",
|
||||||
"the log must offer the resubmit action per event",
|
"the log must offer the resubmit action per event",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,7 +4,9 @@ import (
|
|||||||
"html/template"
|
"html/template"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gorm.io/gorm"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -69,6 +71,29 @@ func (s *Handlers) LoadEventLogViewsForTest(
|
|||||||
return views
|
return views
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// WebhookStatsForTest returns the figures the statistics pane on a
|
||||||
|
// webhook's page shows, from the webhook's entrypoints and targets
|
||||||
|
// loaded as that page loads them.
|
||||||
|
func (s *Handlers) WebhookStatsForTest(webhookID string) *WebhookStats {
|
||||||
|
var entrypoints []database.Entrypoint
|
||||||
|
|
||||||
|
s.db.DB().Where("webhook_id = ?", webhookID).Find(&entrypoints)
|
||||||
|
|
||||||
|
var targets []database.Target
|
||||||
|
|
||||||
|
s.db.DB().Where("webhook_id = ?", webhookID).Find(&targets)
|
||||||
|
|
||||||
|
return s.loadWebhookStats(webhookID, entrypoints, targets)
|
||||||
|
}
|
||||||
|
|
||||||
|
// FinishedByTargetForTest exposes finishedByTarget for use in the
|
||||||
|
// handlers_test package.
|
||||||
|
func FinishedByTargetForTest(
|
||||||
|
webhookDB *gorm.DB, since time.Time,
|
||||||
|
) ([]TargetFinished, error) {
|
||||||
|
return finishedByTarget(webhookDB, since)
|
||||||
|
}
|
||||||
|
|
||||||
// AddTemplateForTest registers a template under a page name so that
|
// AddTemplateForTest registers a template under a page name so that
|
||||||
// the handlers_test package can drive the render path with a
|
// the handlers_test package can drive the render path with a
|
||||||
// template of its own.
|
// template of its own.
|
||||||
|
|||||||
@@ -306,7 +306,7 @@ func postWebhook(
|
|||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
req := httptest.NewRequestWithContext(
|
req := httptest.NewRequestWithContext(
|
||||||
context.Background(), http.MethodPost, "/webhook/x",
|
context.Background(), http.MethodPost, "/h/x",
|
||||||
strings.NewReader("{}"),
|
strings.NewReader("{}"),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ const (
|
|||||||
// maxBodyShift is the bit shift for 1 MB body limit.
|
// maxBodyShift is the bit shift for 1 MB body limit.
|
||||||
maxBodyShift = 20
|
maxBodyShift = 20
|
||||||
// recentEventLimit is the number of recent events to show.
|
// recentEventLimit is the number of recent events to show.
|
||||||
recentEventLimit = 20
|
recentEventLimit = 50
|
||||||
// paginationPerPage is the number of items per page.
|
// paginationPerPage is the number of items per page.
|
||||||
paginationPerPage = 25
|
paginationPerPage = 25
|
||||||
|
|
||||||
@@ -36,6 +36,9 @@ const (
|
|||||||
tmplKeyError = "Error"
|
tmplKeyError = "Error"
|
||||||
// tmplKeyWebhook is the template data key for a webhook.
|
// tmplKeyWebhook is the template data key for a webhook.
|
||||||
tmplKeyWebhook = "Webhook"
|
tmplKeyWebhook = "Webhook"
|
||||||
|
// tmplKeyNext is the template data key for the page to return
|
||||||
|
// to after login.
|
||||||
|
tmplKeyNext = "Next"
|
||||||
)
|
)
|
||||||
|
|
||||||
// errInvalidPassword is returned when a password does not match.
|
// errInvalidPassword is returned when a password does not match.
|
||||||
@@ -59,7 +62,7 @@ type HandlersParams struct {
|
|||||||
Session *session.Session
|
Session *session.Session
|
||||||
Middleware *middleware.Middleware
|
Middleware *middleware.Middleware
|
||||||
Notifier delivery.Notifier
|
Notifier delivery.Notifier
|
||||||
Evictor delivery.WebhookEvictor
|
Archives delivery.Archives
|
||||||
SSRFGuard *delivery.Guard
|
SSRFGuard *delivery.Guard
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -74,7 +77,7 @@ type Handlers struct {
|
|||||||
session *session.Session
|
session *session.Session
|
||||||
mw *middleware.Middleware
|
mw *middleware.Middleware
|
||||||
notifier delivery.Notifier
|
notifier delivery.Notifier
|
||||||
evictor delivery.WebhookEvictor
|
archives delivery.Archives
|
||||||
mtr *metrics.Set
|
mtr *metrics.Set
|
||||||
templates map[string]*template.Template
|
templates map[string]*template.Template
|
||||||
|
|
||||||
@@ -91,18 +94,22 @@ type Handlers struct {
|
|||||||
|
|
||||||
// parsePageTemplate parses a page-specific template set from the
|
// parsePageTemplate parses a page-specific template set from the
|
||||||
// embedded FS. Each page template is combined with the shared
|
// embedded FS. Each page template is combined with the shared
|
||||||
// base, htmlheader, and navbar templates. The page file must be
|
// base, htmlheader, and navbar templates, and with any further files
|
||||||
// listed first so that its root action ({{template "base" .}})
|
// the page includes. The page file must be listed first so that its
|
||||||
// becomes the template set's entry point.
|
// root action ({{template "base" .}}) becomes the template set's entry
|
||||||
func parsePageTemplate(pageFile string) *template.Template {
|
// point.
|
||||||
|
func parsePageTemplate(
|
||||||
|
pageFile string, included ...string,
|
||||||
|
) *template.Template {
|
||||||
|
files := append([]string{
|
||||||
|
pageFile,
|
||||||
|
"base.html",
|
||||||
|
"htmlheader.html",
|
||||||
|
"navbar.html",
|
||||||
|
}, included...)
|
||||||
|
|
||||||
return template.Must(
|
return template.Must(
|
||||||
template.ParseFS(
|
template.ParseFS(templates.Templates, files...),
|
||||||
templates.Templates,
|
|
||||||
pageFile,
|
|
||||||
"base.html",
|
|
||||||
"htmlheader.html",
|
|
||||||
"navbar.html",
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -121,7 +128,7 @@ func New(
|
|||||||
s.session = params.Session
|
s.session = params.Session
|
||||||
s.mw = params.Middleware
|
s.mw = params.Middleware
|
||||||
s.notifier = params.Notifier
|
s.notifier = params.Notifier
|
||||||
s.evictor = params.Evictor
|
s.archives = params.Archives
|
||||||
s.mtr = metrics.Default()
|
s.mtr = metrics.Default()
|
||||||
s.ssrf = params.SSRFGuard
|
s.ssrf = params.SSRFGuard
|
||||||
|
|
||||||
@@ -131,7 +138,7 @@ func New(
|
|||||||
"profile.html": parsePageTemplate("profile.html"),
|
"profile.html": parsePageTemplate("profile.html"),
|
||||||
"sources_list.html": parsePageTemplate("sources_list.html"),
|
"sources_list.html": parsePageTemplate("sources_list.html"),
|
||||||
"sources_new.html": parsePageTemplate("sources_new.html"),
|
"sources_new.html": parsePageTemplate("sources_new.html"),
|
||||||
"source_detail.html": parsePageTemplate("source_detail.html"),
|
"source_detail.html": parsePageTemplate("source_detail.html", "webhook_stats.html"),
|
||||||
"source_edit.html": parsePageTemplate("source_edit.html"),
|
"source_edit.html": parsePageTemplate("source_edit.html"),
|
||||||
"source_logs.html": parsePageTemplate("source_logs.html"),
|
"source_logs.html": parsePageTemplate("source_logs.html"),
|
||||||
"target_edit.html": parsePageTemplate("target_edit.html"),
|
"target_edit.html": parsePageTemplate("target_edit.html"),
|
||||||
|
|||||||
@@ -51,23 +51,67 @@ func (n *recordingNotifier) Tasks() []delivery.Task {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// recordingEvictor is a delivery.WebhookEvictor that records
|
// recordingArchives is a delivery.Archives that records what it
|
||||||
// the webhook ids it was asked to evict, so a test can prove
|
// was asked to do, so a test can prove that a deletion or rename
|
||||||
// that a deletion path reached the delivery engine.
|
// path reached the delivery engine. After FailRenames, every
|
||||||
type recordingEvictor struct {
|
// rename fails with the given error.
|
||||||
mu sync.Mutex
|
type recordingArchives struct {
|
||||||
evicted []string
|
mu sync.Mutex
|
||||||
|
evicted []string
|
||||||
|
evictedTargets []string
|
||||||
|
renames []archiveRename
|
||||||
|
renameErr error
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *recordingEvictor) EvictWebhook(webhookID string) {
|
// errInjectedRename is the failure a test hands FailRenames.
|
||||||
|
var errInjectedRename = errors.New("injected rename failure")
|
||||||
|
|
||||||
|
// archiveRename is one recorded RenameArchive call.
|
||||||
|
type archiveRename struct {
|
||||||
|
TargetID string
|
||||||
|
WebhookName string
|
||||||
|
TargetName string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *recordingArchives) EvictWebhook(webhookID string) {
|
||||||
r.mu.Lock()
|
r.mu.Lock()
|
||||||
defer r.mu.Unlock()
|
defer r.mu.Unlock()
|
||||||
|
|
||||||
r.evicted = append(r.evicted, webhookID)
|
r.evicted = append(r.evicted, webhookID)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (r *recordingArchives) EvictTarget(targetID string) {
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
|
||||||
|
r.evictedTargets = append(r.evictedTargets, targetID)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *recordingArchives) RenameArchive(
|
||||||
|
targetID, webhookName, targetName string,
|
||||||
|
) error {
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
|
||||||
|
r.renames = append(r.renames, archiveRename{
|
||||||
|
TargetID: targetID,
|
||||||
|
WebhookName: webhookName,
|
||||||
|
TargetName: targetName,
|
||||||
|
})
|
||||||
|
|
||||||
|
return r.renameErr
|
||||||
|
}
|
||||||
|
|
||||||
|
// FailRenames makes every later rename fail with err.
|
||||||
|
func (r *recordingArchives) FailRenames(err error) {
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
|
||||||
|
r.renameErr = err
|
||||||
|
}
|
||||||
|
|
||||||
// Evicted returns a copy of the recorded webhook ids.
|
// Evicted returns a copy of the recorded webhook ids.
|
||||||
func (r *recordingEvictor) Evicted() []string {
|
func (r *recordingArchives) Evicted() []string {
|
||||||
r.mu.Lock()
|
r.mu.Lock()
|
||||||
defer r.mu.Unlock()
|
defer r.mu.Unlock()
|
||||||
|
|
||||||
@@ -77,6 +121,28 @@ func (r *recordingEvictor) Evicted() []string {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// EvictedTargets returns a copy of the recorded target ids.
|
||||||
|
func (r *recordingArchives) EvictedTargets() []string {
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
|
||||||
|
out := make([]string, len(r.evictedTargets))
|
||||||
|
copy(out, r.evictedTargets)
|
||||||
|
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// Renames returns a copy of the recorded renames.
|
||||||
|
func (r *recordingArchives) Renames() []archiveRename {
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
|
||||||
|
out := make([]archiveRename, len(r.renames))
|
||||||
|
copy(out, r.renames)
|
||||||
|
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
func newTestApp(
|
func newTestApp(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
targets ...any,
|
targets ...any,
|
||||||
@@ -103,10 +169,10 @@ func newTestApp(
|
|||||||
func(n *recordingNotifier) delivery.Notifier {
|
func(n *recordingNotifier) delivery.Notifier {
|
||||||
return n
|
return n
|
||||||
},
|
},
|
||||||
func() *recordingEvictor {
|
func() *recordingArchives {
|
||||||
return &recordingEvictor{}
|
return &recordingArchives{}
|
||||||
},
|
},
|
||||||
func(r *recordingEvictor) delivery.WebhookEvictor {
|
func(r *recordingArchives) delivery.Archives {
|
||||||
return r
|
return r
|
||||||
},
|
},
|
||||||
middleware.New,
|
middleware.New,
|
||||||
@@ -176,7 +242,7 @@ func TestHandleIndex_Authenticated(t *testing.T) {
|
|||||||
|
|
||||||
assert.Equal(t, http.StatusSeeOther, w2.Code)
|
assert.Equal(t, http.StatusSeeOther, w2.Code)
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t, "/sources", w2.Header().Get("Location"),
|
t, "/hooks", w2.Header().Get("Location"),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -5,13 +5,13 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// HandleIndex returns a handler for the root path that redirects
|
// HandleIndex returns a handler for the root path that redirects
|
||||||
// based on authentication state: authenticated users go to /sources
|
// based on authentication state: authenticated users go to /hooks
|
||||||
// (the dashboard), unauthenticated users go to the login page.
|
// (the dashboard), unauthenticated users go to the login page.
|
||||||
func (s *Handlers) HandleIndex() http.HandlerFunc {
|
func (s *Handlers) HandleIndex() http.HandlerFunc {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
sess, err := s.session.Get(r)
|
sess, err := s.session.Get(r)
|
||||||
if err == nil && s.session.IsAuthenticated(sess) {
|
if err == nil && s.session.IsAuthenticated(sess) {
|
||||||
http.Redirect(w, r, "/sources", http.StatusSeeOther)
|
http.Redirect(w, r, "/hooks", http.StatusSeeOther)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ package handlers_test
|
|||||||
// this package reach a value an UNAUTHENTICATED client picks outright
|
// this package reach a value an UNAUTHENTICATED client picks outright
|
||||||
// and of a length it picks outright:
|
// and of a length it picks outright:
|
||||||
//
|
//
|
||||||
// - the unknown-entrypoint DEBUG line on /webhook/{uuid}, whose
|
// - the unknown-entrypoint DEBUG line on /h/{uuid}, whose
|
||||||
// path segment matched no stored entrypoint and so is bounded by
|
// path segment matched no stored entrypoint and so is bounded by
|
||||||
// nothing;
|
// nothing;
|
||||||
// - the failed-login DEBUG lines, whose username is a form field.
|
// - the failed-login DEBUG lines, whose username is a form field.
|
||||||
@@ -190,12 +190,12 @@ func assertNoClientText(t *testing.T, buf *bytes.Buffer) {
|
|||||||
// route pattern.
|
// route pattern.
|
||||||
func receiverRouter(h *handlers.Handlers) *chi.Mux {
|
func receiverRouter(h *handlers.Handlers) *chi.Mux {
|
||||||
router := chi.NewRouter()
|
router := chi.NewRouter()
|
||||||
router.Post("/webhook/{uuid}", h.HandleWebhook())
|
router.Post("/h/{uuid}", h.HandleWebhook())
|
||||||
|
|
||||||
return router
|
return router
|
||||||
}
|
}
|
||||||
|
|
||||||
// postReceiver sends one POST at /webhook/<segment>.
|
// postReceiver sends one POST at /h/<segment>.
|
||||||
//
|
//
|
||||||
// RawPath is cleared after parsing so chi routes on the decoded path
|
// RawPath is cleared after parsing so chi routes on the decoded path
|
||||||
// and the handler sees the raw bytes rather than their percent-escaped
|
// and the handler sees the raw bytes rather than their percent-escaped
|
||||||
@@ -210,7 +210,7 @@ func postReceiver(
|
|||||||
req := httptest.NewRequestWithContext(
|
req := httptest.NewRequestWithContext(
|
||||||
context.Background(),
|
context.Background(),
|
||||||
http.MethodPost,
|
http.MethodPost,
|
||||||
"/webhook/"+url.PathEscape(segment),
|
"/h/"+url.PathEscape(segment),
|
||||||
strings.NewReader(""),
|
strings.NewReader(""),
|
||||||
)
|
)
|
||||||
req.URL.RawPath = ""
|
req.URL.RawPath = ""
|
||||||
@@ -339,11 +339,9 @@ const storedUserPassword = "correct-horse-battery-staple"
|
|||||||
// storedFillBytes is the raw length of the client-chosen value in
|
// storedFillBytes is the raw length of the client-chosen value in
|
||||||
// those accounts' usernames. It is well past the 512-byte field
|
// those accounts' usernames. It is well past the 512-byte field
|
||||||
// budget, so the line is still truncated, but short enough that the
|
// budget, so the line is still truncated, but short enough that the
|
||||||
// session cookie a successful login writes stays inside
|
// whole username, markers and fill name included, stays within
|
||||||
// securecookie's 4 KB limit: the cookie is written BEFORE the
|
// database.MaxUsernameBytes.
|
||||||
// "user logged in" line, so an 8 KB username answers 500 and never
|
const storedFillBytes = 960
|
||||||
// reaches it.
|
|
||||||
const storedFillBytes = 1024
|
|
||||||
|
|
||||||
// storedFill builds a username fill of storedFillBytes raw bytes out
|
// storedFill builds a username fill of storedFillBytes raw bytes out
|
||||||
// of repetitions of ch, with both markers at its far end.
|
// of repetitions of ch, with both markers at its far end.
|
||||||
@@ -509,7 +507,7 @@ func TestVerificationCapacity_LogLineDoesNotTrackPathSize(
|
|||||||
http.StatusServiceUnavailable,
|
http.StatusServiceUnavailable,
|
||||||
postLoginAtPath(
|
postLoginAtPath(
|
||||||
t, h,
|
t, h,
|
||||||
"/source/"+url.PathEscape(
|
"/hook/"+url.PathEscape(
|
||||||
oversizedFill(fill),
|
oversizedFill(fill),
|
||||||
)+"/login",
|
)+"/login",
|
||||||
),
|
),
|
||||||
|
|||||||
@@ -88,6 +88,8 @@ func TestHandleProfile_OwnProfile_OK(t *testing.T) {
|
|||||||
h.HandleProfile().ServeHTTP(w, req)
|
h.HandleProfile().ServeHTTP(w, req)
|
||||||
|
|
||||||
assert.Equal(t, http.StatusOK, w.Code)
|
assert.Equal(t, http.StatusOK, w.Code)
|
||||||
|
assert.Contains(t, w.Body.String(), "Account Information")
|
||||||
|
assert.NotContains(t, w.Body.String(), "Account Type")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHandleProfile_OtherProfile_Forbidden(t *testing.T) {
|
func TestHandleProfile_OtherProfile_Forbidden(t *testing.T) {
|
||||||
@@ -158,7 +160,10 @@ func TestUserRoute_Unauthenticated_RedirectedByMiddleware(t *testing.T) {
|
|||||||
"handler must not be reached for unauthenticated request",
|
"handler must not be reached for unauthenticated request",
|
||||||
)
|
)
|
||||||
assert.Equal(t, http.StatusSeeOther, w.Code)
|
assert.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
assert.Equal(t, "/pages/login", w.Header().Get("Location"))
|
assert.Equal(
|
||||||
|
t, "/pages/login?next=%2Fuser%2Ftestuser",
|
||||||
|
w.Header().Get("Location"),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// passwordChangeRequest builds a POST request to the password-change
|
// passwordChangeRequest builds a POST request to the password-change
|
||||||
|
|||||||
@@ -0,0 +1,293 @@
|
|||||||
|
package handlers
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"slices"
|
||||||
|
"strconv"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/dustin/go-humanize"
|
||||||
|
"gorm.io/gorm"
|
||||||
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
|
)
|
||||||
|
|
||||||
|
// recentEventColumns is the recent events list's projection. It
|
||||||
|
// leaves out the body, for the reason maxRenderedBodyBytes gives,
|
||||||
|
// and reads its size from body_bytes, recorded when the event was
|
||||||
|
// stored.
|
||||||
|
const recentEventColumns = "id, created_at, method, content_type, " +
|
||||||
|
"resubmitted_from_id, body_bytes"
|
||||||
|
|
||||||
|
// recentAttemptColumns is the part of a recorded attempt the list
|
||||||
|
// uses. The event log's deliveryResultColumns also reads response
|
||||||
|
// bodies, which the list does not show.
|
||||||
|
const recentAttemptColumns = "delivery_id, status_code, created_at"
|
||||||
|
|
||||||
|
// RecentEventView is one row of the recent events list on a
|
||||||
|
// webhook's page.
|
||||||
|
type RecentEventView struct {
|
||||||
|
Method string
|
||||||
|
ContentType string
|
||||||
|
|
||||||
|
// ResubmittedFromID names the event this one was copied from,
|
||||||
|
// empty for an event that arrived on the receiver.
|
||||||
|
ResubmittedFromID string
|
||||||
|
|
||||||
|
// Received is how long ago the event arrived, and ReceivedUTC
|
||||||
|
// the full timestamp the page shows on hover.
|
||||||
|
Received string
|
||||||
|
ReceivedUTC string
|
||||||
|
|
||||||
|
// Size is the size of the stored body.
|
||||||
|
Size string
|
||||||
|
|
||||||
|
// ProcessingTime is how long the event's slowest delivery
|
||||||
|
// took; see processingTime.
|
||||||
|
ProcessingTime string
|
||||||
|
|
||||||
|
// Status is what the webhook's HTTP target answered, and
|
||||||
|
// StatusClass its colour; see targetStatus. Both are empty
|
||||||
|
// unless the webhook has exactly one HTTP target.
|
||||||
|
Status string
|
||||||
|
StatusClass string
|
||||||
|
}
|
||||||
|
|
||||||
|
// recentEventRow is one row of recentEventColumns.
|
||||||
|
type recentEventRow struct {
|
||||||
|
ID string
|
||||||
|
CreatedAt time.Time
|
||||||
|
Method string
|
||||||
|
ContentType string
|
||||||
|
ResubmittedFromID *string
|
||||||
|
BodyBytes uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
// recentAttemptRow is one row of recentAttemptColumns. CreatedAt is
|
||||||
|
// when the attempt's result was recorded, which is when the attempt
|
||||||
|
// finished.
|
||||||
|
type recentAttemptRow struct {
|
||||||
|
DeliveryID string
|
||||||
|
StatusCode int
|
||||||
|
CreatedAt time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// singleHTTPTargetID returns the ID of the webhook's HTTP target
|
||||||
|
// when it has exactly one, and "" when it has none or several.
|
||||||
|
func singleHTTPTargetID(targets []database.Target) string {
|
||||||
|
id := ""
|
||||||
|
count := 0
|
||||||
|
|
||||||
|
for i := range targets {
|
||||||
|
if targets[i].Type == database.TargetTypeHTTP {
|
||||||
|
id = targets[i].ID
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if count != 1 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
return id
|
||||||
|
}
|
||||||
|
|
||||||
|
// loadRecentEvents loads the webhook's recentEventLimit newest
|
||||||
|
// events for its page, newest first. statusTargetID is the
|
||||||
|
// webhook's only HTTP target, or "" when the list shows no status.
|
||||||
|
func loadRecentEvents(
|
||||||
|
webhookDB *gorm.DB, webhookID, statusTargetID string,
|
||||||
|
) ([]RecentEventView, error) {
|
||||||
|
var rows []recentEventRow
|
||||||
|
|
||||||
|
err := webhookDB.Model(&database.Event{}).
|
||||||
|
Select(recentEventColumns).
|
||||||
|
Where("webhook_id = ?", webhookID).
|
||||||
|
Order("created_at DESC").
|
||||||
|
Limit(recentEventLimit).
|
||||||
|
Find(&rows).Error
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
eventIDs := make([]string, len(rows))
|
||||||
|
for i := range rows {
|
||||||
|
eventIDs[i] = rows[i].ID
|
||||||
|
}
|
||||||
|
|
||||||
|
// Oldest first, so an event's last delivery to a target is its
|
||||||
|
// newest: a replay adds a delivery rather than changing the
|
||||||
|
// earlier one.
|
||||||
|
var deliveries []database.Delivery
|
||||||
|
|
||||||
|
err = webhookDB.
|
||||||
|
Select("id, event_id, target_id, status, created_at").
|
||||||
|
Where("event_id IN ?", eventIDs).
|
||||||
|
Order("created_at ASC").
|
||||||
|
Find(&deliveries).Error
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
byEvent := make(map[string][]database.Delivery, len(rows))
|
||||||
|
deliveryIDs := make([]string, len(deliveries))
|
||||||
|
|
||||||
|
for i := range deliveries {
|
||||||
|
eventID := deliveries[i].EventID
|
||||||
|
byEvent[eventID] = append(byEvent[eventID], deliveries[i])
|
||||||
|
deliveryIDs[i] = deliveries[i].ID
|
||||||
|
}
|
||||||
|
|
||||||
|
attempts, err := loadRecentAttempts(webhookDB, deliveryIDs)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
views := make([]RecentEventView, len(rows))
|
||||||
|
for i := range rows {
|
||||||
|
views[i] = rows[i].view(
|
||||||
|
byEvent[rows[i].ID], attempts, statusTargetID,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return views, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// loadRecentAttempts loads the recorded attempts of the listed
|
||||||
|
// events' deliveries, keyed by delivery ID, each delivery's in
|
||||||
|
// attempt order. The IDs go in chunks for the reason
|
||||||
|
// deliveryIDChunkSize gives.
|
||||||
|
func loadRecentAttempts(
|
||||||
|
webhookDB *gorm.DB, deliveryIDs []string,
|
||||||
|
) (map[string][]recentAttemptRow, error) {
|
||||||
|
byDelivery := make(map[string][]recentAttemptRow)
|
||||||
|
|
||||||
|
for chunk := range slices.Chunk(deliveryIDs, deliveryIDChunkSize) {
|
||||||
|
var rows []recentAttemptRow
|
||||||
|
|
||||||
|
err := webhookDB.Model(&database.DeliveryResult{}).
|
||||||
|
Select(recentAttemptColumns).
|
||||||
|
Where("delivery_id IN ?", chunk).
|
||||||
|
Order("attempt_num ASC").
|
||||||
|
Find(&rows).Error
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
for i := range rows {
|
||||||
|
id := rows[i].DeliveryID
|
||||||
|
byDelivery[id] = append(byDelivery[id], rows[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return byDelivery, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// view projects a loaded row for rendering. deliveries is the
|
||||||
|
// event's deliveries, oldest first, and attempts their recorded
|
||||||
|
// attempts keyed by delivery ID.
|
||||||
|
func (r *recentEventRow) view(
|
||||||
|
deliveries []database.Delivery,
|
||||||
|
attempts map[string][]recentAttemptRow,
|
||||||
|
statusTargetID string,
|
||||||
|
) RecentEventView {
|
||||||
|
v := RecentEventView{
|
||||||
|
Method: r.Method,
|
||||||
|
ContentType: r.ContentType,
|
||||||
|
Received: humanize.Time(r.CreatedAt),
|
||||||
|
ReceivedUTC: r.CreatedAt.UTC().Format(time.DateTime) + " UTC",
|
||||||
|
Size: humanize.Bytes(r.BodyBytes),
|
||||||
|
ProcessingTime: processingTime(deliveries, attempts),
|
||||||
|
}
|
||||||
|
|
||||||
|
if r.ResubmittedFromID != nil {
|
||||||
|
v.ResubmittedFromID = *r.ResubmittedFromID
|
||||||
|
}
|
||||||
|
|
||||||
|
if statusTargetID != "" {
|
||||||
|
v.Status, v.StatusClass = targetStatus(
|
||||||
|
deliveries, attempts, statusTargetID,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
|
||||||
|
// processingTime is how long the event's slowest delivery took,
|
||||||
|
// from being queued to its last recorded attempt, time spent
|
||||||
|
// waiting between retries included. A delivery is queued when its
|
||||||
|
// event is received, or when an operator replays it, so a replay
|
||||||
|
// is timed from the replay rather than from the event's arrival.
|
||||||
|
// It is "in progress" while any delivery is pending or retrying,
|
||||||
|
// and empty for an event with no deliveries.
|
||||||
|
func processingTime(
|
||||||
|
deliveries []database.Delivery,
|
||||||
|
attempts map[string][]recentAttemptRow,
|
||||||
|
) string {
|
||||||
|
if len(deliveries) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
var slowest time.Duration
|
||||||
|
|
||||||
|
for i := range deliveries {
|
||||||
|
if !deliveries[i].Status.Terminal() {
|
||||||
|
return "in progress"
|
||||||
|
}
|
||||||
|
|
||||||
|
tries := attempts[deliveries[i].ID]
|
||||||
|
if len(tries) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
last := tries[len(tries)-1].CreatedAt
|
||||||
|
slowest = max(slowest, last.Sub(deliveries[i].CreatedAt))
|
||||||
|
}
|
||||||
|
|
||||||
|
return slowest.Round(time.Millisecond).String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// targetStatus is what the target answered for the event, and the
|
||||||
|
// colour to show it in: the HTTP status code of the last attempt of
|
||||||
|
// the event's newest delivery to the target. Without a code it is
|
||||||
|
// "no response" when that attempt failed before a response
|
||||||
|
// arrived, the delivery's status ("pending") before any attempt,
|
||||||
|
// and "not sent" when the event has no delivery to the target.
|
||||||
|
func targetStatus(
|
||||||
|
deliveries []database.Delivery,
|
||||||
|
attempts map[string][]recentAttemptRow,
|
||||||
|
targetID string,
|
||||||
|
) (string, string) {
|
||||||
|
newest := -1
|
||||||
|
|
||||||
|
for i := range deliveries {
|
||||||
|
if deliveries[i].TargetID == targetID {
|
||||||
|
newest = i
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if newest < 0 {
|
||||||
|
return "not sent", "text-gray-400"
|
||||||
|
}
|
||||||
|
|
||||||
|
tries := attempts[deliveries[newest].ID]
|
||||||
|
if len(tries) == 0 {
|
||||||
|
return string(deliveries[newest].Status), "text-gray-400"
|
||||||
|
}
|
||||||
|
|
||||||
|
code := tries[len(tries)-1].StatusCode
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case code == 0:
|
||||||
|
return "no response", "text-red-600"
|
||||||
|
case code >= http.StatusInternalServerError:
|
||||||
|
return strconv.Itoa(code), "text-red-600"
|
||||||
|
case code >= http.StatusBadRequest:
|
||||||
|
return strconv.Itoa(code), "text-yellow-600"
|
||||||
|
case code >= http.StatusMultipleChoices:
|
||||||
|
return strconv.Itoa(code), "text-gray-500"
|
||||||
|
case code >= http.StatusOK:
|
||||||
|
return strconv.Itoa(code), "text-green-600"
|
||||||
|
default:
|
||||||
|
return strconv.Itoa(code), "text-gray-500"
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,362 @@
|
|||||||
|
package handlers_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/go-chi/chi"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"gorm.io/gorm"
|
||||||
|
"gorm.io/gorm/clause"
|
||||||
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
|
"sneak.berlin/go/webhooker/internal/handlers"
|
||||||
|
"sneak.berlin/go/webhooker/internal/session"
|
||||||
|
)
|
||||||
|
|
||||||
|
// statusTitle marks the status column's cell in a recent events
|
||||||
|
// row; it is absent from the page when the column is not shown.
|
||||||
|
const statusTitle = `title="HTTP status from the HTTP target"`
|
||||||
|
|
||||||
|
// recentEventsFixture is one started app and a webhook whose
|
||||||
|
// recent events list a test fills.
|
||||||
|
type recentEventsFixture struct {
|
||||||
|
h *handlers.Handlers
|
||||||
|
sess *session.Session
|
||||||
|
db *database.Database
|
||||||
|
webhook *database.Webhook
|
||||||
|
webhookDB *gorm.DB
|
||||||
|
}
|
||||||
|
|
||||||
|
func newRecentEventsFixture(t *testing.T) *recentEventsFixture {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
f := &recentEventsFixture{}
|
||||||
|
|
||||||
|
var dbMgr *database.WebhookDBManager
|
||||||
|
|
||||||
|
app := newTestApp(t, &f.h, &f.sess, &f.db, &dbMgr)
|
||||||
|
app.RequireStart()
|
||||||
|
|
||||||
|
t.Cleanup(app.RequireStop)
|
||||||
|
|
||||||
|
f.webhook = seedWebhook(t, f.db)
|
||||||
|
|
||||||
|
webhookDB, err := dbMgr.GetDB(f.webhook.ID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
f.webhookDB = webhookDB
|
||||||
|
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *recentEventsFixture) render(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
return renderSourceDetailPage(t, f.h, f.sess, f.webhook.ID)
|
||||||
|
}
|
||||||
|
|
||||||
|
// event records an event received at receivedAt, with its body's
|
||||||
|
// size as the receiver records it.
|
||||||
|
func (f *recentEventsFixture) event(
|
||||||
|
t *testing.T, contentType, body string, receivedAt time.Time,
|
||||||
|
) *database.Event {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
event := &database.Event{
|
||||||
|
WebhookID: f.webhook.ID,
|
||||||
|
Method: http.MethodPost,
|
||||||
|
Body: body,
|
||||||
|
BodyBytes: int64(len(body)),
|
||||||
|
ContentType: contentType,
|
||||||
|
}
|
||||||
|
event.CreatedAt = receivedAt
|
||||||
|
|
||||||
|
require.NoError(t, f.webhookDB.Omit(
|
||||||
|
clause.Associations,
|
||||||
|
).Create(event).Error)
|
||||||
|
|
||||||
|
return event
|
||||||
|
}
|
||||||
|
|
||||||
|
// delivery records a delivery of the event to the target, queued
|
||||||
|
// when the event was received.
|
||||||
|
func (f *recentEventsFixture) delivery(
|
||||||
|
t *testing.T,
|
||||||
|
event *database.Event,
|
||||||
|
targetID string,
|
||||||
|
status database.DeliveryStatus,
|
||||||
|
) *database.Delivery {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
return f.deliveryQueuedAt(
|
||||||
|
t, event, targetID, status, event.CreatedAt,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// deliveryQueuedAt records a delivery of the event to the target,
|
||||||
|
// queued at queuedAt, as a replay is.
|
||||||
|
func (f *recentEventsFixture) deliveryQueuedAt(
|
||||||
|
t *testing.T,
|
||||||
|
event *database.Event,
|
||||||
|
targetID string,
|
||||||
|
status database.DeliveryStatus,
|
||||||
|
queuedAt time.Time,
|
||||||
|
) *database.Delivery {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
dlv := &database.Delivery{
|
||||||
|
EventID: event.ID,
|
||||||
|
TargetID: targetID,
|
||||||
|
Status: status,
|
||||||
|
}
|
||||||
|
dlv.CreatedAt = queuedAt
|
||||||
|
|
||||||
|
require.NoError(t, f.webhookDB.Omit(
|
||||||
|
clause.Associations,
|
||||||
|
).Create(dlv).Error)
|
||||||
|
|
||||||
|
return dlv
|
||||||
|
}
|
||||||
|
|
||||||
|
// attempt records one attempt of the delivery that finished took
|
||||||
|
// after the delivery was queued, with HTTP status code (0 for no
|
||||||
|
// response).
|
||||||
|
func (f *recentEventsFixture) attempt(
|
||||||
|
t *testing.T, dlv *database.Delivery, code int, took time.Duration,
|
||||||
|
) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
result := &database.DeliveryResult{
|
||||||
|
DeliveryID: dlv.ID,
|
||||||
|
AttemptNum: 1,
|
||||||
|
StatusCode: code,
|
||||||
|
}
|
||||||
|
result.CreatedAt = dlv.CreatedAt.Add(took)
|
||||||
|
|
||||||
|
require.NoError(t, f.webhookDB.Omit(
|
||||||
|
clause.Associations,
|
||||||
|
).Create(result).Error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// statusCell is the status column's cell as the page renders it.
|
||||||
|
func statusCell(class, text string) string {
|
||||||
|
return `<span class="font-medium ` + class + `" ` + statusTitle +
|
||||||
|
`>` + text + `</span>`
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestHandleSourceDetail_ShowsFiftyNewestEvents proves the list is
|
||||||
|
// headed "50 Most Recent Events" and holds the 50 newest events,
|
||||||
|
// newest first, and not one more.
|
||||||
|
func TestHandleSourceDetail_ShowsFiftyNewestEvents(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
f := newRecentEventsFixture(t)
|
||||||
|
base := time.Now().Add(-time.Hour)
|
||||||
|
|
||||||
|
for i := range 51 {
|
||||||
|
f.event(
|
||||||
|
t, fmt.Sprintf("application/x-recent-%02d", i), "{}",
|
||||||
|
base.Add(time.Duration(i)*time.Second),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
body := f.render(t)
|
||||||
|
|
||||||
|
assert.Contains(t, body, ">50 Most Recent Events</h2>")
|
||||||
|
assert.Equal(t, 50, strings.Count(body, `title="Body size"`))
|
||||||
|
assert.NotContains(t, body, "application/x-recent-00")
|
||||||
|
assert.Contains(t, body, "application/x-recent-01")
|
||||||
|
assert.Less(
|
||||||
|
t,
|
||||||
|
strings.Index(body, "application/x-recent-50"),
|
||||||
|
strings.Index(body, "application/x-recent-49"),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestHandleSourceDetail_RecentEventColumns proves a row shows its
|
||||||
|
// time relative with the UTC timestamp on hover, its body size,
|
||||||
|
// and its processing time once every delivery has finished.
|
||||||
|
func TestHandleSourceDetail_RecentEventColumns(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
f := newRecentEventsFixture(t)
|
||||||
|
logTarget := seedTarget(t, f.db, f.webhook.ID, database.TargetTypeLog)
|
||||||
|
|
||||||
|
receivedAt := time.Now().Add(-210 * time.Second).
|
||||||
|
UTC().Truncate(time.Second)
|
||||||
|
|
||||||
|
done := f.event(
|
||||||
|
t, contentTypeJSON, strings.Repeat("x", 2048), receivedAt,
|
||||||
|
)
|
||||||
|
f.attempt(
|
||||||
|
t,
|
||||||
|
f.delivery(t, done, logTarget.ID, database.DeliveryStatusDelivered),
|
||||||
|
0, 1500*time.Millisecond,
|
||||||
|
)
|
||||||
|
|
||||||
|
waiting := f.event(t, "text/plain", "{}", receivedAt)
|
||||||
|
f.delivery(t, waiting, logTarget.ID, database.DeliveryStatusPending)
|
||||||
|
|
||||||
|
body := f.render(t)
|
||||||
|
|
||||||
|
assert.Contains(
|
||||||
|
t, body,
|
||||||
|
`<span title="`+receivedAt.Format(time.DateTime)+
|
||||||
|
` UTC">3 minutes ago</span>`,
|
||||||
|
)
|
||||||
|
assert.Contains(t, body, `<span title="Body size">2.0 kB</span>`)
|
||||||
|
assert.Contains(t, body, ">1.5s</span>")
|
||||||
|
assert.Contains(t, body, ">in progress</span>")
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestHandleSourceDetail_StatusWithSingleHTTPTarget proves that a
|
||||||
|
// webhook with exactly one HTTP target shows, colour-coded, what
|
||||||
|
// that target answered for each event. The log target beside it
|
||||||
|
// does not count against "exactly one".
|
||||||
|
func TestHandleSourceDetail_StatusWithSingleHTTPTarget(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
f := newRecentEventsFixture(t)
|
||||||
|
target := seedTarget(t, f.db, f.webhook.ID, database.TargetTypeHTTP)
|
||||||
|
seedTarget(t, f.db, f.webhook.ID, database.TargetTypeLog)
|
||||||
|
|
||||||
|
now := time.Now()
|
||||||
|
|
||||||
|
for _, code := range []int{204, 302, 404, 503, 0} {
|
||||||
|
dlv := f.delivery(
|
||||||
|
t, f.event(t, contentTypeJSON, "{}", now), target.ID,
|
||||||
|
database.DeliveryStatusDelivered,
|
||||||
|
)
|
||||||
|
f.attempt(t, dlv, code, time.Second)
|
||||||
|
}
|
||||||
|
|
||||||
|
f.delivery(
|
||||||
|
t, f.event(t, contentTypeJSON, "{}", now), target.ID,
|
||||||
|
database.DeliveryStatusPending,
|
||||||
|
)
|
||||||
|
f.event(t, contentTypeJSON, "{}", now)
|
||||||
|
|
||||||
|
// A replay is a newer delivery, and its answer is the one shown.
|
||||||
|
replayed := f.event(t, contentTypeJSON, "{}", now)
|
||||||
|
f.attempt(t, f.delivery(
|
||||||
|
t, replayed, target.ID, database.DeliveryStatusFailed,
|
||||||
|
), 502, time.Second)
|
||||||
|
f.attempt(t, f.deliveryQueuedAt(
|
||||||
|
t, replayed, target.ID, database.DeliveryStatusDelivered,
|
||||||
|
now.Add(time.Minute),
|
||||||
|
), 200, time.Second)
|
||||||
|
|
||||||
|
body := f.render(t)
|
||||||
|
|
||||||
|
assert.Contains(t, body, statusCell("text-green-600", "204"))
|
||||||
|
assert.Contains(t, body, statusCell("text-gray-500", "302"))
|
||||||
|
assert.Contains(t, body, statusCell("text-yellow-600", "404"))
|
||||||
|
assert.Contains(t, body, statusCell("text-red-600", "503"))
|
||||||
|
assert.Contains(t, body, statusCell("text-red-600", "no response"))
|
||||||
|
assert.Contains(t, body, statusCell("text-gray-400", "pending"))
|
||||||
|
assert.Contains(t, body, statusCell("text-gray-400", "not sent"))
|
||||||
|
assert.Contains(t, body, statusCell("text-green-600", "200"))
|
||||||
|
assert.NotContains(t, body, ">502<")
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestHandleSourceDetail_NoStatusWithoutSingleHTTPTarget proves the
|
||||||
|
// status column is absent when the webhook has no HTTP target or
|
||||||
|
// more than one.
|
||||||
|
func TestHandleSourceDetail_NoStatusWithoutSingleHTTPTarget(
|
||||||
|
t *testing.T,
|
||||||
|
) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
cases := map[string][]database.TargetType{
|
||||||
|
"none": {database.TargetTypeLog},
|
||||||
|
"several": {database.TargetTypeHTTP, database.TargetTypeHTTP},
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, types := range cases {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
f := newRecentEventsFixture(t)
|
||||||
|
event := f.event(t, contentTypeJSON, "{}", time.Now())
|
||||||
|
|
||||||
|
for _, tt := range types {
|
||||||
|
target := seedTarget(t, f.db, f.webhook.ID, tt)
|
||||||
|
f.attempt(t, f.delivery(
|
||||||
|
t, event, target.ID,
|
||||||
|
database.DeliveryStatusDelivered,
|
||||||
|
), 200, time.Second)
|
||||||
|
}
|
||||||
|
|
||||||
|
body := f.render(t)
|
||||||
|
|
||||||
|
assert.Contains(t, body, `title="Body size"`)
|
||||||
|
assert.NotContains(t, body, statusTitle)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestHandleWebhook_RecordsBodySize proves the receiver records the
|
||||||
|
// body's size in bytes, not characters, with the event it stores.
|
||||||
|
func TestHandleWebhook_RecordsBodySize(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
f := newRecentEventsFixture(t)
|
||||||
|
seedEntrypoint(t, f.db, f.webhook.ID)
|
||||||
|
|
||||||
|
// Two bytes per character.
|
||||||
|
body := strings.Repeat("é", 1024)
|
||||||
|
|
||||||
|
req := httptest.NewRequestWithContext(
|
||||||
|
context.Background(), http.MethodPost, "/h/x",
|
||||||
|
strings.NewReader(body),
|
||||||
|
)
|
||||||
|
|
||||||
|
rctx := chi.NewRouteContext()
|
||||||
|
rctx.URLParams.Add("uuid", "ep-"+f.webhook.ID)
|
||||||
|
|
||||||
|
req = req.WithContext(context.WithValue(
|
||||||
|
req.Context(), chi.RouteCtxKey, rctx,
|
||||||
|
))
|
||||||
|
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
f.h.HandleWebhook().ServeHTTP(w, req)
|
||||||
|
|
||||||
|
require.Equal(t, http.StatusOK, w.Code)
|
||||||
|
|
||||||
|
var stored database.Event
|
||||||
|
|
||||||
|
require.NoError(t, f.webhookDB.First(&stored).Error)
|
||||||
|
assert.Equal(t, int64(2048), stored.BodyBytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestHandleSourceDetail_FailedLoadIsAnError proves that when the
|
||||||
|
// list cannot be loaded the page answers with an error, rather than
|
||||||
|
// an empty list claiming the webhook has no events.
|
||||||
|
func TestHandleSourceDetail_FailedLoadIsAnError(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
f := newRecentEventsFixture(t)
|
||||||
|
target := seedTarget(t, f.db, f.webhook.ID, database.TargetTypeHTTP)
|
||||||
|
|
||||||
|
f.attempt(t, f.delivery(
|
||||||
|
t, f.event(t, contentTypeJSON, "{}", time.Now()), target.ID,
|
||||||
|
database.DeliveryStatusDelivered,
|
||||||
|
), 200, time.Second)
|
||||||
|
|
||||||
|
// The attempts are the list's last query, so its events and
|
||||||
|
// deliveries have already loaded when it fails.
|
||||||
|
require.NoError(t, f.webhookDB.Exec(
|
||||||
|
"DROP TABLE delivery_results",
|
||||||
|
).Error)
|
||||||
|
|
||||||
|
w := serveSourceDetailPage(t, f.h, f.sess, f.webhook.ID)
|
||||||
|
|
||||||
|
assert.Equal(t, http.StatusInternalServerError, w.Code)
|
||||||
|
assert.NotContains(t, w.Body.String(), "No events received yet.")
|
||||||
|
}
|
||||||
@@ -15,6 +15,7 @@ import (
|
|||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
"gorm.io/gorm/clause"
|
"gorm.io/gorm/clause"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
"sneak.berlin/go/webhooker/internal/handlers"
|
"sneak.berlin/go/webhooker/internal/handlers"
|
||||||
"sneak.berlin/go/webhooker/internal/session"
|
"sneak.berlin/go/webhooker/internal/session"
|
||||||
)
|
)
|
||||||
@@ -147,18 +148,19 @@ func failDeleteOnTable(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// archivePathFor returns the archive database path the
|
// archivePathFor returns the archive database path the
|
||||||
// delivery engine would use for a webhook: beside the webhook's
|
// delivery engine would use for a database target: beside the
|
||||||
// event database in the data directory.
|
// webhook's event database in the data directory.
|
||||||
func archivePathFor(
|
func archivePathFor(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
mgr *database.WebhookDBManager,
|
mgr *database.WebhookDBManager,
|
||||||
webhookID string,
|
wh *database.Webhook,
|
||||||
|
tgt *database.Target,
|
||||||
) string {
|
) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
return filepath.Join(
|
return filepath.Join(
|
||||||
filepath.Dir(mgr.DBPath(webhookID)),
|
filepath.Dir(mgr.DBPath(wh.ID)),
|
||||||
"archive-"+webhookID+".db",
|
delivery.ArchiveFileName(wh.Name, tgt.Name, tgt.ID),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -195,8 +197,8 @@ func postRequest(
|
|||||||
|
|
||||||
// TestHandleSourceDelete_EvictsArchiveWriter proves that
|
// TestHandleSourceDelete_EvictsArchiveWriter proves that
|
||||||
// deleting a webhook reaches the delivery engine and releases
|
// deleting a webhook reaches the delivery engine and releases
|
||||||
// the webhook's archive writer, exercised through the real
|
// the webhook's archive writers, exercised through the real
|
||||||
// deletion handler rather than by calling the evictor directly.
|
// deletion handler rather than by calling the engine directly.
|
||||||
func TestHandleSourceDelete_EvictsArchiveWriter(t *testing.T) {
|
func TestHandleSourceDelete_EvictsArchiveWriter(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -204,7 +206,7 @@ func TestHandleSourceDelete_EvictsArchiveWriter(t *testing.T) {
|
|||||||
h *handlers.Handlers
|
h *handlers.Handlers
|
||||||
sess *session.Session
|
sess *session.Session
|
||||||
db *database.Database
|
db *database.Database
|
||||||
ev *recordingEvictor
|
ev *recordingArchives
|
||||||
)
|
)
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &ev)
|
app := newTestApp(t, &h, &sess, &db, &ev)
|
||||||
@@ -220,7 +222,7 @@ func TestHandleSourceDelete_EvictsArchiveWriter(t *testing.T) {
|
|||||||
)
|
)
|
||||||
|
|
||||||
req := postRequest(
|
req := postRequest(
|
||||||
"/source/"+wh.ID+"/delete",
|
"/hook/"+wh.ID+"/delete",
|
||||||
cookies,
|
cookies,
|
||||||
map[string]string{paramSourceID: wh.ID},
|
map[string]string{paramSourceID: wh.ID},
|
||||||
)
|
)
|
||||||
@@ -254,9 +256,10 @@ func TestHandleSourceDelete_KeepsArchiveFile(t *testing.T) {
|
|||||||
t.Cleanup(app.RequireStop)
|
t.Cleanup(app.RequireStop)
|
||||||
|
|
||||||
wh := seedWebhook(t, db)
|
wh := seedWebhook(t, db)
|
||||||
|
tgt := seedTarget(t, db, wh.ID, database.TargetTypeDatabase)
|
||||||
|
|
||||||
// Place an archive file where the delivery engine would.
|
// Place an archive file where the delivery engine would.
|
||||||
archivePath := archivePathFor(t, mgr, wh.ID)
|
archivePath := archivePathFor(t, mgr, wh, tgt)
|
||||||
require.NoError(
|
require.NoError(
|
||||||
t,
|
t,
|
||||||
writeArchivePlaceholder(archivePath),
|
writeArchivePlaceholder(archivePath),
|
||||||
@@ -267,7 +270,7 @@ func TestHandleSourceDelete_KeepsArchiveFile(t *testing.T) {
|
|||||||
)
|
)
|
||||||
|
|
||||||
req := postRequest(
|
req := postRequest(
|
||||||
"/source/"+wh.ID+"/delete",
|
"/hook/"+wh.ID+"/delete",
|
||||||
cookies,
|
cookies,
|
||||||
map[string]string{paramSourceID: wh.ID},
|
map[string]string{paramSourceID: wh.ID},
|
||||||
)
|
)
|
||||||
@@ -323,7 +326,7 @@ func TestHandleSourceDelete_FailedDeleteKeepsEverything(
|
|||||||
)
|
)
|
||||||
|
|
||||||
req := postRequest(
|
req := postRequest(
|
||||||
"/source/"+wh.ID+"/delete",
|
"/hook/"+wh.ID+"/delete",
|
||||||
cookies,
|
cookies,
|
||||||
map[string]string{paramSourceID: wh.ID},
|
map[string]string{paramSourceID: wh.ID},
|
||||||
)
|
)
|
||||||
@@ -337,7 +340,7 @@ func TestHandleSourceDelete_FailedDeleteKeepsEverything(
|
|||||||
)
|
)
|
||||||
assert.Empty(
|
assert.Empty(
|
||||||
t, w.Header().Get("Location"),
|
t, w.Header().Get("Location"),
|
||||||
"a failed deletion must not redirect to /sources",
|
"a failed deletion must not redirect to /hooks",
|
||||||
)
|
)
|
||||||
|
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
@@ -402,7 +405,7 @@ func TestHandleSourceDelete_RemovesConfigAndEventDatabase(
|
|||||||
)
|
)
|
||||||
|
|
||||||
req := postRequest(
|
req := postRequest(
|
||||||
"/source/"+wh.ID+"/delete",
|
"/hook/"+wh.ID+"/delete",
|
||||||
cookies,
|
cookies,
|
||||||
map[string]string{paramSourceID: wh.ID},
|
map[string]string{paramSourceID: wh.ID},
|
||||||
)
|
)
|
||||||
@@ -411,7 +414,7 @@ func TestHandleSourceDelete_RemovesConfigAndEventDatabase(
|
|||||||
h.HandleSourceDelete().ServeHTTP(w, req)
|
h.HandleSourceDelete().ServeHTTP(w, req)
|
||||||
|
|
||||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
assert.Equal(t, "/sources", w.Header().Get("Location"))
|
assert.Equal(t, "/hooks", w.Header().Get("Location"))
|
||||||
|
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t, int64(0),
|
t, int64(0),
|
||||||
@@ -435,68 +438,17 @@ func TestHandleSourceDelete_RemovesConfigAndEventDatabase(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestHandleTargetDelete_EvictsWhenLastDatabaseTargetGone
|
// TestHandleTargetDelete_EvictsThatTarget proves that deleting a
|
||||||
// proves that removing the last database target releases the
|
// database target releases that target's archive writer and no
|
||||||
// archive writer.
|
// other: the webhook's other database target keeps its own.
|
||||||
func TestHandleTargetDelete_EvictsWhenLastDatabaseTargetGone(
|
func TestHandleTargetDelete_EvictsThatTarget(t *testing.T) {
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
var (
|
var (
|
||||||
h *handlers.Handlers
|
h *handlers.Handlers
|
||||||
sess *session.Session
|
sess *session.Session
|
||||||
db *database.Database
|
db *database.Database
|
||||||
ev *recordingEvictor
|
ev *recordingArchives
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &ev)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
wh := seedWebhook(t, db)
|
|
||||||
tgt := seedTarget(
|
|
||||||
t, db, wh.ID, database.TargetTypeDatabase,
|
|
||||||
)
|
|
||||||
|
|
||||||
cookies := authenticatedCookies(
|
|
||||||
t, sess, deleteTestUserID, deleteTestUsername,
|
|
||||||
)
|
|
||||||
|
|
||||||
req := postRequest(
|
|
||||||
"/source/"+wh.ID+"/targets/"+tgt.ID+"/delete",
|
|
||||||
cookies,
|
|
||||||
map[string]string{
|
|
||||||
paramSourceID: wh.ID,
|
|
||||||
paramTargetID: tgt.ID,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
|
|
||||||
h.HandleTargetDelete().ServeHTTP(w, req)
|
|
||||||
|
|
||||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
|
||||||
assert.Equal(
|
|
||||||
t, []string{wh.ID}, ev.Evicted(),
|
|
||||||
"removing the last database target should evict",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleTargetDelete_KeepsWriterWhenDatabaseTargetRemains
|
|
||||||
// proves that deleting one of several database targets leaves
|
|
||||||
// the still-needed archive writer alone: the surviving target
|
|
||||||
// keeps archiving to the same file, so the writer must stay.
|
|
||||||
func TestHandleTargetDelete_KeepsWriterWhenDatabaseTargetRemains(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
ev *recordingEvictor
|
|
||||||
)
|
)
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &ev)
|
app := newTestApp(t, &h, &sess, &db, &ev)
|
||||||
@@ -515,7 +467,7 @@ func TestHandleTargetDelete_KeepsWriterWhenDatabaseTargetRemains(
|
|||||||
)
|
)
|
||||||
|
|
||||||
req := postRequest(
|
req := postRequest(
|
||||||
"/source/"+wh.ID+"/targets/"+doomed.ID+"/delete",
|
"/hook/"+wh.ID+"/targets/"+doomed.ID+"/delete",
|
||||||
cookies,
|
cookies,
|
||||||
map[string]string{
|
map[string]string{
|
||||||
paramSourceID: wh.ID,
|
paramSourceID: wh.ID,
|
||||||
@@ -527,17 +479,17 @@ func TestHandleTargetDelete_KeepsWriterWhenDatabaseTargetRemains(
|
|||||||
h.HandleTargetDelete().ServeHTTP(w, req)
|
h.HandleTargetDelete().ServeHTTP(w, req)
|
||||||
|
|
||||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
assert.Empty(
|
assert.Equal(
|
||||||
t, ev.Evicted(),
|
t, []string{doomed.ID}, ev.EvictedTargets(),
|
||||||
"a second database target still needs the writer",
|
"deleting a database target should evict its writer",
|
||||||
)
|
)
|
||||||
|
assert.Empty(t, ev.Evicted(), "the webhook is not deleted")
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestHandleTargetDelete_KeepsWriterWhenOtherTypeDeleted proves
|
// TestHandleTargetDelete_IgnoresAnotherWebhooksTarget proves that
|
||||||
// that deleting a target of an unrelated type leaves a
|
// a target id from the URL that is not a target of the webhook
|
||||||
// still-needed archive writer alone: the webhook's database
|
// deletes nothing and so evicts nothing.
|
||||||
// target is untouched, so its writer must stay.
|
func TestHandleTargetDelete_IgnoresAnotherWebhooksTarget(
|
||||||
func TestHandleTargetDelete_KeepsWriterWhenOtherTypeDeleted(
|
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
@@ -546,7 +498,7 @@ func TestHandleTargetDelete_KeepsWriterWhenOtherTypeDeleted(
|
|||||||
h *handlers.Handlers
|
h *handlers.Handlers
|
||||||
sess *session.Session
|
sess *session.Session
|
||||||
db *database.Database
|
db *database.Database
|
||||||
ev *recordingEvictor
|
ev *recordingArchives
|
||||||
)
|
)
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &ev)
|
app := newTestApp(t, &h, &sess, &db, &ev)
|
||||||
@@ -555,19 +507,20 @@ func TestHandleTargetDelete_KeepsWriterWhenOtherTypeDeleted(
|
|||||||
t.Cleanup(app.RequireStop)
|
t.Cleanup(app.RequireStop)
|
||||||
|
|
||||||
wh := seedWebhook(t, db)
|
wh := seedWebhook(t, db)
|
||||||
seedTarget(t, db, wh.ID, database.TargetTypeDatabase)
|
elsewhere := seedTarget(
|
||||||
other := seedTarget(t, db, wh.ID, database.TargetTypeLog)
|
t, db, seedWebhook(t, db).ID, database.TargetTypeDatabase,
|
||||||
|
)
|
||||||
|
|
||||||
cookies := authenticatedCookies(
|
cookies := authenticatedCookies(
|
||||||
t, sess, deleteTestUserID, deleteTestUsername,
|
t, sess, deleteTestUserID, deleteTestUsername,
|
||||||
)
|
)
|
||||||
|
|
||||||
req := postRequest(
|
req := postRequest(
|
||||||
"/source/"+wh.ID+"/targets/"+other.ID+"/delete",
|
"/hook/"+wh.ID+"/targets/"+elsewhere.ID+"/delete",
|
||||||
cookies,
|
cookies,
|
||||||
map[string]string{
|
map[string]string{
|
||||||
paramSourceID: wh.ID,
|
paramSourceID: wh.ID,
|
||||||
paramTargetID: other.ID,
|
paramTargetID: elsewhere.ID,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
w := httptest.NewRecorder()
|
w := httptest.NewRecorder()
|
||||||
@@ -576,7 +529,7 @@ func TestHandleTargetDelete_KeepsWriterWhenOtherTypeDeleted(
|
|||||||
|
|
||||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
assert.Empty(
|
assert.Empty(
|
||||||
t, ev.Evicted(),
|
t, ev.EvictedTargets(),
|
||||||
"a surviving database target must keep its writer",
|
"another webhook's target must not be evicted",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -81,7 +81,7 @@ func (f *baseURLFixture) entrypointURL(
|
|||||||
req := httptest.NewRequestWithContext(
|
req := httptest.NewRequestWithContext(
|
||||||
context.Background(),
|
context.Background(),
|
||||||
http.MethodGet,
|
http.MethodGet,
|
||||||
"/source/"+f.webhook,
|
"/hook/"+f.webhook,
|
||||||
nil,
|
nil,
|
||||||
)
|
)
|
||||||
req.Host = host
|
req.Host = host
|
||||||
@@ -213,7 +213,7 @@ func TestSourceDetailBaseURL_ForwardedProtoSpellings(t *testing.T) {
|
|||||||
|
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t,
|
t,
|
||||||
tc.scheme+"://"+host+"/webhook/"+fixture.path,
|
tc.scheme+"://"+host+"/h/"+fixture.path,
|
||||||
fixture.entrypointURL(
|
fixture.entrypointURL(
|
||||||
t, host, forwardedProto(tc.header),
|
t, host, forwardedProto(tc.header),
|
||||||
),
|
),
|
||||||
@@ -244,7 +244,7 @@ func TestSourceDetailBaseURL_DirectTLSBeatsPlaintextHeader(
|
|||||||
|
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t,
|
t,
|
||||||
"https://"+host+"/webhook/"+fixture.path,
|
"https://"+host+"/h/"+fixture.path,
|
||||||
got,
|
got,
|
||||||
"a connection this process terminated with TLS "+
|
"a connection this process terminated with TLS "+
|
||||||
"outranks a header claiming plaintext",
|
"outranks a header claiming plaintext",
|
||||||
@@ -272,7 +272,7 @@ func TestSourceDetailBaseURL_KeepsHostAuthority(t *testing.T) {
|
|||||||
|
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t,
|
t,
|
||||||
"https://"+host+"/webhook/"+fixture.path,
|
"https://"+host+"/h/"+fixture.path,
|
||||||
fixture.entrypointURL(
|
fixture.entrypointURL(
|
||||||
t, host, forwardedProto("HTTPS"),
|
t, host, forwardedProto("HTTPS"),
|
||||||
),
|
),
|
||||||
|
|||||||
@@ -62,10 +62,27 @@ func renderSourceDetailPage(
|
|||||||
) string {
|
) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
|
w := serveSourceDetailPage(t, h, sess, webhookID)
|
||||||
|
|
||||||
|
require.Equal(t, http.StatusOK, w.Code)
|
||||||
|
|
||||||
|
return w.Body.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// serveSourceDetailPage runs the real source detail handler for a
|
||||||
|
// webhook and returns its response, whatever its status.
|
||||||
|
func serveSourceDetailPage(
|
||||||
|
t *testing.T,
|
||||||
|
h *handlers.Handlers,
|
||||||
|
sess *session.Session,
|
||||||
|
webhookID string,
|
||||||
|
) *httptest.ResponseRecorder {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
req := httptest.NewRequestWithContext(
|
req := httptest.NewRequestWithContext(
|
||||||
context.Background(),
|
context.Background(),
|
||||||
http.MethodGet,
|
http.MethodGet,
|
||||||
"/source/"+webhookID,
|
"/hook/"+webhookID,
|
||||||
nil,
|
nil,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -87,9 +104,7 @@ func renderSourceDetailPage(
|
|||||||
w := httptest.NewRecorder()
|
w := httptest.NewRecorder()
|
||||||
h.HandleSourceDetail().ServeHTTP(w, req)
|
h.HandleSourceDetail().ServeHTTP(w, req)
|
||||||
|
|
||||||
require.Equal(t, http.StatusOK, w.Code)
|
return w
|
||||||
|
|
||||||
return w.Body.String()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestHandleSourceDetail_MasksSlackWebhookURL is the
|
// TestHandleSourceDetail_MasksSlackWebhookURL is the
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ func deleteTargetThroughHandler(
|
|||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
req := postRequest(
|
req := postRequest(
|
||||||
"/source/"+webhookID+"/targets/"+targetID+"/delete",
|
"/hook/"+webhookID+"/targets/"+targetID+"/delete",
|
||||||
authenticatedCookies(
|
authenticatedCookies(
|
||||||
t, sess, deleteTestUserID, deleteTestUsername,
|
t, sess, deleteTestUserID, deleteTestUsername,
|
||||||
),
|
),
|
||||||
|
|||||||
@@ -84,7 +84,7 @@ func renderSourceLogsPageWithQuery(
|
|||||||
req := httptest.NewRequestWithContext(
|
req := httptest.NewRequestWithContext(
|
||||||
context.Background(),
|
context.Background(),
|
||||||
http.MethodGet,
|
http.MethodGet,
|
||||||
"/source/"+webhookID+"/logs"+query,
|
"/hook/"+webhookID+"/events"+query,
|
||||||
nil,
|
nil,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -330,7 +330,7 @@ func (h *Handlers) createWebhookWithEntrypoint(
|
|||||||
)
|
)
|
||||||
|
|
||||||
http.Redirect(
|
http.Redirect(
|
||||||
w, r, "/source/"+webhook.ID, http.StatusSeeOther,
|
w, r, "/hook/"+webhook.ID, http.StatusSeeOther,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -415,16 +415,23 @@ func (h *Handlers) renderSourceDetail(
|
|||||||
"webhook_id = ?", webhook.ID,
|
"webhook_id = ?", webhook.ID,
|
||||||
).Find(&targets)
|
).Find(&targets)
|
||||||
|
|
||||||
var events []database.Event
|
var events []RecentEventView
|
||||||
|
|
||||||
if h.dbMgr.DBExists(webhook.ID) {
|
if h.dbMgr.DBExists(webhook.ID) {
|
||||||
webhookDB, dbErr := h.dbMgr.GetDB(webhook.ID)
|
webhookDB, err := h.dbMgr.GetDB(webhook.ID)
|
||||||
if dbErr == nil {
|
if err != nil {
|
||||||
webhookDB.Where(
|
h.serverError(w, "failed to get webhook database", err)
|
||||||
"webhook_id = ?", webhook.ID,
|
|
||||||
).Order("created_at DESC").Limit(
|
return
|
||||||
recentEventLimit,
|
}
|
||||||
).Find(&events)
|
|
||||||
|
events, err = loadRecentEvents(
|
||||||
|
webhookDB, webhook.ID, singleHTTPTargetID(targets),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
h.serverError(w, "failed to load recent events", err)
|
||||||
|
|
||||||
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -450,6 +457,7 @@ func (h *Handlers) renderSourceDetail(
|
|||||||
"Targets": delivery.NewTargetViews(targets),
|
"Targets": delivery.NewTargetViews(targets),
|
||||||
"Events": events,
|
"Events": events,
|
||||||
"BaseURL": baseURL,
|
"BaseURL": baseURL,
|
||||||
|
"Stats": h.loadWebhookStats(webhook.ID, entrypoints, targets),
|
||||||
}
|
}
|
||||||
|
|
||||||
h.renderTemplate(w, r, "source_detail.html", data)
|
h.renderTemplate(w, r, "source_detail.html", data)
|
||||||
@@ -551,6 +559,7 @@ func (h *Handlers) applyWebhookEdit(
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
oldName := webhook.Name
|
||||||
webhook.Name = name
|
webhook.Name = name
|
||||||
webhook.Description = r.PostFormValue("description")
|
webhook.Description = r.PostFormValue("description")
|
||||||
|
|
||||||
@@ -573,15 +582,31 @@ func (h *Handlers) applyWebhookEdit(
|
|||||||
|
|
||||||
webhook.RetentionDays = retentionDays
|
webhook.RetentionDays = retentionDays
|
||||||
|
|
||||||
err := h.db.DB().Save(webhook).Error
|
// The archive files are renamed before the new name is saved
|
||||||
|
// (see delivery.Engine.RenameArchive). If either step fails,
|
||||||
|
// they go back to the name that is still stored.
|
||||||
|
err := h.renameWebhookArchives(webhook.ID, webhook.Name)
|
||||||
|
if err == nil {
|
||||||
|
err = h.db.DB().Save(webhook).Error
|
||||||
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
restoreErr := h.renameWebhookArchives(webhook.ID, oldName)
|
||||||
|
if restoreErr != nil {
|
||||||
|
h.log.Error(
|
||||||
|
"failed to rename archives back",
|
||||||
|
"webhook_id", webhook.ID,
|
||||||
|
"error", restoreErr,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
h.serverError(w, "failed to update webhook", err)
|
h.serverError(w, "failed to update webhook", err)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
http.Redirect(
|
http.Redirect(
|
||||||
w, r, "/source/"+webhook.ID, http.StatusSeeOther,
|
w, r, "/hook/"+webhook.ID, http.StatusSeeOther,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -664,7 +689,7 @@ func (h *Handlers) deleteWebhookResources(
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
http.Redirect(w, r, "/sources", http.StatusSeeOther)
|
http.Redirect(w, r, "/hooks", http.StatusSeeOther)
|
||||||
}
|
}
|
||||||
|
|
||||||
// commitWebhookDeletion soft-deletes a webhook's entrypoints,
|
// commitWebhookDeletion soft-deletes a webhook's entrypoints,
|
||||||
@@ -709,11 +734,11 @@ func (h *Handlers) commitWebhookDeletion(
|
|||||||
return tx.Commit().Error
|
return tx.Commit().Error
|
||||||
}
|
}
|
||||||
|
|
||||||
// evictArchiveWriter asks the delivery engine to drop its
|
// evictArchiveWriter asks the delivery engine to drop the cached
|
||||||
// cached archive writer for a webhook, closing the archive file
|
// archive writers of a webhook's database targets, closing their
|
||||||
// handle.
|
// archive file handles.
|
||||||
//
|
//
|
||||||
// The archive database file is NOT deleted. Unlike the event
|
// The archive database files are NOT deleted. Unlike the event
|
||||||
// database — which is per-webhook working storage and is
|
// database — which is per-webhook working storage and is
|
||||||
// hard-deleted with the webhook — an archive is explicitly
|
// hard-deleted with the webhook — an archive is explicitly
|
||||||
// long-term storage that an operator may want to keep or move
|
// long-term storage that an operator may want to keep or move
|
||||||
@@ -721,50 +746,57 @@ func (h *Handlers) commitWebhookDeletion(
|
|||||||
// deleting a webhook would be a surprising and unrecoverable
|
// deleting a webhook would be a surprising and unrecoverable
|
||||||
// data loss, so the file is left for the operator to handle.
|
// data loss, so the file is left for the operator to handle.
|
||||||
func (h *Handlers) evictArchiveWriter(webhookID string) {
|
func (h *Handlers) evictArchiveWriter(webhookID string) {
|
||||||
if h.evictor == nil {
|
if h.archives == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
h.evictor.EvictWebhook(webhookID)
|
h.archives.EvictWebhook(webhookID)
|
||||||
}
|
}
|
||||||
|
|
||||||
// evictArchiveWriterIfUnused releases a webhook's archive
|
// evictTargetArchiveWriter is evictArchiveWriter for one deleted
|
||||||
// writer once the webhook has no database target left to feed
|
// target, and leaves its archive file on disk for the same reason.
|
||||||
// it.
|
// A target that is not a database target has no writer, and
|
||||||
//
|
// evicting it does nothing.
|
||||||
// It is called after any child resource of a webhook is
|
func (h *Handlers) evictTargetArchiveWriter(targetID string) {
|
||||||
// deleted, and is correct without knowing which kind was: it
|
if h.archives == nil {
|
||||||
// evicts only when no database target remains, so deleting one
|
return
|
||||||
// of several database targets — or deleting an unrelated
|
}
|
||||||
// target type — leaves a still-needed writer alone. When no
|
|
||||||
// database target ever existed there is no writer and eviction
|
h.archives.EvictTarget(targetID)
|
||||||
// is a no-op. Soft-deleted targets are excluded by GORM's
|
}
|
||||||
// default scope, so the row just deleted is not counted.
|
|
||||||
func (h *Handlers) evictArchiveWriterIfUnused(webhookID string) {
|
// renameWebhookArchives renames the archive file of every database
|
||||||
var remaining int64
|
// target of a webhook for the webhook name webhookName, keeping
|
||||||
|
// each target's own name. It stops at the first failure.
|
||||||
|
func (h *Handlers) renameWebhookArchives(
|
||||||
|
webhookID, webhookName string,
|
||||||
|
) error {
|
||||||
|
if h.archives == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var targets []database.Target
|
||||||
|
|
||||||
err := h.db.DB().
|
err := h.db.DB().
|
||||||
Model(&database.Target{}).
|
|
||||||
Where(
|
Where(
|
||||||
"webhook_id = ? AND type = ?",
|
"webhook_id = ? AND type = ?",
|
||||||
webhookID, database.TargetTypeDatabase,
|
webhookID, database.TargetTypeDatabase,
|
||||||
).
|
).
|
||||||
Count(&remaining).Error
|
Find(&targets).Error
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.log.Error(
|
return err
|
||||||
"failed to count remaining database targets",
|
}
|
||||||
"webhook_id", webhookID,
|
|
||||||
"error", err,
|
for i := range targets {
|
||||||
|
err = h.archives.RenameArchive(
|
||||||
|
targets[i].ID, webhookName, targets[i].Name,
|
||||||
)
|
)
|
||||||
|
if err != nil {
|
||||||
return
|
return err
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if remaining > 0 {
|
return nil
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
h.evictArchiveWriter(webhookID)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ownedWebhook resolves the request's sourceID parameter to a
|
// ownedWebhook resolves the request's sourceID parameter to a
|
||||||
@@ -1257,7 +1289,7 @@ func (h *Handlers) HandleEntrypointCreate() http.HandlerFunc {
|
|||||||
}
|
}
|
||||||
|
|
||||||
http.Redirect(
|
http.Redirect(
|
||||||
w, r, "/source/"+webhook.ID, http.StatusSeeOther,
|
w, r, "/hook/"+webhook.ID, http.StatusSeeOther,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1313,7 +1345,7 @@ func (h *Handlers) processTargetCreate(
|
|||||||
//
|
//
|
||||||
// Every field here is read with PostFormValue, not FormValue.
|
// Every field here is read with PostFormValue, not FormValue.
|
||||||
// FormValue falls back to the query string, which would let
|
// FormValue falls back to the query string, which would let
|
||||||
// `POST /source/{id}/targets?url=https://hooks.slack.com/...`
|
// `POST /hook/{id}/targets?url=https://hooks.slack.com/...`
|
||||||
// configure a target from a value the request line carries — and
|
// configure a target from a value the request line carries — and
|
||||||
// the request line, unlike the body, is what logs, proxies,
|
// the request line, unlike the body, is what logs, proxies,
|
||||||
// Referer headers and error trackers record.
|
// Referer headers and error trackers record.
|
||||||
@@ -1370,7 +1402,7 @@ func (h *Handlers) processTargetCreate(
|
|||||||
}
|
}
|
||||||
|
|
||||||
http.Redirect(
|
http.Redirect(
|
||||||
w, r, "/source/"+webhook.ID, http.StatusSeeOther,
|
w, r, "/hook/"+webhook.ID, http.StatusSeeOther,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1428,7 +1460,7 @@ type targetFormInput struct {
|
|||||||
//
|
//
|
||||||
// Every field is read with PostFormValue, not FormValue. FormValue
|
// Every field is read with PostFormValue, not FormValue. FormValue
|
||||||
// falls back to the query string, which would let
|
// falls back to the query string, which would let
|
||||||
// `POST /source/{id}/targets?url=https://hooks.slack.com/...`
|
// `POST /hook/{id}/targets?url=https://hooks.slack.com/...`
|
||||||
// configure a target from a value the request line carries — and the
|
// configure a target from a value the request line carries — and the
|
||||||
// request line, unlike the body, is what logs, proxies, Referer
|
// request line, unlike the body, is what logs, proxies, Referer
|
||||||
// headers and error trackers record. The headers field is under the
|
// headers and error trackers record. The headers field is under the
|
||||||
@@ -1641,27 +1673,26 @@ func (h *Handlers) HandleEntrypointDelete() http.HandlerFunc {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// HandleTargetDelete handles deleting a target. Deleting the
|
// HandleTargetDelete handles deleting a target. A deleted
|
||||||
// last database target of a webhook leaves its archive writer
|
// database target's archive writer is evicted and its handle
|
||||||
// with nothing to write, so the writer is evicted and its
|
// closed; the archive file is left on disk.
|
||||||
// handle closed; the archive file is left on disk.
|
|
||||||
func (h *Handlers) HandleTargetDelete() http.HandlerFunc {
|
func (h *Handlers) HandleTargetDelete() http.HandlerFunc {
|
||||||
return h.deleteChildResource(
|
return h.deleteChildResource(
|
||||||
"targetID", &database.Target{},
|
"targetID", &database.Target{},
|
||||||
"failed to delete target",
|
"failed to delete target",
|
||||||
h.evictArchiveWriterIfUnused,
|
h.evictTargetArchiveWriter,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// deleteChildResource returns a handler that deletes a child
|
// deleteChildResource returns a handler that deletes a child
|
||||||
// resource (entrypoint or target) belonging to a webhook. The
|
// resource (entrypoint or target) belonging to a webhook. The
|
||||||
// optional afterDelete hook runs with the webhook's id once the
|
// optional afterDelete hook runs with the child's id once the
|
||||||
// delete has succeeded, before the redirect.
|
// delete has removed it, before the redirect.
|
||||||
func (h *Handlers) deleteChildResource(
|
func (h *Handlers) deleteChildResource(
|
||||||
idParam string,
|
idParam string,
|
||||||
model any,
|
model any,
|
||||||
errMsg string,
|
errMsg string,
|
||||||
afterDelete func(webhookID string),
|
afterDelete func(childID string),
|
||||||
) http.HandlerFunc {
|
) http.HandlerFunc {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
userID, ok := h.getUserID(r)
|
userID, ok := h.getUserID(r)
|
||||||
@@ -1701,13 +1732,15 @@ func (h *Handlers) deleteChildResource(
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if afterDelete != nil {
|
// Only for a row this webhook really had: the id came from
|
||||||
afterDelete(webhook.ID)
|
// the URL and may name another webhook's child.
|
||||||
|
if afterDelete != nil && result.RowsAffected > 0 {
|
||||||
|
afterDelete(childID)
|
||||||
}
|
}
|
||||||
|
|
||||||
http.Redirect(
|
http.Redirect(
|
||||||
w, r,
|
w, r,
|
||||||
"/source/"+webhook.ID,
|
"/hook/"+webhook.ID,
|
||||||
http.StatusSeeOther,
|
http.StatusSeeOther,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -1804,7 +1837,7 @@ func (h *Handlers) toggleChildResource(
|
|||||||
|
|
||||||
http.Redirect(
|
http.Redirect(
|
||||||
w, r,
|
w, r,
|
||||||
"/source/"+webhook.ID,
|
"/hook/"+webhook.ID,
|
||||||
http.StatusSeeOther,
|
http.StatusSeeOther,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -105,7 +105,7 @@ func submitCreate(
|
|||||||
form.Set("retention_days", *retention)
|
form.Set("retention_days", *retention)
|
||||||
}
|
}
|
||||||
|
|
||||||
req := formRequest("/sources/new", cookies, form, nil)
|
req := formRequest("/hooks/new", cookies, form, nil)
|
||||||
w := httptest.NewRecorder()
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
h.HandleSourceCreateSubmit().ServeHTTP(w, req)
|
h.HandleSourceCreateSubmit().ServeHTTP(w, req)
|
||||||
@@ -187,6 +187,7 @@ func storedRetentionDays(
|
|||||||
type sourceTestEnv struct {
|
type sourceTestEnv struct {
|
||||||
handlers *handlers.Handlers
|
handlers *handlers.Handlers
|
||||||
db *database.Database
|
db *database.Database
|
||||||
|
archives *recordingArchives
|
||||||
cookies []*http.Cookie
|
cookies []*http.Cookie
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -199,7 +200,9 @@ func setupSourceTest(t *testing.T) *sourceTestEnv {
|
|||||||
|
|
||||||
var db *database.Database
|
var db *database.Database
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db)
|
var archives *recordingArchives
|
||||||
|
|
||||||
|
app := newTestApp(t, &h, &sess, &db, &archives)
|
||||||
app.RequireStart()
|
app.RequireStart()
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
t.Cleanup(app.RequireStop)
|
||||||
@@ -207,6 +210,7 @@ func setupSourceTest(t *testing.T) *sourceTestEnv {
|
|||||||
return &sourceTestEnv{
|
return &sourceTestEnv{
|
||||||
handlers: h,
|
handlers: h,
|
||||||
db: db,
|
db: db,
|
||||||
|
archives: archives,
|
||||||
cookies: authenticatedCookies(
|
cookies: authenticatedCookies(
|
||||||
t, sess, sourceTestUserID, "sourceuser",
|
t, sess, sourceTestUserID, "sourceuser",
|
||||||
),
|
),
|
||||||
@@ -265,7 +269,7 @@ func TestHandleSourceCreate_PrefillsDefaultFromConstant(t *testing.T) {
|
|||||||
|
|
||||||
w := httptest.NewRecorder()
|
w := httptest.NewRecorder()
|
||||||
env.handlers.HandleSourceCreate().ServeHTTP(
|
env.handlers.HandleSourceCreate().ServeHTTP(
|
||||||
w, getRequest(t, "/sources/new", env.cookies, nil),
|
w, getRequest(t, "/hooks/new", env.cookies, nil),
|
||||||
)
|
)
|
||||||
|
|
||||||
require.Equal(t, http.StatusOK, w.Code)
|
require.Equal(t, http.StatusOK, w.Code)
|
||||||
@@ -402,7 +406,7 @@ func TestHandleSourceCreateSubmit_RejectedFormKeepsUserInput(
|
|||||||
form.Set("description", description)
|
form.Set("description", description)
|
||||||
form.Set("retention_days", "nonsense")
|
form.Set("retention_days", "nonsense")
|
||||||
|
|
||||||
req := formRequest("/sources/new", env.cookies, form, nil)
|
req := formRequest("/hooks/new", env.cookies, form, nil)
|
||||||
w := httptest.NewRecorder()
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
env.handlers.HandleSourceCreateSubmit().ServeHTTP(w, req)
|
env.handlers.HandleSourceCreateSubmit().ServeHTTP(w, req)
|
||||||
@@ -430,7 +434,7 @@ func submitEdit(
|
|||||||
form.Set("retention_days", retention)
|
form.Set("retention_days", retention)
|
||||||
|
|
||||||
req := formRequest(
|
req := formRequest(
|
||||||
"/source/"+wh.ID+"/edit",
|
"/hook/"+wh.ID+"/edit",
|
||||||
env.cookies,
|
env.cookies,
|
||||||
form,
|
form,
|
||||||
map[string]string{sourceIDParam: wh.ID},
|
map[string]string{sourceIDParam: wh.ID},
|
||||||
@@ -498,6 +502,74 @@ func TestHandleSourceEditSubmit_EmptyRetentionLeavesValueUnchanged(
|
|||||||
assert.Equal(t, 7, storedRetentionDays(t, env.db, wh.ID))
|
assert.Equal(t, 7, storedRetentionDays(t, env.db, wh.ID))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// renamedWebhookName is the name the rename tests give a webhook.
|
||||||
|
const renamedWebhookName = "Renamed"
|
||||||
|
|
||||||
|
// TestHandleSourceEditSubmit_RenamesArchives proves that renaming a
|
||||||
|
// webhook renames the archive of each of its database targets, and
|
||||||
|
// asks nothing of its other targets.
|
||||||
|
func TestHandleSourceEditSubmit_RenamesArchives(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
env := setupSourceTest(t)
|
||||||
|
wh := seedWebhookWithRetention(t, env.db, 7)
|
||||||
|
first := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
|
||||||
|
second := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
|
||||||
|
seedTarget(t, env.db, wh.ID, database.TargetTypeLog)
|
||||||
|
|
||||||
|
wh.Name = renamedWebhookName
|
||||||
|
|
||||||
|
w := submitEdit(t, env, wh, "")
|
||||||
|
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
|
|
||||||
|
assert.ElementsMatch(
|
||||||
|
t,
|
||||||
|
[]archiveRename{
|
||||||
|
{first.ID, renamedWebhookName, first.Name},
|
||||||
|
{second.ID, renamedWebhookName, second.Name},
|
||||||
|
},
|
||||||
|
env.archives.Renames(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestHandleSourceEditSubmit_FailedRenameKeepsTheName proves that a
|
||||||
|
// webhook whose archive cannot be renamed keeps its stored name, so
|
||||||
|
// the name on disk and the name in the UI do not part, and that the
|
||||||
|
// handler puts back what it may already have moved.
|
||||||
|
func TestHandleSourceEditSubmit_FailedRenameKeepsTheName(
|
||||||
|
t *testing.T,
|
||||||
|
) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
env := setupSourceTest(t)
|
||||||
|
wh := seedWebhookWithRetention(t, env.db, 7)
|
||||||
|
tgt := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
|
||||||
|
|
||||||
|
env.archives.FailRenames(errInjectedRename)
|
||||||
|
|
||||||
|
oldName := wh.Name
|
||||||
|
wh.Name = renamedWebhookName
|
||||||
|
|
||||||
|
w := submitEdit(t, env, wh, "")
|
||||||
|
require.Equal(t, http.StatusInternalServerError, w.Code)
|
||||||
|
|
||||||
|
var stored database.Webhook
|
||||||
|
|
||||||
|
require.NoError(
|
||||||
|
t, env.db.DB().First(&stored, "id = ?", wh.ID).Error,
|
||||||
|
)
|
||||||
|
assert.Equal(t, oldName, stored.Name)
|
||||||
|
|
||||||
|
assert.Equal(
|
||||||
|
t,
|
||||||
|
[]archiveRename{
|
||||||
|
{tgt.ID, renamedWebhookName, tgt.Name},
|
||||||
|
{tgt.ID, oldName, tgt.Name},
|
||||||
|
},
|
||||||
|
env.archives.Renames(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
// TestSourceEditForm_ForeverWebhookRoundTrips walks the exact path that
|
// TestSourceEditForm_ForeverWebhookRoundTrips walks the exact path that
|
||||||
// the removed max="365" cap used to break: render the edit form for a
|
// the removed max="365" cap used to break: render the edit form for a
|
||||||
// retain-forever webhook, confirm the pre-filled sentinel is not capped
|
// retain-forever webhook, confirm the pre-filled sentinel is not capped
|
||||||
@@ -512,7 +584,7 @@ func TestSourceEditForm_ForeverWebhookRoundTrips(t *testing.T) {
|
|||||||
)
|
)
|
||||||
|
|
||||||
req := getRequest(
|
req := getRequest(
|
||||||
t, "/source/"+wh.ID+"/edit", env.cookies,
|
t, "/hook/"+wh.ID+"/edit", env.cookies,
|
||||||
map[string]string{sourceIDParam: wh.ID},
|
map[string]string{sourceIDParam: wh.ID},
|
||||||
)
|
)
|
||||||
w := httptest.NewRecorder()
|
w := httptest.NewRecorder()
|
||||||
@@ -567,7 +639,7 @@ func TestSourceListAndDetail_ShowForeverNotTheSentinelNumber(
|
|||||||
|
|
||||||
listW := httptest.NewRecorder()
|
listW := httptest.NewRecorder()
|
||||||
env.handlers.HandleSourceList().ServeHTTP(
|
env.handlers.HandleSourceList().ServeHTTP(
|
||||||
listW, getRequest(t, "/sources", env.cookies, nil),
|
listW, getRequest(t, "/hooks", env.cookies, nil),
|
||||||
)
|
)
|
||||||
|
|
||||||
require.Equal(t, http.StatusOK, listW.Code)
|
require.Equal(t, http.StatusOK, listW.Code)
|
||||||
@@ -578,7 +650,7 @@ func TestSourceListAndDetail_ShowForeverNotTheSentinelNumber(
|
|||||||
env.handlers.HandleSourceDetail().ServeHTTP(
|
env.handlers.HandleSourceDetail().ServeHTTP(
|
||||||
detailW,
|
detailW,
|
||||||
getRequest(
|
getRequest(
|
||||||
t, "/source/"+wh.ID, env.cookies,
|
t, "/hook/"+wh.ID, env.cookies,
|
||||||
map[string]string{sourceIDParam: wh.ID},
|
map[string]string{sourceIDParam: wh.ID},
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -76,11 +76,11 @@ func postTargetCreate(
|
|||||||
router := chi.NewRouter()
|
router := chi.NewRouter()
|
||||||
router.Use(mw.Logging())
|
router.Use(mw.Logging())
|
||||||
router.Post(
|
router.Post(
|
||||||
"/source/{sourceID}/targets",
|
"/hook/{sourceID}/targets",
|
||||||
env.handlers.HandleTargetCreate(),
|
env.handlers.HandleTargetCreate(),
|
||||||
)
|
)
|
||||||
|
|
||||||
target := "/source/" + webhookID + "/targets"
|
target := "/hook/" + webhookID + "/targets"
|
||||||
if query != "" {
|
if query != "" {
|
||||||
target += "?" + query
|
target += "?" + query
|
||||||
}
|
}
|
||||||
@@ -114,7 +114,7 @@ func postTargetCreate(
|
|||||||
// regression test for the ingress leak. r.FormValue falls back to the
|
// regression test for the ingress leak. r.FormValue falls back to the
|
||||||
// query string when a field is absent from the POST body, so
|
// query string when a field is absent from the POST body, so
|
||||||
//
|
//
|
||||||
// POST /source/{id}/targets?url=https://hooks.slack.com/services/...
|
// POST /hook/{id}/targets?url=https://hooks.slack.com/services/...
|
||||||
//
|
//
|
||||||
// with an empty url field used to create a working target from a value
|
// with an empty url field used to create a working target from a value
|
||||||
// carried on the request line — where logs, proxies, Referer headers
|
// carried on the request line — where logs, proxies, Referer headers
|
||||||
|
|||||||
@@ -47,7 +47,7 @@ type targetEditView struct {
|
|||||||
//
|
//
|
||||||
// This page is the one place the full destination URL and header
|
// This page is the one place the full destination URL and header
|
||||||
// values are shown. It is reachable only through the
|
// values are shown. It is reachable only through the
|
||||||
// /source/{sourceID} route group, which supplies RequireAuth and
|
// /hook/{sourceID} route group, which supplies RequireAuth and
|
||||||
// NoCache, and only for a target of a webhook the session's user
|
// NoCache, and only for a target of a webhook the session's user
|
||||||
// owns; masking (delivery.TargetView) is unchanged everywhere else.
|
// owns; masking (delivery.TargetView) is unchanged everywhere else.
|
||||||
func (h *Handlers) HandleTargetEdit() http.HandlerFunc {
|
func (h *Handlers) HandleTargetEdit() http.HandlerFunc {
|
||||||
@@ -152,18 +152,54 @@ func (h *Handlers) applyTargetEdit(
|
|||||||
target.MaxRetries = retries
|
target.MaxRetries = retries
|
||||||
}
|
}
|
||||||
|
|
||||||
|
oldName := target.Name
|
||||||
target.Name = name
|
target.Name = name
|
||||||
target.Config = configJSON
|
target.Config = configJSON
|
||||||
|
|
||||||
err = h.db.DB().Save(target).Error
|
// The archive file is renamed before the new name is saved (see
|
||||||
|
// delivery.Engine.RenameArchive). If either step fails, it goes
|
||||||
|
// back to the name that is still stored.
|
||||||
|
err = h.renameTargetArchive(target, webhook.Name, name)
|
||||||
|
if err == nil {
|
||||||
|
err = h.db.DB().Save(target).Error
|
||||||
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
restoreErr := h.renameTargetArchive(
|
||||||
|
target, webhook.Name, oldName,
|
||||||
|
)
|
||||||
|
if restoreErr != nil {
|
||||||
|
h.log.Error(
|
||||||
|
"failed to rename archive back",
|
||||||
|
"target_id", target.ID,
|
||||||
|
"error", restoreErr,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
h.serverError(w, "failed to update target", err)
|
h.serverError(w, "failed to update target", err)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
http.Redirect(
|
http.Redirect(
|
||||||
w, r, "/source/"+webhook.ID, http.StatusSeeOther,
|
w, r, "/hook/"+webhook.ID, http.StatusSeeOther,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// renameTargetArchive renames a database target's archive file for
|
||||||
|
// the given webhook and target names. Other target types have no
|
||||||
|
// archive.
|
||||||
|
func (h *Handlers) renameTargetArchive(
|
||||||
|
target *database.Target,
|
||||||
|
webhookName, targetName string,
|
||||||
|
) error {
|
||||||
|
if h.archives == nil ||
|
||||||
|
target.Type != database.TargetTypeDatabase {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return h.archives.RenameArchive(
|
||||||
|
target.ID, webhookName, targetName,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -42,15 +42,15 @@ const (
|
|||||||
func targetRouter(env *sourceTestEnv) *chi.Mux {
|
func targetRouter(env *sourceTestEnv) *chi.Mux {
|
||||||
router := chi.NewRouter()
|
router := chi.NewRouter()
|
||||||
router.Post(
|
router.Post(
|
||||||
"/source/{sourceID}/targets",
|
"/hook/{sourceID}/targets",
|
||||||
env.handlers.HandleTargetCreate(),
|
env.handlers.HandleTargetCreate(),
|
||||||
)
|
)
|
||||||
router.Get(
|
router.Get(
|
||||||
"/source/{sourceID}/targets/{targetID}/edit",
|
"/hook/{sourceID}/targets/{targetID}/edit",
|
||||||
env.handlers.HandleTargetEdit(),
|
env.handlers.HandleTargetEdit(),
|
||||||
)
|
)
|
||||||
router.Post(
|
router.Post(
|
||||||
"/source/{sourceID}/targets/{targetID}/edit",
|
"/hook/{sourceID}/targets/{targetID}/edit",
|
||||||
env.handlers.HandleTargetEditSubmit(),
|
env.handlers.HandleTargetEditSubmit(),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -117,7 +117,7 @@ func seedHTTPTarget(
|
|||||||
|
|
||||||
w := serveTarget(
|
w := serveTarget(
|
||||||
env, http.MethodPost,
|
env, http.MethodPost,
|
||||||
"/source/"+webhook.ID+"/targets", form,
|
"/hook/"+webhook.ID+"/targets", form,
|
||||||
)
|
)
|
||||||
require.Equal(t, http.StatusSeeOther, w.Code, w.Body.String())
|
require.Equal(t, http.StatusSeeOther, w.Code, w.Body.String())
|
||||||
|
|
||||||
@@ -188,7 +188,7 @@ func submitTargetEdit(
|
|||||||
) *httptest.ResponseRecorder {
|
) *httptest.ResponseRecorder {
|
||||||
return serveTarget(
|
return serveTarget(
|
||||||
env, http.MethodPost,
|
env, http.MethodPost,
|
||||||
"/source/"+webhookID+"/targets/"+targetID+"/edit",
|
"/hook/"+webhookID+"/targets/"+targetID+"/edit",
|
||||||
form,
|
form,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -401,7 +401,7 @@ func TestHandleTargetEdit_PrefillsTheStoredValuesUnmasked(
|
|||||||
|
|
||||||
w := serveTarget(
|
w := serveTarget(
|
||||||
env, http.MethodGet,
|
env, http.MethodGet,
|
||||||
"/source/"+webhook.ID+"/targets/"+target.ID+"/edit",
|
"/hook/"+webhook.ID+"/targets/"+target.ID+"/edit",
|
||||||
nil,
|
nil,
|
||||||
)
|
)
|
||||||
require.Equal(t, http.StatusOK, w.Code)
|
require.Equal(t, http.StatusOK, w.Code)
|
||||||
@@ -508,7 +508,7 @@ func assertEditIgnoresQueryString(
|
|||||||
|
|
||||||
w := serveTarget(
|
w := serveTarget(
|
||||||
env, http.MethodPost,
|
env, http.MethodPost,
|
||||||
"/source/"+webhook.ID+"/targets/"+target.ID+
|
"/hook/"+webhook.ID+"/targets/"+target.ID+
|
||||||
"/edit?url="+url.QueryEscape(editReplacedURL)+
|
"/edit?url="+url.QueryEscape(editReplacedURL)+
|
||||||
"&headers="+url.QueryEscape(editAuthHeader),
|
"&headers="+url.QueryEscape(editAuthHeader),
|
||||||
form,
|
form,
|
||||||
@@ -592,7 +592,7 @@ func assertTargetOfAnotherWebhook404s(
|
|||||||
|
|
||||||
get := serveTarget(
|
get := serveTarget(
|
||||||
env, http.MethodGet,
|
env, http.MethodGet,
|
||||||
"/source/"+mine.ID+"/targets/"+target.ID+"/edit", nil,
|
"/hook/"+mine.ID+"/targets/"+target.ID+"/edit", nil,
|
||||||
)
|
)
|
||||||
assert.Equal(t, http.StatusNotFound, get.Code)
|
assert.Equal(t, http.StatusNotFound, get.Code)
|
||||||
|
|
||||||
@@ -630,8 +630,53 @@ func assertWebhookOfAnotherUser404s(
|
|||||||
|
|
||||||
w := serveTarget(
|
w := serveTarget(
|
||||||
env, http.MethodGet,
|
env, http.MethodGet,
|
||||||
"/source/"+other.ID+"/targets/"+target.ID+"/edit", nil,
|
"/hook/"+other.ID+"/targets/"+target.ID+"/edit", nil,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert.Equal(t, http.StatusNotFound, w.Code)
|
assert.Equal(t, http.StatusNotFound, w.Code)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestHandleTargetEditSubmit_RenamesArchive proves that renaming a
|
||||||
|
// database target renames its archive, that a target of another type
|
||||||
|
// has no archive to rename, and that a target whose archive cannot be
|
||||||
|
// renamed keeps its stored name.
|
||||||
|
func TestHandleTargetEditSubmit_RenamesArchive(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
env := setupSourceTest(t)
|
||||||
|
wh := seedWebhookWithRetention(t, env.db, 7)
|
||||||
|
archive := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
|
||||||
|
|
||||||
|
w := submitTargetEdit(
|
||||||
|
env, wh.ID, archive.ID, url.Values{"name": {"Long Term"}},
|
||||||
|
)
|
||||||
|
require.Equal(t, http.StatusSeeOther, w.Code, w.Body.String())
|
||||||
|
assert.Equal(
|
||||||
|
t,
|
||||||
|
[]archiveRename{{archive.ID, wh.Name, "Long Term"}},
|
||||||
|
env.archives.Renames(),
|
||||||
|
)
|
||||||
|
|
||||||
|
httpWebhook, httpTarget := seedHTTPTarget(t, env, "", "")
|
||||||
|
|
||||||
|
w = submitTargetEdit(
|
||||||
|
env, httpWebhook.ID, httpTarget.ID,
|
||||||
|
editForm(editOriginalURL, "", ""),
|
||||||
|
)
|
||||||
|
require.Equal(t, http.StatusSeeOther, w.Code, w.Body.String())
|
||||||
|
assert.Len(
|
||||||
|
t, env.archives.Renames(), 1,
|
||||||
|
"an HTTP target has no archive to rename",
|
||||||
|
)
|
||||||
|
|
||||||
|
env.archives.FailRenames(errInjectedRename)
|
||||||
|
|
||||||
|
w = submitTargetEdit(
|
||||||
|
env, wh.ID, archive.ID, url.Values{"name": {"Again"}},
|
||||||
|
)
|
||||||
|
require.Equal(t, http.StatusInternalServerError, w.Code)
|
||||||
|
assert.Equal(
|
||||||
|
t, "Long Term", storedTarget(t, env, archive.ID).Name,
|
||||||
|
"a target whose archive was not renamed keeps its name",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|||||||
@@ -102,7 +102,7 @@ func createWithRetries(
|
|||||||
|
|
||||||
w := serveTarget(
|
w := serveTarget(
|
||||||
env, http.MethodPost,
|
env, http.MethodPost,
|
||||||
"/source/"+webhook.ID+"/targets",
|
"/hook/"+webhook.ID+"/targets",
|
||||||
createRetriesForm(retries),
|
createRetriesForm(retries),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -54,8 +54,7 @@ func renderPage(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TestNavbarUsesWebhookTerminology pins the user-visible navigation
|
// TestNavbarUsesWebhookTerminology pins the user-visible navigation
|
||||||
// label to "Webhooks". The /sources route is deliberately unchanged, so
|
// label to "Webhooks" and its link to the webhook list at /hooks.
|
||||||
// the assertion targets the link text rather than the href.
|
|
||||||
func TestNavbarUsesWebhookTerminology(t *testing.T) {
|
func TestNavbarUsesWebhookTerminology(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -95,15 +94,11 @@ func TestNavbarUsesWebhookTerminology(t *testing.T) {
|
|||||||
t, body, ">Sources<",
|
t, body, ">Sources<",
|
||||||
"no user-visible element may still be labelled Sources",
|
"no user-visible element may still be labelled Sources",
|
||||||
)
|
)
|
||||||
assert.Contains(
|
assert.Contains(t, body, `href="/hooks"`)
|
||||||
t, body, `href="/sources"`,
|
|
||||||
"the /sources route itself must not change",
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestEditPageUsesWebhookTerminology pins the edit page's heading and
|
// TestEditPageUsesWebhookTerminology pins the edit page's heading and
|
||||||
// its back link. The link's href still points at /source/{id}, which is
|
// its back link to the webhook page at /hook/{id}.
|
||||||
// intentional: only user-visible copy changes.
|
|
||||||
func TestEditPageUsesWebhookTerminology(t *testing.T) {
|
func TestEditPageUsesWebhookTerminology(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -130,7 +125,57 @@ func TestEditPageUsesWebhookTerminology(t *testing.T) {
|
|||||||
|
|
||||||
assert.Contains(t, body, "Edit Webhook")
|
assert.Contains(t, body, "Edit Webhook")
|
||||||
assert.NotContains(t, body, ">Sources<")
|
assert.NotContains(t, body, ">Sources<")
|
||||||
assert.Contains(t, body, `href="/source/wh-1"`)
|
assert.Contains(t, body, `href="/hook/wh-1"`)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestEventLogPageIsCalledFullEventLog pins the one name the event log
|
||||||
|
// page at /hook/{id}/events goes by: both links to it on the webhook
|
||||||
|
// page, and its own heading, read "Full Event Log".
|
||||||
|
func TestEventLogPageIsCalledFullEventLog(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var h *handlers.Handlers
|
||||||
|
|
||||||
|
var sess *session.Session
|
||||||
|
|
||||||
|
app := newTestApp(t, &h, &sess)
|
||||||
|
app.RequireStart()
|
||||||
|
|
||||||
|
t.Cleanup(app.RequireStop)
|
||||||
|
|
||||||
|
// A pointer, as in the handlers: source_detail.html calls
|
||||||
|
// Webhook.RetentionLabel, a pointer method. Both pages only range
|
||||||
|
// over their lists, and a list left out renders as empty, so the
|
||||||
|
// lists are left out.
|
||||||
|
webhook := &database.Webhook{Name: "wh", RetentionDays: 14}
|
||||||
|
webhook.ID = testWebhookID
|
||||||
|
|
||||||
|
detailBody := renderPage(
|
||||||
|
t, h, sess, "source_detail.html", map[string]any{
|
||||||
|
dataKeyWebhook: webhook,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert.Contains(
|
||||||
|
t, detailBody,
|
||||||
|
`<a href="/hook/wh-1/events" class="btn-secondary">Full Event Log</a>`,
|
||||||
|
"the button at the top of the webhook page",
|
||||||
|
)
|
||||||
|
assert.Contains(
|
||||||
|
t, detailBody,
|
||||||
|
`<a href="/hook/wh-1/events" class="btn-text text-sm">Full Event Log</a>`,
|
||||||
|
"the link under recent events",
|
||||||
|
)
|
||||||
|
|
||||||
|
logBody := renderPage(t, h, sess, "source_logs.html", map[string]any{
|
||||||
|
dataKeyWebhook: webhook,
|
||||||
|
"TotalEvents": int64(0),
|
||||||
|
})
|
||||||
|
|
||||||
|
assert.Contains(
|
||||||
|
t, logBody,
|
||||||
|
`<h1 class="text-2xl font-medium text-gray-900">Full Event Log</h1>`,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestCreateFormRetentionCopyMatchesBehaviour pins the create form's
|
// TestCreateFormRetentionCopyMatchesBehaviour pins the create form's
|
||||||
@@ -283,7 +328,7 @@ func TestEntrypointCopyButtonIsProgressiveEnhancement(t *testing.T) {
|
|||||||
t, body,
|
t, body,
|
||||||
`<code id="entrypoint-url-ep-1"`,
|
`<code id="entrypoint-url-ep-1"`,
|
||||||
)
|
)
|
||||||
assert.Contains(t, body, "https://hooks.example.com/webhook/abc123")
|
assert.Contains(t, body, "https://hooks.example.com/h/abc123")
|
||||||
assert.Contains(
|
assert.Contains(
|
||||||
t, body,
|
t, body,
|
||||||
`hidden data-copy-target="entrypoint-url-ep-1"`,
|
`hidden data-copy-target="entrypoint-url-ep-1"`,
|
||||||
|
|||||||
@@ -131,7 +131,7 @@ func (h *Handlers) lookupEntrypoint(
|
|||||||
"path = ?", entrypointUUID,
|
"path = ?", entrypointUUID,
|
||||||
).First(&entrypoint)
|
).First(&entrypoint)
|
||||||
if result.Error != nil {
|
if result.Error != nil {
|
||||||
// The receiver is unauthenticated and /webhook/{uuid}
|
// The receiver is unauthenticated and /h/{uuid}
|
||||||
// matches any single segment, so this value is entirely
|
// matches any single segment, so this value is entirely
|
||||||
// client-chosen on exactly the branch where the lookup
|
// client-chosen on exactly the branch where the lookup
|
||||||
// failed. DEBUG is off by default; the cap is what keeps
|
// failed. DEBUG is off by default; the cap is what keeps
|
||||||
@@ -230,6 +230,7 @@ func (s eventSource) event() *database.Event {
|
|||||||
Method: s.Method,
|
Method: s.Method,
|
||||||
Headers: s.HeadersJSON,
|
Headers: s.HeadersJSON,
|
||||||
Body: string(s.Body),
|
Body: string(s.Body),
|
||||||
|
BodyBytes: int64(len(s.Body)),
|
||||||
ContentType: s.ContentType,
|
ContentType: s.ContentType,
|
||||||
ResubmittedFromID: s.ResubmittedFromID,
|
ResubmittedFromID: s.ResubmittedFromID,
|
||||||
}
|
}
|
||||||
@@ -252,11 +253,12 @@ func requestEventSource(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// createAndFanOut writes the event and one pending delivery per target
|
// createAndFanOut writes the event and one pending delivery per target,
|
||||||
// in a single transaction, then hands the tasks to the delivery
|
// and adds them to the webhook's running totals, in a single
|
||||||
// engine. It is the only path by which an event and its deliveries are
|
// transaction, then hands the tasks to the delivery engine. It is the
|
||||||
// created, so a resubmitted event is retried, SSRF-guarded and
|
// only path by which an event and its deliveries are created, so a
|
||||||
// circuit-broken exactly as a received one is.
|
// resubmitted event is retried, SSRF-guarded and circuit-broken
|
||||||
|
// exactly as a received one is.
|
||||||
//
|
//
|
||||||
// The tasks are returned as well as queued, so a caller can report how
|
// The tasks are returned as well as queued, so a caller can report how
|
||||||
// many targets the event went to.
|
// many targets the event went to.
|
||||||
@@ -296,6 +298,15 @@ func (h *Handlers) createAndFanOut(
|
|||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
err = database.AddEventTotals(tx, database.EventTotals{
|
||||||
|
Events: 1, LastEventAt: &event.CreatedAt,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
tx.Rollback()
|
||||||
|
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
|
||||||
err = tx.Commit().Error
|
err = tx.Commit().Error
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, fmt.Errorf(
|
return nil, nil, fmt.Errorf(
|
||||||
@@ -354,8 +365,9 @@ func (h *Handlers) finishWebhookResponse(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// buildDeliveryTasks creates one pending delivery per target in the
|
// buildDeliveryTasks creates one pending delivery per target in the
|
||||||
// transaction and returns the tasks for the delivery engine. The
|
// transaction, adds each to its target's totals, and returns the tasks
|
||||||
// caller owns the transaction and rolls it back on error.
|
// for the delivery engine. The caller owns the transaction and rolls
|
||||||
|
// it back on error.
|
||||||
func buildDeliveryTasks(
|
func buildDeliveryTasks(
|
||||||
tx *gorm.DB,
|
tx *gorm.DB,
|
||||||
event *database.Event,
|
event *database.Event,
|
||||||
@@ -379,6 +391,13 @@ func buildDeliveryTasks(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
err = database.AddTargetTotals(tx, database.TargetTotals{
|
||||||
|
TargetID: targets[i].ID, Deliveries: 1,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
tasks = append(tasks, delivery.Task{
|
tasks = append(tasks, delivery.Task{
|
||||||
DeliveryID: dlv.ID,
|
DeliveryID: dlv.ID,
|
||||||
EventID: event.ID,
|
EventID: event.ID,
|
||||||
|
|||||||
@@ -0,0 +1,259 @@
|
|||||||
|
package handlers
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gorm.io/gorm"
|
||||||
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The spans of the two recent windows the statistics pane reports on:
|
||||||
|
// the last 10 minutes and the last 24 hours.
|
||||||
|
const (
|
||||||
|
shortWindow = 10 * time.Minute
|
||||||
|
longWindow = 24 * time.Hour
|
||||||
|
)
|
||||||
|
|
||||||
|
// percent turns a fraction into a percentage.
|
||||||
|
const percent = 100
|
||||||
|
|
||||||
|
// WebhookStats holds the figures in the statistics pane at the top of
|
||||||
|
// the webhook page.
|
||||||
|
type WebhookStats struct {
|
||||||
|
Entrypoints int
|
||||||
|
ActiveEntrypoints int
|
||||||
|
Targets int
|
||||||
|
ActiveTargets int
|
||||||
|
|
||||||
|
// Lifetime counts every event, delivery and failure the webhook
|
||||||
|
// has had, and WithinRetention those still stored.
|
||||||
|
Lifetime Counts
|
||||||
|
WithinRetention Counts
|
||||||
|
|
||||||
|
// InProgress counts the deliveries still pending or retrying.
|
||||||
|
InProgress int64
|
||||||
|
|
||||||
|
// LastEventAt is when the newest event arrived, or nil when none
|
||||||
|
// has. Retention does not change it.
|
||||||
|
LastEventAt *time.Time
|
||||||
|
|
||||||
|
Last10Minutes RecentWindow
|
||||||
|
Last24Hours RecentWindow
|
||||||
|
}
|
||||||
|
|
||||||
|
// Counts holds a number of events, of deliveries and of failed
|
||||||
|
// deliveries.
|
||||||
|
type Counts struct {
|
||||||
|
Events int64
|
||||||
|
Deliveries int64
|
||||||
|
Failures int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecentWindow holds what happened in one recent window: the events
|
||||||
|
// received in it, and the deliveries that became delivered or failed in
|
||||||
|
// it.
|
||||||
|
type RecentWindow struct {
|
||||||
|
Events int64
|
||||||
|
Delivered int64
|
||||||
|
Failed int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// TargetFinished is how many of one target's deliveries became
|
||||||
|
// delivered, and how many failed, in a recent window.
|
||||||
|
type TargetFinished struct {
|
||||||
|
TargetID string
|
||||||
|
Delivered int64
|
||||||
|
Failed int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// FailurePercent is the share of the deliveries finished in the window
|
||||||
|
// that failed, or a dash when none finished. Deliveries still pending
|
||||||
|
// or retrying are not counted either way.
|
||||||
|
func (w RecentWindow) FailurePercent() string {
|
||||||
|
finished := w.Delivered + w.Failed
|
||||||
|
if finished == 0 {
|
||||||
|
return "—"
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Sprintf(
|
||||||
|
"%.1f%%", percent*float64(w.Failed)/float64(finished),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// loadWebhookStats gathers the figures for the statistics pane from the
|
||||||
|
// webhook's entrypoints and targets, as the page has already loaded
|
||||||
|
// them, and from its event database. It returns nil, and logs why, when
|
||||||
|
// the event database cannot be read.
|
||||||
|
func (h *Handlers) loadWebhookStats(
|
||||||
|
webhookID string,
|
||||||
|
entrypoints []database.Entrypoint,
|
||||||
|
targets []database.Target,
|
||||||
|
) *WebhookStats {
|
||||||
|
stats := &WebhookStats{
|
||||||
|
Entrypoints: len(entrypoints),
|
||||||
|
Targets: len(targets),
|
||||||
|
}
|
||||||
|
|
||||||
|
for i := range entrypoints {
|
||||||
|
if entrypoints[i].Active {
|
||||||
|
stats.ActiveEntrypoints++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for i := range targets {
|
||||||
|
if targets[i].Active {
|
||||||
|
stats.ActiveTargets++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Opening an event database that does not exist would create it,
|
||||||
|
// and it would hold nothing to count.
|
||||||
|
if !h.dbMgr.DBExists(webhookID) {
|
||||||
|
return stats
|
||||||
|
}
|
||||||
|
|
||||||
|
webhookDB, err := h.dbMgr.GetDB(webhookID)
|
||||||
|
if err == nil {
|
||||||
|
err = readEventStats(webhookDB, time.Now(), stats)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
h.log.Error(
|
||||||
|
"failed to read webhook statistics",
|
||||||
|
"webhook_id", webhookID,
|
||||||
|
"error", err,
|
||||||
|
)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return stats
|
||||||
|
}
|
||||||
|
|
||||||
|
// readEventStats fills in the figures that come from the webhook's
|
||||||
|
// event database. None of them reads every stored row: the totals are
|
||||||
|
// one row for the events and one per target for the deliveries, and
|
||||||
|
// every other figure is read from an index, over only the rows it
|
||||||
|
// counts.
|
||||||
|
func readEventStats(
|
||||||
|
db *gorm.DB, now time.Time, stats *WebhookStats,
|
||||||
|
) error {
|
||||||
|
err := readTotals(db, stats)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = db.Model(&database.Delivery{}).
|
||||||
|
Where("status IN ?", []database.DeliveryStatus{
|
||||||
|
database.DeliveryStatusPending,
|
||||||
|
database.DeliveryStatusRetrying,
|
||||||
|
}).
|
||||||
|
Count(&stats.InProgress).Error
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("counting deliveries in progress: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
stats.Last10Minutes, err = readRecentWindow(
|
||||||
|
db, now.Add(-shortWindow),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
stats.Last24Hours, err = readRecentWindow(
|
||||||
|
db, now.Add(-longWindow),
|
||||||
|
)
|
||||||
|
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// readTotals fills in the lifetime and within-retention figures, and
|
||||||
|
// when the last event arrived, from the running totals: the events'
|
||||||
|
// row, and the targets' rows summed.
|
||||||
|
func readTotals(db *gorm.DB, stats *WebhookStats) error {
|
||||||
|
var events database.EventTotals
|
||||||
|
|
||||||
|
err := db.Take(&events).Error
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("reading event totals: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var targets []database.TargetTotals
|
||||||
|
|
||||||
|
err = db.Find(&targets).Error
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("reading target totals: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
stats.Lifetime.Events = events.Events
|
||||||
|
stats.WithinRetention.Events = events.Events - events.EventsRemoved
|
||||||
|
stats.LastEventAt = events.LastEventAt
|
||||||
|
|
||||||
|
for _, t := range targets {
|
||||||
|
stats.Lifetime.Deliveries += t.Deliveries
|
||||||
|
stats.Lifetime.Failures += t.Failed
|
||||||
|
stats.WithinRetention.Deliveries += t.Deliveries - t.DeliveriesRemoved
|
||||||
|
stats.WithinRetention.Failures += t.Failed - t.FailedRemoved
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// readRecentWindow counts the events received, and the deliveries that
|
||||||
|
// became delivered or failed, since the given time.
|
||||||
|
func readRecentWindow(
|
||||||
|
db *gorm.DB, since time.Time,
|
||||||
|
) (RecentWindow, error) {
|
||||||
|
var w RecentWindow
|
||||||
|
|
||||||
|
err := db.Model(&database.Event{}).
|
||||||
|
Where("created_at >= ?", since).
|
||||||
|
Count(&w.Events).Error
|
||||||
|
if err != nil {
|
||||||
|
return w, fmt.Errorf("counting recent events: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
byTarget, err := finishedByTarget(db, since)
|
||||||
|
if err != nil {
|
||||||
|
return w, err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, f := range byTarget {
|
||||||
|
w.Delivered += f.Delivered
|
||||||
|
w.Failed += f.Failed
|
||||||
|
}
|
||||||
|
|
||||||
|
return w, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// finishedByTarget counts, for each target, the deliveries that became
|
||||||
|
// delivered and those that failed since the given time, in one query
|
||||||
|
// over just that window of the deliveries' status index. A target with
|
||||||
|
// neither is left out.
|
||||||
|
func finishedByTarget(
|
||||||
|
db *gorm.DB, since time.Time,
|
||||||
|
) ([]TargetFinished, error) {
|
||||||
|
var byTarget []TargetFinished
|
||||||
|
|
||||||
|
err := db.Model(&database.Delivery{}).
|
||||||
|
Select("target_id, "+
|
||||||
|
"count(CASE WHEN status = ? THEN 1 END) AS delivered, "+
|
||||||
|
"count(CASE WHEN status = ? THEN 1 END) AS failed",
|
||||||
|
database.DeliveryStatusDelivered,
|
||||||
|
database.DeliveryStatusFailed).
|
||||||
|
Where("status IN ? AND finished_at >= ?",
|
||||||
|
[]database.DeliveryStatus{
|
||||||
|
database.DeliveryStatusDelivered,
|
||||||
|
database.DeliveryStatusFailed,
|
||||||
|
}, since).
|
||||||
|
Group("target_id").
|
||||||
|
Find(&byTarget).Error
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf(
|
||||||
|
"counting deliveries finished by target: %w", err,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return byTarget, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,569 @@
|
|||||||
|
package handlers_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/google/uuid"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"go.uber.org/fx/fxtest"
|
||||||
|
"gorm.io/gorm"
|
||||||
|
"gorm.io/gorm/clause"
|
||||||
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
|
"sneak.berlin/go/webhooker/internal/handlers"
|
||||||
|
"sneak.berlin/go/webhooker/internal/logger"
|
||||||
|
"sneak.berlin/go/webhooker/internal/session"
|
||||||
|
)
|
||||||
|
|
||||||
|
// statsEntrypoint adds an entrypoint to a webhook and returns its path.
|
||||||
|
func statsEntrypoint(
|
||||||
|
t *testing.T, db *database.Database, webhookID string, active bool,
|
||||||
|
) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
ep := &database.Entrypoint{
|
||||||
|
WebhookID: webhookID,
|
||||||
|
Path: uuid.New().String(),
|
||||||
|
}
|
||||||
|
|
||||||
|
require.NoError(t, db.DB().Omit(clause.Associations).Create(ep).Error)
|
||||||
|
require.NoError(t, db.DB().Model(ep).Update("active", active).Error)
|
||||||
|
|
||||||
|
return ep.Path
|
||||||
|
}
|
||||||
|
|
||||||
|
// statsDelivery returns an event's delivery to a target.
|
||||||
|
func statsDelivery(
|
||||||
|
t *testing.T, webhookDB *gorm.DB, eventID, targetID string,
|
||||||
|
) database.Delivery {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
var d database.Delivery
|
||||||
|
|
||||||
|
require.NoError(t, webhookDB.Where(
|
||||||
|
"event_id = ? AND target_id = ?", eventID, targetID,
|
||||||
|
).First(&d).Error)
|
||||||
|
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
|
||||||
|
// statsFinish settles a delivery as the delivery engine does: its
|
||||||
|
// final status and the time it finished, and one more on its target's
|
||||||
|
// delivered or failed total, in one transaction.
|
||||||
|
func statsFinish(
|
||||||
|
t *testing.T,
|
||||||
|
webhookDB *gorm.DB,
|
||||||
|
d database.Delivery,
|
||||||
|
status database.DeliveryStatus,
|
||||||
|
at time.Time,
|
||||||
|
) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
add := database.TargetTotals{TargetID: d.TargetID, Delivered: 1}
|
||||||
|
if status == database.DeliveryStatusFailed {
|
||||||
|
add = database.TargetTotals{TargetID: d.TargetID, Failed: 1}
|
||||||
|
}
|
||||||
|
|
||||||
|
require.NoError(t, webhookDB.Transaction(func(tx *gorm.DB) error {
|
||||||
|
err := tx.Model(&database.Delivery{}).
|
||||||
|
Where("id = ?", d.ID).
|
||||||
|
Updates(map[string]any{"status": status, "finished_at": at}).
|
||||||
|
Error
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return database.AddTargetTotals(tx, add)
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
// statsAge moves an event's arrival back to the given time.
|
||||||
|
func statsAge(
|
||||||
|
t *testing.T, webhookDB *gorm.DB, eventID string, at time.Time,
|
||||||
|
) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
require.NoError(t, webhookDB.Model(&database.Event{}).
|
||||||
|
Where("id = ?", eventID).
|
||||||
|
Update("created_at", at).Error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// statsTargetTotals reads a webhook database's target totals, keyed by
|
||||||
|
// target.
|
||||||
|
func statsTargetTotals(
|
||||||
|
t *testing.T, webhookDB *gorm.DB,
|
||||||
|
) map[string]database.TargetTotals {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
var rows []database.TargetTotals
|
||||||
|
|
||||||
|
require.NoError(t, webhookDB.Find(&rows).Error)
|
||||||
|
|
||||||
|
byTarget := make(map[string]database.TargetTotals, len(rows))
|
||||||
|
for _, row := range rows {
|
||||||
|
byTarget[row.TargetID] = row
|
||||||
|
}
|
||||||
|
|
||||||
|
return byTarget
|
||||||
|
}
|
||||||
|
|
||||||
|
// statsHistory is the webhook seedStatsHistory builds: its event
|
||||||
|
// database, its newest event, and its two active targets.
|
||||||
|
type statsHistory struct {
|
||||||
|
webhook *database.Webhook
|
||||||
|
webhookDB *gorm.DB
|
||||||
|
newest database.Event
|
||||||
|
first, second string
|
||||||
|
}
|
||||||
|
|
||||||
|
// seedStatsHistory builds the webhook the statistics test checks: 14
|
||||||
|
// days of retention, twelve entrypoints (one inactive) and six targets
|
||||||
|
// (four inactive). Ten events arrive through the receiver, and so each
|
||||||
|
// has a delivery to the two active targets. The oldest event is past
|
||||||
|
// retention, the next 30 hours old, the next six hours old, the other
|
||||||
|
// seven just in. Six deliveries are settled as the delivery engine
|
||||||
|
// would, two of them inside a recent window though their event arrived
|
||||||
|
// before it. The newest event's delivery to the second target is
|
||||||
|
// retrying, the rest are left pending, and a replay adds a pending
|
||||||
|
// delivery to the oldest event. Once retention has removed the oldest
|
||||||
|
// event, every figure in the pane differs from every other.
|
||||||
|
func seedStatsHistory(
|
||||||
|
t *testing.T,
|
||||||
|
h *handlers.Handlers,
|
||||||
|
sess *session.Session,
|
||||||
|
db *database.Database,
|
||||||
|
dbMgr *database.WebhookDBManager,
|
||||||
|
) statsHistory {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
wh := &database.Webhook{UserID: deleteTestUserID, Name: "stats", RetentionDays: 14}
|
||||||
|
require.NoError(t, db.DB().Omit(clause.Associations).Create(wh).Error)
|
||||||
|
|
||||||
|
path := statsEntrypoint(t, db, wh.ID, true)
|
||||||
|
for range 10 {
|
||||||
|
statsEntrypoint(t, db, wh.ID, true)
|
||||||
|
}
|
||||||
|
|
||||||
|
statsEntrypoint(t, db, wh.ID, false)
|
||||||
|
|
||||||
|
first := seedConfiguredTarget(
|
||||||
|
t, db, wh.ID, database.TargetTypeHTTP,
|
||||||
|
`{"url":"`+replayTargetURL+`"}`,
|
||||||
|
)
|
||||||
|
second := seedTarget(t, db, wh.ID, database.TargetTypeLog)
|
||||||
|
|
||||||
|
for range 4 {
|
||||||
|
inactive := seedTarget(t, db, wh.ID, database.TargetTypeLog)
|
||||||
|
require.NoError(t, db.DB().Model(inactive).
|
||||||
|
Update("active", false).Error)
|
||||||
|
}
|
||||||
|
|
||||||
|
router := receiverRouter(h)
|
||||||
|
|
||||||
|
for range 10 {
|
||||||
|
require.Equal(t, http.StatusOK, postReceiver(t, router, path))
|
||||||
|
}
|
||||||
|
|
||||||
|
webhookDB, err := dbMgr.GetDB(wh.ID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
events := listEvents(t, webhookDB)
|
||||||
|
require.Len(t, events, 10)
|
||||||
|
|
||||||
|
oldest, yesterday, middle, newest := events[0], events[1], events[2], events[9]
|
||||||
|
now := time.Now()
|
||||||
|
|
||||||
|
statsAge(t, webhookDB, oldest.ID, now.Add(-15*24*time.Hour))
|
||||||
|
statsAge(t, webhookDB, yesterday.ID, now.Add(-30*time.Hour))
|
||||||
|
statsAge(t, webhookDB, middle.ID, now.Add(-6*time.Hour))
|
||||||
|
|
||||||
|
oldestFailure := statsDelivery(t, webhookDB, oldest.ID, first.ID)
|
||||||
|
statsFinish(t, webhookDB, oldestFailure,
|
||||||
|
database.DeliveryStatusFailed, now.Add(-14*24*time.Hour))
|
||||||
|
statsFinish(t, webhookDB,
|
||||||
|
statsDelivery(t, webhookDB, yesterday.ID, first.ID),
|
||||||
|
database.DeliveryStatusFailed, now.Add(-29*time.Hour))
|
||||||
|
statsFinish(t, webhookDB,
|
||||||
|
statsDelivery(t, webhookDB, yesterday.ID, second.ID),
|
||||||
|
database.DeliveryStatusFailed, now.Add(-23*time.Hour))
|
||||||
|
statsFinish(t, webhookDB,
|
||||||
|
statsDelivery(t, webhookDB, middle.ID, second.ID),
|
||||||
|
database.DeliveryStatusFailed, now.Add(-5*time.Hour))
|
||||||
|
statsFinish(t, webhookDB,
|
||||||
|
statsDelivery(t, webhookDB, middle.ID, first.ID),
|
||||||
|
database.DeliveryStatusFailed, now.Add(-time.Minute))
|
||||||
|
statsFinish(t, webhookDB,
|
||||||
|
statsDelivery(t, webhookDB, newest.ID, first.ID),
|
||||||
|
database.DeliveryStatusDelivered, now.Add(-2*time.Minute))
|
||||||
|
|
||||||
|
retrying := statsDelivery(t, webhookDB, newest.ID, second.ID)
|
||||||
|
require.NoError(t, webhookDB.Model(&retrying).
|
||||||
|
Update("status", database.DeliveryStatusRetrying).Error)
|
||||||
|
|
||||||
|
require.Equal(t, http.StatusSeeOther,
|
||||||
|
postReplay(t, h, sess, wh.ID, oldestFailure.ID).Code)
|
||||||
|
|
||||||
|
return statsHistory{
|
||||||
|
webhook: wh, webhookDB: webhookDB, newest: newest,
|
||||||
|
first: first.ID, second: second.ID,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// statsPrune runs the real retention reaper until it has removed one
|
||||||
|
// event from the webhook's database, then stops it.
|
||||||
|
func statsPrune(
|
||||||
|
t *testing.T,
|
||||||
|
db *database.Database,
|
||||||
|
dbMgr *database.WebhookDBManager,
|
||||||
|
log *logger.Logger,
|
||||||
|
webhookDB *gorm.DB,
|
||||||
|
) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
lc := fxtest.NewLifecycle(t)
|
||||||
|
database.NewRetentionReaper(lc, database.RetentionReaperParams{
|
||||||
|
Config: &config.Config{
|
||||||
|
RetentionSweepInterval: 10 * time.Millisecond,
|
||||||
|
},
|
||||||
|
Database: db,
|
||||||
|
DBManager: dbMgr,
|
||||||
|
Logger: log,
|
||||||
|
})
|
||||||
|
|
||||||
|
lc.RequireStart()
|
||||||
|
|
||||||
|
require.Eventually(t, func() bool {
|
||||||
|
var totals database.EventTotals
|
||||||
|
|
||||||
|
err := webhookDB.Take(&totals).Error
|
||||||
|
|
||||||
|
return err == nil && totals.EventsRemoved == 1
|
||||||
|
}, 10*time.Second, 10*time.Millisecond)
|
||||||
|
|
||||||
|
lc.RequireStop()
|
||||||
|
}
|
||||||
|
|
||||||
|
// statsPane returns the text of the statistics pane in a rendered
|
||||||
|
// webhook page, everything from its heading to the next heading on the
|
||||||
|
// page, with the markup taken out and each run of space made one
|
||||||
|
// space. A table then reads header by header and row by row, each
|
||||||
|
// row's label followed by its figures in column order.
|
||||||
|
func statsPane(t *testing.T, page string) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
_, pane, found := strings.Cut(page, ">Statistics</h2>")
|
||||||
|
require.True(t, found, "the page has no statistics pane")
|
||||||
|
|
||||||
|
pane, _, _ = strings.Cut(pane, "<h2")
|
||||||
|
pane = regexp.MustCompile(`<[^>]*>`).ReplaceAllString(pane, " ")
|
||||||
|
|
||||||
|
return strings.Join(strings.Fields(pane), " ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// assertStatsTargets checks, for the history seedStatsHistory builds,
|
||||||
|
// each target's totals and its deliveries finished in the last 24
|
||||||
|
// hours. The first target has ten deliveries and the replay, the
|
||||||
|
// second ten; the inactive targets have none and so no row.
|
||||||
|
func assertStatsTargets(t *testing.T, hist statsHistory) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
first, second := hist.first, hist.second
|
||||||
|
|
||||||
|
assert.Equal(t, map[string]database.TargetTotals{
|
||||||
|
first: {TargetID: first, Deliveries: 11, Delivered: 1, Failed: 3},
|
||||||
|
second: {TargetID: second, Deliveries: 10, Failed: 2},
|
||||||
|
}, statsTargetTotals(t, hist.webhookDB))
|
||||||
|
|
||||||
|
lastDay, err := handlers.FinishedByTargetForTest(
|
||||||
|
hist.webhookDB, time.Now().Add(-24*time.Hour),
|
||||||
|
)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.ElementsMatch(t, []handlers.TargetFinished{
|
||||||
|
{TargetID: first, Delivered: 1, Failed: 1},
|
||||||
|
{TargetID: second, Failed: 2},
|
||||||
|
}, lastDay)
|
||||||
|
}
|
||||||
|
|
||||||
|
// assertStatsPaneAfterPrune checks the rendered statistics pane for the
|
||||||
|
// history seedStatsHistory builds, once retention has removed the
|
||||||
|
// oldest event: each figure after its label, in its column.
|
||||||
|
func assertStatsPaneAfterPrune(
|
||||||
|
t *testing.T,
|
||||||
|
h *handlers.Handlers,
|
||||||
|
sess *session.Session,
|
||||||
|
hist statsHistory,
|
||||||
|
) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
pane := statsPane(t, renderSourceDetailPage(t, h, sess, hist.webhook.ID))
|
||||||
|
lastEvent := hist.newest.CreatedAt.UTC().Format("2006-01-02 15:04:05 UTC")
|
||||||
|
|
||||||
|
assert.Contains(t, pane, "Entrypoints 12 (11 active) "+
|
||||||
|
"Targets 6 (2 active) "+
|
||||||
|
"Deliveries in progress 13 "+
|
||||||
|
"Last event "+lastEvent+" "+
|
||||||
|
"Retention 14 days")
|
||||||
|
assert.Contains(t, pane, "Lifetime Within retention "+
|
||||||
|
"Events 10 9 "+
|
||||||
|
"Deliveries 21 18 "+
|
||||||
|
"Failures 5 4")
|
||||||
|
assert.Contains(t, pane, "Last 10 minutes Last 24 hours "+
|
||||||
|
"Events 7 8 "+
|
||||||
|
"Failures 1 3 "+
|
||||||
|
"Failure percentage 50.0% 75.0%")
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestWebhookStats_EveryFigureAcrossRetentionPrune checks every figure
|
||||||
|
// the statistics pane shows for the history seedStatsHistory builds,
|
||||||
|
// and each target's totals and recent figures, before and after the
|
||||||
|
// real retention reaper removes the oldest event.
|
||||||
|
func TestWebhookStats_EveryFigureAcrossRetentionPrune(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var (
|
||||||
|
h *handlers.Handlers
|
||||||
|
sess *session.Session
|
||||||
|
db *database.Database
|
||||||
|
dbMgr *database.WebhookDBManager
|
||||||
|
log *logger.Logger
|
||||||
|
)
|
||||||
|
|
||||||
|
app := newTestApp(t, &h, &sess, &db, &dbMgr, &log)
|
||||||
|
app.RequireStart()
|
||||||
|
|
||||||
|
t.Cleanup(app.RequireStop)
|
||||||
|
|
||||||
|
hist := seedStatsHistory(t, h, sess, db, dbMgr)
|
||||||
|
first, second := hist.first, hist.second
|
||||||
|
|
||||||
|
stats := h.WebhookStatsForTest(hist.webhook.ID)
|
||||||
|
require.NotNil(t, stats)
|
||||||
|
|
||||||
|
assert.Equal(t, 12, stats.Entrypoints)
|
||||||
|
assert.Equal(t, 11, stats.ActiveEntrypoints)
|
||||||
|
assert.Equal(t, 6, stats.Targets)
|
||||||
|
assert.Equal(t, 2, stats.ActiveTargets)
|
||||||
|
assert.Equal(t, handlers.Counts{Events: 10, Deliveries: 21, Failures: 5},
|
||||||
|
stats.Lifetime)
|
||||||
|
assert.Equal(t, stats.Lifetime, stats.WithinRetention)
|
||||||
|
assert.Equal(t, int64(15), stats.InProgress)
|
||||||
|
require.NotNil(t, stats.LastEventAt)
|
||||||
|
assert.True(t, hist.newest.CreatedAt.Equal(*stats.LastEventAt))
|
||||||
|
assert.Equal(t, handlers.RecentWindow{
|
||||||
|
Events: 7, Delivered: 1, Failed: 1,
|
||||||
|
}, stats.Last10Minutes)
|
||||||
|
assert.Equal(t, handlers.RecentWindow{
|
||||||
|
Events: 8, Delivered: 1, Failed: 3,
|
||||||
|
}, stats.Last24Hours)
|
||||||
|
assert.Equal(t, "50.0%", stats.Last10Minutes.FailurePercent())
|
||||||
|
assert.Equal(t, "75.0%", stats.Last24Hours.FailurePercent())
|
||||||
|
|
||||||
|
assertStatsTargets(t, hist)
|
||||||
|
|
||||||
|
// Retention removes the oldest event with its three deliveries:
|
||||||
|
// the first target's failed one and the pending replay, and the
|
||||||
|
// second target's pending one.
|
||||||
|
statsPrune(t, db, dbMgr, log, hist.webhookDB)
|
||||||
|
|
||||||
|
after := h.WebhookStatsForTest(hist.webhook.ID)
|
||||||
|
require.NotNil(t, after)
|
||||||
|
|
||||||
|
assert.Equal(t, stats.Lifetime, after.Lifetime)
|
||||||
|
assert.Equal(t, handlers.Counts{Events: 9, Deliveries: 18, Failures: 4},
|
||||||
|
after.WithinRetention)
|
||||||
|
assert.Equal(t, int64(13), after.InProgress)
|
||||||
|
assert.Equal(t, stats.LastEventAt, after.LastEventAt)
|
||||||
|
assert.Equal(t, stats.Last10Minutes, after.Last10Minutes)
|
||||||
|
assert.Equal(t, stats.Last24Hours, after.Last24Hours)
|
||||||
|
|
||||||
|
assert.Equal(t, map[string]database.TargetTotals{
|
||||||
|
first: {
|
||||||
|
TargetID: first, Deliveries: 11, Delivered: 1, Failed: 3,
|
||||||
|
DeliveriesRemoved: 2, FailedRemoved: 1,
|
||||||
|
},
|
||||||
|
second: {
|
||||||
|
TargetID: second, Deliveries: 10, Failed: 2,
|
||||||
|
DeliveriesRemoved: 1,
|
||||||
|
},
|
||||||
|
}, statsTargetTotals(t, hist.webhookDB))
|
||||||
|
|
||||||
|
assertStatsPaneAfterPrune(t, h, sess, hist)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestWebhookStats_LastEventSurvivesPruningEveryEvent checks that once
|
||||||
|
// retention has removed every event, the pane still shows when the last
|
||||||
|
// one arrived rather than "none".
|
||||||
|
func TestWebhookStats_LastEventSurvivesPruningEveryEvent(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var (
|
||||||
|
h *handlers.Handlers
|
||||||
|
sess *session.Session
|
||||||
|
db *database.Database
|
||||||
|
dbMgr *database.WebhookDBManager
|
||||||
|
log *logger.Logger
|
||||||
|
)
|
||||||
|
|
||||||
|
app := newTestApp(t, &h, &sess, &db, &dbMgr, &log)
|
||||||
|
app.RequireStart()
|
||||||
|
|
||||||
|
t.Cleanup(app.RequireStop)
|
||||||
|
|
||||||
|
wh := &database.Webhook{
|
||||||
|
UserID: deleteTestUserID, Name: "pruned", RetentionDays: 1,
|
||||||
|
}
|
||||||
|
require.NoError(t, db.DB().Omit(clause.Associations).Create(wh).Error)
|
||||||
|
|
||||||
|
path := statsEntrypoint(t, db, wh.ID, true)
|
||||||
|
require.Equal(t, http.StatusOK,
|
||||||
|
postReceiver(t, receiverRouter(h), path))
|
||||||
|
|
||||||
|
webhookDB, err := dbMgr.GetDB(wh.ID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
events := listEvents(t, webhookDB)
|
||||||
|
require.Len(t, events, 1)
|
||||||
|
|
||||||
|
arrived := events[0].CreatedAt
|
||||||
|
|
||||||
|
statsAge(t, webhookDB, events[0].ID, time.Now().Add(-50*time.Hour))
|
||||||
|
statsPrune(t, db, dbMgr, log, webhookDB)
|
||||||
|
require.Empty(t, listEvents(t, webhookDB))
|
||||||
|
|
||||||
|
stats := h.WebhookStatsForTest(wh.ID)
|
||||||
|
require.NotNil(t, stats)
|
||||||
|
require.NotNil(t, stats.LastEventAt)
|
||||||
|
assert.True(t, arrived.Equal(*stats.LastEventAt))
|
||||||
|
|
||||||
|
pane := statsPane(t, renderSourceDetailPage(t, h, sess, wh.ID))
|
||||||
|
assert.Contains(t, pane,
|
||||||
|
"Last event "+arrived.UTC().Format("2006-01-02 15:04:05 UTC"))
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestWebhookStats_LastEventInUTC checks that the pane shows when the
|
||||||
|
// last event arrived in UTC, as the event list does, when the time was
|
||||||
|
// stored in another zone, as it is on a host whose local time is not
|
||||||
|
// UTC.
|
||||||
|
func TestWebhookStats_LastEventInUTC(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var (
|
||||||
|
h *handlers.Handlers
|
||||||
|
sess *session.Session
|
||||||
|
db *database.Database
|
||||||
|
dbMgr *database.WebhookDBManager
|
||||||
|
)
|
||||||
|
|
||||||
|
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
||||||
|
app.RequireStart()
|
||||||
|
|
||||||
|
t.Cleanup(app.RequireStop)
|
||||||
|
|
||||||
|
wh := seedWebhook(t, db)
|
||||||
|
|
||||||
|
webhookDB, err := dbMgr.GetDB(wh.ID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
arrived := time.Date(2026, time.March, 4, 22, 30, 0, 0,
|
||||||
|
time.FixedZone("EST", -5*60*60))
|
||||||
|
require.NoError(t, database.AddEventTotals(webhookDB,
|
||||||
|
database.EventTotals{Events: 1, LastEventAt: &arrived}))
|
||||||
|
|
||||||
|
pane := statsPane(t, renderSourceDetailPage(t, h, sess, wh.ID))
|
||||||
|
assert.Contains(t, pane, "Last event 2026-03-05 03:30:00 UTC")
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestWebhookStats_PaneShowsRetentionPeriod checks that the statistics
|
||||||
|
// pane itself, not only the line at the foot of the page, shows the
|
||||||
|
// webhook's retention period, for a finite one and for forever.
|
||||||
|
func TestWebhookStats_PaneShowsRetentionPeriod(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var (
|
||||||
|
h *handlers.Handlers
|
||||||
|
sess *session.Session
|
||||||
|
db *database.Database
|
||||||
|
)
|
||||||
|
|
||||||
|
app := newTestApp(t, &h, &sess, &db)
|
||||||
|
app.RequireStart()
|
||||||
|
|
||||||
|
t.Cleanup(app.RequireStop)
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
retentionDays int
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{30, "30 days"},
|
||||||
|
{database.RetentionForeverDays, "forever"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
wh := &database.Webhook{
|
||||||
|
UserID: deleteTestUserID,
|
||||||
|
Name: "retention",
|
||||||
|
RetentionDays: tt.retentionDays,
|
||||||
|
}
|
||||||
|
require.NoError(t,
|
||||||
|
db.DB().Omit(clause.Associations).Create(wh).Error)
|
||||||
|
|
||||||
|
pane := statsPane(t, renderSourceDetailPage(t, h, sess, wh.ID))
|
||||||
|
assert.Contains(t, pane, "Retention "+tt.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestWebhookStats_WebhookWithNoEvents covers a webhook whose event
|
||||||
|
// database has never been opened: every count is zero, the
|
||||||
|
// percentages are a dash, and showing the page does not create the
|
||||||
|
// database.
|
||||||
|
func TestWebhookStats_WebhookWithNoEvents(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var (
|
||||||
|
h *handlers.Handlers
|
||||||
|
sess *session.Session
|
||||||
|
db *database.Database
|
||||||
|
dbMgr *database.WebhookDBManager
|
||||||
|
)
|
||||||
|
|
||||||
|
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
||||||
|
app.RequireStart()
|
||||||
|
|
||||||
|
t.Cleanup(app.RequireStop)
|
||||||
|
|
||||||
|
wh := seedWebhook(t, db)
|
||||||
|
|
||||||
|
assert.Equal(t, &handlers.WebhookStats{}, h.WebhookStatsForTest(wh.ID))
|
||||||
|
assert.Equal(t, "—", handlers.RecentWindow{}.FailurePercent())
|
||||||
|
|
||||||
|
pane := statsPane(t, renderSourceDetailPage(t, h, sess, wh.ID))
|
||||||
|
assert.Contains(t, pane, "Last event none")
|
||||||
|
assert.Contains(t, pane, "Failure percentage — —")
|
||||||
|
assert.False(t, dbMgr.DBExists(wh.ID))
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRecentWindow_FailurePercent pins the percentage: failed
|
||||||
|
// deliveries out of all that finished in the window.
|
||||||
|
func TestRecentWindow_FailurePercent(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
window handlers.RecentWindow
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{handlers.RecentWindow{}, "—"},
|
||||||
|
{handlers.RecentWindow{Events: 4}, "—"},
|
||||||
|
{handlers.RecentWindow{Delivered: 3, Failed: 1}, "25.0%"},
|
||||||
|
{handlers.RecentWindow{Failed: 2}, "100.0%"},
|
||||||
|
{handlers.RecentWindow{Delivered: 2}, "0.0%"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
assert.Equal(t, tt.want, tt.window.FailurePercent(), tt.window)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -201,7 +201,7 @@ func TestTruncate_LeavesShortValuesAlone(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
for _, s := range []string{
|
for _, s := range []string{
|
||||||
"", "GET", "/source/abc/edit", "Mozilla/5.0 (X11)",
|
"", "GET", "/hook/abc/edit", "Mozilla/5.0 (X11)",
|
||||||
} {
|
} {
|
||||||
assert.Equal(t, s, logfield.Truncate(s, budget))
|
assert.Equal(t, s, logfield.Truncate(s, budget))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -119,7 +119,7 @@ func accessLogRouter(m *middleware.Middleware) *chi.Mux {
|
|||||||
)
|
)
|
||||||
|
|
||||||
router.HandleFunc(
|
router.HandleFunc(
|
||||||
"/webhook/{uuid}",
|
"/h/{uuid}",
|
||||||
func(w http.ResponseWriter, r *http.Request) {
|
func(w http.ResponseWriter, r *http.Request) {
|
||||||
// Stands in for the real handler: an unknown entrypoint
|
// Stands in for the real handler: an unknown entrypoint
|
||||||
// UUID 404s, a known one succeeds.
|
// UUID 404s, a known one succeeds.
|
||||||
@@ -271,11 +271,11 @@ func TestAccessLog_InventedReceiverPathsLogRoutePattern(t *testing.T) {
|
|||||||
assertFloodIsBounded(
|
assertFloodIsBounded(
|
||||||
t,
|
t,
|
||||||
func(i int) string {
|
func(i int) string {
|
||||||
return "/webhook/" + attackerMarker +
|
return "/h/" + attackerMarker +
|
||||||
strings.Repeat("x", i) + "?q=" + attackerMarker
|
strings.Repeat("x", i) + "?q=" + attackerMarker
|
||||||
},
|
},
|
||||||
http.StatusNotFound,
|
http.StatusNotFound,
|
||||||
"/webhook/{uuid}",
|
"/h/{uuid}",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -346,10 +346,10 @@ type sizeCase struct {
|
|||||||
func lineSizeCases() map[string]sizeCase {
|
func lineSizeCases() map[string]sizeCase {
|
||||||
cases := map[string]sizeCase{
|
cases := map[string]sizeCase{
|
||||||
"oversized path segment": {
|
"oversized path segment": {
|
||||||
target: "/webhook/" + attackerMarker +
|
target: "/h/" + attackerMarker +
|
||||||
strings.Repeat("x", oversizedSegmentBytes),
|
strings.Repeat("x", oversizedSegmentBytes),
|
||||||
wantStatus: http.StatusNotFound,
|
wantStatus: http.StatusNotFound,
|
||||||
wantURL: "/webhook/{uuid}",
|
wantURL: "/h/{uuid}",
|
||||||
bound: maxLineBytes,
|
bound: maxLineBytes,
|
||||||
},
|
},
|
||||||
// /.well-known/healthcheck answers 200 to anyone and has no
|
// /.well-known/healthcheck answers 200 to anyone and has no
|
||||||
@@ -605,14 +605,14 @@ func TestAccessLog_SuccessKeepsConcretePathAndRedactsQuery(
|
|||||||
router := accessLogRouter(m)
|
router := accessLogRouter(m)
|
||||||
|
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t, http.StatusOK, get(t, router, "/webhook/known?src=ci"),
|
t, http.StatusOK, get(t, router, "/h/known?src=ci"),
|
||||||
)
|
)
|
||||||
|
|
||||||
// The path resolved against a stored entrypoint, so it stays. The
|
// The path resolved against a stored entrypoint, so it stays. The
|
||||||
// query never does: see TestAccessLog_UnauthenticatedSuccess...
|
// query never does: see TestAccessLog_UnauthenticatedSuccess...
|
||||||
entries := accessLogEntries(t, buf)
|
entries := accessLogEntries(t, buf)
|
||||||
require.Len(t, entries, 1)
|
require.Len(t, entries, 1)
|
||||||
assert.Equal(t, "/webhook/known?(redacted)", entries[0]["url"])
|
assert.Equal(t, "/h/known?(redacted)", entries[0]["url"])
|
||||||
assert.NotContains(t, buf.String(), "src=ci")
|
assert.NotContains(t, buf.String(), "src=ci")
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -640,7 +640,7 @@ func TestAccessLog_RetainsEveryOtherField(t *testing.T) {
|
|||||||
assert.Equal(
|
assert.Equal(
|
||||||
t,
|
t,
|
||||||
http.StatusNotFound,
|
http.StatusNotFound,
|
||||||
get(t, router, "/webhook/"+attackerMarker),
|
get(t, router, "/h/"+attackerMarker),
|
||||||
)
|
)
|
||||||
|
|
||||||
entries := accessLogEntries(t, buf)
|
entries := accessLogEntries(t, buf)
|
||||||
|
|||||||
@@ -41,7 +41,7 @@ func (m *Middleware) CSRF() func(http.Handler) http.Handler {
|
|||||||
// CSRF is registered ahead of RequireAuth on every route
|
// CSRF is registered ahead of RequireAuth on every route
|
||||||
// group that uses it, so this WARN is reachable by an
|
// group that uses it, so this WARN is reachable by an
|
||||||
// unauthenticated client: a POST with no token to
|
// unauthenticated client: a POST with no token to
|
||||||
// /source/<any length of any text>/edit lands here. The
|
// /hook/<any length of any text>/edit lands here. The
|
||||||
// method and path are capped against the same budgets as
|
// method and path are capped against the same budgets as
|
||||||
// the access log. remote_addr is set by net/http from the
|
// the access log. remote_addr is set by net/http from the
|
||||||
// accepted connection rather than by the client, and
|
// accepted connection rather than by the client, and
|
||||||
|
|||||||
@@ -383,7 +383,7 @@ func TestLogLines_ClientChosenPathDoesNotSizeTheLine(t *testing.T) {
|
|||||||
t, newHandler,
|
t, newHandler,
|
||||||
)
|
)
|
||||||
|
|
||||||
path := "/source/" +
|
path := "/hook/" +
|
||||||
oversizedPathSegment(fill) + "/edit"
|
oversizedPathSegment(fill) + "/edit"
|
||||||
|
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
@@ -434,7 +434,7 @@ func TestLoginThrottle_LogLineDoesNotTrackPathSize(t *testing.T) {
|
|||||||
req := httptest.NewRequestWithContext(
|
req := httptest.NewRequestWithContext(
|
||||||
context.Background(),
|
context.Background(),
|
||||||
http.MethodPost,
|
http.MethodPost,
|
||||||
"/source/"+
|
"/hook/"+
|
||||||
oversizedPathSegment(fill)+"/login",
|
oversizedPathSegment(fill)+"/login",
|
||||||
nil,
|
nil,
|
||||||
)
|
)
|
||||||
@@ -499,7 +499,7 @@ func TestMaxBodySize_FloodOfOversizePathsDoesNotGrowTheLog(
|
|||||||
http.StatusRequestEntityTooLarge,
|
http.StatusRequestEntityTooLarge,
|
||||||
postOversize(
|
postOversize(
|
||||||
h,
|
h,
|
||||||
"/source/"+segment(i)+"/edit",
|
"/hook/"+segment(i)+"/edit",
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -108,10 +108,10 @@ type failureWindow struct {
|
|||||||
//
|
//
|
||||||
// A limiter that spends budget on arrival cannot protect a
|
// A limiter that spends budget on arrival cannot protect a
|
||||||
// single-admin product: behind the reverse proxy the deployment
|
// single-admin product: behind the reverse proxy the deployment
|
||||||
// requires, with TRUSTED_PROXIES unset, every client keys on the
|
// requires, when TRUSTED_PROXIES does not cover it, every client
|
||||||
// proxy, so a stranger trickling five POSTs a minute keeps the one
|
// keys on the proxy, so a stranger trickling five POSTs a minute
|
||||||
// bucket full and the operator's own correct password is answered 429
|
// keeps the one bucket full and the operator's own correct password
|
||||||
// forever. There is no second administrative path.
|
// is answered 429 forever. There is no second administrative path.
|
||||||
//
|
//
|
||||||
// So budget is spent only by a FAILED verification. A correct
|
// So budget is spent only by a FAILED verification. A correct
|
||||||
// password is never throttled, whatever the counters say, which is
|
// password is never throttled, whatever the counters say, which is
|
||||||
|
|||||||
@@ -40,7 +40,7 @@ const unmatchedMethod = unmatchedRoute
|
|||||||
//
|
//
|
||||||
// The pattern is what bounds the label's domain to the routes the
|
// The pattern is what bounds the label's domain to the routes the
|
||||||
// service registers. The path does not bound it at all — every byte
|
// service registers. The path does not bound it at all — every byte
|
||||||
// after /webhook/ is client-chosen, so labelling by path lets any
|
// after /h/ is client-chosen, so labelling by path lets any
|
||||||
// unauthenticated client mint permanent series at will, and publishes
|
// unauthenticated client mint permanent series at will, and publishes
|
||||||
// the entrypoint UUID (the receiver's only credential) in the scrape
|
// the entrypoint UUID (the receiver's only credential) in the scrape
|
||||||
// while doing it.
|
// while doing it.
|
||||||
|
|||||||
@@ -50,7 +50,7 @@ func realMethods() []string {
|
|||||||
// dimension varying, so any series growth a probe produces is the
|
// dimension varying, so any series growth a probe produces is the
|
||||||
// method label's and nothing else's.
|
// method label's and nothing else's.
|
||||||
func methodProbePath() string {
|
func methodProbePath() string {
|
||||||
return "/webhook/" + uuid.NewString()
|
return "/h/" + uuid.NewString()
|
||||||
}
|
}
|
||||||
|
|
||||||
// inventedMethods returns n distinct RFC 9110 method tokens that no
|
// inventedMethods returns n distinct RFC 9110 method tokens that no
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ const (
|
|||||||
|
|
||||||
// receiverRoutePattern is the one handler label every receiver
|
// receiverRoutePattern is the one handler label every receiver
|
||||||
// request must produce, however the client varies the path.
|
// request must produce, however the client varies the path.
|
||||||
receiverRoutePattern = "/webhook/{uuid}"
|
receiverRoutePattern = "/h/{uuid}"
|
||||||
|
|
||||||
// okRoute is a static route used to pin that the response-writer
|
// okRoute is a static route used to pin that the response-writer
|
||||||
// interceptor still reports status and size after the handler id
|
// interceptor still reports status and size after the handler id
|
||||||
@@ -143,13 +143,13 @@ func drivePaths(
|
|||||||
return drive(t, h, probes)
|
return drive(t, h, probes)
|
||||||
}
|
}
|
||||||
|
|
||||||
// receiverPaths returns n distinct /webhook/ paths, each naming a
|
// receiverPaths returns n distinct /h/ paths, each naming a
|
||||||
// fresh UUID exactly as an unauthenticated flood would.
|
// fresh UUID exactly as an unauthenticated flood would.
|
||||||
func receiverPaths(n int) []string {
|
func receiverPaths(n int) []string {
|
||||||
paths := make([]string, 0, n)
|
paths := make([]string, 0, n)
|
||||||
|
|
||||||
for range n {
|
for range n {
|
||||||
paths = append(paths, "/webhook/"+uuid.NewString())
|
paths = append(paths, "/h/"+uuid.NewString())
|
||||||
}
|
}
|
||||||
|
|
||||||
return paths
|
return paths
|
||||||
@@ -220,7 +220,7 @@ func keys(set map[string]struct{}) []string {
|
|||||||
|
|
||||||
// TestMetrics_DistinctReceiverPathsMintOneLabelSet is the direct
|
// TestMetrics_DistinctReceiverPathsMintOneLabelSet is the direct
|
||||||
// assertion the issue asks for: N requests to N distinct
|
// assertion the issue asks for: N requests to N distinct
|
||||||
// /webhook/<uuid> paths must produce exactly ONE handler label, the
|
// /h/<uuid> paths must produce exactly ONE handler label, the
|
||||||
// route pattern. Before the fix this produced N of them.
|
// route pattern. Before the fix this produced N of them.
|
||||||
func TestMetrics_DistinctReceiverPathsMintOneLabelSet(t *testing.T) {
|
func TestMetrics_DistinctReceiverPathsMintOneLabelSet(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
@@ -250,7 +250,7 @@ func TestMetrics_DistinctReceiverPathsMintOneLabelSet(t *testing.T) {
|
|||||||
// The scrape must not republish the UUIDs it was driven with.
|
// The scrape must not republish the UUIDs it was driven with.
|
||||||
// They are the receiver's only credential.
|
// They are the receiver's only credential.
|
||||||
for _, p := range paths {
|
for _, p := range paths {
|
||||||
id := strings.TrimPrefix(p, "/webhook/")
|
id := strings.TrimPrefix(p, "/h/")
|
||||||
for label := range labels {
|
for label := range labels {
|
||||||
assert.NotContains(
|
assert.NotContains(
|
||||||
t, label, id,
|
t, label, id,
|
||||||
@@ -354,7 +354,7 @@ func TestMetrics_UnmatchedPathsCollapseToTheSentinel(t *testing.T) {
|
|||||||
if i%2 == 0 {
|
if i%2 == 0 {
|
||||||
paths = append(paths, "/"+id)
|
paths = append(paths, "/"+id)
|
||||||
} else {
|
} else {
|
||||||
paths = append(paths, "/webhook/"+id+"/"+id)
|
paths = append(paths, "/h/"+id+"/"+id)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"log/slog"
|
"log/slog"
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"net/url"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -257,7 +258,7 @@ func concreteLogURL(r *http.Request) string {
|
|||||||
//
|
//
|
||||||
// 3xx and 4xx responses get the chi route pattern instead. Those are
|
// 3xx and 4xx responses get the chi route pattern instead. Those are
|
||||||
// the outcomes an unauthenticated client drives for free: 404 or 429
|
// the outcomes an unauthenticated client drives for free: 404 or 429
|
||||||
// on any invented /webhook/ path, 303 to the login page on any
|
// on any invented /h/ path, 303 to the login page on any
|
||||||
// invented /user/ path. Logging the concrete URL there lets a flood
|
// invented /user/ path. Logging the concrete URL there lets a flood
|
||||||
// write attacker-chosen text, of attacker-chosen length, into the
|
// write attacker-chosen text, of attacker-chosen length, into the
|
||||||
// operator's log at one line per request. The pattern comes from the
|
// operator's log at one line per request. The pattern comes from the
|
||||||
@@ -366,6 +367,30 @@ func (s *Middleware) CORS() func(http.Handler) http.Handler {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NextParam is the query parameter on the login redirect, and the
|
||||||
|
// login form field, that holds the page to return to after login.
|
||||||
|
const NextParam = "next"
|
||||||
|
|
||||||
|
// MaxNextBytes bounds the NextParam value. The login page writes it
|
||||||
|
// into its form, and every page is rendered into a buffer first, so
|
||||||
|
// without a bound a request would choose the size of that buffer.
|
||||||
|
const MaxNextBytes = 2048
|
||||||
|
|
||||||
|
// loginURL is the login page RequireAuth redirects to. A GET carries
|
||||||
|
// its own path and query in NextParam so that logging in returns to
|
||||||
|
// it, unless they are longer than MaxNextBytes; loginDestination in
|
||||||
|
// the handlers package checks whether that value is safe to follow.
|
||||||
|
// Other methods carry nothing, since a redirect cannot repeat them.
|
||||||
|
func loginURL(r *http.Request) string {
|
||||||
|
next := r.URL.RequestURI()
|
||||||
|
|
||||||
|
if r.Method != http.MethodGet || len(next) > MaxNextBytes {
|
||||||
|
return "/pages/login"
|
||||||
|
}
|
||||||
|
|
||||||
|
return "/pages/login?" + url.Values{NextParam: {next}}.Encode()
|
||||||
|
}
|
||||||
|
|
||||||
// RequireAuth returns middleware that checks for a valid session.
|
// RequireAuth returns middleware that checks for a valid session.
|
||||||
// Unauthenticated users are redirected to the login page.
|
// Unauthenticated users are redirected to the login page.
|
||||||
func (s *Middleware) RequireAuth() func(http.Handler) http.Handler {
|
func (s *Middleware) RequireAuth() func(http.Handler) http.Handler {
|
||||||
@@ -381,7 +406,7 @@ func (s *Middleware) RequireAuth() func(http.Handler) http.Handler {
|
|||||||
"error", err,
|
"error", err,
|
||||||
)
|
)
|
||||||
http.Redirect(
|
http.Redirect(
|
||||||
w, r, "/pages/login", http.StatusSeeOther,
|
w, r, loginURL(r), http.StatusSeeOther,
|
||||||
)
|
)
|
||||||
|
|
||||||
return
|
return
|
||||||
@@ -409,7 +434,7 @@ func (s *Middleware) RequireAuth() func(http.Handler) http.Handler {
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
http.Redirect(
|
http.Redirect(
|
||||||
w, r, "/pages/login", http.StatusSeeOther,
|
w, r, loginURL(r), http.StatusSeeOther,
|
||||||
)
|
)
|
||||||
|
|
||||||
return
|
return
|
||||||
@@ -560,7 +585,7 @@ func (s *Middleware) MaxBodySize(
|
|||||||
// internal/server/routes.go), so an
|
// internal/server/routes.go), so an
|
||||||
// unauthenticated client reaches it with a path
|
// unauthenticated client reaches it with a path
|
||||||
// of its own choosing and its own length —
|
// of its own choosing and its own length —
|
||||||
// POST /source/<8 KB>/edit with an oversize
|
// POST /hook/<8 KB>/edit with an oversize
|
||||||
// declared Content-Length costs nothing to
|
// declared Content-Length costs nothing to
|
||||||
// send. At WARN, on by default, that is a
|
// send. At WARN, on by default, that is a
|
||||||
// write into the operator's log sized by the
|
// write into the operator's log sized by the
|
||||||
|
|||||||
@@ -338,6 +338,76 @@ func TestRequireAuth_NoSession_RedirectsToLogin(t *testing.T) {
|
|||||||
"unauthenticated request",
|
"unauthenticated request",
|
||||||
)
|
)
|
||||||
assert.Equal(t, http.StatusSeeOther, w.Code)
|
assert.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
|
assert.Equal(
|
||||||
|
t, "/pages/login?next=%2Fdashboard", w.Header().Get("Location"),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRequireAuth_LoginRedirectCarriesOnlyAGet pins what the login
|
||||||
|
// redirect carries: a GET's path and query, so logging in can return
|
||||||
|
// there, and nothing for a POST, which a redirect cannot repeat.
|
||||||
|
func TestRequireAuth_LoginRedirectCarriesOnlyAGet(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
m, _ := testMiddleware(t, config.EnvironmentDev)
|
||||||
|
|
||||||
|
handler := m.RequireAuth()(http.HandlerFunc(
|
||||||
|
func(_ http.ResponseWriter, _ *http.Request) {},
|
||||||
|
))
|
||||||
|
|
||||||
|
get := httptest.NewRequestWithContext(
|
||||||
|
context.Background(),
|
||||||
|
http.MethodGet, "/hook/abc/events?page=2", nil,
|
||||||
|
)
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
handler.ServeHTTP(w, get)
|
||||||
|
|
||||||
|
assert.Equal(
|
||||||
|
t, "/pages/login?next=%2Fhook%2Fabc%2Fevents%3Fpage%3D2",
|
||||||
|
w.Header().Get("Location"),
|
||||||
|
)
|
||||||
|
|
||||||
|
post := httptest.NewRequestWithContext(
|
||||||
|
context.Background(),
|
||||||
|
http.MethodPost, "/hook/abc/delete", nil,
|
||||||
|
)
|
||||||
|
w = httptest.NewRecorder()
|
||||||
|
handler.ServeHTTP(w, post)
|
||||||
|
|
||||||
|
assert.Equal(t, "/pages/login", w.Header().Get("Location"))
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRequireAuth_LoginRedirectLeavesOutALongURL: a GET whose path
|
||||||
|
// and query are longer than the login page accepts goes to the plain
|
||||||
|
// login page, so a long URL does not make the redirect long.
|
||||||
|
func TestRequireAuth_LoginRedirectLeavesOutALongURL(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
m, _ := testMiddleware(t, config.EnvironmentDev)
|
||||||
|
|
||||||
|
handler := m.RequireAuth()(http.HandlerFunc(
|
||||||
|
func(_ http.ResponseWriter, _ *http.Request) {},
|
||||||
|
))
|
||||||
|
|
||||||
|
atLimit := "/" + strings.Repeat("a", middleware.MaxNextBytes-1)
|
||||||
|
|
||||||
|
get := httptest.NewRequestWithContext(
|
||||||
|
context.Background(), http.MethodGet, atLimit, nil,
|
||||||
|
)
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
handler.ServeHTTP(w, get)
|
||||||
|
|
||||||
|
assert.Equal(
|
||||||
|
t, "/pages/login?next=%2F"+atLimit[1:],
|
||||||
|
w.Header().Get("Location"),
|
||||||
|
)
|
||||||
|
|
||||||
|
get = httptest.NewRequestWithContext(
|
||||||
|
context.Background(), http.MethodGet, atLimit+"a", nil,
|
||||||
|
)
|
||||||
|
w = httptest.NewRecorder()
|
||||||
|
handler.ServeHTTP(w, get)
|
||||||
|
|
||||||
assert.Equal(t, "/pages/login", w.Header().Get("Location"))
|
assert.Equal(t, "/pages/login", w.Header().Get("Location"))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -443,7 +513,9 @@ func TestRequireAuth_UnauthenticatedSession_RedirectsToLogin(
|
|||||||
"unauthenticated session",
|
"unauthenticated session",
|
||||||
)
|
)
|
||||||
assert.Equal(t, http.StatusSeeOther, w.Code)
|
assert.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
assert.Equal(t, "/pages/login", w.Header().Get("Location"))
|
assert.Equal(
|
||||||
|
t, "/pages/login?next=%2Fdashboard", w.Header().Get("Location"),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// --- RequireAuth Session Expiry Tests ---
|
// --- RequireAuth Session Expiry Tests ---
|
||||||
@@ -541,7 +613,9 @@ func TestRequireAuth_IdleExpiredSession_RedirectsToLogin(
|
|||||||
"handler should not run for an idle-expired session",
|
"handler should not run for an idle-expired session",
|
||||||
)
|
)
|
||||||
assert.Equal(t, http.StatusSeeOther, w.Code)
|
assert.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
assert.Equal(t, "/pages/login", w.Header().Get("Location"))
|
assert.Equal(
|
||||||
|
t, "/pages/login?next=%2Fdashboard", w.Header().Get("Location"),
|
||||||
|
)
|
||||||
assert.Empty(
|
assert.Empty(
|
||||||
t, sessionCookies(w),
|
t, sessionCookies(w),
|
||||||
"an expired session must not be refreshed",
|
"an expired session must not be refreshed",
|
||||||
@@ -640,7 +714,7 @@ func TestNoCache_SetsHeaders(t *testing.T) {
|
|||||||
|
|
||||||
req := httptest.NewRequestWithContext(
|
req := httptest.NewRequestWithContext(
|
||||||
context.Background(),
|
context.Background(),
|
||||||
http.MethodGet, "/sources", nil,
|
http.MethodGet, "/hooks", nil,
|
||||||
)
|
)
|
||||||
w := httptest.NewRecorder()
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
|||||||
@@ -63,7 +63,7 @@ const (
|
|||||||
|
|
||||||
// receiverAggregateMultiplier scales the configured
|
// receiverAggregateMultiplier scales the configured
|
||||||
// per-entrypoint receiver limit into the aggregate limit one
|
// per-entrypoint receiver limit into the aggregate limit one
|
||||||
// client IP may spend across the whole /webhook/* route. Ten
|
// client IP may spend across the whole /h/* route. Ten
|
||||||
// entrypoints' worth lets a single sender address drive several
|
// entrypoints' worth lets a single sender address drive several
|
||||||
// entrypoints at their full rate, while still capping what one
|
// entrypoints at their full rate, while still capping what one
|
||||||
// address costs the unauthenticated receiver.
|
// address costs the unauthenticated receiver.
|
||||||
@@ -123,9 +123,8 @@ func bucketKey(addr netip.Addr) string {
|
|||||||
return prefix.String()
|
return prefix.String()
|
||||||
}
|
}
|
||||||
|
|
||||||
// isTrustedProxy reports whether addr belongs to a network the
|
// isTrustedProxy reports whether addr belongs to a network in
|
||||||
// operator listed in TRUSTED_PROXIES. The list is empty by default,
|
// TRUSTED_PROXIES, which by default is the RFC 1918 private ranges.
|
||||||
// so by default nothing is trusted.
|
|
||||||
func (m *Middleware) isTrustedProxy(addr netip.Addr) bool {
|
func (m *Middleware) isTrustedProxy(addr netip.Addr) bool {
|
||||||
for _, prefix := range m.params.Config.TrustedProxies {
|
for _, prefix := range m.params.Config.TrustedProxies {
|
||||||
if prefix.Contains(addr) {
|
if prefix.Contains(addr) {
|
||||||
@@ -390,7 +389,7 @@ func (m *Middleware) postRateLimit(
|
|||||||
// It is Config.ReceiverRateLimit requests per minute.
|
// It is Config.ReceiverRateLimit requests per minute.
|
||||||
//
|
//
|
||||||
// That limit alone bounds nothing in aggregate. The route pattern
|
// That limit alone bounds nothing in aggregate. The route pattern
|
||||||
// /webhook/{uuid} matches any single segment, so a client that
|
// /h/{uuid} matches any single segment, so a client that
|
||||||
// invents a fresh path per request mints a fresh bucket per request
|
// invents a fresh path per request mints a fresh bucket per request
|
||||||
// and never refills one — and every such request still reaches the
|
// and never refills one — and every such request still reaches the
|
||||||
// handler's entrypoint lookup before it 404s. The outer limit is
|
// handler's entrypoint lookup before it 404s. The outer limit is
|
||||||
|
|||||||
@@ -275,7 +275,7 @@ func TestReceiverRateLimit_LimitsPerIPAndPath(t *testing.T) {
|
|||||||
// pass.
|
// pass.
|
||||||
for i := range limit {
|
for i := range limit {
|
||||||
w := receiverPost(
|
w := receiverPost(
|
||||||
handler, "9.9.9.9:1234", "/webhook/uuid-a",
|
handler, "9.9.9.9:1234", "/h/uuid-a",
|
||||||
)
|
)
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t, http.StatusOK, w.Code,
|
t, http.StatusOK, w.Code,
|
||||||
@@ -286,7 +286,7 @@ func TestReceiverRateLimit_LimitsPerIPAndPath(t *testing.T) {
|
|||||||
// The next request over the limit is rejected with a 429
|
// The next request over the limit is rejected with a 429
|
||||||
// carrying a Retry-After header.
|
// carrying a Retry-After header.
|
||||||
w := receiverPost(
|
w := receiverPost(
|
||||||
handler, "9.9.9.9:1234", "/webhook/uuid-a",
|
handler, "9.9.9.9:1234", "/h/uuid-a",
|
||||||
)
|
)
|
||||||
assert.Equal(t, http.StatusTooManyRequests, w.Code)
|
assert.Equal(t, http.StatusTooManyRequests, w.Code)
|
||||||
assert.NotEmpty(
|
assert.NotEmpty(
|
||||||
@@ -296,7 +296,7 @@ func TestReceiverRateLimit_LimitsPerIPAndPath(t *testing.T) {
|
|||||||
|
|
||||||
// The same IP is not limited on a different entrypoint.
|
// The same IP is not limited on a different entrypoint.
|
||||||
w = receiverPost(
|
w = receiverPost(
|
||||||
handler, "9.9.9.9:1234", "/webhook/uuid-b",
|
handler, "9.9.9.9:1234", "/h/uuid-b",
|
||||||
)
|
)
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t, http.StatusOK, w.Code,
|
t, http.StatusOK, w.Code,
|
||||||
@@ -305,7 +305,7 @@ func TestReceiverRateLimit_LimitsPerIPAndPath(t *testing.T) {
|
|||||||
|
|
||||||
// A different IP is not limited on the same entrypoint.
|
// A different IP is not limited on the same entrypoint.
|
||||||
w = receiverPost(
|
w = receiverPost(
|
||||||
handler, "8.8.8.8:1234", "/webhook/uuid-a",
|
handler, "8.8.8.8:1234", "/h/uuid-a",
|
||||||
)
|
)
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t, http.StatusOK, w.Code,
|
t, http.StatusOK, w.Code,
|
||||||
@@ -322,7 +322,7 @@ func TestReceiverRateLimit_CountsEveryMethod(t *testing.T) {
|
|||||||
const (
|
const (
|
||||||
limit = 2
|
limit = 2
|
||||||
ip = "7.7.7.7:1234"
|
ip = "7.7.7.7:1234"
|
||||||
path = "/webhook/uuid-c"
|
path = "/h/uuid-c"
|
||||||
)
|
)
|
||||||
|
|
||||||
handler := receiverLimitedHandler(t, limit)
|
handler := receiverLimitedHandler(t, limit)
|
||||||
@@ -384,8 +384,8 @@ const (
|
|||||||
// trustedProxyCIDR is the proxy network the forwarded-path
|
// trustedProxyCIDR is the proxy network the forwarded-path
|
||||||
// tests configure, and trustedPeer an address inside it. A
|
// tests configure, and trustedPeer an address inside it. A
|
||||||
// production deployment is required to run behind a reverse
|
// production deployment is required to run behind a reverse
|
||||||
// proxy with TRUSTED_PROXIES set, so this is the shape the
|
// proxy that TRUSTED_PROXIES covers, either by the default or by
|
||||||
// bucketing has to hold in.
|
// a set value, so this is the shape the bucketing has to hold in.
|
||||||
trustedProxyCIDR = "10.0.0.0/8"
|
trustedProxyCIDR = "10.0.0.0/8"
|
||||||
trustedPeer = "10.0.0.1:44444"
|
trustedPeer = "10.0.0.1:44444"
|
||||||
)
|
)
|
||||||
@@ -426,8 +426,8 @@ func assertSharedBucket(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TestRateLimitKey_SpoofedForwardedFromUntrustedPeer is the test
|
// TestRateLimitKey_SpoofedForwardedFromUntrustedPeer is the test
|
||||||
// this gating exists for: with no trusted proxies configured (the
|
// this gating exists for: from a peer that is not a trusted
|
||||||
// default), a client that rotates a forwarded header on every
|
// proxy, a client that rotates a forwarded header on every
|
||||||
// request must stay in one bucket. If forwarded headers were
|
// request must stay in one bucket. If forwarded headers were
|
||||||
// trusted unconditionally, each spoofed value would mint a fresh
|
// trusted unconditionally, each spoofed value would mint a fresh
|
||||||
// bucket and the limit would stop no one.
|
// bucket and the limit would stop no one.
|
||||||
@@ -715,7 +715,7 @@ func TestReceiverRateLimit_LimitsAggregateAcrossInventedPaths(
|
|||||||
// none of them shares a per-entrypoint bucket with another.
|
// none of them shares a per-entrypoint bucket with another.
|
||||||
for i := range aggregate {
|
for i := range aggregate {
|
||||||
w := receiverPost(
|
w := receiverPost(
|
||||||
handler, ip, fmt.Sprintf("/webhook/invented-%d", i),
|
handler, ip, fmt.Sprintf("/h/invented-%d", i),
|
||||||
)
|
)
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t, http.StatusOK, w.Code,
|
t, http.StatusOK, w.Code,
|
||||||
@@ -724,17 +724,17 @@ func TestReceiverRateLimit_LimitsAggregateAcrossInventedPaths(
|
|||||||
}
|
}
|
||||||
|
|
||||||
w := receiverPost(
|
w := receiverPost(
|
||||||
handler, ip, fmt.Sprintf("/webhook/invented-%d", aggregate),
|
handler, ip, fmt.Sprintf("/h/invented-%d", aggregate),
|
||||||
)
|
)
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t, http.StatusTooManyRequests, w.Code,
|
t, http.StatusTooManyRequests, w.Code,
|
||||||
"a client must not be able to raise its aggregate rate "+
|
"a client must not be able to raise its aggregate rate "+
|
||||||
"against /webhook/* by varying the path",
|
"against /h/* by varying the path",
|
||||||
)
|
)
|
||||||
|
|
||||||
// The aggregate limit is still per client IP: exhausting one
|
// The aggregate limit is still per client IP: exhausting one
|
||||||
// address must not throttle another.
|
// address must not throttle another.
|
||||||
w = receiverPost(handler, "6.6.6.7:1234", "/webhook/invented-0")
|
w = receiverPost(handler, "6.6.6.7:1234", "/h/invented-0")
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t, http.StatusOK, w.Code,
|
t, http.StatusOK, w.Code,
|
||||||
"a different client IP must not be affected",
|
"a different client IP must not be affected",
|
||||||
@@ -771,7 +771,7 @@ func TestReceiverRateLimit_RejectedRequestsCountTowardAggregate(
|
|||||||
// limit requests are served; the rest are rejected by the
|
// limit requests are served; the rest are rejected by the
|
||||||
// per-entrypoint limiter but still count against the aggregate.
|
// per-entrypoint limiter but still count against the aggregate.
|
||||||
for i := range aggregate {
|
for i := range aggregate {
|
||||||
w := receiverPost(handler, ip, "/webhook/exhausted")
|
w := receiverPost(handler, ip, "/h/exhausted")
|
||||||
|
|
||||||
want := http.StatusTooManyRequests
|
want := http.StatusTooManyRequests
|
||||||
if i < limit {
|
if i < limit {
|
||||||
@@ -784,7 +784,7 @@ func TestReceiverRateLimit_RejectedRequestsCountTowardAggregate(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
w := receiverPost(handler, ip, "/webhook/never-used")
|
w := receiverPost(handler, ip, "/h/never-used")
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t, http.StatusTooManyRequests, w.Code,
|
t, http.StatusTooManyRequests, w.Code,
|
||||||
"requests rejected per entrypoint must still count "+
|
"requests rejected per entrypoint must still count "+
|
||||||
@@ -823,7 +823,7 @@ func TestReceiverRateLimit_IgnoresForwardedFromUntrustedPeer(
|
|||||||
const (
|
const (
|
||||||
limit = 3
|
limit = 3
|
||||||
peer = "203.0.113.10:44444"
|
peer = "203.0.113.10:44444"
|
||||||
path = "/webhook/uuid-d"
|
path = "/h/uuid-d"
|
||||||
)
|
)
|
||||||
|
|
||||||
handler := receiverLimitedHandler(t, limit)
|
handler := receiverLimitedHandler(t, limit)
|
||||||
@@ -1097,8 +1097,9 @@ func TestPostRateLimit_IPv4IndependentPerAddress(t *testing.T) {
|
|||||||
// that arrives from trustedPeer — a configured trusted proxy — and
|
// that arrives from trustedPeer — a configured trusted proxy — and
|
||||||
// names forwarded as its client in X-Forwarded-For. That is the
|
// names forwarded as its client in X-Forwarded-For. That is the
|
||||||
// production path: a deployment is required to run behind a reverse
|
// production path: a deployment is required to run behind a reverse
|
||||||
// proxy with TRUSTED_PROXIES set, so the forwarded address, not the
|
// proxy that TRUSTED_PROXIES covers, either by the default or by a
|
||||||
// peer, is what the limiters bucket on there.
|
// set value, so the forwarded address, not the peer, is what the
|
||||||
|
// limiters bucket on there.
|
||||||
func forwardedKeyFor(
|
func forwardedKeyFor(
|
||||||
t *testing.T, m *middleware.Middleware, forwarded string,
|
t *testing.T, m *middleware.Middleware, forwarded string,
|
||||||
) string {
|
) string {
|
||||||
@@ -1178,9 +1179,9 @@ func TestRateLimitKey_ForwardedIPv6BucketsByPrefix(t *testing.T) {
|
|||||||
//
|
//
|
||||||
// Every existing test of this fallback uses an IPv4 proxy, where
|
// Every existing test of this fallback uses an IPv4 proxy, where
|
||||||
// bucketKey is the identity function, so replacing the call with
|
// bucketKey is the identity function, so replacing the call with
|
||||||
// peer.String() leaves the whole suite green. Only operator-listed
|
// peer.String() leaves the whole suite green. Only addresses inside
|
||||||
// addresses reach this line and the fallback is fail-closed, so this
|
// TRUSTED_PROXIES reach this line and the fallback is fail-closed, so
|
||||||
// pins behaviour rather than fixing a defect.
|
// this pins behaviour rather than fixing a defect.
|
||||||
func TestRateLimitKey_TrustedPeerUnusableForwardedMasksPeer(
|
func TestRateLimitKey_TrustedPeerUnusableForwardedMasksPeer(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
) {
|
) {
|
||||||
|
|||||||
@@ -131,16 +131,22 @@ type noopNotifier struct{}
|
|||||||
|
|
||||||
func (n *noopNotifier) Notify([]delivery.Task) {}
|
func (n *noopNotifier) Notify([]delivery.Task) {}
|
||||||
|
|
||||||
type noopEvictor struct{}
|
type noopArchives struct{}
|
||||||
|
|
||||||
func (n *noopEvictor) EvictWebhook(string) {}
|
func (n *noopArchives) EvictWebhook(string) {}
|
||||||
|
|
||||||
|
func (n *noopArchives) EvictTarget(string) {}
|
||||||
|
|
||||||
|
func (n *noopArchives) RenameArchive(_, _, _ string) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// newServerApp starts the real login path against dir: the handlers,
|
// newServerApp starts the real login path against dir: the handlers,
|
||||||
// the middleware that bounds password verification, the session store
|
// the middleware that bounds password verification, the session store
|
||||||
// and the database, exactly as internal/handlers builds them.
|
// and the database, exactly as internal/handlers builds them.
|
||||||
//
|
//
|
||||||
// One application per test function, not per case: every start that
|
// One application per test function, not per case: every start that
|
||||||
// finds no account seeds one at 64 MB of Argon2id, and this package's
|
// finds no account seeds one with an Argon2id hash, and this package's
|
||||||
// budget is not the place to spend that repeatedly.
|
// budget is not the place to spend that repeatedly.
|
||||||
func newServerApp(
|
func newServerApp(
|
||||||
t *testing.T, dir string,
|
t *testing.T, dir string,
|
||||||
@@ -162,7 +168,7 @@ func newServerApp(
|
|||||||
healthcheck.New,
|
healthcheck.New,
|
||||||
session.New,
|
session.New,
|
||||||
func() delivery.Notifier { return &noopNotifier{} },
|
func() delivery.Notifier { return &noopNotifier{} },
|
||||||
func() delivery.WebhookEvictor { return &noopEvictor{} },
|
func() delivery.Archives { return &noopArchives{} },
|
||||||
middleware.New,
|
middleware.New,
|
||||||
delivery.NewGuard,
|
delivery.NewGuard,
|
||||||
handlers.New,
|
handlers.New,
|
||||||
|
|||||||
+28
-13
@@ -92,11 +92,25 @@ func (s *Server) setupGlobalMiddleware() {
|
|||||||
func (s *Server) setupRoutes() {
|
func (s *Server) setupRoutes() {
|
||||||
s.router.Get("/", s.h.HandleIndex())
|
s.router.Get("/", s.h.HandleIndex())
|
||||||
|
|
||||||
s.router.Mount(
|
// Static assets answer GET and HEAD only. chi's default 405
|
||||||
"/s",
|
// carries no Allow header, so this group supplies its own.
|
||||||
http.StripPrefix("/s", http.FileServer(http.FS(static.Static))),
|
staticFiles := http.StripPrefix(
|
||||||
|
"/s", http.FileServer(http.FS(static.Static)),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
s.router.Route("/s", func(r chi.Router) {
|
||||||
|
r.MethodNotAllowed(func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
w.Header().Set("Allow", "GET, HEAD")
|
||||||
|
http.Error(
|
||||||
|
w,
|
||||||
|
"Method Not Allowed",
|
||||||
|
http.StatusMethodNotAllowed,
|
||||||
|
)
|
||||||
|
})
|
||||||
|
r.Method(http.MethodGet, "/*", staticFiles)
|
||||||
|
r.Method(http.MethodHead, "/*", staticFiles)
|
||||||
|
})
|
||||||
|
|
||||||
s.router.Route("/api/v1", func(_ chi.Router) {
|
s.router.Route("/api/v1", func(_ chi.Router) {
|
||||||
// API routes will be added here.
|
// API routes will be added here.
|
||||||
})
|
})
|
||||||
@@ -140,11 +154,12 @@ func (s *Server) setupPageRoutes() {
|
|||||||
r.Use(s.mw.NoCache())
|
r.Use(s.mw.NoCache())
|
||||||
|
|
||||||
// The login POST carries no pre-emptive rate limiter. Behind
|
// The login POST carries no pre-emptive rate limiter. Behind
|
||||||
// the reverse proxy production requires, with TRUSTED_PROXIES
|
// the reverse proxy production requires, when TRUSTED_PROXIES
|
||||||
// unset, every client shares one bucket, so a limiter spent
|
// does not cover it, every client shares one bucket, so a
|
||||||
// on arrival lets any stranger deny the operator the only
|
// limiter spent on arrival lets any stranger deny the operator
|
||||||
// administrative path. The handler verifies credentials first
|
// the only administrative path. The handler verifies
|
||||||
// and charges only failures; see Handlers.authenticateUser.
|
// credentials first and charges only failures; see
|
||||||
|
// Handlers.authenticateUser.
|
||||||
r.Get("/login", s.h.HandleLoginPage())
|
r.Get("/login", s.h.HandleLoginPage())
|
||||||
r.Post("/login", s.h.HandleLoginSubmit())
|
r.Post("/login", s.h.HandleLoginSubmit())
|
||||||
|
|
||||||
@@ -168,7 +183,7 @@ func (s *Server) setupUserRoutes() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *Server) setupSourceRoutes() {
|
func (s *Server) setupSourceRoutes() {
|
||||||
s.router.Route("/sources", func(r chi.Router) {
|
s.router.Route("/hooks", func(r chi.Router) {
|
||||||
// MaxBodySize precedes CSRF and RequireAuth deliberately;
|
// MaxBodySize precedes CSRF and RequireAuth deliberately;
|
||||||
// see maxFormBodySize for why, and for what it costs.
|
// see maxFormBodySize for why, and for what it costs.
|
||||||
r.Use(s.mw.MaxBodySize(maxFormBodySize))
|
r.Use(s.mw.MaxBodySize(maxFormBodySize))
|
||||||
@@ -180,7 +195,7 @@ func (s *Server) setupSourceRoutes() {
|
|||||||
r.Post("/new", s.h.HandleSourceCreateSubmit())
|
r.Post("/new", s.h.HandleSourceCreateSubmit())
|
||||||
})
|
})
|
||||||
|
|
||||||
s.router.Route("/source/{sourceID}", func(r chi.Router) {
|
s.router.Route("/hook/{sourceID}", func(r chi.Router) {
|
||||||
// MaxBodySize precedes CSRF and RequireAuth deliberately;
|
// MaxBodySize precedes CSRF and RequireAuth deliberately;
|
||||||
// see maxFormBodySize for why, and for what it costs.
|
// see maxFormBodySize for why, and for what it costs.
|
||||||
r.Use(s.mw.MaxBodySize(maxFormBodySize))
|
r.Use(s.mw.MaxBodySize(maxFormBodySize))
|
||||||
@@ -191,14 +206,14 @@ func (s *Server) setupSourceRoutes() {
|
|||||||
r.Get("/edit", s.h.HandleSourceEdit())
|
r.Get("/edit", s.h.HandleSourceEdit())
|
||||||
r.Post("/edit", s.h.HandleSourceEditSubmit())
|
r.Post("/edit", s.h.HandleSourceEditSubmit())
|
||||||
r.Post("/delete", s.h.HandleSourceDelete())
|
r.Post("/delete", s.h.HandleSourceDelete())
|
||||||
r.Get("/logs", s.h.HandleSourceLogs())
|
r.Get("/events", s.h.HandleSourceLogs())
|
||||||
// The log page renders each body only up to its cap, so
|
// The log page renders each body only up to its cap, so
|
||||||
// this is the only route that serves a whole one. It
|
// this is the only route that serves a whole one. It
|
||||||
// belongs to this group for its RequireAuth and
|
// belongs to this group for its RequireAuth and
|
||||||
// NoCache; see HandleEventBodyDownload for the headers
|
// NoCache; see HandleEventBodyDownload for the headers
|
||||||
// that keep the bytes it returns inert.
|
// that keep the bytes it returns inert.
|
||||||
r.Get(
|
r.Get(
|
||||||
"/logs/{eventID}/body",
|
"/events/{eventID}/body",
|
||||||
s.h.HandleEventBodyDownload(),
|
s.h.HandleEventBodyDownload(),
|
||||||
)
|
)
|
||||||
// Replay is the one page action that queues outbound work:
|
// Replay is the one page action that queues outbound work:
|
||||||
@@ -265,7 +280,7 @@ func (s *Server) setupSourceRoutes() {
|
|||||||
|
|
||||||
func (s *Server) setupWebhookRoutes() {
|
func (s *Server) setupWebhookRoutes() {
|
||||||
s.router.With(s.mw.ReceiverRateLimit()).HandleFunc(
|
s.router.With(s.mw.ReceiverRateLimit()).HandleFunc(
|
||||||
"/webhook/{uuid}",
|
"/h/{uuid}",
|
||||||
s.h.HandleWebhook(),
|
s.h.HandleWebhook(),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
+579
-33
@@ -7,6 +7,7 @@ import (
|
|||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"net/url"
|
"net/url"
|
||||||
"regexp"
|
"regexp"
|
||||||
|
"slices"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -45,12 +46,18 @@ type noopNotifier struct{}
|
|||||||
|
|
||||||
func (n *noopNotifier) Notify([]delivery.Task) {}
|
func (n *noopNotifier) Notify([]delivery.Task) {}
|
||||||
|
|
||||||
// noopEvictor satisfies handlers.New's delivery.WebhookEvictor
|
// noopArchives satisfies handlers.New's delivery.Archives
|
||||||
// dependency. These tests never delete a webhook, so there is
|
// dependency. No test here checks what gets evicted or renamed, so
|
||||||
// nothing to record.
|
// it records nothing.
|
||||||
type noopEvictor struct{}
|
type noopArchives struct{}
|
||||||
|
|
||||||
func (e *noopEvictor) EvictWebhook(string) {}
|
func (e *noopArchives) EvictWebhook(string) {}
|
||||||
|
|
||||||
|
func (e *noopArchives) EvictTarget(string) {}
|
||||||
|
|
||||||
|
func (e *noopArchives) RenameArchive(_, _, _ string) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// testEnv is the real router from routes.go plus the collaborators
|
// testEnv is the real router from routes.go plus the collaborators
|
||||||
// tests need to seed users and forge sessions.
|
// tests need to seed users and forge sessions.
|
||||||
@@ -111,7 +118,7 @@ func newTestEnvWithConfig(
|
|||||||
healthcheck.New,
|
healthcheck.New,
|
||||||
session.New,
|
session.New,
|
||||||
func() delivery.Notifier { return &noopNotifier{} },
|
func() delivery.Notifier { return &noopNotifier{} },
|
||||||
func() delivery.WebhookEvictor { return &noopEvictor{} },
|
func() delivery.Archives { return &noopArchives{} },
|
||||||
middleware.New,
|
middleware.New,
|
||||||
delivery.NewGuard,
|
delivery.NewGuard,
|
||||||
handlers.New,
|
handlers.New,
|
||||||
@@ -220,13 +227,45 @@ func (e *testEnv) csrfFrom(
|
|||||||
// out of the markup has to be unescaped before it is submitted.
|
// out of the markup has to be unescaped before it is submitted.
|
||||||
token := html.UnescapeString(match[1])
|
token := html.UnescapeString(match[1])
|
||||||
|
|
||||||
combined := make([]*http.Cookie, 0, len(cookies))
|
// A cookie the page sets replaces the one of the same name, as in
|
||||||
combined = append(combined, cookies...)
|
// a browser. Sent both, the server would read the first, older one.
|
||||||
combined = append(combined, w.Result().Cookies()...)
|
set := w.Result().Cookies()
|
||||||
|
combined := make([]*http.Cookie, 0, len(cookies)+len(set))
|
||||||
|
|
||||||
|
for _, c := range cookies {
|
||||||
|
replaced := slices.ContainsFunc(set, func(n *http.Cookie) bool {
|
||||||
|
return n.Name == c.Name
|
||||||
|
})
|
||||||
|
if !replaced {
|
||||||
|
combined = append(combined, c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
combined = append(combined, set...)
|
||||||
|
|
||||||
return token, combined
|
return token, combined
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// urlFrom renders the page at path and returns the link or form
|
||||||
|
// action that pattern's one group captures, so a test requests the
|
||||||
|
// URL the template emitted rather than one it wrote itself.
|
||||||
|
func (e *testEnv) urlFrom(
|
||||||
|
t *testing.T,
|
||||||
|
path, pattern string,
|
||||||
|
cookies []*http.Cookie,
|
||||||
|
) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
w := e.get(path, cookies)
|
||||||
|
require.Equal(t, http.StatusOK, w.Code)
|
||||||
|
|
||||||
|
match := regexp.MustCompile(pattern).
|
||||||
|
FindStringSubmatch(w.Body.String())
|
||||||
|
require.Len(t, match, 2, "%s should render %s", path, pattern)
|
||||||
|
|
||||||
|
return html.UnescapeString(match[1])
|
||||||
|
}
|
||||||
|
|
||||||
// authCookies forges an authenticated session for the given user.
|
// authCookies forges an authenticated session for the given user.
|
||||||
func (e *testEnv) authCookies(
|
func (e *testEnv) authCookies(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
@@ -396,13 +435,15 @@ func (e *testEnv) storedHash(t *testing.T, username string) string {
|
|||||||
|
|
||||||
// --- /s static group ---
|
// --- /s static group ---
|
||||||
|
|
||||||
// TestStaticServesEveryMethod pins what the static mount actually
|
// TestStaticServesOnlyGetAndHead pins the methods the static group
|
||||||
// answers. chi's Mount registers the handler for all methods and
|
// answers: GET and HEAD are served the asset, and the other methods
|
||||||
// http.FileServer only special-cases HEAD (by suppressing the body),
|
// chi routes (POST, PUT, DELETE and the rest) are refused with 405
|
||||||
// so a POST or a DELETE to an asset is served the file rather than
|
// and an Allow header naming those two. A method chi does not route,
|
||||||
// refused. The README documents this; the test is what keeps the two
|
// such as PROPFIND, is refused with 405 by the top-level router
|
||||||
// from drifting.
|
// before it reaches the static group, so it gets no Allow header.
|
||||||
func TestStaticServesEveryMethod(t *testing.T) {
|
// The README documents this; the test is what keeps the two from
|
||||||
|
// drifting.
|
||||||
|
func TestStaticServesOnlyGetAndHead(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := newTestEnv(t)
|
env := newTestEnv(t)
|
||||||
@@ -417,6 +458,7 @@ func TestStaticServesEveryMethod(t *testing.T) {
|
|||||||
http.MethodPost,
|
http.MethodPost,
|
||||||
http.MethodPut,
|
http.MethodPut,
|
||||||
http.MethodDelete,
|
http.MethodDelete,
|
||||||
|
"PROPFIND",
|
||||||
} {
|
} {
|
||||||
t.Run(method, func(t *testing.T) {
|
t.Run(method, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
@@ -428,18 +470,38 @@ func TestStaticServesEveryMethod(t *testing.T) {
|
|||||||
w := httptest.NewRecorder()
|
w := httptest.NewRecorder()
|
||||||
env.router.ServeHTTP(w, req)
|
env.router.ServeHTTP(w, req)
|
||||||
|
|
||||||
assert.Equal(t, http.StatusOK, w.Code,
|
switch method {
|
||||||
"static mount answers every method")
|
case http.MethodGet:
|
||||||
|
assert.Equal(t, http.StatusOK, w.Code)
|
||||||
if method == http.MethodHead {
|
assert.Equal(t, body, w.Body.Bytes(),
|
||||||
|
"the asset itself is returned")
|
||||||
|
case http.MethodHead:
|
||||||
|
assert.Equal(t, http.StatusOK, w.Code)
|
||||||
assert.Empty(t, w.Body.Bytes(),
|
assert.Empty(t, w.Body.Bytes(),
|
||||||
"HEAD must not carry a body")
|
"HEAD must not carry a body")
|
||||||
|
case "PROPFIND":
|
||||||
return
|
assert.Equal(
|
||||||
|
t, http.StatusMethodNotAllowed, w.Code,
|
||||||
|
)
|
||||||
|
assert.Empty(t, w.Header().Get("Allow"),
|
||||||
|
"chi refuses a method it does not route "+
|
||||||
|
"before the static group runs")
|
||||||
|
assert.NotContains(
|
||||||
|
t, w.Body.String(), string(body),
|
||||||
|
"a refused method must not get the asset",
|
||||||
|
)
|
||||||
|
default:
|
||||||
|
assert.Equal(
|
||||||
|
t, http.StatusMethodNotAllowed, w.Code,
|
||||||
|
)
|
||||||
|
assert.Equal(
|
||||||
|
t, "GET, HEAD", w.Header().Get("Allow"),
|
||||||
|
)
|
||||||
|
assert.NotContains(
|
||||||
|
t, w.Body.String(), string(body),
|
||||||
|
"a refused method must not get the asset",
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
assert.Equal(t, body, w.Body.Bytes(),
|
|
||||||
"the asset itself is returned")
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -591,6 +653,97 @@ func TestPagesLogin_CorrectPasswordSurvivesASpentBudget(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestPagesLogin_CookiesFromAnEarlierDatabase is
|
||||||
|
// https://git.eeqj.de/sneak/webhooker/issues/359. A new database
|
||||||
|
// brings a new session key, and the operator's browser still holds
|
||||||
|
// the session and CSRF cookies signed with the old one. Logging in
|
||||||
|
// must work as from a fresh browser and leave cookies the new key
|
||||||
|
// accepts.
|
||||||
|
func TestPagesLogin_CookiesFromAnEarlierDatabase(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const (
|
||||||
|
username = "operator"
|
||||||
|
password = "correct-horse-battery-staple"
|
||||||
|
)
|
||||||
|
|
||||||
|
earlier := newTestEnv(t)
|
||||||
|
earlierID, _ := earlier.seedUser(t, username, password)
|
||||||
|
_, stale := earlier.csrfFrom(t, "/pages/login", nil)
|
||||||
|
stale = append(stale, earlier.authCookies(t, earlierID, username)...)
|
||||||
|
|
||||||
|
env := newTestEnv(t)
|
||||||
|
env.seedUser(t, username, password)
|
||||||
|
|
||||||
|
token, cookies := env.csrfFrom(t, "/pages/login", stale)
|
||||||
|
|
||||||
|
form := url.Values{}
|
||||||
|
form.Set("csrf_token", token)
|
||||||
|
form.Set("username", username)
|
||||||
|
form.Set("password", password)
|
||||||
|
|
||||||
|
w := env.post("/pages/login", form, cookies)
|
||||||
|
require.Equal(
|
||||||
|
t, http.StatusSeeOther, w.Code,
|
||||||
|
"a session cookie from another key must not fail the login",
|
||||||
|
)
|
||||||
|
|
||||||
|
// The response deletes the old session cookie and then sets the
|
||||||
|
// new one; a browser keeps the last.
|
||||||
|
var fresh *http.Cookie
|
||||||
|
|
||||||
|
for _, c := range w.Result().Cookies() {
|
||||||
|
if c.Name == session.SessionName {
|
||||||
|
fresh = c
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
require.NotNil(t, fresh, "login must set a session cookie")
|
||||||
|
assert.Equal(
|
||||||
|
t, "/hooks",
|
||||||
|
env.get("/", []*http.Cookie{fresh}).Header().Get("Location"),
|
||||||
|
"the new session cookie must authenticate",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestPagesLogin_ReturnsToTheRequestedPage is
|
||||||
|
// https://git.eeqj.de/sneak/webhooker/issues/384: a page opened while
|
||||||
|
// logged out leads to the login page, and logging in from there lands
|
||||||
|
// on that page, query included.
|
||||||
|
func TestPagesLogin_ReturnsToTheRequestedPage(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const (
|
||||||
|
username = "operator"
|
||||||
|
password = "correct-horse-battery-staple"
|
||||||
|
)
|
||||||
|
|
||||||
|
env := newTestEnv(t)
|
||||||
|
userID, _ := env.seedUser(t, username, password)
|
||||||
|
asked := "/hook/" + env.seedWebhook(t, userID).ID + "/events?page=2"
|
||||||
|
|
||||||
|
bounced := env.get(asked, nil)
|
||||||
|
require.Equal(t, http.StatusSeeOther, bounced.Code)
|
||||||
|
|
||||||
|
loginPage := bounced.Header().Get("Location")
|
||||||
|
|
||||||
|
match := regexp.MustCompile(`name="next" value="([^"]*)"`).
|
||||||
|
FindStringSubmatch(env.get(loginPage, nil).Body.String())
|
||||||
|
require.Len(t, match, 2, "the login form must carry the page")
|
||||||
|
|
||||||
|
token, cookies := env.csrfFrom(t, loginPage, nil)
|
||||||
|
|
||||||
|
form := url.Values{}
|
||||||
|
form.Set("csrf_token", token)
|
||||||
|
form.Set("username", username)
|
||||||
|
form.Set("password", password)
|
||||||
|
form.Set("next", html.UnescapeString(match[1]))
|
||||||
|
|
||||||
|
w := env.post("/pages/login", form, cookies)
|
||||||
|
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
|
assert.Equal(t, asked, w.Header().Get("Location"))
|
||||||
|
}
|
||||||
|
|
||||||
// --- /user/{username} group ---
|
// --- /user/{username} group ---
|
||||||
|
|
||||||
// TestPasswordChange_OversizeBody_RejectedAndPasswordUnchanged
|
// TestPasswordChange_OversizeBody_RejectedAndPasswordUnchanged
|
||||||
@@ -652,7 +805,344 @@ func TestPasswordChange_UnderLimit_Succeeds(t *testing.T) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// --- /source/{sourceID} group ---
|
// --- /hooks group ---
|
||||||
|
|
||||||
|
// TestHooks_ListAndNewWebhookForm gets the webhook list through the
|
||||||
|
// production router, follows both of its links to the new-webhook
|
||||||
|
// form, then submits the form to the action and with the token the
|
||||||
|
// page rendered. A mistyped route, link or form action fails here;
|
||||||
|
// the handler tests cannot catch any of them, because they never
|
||||||
|
// route a request.
|
||||||
|
func TestHooks_ListAndNewWebhookForm(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
env := newTestEnv(t)
|
||||||
|
|
||||||
|
userID, _ := env.seedUser(t, "lister", "somepassword")
|
||||||
|
cookies := env.authCookies(t, userID, "lister")
|
||||||
|
|
||||||
|
// The list shows its "Create Webhook" link only while it is empty.
|
||||||
|
createLink := env.urlFrom(
|
||||||
|
t, "/hooks", `href="([^"]+)"[^>]*>Create Webhook<`, cookies,
|
||||||
|
)
|
||||||
|
|
||||||
|
existing := env.seedWebhook(t, userID)
|
||||||
|
|
||||||
|
list := env.get("/hooks", cookies)
|
||||||
|
require.Equal(t, http.StatusOK, list.Code)
|
||||||
|
assert.Contains(
|
||||||
|
t, list.Body.String(), `href="/hook/`+existing.ID+`"`,
|
||||||
|
"the list should link the user's webhook",
|
||||||
|
)
|
||||||
|
|
||||||
|
// The "New Webhook" link has an icon between its href and its text.
|
||||||
|
newLink := env.urlFrom(
|
||||||
|
t, "/hooks", `href="([^"]+)"[^>]*>(?:\s*<[^>]*>)*\s*New Webhook`,
|
||||||
|
cookies,
|
||||||
|
)
|
||||||
|
|
||||||
|
token, cookies := env.csrfFrom(t, newLink, cookies)
|
||||||
|
action := env.urlFrom(t, newLink, `action="(/hooks[^"]*)"`, cookies)
|
||||||
|
assert.Equal(
|
||||||
|
t, action,
|
||||||
|
env.urlFrom(t, createLink, `action="(/hooks[^"]*)"`, cookies),
|
||||||
|
"both links should open the new-webhook form",
|
||||||
|
)
|
||||||
|
|
||||||
|
form := url.Values{}
|
||||||
|
form.Set("csrf_token", token)
|
||||||
|
form.Set("name", "created")
|
||||||
|
|
||||||
|
w := env.post(action, form, cookies)
|
||||||
|
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
|
|
||||||
|
var created database.Webhook
|
||||||
|
|
||||||
|
require.NoError(t,
|
||||||
|
env.db.DB().Where("name = ?", "created").First(&created).Error,
|
||||||
|
)
|
||||||
|
assert.Equal(
|
||||||
|
t, "/hook/"+created.ID, w.Header().Get("Location"),
|
||||||
|
"creating a webhook should redirect to its page",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- /hook/{sourceID} group ---
|
||||||
|
|
||||||
|
// TestHook_EditFormAndDelete follows the webhook page's Edit link to
|
||||||
|
// the edit form and submits it, then deletes the webhook with the
|
||||||
|
// form on its page, every URL and token taken from the rendered
|
||||||
|
// pages.
|
||||||
|
func TestHook_EditFormAndDelete(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
env := newTestEnv(t)
|
||||||
|
|
||||||
|
userID, _ := env.seedUser(t, "editor", "somepassword")
|
||||||
|
cookies := env.authCookies(t, userID, "editor")
|
||||||
|
wh := env.seedWebhook(t, userID)
|
||||||
|
page := "/hook/" + wh.ID
|
||||||
|
|
||||||
|
editPage := env.urlFrom(t, page, `href="(/hook/[^/"]+/edit)"`, cookies)
|
||||||
|
token, cookies := env.csrfFrom(t, editPage, cookies)
|
||||||
|
|
||||||
|
form := url.Values{}
|
||||||
|
form.Set("csrf_token", token)
|
||||||
|
form.Set("name", "renamed")
|
||||||
|
|
||||||
|
w := env.post(
|
||||||
|
env.urlFrom(t, editPage, `action="(/hook/[^/"]+/edit)"`, cookies),
|
||||||
|
form, cookies,
|
||||||
|
)
|
||||||
|
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
|
assert.Equal(t, page, w.Header().Get("Location"))
|
||||||
|
|
||||||
|
var edited database.Webhook
|
||||||
|
|
||||||
|
require.NoError(t, env.db.DB().First(&edited, "id = ?", wh.ID).Error)
|
||||||
|
assert.Equal(t, "renamed", edited.Name)
|
||||||
|
|
||||||
|
form = url.Values{}
|
||||||
|
form.Set("csrf_token", token)
|
||||||
|
|
||||||
|
w = env.post(
|
||||||
|
env.urlFrom(t, page, `action="(/hook/[^/"]+/delete)"`, cookies),
|
||||||
|
form, cookies,
|
||||||
|
)
|
||||||
|
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
|
assert.Equal(t, "/hooks", w.Header().Get("Location"))
|
||||||
|
assert.Equal(
|
||||||
|
t, http.StatusNotFound, env.get(page, cookies).Code,
|
||||||
|
"a deleted webhook's page should be gone",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestHook_EntrypointActions adds, deactivates and deletes an
|
||||||
|
// entrypoint with the forms on the webhook page, each submitted to
|
||||||
|
// the action and with the token the page rendered.
|
||||||
|
func TestHook_EntrypointActions(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
env := newTestEnv(t)
|
||||||
|
|
||||||
|
userID, _ := env.seedUser(t, "epuser", "somepassword")
|
||||||
|
cookies := env.authCookies(t, userID, "epuser")
|
||||||
|
wh := env.seedWebhook(t, userID)
|
||||||
|
page := "/hook/" + wh.ID
|
||||||
|
|
||||||
|
token, cookies := env.csrfFrom(t, page, cookies)
|
||||||
|
|
||||||
|
form := url.Values{}
|
||||||
|
form.Set("csrf_token", token)
|
||||||
|
|
||||||
|
// submit posts the webhook page's form whose action pattern
|
||||||
|
// captures, and requires the redirect back to that page.
|
||||||
|
submit := func(pattern string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
w := env.post(env.urlFrom(t, page, pattern, cookies), form, cookies)
|
||||||
|
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
|
require.Equal(t, page, w.Header().Get("Location"))
|
||||||
|
}
|
||||||
|
|
||||||
|
submit(`action="(/hook/[^/"]+/entrypoints)"`)
|
||||||
|
|
||||||
|
var added database.Entrypoint
|
||||||
|
|
||||||
|
require.NoError(t,
|
||||||
|
env.db.DB().First(&added, "webhook_id = ?", wh.ID).Error,
|
||||||
|
)
|
||||||
|
require.True(t, added.Active)
|
||||||
|
|
||||||
|
submit(`action="(/hook/[^/"]+/entrypoints/[^/"]+/toggle)"`)
|
||||||
|
|
||||||
|
var toggled database.Entrypoint
|
||||||
|
|
||||||
|
require.NoError(t,
|
||||||
|
env.db.DB().First(&toggled, "id = ?", added.ID).Error,
|
||||||
|
)
|
||||||
|
assert.False(t, toggled.Active, "the toggle should deactivate it")
|
||||||
|
|
||||||
|
submit(`action="(/hook/[^/"]+/entrypoints/[^/"]+/delete)"`)
|
||||||
|
|
||||||
|
var left int64
|
||||||
|
|
||||||
|
require.NoError(t, env.db.DB().Model(&database.Entrypoint{}).
|
||||||
|
Where("webhook_id = ?", wh.ID).Count(&left).Error)
|
||||||
|
assert.Zero(t, left, "the delete should remove the entrypoint")
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestHook_TargetActions adds a target with the form on the webhook
|
||||||
|
// page, follows its Edit link to the target edit form and submits
|
||||||
|
// it, then deactivates and deletes it, every URL and token taken from
|
||||||
|
// the rendered pages.
|
||||||
|
func TestHook_TargetActions(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
env := newTestEnv(t)
|
||||||
|
|
||||||
|
userID, _ := env.seedUser(t, "tgtuser", "somepassword")
|
||||||
|
cookies := env.authCookies(t, userID, "tgtuser")
|
||||||
|
wh := env.seedWebhook(t, userID)
|
||||||
|
page := "/hook/" + wh.ID
|
||||||
|
|
||||||
|
token, cookies := env.csrfFrom(t, page, cookies)
|
||||||
|
|
||||||
|
// submit posts form, with the token, to the action pattern
|
||||||
|
// captures on the page at from, and requires the redirect back to
|
||||||
|
// the webhook page.
|
||||||
|
submit := func(from, pattern string, form url.Values) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
form.Set("csrf_token", token)
|
||||||
|
|
||||||
|
w := env.post(env.urlFrom(t, from, pattern, cookies), form, cookies)
|
||||||
|
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
|
require.Equal(t, page, w.Header().Get("Location"))
|
||||||
|
}
|
||||||
|
|
||||||
|
submit(page, `action="(/hook/[^/"]+/targets)"`, url.Values{
|
||||||
|
"name": {"added"},
|
||||||
|
"type": {string(database.TargetTypeLog)},
|
||||||
|
})
|
||||||
|
|
||||||
|
editPage := env.urlFrom(
|
||||||
|
t, page, `href="(/hook/[^/"]+/targets/[^/"]+/edit)"`, cookies,
|
||||||
|
)
|
||||||
|
submit(editPage, `action="(/hook/[^/"]+/targets/[^/"]+/edit)"`,
|
||||||
|
url.Values{"name": {"renamed"}})
|
||||||
|
|
||||||
|
var edited database.Target
|
||||||
|
|
||||||
|
require.NoError(t,
|
||||||
|
env.db.DB().First(&edited, "webhook_id = ?", wh.ID).Error,
|
||||||
|
)
|
||||||
|
assert.Equal(t, "renamed", edited.Name)
|
||||||
|
require.True(t, edited.Active)
|
||||||
|
|
||||||
|
submit(page, `action="(/hook/[^/"]+/targets/[^/"]+/toggle)"`,
|
||||||
|
url.Values{})
|
||||||
|
|
||||||
|
var toggled database.Target
|
||||||
|
|
||||||
|
require.NoError(t,
|
||||||
|
env.db.DB().First(&toggled, "id = ?", edited.ID).Error,
|
||||||
|
)
|
||||||
|
assert.False(t, toggled.Active, "the toggle should deactivate it")
|
||||||
|
|
||||||
|
submit(page, `action="(/hook/[^/"]+/targets/[^/"]+/delete)"`,
|
||||||
|
url.Values{})
|
||||||
|
|
||||||
|
var left int64
|
||||||
|
|
||||||
|
require.NoError(t, env.db.DB().Model(&database.Target{}).
|
||||||
|
Where("webhook_id = ?", wh.ID).Count(&left).Error)
|
||||||
|
assert.Zero(t, left, "the delete should remove the target")
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestHook_ResubmitFromEventLog follows the webhook page's "Full
|
||||||
|
// Event Log" link, then resubmits a stored event with the form on
|
||||||
|
// that page, submitted to the action and with the token the page
|
||||||
|
// rendered.
|
||||||
|
func TestHook_ResubmitFromEventLog(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
env := newTestEnv(t)
|
||||||
|
|
||||||
|
userID, _ := env.seedUser(t, "resubmitter", "somepassword")
|
||||||
|
cookies := env.authCookies(t, userID, "resubmitter")
|
||||||
|
wh := env.seedWebhook(t, userID)
|
||||||
|
env.seedEvent(t, wh.ID, `{"resubmit":"me"}`)
|
||||||
|
|
||||||
|
logsPath := env.urlFrom(
|
||||||
|
t, "/hook/"+wh.ID, `href="([^"]+)"[^>]*>Full Event Log<`, cookies,
|
||||||
|
)
|
||||||
|
|
||||||
|
token, cookies := env.csrfFrom(t, logsPath, cookies)
|
||||||
|
|
||||||
|
form := url.Values{}
|
||||||
|
form.Set("csrf_token", token)
|
||||||
|
|
||||||
|
w := env.post(
|
||||||
|
env.urlFrom(t, logsPath, `action="(/hook/[^"]+/resubmit)"`, cookies),
|
||||||
|
form, cookies,
|
||||||
|
)
|
||||||
|
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
|
assert.Equal(
|
||||||
|
t, logsPath+"?resubmit=no-targets", w.Header().Get("Location"),
|
||||||
|
)
|
||||||
|
|
||||||
|
webhookDB, err := env.dbMgr.GetDB(wh.ID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
var events int64
|
||||||
|
|
||||||
|
require.NoError(t,
|
||||||
|
webhookDB.Model(&database.Event{}).Count(&events).Error,
|
||||||
|
)
|
||||||
|
assert.Equal(t, int64(2), events, "the resubmit stores a new event")
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestHook_LinksBetweenPages follows each link to a webhook page that
|
||||||
|
// the tests above do not: the navbar's "Webhooks" links, the back and
|
||||||
|
// Cancel links, the list's link to a webhook, the "Full Event Log"
|
||||||
|
// link beside the recent events, and the event log's page links. Each
|
||||||
|
// must point where it should, and that page must render.
|
||||||
|
func TestHook_LinksBetweenPages(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
env := newTestEnv(t)
|
||||||
|
|
||||||
|
userID, _ := env.seedUser(t, "navigator", "somepassword")
|
||||||
|
cookies := env.authCookies(t, userID, "navigator")
|
||||||
|
wh := env.seedWebhook(t, userID)
|
||||||
|
tgt := env.seedTarget(t, wh.ID)
|
||||||
|
|
||||||
|
// The event log shows 25 events a page; one more gives it a second
|
||||||
|
// page, so it renders its Next and Previous links.
|
||||||
|
for range 26 {
|
||||||
|
env.seedEvent(t, wh.ID, "paged")
|
||||||
|
}
|
||||||
|
|
||||||
|
list := "/hooks"
|
||||||
|
newForm := list + "/new"
|
||||||
|
page := "/hook/" + wh.ID
|
||||||
|
targetEdit := page + "/targets/" + tgt.ID + "/edit"
|
||||||
|
events := page + "/events"
|
||||||
|
back := `href="([^"]+)"[^>]*>← Back to `
|
||||||
|
cancel := `href="([^"]+)"[^>]*>Cancel<`
|
||||||
|
|
||||||
|
for _, link := range []struct{ from, pattern, want string }{
|
||||||
|
// The navbar on the profile page: its desktop link, then its
|
||||||
|
// mobile menu link.
|
||||||
|
{
|
||||||
|
"/user/navigator/",
|
||||||
|
`href="([^"]+)" class="btn-text">Webhooks<`,
|
||||||
|
list,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"/user/navigator/",
|
||||||
|
`href="([^"]+)" class="btn-text w-full[^"]*">Webhooks<`,
|
||||||
|
list,
|
||||||
|
},
|
||||||
|
{list, `href="(/hook/[^"]+)"`, page},
|
||||||
|
{newForm, back, list},
|
||||||
|
{newForm, cancel, list},
|
||||||
|
{page, back, list},
|
||||||
|
{page, `Recent Events</h2>\s*<a href="([^"]+)"`, events},
|
||||||
|
{page + "/edit", back, page},
|
||||||
|
{page + "/edit", cancel, page},
|
||||||
|
{targetEdit, back, page},
|
||||||
|
{targetEdit, cancel, page},
|
||||||
|
{events, back, page},
|
||||||
|
{events, `href="([^"]+)"[^>]*>Next →<`, events + "?page=2"},
|
||||||
|
{events + "?page=2", `href="([^"]+)"[^>]*>← Previous<`, events + "?page=1"},
|
||||||
|
} {
|
||||||
|
got := env.urlFrom(t, link.from, link.pattern, cookies)
|
||||||
|
assert.Equal(t, link.want, got, "%s: %s", link.from, link.pattern)
|
||||||
|
assert.Equal(t, http.StatusOK, env.get(got, cookies).Code, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestSourceLogs_TruncationLinkDownloadsTheBody walks the whole
|
// TestSourceLogs_TruncationLinkDownloadsTheBody walks the whole
|
||||||
// feature the way a user does: render the event log page through
|
// feature the way a user does: render the event log page through
|
||||||
@@ -680,11 +1170,11 @@ func TestSourceLogs_TruncationLinkDownloadsTheBody(t *testing.T) {
|
|||||||
wh := env.seedWebhook(t, userID)
|
wh := env.seedWebhook(t, userID)
|
||||||
env.seedEvent(t, wh.ID, stored)
|
env.seedEvent(t, wh.ID, stored)
|
||||||
|
|
||||||
page := env.get("/source/"+wh.ID+"/logs", cookies)
|
page := env.get("/hook/"+wh.ID+"/events", cookies)
|
||||||
require.Equal(t, http.StatusOK, page.Code)
|
require.Equal(t, http.StatusOK, page.Code)
|
||||||
|
|
||||||
link := regexp.MustCompile(
|
link := regexp.MustCompile(
|
||||||
`href="(/source/[^"]+/body)"`,
|
`href="(/hook/[^"]+/body)"`,
|
||||||
).FindStringSubmatch(page.Body.String())
|
).FindStringSubmatch(page.Body.String())
|
||||||
require.Len(
|
require.Len(
|
||||||
t, link, 2,
|
t, link, 2,
|
||||||
@@ -730,7 +1220,7 @@ func TestSourceLogsBody_OtherUser404s(t *testing.T) {
|
|||||||
const payload = "OWNERS-PAYLOAD-77c1"
|
const payload = "OWNERS-PAYLOAD-77c1"
|
||||||
|
|
||||||
evt := env.seedEvent(t, wh.ID, payload)
|
evt := env.seedEvent(t, wh.ID, payload)
|
||||||
path := "/source/" + wh.ID + "/logs/" + evt.ID + "/body"
|
path := "/hook/" + wh.ID + "/events/" + evt.ID + "/body"
|
||||||
|
|
||||||
intruderID, _ := env.seedUser(t, "intruder", "somepassword")
|
intruderID, _ := env.seedUser(t, "intruder", "somepassword")
|
||||||
intruder := env.authCookies(t, intruderID, "intruder")
|
intruder := env.authCookies(t, intruderID, "intruder")
|
||||||
@@ -741,7 +1231,10 @@ func TestSourceLogsBody_OtherUser404s(t *testing.T) {
|
|||||||
|
|
||||||
anon := env.get(path, nil)
|
anon := env.get(path, nil)
|
||||||
assert.Equal(t, http.StatusSeeOther, anon.Code)
|
assert.Equal(t, http.StatusSeeOther, anon.Code)
|
||||||
assert.Equal(t, "/pages/login", anon.Header().Get("Location"))
|
assert.Equal(
|
||||||
|
t, "/pages/login?next="+url.QueryEscape(path),
|
||||||
|
anon.Header().Get("Location"),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestDeliveryReplay_PostOnlyAndCSRFProtected walks the replay action
|
// TestDeliveryReplay_PostOnlyAndCSRFProtected walks the replay action
|
||||||
@@ -764,7 +1257,7 @@ func TestDeliveryReplay_PostOnlyAndCSRFProtected(t *testing.T) {
|
|||||||
evt := env.seedEvent(t, wh.ID, `{"replay":"me"}`)
|
evt := env.seedEvent(t, wh.ID, `{"replay":"me"}`)
|
||||||
dlv := env.seedFailedDelivery(t, wh.ID, evt.ID, tgt.ID)
|
dlv := env.seedFailedDelivery(t, wh.ID, evt.ID, tgt.ID)
|
||||||
|
|
||||||
path := "/source/" + wh.ID + "/deliveries/" + dlv.ID +
|
path := "/hook/" + wh.ID + "/deliveries/" + dlv.ID +
|
||||||
"/replay"
|
"/replay"
|
||||||
|
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
@@ -790,7 +1283,7 @@ func TestDeliveryReplay_PostOnlyAndCSRFProtected(t *testing.T) {
|
|||||||
// The token and the action URL both come out of the rendered
|
// The token and the action URL both come out of the rendered
|
||||||
// page, so a typo in either the route pattern or the template
|
// page, so a typo in either the route pattern or the template
|
||||||
// fails here.
|
// fails here.
|
||||||
logsPath := "/source/" + wh.ID + "/logs"
|
logsPath := "/hook/" + wh.ID + "/events"
|
||||||
|
|
||||||
token, cookies := env.csrfFrom(t, logsPath, cookies)
|
token, cookies := env.csrfFrom(t, logsPath, cookies)
|
||||||
|
|
||||||
@@ -798,7 +1291,7 @@ func TestDeliveryReplay_PostOnlyAndCSRFProtected(t *testing.T) {
|
|||||||
require.Equal(t, http.StatusOK, page.Code)
|
require.Equal(t, http.StatusOK, page.Code)
|
||||||
|
|
||||||
action := regexp.MustCompile(
|
action := regexp.MustCompile(
|
||||||
`action="(/source/[^"]+/replay)"`,
|
`action="(/hook/[^"]+/replay)"`,
|
||||||
).FindStringSubmatch(page.Body.String())
|
).FindStringSubmatch(page.Body.String())
|
||||||
require.Len(
|
require.Len(
|
||||||
t, action, 2,
|
t, action, 2,
|
||||||
@@ -823,6 +1316,59 @@ func TestDeliveryReplay_PostOnlyAndCSRFProtected(t *testing.T) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// --- /h/{uuid} receiver ---
|
||||||
|
|
||||||
|
// TestReceiver_EntrypointURLIsRateLimited takes the entrypoint URL
|
||||||
|
// the webhook page shows and posts to it through the production
|
||||||
|
// router until the receiver rate limit refuses it. The URL has to
|
||||||
|
// reach the receiver, and the limit has to apply to it.
|
||||||
|
func TestReceiver_EntrypointURLIsRateLimited(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const limit = 2
|
||||||
|
|
||||||
|
env := newTestEnvWithConfig(t, &config.Config{
|
||||||
|
DataDir: t.TempDir(),
|
||||||
|
Environment: config.EnvironmentDev,
|
||||||
|
ReceiverRateLimit: limit,
|
||||||
|
})
|
||||||
|
|
||||||
|
userID, _ := env.seedUser(t, "receiver", "somepassword")
|
||||||
|
cookies := env.authCookies(t, userID, "receiver")
|
||||||
|
|
||||||
|
wh := env.seedWebhook(t, userID)
|
||||||
|
require.NoError(t, env.db.DB().Omit(clause.Associations).Create(
|
||||||
|
&database.Entrypoint{
|
||||||
|
WebhookID: wh.ID,
|
||||||
|
Path: "6f1e2a9c-4b7d-4e3a-9c2f-1d8b5a7e3c60",
|
||||||
|
Active: true,
|
||||||
|
},
|
||||||
|
).Error)
|
||||||
|
|
||||||
|
page := env.get("/hook/"+wh.ID, cookies)
|
||||||
|
require.Equal(t, http.StatusOK, page.Code)
|
||||||
|
|
||||||
|
shown := regexp.MustCompile(`(/h/[^<]+)</code>`).
|
||||||
|
FindStringSubmatch(page.Body.String())
|
||||||
|
require.Len(
|
||||||
|
t, shown, 2, "the webhook page should show the entrypoint URL",
|
||||||
|
)
|
||||||
|
|
||||||
|
for i := range limit {
|
||||||
|
assert.Equal(
|
||||||
|
t, http.StatusOK,
|
||||||
|
env.post(shown[1], url.Values{}, nil).Code,
|
||||||
|
"request %d should reach the receiver", i,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.Equal(
|
||||||
|
t, http.StatusTooManyRequests,
|
||||||
|
env.post(shown[1], url.Values{}, nil).Code,
|
||||||
|
"the receiver rate limit must apply to the entrypoint URL",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
// metricsConfig is a Config differing from the routing default only
|
// metricsConfig is a Config differing from the routing default only
|
||||||
// in the two /metrics credentials.
|
// in the two /metrics credentials.
|
||||||
func metricsConfig(
|
func metricsConfig(
|
||||||
|
|||||||
@@ -55,7 +55,7 @@ func sentryClientOptions(dsn, release string) sentry.ClientOptions {
|
|||||||
//
|
//
|
||||||
// URL is the third such field. NewRequest builds it as
|
// URL is the third such field. NewRequest builds it as
|
||||||
// scheme://host/path (interfaces.go:183), and on the receiver route
|
// scheme://host/path (interfaces.go:183), and on the receiver route
|
||||||
// that path is /webhook/<uuid> in full — a write capability, not an
|
// that path is /h/<uuid> in full — a write capability, not an
|
||||||
// identifier. It is rebuilt here from the chi route pattern, on every
|
// identifier. It is rebuilt here from the chi route pattern, on every
|
||||||
// route, keeping the scheme and the host.
|
// route, keeping the scheme and the host.
|
||||||
//
|
//
|
||||||
|
|||||||
@@ -153,7 +153,7 @@ func (c sentryCase) router() http.Handler {
|
|||||||
sentryhttp.New(sentryhttp.Options{Repanic: true}).Handle,
|
sentryhttp.New(sentryhttp.Options{Repanic: true}).Handle,
|
||||||
)
|
)
|
||||||
router.HandleFunc("/pages/login", handler)
|
router.HandleFunc("/pages/login", handler)
|
||||||
router.HandleFunc("/webhook/{uuid}", handler)
|
router.HandleFunc("/h/{uuid}", handler)
|
||||||
|
|
||||||
return router
|
return router
|
||||||
}
|
}
|
||||||
@@ -191,7 +191,7 @@ func sentryLoginRequest(client *sentry.Client) *http.Request {
|
|||||||
// concrete path carries the entrypoint capability.
|
// concrete path carries the entrypoint capability.
|
||||||
func sentryReceiverRequest(client *sentry.Client) *http.Request {
|
func sentryReceiverRequest(client *sentry.Client) *http.Request {
|
||||||
return sentryRequest(
|
return sentryRequest(
|
||||||
client, "/webhook/"+sentryReceiverUUID, "payload=hello",
|
client, "/h/"+sentryReceiverUUID, "payload=hello",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -316,7 +316,7 @@ func TestSentryScrub_ReplacesTheCapabilityPathWithTheRoutePattern(
|
|||||||
t, marshalEvent(t, event), sentryReceiverUUID,
|
t, marshalEvent(t, event), sentryReceiverUUID,
|
||||||
)
|
)
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t, "http://example.com/webhook/{uuid}", event.Request.URL,
|
t, "http://example.com/h/{uuid}", event.Request.URL,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -401,7 +401,7 @@ func TestSentryScrub_TransactionDispatchIsUnscrubbedWithoutTheHook(
|
|||||||
func TestSentryScrub_FallsBackWithoutARoutePattern(t *testing.T) {
|
func TestSentryScrub_FallsBackWithoutARoutePattern(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
concrete := "https://example.com/webhook/" + sentryReceiverUUID
|
concrete := "https://example.com/h/" + sentryReceiverUUID
|
||||||
|
|
||||||
// A request with no chi routing context on it at all, which is
|
// A request with no chi routing context on it at all, which is
|
||||||
// what an event captured outside the router would carry.
|
// what an event captured outside the router would carry.
|
||||||
@@ -426,7 +426,7 @@ func TestSentryScrub_FallsBackWithoutARoutePattern(t *testing.T) {
|
|||||||
|
|
||||||
event := sentry.NewEvent()
|
event := sentry.NewEvent()
|
||||||
event.Request = &sentry.Request{URL: concrete}
|
event.Request = &sentry.Request{URL: concrete}
|
||||||
event.Transaction = "POST /webhook/" +
|
event.Transaction = "POST /h/" +
|
||||||
sentryReceiverUUID
|
sentryReceiverUUID
|
||||||
|
|
||||||
scrubbed := server.ScrubSentryRequestForTest(
|
scrubbed := server.ScrubSentryRequestForTest(
|
||||||
@@ -459,9 +459,9 @@ func TestSentryScrub_WithholdsUnparseableValues(t *testing.T) {
|
|||||||
|
|
||||||
event := sentry.NewEvent()
|
event := sentry.NewEvent()
|
||||||
event.Request = &sentry.Request{
|
event.Request = &sentry.Request{
|
||||||
URL: "/webhook/" + sentryReceiverUUID,
|
URL: "/h/" + sentryReceiverUUID,
|
||||||
}
|
}
|
||||||
event.Transaction = "/webhook/" + sentryReceiverUUID
|
event.Transaction = "/h/" + sentryReceiverUUID
|
||||||
|
|
||||||
scrubbed := server.ScrubSentryRequestForTest(event, nil)
|
scrubbed := server.ScrubSentryRequestForTest(event, nil)
|
||||||
require.NotNil(t, scrubbed)
|
require.NotNil(t, scrubbed)
|
||||||
|
|||||||
@@ -13,9 +13,9 @@ import (
|
|||||||
|
|
||||||
// TestBaseTemplateScriptsAreServed walks every /s/ script the base
|
// TestBaseTemplateScriptsAreServed walks every /s/ script the base
|
||||||
// template loads on each page and fetches it through the real router.
|
// template loads on each page and fetches it through the real router.
|
||||||
// Alpine.js is fetched at build time rather than committed, so nothing
|
// Alpine.js is extracted from its tarball in 3p/ at build time, so the
|
||||||
// in the repo guarantees it is present: this is the check that the page
|
// file is not in the tree: this is the check that the page still gets
|
||||||
// still gets the JavaScript it asks for.
|
// the JavaScript it asks for.
|
||||||
func TestBaseTemplateScriptsAreServed(t *testing.T) {
|
func TestBaseTemplateScriptsAreServed(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -19,8 +19,8 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// The tests below exercise the securecookie codecs underneath the
|
// The tests below exercise the securecookie codecs underneath the
|
||||||
// store and nothing else: Session.Get only decodes, so no server-side
|
// store and nothing else: they decode through the store itself, so no
|
||||||
// expiry check takes part in the result. They exist because
|
// server-side expiry check takes part in the result. They exist because
|
||||||
// NewCookieStore gives its codecs a 30-day max age that assigning
|
// NewCookieStore gives its codecs a 30-day max age that assigning
|
||||||
// store.Options does not override, which would let the codec accept a
|
// store.Options does not override, which would let the codec accept a
|
||||||
// cookie weeks past the cap the cookie attribute advertises.
|
// cookie weeks past the cap the cookie attribute advertises.
|
||||||
@@ -75,10 +75,11 @@ func restamp(
|
|||||||
return base64.URLEncoding.EncodeToString(payload)
|
return base64.URLEncoding.EncodeToString(payload)
|
||||||
}
|
}
|
||||||
|
|
||||||
// decodeCookie feeds value back through the store's decode path.
|
// decodeCookie feeds value back through the store's decode path. It
|
||||||
|
// asks the store rather than Session.Get, which treats a cookie that
|
||||||
|
// does not decode as absent and so hides the codec's reason.
|
||||||
func decodeCookie(
|
func decodeCookie(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
s *session.Session,
|
|
||||||
value string,
|
value string,
|
||||||
) (*sessions.Session, error) {
|
) (*sessions.Session, error) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
@@ -94,7 +95,7 @@ func decodeCookie(
|
|||||||
SameSite: http.SameSiteLaxMode,
|
SameSite: http.SameSiteLaxMode,
|
||||||
})
|
})
|
||||||
|
|
||||||
sess, err := s.Get(req)
|
sess, err := session.NewStore(testKey()).Get(req, session.SessionName)
|
||||||
require.NotNil(t, sess)
|
require.NotNil(t, sess)
|
||||||
|
|
||||||
return sess, err
|
return sess, err
|
||||||
@@ -105,7 +106,7 @@ func TestCodec_AcceptsCookieInsideAbsoluteCap(t *testing.T) {
|
|||||||
|
|
||||||
s := testSession(t)
|
s := testSession(t)
|
||||||
|
|
||||||
sess, err := decodeCookie(t, s, restamp(
|
sess, err := decodeCookie(t, restamp(
|
||||||
t,
|
t,
|
||||||
issuedCookie(t, s),
|
issuedCookie(t, s),
|
||||||
time.Now().Add(-(testAbsoluteMaxAge-time.Hour)),
|
time.Now().Add(-(testAbsoluteMaxAge-time.Hour)),
|
||||||
@@ -126,7 +127,7 @@ func TestCodec_RejectsCookiePastAbsoluteCap(t *testing.T) {
|
|||||||
|
|
||||||
s := testSession(t)
|
s := testSession(t)
|
||||||
|
|
||||||
sess, err := decodeCookie(t, s, restamp(
|
sess, err := decodeCookie(t, restamp(
|
||||||
t,
|
t,
|
||||||
issuedCookie(t, s),
|
issuedCookie(t, s),
|
||||||
time.Now().Add(-(testAbsoluteMaxAge+time.Hour)),
|
time.Now().Add(-(testAbsoluteMaxAge+time.Hour)),
|
||||||
|
|||||||
@@ -224,10 +224,22 @@ func New(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Get retrieves a session for the request.
|
// Get retrieves a session for the request.
|
||||||
|
//
|
||||||
|
// A session cookie that does not decode -- one signed with an earlier
|
||||||
|
// session key, say, because the database was made anew -- is treated
|
||||||
|
// as absent: the caller gets a new, empty session and no error, and
|
||||||
|
// the next save replaces the cookie.
|
||||||
func (s *Session) Get(
|
func (s *Session) Get(
|
||||||
r *http.Request,
|
r *http.Request,
|
||||||
) (*sessions.Session, error) {
|
) (*sessions.Session, error) {
|
||||||
return s.store.Get(r, SessionName)
|
sess, err := s.store.Get(r, SessionName)
|
||||||
|
if sess == nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// For a cookie that does not decode, gorilla/sessions returns a
|
||||||
|
// new, empty session alongside the error that is dropped here.
|
||||||
|
return sess, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetKey returns the raw 32-byte authentication key used for
|
// GetKey returns the raw 32-byte authentication key used for
|
||||||
|
|||||||
Executable
+16
@@ -0,0 +1,16 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# script/assets: extract Alpine.js from its npm package tarball, committed
|
||||||
|
# in 3p/, to static/js/alpine.min.js, where go:embed reads it. The
|
||||||
|
# extracted file is not committed. script/test, make build and make dev run
|
||||||
|
# this first.
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
|
|
||||||
|
main() {
|
||||||
|
cd "$ROOT"
|
||||||
|
tar -xzOf 3p/alpinejs-3.14.9.tgz package/dist/cdn.min.js \
|
||||||
|
>static/js/alpine.min.js
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
+1
-8
@@ -4,9 +4,7 @@
|
|||||||
# installed tools are skipped. Base tooling comes from nix, apt, brew,
|
# installed tools are skipped. Base tooling comes from nix, apt, brew,
|
||||||
# or apk (detected in that order); assumes NOTHING is present (not git,
|
# or apk (detected in that order); assumes NOTHING is present (not git,
|
||||||
# make, or go). golangci-lint is deliberately not installed: linting runs
|
# make, or go). golangci-lint is deliberately not installed: linting runs
|
||||||
# only in docker, via script/lint and Dockerfile.lint. Finishes by running
|
# only in docker, via script/lint and Dockerfile.lint.
|
||||||
# script/fetch-assets, which installs the hash-pinned third-party browser
|
|
||||||
# assets the repo does not commit.
|
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
@@ -69,11 +67,6 @@ main() {
|
|||||||
|
|
||||||
go mod download
|
go mod download
|
||||||
|
|
||||||
# Third-party browser assets are not committed; fetch and verify them
|
|
||||||
# so a fresh clone can build and test.
|
|
||||||
if missing curl; then pkg_install curl curl curl curl; fi
|
|
||||||
"$ROOT/script/fetch-assets"
|
|
||||||
|
|
||||||
echo "bootstrap complete"
|
echo "bootstrap complete"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+2
-1
@@ -1,6 +1,7 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
# script/check: run all checks (test, lint, fmt-check). Our own
|
# script/check: run all checks (test, lint, fmt-check). Our own
|
||||||
# extension to scripts-to-rule-them-all. Must not modify any files.
|
# extension to scripts-to-rule-them-all.
|
||||||
|
# Writes only the ignored static/js/alpine.min.js, through script/test.
|
||||||
# Generic: usually needs no adaptation.
|
# Generic: usually needs no adaptation.
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
|
|||||||
@@ -1,104 +0,0 @@
|
|||||||
#!/bin/sh
|
|
||||||
# script/fetch-assets: download the third-party browser assets the web UI
|
|
||||||
# ships and install them under static/. Minified bundles are not committed
|
|
||||||
# (REPO_POLICIES.md: no build artifacts in version control), so the build
|
|
||||||
# fetches them here. Every download is verified against a hardcoded sha256
|
|
||||||
# before it is installed, and any mismatch aborts. Idempotent: an asset
|
|
||||||
# already present with its pinned hash is left alone.
|
|
||||||
set -eu
|
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
|
||||||
|
|
||||||
# The sha256 of each installed asset lives in static/vendor.sha256, in
|
|
||||||
# sha256sum(1) format, with paths relative to static/. That file is the
|
|
||||||
# single source of truth: this script verifies against it, and
|
|
||||||
# static/vendor_test.go asserts the bytes embedded into the binary match
|
|
||||||
# it, so the hash cannot rot into a value nothing checks.
|
|
||||||
MANIFEST="static/vendor.sha256"
|
|
||||||
|
|
||||||
# Alpine.js 3.14.9, 2026-08-17. Fetched from registry.npmjs.org, the
|
|
||||||
# publisher of record; the jsDelivr and unpkg copies are mirrors of this
|
|
||||||
# same tarball. dist/cdn.min.js is the browser build Alpine publishes for
|
|
||||||
# a <script> tag.
|
|
||||||
ALPINE_VERSION="3.14.9"
|
|
||||||
ALPINE_URL="https://registry.npmjs.org/alpinejs/-/alpinejs-${ALPINE_VERSION}.tgz"
|
|
||||||
# sha256 of alpinejs-3.14.9.tgz
|
|
||||||
ALPINE_TARBALL_SHA256="97dad7c0c81e659cfc8e7700055da9770f8186187cb9a8a76efb57e00d5ce52a"
|
|
||||||
ALPINE_MEMBER="package/dist/cdn.min.js"
|
|
||||||
ALPINE_DEST="js/alpine.min.js"
|
|
||||||
|
|
||||||
sha256_of() {
|
|
||||||
if command -v sha256sum >/dev/null 2>&1; then
|
|
||||||
sha256sum "$1" | cut -d' ' -f1
|
|
||||||
else
|
|
||||||
shasum -a 256 "$1" | cut -d' ' -f1
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# expected_sha256 <path-relative-to-static>
|
|
||||||
expected_sha256() {
|
|
||||||
awk -v want="$1" '$2 == want { print $1; found = 1 }
|
|
||||||
END { if (!found) exit 1 }' "$ROOT/$MANIFEST"
|
|
||||||
}
|
|
||||||
|
|
||||||
# verify <file> <expected-sha256> <what>
|
|
||||||
verify() {
|
|
||||||
actual="$(sha256_of "$1")"
|
|
||||||
if [ "$actual" != "$2" ]; then
|
|
||||||
echo "fetch-assets: sha256 mismatch for $3" >&2
|
|
||||||
echo " expected: $2" >&2
|
|
||||||
echo " actual: $actual" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# up_to_date <path-relative-to-static> <expected-sha256>
|
|
||||||
up_to_date() {
|
|
||||||
[ -f "$ROOT/static/$1" ] || return 1
|
|
||||||
[ "$(sha256_of "$ROOT/static/$1")" = "$2" ]
|
|
||||||
}
|
|
||||||
|
|
||||||
fetch_alpine() {
|
|
||||||
want="$(expected_sha256 "$ALPINE_DEST")"
|
|
||||||
|
|
||||||
if up_to_date "$ALPINE_DEST" "$want"; then
|
|
||||||
echo "fetch-assets: static/$ALPINE_DEST already at $want"
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "fetch-assets: fetching Alpine.js $ALPINE_VERSION from $ALPINE_URL"
|
|
||||||
tmp="$(mktemp -d)"
|
|
||||||
trap 'rm -rf "$tmp"' EXIT INT TERM
|
|
||||||
curl -fsSL -o "$tmp/alpine.tgz" "$ALPINE_URL"
|
|
||||||
verify "$tmp/alpine.tgz" "$ALPINE_TARBALL_SHA256" "alpinejs-${ALPINE_VERSION}.tgz"
|
|
||||||
tar -xzOf "$tmp/alpine.tgz" "$ALPINE_MEMBER" >"$tmp/alpine.min.js"
|
|
||||||
verify "$tmp/alpine.min.js" "$want" "$ALPINE_MEMBER from alpinejs-${ALPINE_VERSION}.tgz"
|
|
||||||
|
|
||||||
mkdir -p "$(dirname "$ROOT/static/$ALPINE_DEST")"
|
|
||||||
cp "$tmp/alpine.min.js" "$ROOT/static/$ALPINE_DEST"
|
|
||||||
rm -rf "$tmp"
|
|
||||||
trap - EXIT INT TERM
|
|
||||||
echo "fetch-assets: installed static/$ALPINE_DEST ($want)"
|
|
||||||
}
|
|
||||||
|
|
||||||
# Re-check every manifest entry against what is now on disk, so an entry
|
|
||||||
# no script installs fails loudly instead of passing silently.
|
|
||||||
verify_manifest() {
|
|
||||||
while read -r want path; do
|
|
||||||
case "$want" in '' | '#'*) continue ;; esac
|
|
||||||
if [ ! -f "$ROOT/static/$path" ]; then
|
|
||||||
echo "fetch-assets: $MANIFEST lists static/$path, which is missing" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
verify "$ROOT/static/$path" "$want" "static/$path"
|
|
||||||
done <"$ROOT/$MANIFEST"
|
|
||||||
}
|
|
||||||
|
|
||||||
main() {
|
|
||||||
cd "$ROOT"
|
|
||||||
fetch_alpine
|
|
||||||
verify_manifest
|
|
||||||
echo "fetch-assets: all assets in $MANIFEST verified"
|
|
||||||
}
|
|
||||||
|
|
||||||
main "$@"
|
|
||||||
+12
-1
@@ -22,13 +22,24 @@
|
|||||||
# The one figure above 90s is GOMAXPROCS 1, a synthetic core floor rather than
|
# The one figure above 90s is GOMAXPROCS 1, a synthetic core floor rather than
|
||||||
# a condition CI runs under. If a CPU-limited runner ever puts a real run near
|
# a condition CI runs under. If a CPU-limited runner ever puts a real run near
|
||||||
# 67s, that is the datum to revisit the org figure with.
|
# 67s, that is the datum to revisit the org figure with.
|
||||||
|
#
|
||||||
|
# -p 4 -parallel 8 keep the run under 2 GB of memory: at most four test
|
||||||
|
# binaries build or run at once, each with at most eight parallel tests. Under
|
||||||
|
# -race every test binary and every link costs a few hundred MB, so the
|
||||||
|
# defaults (one per core) add up to several GB on a many-core host.
|
||||||
|
#
|
||||||
|
# No -v: the Docker build cuts each step's log off at 2 MiB, and verbose output
|
||||||
|
# from the whole suite passes that before a failure is printed. Without it, go
|
||||||
|
# test prints one result line per package and, for a package that fails,
|
||||||
|
# everything its tests wrote, application log lines included.
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
|
|
||||||
main() {
|
main() {
|
||||||
cd "$ROOT"
|
cd "$ROOT"
|
||||||
go test -v -race -timeout 90s ./...
|
"$ROOT/script/assets"
|
||||||
|
go test -race -p 4 -parallel 8 -timeout 90s ./...
|
||||||
}
|
}
|
||||||
|
|
||||||
main "$@"
|
main "$@"
|
||||||
|
|||||||
@@ -1 +0,0 @@
|
|||||||
3ed1eed252488921df65e363d6715deb04d7f92aaedb9e52199fdf73cb1e0ad3 js/alpine.min.js
|
|
||||||
@@ -1,92 +0,0 @@
|
|||||||
package static_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bufio"
|
|
||||||
"crypto/sha256"
|
|
||||||
"encoding/hex"
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
|
|
||||||
"sneak.berlin/go/webhooker/static"
|
|
||||||
)
|
|
||||||
|
|
||||||
const manifestPath = "vendor.sha256"
|
|
||||||
|
|
||||||
// fetchHint is appended to every failure here: the assets the manifest
|
|
||||||
// covers are fetched by the build, not committed, so a fresh clone that
|
|
||||||
// has not run script/fetch-assets fails this test and should be told why.
|
|
||||||
const fetchHint = "run `script/fetch-assets` (or `make assets`) to install " +
|
|
||||||
"the pinned third-party assets"
|
|
||||||
|
|
||||||
// TestVendoredAssetsMatchManifest asserts that every asset listed in
|
|
||||||
// static/vendor.sha256 is embedded in the binary with exactly the pinned
|
|
||||||
// bytes. script/fetch-assets verifies the same hashes at download time;
|
|
||||||
// this test verifies them again on what actually ships, so a build that
|
|
||||||
// skipped, cached, or subverted the fetch cannot produce a binary serving
|
|
||||||
// unpinned third-party JavaScript.
|
|
||||||
func TestVendoredAssetsMatchManifest(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
entries := readManifest(t)
|
|
||||||
require.NotEmpty(t, entries, "%s lists no assets", manifestPath)
|
|
||||||
|
|
||||||
for path, want := range entries {
|
|
||||||
t.Run(path, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
data, err := static.Static.ReadFile(path)
|
|
||||||
require.NoErrorf(
|
|
||||||
t, err,
|
|
||||||
"%s is listed in %s but is not embedded; %s",
|
|
||||||
path, manifestPath, fetchHint,
|
|
||||||
)
|
|
||||||
|
|
||||||
sum := sha256.Sum256(data)
|
|
||||||
got := hex.EncodeToString(sum[:])
|
|
||||||
require.Equalf(
|
|
||||||
t, want, got,
|
|
||||||
"embedded %s does not match its pinned sha256 in %s; %s",
|
|
||||||
path, manifestPath, fetchHint,
|
|
||||||
)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// readManifest parses static/vendor.sha256, which is in sha256sum(1)
|
|
||||||
// format with paths relative to static/.
|
|
||||||
func readManifest(t *testing.T) map[string]string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
f, err := os.Open(manifestPath)
|
|
||||||
require.NoError(t, err, "opening %s", manifestPath)
|
|
||||||
|
|
||||||
defer func() { require.NoError(t, f.Close()) }()
|
|
||||||
|
|
||||||
entries := make(map[string]string)
|
|
||||||
scanner := bufio.NewScanner(f)
|
|
||||||
|
|
||||||
for scanner.Scan() {
|
|
||||||
line := strings.TrimSpace(scanner.Text())
|
|
||||||
if line == "" || strings.HasPrefix(line, "#") {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
fields := strings.Fields(line)
|
|
||||||
require.Lenf(
|
|
||||||
t, fields, 2,
|
|
||||||
"%s: malformed entry %q, want \"<sha256> <path>\"",
|
|
||||||
manifestPath, line,
|
|
||||||
)
|
|
||||||
|
|
||||||
sum, path := fields[0], fields[1]
|
|
||||||
require.Lenf(t, sum, 64, "%s: %q is not a sha256", manifestPath, sum)
|
|
||||||
entries[path] = sum
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NoError(t, scanner.Err(), "reading %s", manifestPath)
|
|
||||||
|
|
||||||
return entries
|
|
||||||
}
|
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user