Compare commits
48
Commits
ddbe8cd8df
...
next
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
74a96b2226 | ||
|
|
8b617efa63 | ||
|
|
d8c60c9b67 | ||
|
|
ea8384f4a2 | ||
|
|
17e6c85dd8 | ||
|
|
bcdd4791ec | ||
|
|
f1da5e73dd | ||
|
|
d2ecb83923 | ||
|
|
643077021d | ||
|
|
22fa502638 | ||
|
|
6395210474 | ||
|
|
3489d6909a | ||
|
|
f282c6363d | ||
|
|
61371d388e | ||
|
|
19a6705c63 | ||
|
|
93911f28f9 | ||
|
|
9305af4f85 | ||
|
|
ff24638ba4 | ||
|
|
da75950e91 | ||
|
|
719d7013ee | ||
|
|
faf7ca1a5e | ||
|
|
0f5f6ba6bf | ||
|
|
820d9391ff | ||
|
|
2967c475a1 | ||
|
|
35d2f28c67 | ||
|
|
9526e961b5 | ||
|
|
4915d60d8e | ||
|
|
0945831442 | ||
|
|
f82b730c31 | ||
|
|
1a1fee0874 | ||
|
|
290925f184 | ||
|
|
73353bc8e5 | ||
|
|
40f59ec4d2 | ||
|
|
806c95e305 | ||
|
|
45bd7e9b94 | ||
|
|
bf3df0312b | ||
|
|
503c57efd9 | ||
|
|
d084f4f912 | ||
|
|
e67fffb05d | ||
|
|
4958a6f2e4 | ||
|
|
385fbc1a6a | ||
|
|
c22ca6218e | ||
|
|
0ccb01cada | ||
|
|
1f22b30de3 | ||
|
|
debe588bba | ||
|
|
e8379272ae | ||
|
|
3e209bfe4e | ||
|
|
88b961c115 |
@@ -15,6 +15,9 @@ bin/
|
||||
# Extracted from 3p/ by `make assets` inside the build; a host copy is not
|
||||
# needed. The tarball in 3p/ must stay in the context.
|
||||
static/js/alpine.min.js
|
||||
# The js-deps stage installs ESLint; a host copy would overwrite it at the
|
||||
# js-lint stage's `COPY . .`.
|
||||
node_modules/
|
||||
.env
|
||||
.env.*
|
||||
*.db
|
||||
|
||||
@@ -28,10 +28,10 @@ jobs:
|
||||
|
||||
- name: Fingerprint the build context
|
||||
# Writes the hash of the commit being checked into the context, which
|
||||
# invalidates the `COPY . .` layer of both check stages: a commit
|
||||
# that was never linted, format-checked, tested and built cannot
|
||||
# report success from cache.
|
||||
# invalidates the `COPY . .` layer of every check stage: a commit
|
||||
# that was never linted, format-checked, stylesheet-checked, tested
|
||||
# and built cannot report success from cache.
|
||||
run: git rev-parse HEAD > .ci-fingerprint
|
||||
|
||||
- name: Build Docker image (runs make fmt-check, golangci-lint, make test, make build)
|
||||
- name: Build Docker image (runs make fmt-check, golangci-lint, the stylesheet check, ESLint, make test, make build)
|
||||
run: script/cibuild
|
||||
|
||||
@@ -15,6 +15,9 @@ bin/
|
||||
# Go vendor directory
|
||||
vendor/
|
||||
|
||||
# ESLint and its dependencies, installed from yarn.lock
|
||||
node_modules/
|
||||
|
||||
# IDE specific files
|
||||
.idea/
|
||||
*.swp
|
||||
|
||||
Binary file not shown.
Binary file not shown.
+56
-2
@@ -25,8 +25,60 @@ COPY . .
|
||||
# would need a docker daemon inside the build. Keep these steps in step with
|
||||
# Dockerfile.lint, including --network=none (see its header for why).
|
||||
RUN make fmt-check
|
||||
RUN script/assets
|
||||
RUN --network=none golangci-lint config verify --config .golangci.yml
|
||||
RUN --network=none golangci-lint run --config .golangci.yml ./...
|
||||
RUN --network=none golangci-lint run --config .golangci.yml --build-tags browser ./...
|
||||
|
||||
# Stylesheet stages. static/css/tailwind.css is generated, by this pinned
|
||||
# tailwindcss, from static/css/input.css and the files its @source lines
|
||||
# name. `make css` (script/css) writes it out from the css-output stage.
|
||||
# The css-check stage fails when the committed file differs from what is
|
||||
# generated; `make check` runs it, and so does the build stage below.
|
||||
#
|
||||
# tailwindcss v4.2.1 standalone CLI, released 2026-02-23: one binary per
|
||||
# architecture, each pinned by its sha256 from the release's sha256sums.txt.
|
||||
# debian:bookworm-slim, 2026-10-02: the binary needs glibc.
|
||||
FROM debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251 AS tailwind-amd64
|
||||
ADD --checksum=sha256:39e8d4e24b3c83b0a6e69e100a972fbc75d5fef8dce47b3ddac3cf92dea81fe3 --chmod=755 \
|
||||
https://github.com/tailwindlabs/tailwindcss/releases/download/v4.2.1/tailwindcss-linux-x64 /usr/local/bin/tailwindcss
|
||||
|
||||
FROM debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251 AS tailwind-arm64
|
||||
ADD --checksum=sha256:d87e6486bb3f70b04ef1dcaacc4ee6548a5a15fbf521b31bc24d2c774f68a951 --chmod=755 \
|
||||
https://github.com/tailwindlabs/tailwindcss/releases/download/v4.2.1/tailwindcss-linux-arm64 /usr/local/bin/tailwindcss
|
||||
|
||||
# TARGETARCH, set by docker, is the architecture being built for.
|
||||
FROM tailwind-${TARGETARCH} AS css
|
||||
WORKDIR /src
|
||||
COPY . .
|
||||
RUN tailwindcss -i static/css/input.css -o /out/tailwind.css --minify
|
||||
|
||||
FROM scratch AS css-output
|
||||
COPY --from=css /out/tailwind.css /
|
||||
|
||||
# Both files are split after each "}", one rule per line, so that when they
|
||||
# differ the diff shows the rules that differ.
|
||||
FROM css AS css-check
|
||||
RUN sed 's/}/}\n/g' static/css/tailwind.css > /tmp/committed.css \
|
||||
&& sed 's/}/}\n/g' /out/tailwind.css > /tmp/generated.css \
|
||||
&& diff -U0 /tmp/committed.css /tmp/generated.css || { \
|
||||
echo "static/css/tailwind.css is not what make css generates; run make css" >&2; \
|
||||
exit 1; \
|
||||
}
|
||||
|
||||
# JavaScript lint stages: ESLint, at the version package.json and yarn.lock
|
||||
# pin, checks static/js/ against eslint.config.mjs. js-deps installs it and
|
||||
# stays cached until those two files change. script/lint forces only js-lint
|
||||
# to re-run, and the build stage below runs it too. COPY . . brings in the CI
|
||||
# cache barrier described in the lint stage above.
|
||||
# node:24.21.0-alpine (LTS, with yarn 1.22.22), 2026-09-18
|
||||
FROM node:24.21.0-alpine@sha256:ebfe2f90462722a7a4de65e91990e97fe0d401c70e0e762c5b53302f905ec1c1 AS js-deps
|
||||
WORKDIR /src
|
||||
COPY package.json yarn.lock ./
|
||||
RUN yarn install --frozen-lockfile --ignore-scripts
|
||||
|
||||
FROM js-deps AS js-lint
|
||||
COPY . .
|
||||
RUN --network=none node_modules/.bin/eslint static/js
|
||||
|
||||
# Build stage
|
||||
# golang:1.26.1-bookworm (Debian-based), 2026-03-17
|
||||
@@ -34,8 +86,10 @@ RUN --network=none golangci-lint run --config .golangci.yml ./...
|
||||
# mattn/go-sqlite3 (CGO), which does not compile on Alpine musl.
|
||||
FROM golang:1.26.1-bookworm@sha256:4465644228bc2857a954b092167e12aa59c006a3492282a6c820bf4755fd64a4 AS builder
|
||||
|
||||
# Depend on lint stage passing
|
||||
# Depend on the lint, stylesheet check and JavaScript lint stages passing
|
||||
COPY --from=lint /src/go.sum /dev/null
|
||||
COPY --from=css-check /out/tailwind.css /dev/null
|
||||
COPY --from=js-lint /src/yarn.lock /dev/null
|
||||
|
||||
# jq is a runtime dependency of script/ci-mark-superseded, which the test
|
||||
# suite executes. git is what script/version derives the version with.
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
# Browser test image, built by script/test-browser (make test-browser). It
|
||||
# runs the test in internal/server that loads the pages in a headless
|
||||
# browser under the real Content-Security-Policy. That test is built only
|
||||
# with the browser build tag, so make test leaves it out. Here the browser
|
||||
# comes from a digest-pinned image, and if it is missing the test fails.
|
||||
|
||||
# golang:1.26.1-bookworm, 2026-03-17: the builder stage's image in Dockerfile.
|
||||
FROM golang:1.26.1-bookworm@sha256:4465644228bc2857a954b092167e12aa59c006a3492282a6c820bf4755fd64a4 AS build
|
||||
|
||||
WORKDIR /src
|
||||
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
COPY . .
|
||||
|
||||
# The test binary embeds the templates and static files, so the browser
|
||||
# stage needs nothing else. -p 4 keeps the compile's memory down, as in
|
||||
# script/test.
|
||||
RUN make assets && go test -c -p 4 -tags browser -o /browser.test ./internal/server
|
||||
|
||||
# chromedp/headless-shell:151.0.7922.109 (Debian trixie), 2026-08-11. The
|
||||
# browser is on PATH as headless-shell, where the test's browser library
|
||||
# looks for it.
|
||||
FROM chromedp/headless-shell:151.0.7922.109@sha256:2d349b544a1ea6b5b5fd7c0fe99215ff662339c57407ee2e8c0a11af93516b04 AS browser
|
||||
|
||||
COPY --from=build /browser.test /browser.test
|
||||
|
||||
RUN /browser.test -test.v -test.timeout 90s -test.run '^TestAlpineRunsUnderTheSecurityPolicy$'
|
||||
+7
-1
@@ -31,7 +31,13 @@ FROM deps AS lint
|
||||
|
||||
COPY . .
|
||||
|
||||
# static/static.go embeds the Alpine.js file this extracts from 3p/; without
|
||||
# it the static package does not compile and cannot be linted.
|
||||
RUN script/assets
|
||||
|
||||
# `run` silently ignores config keys it does not recognize, so a typo would
|
||||
# disable a setting without a word. `config verify` is what catches that.
|
||||
RUN --network=none golangci-lint config verify --config .golangci.yml
|
||||
RUN --network=none golangci-lint run --config .golangci.yml ./...
|
||||
# --build-tags browser also lints the browser test, which is built only with
|
||||
# that tag (make test-browser).
|
||||
RUN --network=none golangci-lint run --config .golangci.yml --build-tags browser ./...
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
.PHONY: bootstrap setup assets test lint fmt fmt-check check build run dev deps docker clean hooks css version
|
||||
.PHONY: bootstrap setup assets test test-browser lint fmt fmt-check check build run dev deps docker clean hooks css css-check version
|
||||
|
||||
# Default target
|
||||
.DEFAULT_GOAL := check
|
||||
@@ -33,6 +33,9 @@ assets:
|
||||
test:
|
||||
@script/test
|
||||
|
||||
test-browser:
|
||||
@script/test-browser
|
||||
|
||||
lint:
|
||||
@script/lint
|
||||
|
||||
@@ -71,4 +74,7 @@ hooks:
|
||||
@script/install-precommit
|
||||
|
||||
css:
|
||||
tailwindcss -i static/css/input.css -o static/css/tailwind.css --minify
|
||||
@script/css
|
||||
|
||||
css-check:
|
||||
@script/css-check
|
||||
|
||||
+27
-7
@@ -8,6 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"go.uber.org/fx"
|
||||
"go.uber.org/fx/fxevent"
|
||||
"sneak.berlin/go/webhooker/internal/config"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
"sneak.berlin/go/webhooker/internal/datadir"
|
||||
@@ -37,17 +38,19 @@ import (
|
||||
// hook that used the whole budget would exhaust it at that instant,
|
||||
// and fx would skip every hook after the server — the delivery
|
||||
// engine, the healthcheck, the webhook DB manager and the database
|
||||
// close. That hook is the 3s HTTP drain plus the Sentry flush that
|
||||
// follows it in the same hook, so the flush is clamped to the stop
|
||||
// close. That hook is the HTTP drain plus the Sentry flush that
|
||||
// follows it in the same hook, and each is clamped to the stop
|
||||
// context's remaining time less server.TailHookReserve rather than
|
||||
// running for its own fixed 2s; the reserve is what the tail hooks
|
||||
// live on, and they are microsecond-scale in normal operation.
|
||||
// running for its own fixed 3s and 2s; the reserve is what the tail
|
||||
// hooks live on, and they are microsecond-scale in normal operation.
|
||||
// TestStopTimeout_LeavesHeadroomForTailHooks pins the arithmetic
|
||||
// across every drain length.
|
||||
// across every drain length and every amount of budget the hooks
|
||||
// before the server may already have spent.
|
||||
//
|
||||
// This does not make the database close unconditional: the
|
||||
// ArchiveSweeper and RetentionReaper hooks run before the server
|
||||
// and can still consume the whole budget on their own.
|
||||
// ArchiveSweeper and RetentionReaper hooks run before the server.
|
||||
// What they spend comes out of the drain first, but past 3s it comes
|
||||
// out of the reserve, and they can consume the whole budget.
|
||||
const stopTimeout = 5 * time.Second
|
||||
|
||||
// exitUsage is the status for a command line this binary cannot make
|
||||
@@ -168,6 +171,19 @@ func run(stderr io.Writer) int {
|
||||
func newApp() *fx.App {
|
||||
return fx.New(
|
||||
fx.StopTimeout(stopTimeout),
|
||||
// fx's own events go through the service's logger, not fx's
|
||||
// console logger on standard error. The exception is a failure
|
||||
// before this logger is built, such as an invalid configuration
|
||||
// value, which fx's console logger still prints there. fx holds
|
||||
// its events back until this logger is built and then replays
|
||||
// them, so it takes the configuration, which sets the level
|
||||
// DEBUG=true asks for: without it the replay would run at INFO
|
||||
// and drop every record of how the graph was built.
|
||||
fx.WithLogger(
|
||||
func(l *logger.Logger, _ *config.Config) fxevent.Logger {
|
||||
return logger.NewFxLogger(l.Get())
|
||||
},
|
||||
),
|
||||
fx.Provide(
|
||||
globals.New,
|
||||
logger.New,
|
||||
@@ -196,6 +212,10 @@ func newApp() *fx.App {
|
||||
// or renaming a webhook or target reaches its archive
|
||||
// files.
|
||||
func(e *delivery.Engine) delivery.Archives { return e },
|
||||
// Wire *delivery.Engine as delivery.CircuitBreakers so
|
||||
// the pages can show a target whose deliveries are
|
||||
// paused.
|
||||
func(e *delivery.Engine) delivery.CircuitBreakers { return e },
|
||||
server.New,
|
||||
),
|
||||
fx.Invoke(
|
||||
|
||||
+129
-9
@@ -2,12 +2,19 @@ package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/webhooker/internal/config"
|
||||
"sneak.berlin/go/webhooker/internal/datadir"
|
||||
"sneak.berlin/go/webhooker/internal/resetpw"
|
||||
"sneak.berlin/go/webhooker/internal/server"
|
||||
@@ -30,6 +37,7 @@ const dockerStopGrace = 10 * time.Second
|
||||
// fx.New applies options before it executes invokes, so the timeout
|
||||
// is set whether or not the graph itself can be constructed here.
|
||||
func TestNewApp_StopTimeout(t *testing.T) {
|
||||
config.ClearEnvForTest(t)
|
||||
t.Setenv("DATA_DIR", t.TempDir())
|
||||
|
||||
got := newApp().StopTimeout()
|
||||
@@ -38,6 +46,100 @@ func TestNewApp_StopTimeout(t *testing.T) {
|
||||
require.Less(t, got, dockerStopGrace)
|
||||
}
|
||||
|
||||
// freePort returns a loopback TCP port that was free a moment ago, by
|
||||
// taking one and releasing it.
|
||||
func freePort(t *testing.T) int {
|
||||
t.Helper()
|
||||
|
||||
var listenCfg net.ListenConfig
|
||||
|
||||
l, err := listenCfg.Listen(t.Context(), "tcp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
|
||||
addr, ok := l.Addr().(*net.TCPAddr)
|
||||
require.True(t, ok, "listener is not TCP")
|
||||
require.NoError(t, l.Close())
|
||||
|
||||
return addr.Port
|
||||
}
|
||||
|
||||
// TestNewApp_SendsFxEventsToTheLogger starts and stops the app main
|
||||
// runs, with DEBUG=true, and reads back what reached the service's
|
||||
// logger. fx's own events must arrive there as structured records:
|
||||
// the start at INFO, and at DEBUG the records of how the graph was
|
||||
// built.
|
||||
//
|
||||
// fx holds its events back until its logger is built and then replays
|
||||
// them all at once, so the earliest of them arriving shows the replay
|
||||
// ran at DEBUG: that globals.New was provided, which fx records before
|
||||
// anything is built, and the run of logger.New, which happens before
|
||||
// the configuration sets the level.
|
||||
func TestNewApp_SendsFxEventsToTheLogger(t *testing.T) {
|
||||
config.ClearEnvForTest(t)
|
||||
t.Setenv("DATA_DIR", t.TempDir())
|
||||
t.Setenv("PORT", strconv.Itoa(freePort(t)))
|
||||
t.Setenv("DEBUG", "true")
|
||||
|
||||
// internal/logger writes to whatever os.Stdout is when it builds
|
||||
// its handler. A file is not a terminal, so that handler is the
|
||||
// JSON one the service uses in production.
|
||||
out, err := os.CreateTemp(t.TempDir(), "stdout")
|
||||
require.NoError(t, err)
|
||||
|
||||
stdout := os.Stdout
|
||||
os.Stdout = out
|
||||
|
||||
t.Cleanup(func() {
|
||||
os.Stdout = stdout
|
||||
_ = out.Close()
|
||||
})
|
||||
|
||||
app := newApp()
|
||||
require.NoError(t, app.Start(t.Context()))
|
||||
require.NoError(t, app.Stop(t.Context()))
|
||||
|
||||
_, err = out.Seek(0, io.SeekStart)
|
||||
require.NoError(t, err)
|
||||
|
||||
written, err := io.ReadAll(out)
|
||||
require.NoError(t, err)
|
||||
|
||||
type record struct {
|
||||
Level string `json:"level"`
|
||||
Msg string `json:"msg"`
|
||||
Name string `json:"name"`
|
||||
Constructor string `json:"constructor"`
|
||||
}
|
||||
|
||||
var records []record
|
||||
|
||||
for line := range strings.Lines(string(written)) {
|
||||
var r record
|
||||
|
||||
// The first-boot banner is plain text, not a record.
|
||||
if json.Unmarshal([]byte(line), &r) == nil {
|
||||
records = append(records, r)
|
||||
}
|
||||
}
|
||||
|
||||
const pkg = "sneak.berlin/go/webhooker/internal/"
|
||||
|
||||
info := slog.LevelInfo.String()
|
||||
debug := slog.LevelDebug.String()
|
||||
|
||||
assert.Contains(t, records, record{Level: info, Msg: "started"})
|
||||
assert.Contains(t, records, record{
|
||||
Level: debug, Msg: "provided", Constructor: pkg + "globals.New()",
|
||||
})
|
||||
assert.Contains(t, records, record{
|
||||
Level: debug, Msg: "run", Name: pkg + "logger.New()",
|
||||
})
|
||||
assert.Contains(t, records, record{Level: debug, Msg: "invoking"})
|
||||
assert.Contains(t, records, record{
|
||||
Level: debug, Msg: "initialized custom fxevent.Logger",
|
||||
})
|
||||
}
|
||||
|
||||
// TestRunRefusesLockedDataDir pins what an operator's second start
|
||||
// does. The entry point must refuse before it builds the fx graph —
|
||||
// nothing may open a database in a DATA_DIR another process holds —
|
||||
@@ -150,22 +252,40 @@ const tailHeadroom = 2 * time.Second
|
||||
// can produce, since a shorter drain leaves the flush more room and
|
||||
// the worst case is not necessarily at either extreme.
|
||||
//
|
||||
// Shrinking either budget, or unbounding the flush again, must fail
|
||||
// here rather than silently recreating a hook that swallows the
|
||||
// whole sequence.
|
||||
// Nor does the hook start on a full budget: the ArchiveSweeper and
|
||||
// RetentionReaper hooks run before it, and whatever they spent is
|
||||
// gone. The outer sweep walks every amount they can spend. Once they
|
||||
// have eaten into the headroom themselves, the hook must spend
|
||||
// nothing of what is left. A drain that starts on the full budget
|
||||
// must still get all of ShutdownTimeout, so a smaller stopTimeout
|
||||
// cannot silently shorten every drain.
|
||||
//
|
||||
// Shrinking either budget, or unbounding the drain or the flush
|
||||
// again, must fail here rather than silently recreating a hook that
|
||||
// swallows the whole sequence.
|
||||
func TestStopTimeout_LeavesHeadroomForTailHooks(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
require.Less(t, server.ShutdownTimeout, stopTimeout)
|
||||
require.Equal(
|
||||
t, server.ShutdownTimeout, server.DrainBudget(stopTimeout),
|
||||
"a drain that starts on the full stop budget is cut short",
|
||||
)
|
||||
|
||||
const step = 10 * time.Millisecond
|
||||
|
||||
for drain := time.Duration(0); drain <= server.ShutdownTimeout; drain += step {
|
||||
hook := drain + server.SentryFlushBudget(stopTimeout-drain)
|
||||
for spent := time.Duration(0); spent <= stopTimeout; spent += step {
|
||||
remaining := stopTimeout - spent
|
||||
longest := max(server.DrainBudget(remaining), 0)
|
||||
|
||||
require.LessOrEqual(
|
||||
t, hook+tailHeadroom, stopTimeout,
|
||||
"a %s drain leaves the tail hooks short", drain,
|
||||
)
|
||||
for drain := time.Duration(0); drain <= longest; drain += step {
|
||||
hook := drain + server.SentryFlushBudget(remaining-drain)
|
||||
|
||||
require.GreaterOrEqual(
|
||||
t, remaining-hook, min(remaining, tailHeadroom),
|
||||
"a %s drain after %s of earlier hooks leaves "+
|
||||
"the tail hooks short", drain, spent,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
// ESLint configuration for static/js/. script/lint and the image build run
|
||||
// ESLint in the Dockerfile's js-lint stage, never on the host.
|
||||
//
|
||||
// The rules are the ones the JavaScript styleguide linked from
|
||||
// REPO_POLICIES.md states that a linter can check: const for everything,
|
||||
// let only for a variable that is reassigned, never var.
|
||||
export default [
|
||||
// Alpine.js, extracted from 3p/ by make assets; not ours to lint.
|
||||
{ ignores: ["static/js/alpine.min.js"] },
|
||||
{
|
||||
// The pages load static/js/app.js as a classic script, not a module.
|
||||
languageOptions: { sourceType: "script" },
|
||||
rules: {
|
||||
"no-var": "error",
|
||||
"prefer-const": "error",
|
||||
},
|
||||
},
|
||||
];
|
||||
@@ -4,6 +4,8 @@ go 1.26.1
|
||||
|
||||
require (
|
||||
github.com/99designs/basicauth-go v0.0.0-20230316000542-bf6f9cbbf0f8
|
||||
github.com/chromedp/cdproto v0.0.0-20260922220944-a19bff23514f
|
||||
github.com/chromedp/chromedp v0.16.0
|
||||
github.com/dustin/go-humanize v1.0.1
|
||||
github.com/getsentry/sentry-go v0.25.0
|
||||
github.com/go-chi/chi v1.5.5
|
||||
@@ -18,7 +20,7 @@ require (
|
||||
github.com/prometheus/client_model v0.5.0
|
||||
github.com/slok/go-http-metrics v0.11.0
|
||||
github.com/stretchr/testify v1.11.1
|
||||
go.uber.org/fx v1.20.1
|
||||
go.uber.org/fx v1.24.0
|
||||
golang.org/x/crypto v0.38.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
gorm.io/driver/sqlite v1.5.4
|
||||
@@ -29,13 +31,17 @@ require (
|
||||
require (
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/chromedp/sysutil v1.1.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
github.com/go-json-experiment/json v0.0.0-20260820222146-c27c302e5fc3 // indirect
|
||||
github.com/gobwas/httphead v0.1.0 // indirect
|
||||
github.com/gobwas/pool v0.2.1 // indirect
|
||||
github.com/gobwas/ws v1.4.0 // indirect
|
||||
github.com/gorilla/securecookie v1.1.2 // indirect
|
||||
github.com/jinzhu/inflection v1.0.0 // indirect
|
||||
github.com/jinzhu/now v1.1.5 // indirect
|
||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-sqlite3 v1.14.17 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
|
||||
@@ -44,13 +50,12 @@ require (
|
||||
github.com/prometheus/procfs v0.12.0 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/zeebo/xxh3 v1.0.2 // indirect
|
||||
go.uber.org/atomic v1.9.0 // indirect
|
||||
go.uber.org/dig v1.17.0 // indirect
|
||||
go.uber.org/multierr v1.9.0 // indirect
|
||||
go.uber.org/zap v1.23.0 // indirect
|
||||
go.uber.org/dig v1.19.0 // indirect
|
||||
go.uber.org/multierr v1.10.0 // indirect
|
||||
go.uber.org/zap v1.26.0 // indirect
|
||||
golang.org/x/mod v0.17.0 // indirect
|
||||
golang.org/x/sync v0.14.0 // indirect
|
||||
golang.org/x/sys v0.37.0 // indirect
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
golang.org/x/text v0.25.0 // indirect
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
|
||||
google.golang.org/protobuf v1.31.0 // indirect
|
||||
|
||||
@@ -1,14 +1,15 @@
|
||||
github.com/99designs/basicauth-go v0.0.0-20230316000542-bf6f9cbbf0f8 h1:nMpu1t4amK3vJWBibQ5X/Nv0aXL+b69TQf2uK5PH7Go=
|
||||
github.com/99designs/basicauth-go v0.0.0-20230316000542-bf6f9cbbf0f8/go.mod h1:3cARGAK9CfW3HoxCy1a0G4TKrdiKke8ftOMEOHyySYs=
|
||||
github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
|
||||
github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
|
||||
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/chromedp/cdproto v0.0.0-20260922220944-a19bff23514f h1:8PK9FM4bE0C8GMoWBW5lVsef3U7sPICjDg6JqngyYhk=
|
||||
github.com/chromedp/cdproto v0.0.0-20260922220944-a19bff23514f/go.mod h1:3v4FIp5njIUyPDvqXsxEOxnB34lijG0up98/5kM1KaE=
|
||||
github.com/chromedp/chromedp v0.16.0 h1:rOO4deOm4CbZgBCa8mD9g2rDyIoNs0BkgvNrlbp5ouk=
|
||||
github.com/chromedp/chromedp v0.16.0/go.mod h1:rbuGKFT1vMcFcFqKfPIO1GpX/N+2s8onm2qMxZLbU5U=
|
||||
github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM=
|
||||
github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
@@ -23,6 +24,14 @@ github.com/go-chi/httprate v0.15.0 h1:j54xcWV9KGmPf/X4H32/aTH+wBlrvxL7P+SdnRqxh5
|
||||
github.com/go-chi/httprate v0.15.0/go.mod h1:rzGHhVrsBn3IMLYDOZQsSU4fJNWcjui4fWKJcCId1R4=
|
||||
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
|
||||
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
|
||||
github.com/go-json-experiment/json v0.0.0-20260820222146-c27c302e5fc3 h1:UADEEmDKgfXbtnGJZ97beY5XLo9ZechG1nlU4KnRrkE=
|
||||
github.com/go-json-experiment/json v0.0.0-20260820222146-c27c302e5fc3/go.mod h1:tphK2c80bpPhMOI4v6bIc2xWywPfbqi1Z06+RcrMkDg=
|
||||
github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
|
||||
github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
|
||||
github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
|
||||
github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
|
||||
github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs=
|
||||
github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc=
|
||||
github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw=
|
||||
github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0=
|
||||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||
@@ -55,17 +64,20 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo=
|
||||
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
|
||||
github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
|
||||
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg=
|
||||
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k=
|
||||
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw=
|
||||
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
|
||||
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
|
||||
github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk=
|
||||
@@ -82,28 +94,24 @@ github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjR
|
||||
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
|
||||
github.com/slok/go-http-metrics v0.11.0 h1:ABJUpekCZSkQT1wQrFvS4kGbhea/w6ndFJaWJeh3zL0=
|
||||
github.com/slok/go-http-metrics v0.11.0/go.mod h1:ZGKeYG1ET6TEJpQx18BqAJAvxw9jBAZXCHU7bWQqqAc=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
|
||||
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
||||
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
|
||||
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
|
||||
go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
|
||||
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||
go.uber.org/dig v1.17.0 h1:5Chju+tUvcC+N7N6EV08BJz41UZuO3BmHcN4A287ZLI=
|
||||
go.uber.org/dig v1.17.0/go.mod h1:rTxpf7l5I0eBTlE6/9RL+lDybC7WFwY2QH55ZSjy1mU=
|
||||
go.uber.org/fx v1.20.1 h1:zVwVQGS8zYvhh9Xxcu4w1M6ESyeMzebzj2NbSayZ4Mk=
|
||||
go.uber.org/fx v1.20.1/go.mod h1:iSYNbHf2y55acNCwCXKx7LbWb5WG1Bnue5RDXz1OREg=
|
||||
go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
|
||||
go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
|
||||
go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
|
||||
go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
|
||||
go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY=
|
||||
go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY=
|
||||
go.uber.org/dig v1.19.0 h1:BACLhebsYdpQ7IROQ1AGPjrXcP5dF80U3gKoFzbaq/4=
|
||||
go.uber.org/dig v1.19.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE=
|
||||
go.uber.org/fx v1.24.0 h1:wE8mruvpg2kiiL1Vqd0CC+tr0/24XIB10Iwp2lLWzkg=
|
||||
go.uber.org/fx v1.24.0/go.mod h1:AmDeGyS+ZARGKM4tlH4FY2Jr63VjbEDJHtqXTGP5hbo=
|
||||
go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
|
||||
go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=
|
||||
go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
|
||||
go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||
go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
|
||||
go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
|
||||
golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8=
|
||||
golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw=
|
||||
golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
|
||||
@@ -111,8 +119,8 @@ golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ=
|
||||
golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
|
||||
golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4=
|
||||
golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=
|
||||
|
||||
+19
-10
@@ -80,8 +80,7 @@ const (
|
||||
// process over a Docker network or a private LAN connects from.
|
||||
defaultTrustedProxies = "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16"
|
||||
|
||||
// maxPort is the highest valid TCP port number. The lower
|
||||
// bound (at least 1) is enforced by envPositiveInt.
|
||||
// maxPort is the highest valid TCP port number.
|
||||
maxPort = 65535
|
||||
|
||||
// mappedV4Offset is the number of leading bits an IPv4-mapped
|
||||
@@ -105,7 +104,7 @@ var ErrInvalidEnvironment = errors.New("invalid environment")
|
||||
var ErrNonPositiveValue = errors.New("value must be positive")
|
||||
|
||||
// ErrInvalidPort is returned when an environment variable holding a
|
||||
// TCP port number is set above the valid port range.
|
||||
// TCP port number is set to a number outside 1 to 65535.
|
||||
var ErrInvalidPort = errors.New("invalid port")
|
||||
|
||||
// ErrInvalidCIDR is returned when an environment variable holding a
|
||||
@@ -363,17 +362,27 @@ func envPositiveInt(
|
||||
// envPort returns the value of the named environment variable parsed
|
||||
// as a TCP port number. Returns defaultValue if not set. A set value
|
||||
// that is unparseable, below 1, or above maxPort is a hard error
|
||||
// naming the key and the bad value.
|
||||
// naming the key and the bad value; every out-of-range value wraps
|
||||
// ErrInvalidPort, including one too large or too small for an int.
|
||||
func envPort(key string, defaultValue int) (int, error) {
|
||||
port, err := envPositiveInt(key, defaultValue)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
v := os.Getenv(key)
|
||||
if v == "" {
|
||||
return defaultValue, nil
|
||||
}
|
||||
|
||||
if port > maxPort {
|
||||
// strconv.ErrRange means a number too large or too small for an
|
||||
// int, which is outside the port range as well.
|
||||
port, err := strconv.Atoi(v)
|
||||
if err != nil && !errors.Is(err, strconv.ErrRange) {
|
||||
return 0, fmt.Errorf(
|
||||
"%w: %s must be at most %d, got %d",
|
||||
ErrInvalidPort, key, maxPort, port,
|
||||
"invalid integer for %s: %q: %w", key, v, err,
|
||||
)
|
||||
}
|
||||
|
||||
if err != nil || port < 1 || port > maxPort {
|
||||
return 0, fmt.Errorf(
|
||||
"%w: %s must be from 1 to %d, got %q",
|
||||
ErrInvalidPort, key, maxPort, v,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -3,7 +3,6 @@ package config_test
|
||||
import (
|
||||
"bytes"
|
||||
"log/slog"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -71,14 +70,12 @@ func TestEnvironmentConfig(t *testing.T) {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Cannot use t.Parallel() here because t.Setenv
|
||||
// is incompatible with parallel subtests.
|
||||
config.ClearEnvForTest(t)
|
||||
|
||||
if tt.envValue != "" {
|
||||
t.Setenv(
|
||||
"WEBHOOKER_ENVIRONMENT", tt.envValue,
|
||||
)
|
||||
} else {
|
||||
require.NoError(t, os.Unsetenv(
|
||||
"WEBHOOKER_ENVIRONMENT",
|
||||
))
|
||||
}
|
||||
|
||||
for k, v := range tt.envVars {
|
||||
@@ -199,14 +196,11 @@ func TestRetentionSweepInterval(t *testing.T) {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Cannot use t.Parallel() here because t.Setenv
|
||||
// is incompatible with parallel subtests.
|
||||
config.ClearEnvForTest(t)
|
||||
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
||||
|
||||
if tt.set {
|
||||
t.Setenv("RETENTION_SWEEP_INTERVAL", tt.value)
|
||||
} else {
|
||||
require.NoError(t, os.Unsetenv(
|
||||
"RETENTION_SWEEP_INTERVAL",
|
||||
))
|
||||
}
|
||||
|
||||
if tt.expectError {
|
||||
@@ -341,14 +335,11 @@ func TestSessionIdleTimeout(t *testing.T) {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Cannot use t.Parallel() here because t.Setenv
|
||||
// is incompatible with parallel subtests.
|
||||
config.ClearEnvForTest(t)
|
||||
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
||||
|
||||
if tt.set {
|
||||
t.Setenv("SESSION_IDLE_TIMEOUT", tt.value)
|
||||
} else {
|
||||
require.NoError(t, os.Unsetenv(
|
||||
"SESSION_IDLE_TIMEOUT",
|
||||
))
|
||||
}
|
||||
|
||||
if tt.expectError {
|
||||
@@ -397,16 +388,12 @@ func TestDefaultDataDir(t *testing.T) {
|
||||
t.Run("env="+name, func(t *testing.T) {
|
||||
// Cannot use t.Parallel() here because t.Setenv
|
||||
// is incompatible with parallel subtests.
|
||||
config.ClearEnvForTest(t)
|
||||
|
||||
if env != "" {
|
||||
t.Setenv("WEBHOOKER_ENVIRONMENT", env)
|
||||
} else {
|
||||
require.NoError(t, os.Unsetenv(
|
||||
"WEBHOOKER_ENVIRONMENT",
|
||||
))
|
||||
}
|
||||
|
||||
require.NoError(t, os.Unsetenv("DATA_DIR"))
|
||||
|
||||
var cfg *config.Config
|
||||
|
||||
app := fxtest.New(
|
||||
@@ -446,9 +433,9 @@ func TestDataDirHelper(t *testing.T) {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
// Cannot use t.Parallel() here because t.Setenv
|
||||
// is incompatible with parallel subtests.
|
||||
if set == "" {
|
||||
require.NoError(t, os.Unsetenv("DATA_DIR"))
|
||||
} else {
|
||||
config.ClearEnvForTest(t)
|
||||
|
||||
if set != "" {
|
||||
t.Setenv("DATA_DIR", set)
|
||||
}
|
||||
|
||||
@@ -511,14 +498,11 @@ func TestReceiverRateLimit(t *testing.T) {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Cannot use t.Parallel() here because t.Setenv
|
||||
// is incompatible with parallel subtests.
|
||||
config.ClearEnvForTest(t)
|
||||
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
||||
|
||||
if tt.set {
|
||||
t.Setenv("RECEIVER_RATE_LIMIT", tt.value)
|
||||
} else {
|
||||
require.NoError(t, os.Unsetenv(
|
||||
"RECEIVER_RATE_LIMIT",
|
||||
))
|
||||
}
|
||||
|
||||
if tt.expectError {
|
||||
@@ -630,12 +614,11 @@ func TestTrustedProxies(t *testing.T) {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Cannot use t.Parallel() here because t.Setenv
|
||||
// is incompatible with parallel subtests.
|
||||
config.ClearEnvForTest(t)
|
||||
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
||||
|
||||
if tt.set {
|
||||
t.Setenv("TRUSTED_PROXIES", tt.value)
|
||||
} else {
|
||||
require.NoError(t, os.Unsetenv("TRUSTED_PROXIES"))
|
||||
}
|
||||
|
||||
if tt.expectError {
|
||||
@@ -742,14 +725,11 @@ func TestAllowedEgressCIDRs(t *testing.T) {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Cannot use t.Parallel() here because t.Setenv
|
||||
// is incompatible with parallel subtests.
|
||||
config.ClearEnvForTest(t)
|
||||
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
||||
|
||||
if tt.set {
|
||||
t.Setenv("ALLOWED_EGRESS_CIDRS", tt.value)
|
||||
} else {
|
||||
require.NoError(
|
||||
t, os.Unsetenv("ALLOWED_EGRESS_CIDRS"),
|
||||
)
|
||||
}
|
||||
|
||||
if tt.expectError {
|
||||
@@ -817,13 +797,10 @@ func TestEgressAllowlistWarning(t *testing.T) {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Cannot use t.Parallel() here because t.Setenv
|
||||
// is incompatible with parallel subtests.
|
||||
config.ClearEnvForTest(t)
|
||||
t.Setenv("WEBHOOKER_ENVIRONMENT", config.EnvironmentDev)
|
||||
|
||||
if tt.allowed == "" {
|
||||
require.NoError(
|
||||
t, os.Unsetenv("ALLOWED_EGRESS_CIDRS"),
|
||||
)
|
||||
} else {
|
||||
if tt.allowed != "" {
|
||||
t.Setenv("ALLOWED_EGRESS_CIDRS", tt.allowed)
|
||||
}
|
||||
|
||||
@@ -956,20 +933,14 @@ func TestMetricsAuthConfig(t *testing.T) {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Cannot use t.Parallel() here because t.Setenv
|
||||
// is incompatible with parallel subtests.
|
||||
config.ClearEnvForTest(t)
|
||||
|
||||
if tt.username.set {
|
||||
t.Setenv("METRICS_USERNAME", tt.username.value)
|
||||
} else {
|
||||
require.NoError(
|
||||
t, os.Unsetenv("METRICS_USERNAME"),
|
||||
)
|
||||
}
|
||||
|
||||
if tt.password.set {
|
||||
t.Setenv("METRICS_PASSWORD", tt.password.value)
|
||||
} else {
|
||||
require.NoError(
|
||||
t, os.Unsetenv("METRICS_PASSWORD"),
|
||||
)
|
||||
}
|
||||
|
||||
if tt.expectError {
|
||||
|
||||
@@ -22,17 +22,6 @@ const malformedDotEnv = "PORT 19615\n" +
|
||||
"this is not = valid ! syntax\n" +
|
||||
"\"unclosed\n"
|
||||
|
||||
// unsetDotEnvKey makes dotEnvKey genuinely absent for the duration of
|
||||
// the test and restores it afterwards. t.Setenv registers the restore;
|
||||
// the Unsetenv that follows is what the test actually needs, because a
|
||||
// variable set to the empty string is still present in os.Environ and
|
||||
// godotenv would refuse to overwrite it.
|
||||
func unsetDotEnvKey(t *testing.T) {
|
||||
t.Helper()
|
||||
t.Setenv(dotEnvKey, "placeholder")
|
||||
require.NoError(t, os.Unsetenv(dotEnvKey))
|
||||
}
|
||||
|
||||
// writeDotEnv writes contents to a .env file in a fresh temporary
|
||||
// directory and returns its path.
|
||||
func writeDotEnv(t *testing.T, contents string) string {
|
||||
@@ -50,9 +39,9 @@ func writeDotEnv(t *testing.T, contents string) string {
|
||||
// normally rather than be refused for a file it was never meant to
|
||||
// have.
|
||||
//
|
||||
//nolint:paralleltest // unsetDotEnvKey uses t.Setenv.
|
||||
//nolint:paralleltest // ClearEnvForTest uses t.Setenv.
|
||||
func TestLoadDotEnv_MissingFileIsFine(t *testing.T) {
|
||||
unsetDotEnvKey(t)
|
||||
config.ClearEnvForTest(t)
|
||||
|
||||
absent := filepath.Join(t.TempDir(), config.DotEnvPath)
|
||||
require.NoError(t, config.LoadDotEnvFileForTest(absent))
|
||||
@@ -65,9 +54,9 @@ func TestLoadDotEnv_MissingFileIsFine(t *testing.T) {
|
||||
// reaches the environment, which is the whole reason the file is read
|
||||
// at all.
|
||||
//
|
||||
//nolint:paralleltest // unsetDotEnvKey uses t.Setenv.
|
||||
//nolint:paralleltest // ClearEnvForTest uses t.Setenv.
|
||||
func TestLoadDotEnv_AppliesValues(t *testing.T) {
|
||||
unsetDotEnvKey(t)
|
||||
config.ClearEnvForTest(t)
|
||||
|
||||
path := writeDotEnv(t, "# a comment\n"+dotEnvKey+"=from-dot-env\n")
|
||||
|
||||
@@ -93,9 +82,9 @@ func TestLoadDotEnv_RealEnvironmentWins(t *testing.T) {
|
||||
// reverts to its default; the process used to start that way with no
|
||||
// log line naming the file at all.
|
||||
//
|
||||
//nolint:paralleltest // unsetDotEnvKey uses t.Setenv.
|
||||
//nolint:paralleltest // ClearEnvForTest uses t.Setenv.
|
||||
func TestLoadDotEnv_MalformedFileAborts(t *testing.T) {
|
||||
unsetDotEnvKey(t)
|
||||
config.ClearEnvForTest(t)
|
||||
|
||||
path := writeDotEnv(
|
||||
t, malformedDotEnv+dotEnvKey+"=from-dot-env\n",
|
||||
@@ -143,7 +132,7 @@ func TestLoadDotEnv_UnreadableFileAborts(t *testing.T) {
|
||||
//
|
||||
//nolint:paralleltest // t.Chdir moves the whole process.
|
||||
func TestLoadDotEnv_ReadsTheWorkingDirectory(t *testing.T) {
|
||||
unsetDotEnvKey(t)
|
||||
config.ClearEnvForTest(t)
|
||||
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(
|
||||
|
||||
+78
-91
@@ -1,7 +1,6 @@
|
||||
package config_test
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -121,10 +120,10 @@ func TestEnvBool(t *testing.T) {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Cannot use t.Parallel() here because t.Setenv
|
||||
// is incompatible with parallel subtests.
|
||||
config.ClearEnvForTest(t)
|
||||
|
||||
if tt.set {
|
||||
t.Setenv(testEnvKey, tt.value)
|
||||
} else {
|
||||
require.NoError(t, os.Unsetenv(testEnvKey))
|
||||
}
|
||||
|
||||
got, err := config.EnvBoolForTest(
|
||||
@@ -145,17 +144,62 @@ func TestEnvBool(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// envIntCase is one row of the envPositiveInt and envPort tables.
|
||||
type envIntCase struct {
|
||||
name string
|
||||
set bool
|
||||
value string
|
||||
expectError bool
|
||||
errIs error
|
||||
expected int
|
||||
}
|
||||
|
||||
// runEnvIntCases runs each row through parse, which is
|
||||
// envPositiveInt or envPort, with testEnvKey set to the row's value
|
||||
// or left unset.
|
||||
func runEnvIntCases(
|
||||
t *testing.T,
|
||||
parse func(key string, defaultValue int) (int, error),
|
||||
defaultValue int,
|
||||
tests []envIntCase,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Cannot use t.Parallel() here because t.Setenv
|
||||
// is incompatible with parallel subtests.
|
||||
config.ClearEnvForTest(t)
|
||||
|
||||
if tt.set {
|
||||
t.Setenv(testEnvKey, tt.value)
|
||||
}
|
||||
|
||||
got, err := parse(testEnvKey, defaultValue)
|
||||
|
||||
if tt.expectError {
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), testEnvKey)
|
||||
assert.Contains(t, err.Error(), tt.value)
|
||||
|
||||
if tt.errIs != nil {
|
||||
require.ErrorIs(t, err, tt.errIs)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.expected, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
//nolint:paralleltest // runEnvIntCases uses t.Setenv.
|
||||
func TestEnvPositiveInt(t *testing.T) {
|
||||
const defaultValue = 7
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
set bool
|
||||
value string
|
||||
expectError bool
|
||||
errIs error
|
||||
expected int
|
||||
}{
|
||||
runEnvIntCases(t, config.EnvPositiveIntForTest, defaultValue, []envIntCase{
|
||||
{
|
||||
name: "unset returns the default integer",
|
||||
expected: defaultValue,
|
||||
@@ -192,51 +236,14 @@ func TestEnvPositiveInt(t *testing.T) {
|
||||
expectError: true,
|
||||
errIs: config.ErrNonPositiveValue,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Cannot use t.Parallel() here because t.Setenv
|
||||
// is incompatible with parallel subtests.
|
||||
if tt.set {
|
||||
t.Setenv(testEnvKey, tt.value)
|
||||
} else {
|
||||
require.NoError(t, os.Unsetenv(testEnvKey))
|
||||
}
|
||||
|
||||
got, err := config.EnvPositiveIntForTest(
|
||||
testEnvKey, defaultValue,
|
||||
)
|
||||
|
||||
if tt.expectError {
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), testEnvKey)
|
||||
assert.Contains(t, err.Error(), tt.value)
|
||||
|
||||
if tt.errIs != nil {
|
||||
require.ErrorIs(t, err, tt.errIs)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.expected, got)
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
//nolint:paralleltest // runEnvIntCases uses t.Setenv.
|
||||
func TestEnvPort(t *testing.T) {
|
||||
const defaultValue = 8080
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
set bool
|
||||
value string
|
||||
expectError bool
|
||||
errIs error
|
||||
expected int
|
||||
}{
|
||||
runEnvIntCases(t, config.EnvPortForTest, defaultValue, []envIntCase{
|
||||
{
|
||||
name: "unset returns the default port",
|
||||
expected: defaultValue,
|
||||
@@ -264,7 +271,14 @@ func TestEnvPort(t *testing.T) {
|
||||
set: true,
|
||||
value: "0",
|
||||
expectError: true,
|
||||
errIs: config.ErrNonPositiveValue,
|
||||
errIs: config.ErrInvalidPort,
|
||||
},
|
||||
{
|
||||
name: "negative is rejected",
|
||||
set: true,
|
||||
value: "-1",
|
||||
expectError: true,
|
||||
errIs: config.ErrInvalidPort,
|
||||
},
|
||||
{
|
||||
name: "above the port range is rejected",
|
||||
@@ -273,37 +287,14 @@ func TestEnvPort(t *testing.T) {
|
||||
expectError: true,
|
||||
errIs: config.ErrInvalidPort,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Cannot use t.Parallel() here because t.Setenv
|
||||
// is incompatible with parallel subtests.
|
||||
if tt.set {
|
||||
t.Setenv(testEnvKey, tt.value)
|
||||
} else {
|
||||
require.NoError(t, os.Unsetenv(testEnvKey))
|
||||
}
|
||||
|
||||
got, err := config.EnvPortForTest(
|
||||
testEnvKey, defaultValue,
|
||||
)
|
||||
|
||||
if tt.expectError {
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), testEnvKey)
|
||||
|
||||
if tt.errIs != nil {
|
||||
require.ErrorIs(t, err, tt.errIs)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.expected, got)
|
||||
})
|
||||
}
|
||||
{
|
||||
name: "too large for an int is rejected",
|
||||
set: true,
|
||||
value: "99999999999999999999",
|
||||
expectError: true,
|
||||
errIs: config.ErrInvalidPort,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// TestEnvBindAddress covers BIND_ADDRESS parsing.
|
||||
@@ -319,10 +310,10 @@ func TestEnvBindAddress(t *testing.T) {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Cannot use t.Parallel() here because t.Setenv
|
||||
// is incompatible with parallel subtests.
|
||||
config.ClearEnvForTest(t)
|
||||
|
||||
if tt.set {
|
||||
t.Setenv(testEnvKey, tt.value)
|
||||
} else {
|
||||
require.NoError(t, os.Unsetenv(testEnvKey))
|
||||
}
|
||||
|
||||
got, err := config.EnvBindAddressForTest(
|
||||
@@ -485,6 +476,7 @@ func TestNewRejectsBadEnvValues(t *testing.T) {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Cannot use t.Parallel() here because t.Setenv
|
||||
// is incompatible with parallel subtests.
|
||||
config.ClearEnvForTest(t)
|
||||
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
||||
t.Setenv(tt.key, tt.value)
|
||||
|
||||
@@ -646,14 +638,9 @@ func sentryEnvValueCases() []badEnvValueCase {
|
||||
// break the legitimate unset case: absent variables still get their
|
||||
// documented defaults.
|
||||
func TestNewUsesDefaultsWhenUnset(t *testing.T) {
|
||||
config.ClearEnvForTest(t)
|
||||
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
||||
|
||||
for _, key := range []string{
|
||||
envKeyPort, envKeyDebug, envKeyBindAddress, envKeySentryDSN,
|
||||
} {
|
||||
require.NoError(t, os.Unsetenv(key))
|
||||
}
|
||||
|
||||
cfg, err := buildConfig(t)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, cfg)
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package config_test
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -101,10 +100,10 @@ func TestEnvSentryDSN(t *testing.T) {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Cannot use t.Parallel() here because t.Setenv
|
||||
// is incompatible with parallel subtests.
|
||||
config.ClearEnvForTest(t)
|
||||
|
||||
if tt.set {
|
||||
t.Setenv(envKeySentryDSN, tt.value)
|
||||
} else {
|
||||
require.NoError(t, os.Unsetenv(envKeySentryDSN))
|
||||
}
|
||||
|
||||
got, err := config.EnvSentryDSNForTest(envKeySentryDSN)
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// ClearEnvForTest unsets every variable in the process environment
|
||||
// for the rest of the test, so a test sees only the variables it sets
|
||||
// itself, not whatever the developer's shell exports. When the test
|
||||
// ends it leaves the environment exactly as it found it: each variable
|
||||
// it unset is put back, and any variable added since is removed.
|
||||
func ClearEnvForTest(t *testing.T) {
|
||||
t.Helper()
|
||||
|
||||
present := make(map[string]bool)
|
||||
|
||||
for _, entry := range os.Environ() {
|
||||
key, _, _ := strings.Cut(entry, "=")
|
||||
present[key] = true
|
||||
|
||||
// t.Setenv registers the restore; the Unsetenv after it is
|
||||
// what makes the key absent, since a key set to the empty
|
||||
// string is still present, and godotenv will not overwrite a
|
||||
// present key.
|
||||
t.Setenv(key, "")
|
||||
|
||||
err := os.Unsetenv(key)
|
||||
if err != nil {
|
||||
t.Fatalf("unsetting %s: %v", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A variable the test adds other than through t.Setenv, as loading
|
||||
// a .env file does, has no restore of its own.
|
||||
t.Cleanup(func() {
|
||||
for _, entry := range os.Environ() {
|
||||
key, _, _ := strings.Cut(entry, "=")
|
||||
if present[key] {
|
||||
continue
|
||||
}
|
||||
|
||||
err := os.Unsetenv(key)
|
||||
if err != nil {
|
||||
t.Errorf("unsetting %s: %v", key, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package config_test
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/webhooker/internal/config"
|
||||
)
|
||||
|
||||
// TestClearEnvForTest_RemovesAddedVariables pins that a variable set
|
||||
// after the clear other than through t.Setenv, as a test's .env file
|
||||
// sets one, is gone once the test ends, so it cannot reach the tests
|
||||
// that run after it.
|
||||
//
|
||||
//nolint:paralleltest // ClearEnvForTest uses t.Setenv.
|
||||
func TestClearEnvForTest_RemovesAddedVariables(t *testing.T) {
|
||||
// The outer clear keeps a value of the key exported in the shell
|
||||
// from making it a variable the inner clear has to put back.
|
||||
config.ClearEnvForTest(t)
|
||||
|
||||
t.Run("loads a .env file after the clear", func(t *testing.T) {
|
||||
config.ClearEnvForTest(t)
|
||||
|
||||
path := writeDotEnv(t, dotEnvKey+"=from-dot-env\n")
|
||||
require.NoError(t, config.LoadDotEnvFileForTest(path))
|
||||
require.Equal(t, "from-dot-env", os.Getenv(dotEnvKey))
|
||||
})
|
||||
|
||||
_, present := os.LookupEnv(dotEnvKey)
|
||||
assert.False(
|
||||
t, present,
|
||||
"a variable set after the clear must not outlive the test",
|
||||
)
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -119,3 +120,26 @@ func TestNewDatabase_IsLoggedWithItsPath(t *testing.T) {
|
||||
t, second, created, "an existing database is not new",
|
||||
)
|
||||
}
|
||||
|
||||
// TestZeroLengthDatabase_IsLoggedAsNew covers what
|
||||
// https://git.eeqj.de/sneak/webhooker/issues/290 found: SQLite opens a
|
||||
// zero-length file as an empty database, so a start on one is a first
|
||||
// start, and it must say so exactly as a start with no file does.
|
||||
func TestZeroLengthDatabase_IsLoggedAsNew(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, database.MainDBFileName)
|
||||
require.NoError(t, os.WriteFile(path, nil, database.SQLiteFilePerm))
|
||||
|
||||
var out bytes.Buffer
|
||||
|
||||
db, err := database.Open(dir, slog.New(slog.NewTextHandler(&out, nil)))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, db.Close())
|
||||
|
||||
assert.Contains(
|
||||
t, out.String(),
|
||||
`level=WARN msg="created a new, empty database" path=`+path,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -203,8 +202,7 @@ func (d *Database) connectTo(dataDir string) error {
|
||||
// Checked before opening, which creates the file. A DATA_DIR that
|
||||
// is unexpectedly empty -- its volume not mounted, say -- looks
|
||||
// exactly like a first start, so a new database is a warning.
|
||||
_, statErr := os.Stat(dbPath)
|
||||
created := errors.Is(statErr, fs.ErrNotExist)
|
||||
created := missingOrEmpty(dbPath)
|
||||
|
||||
// Opened through OpenSQLite so this handle carries the same WAL
|
||||
// journaling, busy timeout, immediate-transaction locking, and pool
|
||||
@@ -213,13 +211,15 @@ func (d *Database) connectTo(dataDir string) error {
|
||||
if err != nil {
|
||||
d.log.Error(
|
||||
"failed to open database",
|
||||
"path", dbPath,
|
||||
"error", err,
|
||||
)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// Then use it with GORM
|
||||
// Then use it with GORM. Its errors are SQLite's alone and name no
|
||||
// file, so the path is added to them here.
|
||||
db, err := gorm.Open(sqlite.Dialector{
|
||||
Conn: sqlDB,
|
||||
}, &gorm.Config{
|
||||
@@ -229,10 +229,11 @@ func (d *Database) connectTo(dataDir string) error {
|
||||
if err != nil {
|
||||
d.log.Error(
|
||||
"failed to connect to database",
|
||||
"path", dbPath,
|
||||
"error", err,
|
||||
)
|
||||
|
||||
return err
|
||||
return fmt.Errorf("connecting to %s: %w", dbPath, err)
|
||||
}
|
||||
|
||||
d.db = db
|
||||
@@ -243,8 +244,12 @@ func (d *Database) connectTo(dataDir string) error {
|
||||
d.log.Info("connected to database", "path", dbPath)
|
||||
}
|
||||
|
||||
// Run migrations
|
||||
return d.migrate()
|
||||
err = d.migrate()
|
||||
if err != nil {
|
||||
return fmt.Errorf("migrating %s: %w", dbPath, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Database) migrate() error {
|
||||
|
||||
@@ -1,9 +1,15 @@
|
||||
package database_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/fx/fxtest"
|
||||
"sneak.berlin/go/webhooker/internal/config"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
@@ -100,3 +106,22 @@ func TestDatabaseConnection(t *testing.T) {
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpen_UnreadableDatabaseIsNamed pins
|
||||
// https://git.eeqj.de/sneak/webhooker/issues/459: when SQLite cannot
|
||||
// read webhooker.db, the error that stops the server and `webhooker
|
||||
// resetpw` names the file, not only SQLite's own message.
|
||||
func TestOpen_UnreadableDatabaseIsNamed(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, database.MainDBFileName)
|
||||
require.NoError(t, os.WriteFile(
|
||||
path, bytes.Repeat([]byte("junk"), 1024), database.SQLiteFilePerm,
|
||||
))
|
||||
|
||||
_, err := database.Open(dir, slog.New(slog.DiscardHandler))
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), path)
|
||||
assert.Contains(t, err.Error(), "file is not a database")
|
||||
}
|
||||
|
||||
@@ -199,6 +199,77 @@ func TestStatisticsQueriesUseTheirIndexes(t *testing.T) {
|
||||
"(deleted_at=? AND created_at>?)")
|
||||
}
|
||||
|
||||
// TestResubmitCountUsesItsIndex does the same for the event log's count
|
||||
// of the events resubmitted from each of a page's events (resubmitCounts
|
||||
// in the handlers). It passes a full page of 25 ids: with an index on
|
||||
// resubmitted_from_id alone, SQLite uses it for three ids and turns to
|
||||
// the deleted_at index from five.
|
||||
func TestResubmitCountUsesItsIndex(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
mgr, lc := setupTestWebhookDBManager(t)
|
||||
ctx := context.Background()
|
||||
require.NoError(t, lc.Start(ctx))
|
||||
|
||||
defer func() { require.NoError(t, lc.Stop(ctx)) }()
|
||||
|
||||
db, err := mgr.GetDB(uuid.New().String())
|
||||
require.NoError(t, err)
|
||||
|
||||
dry := db.Session(&gorm.Session{DryRun: true})
|
||||
|
||||
page := make([]string, 25)
|
||||
for i := range page {
|
||||
page[i] = uuid.New().String()
|
||||
}
|
||||
|
||||
var counts []struct{ Total int }
|
||||
|
||||
assertPlanUses(t, db, dry.Model(&database.Event{}).
|
||||
Select("resubmitted_from_id, count(*) AS total").
|
||||
Where("resubmitted_from_id IN ?", page).
|
||||
Group("resubmitted_from_id").Find(&counts),
|
||||
"idx_events_resubmitted_from_id "+
|
||||
"(resubmitted_from_id=? AND deleted_at=?)")
|
||||
}
|
||||
|
||||
// TestEntrypointEventsUseTheirIndex does the same for the webhook
|
||||
// page's count, for each entrypoint, of the events that arrived on its
|
||||
// URL since the retention cutoff (addEntrypointEvents in the
|
||||
// handlers), which must come from the index alone. It passes 25
|
||||
// entrypoints, as TestResubmitCountUsesItsIndex passes 25 events.
|
||||
func TestEntrypointEventsUseTheirIndex(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
mgr, lc := setupTestWebhookDBManager(t)
|
||||
ctx := context.Background()
|
||||
require.NoError(t, lc.Start(ctx))
|
||||
|
||||
defer func() { require.NoError(t, lc.Stop(ctx)) }()
|
||||
|
||||
db, err := mgr.GetDB(uuid.New().String())
|
||||
require.NoError(t, err)
|
||||
|
||||
dry := db.Session(&gorm.Session{DryRun: true})
|
||||
|
||||
entrypoints := make([]string, 25)
|
||||
for i := range entrypoints {
|
||||
entrypoints[i] = uuid.New().String()
|
||||
}
|
||||
|
||||
var rows []struct{ Events int }
|
||||
|
||||
assertPlanUses(t, db, dry.Model(&database.Event{}).
|
||||
Select("entrypoint_id, count(*) AS events").
|
||||
Where("entrypoint_id IN ? AND resubmitted_from_id IS NULL",
|
||||
entrypoints).
|
||||
Where("created_at >= ?", time.Now()).
|
||||
Group("entrypoint_id").Find(&rows),
|
||||
"COVERING INDEX idx_events_entrypoint_id "+
|
||||
"(entrypoint_id=? AND deleted_at=? AND "+
|
||||
"resubmitted_from_id=? AND created_at>?)")
|
||||
}
|
||||
|
||||
// assertPlanUses asserts that SQLite's plan for a statement GORM built
|
||||
// in a dry run, run with the same SQL and arguments GORM would send,
|
||||
// names each of the given indexes.
|
||||
|
||||
@@ -15,6 +15,7 @@ type APIKey struct {
|
||||
Description string `json:"description"`
|
||||
LastUsedAt *time.Time `json:"lastUsedAt,omitempty"`
|
||||
|
||||
// Relations
|
||||
User User `json:"user,omitzero"`
|
||||
// Relations. No model marshals the record it belongs to:
|
||||
// User.APIKeys leads back here, and the JSON could loop.
|
||||
User User `json:"-"`
|
||||
}
|
||||
|
||||
@@ -56,8 +56,14 @@ type Delivery struct {
|
||||
// the index.
|
||||
FinishedAt *time.Time `gorm:"index:idx_deliveries_status,priority:3" json:"finishedAt,omitempty"`
|
||||
|
||||
// Relations
|
||||
Event Event `json:"event,omitzero"`
|
||||
Target Target `json:"target,omitzero"`
|
||||
// Replay is set on a delivery created by the event log's Replay
|
||||
// action, so the pages can tell it from the delivery it repeats.
|
||||
Replay bool `gorm:"not null;default:false" json:"replay"`
|
||||
|
||||
// Relations. No model marshals the record it belongs to:
|
||||
// Event.Deliveries and Target.Deliveries lead back here, and the
|
||||
// JSON could loop.
|
||||
Event Event `json:"-"`
|
||||
Target Target `json:"-"`
|
||||
DeliveryResults []DeliveryResult `json:"deliveryResults,omitempty"`
|
||||
}
|
||||
|
||||
@@ -23,6 +23,7 @@ type DeliveryResult struct {
|
||||
Error string `json:"error,omitempty"`
|
||||
Duration int64 `json:"durationMs"` // Duration in milliseconds
|
||||
|
||||
// Relations
|
||||
Delivery Delivery `json:"delivery,omitzero"`
|
||||
// Relations. No model marshals the record it belongs to:
|
||||
// Delivery.DeliveryResults leads back here, and the JSON could loop.
|
||||
Delivery Delivery `json:"-"`
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ type Entrypoint struct {
|
||||
Description string `json:"description"`
|
||||
Active bool `gorm:"default:true" json:"active"`
|
||||
|
||||
// Relations
|
||||
Webhook Webhook `json:"webhook,omitzero"`
|
||||
// Relations. No model marshals the record it belongs to:
|
||||
// Webhook.Entrypoints leads back here, and the JSON could loop.
|
||||
Webhook Webhook `json:"-"`
|
||||
}
|
||||
|
||||
@@ -19,11 +19,16 @@ type Event struct {
|
||||
// narrows by a < only on the last column it uses. Its final delete
|
||||
// has no deleted_at condition and uses the index on created_at
|
||||
// alone. The other tables keep the unindexed BaseModel created_at.
|
||||
CreatedAt time.Time `gorm:"index;index:idx_events_deleted_at_created_at,priority:2" json:"createdAt"`
|
||||
DeletedAt gorm.DeletedAt `gorm:"index:idx_events_deleted_at_created_at,priority:1" json:"deletedAt,omitzero"`
|
||||
// DeletedAt is also the second column of the resubmitted_from_id
|
||||
// index, for the reason DeliveryResult gives. The entrypoint_id
|
||||
// index, for the webhook page's entrypoint list, has it second too,
|
||||
// resubmitted_from_id third, and created_at last, which the list
|
||||
// compares with a range.
|
||||
CreatedAt time.Time `gorm:"index;index:idx_events_deleted_at_created_at,priority:2;index:idx_events_entrypoint_id,priority:4" json:"createdAt"`
|
||||
DeletedAt gorm.DeletedAt `gorm:"index:idx_events_deleted_at_created_at,priority:1;index:idx_events_resubmitted_from_id,priority:2;index:idx_events_entrypoint_id,priority:2" json:"deletedAt,omitzero"`
|
||||
|
||||
WebhookID string `gorm:"type:uuid;not null" json:"webhookId"`
|
||||
EntrypointID string `gorm:"type:uuid;not null" json:"entrypointId"`
|
||||
WebhookID string `gorm:"type:uuid;not null" json:"webhookId"`
|
||||
EntrypointID string `gorm:"type:uuid;not null;index:idx_events_entrypoint_id,priority:1" json:"entrypointId"`
|
||||
|
||||
// Request data
|
||||
Method string `gorm:"not null" json:"method"`
|
||||
@@ -32,8 +37,8 @@ type Event struct {
|
||||
ContentType string `json:"contentType"`
|
||||
|
||||
// BodyBytes is the size of Body in bytes, recorded when the event
|
||||
// is stored so the recent events list can show it without reading
|
||||
// the body.
|
||||
// is stored, so that the recent events list, which reads only the
|
||||
// start of each body, knows the whole body's size.
|
||||
BodyBytes int64 `gorm:"not null" json:"bodyBytes"`
|
||||
|
||||
// ResubmittedFromID names the event this one was copied from by
|
||||
@@ -42,10 +47,11 @@ type Event struct {
|
||||
// existed. It is not a foreign key: the source event can be
|
||||
// reaped by retention while its copies remain, and the id is
|
||||
// kept as the record of where the copy came from either way.
|
||||
ResubmittedFromID *string `gorm:"type:uuid;index" json:"resubmittedFromId,omitempty"`
|
||||
ResubmittedFromID *string `gorm:"type:uuid;index:idx_events_resubmitted_from_id,priority:1;index:idx_events_entrypoint_id,priority:3" json:"resubmittedFromId,omitempty"`
|
||||
|
||||
// Relations
|
||||
Webhook Webhook `json:"webhook,omitzero"`
|
||||
Entrypoint Entrypoint `json:"entrypoint,omitzero"`
|
||||
// Relations. No model marshals the record it belongs to, so
|
||||
// Webhook and Entrypoint are left out of the JSON.
|
||||
Webhook Webhook `json:"-"`
|
||||
Entrypoint Entrypoint `json:"-"`
|
||||
Deliveries []Delivery `json:"deliveries,omitempty"`
|
||||
}
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
package database_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
)
|
||||
|
||||
// TestPreloadedModelsMarshalWithoutTheirParent pins that a child's
|
||||
// reference to the record it belongs to is left out of the JSON, so a
|
||||
// webhook and its targets cannot marshal each other in a loop, and that
|
||||
// GORM still preloads that reference, since it ignores json tags.
|
||||
func TestPreloadedModelsMarshalWithoutTheirParent(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db := startedTestDB(t)
|
||||
|
||||
stored := database.Webhook{
|
||||
UserID: uuid.New().String(),
|
||||
Name: testWebhookName,
|
||||
Entrypoints: []database.Entrypoint{{Path: uuid.New().String()}},
|
||||
Targets: []database.Target{{
|
||||
Name: "log",
|
||||
Type: database.TargetTypeLog,
|
||||
}},
|
||||
}
|
||||
require.NoError(t, db.Create(&stored).Error)
|
||||
|
||||
entrypointID := stored.Entrypoints[0].ID
|
||||
targetID := stored.Targets[0].ID
|
||||
|
||||
var webhook database.Webhook
|
||||
|
||||
require.NoError(t, db.
|
||||
Preload("Entrypoints.Webhook").
|
||||
Preload("Targets.Webhook").
|
||||
First(&webhook, "id = ?", stored.ID).Error)
|
||||
|
||||
require.Len(t, webhook.Entrypoints, 1)
|
||||
require.Len(t, webhook.Targets, 1)
|
||||
assert.Equal(t, stored.ID, webhook.Entrypoints[0].Webhook.ID)
|
||||
assert.Equal(t, stored.ID, webhook.Targets[0].Webhook.ID)
|
||||
|
||||
encoded := marshalModel(t, webhook)
|
||||
|
||||
assert.Contains(t, encoded, entrypointID)
|
||||
assert.Contains(t, encoded, targetID)
|
||||
|
||||
// Each child holds the parent's id as its webhookId, so the parent
|
||||
// is looked for by its own id field.
|
||||
parentIDField := `"id":"` + stored.ID + `"`
|
||||
|
||||
assert.NotContains(t, marshalModel(t, webhook.Entrypoints[0]), parentIDField)
|
||||
assert.NotContains(t, marshalModel(t, webhook.Targets[0]), parentIDField)
|
||||
|
||||
var target database.Target
|
||||
|
||||
require.NoError(t, db.
|
||||
Preload("Webhook").
|
||||
First(&target, "id = ?", targetID).Error)
|
||||
|
||||
assert.Equal(t, stored.ID, target.Webhook.ID)
|
||||
|
||||
encoded = marshalModel(t, target)
|
||||
|
||||
assert.Contains(t, encoded, stored.ID)
|
||||
assert.NotContains(t, encoded, parentIDField)
|
||||
}
|
||||
|
||||
// TestModelsMarshalWithoutTheirParent covers the other references to a
|
||||
// parent: each model is built with its parent set, and the parent's id
|
||||
// must not appear in the JSON.
|
||||
func TestModelsMarshalWithoutTheirParent(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
parent := database.BaseModel{ID: uuid.New().String()}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
model any
|
||||
}{
|
||||
{
|
||||
name: "Webhook.User",
|
||||
model: database.Webhook{User: database.User{BaseModel: parent}},
|
||||
},
|
||||
{
|
||||
name: "APIKey.User",
|
||||
model: database.APIKey{User: database.User{BaseModel: parent}},
|
||||
},
|
||||
{
|
||||
name: "Delivery.Event",
|
||||
model: database.Delivery{Event: database.Event{BaseModel: parent}},
|
||||
},
|
||||
{
|
||||
name: "Delivery.Target",
|
||||
model: database.Delivery{Target: database.Target{BaseModel: parent}},
|
||||
},
|
||||
{
|
||||
name: "DeliveryResult.Delivery",
|
||||
model: database.DeliveryResult{
|
||||
Delivery: database.Delivery{BaseModel: parent},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Event.Webhook",
|
||||
model: database.Event{Webhook: database.Webhook{BaseModel: parent}},
|
||||
},
|
||||
{
|
||||
name: "Event.Entrypoint",
|
||||
model: database.Event{
|
||||
Entrypoint: database.Entrypoint{BaseModel: parent},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
assert.NotContains(t, marshalModel(t, tc.model), parent.ID)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -31,10 +31,10 @@ type Target struct {
|
||||
|
||||
// For HTTP targets (max_retries=0 means fire-and-forget,
|
||||
// >0 enables retries with backoff)
|
||||
MaxRetries int `json:"maxRetries,omitempty"`
|
||||
MaxQueueSize int `json:"maxQueueSize,omitempty"`
|
||||
MaxRetries int `json:"maxRetries,omitempty"`
|
||||
|
||||
// Relations
|
||||
Webhook Webhook `json:"webhook,omitzero"`
|
||||
// Relations. No model marshals the record it belongs to:
|
||||
// Webhook.Targets leads back here, and the JSON could loop.
|
||||
Webhook Webhook `json:"-"`
|
||||
Deliveries []Delivery `json:"deliveries,omitempty"`
|
||||
}
|
||||
|
||||
@@ -52,6 +52,21 @@ func (TargetTotals) TableName() string {
|
||||
return "target_totals"
|
||||
}
|
||||
|
||||
// EntrypointTotals is one row per entrypoint, created by the first
|
||||
// event that arrives on its URL: when the newest such event arrived,
|
||||
// which retention leaves as it is. A resubmitted copy did not arrive
|
||||
// on the URL and does not change it.
|
||||
type EntrypointTotals struct {
|
||||
EntrypointID string `gorm:"type:uuid;primaryKey"`
|
||||
|
||||
LastEventAt time.Time `gorm:"not null"`
|
||||
}
|
||||
|
||||
// TableName names the table AddEntrypointTotals updates.
|
||||
func (EntrypointTotals) TableName() string {
|
||||
return "entrypoint_totals"
|
||||
}
|
||||
|
||||
// AddEventTotals adds each count in add to the webhook's event totals,
|
||||
// and records add.LastEventAt as when the newest event arrived if it is
|
||||
// set. Call it on the transaction that writes or deletes the events it
|
||||
@@ -97,3 +112,25 @@ func AddTargetTotals(tx *gorm.DB, add TargetTotals) error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// AddEntrypointTotals records add.LastEventAt as when the newest event
|
||||
// arrived on the URL of the entrypoint add.EntrypointID names, creating
|
||||
// its row the first time. Call it on the transaction that stores the
|
||||
// event.
|
||||
func AddEntrypointTotals(tx *gorm.DB, add EntrypointTotals) error {
|
||||
err := tx.Exec(
|
||||
`INSERT INTO entrypoint_totals (entrypoint_id, last_event_at)
|
||||
VALUES (?, ?)
|
||||
ON CONFLICT (entrypoint_id) DO UPDATE SET
|
||||
last_event_at = excluded.last_event_at`,
|
||||
add.EntrypointID, add.LastEventAt,
|
||||
).Error
|
||||
if err != nil {
|
||||
return fmt.Errorf(
|
||||
"adding to totals of entrypoint %s: %w",
|
||||
add.EntrypointID, err,
|
||||
)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -66,8 +66,9 @@ type Webhook struct {
|
||||
// must equal DefaultRetentionDays.
|
||||
RetentionDays int `gorm:"default:30" json:"retentionDays"`
|
||||
|
||||
// Relations
|
||||
User User `json:"user,omitzero"`
|
||||
// Relations. No model marshals the record it belongs to:
|
||||
// User.Webhooks leads back here, and the JSON could loop.
|
||||
User User `json:"-"`
|
||||
Entrypoints []Entrypoint `json:"entrypoints,omitempty"`
|
||||
Targets []Target `json:"targets,omitempty"`
|
||||
}
|
||||
@@ -110,6 +111,13 @@ func (w *Webhook) RetainsForever() bool {
|
||||
return retainsForever(w.RetentionDays)
|
||||
}
|
||||
|
||||
// RetentionCutoff returns the time before which this webhook's events
|
||||
// have expired, as the reaper computes it, and false when the webhook
|
||||
// retains them forever.
|
||||
func (w *Webhook) RetentionCutoff(now time.Time) (time.Time, bool) {
|
||||
return retentionCutoff(now, w.RetentionDays)
|
||||
}
|
||||
|
||||
// RetentionLabel returns the webhook's retention policy as display
|
||||
// text, so that no template has to know about the sentinel value.
|
||||
func (w *Webhook) RetentionLabel() string {
|
||||
|
||||
@@ -3,7 +3,7 @@ package database
|
||||
// Migrate runs database migrations for the main application database.
|
||||
// Only configuration-tier models are stored in the main database.
|
||||
// Event-tier models (Event, Delivery, DeliveryResult, EventTotals,
|
||||
// TargetTotals) live in
|
||||
// TargetTotals, EntrypointTotals) live in
|
||||
// per-webhook dedicated databases managed by WebhookDBManager.
|
||||
func (d *Database) Migrate() error {
|
||||
return d.db.AutoMigrate(
|
||||
|
||||
@@ -184,18 +184,8 @@ func (r *RetentionReaper) sweep(ctx context.Context) {
|
||||
|
||||
wh := webhooks[i]
|
||||
|
||||
// Skip retain-forever webhooks before building any query.
|
||||
// RetainsForever covers both the RetentionForeverDays
|
||||
// sentinel and the non-positive values that predate it: the
|
||||
// sentinel is a positive number, so without this the reaper
|
||||
// would compute a cutoff a thousand years in the past and
|
||||
// issue a DELETE matching nothing on every single sweep.
|
||||
if wh.RetainsForever() {
|
||||
continue
|
||||
}
|
||||
|
||||
// Nothing to reap if the per-webhook database has never
|
||||
// been created.
|
||||
// A missing database has nothing to reap. Restart recovery
|
||||
// reports a lost one (see WebhookDBManager.GetDB).
|
||||
if !r.dbManager.DBExists(wh.ID) {
|
||||
continue
|
||||
}
|
||||
@@ -212,6 +202,13 @@ func (r *RetentionReaper) reapWebhook(
|
||||
webhookID string,
|
||||
retentionDays int,
|
||||
) {
|
||||
// A retain-forever webhook has no cutoff, so its database is not
|
||||
// even opened.
|
||||
cutoff, ok := retentionCutoff(time.Now(), retentionDays)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
db, err := r.dbManager.GetDB(webhookID)
|
||||
if err != nil {
|
||||
r.log.Error(
|
||||
@@ -223,11 +220,6 @@ func (r *RetentionReaper) reapWebhook(
|
||||
return
|
||||
}
|
||||
|
||||
cutoff, ok := retentionCutoff(time.Now(), retentionDays)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
deleted, err := reapExpired(ctx, db, cutoff)
|
||||
if err != nil {
|
||||
r.log.Error(
|
||||
|
||||
@@ -362,7 +362,7 @@ func TestRetentionReaper_HugeFiniteRetentionRetainsRecentEvents(
|
||||
t,
|
||||
overflowingRetentionDays,
|
||||
database.RetentionForeverDays,
|
||||
"the test value must not be rescued by the forever skip",
|
||||
"the test value must not be treated as retain-forever",
|
||||
)
|
||||
|
||||
webhookID := createWebhook(
|
||||
|
||||
@@ -182,6 +182,29 @@ func TestOpenSQLiteTightensFilesLeftWorldReadable(t *testing.T) {
|
||||
requireDatabaseSetOwnerOnly(t, path)
|
||||
}
|
||||
|
||||
// TestOpenSQLiteRefusesADirectorySidecar covers a directory in place
|
||||
// of -wal or -shm. Beside a -shm directory SQLite opens the database
|
||||
// read-only without a word, and every write then fails naming no file,
|
||||
// so the open must stop instead, naming the directory.
|
||||
func TestOpenSQLiteRefusesADirectorySidecar(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, suffix := range []string{"-wal", "-shm"} {
|
||||
t.Run(suffix, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
path := filepath.Join(t.TempDir(), database.MainDBFileName)
|
||||
require.NoError(t, os.Mkdir(path+suffix, 0o700))
|
||||
|
||||
_, err := database.OpenSQLite(
|
||||
path, database.SQLiteModeCreate,
|
||||
)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), path+suffix)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenSQLiteExistingModeDoesNotCreateTheFile guards the mechanism
|
||||
// the fix uses: OpenSQLite now creates the database file itself, and
|
||||
// must not do so for a caller that asked for an existing database. An
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"io/fs"
|
||||
"net/url"
|
||||
"os"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
_ "modernc.org/sqlite" // Pure Go SQLite driver
|
||||
@@ -93,7 +94,8 @@ const (
|
||||
const SQLiteFilePerm fs.FileMode = 0o600
|
||||
|
||||
// reserveSQLiteFile puts path at SQLiteFilePerm before the driver ever
|
||||
// touches it, and tightens any sidecar already on disk.
|
||||
// touches it, and tightens any sidecar already on disk. A directory in
|
||||
// place of any of them is an error naming it.
|
||||
//
|
||||
// The mode has to be settled here rather than by a chmod after opening,
|
||||
// because SQLite picks it: robust_open substitutes
|
||||
@@ -143,7 +145,15 @@ func reserveSQLiteFile(path string, create bool) error {
|
||||
for _, p := range append(
|
||||
[]string{path}, sqliteSidecarPaths(path)...,
|
||||
) {
|
||||
err := os.Chmod(p, SQLiteFilePerm)
|
||||
// Chmod accepts a directory, and SQLite opens a database whose
|
||||
// -shm is one read-only, without a word: every write then
|
||||
// fails naming no file.
|
||||
info, err := os.Stat(p)
|
||||
if err == nil && info.IsDir() {
|
||||
return fmt.Errorf("securing %s: %w", p, syscall.EISDIR)
|
||||
}
|
||||
|
||||
err = os.Chmod(p, SQLiteFilePerm)
|
||||
if err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||
return fmt.Errorf("securing %s: %w", p, err)
|
||||
}
|
||||
@@ -152,6 +162,20 @@ func reserveSQLiteFile(path string, create bool) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// missingOrEmpty reports whether opening path in SQLiteModeCreate
|
||||
// would start a new, empty database: the file is not there, or it is
|
||||
// zero-length, which SQLite opens as an empty database. A file left at
|
||||
// zero length by an interrupted first start or a truncated copy holds
|
||||
// as little as a missing one, and must be reported the same way.
|
||||
func missingOrEmpty(path string) bool {
|
||||
info, err := os.Stat(path)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return true
|
||||
}
|
||||
|
||||
return err == nil && info.Size() == 0
|
||||
}
|
||||
|
||||
// sqliteSidecarPaths returns the files SQLite maintains beside a
|
||||
// database under WAL. They carry the same rows as the database itself,
|
||||
// so a fix that tightens only the main file has fixed nothing.
|
||||
|
||||
@@ -102,6 +102,13 @@ func TestWebhookDBManager_TotalsSurviveReopen(t *testing.T) {
|
||||
// seedExpiredEvents stores count events created at the given time,
|
||||
// each with a delivered delivery to one target and a failed delivery
|
||||
// to the other, and one attempt for each delivery.
|
||||
//
|
||||
// It and seedBareEvents insert 50 rows per statement, not more. The
|
||||
// SQLite driver looks up each parameter's value by scanning the
|
||||
// statement's arguments from the first until it reaches that
|
||||
// parameter's, so the time to bind a statement grows with the square of
|
||||
// its parameter count: at 500 rows, several thousand parameters, the
|
||||
// seeding took most of these tests' time under -race.
|
||||
func seedExpiredEvents(
|
||||
t *testing.T,
|
||||
db *gorm.DB,
|
||||
@@ -138,8 +145,8 @@ func seedExpiredEvents(
|
||||
)
|
||||
}
|
||||
|
||||
require.NoError(t, db.CreateInBatches(events, 500).Error)
|
||||
require.NoError(t, db.CreateInBatches(deliveries, 500).Error)
|
||||
require.NoError(t, db.CreateInBatches(events, 50).Error)
|
||||
require.NoError(t, db.CreateInBatches(deliveries, 50).Error)
|
||||
|
||||
results := make([]database.DeliveryResult, len(deliveries))
|
||||
for i := range deliveries {
|
||||
@@ -148,7 +155,7 @@ func seedExpiredEvents(
|
||||
}
|
||||
}
|
||||
|
||||
require.NoError(t, db.CreateInBatches(results, 500).Error)
|
||||
require.NoError(t, db.CreateInBatches(results, 50).Error)
|
||||
}
|
||||
|
||||
// seedBareEvents stores count events created at the given time, with
|
||||
@@ -172,7 +179,7 @@ func seedBareEvents(
|
||||
events[i].CreatedAt = createdAt
|
||||
}
|
||||
|
||||
require.NoError(t, db.CreateInBatches(events, 500).Error)
|
||||
require.NoError(t, db.CreateInBatches(events, 50).Error)
|
||||
}
|
||||
|
||||
// TestRetentionReaper_PrunesMoreThanOneBatch verifies that a prune
|
||||
|
||||
@@ -33,10 +33,23 @@ var errInvalidCachedDBType = errors.New(
|
||||
"invalid cached database type",
|
||||
)
|
||||
|
||||
// ErrEventDBNotRemoved is in DeleteDB's error when the event
|
||||
// database file itself could not be removed: it is still on disk.
|
||||
var ErrEventDBNotRemoved = errors.New(
|
||||
"event database file not removed",
|
||||
)
|
||||
|
||||
// ErrSidecarNotRemoved is in DeleteDB's error when the event
|
||||
// database file was removed, so its events are gone, but its -wal
|
||||
// or -shm sidecar could not be.
|
||||
var ErrSidecarNotRemoved = errors.New(
|
||||
"event database file removed, but a -wal or -shm sidecar was not",
|
||||
)
|
||||
|
||||
// WebhookDBManager manages per-webhook SQLite database files
|
||||
// for event storage. Each webhook gets its own dedicated
|
||||
// database containing Events, Deliveries, DeliveryResults and the
|
||||
// running totals of them (EventTotals, TargetTotals).
|
||||
// running totals of them (EventTotals, TargetTotals, EntrypointTotals).
|
||||
// Database connections are opened lazily and cached.
|
||||
type WebhookDBManager struct {
|
||||
dataDir string
|
||||
@@ -85,34 +98,37 @@ func NewWebhookDBManager(
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// GetDB returns the database connection for a webhook,
|
||||
// creating the database file lazily if it doesn't exist.
|
||||
// GetDB returns the database connection for a webhook, opening it on
|
||||
// first use.
|
||||
//
|
||||
// The file is made by CreateDB when the webhook is created. One that is
|
||||
// missing or zero-length here means the webhook's events and pending
|
||||
// deliveries are gone: an empty database is created in its place so
|
||||
// the webhook keeps receiving, and that is logged as a warning naming
|
||||
// the file, as a new main database is.
|
||||
func (m *WebhookDBManager) GetDB(
|
||||
webhookID string,
|
||||
) (*gorm.DB, error) {
|
||||
// Fast path: already open
|
||||
if val, ok := m.dbs.Load(webhookID); ok {
|
||||
return asGormDB(val, webhookID)
|
||||
}
|
||||
return m.getDB(webhookID, false)
|
||||
}
|
||||
|
||||
// Slow path: open the database under the lock, looking in the
|
||||
// cache again first. A caller that raced another one here then
|
||||
// waits for its handle instead of opening a second one.
|
||||
// GetDBIf is GetDB, done only when check reports true. check runs under
|
||||
// the lock DeleteDB holds while it removes the files, so a caller can
|
||||
// confirm the webhook still exists and open its database with no delete
|
||||
// in between. The handle is nil when check reports false. check must
|
||||
// not call the manager.
|
||||
func (m *WebhookDBManager) GetDBIf(
|
||||
webhookID string, check func() (bool, error),
|
||||
) (*gorm.DB, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
if val, ok := m.dbs.Load(webhookID); ok {
|
||||
return asGormDB(val, webhookID)
|
||||
}
|
||||
|
||||
db, err := m.openDB(webhookID)
|
||||
if err != nil {
|
||||
ok, err := check()
|
||||
if err != nil || !ok {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
m.dbs.Store(webhookID, db)
|
||||
|
||||
return db, nil
|
||||
return m.getDBLocked(webhookID, false)
|
||||
}
|
||||
|
||||
// asGormDB returns a value read from the cache as the database
|
||||
@@ -130,12 +146,12 @@ func asGormDB(val any, webhookID string) (*gorm.DB, error) {
|
||||
return db, nil
|
||||
}
|
||||
|
||||
// CreateDB explicitly creates a new per-webhook database file
|
||||
// and runs migrations.
|
||||
// CreateDB creates a new webhook's database file and runs
|
||||
// migrations.
|
||||
func (m *WebhookDBManager) CreateDB(
|
||||
webhookID string,
|
||||
) error {
|
||||
_, err := m.GetDB(webhookID)
|
||||
_, err := m.getDB(webhookID, true)
|
||||
|
||||
return err
|
||||
}
|
||||
@@ -151,7 +167,10 @@ func (m *WebhookDBManager) DBExists(
|
||||
}
|
||||
|
||||
// DeleteDB closes the connection and deletes the database file
|
||||
// for a webhook. The file is permanently removed.
|
||||
// for a webhook, with its -wal and -shm sidecars. The files are
|
||||
// permanently removed. Each file is tried even when another could
|
||||
// not be removed, and the error wraps ErrEventDBNotRemoved or
|
||||
// ErrSidecarNotRemoved to say which was left, naming each file.
|
||||
func (m *WebhookDBManager) DeleteDB(
|
||||
webhookID string,
|
||||
) error {
|
||||
@@ -170,16 +189,23 @@ func (m *WebhookDBManager) DeleteDB(
|
||||
}
|
||||
}
|
||||
|
||||
// Delete the main DB file and WAL/SHM files
|
||||
path := m.dbPath(webhookID)
|
||||
for _, suffix := range []string{"", "-wal", "-shm"} {
|
||||
err := os.Remove(path + suffix)
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return fmt.Errorf(
|
||||
"deleting webhook database file %s%s: %w",
|
||||
path, suffix, err,
|
||||
)
|
||||
}
|
||||
|
||||
dbErr := removeFile(path)
|
||||
sidecarErr := errors.Join(
|
||||
removeFile(path+"-wal"),
|
||||
removeFile(path+"-shm"),
|
||||
)
|
||||
|
||||
if dbErr != nil {
|
||||
return fmt.Errorf(
|
||||
"%w: %w",
|
||||
ErrEventDBNotRemoved, errors.Join(dbErr, sidecarErr),
|
||||
)
|
||||
}
|
||||
|
||||
if sidecarErr != nil {
|
||||
return fmt.Errorf("%w: %w", ErrSidecarNotRemoved, sidecarErr)
|
||||
}
|
||||
|
||||
m.log.Info(
|
||||
@@ -190,6 +216,17 @@ func (m *WebhookDBManager) DeleteDB(
|
||||
return nil
|
||||
}
|
||||
|
||||
// removeFile removes path. A file that is already gone counts as
|
||||
// removed; the error from any other failure names the file.
|
||||
func removeFile(path string) error {
|
||||
err := os.Remove(path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// CloseAll closes all open per-webhook database connections.
|
||||
// Called during application shutdown.
|
||||
func (m *WebhookDBManager) CloseAll() error {
|
||||
@@ -232,6 +269,54 @@ func (m *WebhookDBManager) DBPath(
|
||||
return m.dbPath(webhookID)
|
||||
}
|
||||
|
||||
// getDB is GetDB, and CreateDB when isNew is true: the webhook has just
|
||||
// been created, so a missing file is expected rather than lost.
|
||||
func (m *WebhookDBManager) getDB(
|
||||
webhookID string, isNew bool,
|
||||
) (*gorm.DB, error) {
|
||||
// Fast path: already open
|
||||
if val, ok := m.dbs.Load(webhookID); ok {
|
||||
return asGormDB(val, webhookID)
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
return m.getDBLocked(webhookID, isNew)
|
||||
}
|
||||
|
||||
// getDBLocked is getDB's slow path, run with m.mu held. It looks in the
|
||||
// cache again first: a caller that raced another one to the lock then
|
||||
// gets its handle instead of opening a second one.
|
||||
func (m *WebhookDBManager) getDBLocked(
|
||||
webhookID string, isNew bool,
|
||||
) (*gorm.DB, error) {
|
||||
if val, ok := m.dbs.Load(webhookID); ok {
|
||||
return asGormDB(val, webhookID)
|
||||
}
|
||||
|
||||
// Checked before opening, which creates the file. See GetDB.
|
||||
path := m.dbPath(webhookID)
|
||||
replaced := !isNew && missingOrEmpty(path)
|
||||
|
||||
db, err := m.openDB(webhookID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if replaced {
|
||||
m.log.Warn(
|
||||
"created a new, empty database",
|
||||
"webhook_id", webhookID,
|
||||
"path", path,
|
||||
)
|
||||
}
|
||||
|
||||
m.dbs.Store(webhookID, db)
|
||||
|
||||
return db, nil
|
||||
}
|
||||
|
||||
func (m *WebhookDBManager) dbPath(
|
||||
webhookID string,
|
||||
) string {
|
||||
@@ -296,7 +381,7 @@ func (m *WebhookDBManager) openDB(
|
||||
// Run migrations for event-tier models only
|
||||
err = db.AutoMigrate(
|
||||
&Event{}, &Delivery{}, &DeliveryResult{},
|
||||
&EventTotals{}, &TargetTotals{},
|
||||
&EventTotals{}, &TargetTotals{}, &EntrypointTotals{},
|
||||
)
|
||||
if err != nil {
|
||||
_ = sqlDB.Close()
|
||||
|
||||
@@ -182,17 +182,91 @@ func TestWebhookDBManager_DeleteDB(t *testing.T) {
|
||||
}
|
||||
require.NoError(t, db.Create(event).Error)
|
||||
|
||||
// Under WAL, an open database that has been written to has both
|
||||
// sidecars beside it.
|
||||
dbPath := mgr.DBPath(webhookID)
|
||||
require.FileExists(t, dbPath+"-wal")
|
||||
require.FileExists(t, dbPath+"-shm")
|
||||
|
||||
// Delete the DB
|
||||
require.NoError(t, mgr.DeleteDB(webhookID))
|
||||
|
||||
// File should no longer exist
|
||||
assert.False(t, mgr.DBExists(webhookID))
|
||||
|
||||
// Verify the file is actually gone from disk
|
||||
// Verify the files are actually gone from disk
|
||||
assert.NoFileExists(t, dbPath)
|
||||
assert.NoFileExists(t, dbPath+"-wal")
|
||||
assert.NoFileExists(t, dbPath+"-shm")
|
||||
}
|
||||
|
||||
// blockRemoval puts a non-empty directory at path, which os.Remove
|
||||
// cannot remove whoever runs the test, root included.
|
||||
func blockRemoval(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(path, "keep"), 0o700))
|
||||
}
|
||||
|
||||
// TestWebhookDBManager_DeleteDBKeepsDatabaseFile proves that when the
|
||||
// event database file cannot be removed, the error says so, and both
|
||||
// sidecars are still removed.
|
||||
func TestWebhookDBManager_DeleteDBKeepsDatabaseFile(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
mgr, lc := setupTestWebhookDBManager(t)
|
||||
ctx := context.Background()
|
||||
require.NoError(t, lc.Start(ctx))
|
||||
|
||||
defer func() { require.NoError(t, lc.Stop(ctx)) }()
|
||||
|
||||
webhookID := uuid.New().String()
|
||||
dbPath := mgr.DBPath(webhookID)
|
||||
|
||||
_, err = os.Stat(dbPath)
|
||||
assert.True(t, os.IsNotExist(err))
|
||||
blockRemoval(t, dbPath)
|
||||
require.NoError(t, os.WriteFile(dbPath+"-wal", nil, 0o600))
|
||||
require.NoError(t, os.WriteFile(dbPath+"-shm", nil, 0o600))
|
||||
|
||||
err := mgr.DeleteDB(webhookID)
|
||||
|
||||
require.ErrorIs(t, err, database.ErrEventDBNotRemoved)
|
||||
require.NotErrorIs(t, err, database.ErrSidecarNotRemoved)
|
||||
assert.Contains(t, err.Error(), dbPath)
|
||||
assert.NoFileExists(t, dbPath+"-wal")
|
||||
assert.NoFileExists(t, dbPath+"-shm")
|
||||
}
|
||||
|
||||
// TestWebhookDBManager_DeleteDBKeepsSidecar proves that when the
|
||||
// event database file is removed but a sidecar is not, the error
|
||||
// says the database file is gone, and the other sidecar is still
|
||||
// removed.
|
||||
func TestWebhookDBManager_DeleteDBKeepsSidecar(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
mgr, lc := setupTestWebhookDBManager(t)
|
||||
ctx := context.Background()
|
||||
require.NoError(t, lc.Start(ctx))
|
||||
|
||||
defer func() { require.NoError(t, lc.Stop(ctx)) }()
|
||||
|
||||
webhookID := uuid.New().String()
|
||||
dbPath := mgr.DBPath(webhookID)
|
||||
|
||||
require.NoError(t, mgr.CreateDB(webhookID))
|
||||
// Closing removes the sidecars, so the ones below are the only
|
||||
// ones there.
|
||||
require.NoError(t, mgr.CloseAll())
|
||||
|
||||
blockRemoval(t, dbPath+"-wal")
|
||||
require.NoError(t, os.WriteFile(dbPath+"-shm", nil, 0o600))
|
||||
|
||||
err := mgr.DeleteDB(webhookID)
|
||||
|
||||
require.ErrorIs(t, err, database.ErrSidecarNotRemoved)
|
||||
require.NotErrorIs(t, err, database.ErrEventDBNotRemoved)
|
||||
assert.Contains(t, err.Error(), dbPath+"-wal")
|
||||
assert.NoFileExists(t, dbPath)
|
||||
assert.NoFileExists(t, dbPath+"-shm")
|
||||
}
|
||||
|
||||
func TestWebhookDBManager_LazyCreation(t *testing.T) {
|
||||
@@ -215,6 +289,75 @@ func TestWebhookDBManager_LazyCreation(t *testing.T) {
|
||||
assert.True(t, mgr.DBExists(webhookID))
|
||||
}
|
||||
|
||||
// A webhook's database is made by CreateDB along with the webhook. One
|
||||
// that GetDB finds missing or zero-length has lost the webhook's events
|
||||
// and pending deliveries, so the empty database made in its place is
|
||||
// logged as a warning naming the file
|
||||
// (https://git.eeqj.de/sneak/webhooker/issues/290). CreateDB, and
|
||||
// reopening a database that is there, log no such warning.
|
||||
func TestWebhookDBManager_LostDatabaseIsLogged(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const created = `level=WARN msg="created a new, empty database"`
|
||||
|
||||
open := func(
|
||||
t *testing.T, prepare func(*database.WebhookDBManager, string),
|
||||
) (string, string) {
|
||||
t.Helper()
|
||||
|
||||
var logs bytes.Buffer
|
||||
|
||||
mgr := database.NewTestWebhookDBManagerWithLogger(
|
||||
t.TempDir(),
|
||||
slog.New(slog.NewTextHandler(&logs, nil)),
|
||||
)
|
||||
|
||||
webhookID := uuid.New().String()
|
||||
prepare(mgr, webhookID)
|
||||
|
||||
_, err := mgr.GetDB(webhookID)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, mgr.CloseAll())
|
||||
|
||||
return logs.String(),
|
||||
" webhook_id=" + webhookID + " path=" + mgr.DBPath(webhookID)
|
||||
}
|
||||
|
||||
t.Run("missing", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
logs, fields := open(
|
||||
t, func(*database.WebhookDBManager, string) {},
|
||||
)
|
||||
assert.Contains(t, logs, created+fields)
|
||||
})
|
||||
|
||||
t.Run("zero-length", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
logs, fields := open(
|
||||
t, func(mgr *database.WebhookDBManager, webhookID string) {
|
||||
require.NoError(t, os.WriteFile(
|
||||
mgr.DBPath(webhookID), nil, database.SQLiteFilePerm,
|
||||
))
|
||||
},
|
||||
)
|
||||
assert.Contains(t, logs, created+fields)
|
||||
})
|
||||
|
||||
t.Run("created with the webhook, then reopened", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
logs, _ := open(
|
||||
t, func(mgr *database.WebhookDBManager, webhookID string) {
|
||||
require.NoError(t, mgr.CreateDB(webhookID))
|
||||
require.NoError(t, mgr.CloseAll())
|
||||
},
|
||||
)
|
||||
assert.NotContains(t, logs, created)
|
||||
})
|
||||
}
|
||||
|
||||
func TestWebhookDBManager_DeliveryWorkflow(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -45,8 +45,13 @@ type ArchiveSweeper struct {
|
||||
eng *Engine
|
||||
log *slog.Logger
|
||||
interval time.Duration
|
||||
cancel context.CancelFunc
|
||||
wg sync.WaitGroup
|
||||
|
||||
// cancel needs no lock: fx calls the stop hook only after the
|
||||
// start hook has returned, so stop never reads it while start
|
||||
// is still setting it.
|
||||
cancel context.CancelFunc
|
||||
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
// NewArchiveSweeper creates the archive sweeper and registers
|
||||
@@ -163,10 +168,18 @@ func (s *ArchiveSweeper) sweep(ctx context.Context) {
|
||||
var targets []database.Target
|
||||
|
||||
err := s.db.DB().
|
||||
WithContext(ctx).
|
||||
Model(&database.Target{}).
|
||||
Where("type = ?", database.TargetTypeDatabase).
|
||||
Find(&targets).Error
|
||||
if err != nil {
|
||||
// The app stopping as a sweep starts cancels the listing.
|
||||
// Stopping is not a failure, so it must not produce an
|
||||
// error line.
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
|
||||
s.log.Error(
|
||||
"archive sweep: failed to list database targets",
|
||||
"error", err,
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
package delivery_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -20,6 +22,7 @@ import (
|
||||
_ "modernc.org/sqlite" // Pure Go SQLite driver.
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
"sneak.berlin/go/webhooker/internal/delivery"
|
||||
"sneak.berlin/go/webhooker/internal/gormlog"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -68,7 +71,8 @@ func setupArchiveTest(t *testing.T) *archiveEnv {
|
||||
t.Cleanup(func() { _ = sqlDB.Close() })
|
||||
|
||||
gdb, err := gorm.Open(
|
||||
sqlite.Dialector{Conn: sqlDB}, &gorm.Config{},
|
||||
sqlite.Dialector{Conn: sqlDB},
|
||||
&gorm.Config{Logger: gormlog.New(slog.New(slog.DiscardHandler))},
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -159,6 +163,17 @@ func (env *archiveEnv) seedArchiveRows(
|
||||
t.Helper()
|
||||
|
||||
path := env.archivePath(tgt)
|
||||
seedArchiveFile(t, path, tgt.WebhookID, archivedAt...)
|
||||
|
||||
return path
|
||||
}
|
||||
|
||||
// seedArchiveFile creates the archive file at path and inserts one row
|
||||
// per supplied archived-at timestamp, as seedArchiveRows does.
|
||||
func seedArchiveFile(
|
||||
t *testing.T, path, webhookID string, archivedAt ...time.Time,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
sqlDB, err := sql.Open(
|
||||
"sqlite", fmt.Sprintf("file:%s?mode=rwc", path),
|
||||
@@ -166,7 +181,8 @@ func (env *archiveEnv) seedArchiveRows(
|
||||
require.NoError(t, err)
|
||||
|
||||
gdb, err := gorm.Open(
|
||||
sqlite.Dialector{Conn: sqlDB}, &gorm.Config{},
|
||||
sqlite.Dialector{Conn: sqlDB},
|
||||
&gorm.Config{Logger: gormlog.New(slog.New(slog.DiscardHandler))},
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -177,7 +193,7 @@ func (env *archiveEnv) seedArchiveRows(
|
||||
for i, at := range archivedAt {
|
||||
row := delivery.ExportArchivedEvent{
|
||||
EventID: fmt.Sprintf("ev-%d", i),
|
||||
WebhookID: tgt.WebhookID,
|
||||
WebhookID: webhookID,
|
||||
Method: http.MethodPost,
|
||||
Body: `{"seeded":true}`,
|
||||
ArchivedAt: at,
|
||||
@@ -186,8 +202,6 @@ func (env *archiveEnv) seedArchiveRows(
|
||||
}
|
||||
|
||||
require.NoError(t, sqlDB.Close())
|
||||
|
||||
return path
|
||||
}
|
||||
|
||||
// archivedEventIDs returns the event ids currently stored in an
|
||||
@@ -225,7 +239,8 @@ func countArchivedRows(path string) (int64, error) {
|
||||
defer func() { _ = sqlDB.Close() }()
|
||||
|
||||
gdb, err := gorm.Open(
|
||||
sqlite.Dialector{Conn: sqlDB}, &gorm.Config{},
|
||||
sqlite.Dialector{Conn: sqlDB},
|
||||
&gorm.Config{Logger: gormlog.New(slog.New(slog.DiscardHandler))},
|
||||
)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
@@ -681,6 +696,64 @@ func TestArchiveSweep_ClosesHandleOfRegisteredWriter(
|
||||
)
|
||||
}
|
||||
|
||||
// TestArchiveSweep_ClosesHandleBeforeReopening proves the sweep
|
||||
// closes the handle it finds open before it reopens the file.
|
||||
// TestArchiveSweep_LeavesArchiveClosed cannot see this: without the
|
||||
// close, the reopen replaces the handle without closing it, the
|
||||
// sweep then closes only the new one, and one connection leaks per
|
||||
// archive per sweep.
|
||||
func TestArchiveSweep_ClosesHandleBeforeReopening(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
path := filepath.Join(t.TempDir(), "archive.db")
|
||||
|
||||
w := delivery.NewExportArchiveWriter(
|
||||
path, archiveTestLogger(), 0,
|
||||
)
|
||||
|
||||
require.NoError(t, w.Open(time.Hour))
|
||||
|
||||
before, err := w.DB().DB()
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, w.SweepExpired(time.Hour))
|
||||
|
||||
assert.Error(
|
||||
t, before.PingContext(t.Context()),
|
||||
"the handle open before the sweep must be closed by it",
|
||||
)
|
||||
}
|
||||
|
||||
// TestArchiveSweep_CancelledSweepLogsNoError proves a sweep whose
|
||||
// context is already cancelled, as when the app stops just as a
|
||||
// sweep starts, returns without an error line: stopping is not a
|
||||
// failure.
|
||||
func TestArchiveSweep_CancelledSweepLogsNoError(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
|
||||
var errorLines bytes.Buffer
|
||||
|
||||
sweeper := delivery.NewTestArchiveSweeper(
|
||||
env.mainDB, env.eng,
|
||||
slog.New(slog.NewTextHandler(
|
||||
&errorLines,
|
||||
&slog.HandlerOptions{Level: slog.LevelError},
|
||||
)),
|
||||
)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
sweeper.ExportSweep(ctx)
|
||||
|
||||
assert.Empty(
|
||||
t, errorLines.String(),
|
||||
"a cancelled sweep must not log at error level",
|
||||
)
|
||||
}
|
||||
|
||||
// TestArchiveSweep_NeverExpiryUntouched proves the sweep is a
|
||||
// no-op for the default retention policy, so archives with no
|
||||
// expiry (or the literal "never") behave exactly as before.
|
||||
|
||||
@@ -102,6 +102,20 @@ func (cb *CircuitBreaker) CooldownRemaining() time.Duration {
|
||||
return remaining
|
||||
}
|
||||
|
||||
// StateAndCooldown returns the circuit state and, while the circuit is
|
||||
// open, what is left of the cooldown, or zero once that has passed.
|
||||
// Both are read under one lock, so they always agree.
|
||||
func (cb *CircuitBreaker) StateAndCooldown() (CircuitState, time.Duration) {
|
||||
cb.mu.Lock()
|
||||
defer cb.mu.Unlock()
|
||||
|
||||
if cb.state != CircuitOpen {
|
||||
return cb.state, 0
|
||||
}
|
||||
|
||||
return cb.state, max(cb.cooldown-time.Since(cb.lastFailure), 0)
|
||||
}
|
||||
|
||||
// RecordSuccess records a successful delivery and resets
|
||||
// the circuit breaker to closed state.
|
||||
func (cb *CircuitBreaker) RecordSuccess() {
|
||||
|
||||
+68
-14
@@ -143,6 +143,15 @@ type Archives interface {
|
||||
Rename(targetID, webhookName, targetName string) error
|
||||
}
|
||||
|
||||
// CircuitBreakers is how the handlers read a target's circuit
|
||||
// breaker, so the webhook page and the event log can say that
|
||||
// deliveries to the target are paused and until when. Like Archives,
|
||||
// it keeps the handlers free of the engine's internals and is
|
||||
// trivially faked in tests.
|
||||
type CircuitBreakers interface {
|
||||
StateAndCooldown(targetID string) (CircuitState, time.Duration)
|
||||
}
|
||||
|
||||
// EngineParams are the fx dependencies for the delivery
|
||||
// engine.
|
||||
type EngineParams struct {
|
||||
@@ -186,9 +195,11 @@ type Engine struct {
|
||||
// targets maps each target type to its implementation.
|
||||
targets map[database.TargetType]Target
|
||||
|
||||
// httpTarget is retained so tests can reach the HTTP
|
||||
// target's shared client and circuit breakers.
|
||||
httpTarget *httpTarget
|
||||
// httpTarget and slackTarget are retained so StateAndCooldown
|
||||
// can read their circuit breakers, and so tests can reach the
|
||||
// HTTP target's shared client.
|
||||
httpTarget *httpTarget
|
||||
slackTarget *slackTarget
|
||||
|
||||
// dbTarget is retained so the engine can reach the archive
|
||||
// writer registry for eviction, renames and the idle sweep.
|
||||
@@ -284,12 +295,13 @@ func (e *Engine) EvictTarget(targetID string) {
|
||||
e.dbTarget.evict(targetID)
|
||||
}
|
||||
|
||||
// Rename implements Archives. It renames a database target's
|
||||
// archive file to ArchiveFileName(webhookName, targetName,
|
||||
// targetID), under the lock the target's archive writes and the
|
||||
// idle sweep take. It never replaces a file: if one already has the
|
||||
// new name, the error is ErrArchiveNameTaken. The caller renames
|
||||
// before it saves the new name: see databaseTarget.rename.
|
||||
// Rename implements Archives. It renames every one of a database
|
||||
// target's archive files to ArchiveFileName(webhookName, targetName,
|
||||
// targetID), each keeping the period in its name, under the lock the
|
||||
// target's archive writes and the idle sweep take. It never replaces
|
||||
// a file: if one already has a new name, the error is
|
||||
// ErrArchiveNameTaken. The caller renames before it saves the new
|
||||
// name: see databaseTarget.rename.
|
||||
func (e *Engine) Rename(
|
||||
targetID, webhookName, targetName string,
|
||||
) error {
|
||||
@@ -300,6 +312,28 @@ func (e *Engine) Rename(
|
||||
return e.dbTarget.rename(targetID, webhookName, targetName)
|
||||
}
|
||||
|
||||
// StateAndCooldown implements CircuitBreakers. It returns the state of
|
||||
// the target's circuit breaker and, while the breaker is open, what is
|
||||
// left of its cooldown; the cooldown is zero once that has passed and
|
||||
// in any other state. A target with no breaker reads as closed with no
|
||||
// cooldown, and reading never creates one.
|
||||
func (e *Engine) StateAndCooldown(
|
||||
targetID string,
|
||||
) (CircuitState, time.Duration) {
|
||||
for _, core := range []*httpCore{
|
||||
e.httpTarget.httpCore, e.slackTarget.httpCore,
|
||||
} {
|
||||
val, ok := core.circuitBreakers.Load(targetID)
|
||||
if ok {
|
||||
cb, _ := val.(*CircuitBreaker)
|
||||
|
||||
return cb.StateAndCooldown()
|
||||
}
|
||||
}
|
||||
|
||||
return CircuitClosed, 0
|
||||
}
|
||||
|
||||
// ScheduleRetry schedules a task to be re-enqueued onto the
|
||||
// retry channel after delay. It implements the Scheduler
|
||||
// interface the targets use to own their durable retries.
|
||||
@@ -699,10 +733,9 @@ func (e *Engine) recoverInFlight(ctx context.Context) {
|
||||
default:
|
||||
}
|
||||
|
||||
if !e.dbManager.DBExists(webhookID) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Opened even when its file is missing, so that a lost
|
||||
// database is reported at start, not when the webhook next
|
||||
// receives an event, which for a quiet webhook may be never.
|
||||
e.recoverWebhookDeliveries(ctx, webhookID)
|
||||
}
|
||||
}
|
||||
@@ -710,7 +743,24 @@ func (e *Engine) recoverInFlight(ctx context.Context) {
|
||||
func (e *Engine) recoverWebhookDeliveries(
|
||||
ctx context.Context, webhookID string,
|
||||
) {
|
||||
webhookDB, err := e.dbManager.GetDB(webhookID)
|
||||
// The web interface is already serving, so the webhook may have
|
||||
// been deleted since the list was read. Opening its database then
|
||||
// would create the file again after the delete removed it.
|
||||
stillExists := func() (bool, error) {
|
||||
var count int64
|
||||
|
||||
err := e.database.DB().
|
||||
Model(&database.Webhook{}).
|
||||
Where("id = ?", webhookID).
|
||||
Count(&count).Error
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("confirming webhook exists: %w", err)
|
||||
}
|
||||
|
||||
return count > 0, nil
|
||||
}
|
||||
|
||||
webhookDB, err := e.dbManager.GetDBIf(webhookID, stillExists)
|
||||
if err != nil {
|
||||
e.log.Error(
|
||||
"failed to get webhook database for recovery",
|
||||
@@ -721,6 +771,10 @@ func (e *Engine) recoverWebhookDeliveries(
|
||||
return
|
||||
}
|
||||
|
||||
if webhookDB == nil {
|
||||
return
|
||||
}
|
||||
|
||||
e.recoverPendingDeliveries(
|
||||
ctx, webhookDB, webhookID,
|
||||
)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package delivery_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
@@ -23,6 +24,7 @@ import (
|
||||
_ "modernc.org/sqlite"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
"sneak.berlin/go/webhooker/internal/delivery"
|
||||
"sneak.berlin/go/webhooker/internal/gormlog"
|
||||
)
|
||||
|
||||
// iSetup holds common integration test dependencies.
|
||||
@@ -80,7 +82,8 @@ func iMainDB(t *testing.T) *gorm.DB {
|
||||
t.Cleanup(func() { _ = sqlDB.Close() })
|
||||
|
||||
db, err := gorm.Open(
|
||||
sqlite.Dialector{Conn: sqlDB}, &gorm.Config{},
|
||||
sqlite.Dialector{Conn: sqlDB},
|
||||
&gorm.Config{Logger: gormlog.New(slog.New(slog.DiscardHandler))},
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -355,9 +358,14 @@ func TestProcessRetryTask_SuccessfulRetry(t *testing.T) {
|
||||
|
||||
s := newISetup(t)
|
||||
|
||||
var receivedBody string
|
||||
|
||||
ts := httptest.NewServer(
|
||||
http.HandlerFunc(
|
||||
func(w http.ResponseWriter, _ *http.Request) {
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
receivedBody = string(body)
|
||||
|
||||
w.WriteHeader(http.StatusOK)
|
||||
},
|
||||
),
|
||||
@@ -397,6 +405,8 @@ func TestProcessRetryTask_SuccessfulRetry(t *testing.T) {
|
||||
context.TODO(), &task,
|
||||
)
|
||||
|
||||
assert.Equal(t, event.Body, receivedBody)
|
||||
|
||||
iAssertStatus(t, s.WebhookDB, d.ID,
|
||||
database.DeliveryStatusDelivered,
|
||||
)
|
||||
@@ -443,9 +453,14 @@ func TestProcessRetryTask_LargeBody_FetchFromDB(
|
||||
|
||||
s := newISetup(t)
|
||||
|
||||
var receivedBody string
|
||||
|
||||
ts := httptest.NewServer(
|
||||
http.HandlerFunc(
|
||||
func(w http.ResponseWriter, _ *http.Request) {
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
receivedBody = string(body)
|
||||
|
||||
w.WriteHeader(http.StatusOK)
|
||||
},
|
||||
),
|
||||
@@ -482,6 +497,8 @@ func TestProcessRetryTask_LargeBody_FetchFromDB(
|
||||
context.TODO(), &task,
|
||||
)
|
||||
|
||||
assert.Equal(t, largeBody, receivedBody)
|
||||
|
||||
iAssertStatus(t, s.WebhookDB, d.ID,
|
||||
database.DeliveryStatusDelivered,
|
||||
)
|
||||
@@ -1120,6 +1137,85 @@ func TestRecoverInFlight_WithPendingDeliveries(
|
||||
}
|
||||
}
|
||||
|
||||
// TestRecoverInFlight_ReportsAMissingWebhookDatabase covers a webhook
|
||||
// whose database file is gone, after a partial restore say. Restart
|
||||
// recovery opens every webhook's database, so the empty one made in its
|
||||
// place is reported at start, naming the file
|
||||
// (https://git.eeqj.de/sneak/webhooker/issues/290).
|
||||
func TestRecoverInFlight_ReportsAMissingWebhookDatabase(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
mainDB := iMainDB(t)
|
||||
webhookID := uuid.New().String()
|
||||
iCreateWebhook(t, mainDB, webhookID, "lost-database")
|
||||
|
||||
var logs bytes.Buffer
|
||||
|
||||
dbMgr := database.NewTestWebhookDBManagerWithLogger(
|
||||
t.TempDir(), slog.New(slog.NewTextHandler(&logs, nil)),
|
||||
)
|
||||
t.Cleanup(func() { _ = dbMgr.CloseAll() })
|
||||
|
||||
engine := delivery.NewTestEngineWithDB(
|
||||
database.NewTestDatabase(mainDB), dbMgr,
|
||||
slog.New(slog.DiscardHandler),
|
||||
&http.Client{Timeout: 5 * time.Second}, 1,
|
||||
)
|
||||
|
||||
engine.ExportRecoverInFlight(context.Background())
|
||||
|
||||
assert.Contains(
|
||||
t, logs.String(),
|
||||
`level=WARN msg="created a new, empty database" webhook_id=`+
|
||||
webhookID+" path="+dbMgr.DBPath(webhookID),
|
||||
)
|
||||
}
|
||||
|
||||
// TestRecoverInFlight_SkipsAWebhookDeletedAfterTheListIsRead covers a
|
||||
// webhook deleted from the web interface while restart recovery runs.
|
||||
// Its database file is gone, and recovery must not create it again.
|
||||
func TestRecoverInFlight_SkipsAWebhookDeletedAfterTheListIsRead(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
mainDB := iMainDB(t)
|
||||
webhookID := uuid.New().String()
|
||||
iCreateWebhook(t, mainDB, webhookID, "deleted-during-recovery")
|
||||
|
||||
// The first query to return is recovery's read of the list of
|
||||
// webhooks. Deleting the webhook right after it puts the delete
|
||||
// between that read and the opening of the webhook's database.
|
||||
deleted := false
|
||||
|
||||
require.NoError(t, mainDB.Callback().Query().After("gorm:query").
|
||||
Register("delete-after-list", func(*gorm.DB) {
|
||||
if deleted {
|
||||
return
|
||||
}
|
||||
|
||||
deleted = true
|
||||
|
||||
require.NoError(t, mainDB.Delete(
|
||||
&database.Webhook{}, "id = ?", webhookID,
|
||||
).Error)
|
||||
}))
|
||||
|
||||
dbMgr := database.NewTestWebhookDBManager(t.TempDir())
|
||||
t.Cleanup(func() { _ = dbMgr.CloseAll() })
|
||||
|
||||
engine := delivery.NewTestEngineWithDB(
|
||||
database.NewTestDatabase(mainDB), dbMgr,
|
||||
slog.New(slog.DiscardHandler),
|
||||
&http.Client{Timeout: 5 * time.Second}, 1,
|
||||
)
|
||||
|
||||
engine.ExportRecoverInFlight(context.Background())
|
||||
|
||||
require.True(t, deleted)
|
||||
assert.False(t, dbMgr.DBExists(webhookID))
|
||||
}
|
||||
|
||||
// --- HTTP Config with custom headers ---
|
||||
|
||||
func TestDeliverHTTP_CustomTargetHeaders(t *testing.T) {
|
||||
@@ -1411,6 +1507,32 @@ func TestDeliverHTTP_InvalidConfig(t *testing.T) {
|
||||
)
|
||||
}
|
||||
|
||||
// TestDeliverHTTP_InvalidConfigUnrecordedStaysPending: a delivery is
|
||||
// failed for an invalid config only once the reason is recorded.
|
||||
// Unrecorded, it stays pending, where the sweep finds it again.
|
||||
func TestDeliverHTTP_InvalidConfigUnrecordedStaysPending(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db := testWebhookDB(t)
|
||||
e := testEngine(t, 1)
|
||||
|
||||
event, del := iSeedEventAndDelivery(
|
||||
t, db, `{"config":"invalid"}`, "",
|
||||
)
|
||||
|
||||
task, d := iHTTPTaskAndDelivery(
|
||||
event, del, "bad-config", `not-json`, 0, 1,
|
||||
)
|
||||
|
||||
require.NoError(t, db.Exec("drop table delivery_results").Error)
|
||||
|
||||
e.ExportDeliverHTTP(context.TODO(), db, d, task)
|
||||
|
||||
iAssertStatus(t, db, del.ID,
|
||||
database.DeliveryStatusPending,
|
||||
)
|
||||
}
|
||||
|
||||
// --- Notify batching ---
|
||||
|
||||
func TestNotify_MultipleTasks(t *testing.T) {
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -25,6 +26,7 @@ import (
|
||||
_ "modernc.org/sqlite"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
"sneak.berlin/go/webhooker/internal/delivery"
|
||||
"sneak.berlin/go/webhooker/internal/gormlog"
|
||||
"sneak.berlin/go/webhooker/internal/metrics"
|
||||
)
|
||||
|
||||
@@ -49,7 +51,8 @@ func testWebhookDB(t *testing.T) *gorm.DB {
|
||||
t.Cleanup(func() { _ = sqlDB.Close() })
|
||||
|
||||
db, err := gorm.Open(
|
||||
sqlite.Dialector{Conn: sqlDB}, &gorm.Config{},
|
||||
sqlite.Dialector{Conn: sqlDB},
|
||||
&gorm.Config{Logger: gormlog.New(slog.New(slog.DiscardHandler))},
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -1015,6 +1018,62 @@ func TestGetCircuitBreaker_CreatesOnDemand(t *testing.T) {
|
||||
)
|
||||
}
|
||||
|
||||
// TestStateAndCooldown_ReadsHTTPAndSlackBreakers proves the engine
|
||||
// reads the state of an http or a slack target's circuit breaker, with
|
||||
// what is left of its cooldown while it is open, and no cooldown while
|
||||
// it is half-open, once it closes, or for a target with no breaker.
|
||||
func TestStateAndCooldown_ReadsHTTPAndSlackBreakers(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
e := testEngine(t, 1)
|
||||
|
||||
httpID := uuid.New().String()
|
||||
slackID := uuid.New().String()
|
||||
|
||||
state, cooldown := e.StateAndCooldown(httpID)
|
||||
assert.Equal(t, delivery.CircuitClosed, state, "no breaker")
|
||||
assert.Zero(t, cooldown, "no breaker")
|
||||
|
||||
httpCB := delivery.NewTestCircuitBreaker(1, time.Hour)
|
||||
e.ExportSetCircuitBreaker(httpID, httpCB)
|
||||
|
||||
slackCB := delivery.NewTestCircuitBreaker(1, time.Hour)
|
||||
e.ExportSetSlackCircuitBreaker(slackID, slackCB)
|
||||
|
||||
httpCB.RecordFailure()
|
||||
slackCB.RecordFailure()
|
||||
|
||||
for _, id := range []string{httpID, slackID} {
|
||||
state, cooldown := e.StateAndCooldown(id)
|
||||
assert.Equal(t, delivery.CircuitOpen, state)
|
||||
assert.Greater(t, cooldown, 59*time.Minute)
|
||||
assert.LessOrEqual(t, cooldown, time.Hour)
|
||||
}
|
||||
|
||||
httpCB.RecordSuccess()
|
||||
slackCB.RecordSuccess()
|
||||
|
||||
for _, id := range []string{httpID, slackID} {
|
||||
state, cooldown := e.StateAndCooldown(id)
|
||||
assert.Equal(t, delivery.CircuitClosed, state, "closed")
|
||||
assert.Zero(t, cooldown, "closed")
|
||||
}
|
||||
|
||||
// A breaker with no cooldown goes half-open on the first Allow
|
||||
// after it trips, letting that one delivery through to test the
|
||||
// target.
|
||||
halfOpenID := uuid.New().String()
|
||||
halfOpenCB := delivery.NewTestCircuitBreaker(1, 0)
|
||||
e.ExportSetCircuitBreaker(halfOpenID, halfOpenCB)
|
||||
|
||||
halfOpenCB.RecordFailure()
|
||||
require.True(t, halfOpenCB.Allow())
|
||||
|
||||
state, cooldown = e.StateAndCooldown(halfOpenID)
|
||||
assert.Equal(t, delivery.CircuitHalfOpen, state)
|
||||
assert.Zero(t, cooldown, "half-open")
|
||||
}
|
||||
|
||||
func TestParseHTTPConfig_Valid(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -1056,6 +1115,21 @@ func TestParseHTTPConfig_MissingURL(t *testing.T) {
|
||||
)
|
||||
}
|
||||
|
||||
func TestParseHTTPConfig_Undecodable(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
e := testEngine(t, 1)
|
||||
|
||||
_, err := e.ExportParseHTTPConfig(
|
||||
`{"url":"https://example.com/hook","timeout":"soon"}`,
|
||||
)
|
||||
|
||||
assert.Error(t, err,
|
||||
"config that does not decode should return error, "+
|
||||
"even when the part that did names a URL",
|
||||
)
|
||||
}
|
||||
|
||||
func TestScheduleRetry_SendsToRetryChannel(
|
||||
t *testing.T,
|
||||
) {
|
||||
@@ -1241,6 +1315,33 @@ func TestDoHTTPRequest_ForwardsHeaders(t *testing.T) {
|
||||
)
|
||||
}
|
||||
|
||||
// A response that ends before the length it announced is an error, not
|
||||
// a short body.
|
||||
func TestDoHTTPRequest_CutShortResponseIsAnError(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ts := httptest.NewServer(
|
||||
http.HandlerFunc(
|
||||
func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Length", "100")
|
||||
_, _ = w.Write([]byte("cut short"))
|
||||
},
|
||||
),
|
||||
)
|
||||
defer ts.Close()
|
||||
|
||||
e := testEngine(t, 1)
|
||||
|
||||
_, body, _, err := e.ExportDoHTTPRequest(
|
||||
context.TODO(),
|
||||
&delivery.HTTPTargetConfig{URL: ts.URL},
|
||||
&database.Event{},
|
||||
)
|
||||
|
||||
require.ErrorIs(t, err, io.ErrUnexpectedEOF)
|
||||
assert.Empty(t, body)
|
||||
}
|
||||
|
||||
// The event's stored inbound headers carry the same Content-Type the
|
||||
// receiver saved as the event's ContentType, so a delivery could send
|
||||
// it twice. It must go out exactly once, with a Content-Type configured
|
||||
@@ -1317,6 +1418,34 @@ func TestApplyRequestHeaders_SendsOneContentType(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Stored inbound headers that do not decode forward nothing, not the
|
||||
// part of them that happened to decode.
|
||||
func TestApplyRequestHeaders_UndecodableInboundForwardsNothing(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
req, err := http.NewRequestWithContext(
|
||||
context.Background(),
|
||||
http.MethodPost,
|
||||
"https://target.example.com/hook",
|
||||
http.NoBody,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
names := delivery.ExportApplyRequestHeaders(
|
||||
req,
|
||||
&database.Event{
|
||||
Headers: `{"X-Custom":["value1"],"X-Broken":"not a list"}`,
|
||||
},
|
||||
&delivery.HTTPTargetConfig{},
|
||||
"webhooker/dev",
|
||||
)
|
||||
|
||||
assert.Empty(t, names)
|
||||
assert.Empty(t, req.Header.Get("X-Custom"))
|
||||
}
|
||||
|
||||
func TestProcessDelivery_RoutesToCorrectHandler(
|
||||
t *testing.T,
|
||||
) {
|
||||
|
||||
@@ -212,6 +212,14 @@ func (e *Engine) ExportSetCircuitBreaker(
|
||||
e.httpTarget.circuitBreakers.Store(targetID, cb)
|
||||
}
|
||||
|
||||
// ExportSetSlackCircuitBreaker is ExportSetCircuitBreaker for the
|
||||
// slack target.
|
||||
func (e *Engine) ExportSetSlackCircuitBreaker(
|
||||
targetID string, cb *CircuitBreaker,
|
||||
) {
|
||||
e.slackTarget.circuitBreakers.Store(targetID, cb)
|
||||
}
|
||||
|
||||
// ExportParseHTTPConfig exposes parseHTTPConfig.
|
||||
func (e *Engine) ExportParseHTTPConfig(
|
||||
configJSON string,
|
||||
@@ -493,23 +501,38 @@ func NewExportArchiveWriter(
|
||||
return &ExportArchiveWriter{w: w}
|
||||
}
|
||||
|
||||
// Write archives a row through the writer.
|
||||
// Write archives a row through the writer, into the file named
|
||||
// without a period.
|
||||
func (e *ExportArchiveWriter) Write(
|
||||
row ExportArchivedEvent, expiry time.Duration,
|
||||
) error {
|
||||
return e.w.write(row, expiry)
|
||||
return e.w.write(row, expiry, "")
|
||||
}
|
||||
|
||||
// WritePeriod archives a row through the writer, into the file for
|
||||
// period.
|
||||
func (e *ExportArchiveWriter) WritePeriod(
|
||||
row ExportArchivedEvent, expiry time.Duration, period string,
|
||||
) error {
|
||||
return e.w.write(row, expiry, period)
|
||||
}
|
||||
|
||||
// Open opens the archive file, pruning when expiry is positive.
|
||||
func (e *ExportArchiveWriter) Open(expiry time.Duration) error {
|
||||
return e.w.open(expiry)
|
||||
return e.w.open(e.w.path, expiry)
|
||||
}
|
||||
|
||||
// Reopen closes and reopens the archive file.
|
||||
func (e *ExportArchiveWriter) Reopen(
|
||||
expiry time.Duration,
|
||||
) error {
|
||||
return e.w.reopen(expiry)
|
||||
return e.w.reopen(e.w.path, expiry)
|
||||
}
|
||||
|
||||
// SetNow replaces the clock the writer measures its reopen
|
||||
// debounce on.
|
||||
func (e *ExportArchiveWriter) SetNow(now func() time.Time) {
|
||||
e.w.now = now
|
||||
}
|
||||
|
||||
// Reopens reports how many times the file has been opened.
|
||||
@@ -533,7 +556,7 @@ func (e *ExportArchiveWriter) Path() string {
|
||||
func (e *ExportArchiveWriter) OpenExisting(
|
||||
expiry time.Duration,
|
||||
) error {
|
||||
return e.w.openMode(archiveModeExisting, expiry)
|
||||
return e.w.openMode(e.w.path, archiveModeExisting, expiry)
|
||||
}
|
||||
|
||||
// SweepExpired runs an idle sweep of the archive.
|
||||
|
||||
@@ -376,3 +376,97 @@ func TestFailedResultWriteLeavesDeliveryRecoverable(
|
||||
database.DeliveryStatusPending,
|
||||
)
|
||||
}
|
||||
|
||||
// TestFailedResultWriteWithRetriesLeavesDeliveryRecoverable is the same
|
||||
// rule for a target with retries: whatever the receiver answered, the
|
||||
// delivery stays pending and no retry is scheduled. The circuit breaker
|
||||
// still learns the answer, because it describes the target's health,
|
||||
// not the database's.
|
||||
func TestFailedResultWriteWithRetriesLeavesDeliveryRecoverable(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
// The "send succeeded" case starts with the breaker tripped open,
|
||||
// so the delivery goes out as its probe and only a recorded
|
||||
// success closes it again.
|
||||
tests := []struct {
|
||||
name string
|
||||
answer int
|
||||
tripped bool
|
||||
wantBreaker delivery.CircuitState
|
||||
}{
|
||||
{"send succeeded", http.StatusOK, true, delivery.CircuitClosed},
|
||||
{"send failed", http.StatusBadGateway, false, delivery.CircuitOpen},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s := newISetup(t)
|
||||
targetID := uuid.New().String()
|
||||
|
||||
ts := httptest.NewServer(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(tc.answer)
|
||||
},
|
||||
))
|
||||
defer ts.Close()
|
||||
|
||||
event := iSeedEvent(
|
||||
t, s.WebhookDB, s.WebhookID, `{"unwritable":true}`,
|
||||
)
|
||||
|
||||
d := iSeedDelivery(
|
||||
t, s.WebhookDB, event.ID, targetID,
|
||||
database.DeliveryStatusPending,
|
||||
)
|
||||
|
||||
require.NoError(
|
||||
t,
|
||||
s.WebhookDB.Exec("drop table delivery_results").Error,
|
||||
)
|
||||
|
||||
// A single failure opens this breaker, and with no
|
||||
// cooldown an open breaker lets the next delivery
|
||||
// through as a probe.
|
||||
cb := delivery.NewTestCircuitBreaker(1, 0)
|
||||
if tc.tripped {
|
||||
cb.RecordFailure()
|
||||
}
|
||||
|
||||
s.Engine.ExportSetCircuitBreaker(targetID, cb)
|
||||
|
||||
full := &database.Delivery{
|
||||
EventID: event.ID,
|
||||
TargetID: targetID,
|
||||
Status: database.DeliveryStatusPending,
|
||||
Event: event,
|
||||
Target: database.Target{
|
||||
Name: "unwritable",
|
||||
Type: database.TargetTypeHTTP,
|
||||
Config: iHTTPConfig(ts.URL),
|
||||
MaxRetries: 3,
|
||||
},
|
||||
}
|
||||
full.ID = d.ID
|
||||
|
||||
sched := &recordingScheduler{}
|
||||
|
||||
s.Engine.ExportDeliverHTTPWithScheduler(
|
||||
context.Background(), s.WebhookDB, full,
|
||||
&delivery.Task{
|
||||
DeliveryID: d.ID,
|
||||
TargetID: targetID,
|
||||
AttemptNum: 1,
|
||||
},
|
||||
sched,
|
||||
)
|
||||
|
||||
iAssertStatus(t, s.WebhookDB, d.ID, database.DeliveryStatusPending)
|
||||
assert.Empty(t, sched.delays, "no retry may be scheduled")
|
||||
assert.Equal(t, tc.wantBreaker, cb.State())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -105,6 +105,7 @@ func (e *Engine) initTargets(client *http.Client) {
|
||||
dbT := &databaseTarget{eng: e}
|
||||
|
||||
e.httpTarget = httpT
|
||||
e.slackTarget = slackT
|
||||
e.dbTarget = dbT
|
||||
|
||||
e.targets = map[database.TargetType]Target{
|
||||
|
||||
@@ -41,6 +41,8 @@ type TargetConfigForm struct {
|
||||
Timeout string
|
||||
// Expiry is the database (archive) target's row expiry.
|
||||
Expiry string
|
||||
// Rotation is the database (archive) target's rotation.
|
||||
Rotation string
|
||||
}
|
||||
|
||||
// NewTargetConfigForm parses a target's stored configuration into
|
||||
@@ -85,11 +87,13 @@ func NewTargetConfigForm(
|
||||
}
|
||||
}
|
||||
|
||||
// databaseConfigForm parses an archive target's optional expiry.
|
||||
// An absent or empty configuration is the keep-forever default and
|
||||
// yields an empty field, so re-saving the form unchanged stores the
|
||||
// same empty configuration it started with. An expiry that is set
|
||||
// but not a valid duration is an error, not a blank field.
|
||||
// databaseConfigForm parses an archive target's optional expiry and
|
||||
// rotation. An absent, empty or never expiry yields an empty expiry,
|
||||
// on which the edit form starts at never; saving it unchanged stores
|
||||
// never, which means the same as an empty expiry. An absent rotation
|
||||
// is empty too, and the form starts at none. An expiry that is set
|
||||
// but not a valid duration, or a rotation that is not one of the
|
||||
// four, is an error, not a blank field.
|
||||
func databaseConfigForm(
|
||||
configJSON string,
|
||||
) (TargetConfigForm, error) {
|
||||
@@ -106,8 +110,15 @@ func databaseConfigForm(
|
||||
)
|
||||
}
|
||||
|
||||
err = ValidateArchiveRotation(cfg.Rotation)
|
||||
if err != nil {
|
||||
return TargetConfigForm{}, err
|
||||
}
|
||||
|
||||
form := TargetConfigForm{Rotation: cfg.Rotation}
|
||||
|
||||
if cfg.Expiry == "" || cfg.Expiry == archiveExpiryNever {
|
||||
return TargetConfigForm{}, nil
|
||||
return form, nil
|
||||
}
|
||||
|
||||
err = ValidateArchiveExpiry(cfg.Expiry)
|
||||
@@ -115,5 +126,7 @@ func databaseConfigForm(
|
||||
return TargetConfigForm{}, err
|
||||
}
|
||||
|
||||
return TargetConfigForm{Expiry: cfg.Expiry}, nil
|
||||
form.Expiry = cfg.Expiry
|
||||
|
||||
return form, nil
|
||||
}
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
package delivery
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
)
|
||||
@@ -97,7 +97,7 @@ func targetConfigFields(
|
||||
) []ConfigField {
|
||||
switch t.Type {
|
||||
case database.TargetTypeSlack:
|
||||
return slackConfigFields(t.Config)
|
||||
return slackConfigFields(t)
|
||||
case database.TargetTypeHTTP:
|
||||
return httpConfigFields(t)
|
||||
case database.TargetTypeDatabase:
|
||||
@@ -119,10 +119,11 @@ func unavailableConfigFields() []ConfigField {
|
||||
}}
|
||||
}
|
||||
|
||||
// slackConfigFields describes a Slack target. Only the masked
|
||||
// webhook URL is shown; the full URL is the credential.
|
||||
func slackConfigFields(configJSON string) []ConfigField {
|
||||
cfg, err := parseSlackConfig(configJSON)
|
||||
// slackConfigFields describes a Slack target: its masked
|
||||
// webhook URL and its retry count. Only the masked URL is
|
||||
// shown; the full URL is the credential.
|
||||
func slackConfigFields(t *database.Target) []ConfigField {
|
||||
cfg, err := parseSlackConfig(t.Config)
|
||||
if err != nil {
|
||||
return unavailableConfigFields()
|
||||
}
|
||||
@@ -130,7 +131,7 @@ func slackConfigFields(configJSON string) []ConfigField {
|
||||
return []ConfigField{{
|
||||
Label: "Webhook URL",
|
||||
Value: cfg.MaskedWebhookURL(),
|
||||
}}
|
||||
}, maxRetriesField(t)}
|
||||
}
|
||||
|
||||
// httpConfigFields describes an HTTP target: its destination
|
||||
@@ -170,63 +171,89 @@ func httpConfigFields(t *database.Target) []ConfigField {
|
||||
})
|
||||
}
|
||||
|
||||
return append(fields, retryFields(t)...)
|
||||
fields = append(fields, maxRetriesField(t))
|
||||
|
||||
return fields
|
||||
}
|
||||
|
||||
// retryFields describes a target's retry settings, which live
|
||||
// maxRetriesField describes a target's retry count, which lives
|
||||
// on the target row rather than in its configuration blob.
|
||||
func retryFields(t *database.Target) []ConfigField {
|
||||
func maxRetriesField(t *database.Target) ConfigField {
|
||||
retries := strconv.Itoa(t.MaxRetries)
|
||||
if t.MaxRetries == 0 {
|
||||
retries += " (fire-and-forget)"
|
||||
}
|
||||
|
||||
fields := []ConfigField{{
|
||||
return ConfigField{
|
||||
Label: "Max Retries",
|
||||
Value: retries,
|
||||
}}
|
||||
|
||||
if t.MaxQueueSize > 0 {
|
||||
fields = append(fields, ConfigField{
|
||||
Label: "Max Queue Size",
|
||||
Value: strconv.Itoa(t.MaxQueueSize),
|
||||
})
|
||||
}
|
||||
|
||||
return fields
|
||||
}
|
||||
|
||||
// databaseConfigFields describes an archive target. Its
|
||||
// configuration is optional, and an absent or empty expiry
|
||||
// means the archive is kept forever. An expiry that is set
|
||||
// but not a valid duration is reported as unavailable rather
|
||||
// than echoed back.
|
||||
// databaseConfigFields describes an archive target by its
|
||||
// expiry in plain units, such as "30 days", or "never" when
|
||||
// the archive is kept forever, and by its rotation. An expiry
|
||||
// that is set but not a valid duration, or a rotation that is
|
||||
// not one of the four, is reported as unavailable rather than
|
||||
// echoed back.
|
||||
func databaseConfigFields(configJSON string) []ConfigField {
|
||||
expiry := archiveExpiryNever
|
||||
expiry, err := parseArchiveExpiry(configJSON)
|
||||
if err != nil {
|
||||
return unavailableConfigFields()
|
||||
}
|
||||
|
||||
if configJSON != "" {
|
||||
var cfg databaseTargetConfig
|
||||
rotation, err := parseArchiveRotation(configJSON)
|
||||
if err != nil {
|
||||
return unavailableConfigFields()
|
||||
}
|
||||
|
||||
err := json.Unmarshal([]byte(configJSON), &cfg)
|
||||
if err != nil {
|
||||
return unavailableConfigFields()
|
||||
}
|
||||
|
||||
if cfg.Expiry != "" {
|
||||
if ValidateArchiveExpiry(cfg.Expiry) != nil {
|
||||
return unavailableConfigFields()
|
||||
}
|
||||
|
||||
expiry = cfg.Expiry
|
||||
}
|
||||
value := archiveExpiryNever
|
||||
if expiry > 0 {
|
||||
value = plainDuration(expiry)
|
||||
}
|
||||
|
||||
return []ConfigField{{
|
||||
Label: "Archive Expiry",
|
||||
Value: expiry,
|
||||
Value: value,
|
||||
}, {
|
||||
Label: "Archive Rotation",
|
||||
Value: rotation,
|
||||
}}
|
||||
}
|
||||
|
||||
// plainDuration writes a positive duration as a count of the
|
||||
// largest whole unit it divides into: "30 days", "12 hours",
|
||||
// "1 minute". A duration with a fraction of a second is
|
||||
// written as Go writes it.
|
||||
func plainDuration(d time.Duration) string {
|
||||
const day = 24 * time.Hour
|
||||
|
||||
units := []struct {
|
||||
size time.Duration
|
||||
name string
|
||||
}{
|
||||
{day, "day"},
|
||||
{time.Hour, "hour"},
|
||||
{time.Minute, "minute"},
|
||||
{time.Second, "second"},
|
||||
}
|
||||
|
||||
for _, unit := range units {
|
||||
if d%unit.size != 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
count := int64(d / unit.size)
|
||||
if count == 1 {
|
||||
return "1 " + unit.name
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%d %ss", count, unit.name)
|
||||
}
|
||||
|
||||
return d.String()
|
||||
}
|
||||
|
||||
// MaskedWebhookURL returns the Slack webhook URL reduced to
|
||||
// its scheme and host, with the path, query and any userinfo
|
||||
// elided. The path segments are the credential, so none of
|
||||
|
||||
@@ -32,6 +32,7 @@ const (
|
||||
viewMaskedOrigin = viewExampleOrigin + "/..."
|
||||
viewUnavailable = "(unavailable)"
|
||||
viewExpiryNever = "never"
|
||||
viewMaxRetries = "Max Retries"
|
||||
)
|
||||
|
||||
func TestMaskedWebhookURL(t *testing.T) {
|
||||
@@ -157,9 +158,7 @@ func TestNewTargetViews_DeletedTarget(t *testing.T) {
|
||||
t, slackTargetName+" (deleted)", view.DisplayName(),
|
||||
)
|
||||
assert.Equal(
|
||||
t,
|
||||
map[string]string{"Webhook URL": slackMaskedURL},
|
||||
fieldMap(view.Config),
|
||||
t, viewFor(t, slackTarget()).Config, view.Config,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -189,7 +188,30 @@ func TestNewTargetViews_Slack(t *testing.T) {
|
||||
|
||||
assert.Equal(
|
||||
t,
|
||||
map[string]string{"Webhook URL": slackMaskedURL},
|
||||
map[string]string{
|
||||
"Webhook URL": slackMaskedURL,
|
||||
viewMaxRetries: "0 (fire-and-forget)",
|
||||
},
|
||||
fieldMap(view.Config),
|
||||
)
|
||||
}
|
||||
|
||||
// TestNewTargetViews_SlackRetries proves a Slack target shows
|
||||
// its retry count the same way an HTTP target does.
|
||||
func TestNewTargetViews_SlackRetries(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
target := slackTarget()
|
||||
target.MaxRetries = 2
|
||||
|
||||
view := viewFor(t, target)
|
||||
|
||||
assert.Equal(
|
||||
t,
|
||||
map[string]string{
|
||||
"Webhook URL": slackMaskedURL,
|
||||
viewMaxRetries: "2",
|
||||
},
|
||||
fieldMap(view.Config),
|
||||
)
|
||||
}
|
||||
@@ -202,8 +224,7 @@ func TestNewTargetViews_HTTP(t *testing.T) {
|
||||
Config: `{"url":"` + viewExampleHook + `",` +
|
||||
`"timeout":30,` +
|
||||
`"headers":{"Authorization":"Bearer sekrit"}}`,
|
||||
MaxRetries: 5,
|
||||
MaxQueueSize: 100,
|
||||
MaxRetries: 5,
|
||||
})
|
||||
|
||||
fields := fieldMap(view.Config)
|
||||
@@ -214,8 +235,7 @@ func TestNewTargetViews_HTTP(t *testing.T) {
|
||||
"Destination URL": viewMaskedOrigin,
|
||||
"Timeout": "30s",
|
||||
"Headers": "1 configured",
|
||||
"Max Retries": "5",
|
||||
"Max Queue Size": "100",
|
||||
viewMaxRetries: "5",
|
||||
},
|
||||
fields,
|
||||
)
|
||||
@@ -238,7 +258,7 @@ func TestNewTargetViews_HTTPFireAndForget(t *testing.T) {
|
||||
t,
|
||||
map[string]string{
|
||||
"Destination URL": viewMaskedOrigin,
|
||||
"Max Retries": "0 (fire-and-forget)",
|
||||
viewMaxRetries: "0 (fire-and-forget)",
|
||||
},
|
||||
fieldMap(view.Config),
|
||||
)
|
||||
@@ -281,14 +301,20 @@ func TestNewTargetViews_Database(t *testing.T) {
|
||||
}{
|
||||
"empty config": {config: "", want: viewExpiryNever},
|
||||
"empty expiry": {config: `{}`, want: viewExpiryNever},
|
||||
"explicit": {
|
||||
config: `{"expiry":"720h"}`,
|
||||
want: "720h",
|
||||
},
|
||||
"never literal": {
|
||||
config: `{"expiry":"` + viewExpiryNever + `"}`,
|
||||
want: viewExpiryNever,
|
||||
},
|
||||
"1h": {config: `{"expiry":"1h"}`, want: "1 hour"},
|
||||
"12h": {config: `{"expiry":"12h"}`, want: "12 hours"},
|
||||
"24h": {config: `{"expiry":"24h"}`, want: "1 day"},
|
||||
"720h": {config: `{"expiry":"720h"}`, want: "30 days"},
|
||||
"2160h": {config: `{"expiry":"2160h"}`, want: "90 days"},
|
||||
"8760h": {config: `{"expiry":"8760h"}`, want: "365 days"},
|
||||
"36h": {config: `{"expiry":"36h"}`, want: "36 hours"},
|
||||
"1h30m": {config: `{"expiry":"1h30m"}`, want: "90 minutes"},
|
||||
"45s": {config: `{"expiry":"45s"}`, want: "45 seconds"},
|
||||
"1.5s": {config: `{"expiry":"1.5s"}`, want: "1.5s"},
|
||||
}
|
||||
|
||||
for name, tc := range tests {
|
||||
@@ -302,13 +328,49 @@ func TestNewTargetViews_Database(t *testing.T) {
|
||||
|
||||
assert.Equal(
|
||||
t,
|
||||
map[string]string{"Archive Expiry": tc.want},
|
||||
map[string]string{
|
||||
"Archive Expiry": tc.want,
|
||||
"Archive Rotation": rotationNone,
|
||||
},
|
||||
fieldMap(view.Config),
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewTargetViews_DatabaseRotation proves the target list shows a
|
||||
// database target's rotation, none when it has none stored.
|
||||
func TestNewTargetViews_DatabaseRotation(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Each stored config, and the rotation the list shows for it.
|
||||
tests := map[string]string{
|
||||
"": rotationNone,
|
||||
`{"rotation":""}`: rotationNone,
|
||||
}
|
||||
|
||||
for _, rotation := range []string{
|
||||
rotationNone, rotationMonthly, rotationDaily, rotationHourly,
|
||||
} {
|
||||
tests[`{"rotation":"`+rotation+`"}`] = rotation
|
||||
}
|
||||
|
||||
for config, want := range tests {
|
||||
t.Run(config, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
view := viewFor(t, database.Target{
|
||||
Type: database.TargetTypeDatabase,
|
||||
Config: config,
|
||||
})
|
||||
|
||||
assert.Equal(
|
||||
t, want, fieldMap(view.Config)["Archive Rotation"],
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewTargetViews_Log(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -356,6 +418,10 @@ func TestNewTargetViews_Unpresentable(t *testing.T) {
|
||||
Type: database.TargetTypeDatabase,
|
||||
Config: `{"expiry":"a fortnight"}`,
|
||||
},
|
||||
"invalid archive rotation": {
|
||||
Type: database.TargetTypeDatabase,
|
||||
Config: weeklyConfig,
|
||||
},
|
||||
}
|
||||
|
||||
for name, target := range tests {
|
||||
|
||||
@@ -3,7 +3,6 @@ package delivery
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -21,7 +20,8 @@ const archiveNameMaxLen = 40
|
||||
// from the per-webhook event database. The event is already
|
||||
// persisted in the per-webhook event DB by the time delivery runs;
|
||||
// the database target additionally writes a durable long-term copy
|
||||
// into the file ArchiveFileName names and then records a single
|
||||
// into the file ArchiveFileName names, with a period added when the
|
||||
// target rotates (see archivePeriodPath), and then records a single
|
||||
// attempt whose outcome reflects whether the archive write
|
||||
// succeeded. See archiveWriter for the close/reopen, auto-recreate,
|
||||
// and expiry semantics.
|
||||
@@ -147,8 +147,9 @@ func (t *databaseTarget) Deliver(
|
||||
}
|
||||
|
||||
// archive writes the full event as a row into the target's
|
||||
// archive database, honouring the optional per-target expiry
|
||||
// parsed from the target config JSON.
|
||||
// archive database, honouring the optional per-target expiry and
|
||||
// rotation parsed from the target config JSON. With rotation, the
|
||||
// event goes to the file for the period of its receive time.
|
||||
func (t *databaseTarget) archive(d *database.Delivery) error {
|
||||
webhookID := d.Event.WebhookID
|
||||
if webhookID == "" {
|
||||
@@ -160,6 +161,19 @@ func (t *databaseTarget) archive(d *database.Delivery) error {
|
||||
return err
|
||||
}
|
||||
|
||||
rotation, err := parseArchiveRotation(d.Target.Config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// An event whose stored row was gone before its delivery ran has
|
||||
// no receive time (see Engine.hydrateEvent), and goes to the
|
||||
// file for now.
|
||||
receivedAt := d.Event.CreatedAt
|
||||
if receivedAt.IsZero() {
|
||||
receivedAt = time.Now()
|
||||
}
|
||||
|
||||
w, err := t.writerFor(d.TargetID)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -175,7 +189,7 @@ func (t *databaseTarget) archive(d *database.Delivery) error {
|
||||
ContentType: d.Event.ContentType,
|
||||
}
|
||||
|
||||
return w.write(row, expiry)
|
||||
return w.write(row, expiry, archivePeriod(rotation, receivedAt))
|
||||
}
|
||||
|
||||
// writerFor returns the archive writer for a database target,
|
||||
@@ -276,11 +290,10 @@ func (t *databaseTarget) releaseSweepWriter(
|
||||
delete(t.writers, targetID)
|
||||
}
|
||||
|
||||
// newWriter builds the writer for a database target's archive. The
|
||||
// file lives beside the webhook's event database in the data
|
||||
// directory and is named for the webhook and the target as the main
|
||||
// database has them now; from then on only rename changes the name
|
||||
// the writer uses. It does not touch the archive file.
|
||||
// newWriter builds the writer for a database target's archive. Its
|
||||
// path is the one ArchivePath gives for the webhook and the target as
|
||||
// the main database names them now; from then on only rename changes
|
||||
// the name the writer uses. It does not touch the archive files.
|
||||
func (t *databaseTarget) newWriter(
|
||||
targetID string,
|
||||
) (*archiveWriter, error) {
|
||||
@@ -299,21 +312,19 @@ func (t *databaseTarget) newWriter(
|
||||
)
|
||||
}
|
||||
|
||||
dir := filepath.Dir(t.eng.dbManager.DBPath(target.WebhookID))
|
||||
name := ArchiveFileName(
|
||||
target.Webhook.Name, target.Name, target.ID,
|
||||
w := newArchiveWriter(
|
||||
ArchivePath(t.eng.dbManager, &target.Webhook, &target),
|
||||
t.eng.log,
|
||||
)
|
||||
|
||||
w := newArchiveWriter(filepath.Join(dir, name), t.eng.log)
|
||||
w.webhookID = target.WebhookID
|
||||
|
||||
return w, nil
|
||||
}
|
||||
|
||||
// rename moves a database target's archive file to the name for
|
||||
// webhookName and targetName. It goes through the target's writer,
|
||||
// so the move holds the lock that writes and the idle sweep take,
|
||||
// and later writes use the new name.
|
||||
// rename moves every one of a database target's archive files to the
|
||||
// name for webhookName and targetName. It goes through the target's
|
||||
// writer, so the move holds the lock that writes and the idle sweep
|
||||
// take, and later writes use the new name.
|
||||
//
|
||||
// The writer is created if there is none, and it stays cached. The
|
||||
// handlers rename before they save the new name, so until the save
|
||||
|
||||
@@ -85,6 +85,10 @@ type databaseTargetConfig struct {
|
||||
// archived rows are pruned, or "never" (the default) to
|
||||
// keep them forever.
|
||||
Expiry string `json:"expiry"`
|
||||
|
||||
// Rotation is none (the default), monthly, daily or hourly: see
|
||||
// archivePeriod.
|
||||
Rotation string `json:"rotation"`
|
||||
}
|
||||
|
||||
// archivedEvent is one fully captured webhook event stored in a
|
||||
@@ -178,7 +182,7 @@ func ValidateArchiveExpiry(expiry string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// archiveWriter owns one database target's archive SQLite file.
|
||||
// archiveWriter owns one database target's archive SQLite files.
|
||||
// It serialises writes, and after each write closes and reopens
|
||||
// the file (debounced to at most once per debounce window) so
|
||||
// an operator can move the file away for offline archiving. The
|
||||
@@ -186,14 +190,26 @@ func ValidateArchiveExpiry(expiry string) error {
|
||||
// file is opened create-if-missing and its schema is migrated
|
||||
// on every open.
|
||||
type archiveWriter struct {
|
||||
mu sync.Mutex
|
||||
path string
|
||||
mu sync.Mutex
|
||||
|
||||
// path is the target's archive file as ArchivePath names it. A
|
||||
// target that rotates writes to the files archivePeriodPath names
|
||||
// for path and a period instead.
|
||||
path string
|
||||
|
||||
// current is the file db is open on.
|
||||
current string
|
||||
|
||||
log *slog.Logger
|
||||
debounce time.Duration
|
||||
db *gorm.DB
|
||||
lastReopen time.Time
|
||||
reopens int
|
||||
|
||||
// now is the clock the reopen debounce is measured on. It is
|
||||
// time.Now outside tests.
|
||||
now func() time.Time
|
||||
|
||||
// evicted marks a writer that has been removed from the
|
||||
// registry. Its handle is closed and it must never open the
|
||||
// file again: nothing holds it any more, so a reopen would
|
||||
@@ -228,15 +244,18 @@ func newArchiveWriter(
|
||||
path: path,
|
||||
log: log,
|
||||
debounce: archiveReopenDebounce,
|
||||
now: time.Now,
|
||||
}
|
||||
}
|
||||
|
||||
// write appends the event as a row, then applies the debounced
|
||||
// close/reopen. It recreates the archive file if it was moved
|
||||
// or removed since the last open. A positive expiry prunes rows
|
||||
// older than it on each (re)open.
|
||||
// write appends the event as a row to the archive file for period
|
||||
// (see archivePeriodPath), then applies the debounced close/reopen.
|
||||
// When period names a different file from the one open, the open one
|
||||
// is closed first. It recreates the archive file if it was moved or
|
||||
// removed since the last open. A positive expiry prunes rows older
|
||||
// than it on each (re)open.
|
||||
func (w *archiveWriter) write(
|
||||
row archivedEvent, expiry time.Duration,
|
||||
row archivedEvent, expiry time.Duration, period string,
|
||||
) error {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
@@ -247,8 +266,10 @@ func (w *archiveWriter) write(
|
||||
)
|
||||
}
|
||||
|
||||
if w.db == nil || !fileExists(w.path) {
|
||||
err := w.reopen(expiry)
|
||||
file := archivePeriodPath(w.path, period)
|
||||
|
||||
if w.db == nil || w.current != file || !fileExists(file) {
|
||||
err := w.reopen(file, expiry)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -259,41 +280,41 @@ func (w *archiveWriter) write(
|
||||
err := w.db.Create(&row).Error
|
||||
if err != nil {
|
||||
return fmt.Errorf(
|
||||
"archiving event to %s: %w", w.path, err,
|
||||
"archiving event to %s: %w", file, err,
|
||||
)
|
||||
}
|
||||
|
||||
if time.Since(w.lastReopen) >= w.debounce {
|
||||
return w.reopen(expiry)
|
||||
if w.now().Sub(w.lastReopen) >= w.debounce {
|
||||
return w.reopen(file, expiry)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// open opens (creating if missing) the archive file, migrates
|
||||
// open opens (creating if missing) an archive file, migrates
|
||||
// its schema, records the reopen time, and prunes expired rows
|
||||
// when expiry is positive.
|
||||
func (w *archiveWriter) open(expiry time.Duration) error {
|
||||
return w.openMode(archiveModeCreate, expiry)
|
||||
func (w *archiveWriter) open(file string, expiry time.Duration) error {
|
||||
return w.openMode(file, archiveModeCreate, expiry)
|
||||
}
|
||||
|
||||
// openMode opens the archive file with the given SQLite URI
|
||||
// openMode opens an archive file with the given SQLite URI
|
||||
// mode, migrates its schema, records the reopen time, and
|
||||
// prunes expired rows when expiry is positive. The write path
|
||||
// passes archiveModeCreate so a missing file is recreated; the
|
||||
// idle sweep passes archiveModeExisting so a missing file is an
|
||||
// error rather than a newly conjured empty archive.
|
||||
func (w *archiveWriter) openMode(
|
||||
mode string, expiry time.Duration,
|
||||
file, mode string, expiry time.Duration,
|
||||
) error {
|
||||
// Opened through database.OpenSQLite so an archive file carries
|
||||
// the same WAL journaling, busy timeout, immediate-transaction
|
||||
// locking, and pool bounds as every other database file. See
|
||||
// internal/database/sqlite_open.go.
|
||||
sqlDB, err := database.OpenSQLite(w.path, mode)
|
||||
sqlDB, err := database.OpenSQLite(file, mode)
|
||||
if err != nil {
|
||||
return fmt.Errorf(
|
||||
"opening archive database %s: %w", w.path, err,
|
||||
"opening archive database %s: %w", file, err,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -309,7 +330,7 @@ func (w *archiveWriter) openMode(
|
||||
|
||||
return fmt.Errorf(
|
||||
"connecting to archive database %s: %w",
|
||||
w.path, err,
|
||||
file, err,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -318,12 +339,13 @@ func (w *archiveWriter) openMode(
|
||||
_ = sqlDB.Close()
|
||||
|
||||
return fmt.Errorf(
|
||||
"migrating archive database %s: %w", w.path, err,
|
||||
"migrating archive database %s: %w", file, err,
|
||||
)
|
||||
}
|
||||
|
||||
w.db = gdb
|
||||
w.lastReopen = time.Now()
|
||||
w.current = file
|
||||
w.lastReopen = w.now()
|
||||
w.reopens++
|
||||
|
||||
if expiry > 0 {
|
||||
@@ -333,12 +355,12 @@ func (w *archiveWriter) openMode(
|
||||
return nil
|
||||
}
|
||||
|
||||
// reopen closes any open handle and opens the file afresh. The
|
||||
// reopen closes any open handle and opens file afresh. The
|
||||
// fresh open recreates the file if it was moved away.
|
||||
func (w *archiveWriter) reopen(expiry time.Duration) error {
|
||||
func (w *archiveWriter) reopen(file string, expiry time.Duration) error {
|
||||
w.close()
|
||||
|
||||
return w.open(expiry)
|
||||
return w.open(file, expiry)
|
||||
}
|
||||
|
||||
// close closes the underlying handle, if any.
|
||||
@@ -355,22 +377,56 @@ func (w *archiveWriter) close() {
|
||||
w.db = nil
|
||||
}
|
||||
|
||||
// sweepExpired prunes an archive that may have gone idle, with
|
||||
// no write to trigger the usual on-reopen prune. It takes the
|
||||
// writer's own mutex for the whole operation, so a sweep is
|
||||
// ordered against concurrent writes rather than reaching around
|
||||
// them to the file.
|
||||
// sweepExpired prunes the target's archive files, which may have
|
||||
// gone idle, with no write to trigger the usual on-reopen prune. It
|
||||
// lists the files under the writer's own mutex, then takes the mutex
|
||||
// again for one file at a time, so a write waits for at most one
|
||||
// file's prune, and each prune is ordered against concurrent writes
|
||||
// rather than reaching around them to the file.
|
||||
//
|
||||
// It never creates the archive file: a missing file is skipped,
|
||||
// and the reopen uses archiveModeExisting so SQLite itself
|
||||
// refuses to create one if the file disappears between the
|
||||
// check and the open.
|
||||
// It never creates an archive file: it prunes only the files
|
||||
// archiveFiles lists, skips one that is gone by the time it is
|
||||
// reached (moved away, or renamed since the listing), and opens each
|
||||
// with archiveModeExisting so SQLite itself refuses to create one if
|
||||
// the file disappears between the check and the open. A file named
|
||||
// for a period that the prune leaves empty is deleted.
|
||||
//
|
||||
// The archive is left CLOSED afterwards. An idle archive holding
|
||||
// no handle is what keeps the operator's move-the-file-away
|
||||
// workflow working; the next write reopens (and recreates) the
|
||||
// file as it always has.
|
||||
func (w *archiveWriter) sweepExpired(expiry time.Duration) error {
|
||||
w.mu.Lock()
|
||||
files, err := archiveFiles(w.path)
|
||||
w.mu.Unlock()
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var errs []error
|
||||
|
||||
for _, file := range files {
|
||||
err = w.sweepFile(file, expiry)
|
||||
if errors.Is(err, errArchiveWriterEvicted) {
|
||||
return err
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// sweepFile prunes one of the target's archive files for sweepExpired,
|
||||
// holding w.mu while it does. It skips a file that is gone, and deletes
|
||||
// the file, with its -wal and -shm, when it is named for a period and
|
||||
// the prune leaves it empty.
|
||||
func (w *archiveWriter) sweepFile(
|
||||
file archiveFile, expiry time.Duration,
|
||||
) error {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
|
||||
@@ -380,7 +436,7 @@ func (w *archiveWriter) sweepExpired(expiry time.Duration) error {
|
||||
)
|
||||
}
|
||||
|
||||
if !fileExists(w.path) {
|
||||
if !fileExists(file.path) {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -388,26 +444,56 @@ func (w *archiveWriter) sweepExpired(expiry time.Duration) error {
|
||||
// freshly opened file, matching the write path's semantics.
|
||||
w.close()
|
||||
|
||||
err := w.openMode(archiveModeExisting, expiry)
|
||||
err := w.openMode(file.path, archiveModeExisting, expiry)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if file.period == "" {
|
||||
w.close()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
var rows int64
|
||||
|
||||
err = w.db.Model(&archivedEvent{}).Count(&rows).Error
|
||||
|
||||
w.close()
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf(
|
||||
"counting rows in archive %s: %w", file.path, err,
|
||||
)
|
||||
}
|
||||
|
||||
if rows > 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, suffix := range []string{"", "-wal", "-shm"} {
|
||||
err = os.Remove(file.path + suffix)
|
||||
if err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||
return fmt.Errorf("deleting empty archive file: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
w.log.Info("deleted empty archive file", "path", file.path)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// rename gives the archive file a new name in the same directory,
|
||||
// and the writer uses the file under that name from now on. The
|
||||
// handle is closed first, which folds the -wal into the .db; any
|
||||
// -wal or -shm still beside the file (left by a crash) is moved with
|
||||
// it, because SQLite finds them by name. A missing file is not an
|
||||
// error: the operator may have moved it away, and the next write
|
||||
// creates it under the new name.
|
||||
// rename gives every one of the target's archive files the new
|
||||
// name, keeping the period in the name of each (see
|
||||
// archivePeriodPath), and the writer uses the files under that name
|
||||
// from now on. The handle is closed first, which folds the -wal into
|
||||
// the .db; any -wal or -shm still beside a file (left by a crash) is
|
||||
// moved with it, because SQLite finds them by name. A target with no
|
||||
// files is not an error: the operator may have moved them away, and
|
||||
// the next write creates its file under the new name.
|
||||
//
|
||||
// If a file already has the new name, nothing is moved and the
|
||||
// error is ErrArchiveNameTaken. If one file fails to move, those
|
||||
// If a file already has one of the new names, nothing is moved and
|
||||
// the error is ErrArchiveNameTaken. If one file fails to move, those
|
||||
// already moved are moved back before the error is returned, so the
|
||||
// archive is never split across two names.
|
||||
func (w *archiveWriter) rename(name string) error {
|
||||
@@ -425,38 +511,53 @@ func (w *archiveWriter) rename(name string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
suffixes := []string{"", "-wal", "-shm"}
|
||||
files, err := archiveFiles(w.path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, suffix := range suffixes {
|
||||
if fileExists(path + suffix) {
|
||||
// from[i] moves to to[i].
|
||||
var from, to []string
|
||||
|
||||
for _, file := range files {
|
||||
renamed := archivePeriodPath(path, file.period)
|
||||
|
||||
for _, suffix := range []string{"", "-wal", "-shm"} {
|
||||
from = append(from, file.path+suffix)
|
||||
to = append(to, renamed+suffix)
|
||||
}
|
||||
}
|
||||
|
||||
for _, taken := range to {
|
||||
if fileExists(taken) {
|
||||
return fmt.Errorf(
|
||||
"%w: %s", ErrArchiveNameTaken, name+suffix,
|
||||
"%w: %s", ErrArchiveNameTaken, filepath.Base(taken),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
w.close()
|
||||
|
||||
for i, suffix := range suffixes {
|
||||
err := os.Rename(w.path+suffix, path+suffix)
|
||||
for i := range from {
|
||||
err = os.Rename(from[i], to[i])
|
||||
if err == nil || errors.Is(err, fs.ErrNotExist) {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, moved := range suffixes[:i] {
|
||||
backErr := os.Rename(path+moved, w.path+moved)
|
||||
for j := range i {
|
||||
backErr := os.Rename(to[j], from[j])
|
||||
if backErr != nil && !errors.Is(backErr, fs.ErrNotExist) {
|
||||
w.log.Error(
|
||||
"failed to move archive file back",
|
||||
"from", path+moved,
|
||||
"to", w.path+moved,
|
||||
"from", to[j],
|
||||
"to", from[j],
|
||||
"error", backErr,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Errorf(
|
||||
"renaming archive %s to %s: %w", w.path+suffix, path+suffix, err,
|
||||
"renaming archive %s to %s: %w", from[i], to[i], err,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -494,7 +595,7 @@ func (w *archiveWriter) prune(expiry time.Duration) {
|
||||
if res.Error != nil {
|
||||
w.log.Error(
|
||||
"failed to prune expired archive rows",
|
||||
"path", w.path,
|
||||
"path", w.current,
|
||||
"error", res.Error,
|
||||
)
|
||||
|
||||
@@ -504,12 +605,81 @@ func (w *archiveWriter) prune(expiry time.Duration) {
|
||||
if res.RowsAffected > 0 {
|
||||
w.log.Info(
|
||||
"pruned expired archive rows",
|
||||
"path", w.path,
|
||||
"path", w.current,
|
||||
"rows_deleted", res.RowsAffected,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// ArchiveFileInfo is what the metadata of a database target's archive
|
||||
// files says about them.
|
||||
type ArchiveFileInfo struct {
|
||||
// Files counts the files.
|
||||
Files int
|
||||
|
||||
// Size is the bytes on disk of the files and their -wal together.
|
||||
Size int64
|
||||
|
||||
// Written is when a file or a -wal was last modified, whichever is
|
||||
// latest: a write lands in the -wal first.
|
||||
Written time.Time
|
||||
}
|
||||
|
||||
// StatArchive reads the metadata of a database target's archive
|
||||
// files, given the path ArchivePath gives it (see archiveFiles), and
|
||||
// of their -wal, without opening them. With no files, which is so
|
||||
// before the first write and after the operator moved them away, the
|
||||
// error wraps fs.ErrNotExist.
|
||||
func StatArchive(path string) (ArchiveFileInfo, error) {
|
||||
files, err := archiveFiles(path)
|
||||
if err != nil {
|
||||
return ArchiveFileInfo{}, err
|
||||
}
|
||||
|
||||
var info ArchiveFileInfo
|
||||
|
||||
for _, file := range files {
|
||||
db, err := os.Stat(file.path)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
continue
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return ArchiveFileInfo{}, err
|
||||
}
|
||||
|
||||
info.Files++
|
||||
info.Size += db.Size()
|
||||
|
||||
if db.ModTime().After(info.Written) {
|
||||
info.Written = db.ModTime()
|
||||
}
|
||||
|
||||
wal, err := os.Stat(file.path + "-wal")
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
continue
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return ArchiveFileInfo{}, err
|
||||
}
|
||||
|
||||
info.Size += wal.Size()
|
||||
|
||||
if wal.ModTime().After(info.Written) {
|
||||
info.Written = wal.ModTime()
|
||||
}
|
||||
}
|
||||
|
||||
if info.Files == 0 {
|
||||
return ArchiveFileInfo{}, fmt.Errorf(
|
||||
"no archive file for %s: %w", path, fs.ErrNotExist,
|
||||
)
|
||||
}
|
||||
|
||||
return info, nil
|
||||
}
|
||||
|
||||
// fileExists reports whether a path currently exists.
|
||||
func fileExists(path string) bool {
|
||||
_, err := os.Stat(path)
|
||||
|
||||
@@ -0,0 +1,386 @@
|
||||
package delivery
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
"sneak.berlin/go/webhooker/internal/gormlog"
|
||||
)
|
||||
|
||||
// archiveTableQuery counts the archive's table: 0 when the archive
|
||||
// writer has created the file but not yet the table in it.
|
||||
const archiveTableQuery = "SELECT count(*) FROM sqlite_master " +
|
||||
"WHERE type = 'table' AND name = 'archived_events'"
|
||||
|
||||
// ArchivePath returns where a database target's archive file is: in
|
||||
// the data directory, beside the webhook's event database, under the
|
||||
// name ArchiveFileName gives it.
|
||||
func ArchivePath(
|
||||
dbMgr *database.WebhookDBManager,
|
||||
webhook *database.Webhook,
|
||||
target *database.Target,
|
||||
) string {
|
||||
return filepath.Join(
|
||||
filepath.Dir(dbMgr.DBPath(webhook.ID)),
|
||||
ArchiveFileName(webhook.Name, target.Name, target.ID),
|
||||
)
|
||||
}
|
||||
|
||||
// ArchiveExportFileName returns the name a database target's archive
|
||||
// downloads under:
|
||||
// archive-WEBHOOKNAME-TARGETNAME-YYYYMMDDTHHMMSSZ.json.gz, the names
|
||||
// made safe as in ArchiveFileName and the time in UTC.
|
||||
func ArchiveExportFileName(
|
||||
webhookName, targetName string, at time.Time,
|
||||
) string {
|
||||
return "archive-" + archiveNamePart(webhookName) + "-" +
|
||||
archiveNamePart(targetName) + "-" +
|
||||
at.UTC().Format("20060102T150405Z") + ".json.gz"
|
||||
}
|
||||
|
||||
// ArchiveExport is a database target's archive listed for download. It
|
||||
// opens one of the target's files at a time, only when its rows are
|
||||
// about to be written out, and closes it before it opens the next, so
|
||||
// an export holds at most one file open however many the target has.
|
||||
//
|
||||
// Each file is read on its own connection inside one read-only
|
||||
// transaction, so its rows are written out as the file stood when it
|
||||
// was opened. Archives are in WAL mode, where a reader works from a
|
||||
// snapshot and never blocks a writer: archive writes go on while a file
|
||||
// is open, and the export does not see them. SQLite cannot checkpoint
|
||||
// a -wal past an open snapshot, so the open file's -wal grows until the
|
||||
// export has written that file out.
|
||||
type ArchiveExport struct {
|
||||
// periods are the periods of the target's files when the export
|
||||
// was listed, "" for the file without one, in the order
|
||||
// archiveFiles lists them.
|
||||
periods []string
|
||||
|
||||
// lock is held while currentPath is called and a file is opened,
|
||||
// so that a rename, which holds it too, cannot move the file in
|
||||
// between.
|
||||
lock sync.Locker
|
||||
|
||||
// currentPath returns the path ArchivePath gives the target under
|
||||
// the names stored for it now, which a rename may have changed since
|
||||
// the export was listed.
|
||||
currentPath func() (string, error)
|
||||
|
||||
log *slog.Logger
|
||||
}
|
||||
|
||||
// exportFile is one archive file opened for an export.
|
||||
type exportFile struct {
|
||||
db *sql.DB
|
||||
tx *gorm.DB
|
||||
|
||||
// period is the period in the file's name, "" for none.
|
||||
period string
|
||||
|
||||
// empty is true for a file without the archive's table yet.
|
||||
empty bool
|
||||
}
|
||||
|
||||
// exportedName is how an export names its webhook and its target.
|
||||
type exportedName struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// NewArchiveExport lists a database target's archive files for export,
|
||||
// given the path ArchivePath gives it (see archiveFiles). It opens none
|
||||
// of them. Its caller holds lock, which every rename of the target's
|
||||
// files runs under, from reading the names path is made of until it
|
||||
// returns, so the files it lists are the ones those names give.
|
||||
//
|
||||
// WriteGzipJSON, called without lock held, finds each file again by its
|
||||
// period under the path currentPath gives, holding lock while it does
|
||||
// and while it opens the file, so a rename during the export loses no
|
||||
// file. A file that is gone by then, emptied by the sweep or moved
|
||||
// away, is skipped. The export never creates a file: with no files, it
|
||||
// has no rows.
|
||||
func NewArchiveExport(
|
||||
path string,
|
||||
lock sync.Locker,
|
||||
currentPath func() (string, error),
|
||||
log *slog.Logger,
|
||||
) (*ArchiveExport, error) {
|
||||
files, err := archiveFiles(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
x := &ArchiveExport{lock: lock, currentPath: currentPath, log: log}
|
||||
|
||||
for _, file := range files {
|
||||
x.periods = append(x.periods, file.period)
|
||||
}
|
||||
|
||||
return x, nil
|
||||
}
|
||||
|
||||
// openExportFile opens one archive file for an export and takes its
|
||||
// snapshot. The transaction lasts as long as ctx does.
|
||||
func openExportFile(
|
||||
ctx context.Context, file archiveFile, log *slog.Logger,
|
||||
) (*exportFile, error) {
|
||||
db, err := database.OpenSQLite(file.path, archiveModeExisting)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("opening archive %s: %w", file.path, err)
|
||||
}
|
||||
|
||||
gdb, err := gorm.Open(
|
||||
sqlite.Dialector{Conn: db}, &gorm.Config{
|
||||
// Never leave this at GORM's default. See
|
||||
// internal/gormlog.
|
||||
Logger: gormlog.New(log),
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
_ = db.Close()
|
||||
|
||||
return nil, fmt.Errorf("opening archive %s: %w", file.path, err)
|
||||
}
|
||||
|
||||
// ReadOnly makes the driver begin a deferred transaction in place
|
||||
// of the BEGIN IMMEDIATE the connection string asks for, so the
|
||||
// export never takes the archive's write lock.
|
||||
tx := gdb.WithContext(ctx).Begin(&sql.TxOptions{ReadOnly: true})
|
||||
if tx.Error != nil {
|
||||
_ = db.Close()
|
||||
|
||||
return nil, fmt.Errorf(
|
||||
"reading archive %s: %w", file.path, tx.Error,
|
||||
)
|
||||
}
|
||||
|
||||
// The transaction's first read is what takes the snapshot.
|
||||
var tables int
|
||||
|
||||
err = tx.Raw(archiveTableQuery).Row().Scan(&tables)
|
||||
if err != nil {
|
||||
_ = tx.Rollback()
|
||||
_ = db.Close()
|
||||
|
||||
return nil, fmt.Errorf("reading archive %s: %w", file.path, err)
|
||||
}
|
||||
|
||||
return &exportFile{
|
||||
db: db, tx: tx, period: file.period, empty: tables == 0,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// WriteGzipJSON writes the export to w as one gzipped JSON object:
|
||||
// webhook and target, each an id and a name; exported_at; and
|
||||
// archived_events, one object per archived row, keyed by column name,
|
||||
// the files in the order archiveFiles lists them. A row from a file
|
||||
// named for a period has "period" beside its columns. A body that is
|
||||
// not valid UTF-8 cannot be a JSON string, so it is written in base64,
|
||||
// with "body_encoding": "base64" beside it.
|
||||
//
|
||||
// Each row is written out before the next is read, so neither the
|
||||
// archive nor its JSON is ever held in memory whole, and each file is
|
||||
// closed once its rows are written, before the next is opened. When it
|
||||
// returns, no file is open. After an error the gzip stream is left
|
||||
// unfinished, so what was written does not decompress as a whole file.
|
||||
func (x *ArchiveExport) WriteGzipJSON(
|
||||
ctx context.Context,
|
||||
w io.Writer,
|
||||
webhook *database.Webhook,
|
||||
target *database.Target,
|
||||
exportedAt time.Time,
|
||||
) error {
|
||||
head, err := json.Marshal(map[string]any{
|
||||
"webhook": exportedName{ID: webhook.ID, Name: webhook.Name},
|
||||
"target": exportedName{ID: target.ID, Name: target.Name},
|
||||
"exported_at": exportedAt.UTC(),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("encoding archive export: %w", err)
|
||||
}
|
||||
|
||||
zw := gzip.NewWriter(w)
|
||||
|
||||
err = x.writeJSON(ctx, zw, head)
|
||||
if err != nil {
|
||||
return fmt.Errorf("writing archive export: %w", err)
|
||||
}
|
||||
|
||||
return zw.Close()
|
||||
}
|
||||
|
||||
// openFile finds the target's archive file for period under the path
|
||||
// currentPath gives now, and opens it for the export, holding x.lock
|
||||
// for both. For a file that is gone, the error wraps fs.ErrNotExist.
|
||||
func (x *ArchiveExport) openFile(
|
||||
ctx context.Context, period string,
|
||||
) (*exportFile, error) {
|
||||
x.lock.Lock()
|
||||
defer x.lock.Unlock()
|
||||
|
||||
path, err := x.currentPath()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("finding archive file: %w", err)
|
||||
}
|
||||
|
||||
file := archiveFile{path: archivePeriodPath(path, period), period: period}
|
||||
|
||||
_, err = os.Stat(file.path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return openExportFile(ctx, file, x.log)
|
||||
}
|
||||
|
||||
// close ends the file's transaction and closes its connection.
|
||||
func (f *exportFile) close() error {
|
||||
_ = f.tx.Rollback()
|
||||
|
||||
return f.db.Close()
|
||||
}
|
||||
|
||||
// writeJSON writes head with archived_events added as its last key,
|
||||
// the rows going into it one at a time.
|
||||
func (x *ArchiveExport) writeJSON(
|
||||
ctx context.Context, w io.Writer, head []byte,
|
||||
) error {
|
||||
// head goes out without its closing brace, so that
|
||||
// archived_events can follow it.
|
||||
_, err := w.Write(head[:len(head)-1])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = io.WriteString(w, `,"archived_events":[`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = x.writeRows(ctx, w)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = io.WriteString(w, "\n]}\n")
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// writeRows writes the archived rows of each file to w, one per line,
|
||||
// separated by commas, opening each file in turn and closing it once
|
||||
// its rows are written.
|
||||
func (x *ArchiveExport) writeRows(ctx context.Context, w io.Writer) error {
|
||||
sep := "\n"
|
||||
|
||||
for _, period := range x.periods {
|
||||
f, err := x.openFile(ctx, period)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
continue
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
sep, err = f.writeRows(ctx, w, sep)
|
||||
|
||||
err = errors.Join(err, f.close())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeRows writes the file's archived rows to w, oldest first, the
|
||||
// first after sep and each other after ",\n". It returns what goes
|
||||
// before the next row: sep again when the file had no rows.
|
||||
func (f *exportFile) writeRows(
|
||||
ctx context.Context, w io.Writer, sep string,
|
||||
) (string, error) {
|
||||
if f.empty {
|
||||
return sep, nil
|
||||
}
|
||||
|
||||
rows, err := f.tx.WithContext(ctx).
|
||||
Model(&archivedEvent{}).Order("id").Rows()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
for ; rows.Next(); sep = ",\n" {
|
||||
var ev archivedEvent
|
||||
|
||||
err = f.tx.ScanRows(rows, &ev)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
_, err = io.WriteString(w, sep)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
err = writeRow(w, &ev, f.period)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
||||
return sep, rows.Err()
|
||||
}
|
||||
|
||||
// writeRow writes an archived row to w as a JSON object keyed by
|
||||
// column name, its body in base64 when it is not valid UTF-8, with
|
||||
// the period of its file beside them unless that is "".
|
||||
func writeRow(w io.Writer, ev *archivedEvent, period string) error {
|
||||
row := map[string]any{
|
||||
"id": ev.ID,
|
||||
"event_id": ev.EventID,
|
||||
"webhook_id": ev.WebhookID,
|
||||
"entrypoint_id": ev.EntrypointID,
|
||||
"method": ev.Method,
|
||||
"headers": ev.Headers,
|
||||
"body": ev.Body,
|
||||
"content_type": ev.ContentType,
|
||||
"archived_at": ev.ArchivedAt.UTC(),
|
||||
}
|
||||
|
||||
if !utf8.ValidString(ev.Body) {
|
||||
row["body"] = base64.StdEncoding.EncodeToString([]byte(ev.Body))
|
||||
row["body_encoding"] = "base64"
|
||||
}
|
||||
|
||||
if period != "" {
|
||||
row["period"] = period
|
||||
}
|
||||
|
||||
line, err := json.Marshal(row)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = w.Write(line)
|
||||
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,563 @@
|
||||
package delivery_test
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
"sneak.berlin/go/webhooker/internal/delivery"
|
||||
)
|
||||
|
||||
// The webhook and the target the export tests' archives belong to.
|
||||
const (
|
||||
exportWebhookID = "wh-export"
|
||||
exportWebhookName = "Orders (EU)"
|
||||
exportTargetID = "tgt-export"
|
||||
exportTargetName = "Long-term archive"
|
||||
)
|
||||
|
||||
// binaryBody is a body that is not valid UTF-8.
|
||||
const binaryBody = "\xff\xfe\x00\x01binary\x80"
|
||||
|
||||
// writeExportTo writes export to w as the archive of the export tests'
|
||||
// webhook and target, exported at 2026-10-02T12:03:04Z.
|
||||
func writeExportTo(
|
||||
t *testing.T, export *delivery.ArchiveExport, w io.Writer,
|
||||
) error {
|
||||
t.Helper()
|
||||
|
||||
return export.WriteGzipJSON(
|
||||
t.Context(), w,
|
||||
&database.Webhook{
|
||||
BaseModel: database.BaseModel{ID: exportWebhookID},
|
||||
Name: exportWebhookName,
|
||||
},
|
||||
&database.Target{
|
||||
BaseModel: database.BaseModel{ID: exportTargetID},
|
||||
Name: exportTargetName,
|
||||
},
|
||||
time.Date(2026, 10, 2, 12, 3, 4, 0, time.UTC),
|
||||
)
|
||||
}
|
||||
|
||||
// listExport lists the archive at path for export, as the archive of a
|
||||
// target whose names do not change.
|
||||
func listExport(t *testing.T, path string) *delivery.ArchiveExport {
|
||||
t.Helper()
|
||||
|
||||
return newExport(t, path, &sync.Mutex{}, func() (string, error) {
|
||||
return path, nil
|
||||
})
|
||||
}
|
||||
|
||||
// newExport lists the archive at path for export, to find each file
|
||||
// again under the path currentPath gives, holding lock while it does.
|
||||
// Nothing else takes lock while it lists, so it does not hold lock.
|
||||
func newExport(
|
||||
t *testing.T,
|
||||
path string,
|
||||
lock sync.Locker,
|
||||
currentPath func() (string, error),
|
||||
) *delivery.ArchiveExport {
|
||||
t.Helper()
|
||||
|
||||
export, err := delivery.NewArchiveExport(
|
||||
path, lock, currentPath, archiveTestLogger(),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
return export
|
||||
}
|
||||
|
||||
// exportArchive runs a whole export of the archive at path and returns
|
||||
// its JSON, decompressed and parsed.
|
||||
func exportArchive(t *testing.T, path string) map[string]any {
|
||||
t.Helper()
|
||||
|
||||
return writeExport(t, listExport(t, path))
|
||||
}
|
||||
|
||||
// writeExport writes an opened export and returns its JSON,
|
||||
// decompressed and parsed. Reading to the end makes the gzip reader
|
||||
// check that the stream was finished.
|
||||
func writeExport(
|
||||
t *testing.T, export *delivery.ArchiveExport,
|
||||
) map[string]any {
|
||||
t.Helper()
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
require.NoError(t, writeExportTo(t, export, &buf))
|
||||
|
||||
zr, err := gzip.NewReader(&buf)
|
||||
require.NoError(t, err)
|
||||
|
||||
raw, err := io.ReadAll(zr)
|
||||
require.NoError(t, err)
|
||||
|
||||
var got map[string]any
|
||||
|
||||
require.NoError(t, json.Unmarshal(raw, &got))
|
||||
|
||||
return got
|
||||
}
|
||||
|
||||
// exportedEvents returns an export's archived_events.
|
||||
func exportedEvents(t *testing.T, got map[string]any) []map[string]any {
|
||||
t.Helper()
|
||||
|
||||
list, ok := got["archived_events"].([]any)
|
||||
require.True(t, ok, "archived_events must be an array: %v", got)
|
||||
|
||||
events := make([]map[string]any, len(list))
|
||||
|
||||
for i, v := range list {
|
||||
events[i], ok = v.(map[string]any)
|
||||
require.True(t, ok, "an archived event must be an object: %v", v)
|
||||
}
|
||||
|
||||
return events
|
||||
}
|
||||
|
||||
// exportedEventIDs returns the event_id of each of an export's
|
||||
// archived_events.
|
||||
func exportedEventIDs(t *testing.T, got map[string]any) []string {
|
||||
t.Helper()
|
||||
|
||||
events := exportedEvents(t, got)
|
||||
ids := make([]string, 0, len(events))
|
||||
|
||||
for _, ev := range events {
|
||||
ids = append(ids, fmt.Sprint(ev["event_id"]))
|
||||
}
|
||||
|
||||
return ids
|
||||
}
|
||||
|
||||
// TestArchiveExport_MatchesStoredRows proves an export holds the
|
||||
// webhook, the target, the time, and every column of every stored
|
||||
// row: a body that is valid UTF-8 as a string, and one that is not in
|
||||
// base64, marked as such.
|
||||
func TestArchiveExport_MatchesStoredRows(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
path := filepath.Join(t.TempDir(), "archive.db")
|
||||
w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
||||
bodies := []string{`{"order":1}`, "plain text", "", binaryBody}
|
||||
|
||||
for i, body := range bodies {
|
||||
require.NoError(t, w.Write(delivery.ExportArchivedEvent{
|
||||
EventID: fmt.Sprintf("ev-%d", i),
|
||||
WebhookID: exportWebhookID,
|
||||
EntrypointID: "ep-1",
|
||||
Method: "POST",
|
||||
Headers: `{"X-Test":["yes"]}`,
|
||||
Body: body,
|
||||
ContentType: testContentType,
|
||||
}, 0))
|
||||
}
|
||||
|
||||
var stored []delivery.ExportArchivedEvent
|
||||
|
||||
require.NoError(t, openArchiveDBForRead(t, path).
|
||||
Order("id").Find(&stored).Error)
|
||||
|
||||
got := exportArchive(t, path)
|
||||
|
||||
assert.Equal(t,
|
||||
map[string]any{"id": exportWebhookID, "name": exportWebhookName},
|
||||
got["webhook"],
|
||||
)
|
||||
assert.Equal(t,
|
||||
map[string]any{"id": exportTargetID, "name": exportTargetName},
|
||||
got["target"],
|
||||
)
|
||||
assert.Equal(t, "2026-10-02T12:03:04Z", got["exported_at"])
|
||||
|
||||
events := exportedEvents(t, got)
|
||||
require.Len(t, events, len(bodies))
|
||||
|
||||
for i, row := range stored {
|
||||
assertExportedRow(t, row, events[i])
|
||||
}
|
||||
}
|
||||
|
||||
// assertExportedRow checks that ev, from an export, holds every column
|
||||
// of the stored row.
|
||||
func assertExportedRow(
|
||||
t *testing.T, row delivery.ExportArchivedEvent, ev map[string]any,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
archivedAt, err := time.Parse(
|
||||
time.RFC3339Nano, fmt.Sprint(ev["archived_at"]),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, archivedAt.Equal(row.ArchivedAt))
|
||||
|
||||
assert.EqualValues(t, row.ID, ev["id"])
|
||||
assert.Equal(t, row.EventID, ev["event_id"])
|
||||
assert.Equal(t, row.WebhookID, ev["webhook_id"])
|
||||
assert.Equal(t, row.EntrypointID, ev["entrypoint_id"])
|
||||
assert.Equal(t, row.Method, ev["method"])
|
||||
assert.Equal(t, row.Headers, ev["headers"])
|
||||
assert.Equal(t, row.ContentType, ev["content_type"])
|
||||
|
||||
if row.Body != binaryBody {
|
||||
assert.Equal(t, row.Body, ev["body"])
|
||||
assert.Len(t, ev, 9, "the nine columns and nothing else: %v", ev)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
body, err := base64.StdEncoding.DecodeString(fmt.Sprint(ev["body"]))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, binaryBody, string(body))
|
||||
assert.Equal(t, "base64", ev["body_encoding"])
|
||||
assert.Len(t, ev, 10, "the nine columns and body_encoding: %v", ev)
|
||||
}
|
||||
|
||||
// TestArchiveExport_Empty proves an archive with nothing in it exports
|
||||
// as an empty archived_events: no file, which the export must not
|
||||
// create; a file the archive writer has not yet put its table in; and
|
||||
// a table with no rows.
|
||||
func TestArchiveExport_Empty(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
missing := filepath.Join(dir, "missing.db")
|
||||
noTable := filepath.Join(dir, "no-table.db")
|
||||
noRows := filepath.Join(dir, "no-rows.db")
|
||||
|
||||
require.NoError(t, os.WriteFile(noTable, nil, 0o600))
|
||||
require.NoError(t,
|
||||
delivery.NewExportArchiveWriter(noRows, archiveTestLogger(), 0).
|
||||
Open(0),
|
||||
)
|
||||
|
||||
for _, path := range []string{missing, noTable, noRows} {
|
||||
assert.Empty(t, exportedEvents(t, exportArchive(t, path)), path)
|
||||
}
|
||||
|
||||
for _, suffix := range archiveFileSuffixes() {
|
||||
assert.NoFileExists(t, missing+suffix)
|
||||
}
|
||||
}
|
||||
|
||||
// unlockHook is a sync.Locker that runs fn each time it is unlocked. An
|
||||
// export unlocks its lock right after it opens a file.
|
||||
type unlockHook struct {
|
||||
sync.Mutex
|
||||
|
||||
fn func()
|
||||
}
|
||||
|
||||
func (u *unlockHook) Unlock() {
|
||||
u.Mutex.Unlock()
|
||||
u.fn()
|
||||
}
|
||||
|
||||
// TestArchiveExport_ReadsOneSnapshot proves an export writes a file
|
||||
// out as it was when the export opened it, and holds up no archive
|
||||
// write: a row written after the export was listed but before the file
|
||||
// was opened is in the export, and one written while the file is open
|
||||
// is stored, and is not. A write held up for the whole busy timeout
|
||||
// would fail.
|
||||
func TestArchiveExport_ReadsOneSnapshot(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
path := filepath.Join(t.TempDir(), "archive.db")
|
||||
w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
||||
|
||||
require.NoError(t, w.Write(delivery.ExportArchivedEvent{EventID: "listed"}, 0))
|
||||
|
||||
opened := &unlockHook{fn: func() {
|
||||
require.NoError(t, w.Write(delivery.ExportArchivedEvent{EventID: "during"}, 0))
|
||||
}}
|
||||
export := newExport(t, path, opened, func() (string, error) {
|
||||
return path, nil
|
||||
})
|
||||
|
||||
require.NoError(t, w.Write(delivery.ExportArchivedEvent{EventID: "before-open"}, 0))
|
||||
|
||||
assert.Equal(t,
|
||||
[]string{"listed", "before-open"},
|
||||
exportedEventIDs(t, writeExport(t, export)),
|
||||
)
|
||||
|
||||
var stored int64
|
||||
|
||||
require.NoError(t, openArchiveDBForRead(t, path).
|
||||
Model(&delivery.ExportArchivedEvent{}).Count(&stored).Error)
|
||||
assert.Equal(t, int64(3), stored)
|
||||
}
|
||||
|
||||
// TestArchiveExport_FindsFilesAfterRename proves that renaming the
|
||||
// archive after an export has listed it, as renaming its webhook or
|
||||
// target does, loses no file: the export finds each file again by its
|
||||
// period under the new name. A file moved away by then is skipped.
|
||||
func TestArchiveExport_FindsFilesAfterRename(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "archive-old.db")
|
||||
w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
||||
|
||||
for _, period := range []string{"", dayPeriod, hourPeriod} {
|
||||
require.NoError(t, w.WritePeriod(
|
||||
delivery.ExportArchivedEvent{EventID: "in-" + period}, 0, period,
|
||||
))
|
||||
}
|
||||
|
||||
current := path
|
||||
export := newExport(t, path, &sync.Mutex{}, func() (string, error) {
|
||||
return current, nil
|
||||
})
|
||||
|
||||
require.NoError(t, w.Rename("archive-new.db"))
|
||||
|
||||
current = filepath.Join(dir, "archive-new.db")
|
||||
|
||||
removeArchiveFiles(t, periodPath(current, dayPeriod))
|
||||
|
||||
assert.Equal(t,
|
||||
[]string{"in-", "in-" + hourPeriod},
|
||||
exportedEventIDs(t, writeExport(t, export)),
|
||||
)
|
||||
}
|
||||
|
||||
// TestArchiveExport_EveryFileOldestFirst writes a row to a target's
|
||||
// file without a period and to its files for a month, an hour and a
|
||||
// day, and proves the export holds every row: the file without a
|
||||
// period first, then the others oldest period first, each row from a
|
||||
// file named for a period carrying that period. A file made after the
|
||||
// export was listed is not in it.
|
||||
func TestArchiveExport_EveryFileOldestFirst(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
path := filepath.Join(t.TempDir(), "archive-wh.db")
|
||||
w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
||||
|
||||
// Written in an order that is not the export's.
|
||||
for _, period := range []string{nextDayPeriod, "", hourPeriod, "2026-03"} {
|
||||
require.NoError(t, w.WritePeriod(
|
||||
delivery.ExportArchivedEvent{EventID: "in-" + period}, 0, period,
|
||||
))
|
||||
}
|
||||
|
||||
export := listExport(t, path)
|
||||
|
||||
require.NoError(t, w.WritePeriod(
|
||||
delivery.ExportArchivedEvent{EventID: "later"}, 0, "2026-03-06",
|
||||
))
|
||||
|
||||
events := exportedEvents(t, writeExport(t, export))
|
||||
ids := make([]string, 0, len(events))
|
||||
periods := make([]any, 0, len(events))
|
||||
|
||||
for _, ev := range events {
|
||||
ids = append(ids, fmt.Sprint(ev["event_id"]))
|
||||
periods = append(periods, ev["period"])
|
||||
}
|
||||
|
||||
assert.Equal(t,
|
||||
[]string{"in-", "in-2026-03", "in-" + hourPeriod, "in-" + nextDayPeriod},
|
||||
ids,
|
||||
)
|
||||
assert.Equal(t,
|
||||
[]any{nil, "2026-03", hourPeriod, nextDayPeriod}, periods,
|
||||
)
|
||||
assert.NotContains(t, events[0], "period",
|
||||
"a row from the file without a period has no period")
|
||||
}
|
||||
|
||||
// openFilesPeak is an io.Writer that discards what it is given and
|
||||
// records the most archive files in dir the process had open at any
|
||||
// write, as /proc/self/fd lists the files a process has open.
|
||||
type openFilesPeak struct {
|
||||
dir string
|
||||
max int
|
||||
}
|
||||
|
||||
func (p *openFilesPeak) Write(b []byte) (int, error) {
|
||||
fds, err := os.ReadDir("/proc/self/fd")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
open := map[string]bool{}
|
||||
|
||||
for _, fd := range fds {
|
||||
file, err := os.Readlink(filepath.Join("/proc/self/fd", fd.Name()))
|
||||
if err == nil && filepath.Dir(file) == p.dir &&
|
||||
strings.HasSuffix(file, ".db") {
|
||||
open[file] = true
|
||||
}
|
||||
}
|
||||
|
||||
p.max = max(p.max, len(open))
|
||||
|
||||
return len(b), nil
|
||||
}
|
||||
|
||||
// TestArchiveExport_OneFileOpenAtATime exports a target with a file for
|
||||
// each of 24 hours and proves the export never had more than one of
|
||||
// them open, and had one open while it wrote. Each file holds a row of
|
||||
// 48 KiB of random base64, which gzip shrinks little, so the export
|
||||
// writes output while it reads each file.
|
||||
func TestArchiveExport_OneFileOpenAtATime(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if runtime.GOOS != "linux" {
|
||||
t.Skip("only Linux lists a process's open files in /proc/self/fd")
|
||||
}
|
||||
|
||||
// Readlink gives each open file's path with no symbolic link in it.
|
||||
dir, err := filepath.EvalSymlinks(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
|
||||
path := filepath.Join(dir, "archive-wh.db")
|
||||
w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
||||
random := make([]byte, 36<<10)
|
||||
|
||||
for hour := range 24 {
|
||||
_, _ = rand.Read(random)
|
||||
|
||||
require.NoError(t, w.WritePeriod(delivery.ExportArchivedEvent{
|
||||
Body: base64.StdEncoding.EncodeToString(random),
|
||||
}, 0, fmt.Sprintf("2026-10-01-%02d", hour)))
|
||||
}
|
||||
|
||||
// The writer's own handle on the last file is not the export's.
|
||||
w.Evict()
|
||||
|
||||
// Through a buffer, the open files are listed once per 8 KiB of
|
||||
// output, a few times for each file, rather than at each of gzip's
|
||||
// small writes, which takes far longer.
|
||||
peak := &openFilesPeak{dir: dir}
|
||||
buffered := bufio.NewWriterSize(peak, 8<<10)
|
||||
|
||||
require.NoError(t, writeExportTo(t, listExport(t, path), buffered))
|
||||
require.NoError(t, buffered.Flush())
|
||||
assert.Equal(t, 1, peak.max)
|
||||
}
|
||||
|
||||
// heapPeak is an io.Writer that discards what it is given and records
|
||||
// the largest heap it saw at a write. It collects garbage before each
|
||||
// reading, so the heap it reads is what is still held.
|
||||
type heapPeak struct {
|
||||
max uint64
|
||||
}
|
||||
|
||||
func (p *heapPeak) Write(b []byte) (int, error) {
|
||||
var m runtime.MemStats
|
||||
|
||||
runtime.GC()
|
||||
runtime.ReadMemStats(&m)
|
||||
p.max = max(p.max, m.HeapAlloc)
|
||||
|
||||
return len(b), nil
|
||||
}
|
||||
|
||||
// exportHeapGrowth exports an archive of rows random bodies, each
|
||||
// bodySize bytes of base64, and returns how far the heap rose above
|
||||
// where it stood when the export began, at its highest.
|
||||
func exportHeapGrowth(t *testing.T, rows, bodySize int) uint64 {
|
||||
t.Helper()
|
||||
|
||||
path := filepath.Join(t.TempDir(), "archive.db")
|
||||
w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
||||
|
||||
// Base64 makes four characters of every three bytes.
|
||||
random := make([]byte, bodySize/4*3)
|
||||
|
||||
for range rows {
|
||||
_, _ = rand.Read(random)
|
||||
|
||||
require.NoError(t, w.Write(delivery.ExportArchivedEvent{
|
||||
Body: base64.StdEncoding.EncodeToString(random),
|
||||
}, 0))
|
||||
}
|
||||
|
||||
export := listExport(t, path)
|
||||
|
||||
runtime.GC()
|
||||
|
||||
var start runtime.MemStats
|
||||
|
||||
runtime.ReadMemStats(&start)
|
||||
|
||||
// Through a buffer, the heap is read once per 8 KiB of output
|
||||
// rather than at each of gzip's small writes, which takes far
|
||||
// longer.
|
||||
peak := &heapPeak{max: start.HeapAlloc}
|
||||
buffered := bufio.NewWriterSize(peak, 8<<10)
|
||||
|
||||
require.NoError(t, writeExportTo(t, export, buffered))
|
||||
require.NoError(t, buffered.Flush())
|
||||
|
||||
return peak.max - start.HeapAlloc
|
||||
}
|
||||
|
||||
// TestArchiveExport_Streams proves an export holds neither the archive
|
||||
// nor its output in memory whole: exporting 384 KiB more of archive
|
||||
// raises the heap's peak by less than half of that. The export's own
|
||||
// memory, mostly gzip's compressor, is the same for both archives, so
|
||||
// it cancels out. The bodies are random bytes in base64, which gzip
|
||||
// shrinks by only a quarter, so an export that read every row before
|
||||
// writing, or built the JSON or the gzipped file before writing it,
|
||||
// would raise the peak by at least three quarters of the difference.
|
||||
//
|
||||
// The smaller archive has two rows so that its export, too, writes
|
||||
// out more than the 8 KiB buffer in exportHeapGrowth before it ends:
|
||||
// the heap must be read while the export's own memory is held.
|
||||
//
|
||||
//nolint:paralleltest // It measures the heap, which tests share.
|
||||
func TestArchiveExport_Streams(t *testing.T) {
|
||||
const (
|
||||
bodySize = 16 << 10
|
||||
smallRows = 2
|
||||
largeRows = smallRows + 24
|
||||
limit = (largeRows - smallRows) * bodySize / 2
|
||||
)
|
||||
|
||||
small := exportHeapGrowth(t, smallRows, bodySize)
|
||||
large := exportHeapGrowth(t, largeRows, bodySize)
|
||||
|
||||
assert.Less(t, large, small+limit,
|
||||
"the heap rose by %d for %d rows and by %d for %d rows",
|
||||
small, smallRows, large, largeRows,
|
||||
)
|
||||
}
|
||||
|
||||
// TestArchiveExportFileName proves the download is named for the
|
||||
// webhook and the target, with the names made safe as for the archive
|
||||
// file, and the export time in UTC.
|
||||
func TestArchiveExportFileName(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cest := time.FixedZone("CEST", int((2 * time.Hour).Seconds()))
|
||||
|
||||
assert.Equal(t,
|
||||
"archive-orders-eu-long-term-archive-20261002T120304Z.json.gz",
|
||||
delivery.ArchiveExportFileName(
|
||||
exportWebhookName, exportTargetName,
|
||||
time.Date(2026, 10, 2, 14, 3, 4, 0, cest),
|
||||
),
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
package delivery
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
)
|
||||
|
||||
// The archive rotations: how often a database target starts a new
|
||||
// archive file. Every rotation but none puts the period of an event's
|
||||
// receive time, in UTC, in the name of the file the event goes to,
|
||||
// written in the layout beside it.
|
||||
const (
|
||||
archiveRotationNone = "none"
|
||||
archiveRotationMonthly = "monthly"
|
||||
archiveRotationDaily = "daily"
|
||||
archiveRotationHourly = "hourly"
|
||||
|
||||
archiveMonthLayout = "2006-01"
|
||||
archiveDayLayout = "2006-01-02"
|
||||
archiveHourLayout = "2006-01-02-15"
|
||||
)
|
||||
|
||||
// errArchiveRotationUnknown is returned for a rotation that is not one
|
||||
// of the four.
|
||||
var errArchiveRotationUnknown = errors.New(
|
||||
"rotation must be none, monthly, daily or hourly",
|
||||
)
|
||||
|
||||
// archiveFile is one of a database target's archive files, and the
|
||||
// period in its name: "" for the file named without one.
|
||||
type archiveFile struct {
|
||||
path string
|
||||
period string
|
||||
}
|
||||
|
||||
// ValidateArchiveRotation checks a user-supplied archive rotation for
|
||||
// a database target: empty or none (both meaning one file), monthly,
|
||||
// daily or hourly.
|
||||
func ValidateArchiveRotation(rotation string) error {
|
||||
switch rotation {
|
||||
case "", archiveRotationNone, archiveRotationMonthly,
|
||||
archiveRotationDaily, archiveRotationHourly:
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("%w: %q", errArchiveRotationUnknown, rotation)
|
||||
}
|
||||
}
|
||||
|
||||
// parseArchiveRotation reads the rotation from a database target's
|
||||
// config JSON. An empty config or an empty rotation is none.
|
||||
func parseArchiveRotation(configJSON string) (string, error) {
|
||||
if configJSON == "" {
|
||||
return archiveRotationNone, nil
|
||||
}
|
||||
|
||||
var cfg databaseTargetConfig
|
||||
|
||||
err := json.Unmarshal([]byte(configJSON), &cfg)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("parsing database target config: %w", err)
|
||||
}
|
||||
|
||||
err = ValidateArchiveRotation(cfg.Rotation)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if cfg.Rotation == "" {
|
||||
return archiveRotationNone, nil
|
||||
}
|
||||
|
||||
return cfg.Rotation, nil
|
||||
}
|
||||
|
||||
// archivePeriod returns the period, in UTC, that a rotation puts an
|
||||
// event received at receivedAt in: "2026-10" for monthly,
|
||||
// "2026-10-01" for daily, "2026-10-01-19" for hourly, and "" for none.
|
||||
func archivePeriod(rotation string, receivedAt time.Time) string {
|
||||
switch rotation {
|
||||
case archiveRotationMonthly:
|
||||
return receivedAt.UTC().Format(archiveMonthLayout)
|
||||
case archiveRotationDaily:
|
||||
return receivedAt.UTC().Format(archiveDayLayout)
|
||||
case archiveRotationHourly:
|
||||
return receivedAt.UTC().Format(archiveHourLayout)
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// archivePeriodPath returns the path of a database target's archive
|
||||
// file for a period: path, as ArchivePath gives it, with "-" and the
|
||||
// period put before its ".db". The period "" gives path itself.
|
||||
func archivePeriodPath(path, period string) string {
|
||||
if period == "" {
|
||||
return path
|
||||
}
|
||||
|
||||
return strings.TrimSuffix(path, ".db") + "-" + period + ".db"
|
||||
}
|
||||
|
||||
// ArchivePathAt returns the archive file a database target writes an
|
||||
// event received at receivedAt to: ArchivePath's file, with the
|
||||
// period in its name when the target rotates.
|
||||
func ArchivePathAt(
|
||||
dbMgr *database.WebhookDBManager,
|
||||
webhook *database.Webhook,
|
||||
target *database.Target,
|
||||
receivedAt time.Time,
|
||||
) (string, error) {
|
||||
rotation, err := parseArchiveRotation(target.Config)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return archivePeriodPath(
|
||||
ArchivePath(dbMgr, webhook, target),
|
||||
archivePeriod(rotation, receivedAt),
|
||||
), nil
|
||||
}
|
||||
|
||||
// archiveFiles lists the database target's archive files that exist,
|
||||
// given the path ArchivePath gives it: the file at path, then each
|
||||
// file archivePeriodPath names for path and a period, oldest period
|
||||
// first. Which rotation wrote a file does not matter, so the files of
|
||||
// an earlier rotation setting are listed too.
|
||||
func archiveFiles(path string) ([]archiveFile, error) {
|
||||
dir := filepath.Dir(path)
|
||||
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("listing archive files: %w", err)
|
||||
}
|
||||
|
||||
stem := strings.TrimSuffix(filepath.Base(path), ".db")
|
||||
|
||||
var files []archiveFile
|
||||
|
||||
for _, entry := range entries {
|
||||
period, ok := archiveFilePeriod(stem, entry.Name())
|
||||
if ok {
|
||||
files = append(files, archiveFile{
|
||||
path: filepath.Join(dir, entry.Name()),
|
||||
period: period,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A month sorts before the days and hours in it.
|
||||
slices.SortFunc(files, func(a, b archiveFile) int {
|
||||
return strings.Compare(a.period, b.period)
|
||||
})
|
||||
|
||||
return files, nil
|
||||
}
|
||||
|
||||
// archiveFilePeriod reports whether name is the name of an archive
|
||||
// file of the target whose file name without a period is stem+".db",
|
||||
// and the period in it.
|
||||
func archiveFilePeriod(stem, name string) (string, bool) {
|
||||
rest, ok := strings.CutPrefix(name, stem)
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
|
||||
rest, ok = strings.CutSuffix(rest, ".db")
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
|
||||
if rest == "" {
|
||||
return "", true
|
||||
}
|
||||
|
||||
period, ok := strings.CutPrefix(rest, "-")
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
|
||||
for _, layout := range []string{
|
||||
archiveMonthLayout, archiveDayLayout, archiveHourLayout,
|
||||
} {
|
||||
_, err := time.Parse(layout, period)
|
||||
if err == nil {
|
||||
return period, true
|
||||
}
|
||||
}
|
||||
|
||||
return "", false
|
||||
}
|
||||
@@ -0,0 +1,389 @@
|
||||
package delivery_test
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
"sneak.berlin/go/webhooker/internal/delivery"
|
||||
)
|
||||
|
||||
// The archive rotations, and the configs of a daily target and of one
|
||||
// whose rotation is not one of the four.
|
||||
const (
|
||||
rotationNone = "none"
|
||||
rotationMonthly = "monthly"
|
||||
rotationDaily = "daily"
|
||||
rotationHourly = "hourly"
|
||||
|
||||
dailyConfig = `{"rotation":"daily"}`
|
||||
weeklyConfig = `{"rotation":"weekly"}`
|
||||
)
|
||||
|
||||
// The periods the tests archive into most: two days, and an hour of
|
||||
// the first.
|
||||
const (
|
||||
dayPeriod = "2026-03-04"
|
||||
nextDayPeriod = "2026-03-05"
|
||||
hourPeriod = "2026-03-04-05"
|
||||
)
|
||||
|
||||
// periodPath returns the archive file for a period of the target whose
|
||||
// file without a period is path.
|
||||
func periodPath(path, period string) string {
|
||||
return strings.TrimSuffix(path, ".db") + "-" + period + ".db"
|
||||
}
|
||||
|
||||
// deliverReceivedAt delivers to a database target an event whose
|
||||
// receive time is receivedAt, and returns the event's id. The receive
|
||||
// time is what decides a rotated archive's file, so setting it is how
|
||||
// these tests move the clock across a period boundary.
|
||||
func (env *archiveEnv) deliverReceivedAt(
|
||||
t *testing.T, tgt *database.Target, receivedAt time.Time,
|
||||
) string {
|
||||
t.Helper()
|
||||
|
||||
webhookDB := testWebhookDB(t)
|
||||
event := seedEvent(t, webhookDB, `{"n":1}`)
|
||||
event.CreatedAt = receivedAt
|
||||
|
||||
env.eng.ExportDeliverDatabase(
|
||||
webhookDB, seedDatabaseTargetDelivery(t, webhookDB, event, tgt),
|
||||
)
|
||||
|
||||
return event.ID
|
||||
}
|
||||
|
||||
// TestDeliverDatabase_RotatesAtEachPeriodBoundary delivers, for each
|
||||
// rotation, an event received in the last second of a period and one
|
||||
// received in the first second of the next, and checks each lands in
|
||||
// the file named for its own period, in UTC. Rotation none keeps both
|
||||
// in the one file.
|
||||
func TestDeliverDatabase_RotatesAtEachPeriodBoundary(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
berlin := time.FixedZone("CEST", 2*60*60)
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
rotation string
|
||||
before, after time.Time
|
||||
// periods are the periods of before and after.
|
||||
periods [2]string
|
||||
}{
|
||||
{
|
||||
rotationMonthly, rotationMonthly,
|
||||
time.Date(2026, 1, 31, 23, 59, 59, 0, time.UTC),
|
||||
time.Date(2026, 2, 1, 0, 0, 0, 0, time.UTC),
|
||||
[2]string{"2026-01", "2026-02"},
|
||||
},
|
||||
{
|
||||
rotationDaily, rotationDaily,
|
||||
time.Date(2026, 3, 4, 23, 59, 59, 0, time.UTC),
|
||||
time.Date(2026, 3, 5, 0, 0, 0, 0, time.UTC),
|
||||
[2]string{dayPeriod, nextDayPeriod},
|
||||
},
|
||||
{
|
||||
// The same instants, received in a zone two hours ahead
|
||||
// of UTC, where they fall on 5 March: the period is UTC's.
|
||||
"daily in another zone", rotationDaily,
|
||||
time.Date(2026, 3, 5, 1, 59, 59, 0, berlin),
|
||||
time.Date(2026, 3, 5, 2, 0, 0, 0, berlin),
|
||||
[2]string{dayPeriod, nextDayPeriod},
|
||||
},
|
||||
{
|
||||
rotationHourly, rotationHourly,
|
||||
time.Date(2026, 3, 4, 5, 59, 59, 0, time.UTC),
|
||||
time.Date(2026, 3, 4, 6, 0, 0, 0, time.UTC),
|
||||
[2]string{hourPeriod, "2026-03-04-06"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
tgt := env.seedDatabaseTarget(t, `{"rotation":"`+tc.rotation+`"}`)
|
||||
path := env.archivePath(tgt)
|
||||
|
||||
first := env.deliverReceivedAt(t, tgt, tc.before)
|
||||
second := env.deliverReceivedAt(t, tgt, tc.after)
|
||||
|
||||
assert.Equal(t, []string{first},
|
||||
archivedEventIDs(t, periodPath(path, tc.periods[0])))
|
||||
assert.Equal(t, []string{second},
|
||||
archivedEventIDs(t, periodPath(path, tc.periods[1])))
|
||||
assert.NoFileExists(t, path,
|
||||
"a rotated target never writes the file without a period")
|
||||
})
|
||||
}
|
||||
|
||||
t.Run(rotationNone, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
tgt := env.seedDatabaseTarget(t, `{"rotation":"`+rotationNone+`"}`)
|
||||
|
||||
first := env.deliverReceivedAt(t, tgt, cases[0].before)
|
||||
second := env.deliverReceivedAt(t, tgt, cases[0].after)
|
||||
|
||||
assert.ElementsMatch(t, []string{first, second},
|
||||
archivedEventIDs(t, env.archivePath(tgt)))
|
||||
})
|
||||
}
|
||||
|
||||
// TestDeliverDatabase_EventWithoutReceiveTime proves an event whose
|
||||
// receive time is not known goes to the file for the time it is
|
||||
// archived, rather than to one for the year 1.
|
||||
func TestDeliverDatabase_EventWithoutReceiveTime(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
tgt := env.seedDatabaseTarget(t, `{"rotation":"`+rotationMonthly+`"}`)
|
||||
path := env.archivePath(tgt)
|
||||
|
||||
before := time.Now().UTC().Format("2006-01")
|
||||
id := env.deliverReceivedAt(t, tgt, time.Time{})
|
||||
|
||||
file := periodPath(path, time.Now().UTC().Format("2006-01"))
|
||||
|
||||
_, err := os.Stat(file)
|
||||
if err != nil {
|
||||
// The month turned during the delivery.
|
||||
file = periodPath(path, before)
|
||||
}
|
||||
|
||||
assert.Equal(t, []string{id}, archivedEventIDs(t, file))
|
||||
assert.NoFileExists(t, periodPath(path, "0001-01"))
|
||||
}
|
||||
|
||||
// TestDeliverDatabase_RotationChangeKeepsOldFiles changes a target's
|
||||
// rotation from none to daily between two events, and checks the
|
||||
// second goes to the daily file while the first stays where it was.
|
||||
func TestDeliverDatabase_RotationChangeKeepsOldFiles(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
tgt := env.seedDatabaseTarget(t, "")
|
||||
path := env.archivePath(tgt)
|
||||
at := time.Date(2026, 3, 4, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
first := env.deliverReceivedAt(t, tgt, at)
|
||||
|
||||
tgt.Config = dailyConfig
|
||||
second := env.deliverReceivedAt(t, tgt, at)
|
||||
|
||||
assert.Equal(t, []string{first}, archivedEventIDs(t, path))
|
||||
assert.Equal(t, []string{second},
|
||||
archivedEventIDs(t, periodPath(path, dayPeriod)))
|
||||
}
|
||||
|
||||
// TestArchiveSweep_PrunesEveryFile gives a daily target three files:
|
||||
// the file without a period, left from before it rotated, and two
|
||||
// daily files. Each holds a row older than the expiry, and one daily
|
||||
// file also a newer row. The sweep prunes the old row from every file,
|
||||
// deletes the daily file it leaves empty, and keeps the file without a
|
||||
// period although it is empty too.
|
||||
func TestArchiveSweep_PrunesEveryFile(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
tgt := env.seedDatabaseTarget(
|
||||
t, `{"expiry":"1h","rotation":"`+rotationDaily+`"}`,
|
||||
)
|
||||
path := env.archivePath(tgt)
|
||||
emptied := periodPath(path, dayPeriod)
|
||||
kept := periodPath(path, nextDayPeriod)
|
||||
|
||||
now := time.Now()
|
||||
old := now.Add(-48 * time.Hour)
|
||||
|
||||
seedArchiveFile(t, path, tgt.WebhookID, old)
|
||||
seedArchiveFile(t, emptied, tgt.WebhookID, old)
|
||||
seedArchiveFile(t, kept, tgt.WebhookID, old, now.Add(-time.Minute))
|
||||
|
||||
env.sweeper.ExportSweep(t.Context())
|
||||
|
||||
assert.Empty(t, archivedEventIDs(t, path))
|
||||
assert.Equal(t, []string{sweepRowNew}, archivedEventIDs(t, kept))
|
||||
|
||||
for _, suffix := range archiveFileSuffixes() {
|
||||
assert.NoFileExists(t, emptied+suffix)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRename_MovesEveryFile renames a daily target that also has a
|
||||
// file without a period, and checks every file moves to the new name
|
||||
// with its period, rows and all, and that a later write uses the new
|
||||
// name.
|
||||
func TestRename_MovesEveryFile(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
tgt := env.seedDatabaseTarget(t, "")
|
||||
oldPath := env.archivePath(tgt)
|
||||
day := time.Date(2026, 3, 4, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
unrotated := env.deliverReceivedAt(t, tgt, day)
|
||||
|
||||
tgt.Config = dailyConfig
|
||||
first := env.deliverReceivedAt(t, tgt, day)
|
||||
second := env.deliverReceivedAt(t, tgt, day.Add(24*time.Hour))
|
||||
|
||||
require.NoError(t, env.eng.Rename(tgt.ID, "Orders", "Long Term"))
|
||||
|
||||
newPath := filepath.Join(
|
||||
env.dataDir, "archive-orders-long-term-"+tgt.ID+".db",
|
||||
)
|
||||
|
||||
for _, old := range []string{
|
||||
oldPath,
|
||||
periodPath(oldPath, dayPeriod),
|
||||
periodPath(oldPath, nextDayPeriod),
|
||||
} {
|
||||
assert.NoFileExists(t, old)
|
||||
}
|
||||
|
||||
assert.Equal(t, []string{unrotated}, archivedEventIDs(t, newPath))
|
||||
assert.Equal(t, []string{first},
|
||||
archivedEventIDs(t, periodPath(newPath, dayPeriod)))
|
||||
assert.Equal(t, []string{second},
|
||||
archivedEventIDs(t, periodPath(newPath, nextDayPeriod)))
|
||||
|
||||
third := env.deliverReceivedAt(t, tgt, day.Add(48*time.Hour))
|
||||
assert.Equal(t, []string{third},
|
||||
archivedEventIDs(t, periodPath(newPath, "2026-03-06")))
|
||||
}
|
||||
|
||||
// TestRename_NeverReplacesARotatedFile plants a file at the new name
|
||||
// of a target's daily file, and proves the rename is refused and moves
|
||||
// none of the target's files.
|
||||
func TestRename_NeverReplacesARotatedFile(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
tgt := env.seedDatabaseTarget(t, dailyConfig)
|
||||
oldPath := env.archivePath(tgt)
|
||||
day := time.Date(2026, 3, 4, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
env.deliverReceivedAt(t, tgt, day)
|
||||
env.deliverReceivedAt(t, tgt, day.Add(24*time.Hour))
|
||||
|
||||
newPath := filepath.Join(
|
||||
env.dataDir, "archive-orders-long-term-"+tgt.ID+".db",
|
||||
)
|
||||
planted := periodPath(newPath, nextDayPeriod)
|
||||
require.NoError(t, os.WriteFile(planted, []byte("planted"), 0o600))
|
||||
|
||||
require.ErrorIs(
|
||||
t, env.eng.Rename(tgt.ID, "Orders", "Long Term"),
|
||||
delivery.ErrArchiveNameTaken,
|
||||
)
|
||||
|
||||
assert.FileExists(t, periodPath(oldPath, dayPeriod))
|
||||
assert.FileExists(t, periodPath(oldPath, nextDayPeriod))
|
||||
assert.NoFileExists(t, periodPath(newPath, dayPeriod))
|
||||
}
|
||||
|
||||
// TestStatArchive_EveryFile proves StatArchive counts and adds up every
|
||||
// one of a target's files, takes the latest write of any of them, and
|
||||
// leaves out files whose names only look like the target's.
|
||||
func TestStatArchive_EveryFile(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "archive-wh.db")
|
||||
files := []string{
|
||||
path, periodPath(path, "2026-03"), periodPath(path, hourPeriod),
|
||||
}
|
||||
written := time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC)
|
||||
|
||||
var size int64
|
||||
|
||||
for i, file := range files {
|
||||
require.NoError(t, os.WriteFile(file, make([]byte, 100*(i+1)), 0o600))
|
||||
|
||||
size += int64(100 * (i + 1))
|
||||
at := written.Add(-time.Duration(i) * time.Hour)
|
||||
require.NoError(t, os.Chtimes(file, at, at))
|
||||
}
|
||||
|
||||
for _, other := range []string{
|
||||
"archive-wh-2026-13.db", "archive-wh-2026-3.db",
|
||||
"archive-wh-other.db", "archive-wh-2026-03.json",
|
||||
"archive-whx.db",
|
||||
} {
|
||||
require.NoError(t,
|
||||
os.WriteFile(filepath.Join(dir, other), []byte("x"), 0o600))
|
||||
}
|
||||
|
||||
got, err := delivery.StatArchive(path)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, len(files), got.Files)
|
||||
assert.Equal(t, size, got.Size)
|
||||
assert.True(t, written.Equal(got.Written), got.Written)
|
||||
}
|
||||
|
||||
// TestArchivePathAt names the file each rotation writes an event to.
|
||||
func TestArchivePathAt(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dataDir := t.TempDir()
|
||||
dbMgr := database.NewTestWebhookDBManager(dataDir)
|
||||
webhook := &database.Webhook{
|
||||
BaseModel: database.BaseModel{ID: "wh-id"}, Name: "Orders",
|
||||
}
|
||||
at := time.Date(2026, 10, 1, 19, 30, 0, 0, time.UTC)
|
||||
|
||||
cases := map[string]string{
|
||||
"": "",
|
||||
`{"rotation":"` + rotationNone + `"}`: "",
|
||||
`{"rotation":"` + rotationMonthly + `"}`: "-2026-10",
|
||||
dailyConfig: "-2026-10-01",
|
||||
`{"rotation":"` + rotationHourly + `"}`: "-2026-10-01-19",
|
||||
}
|
||||
|
||||
for config, period := range cases {
|
||||
target := &database.Target{
|
||||
BaseModel: database.BaseModel{ID: "tgt-id"},
|
||||
Name: "Archive",
|
||||
Config: config,
|
||||
}
|
||||
|
||||
got, err := delivery.ArchivePathAt(dbMgr, webhook, target, at)
|
||||
require.NoError(t, err, config)
|
||||
assert.Equal(t,
|
||||
filepath.Join(
|
||||
dataDir, "archive-orders-archive-tgt-id"+period+".db",
|
||||
),
|
||||
got, config,
|
||||
)
|
||||
}
|
||||
|
||||
_, err := delivery.ArchivePathAt(dbMgr, webhook, &database.Target{
|
||||
Config: weeklyConfig,
|
||||
}, at)
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
// TestValidateArchiveRotation accepts the four rotations, and empty,
|
||||
// and refuses anything else.
|
||||
func TestValidateArchiveRotation(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, ok := range []string{
|
||||
"", rotationNone, rotationMonthly, rotationDaily, rotationHourly,
|
||||
} {
|
||||
require.NoError(t, delivery.ValidateArchiveRotation(ok), ok)
|
||||
}
|
||||
|
||||
for _, bad := range []string{"weekly", "Daily", "hourly "} {
|
||||
require.Error(t, delivery.ValidateArchiveRotation(bad), bad)
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package delivery_test
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -17,6 +18,7 @@ import (
|
||||
_ "modernc.org/sqlite" // Pure Go SQLite driver.
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
"sneak.berlin/go/webhooker/internal/delivery"
|
||||
"sneak.berlin/go/webhooker/internal/gormlog"
|
||||
)
|
||||
|
||||
func archiveTestLogger() *slog.Logger {
|
||||
@@ -42,7 +44,8 @@ func openArchiveDBForRead(
|
||||
t.Cleanup(func() { _ = sqlDB.Close() })
|
||||
|
||||
gdb, err := gorm.Open(
|
||||
sqlite.Dialector{Conn: sqlDB}, &gorm.Config{},
|
||||
sqlite.Dialector{Conn: sqlDB},
|
||||
&gorm.Config{Logger: gormlog.New(slog.New(slog.DiscardHandler))},
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -181,16 +184,67 @@ func TestArchiveWriter_RecreatesAfterRemoval(
|
||||
assert.Equal(t, "b", got[0].EventID)
|
||||
}
|
||||
|
||||
// TestStatArchive proves StatArchive finds no file before the first
|
||||
// write; after a write still held in the -wal, counts the -wal in the
|
||||
// size and takes its later time as the last write; and finds no file
|
||||
// again once the file has been moved away.
|
||||
func TestStatArchive(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
path := filepath.Join(t.TempDir(), "archive-wh.db")
|
||||
|
||||
_, err := delivery.StatArchive(path)
|
||||
require.ErrorIs(t, err, fs.ErrNotExist)
|
||||
|
||||
// With the clock stopped, the reopen debounce never passes, so
|
||||
// the handle stays open after the write.
|
||||
stopped := time.Now()
|
||||
w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
||||
w.SetNow(func() time.Time { return stopped })
|
||||
|
||||
require.NoError(t, w.Write(delivery.ExportArchivedEvent{EventID: "a"}, 0))
|
||||
|
||||
written := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
||||
earlier := written.Add(-time.Hour)
|
||||
require.NoError(t, os.Chtimes(path, earlier, earlier))
|
||||
require.NoError(t, os.Chtimes(path+"-wal", written, written))
|
||||
|
||||
file, err := os.Stat(path)
|
||||
require.NoError(t, err)
|
||||
|
||||
wal, err := os.Stat(path + "-wal")
|
||||
require.NoError(t, err)
|
||||
require.Positive(t, wal.Size())
|
||||
|
||||
got, err := delivery.StatArchive(path)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, got.Files)
|
||||
assert.Equal(t, file.Size()+wal.Size(), got.Size)
|
||||
assert.True(t, written.Equal(got.Written), got.Written)
|
||||
|
||||
removeArchiveFiles(t, path)
|
||||
|
||||
_, err = delivery.StatArchive(path)
|
||||
require.ErrorIs(t, err, fs.ErrNotExist)
|
||||
}
|
||||
|
||||
func TestArchiveWriter_ReopenDebounce(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// A generous debounce keeps the two rapid writes inside
|
||||
// the window even on a heavily loaded test machine.
|
||||
const debounce = 2 * time.Second
|
||||
|
||||
path := filepath.Join(t.TempDir(), "archive-wh.db")
|
||||
w := delivery.NewExportArchiveWriter(
|
||||
path, archiveTestLogger(), 2*time.Second,
|
||||
path, archiveTestLogger(), debounce,
|
||||
)
|
||||
|
||||
// The writer measures its reopen debounce on this clock, which
|
||||
// only the test moves, so how long the host takes between
|
||||
// writes cannot change the result.
|
||||
now := time.Now()
|
||||
|
||||
w.SetNow(func() time.Time { return now })
|
||||
|
||||
require.NoError(t, w.Write(
|
||||
delivery.ExportArchivedEvent{EventID: "a"}, 0,
|
||||
))
|
||||
@@ -202,7 +256,7 @@ func TestArchiveWriter_ReopenDebounce(t *testing.T) {
|
||||
// initial open — no extra close/reopen.
|
||||
assert.Equal(t, 1, w.Reopens())
|
||||
|
||||
time.Sleep(2100 * time.Millisecond)
|
||||
now = now.Add(debounce)
|
||||
|
||||
require.NoError(t, w.Write(
|
||||
delivery.ExportArchivedEvent{EventID: "c"}, 0,
|
||||
@@ -667,3 +721,41 @@ func TestArchiveWriter_RenameMovesBackOnFailure(t *testing.T) {
|
||||
assert.NoFileExists(t, filepath.Join(dir, newName))
|
||||
assert.Equal(t, oldPath, w.Path())
|
||||
}
|
||||
|
||||
// TestArchiveWriter_RenameMovesBackEveryFile renames a target with a
|
||||
// file without a period and a file for a month, and proves that when
|
||||
// the month's file fails to move, the file already moved is moved back:
|
||||
// both files are under the old name with their rows, and nothing is
|
||||
// under the new name. The new name is 251 bytes, so the file without a
|
||||
// period, with its -wal and -shm, can take it, but the month's file,
|
||||
// eight bytes longer, cannot.
|
||||
func TestArchiveWriter_RenameMovesBackEveryFile(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
oldPath := filepath.Join(dir, "archive-old.db")
|
||||
monthPath := filepath.Join(dir, "archive-old-2026-03.db")
|
||||
w := delivery.NewExportArchiveWriter(oldPath, archiveTestLogger(), 0)
|
||||
|
||||
require.NoError(t, w.WritePeriod(
|
||||
delivery.ExportArchivedEvent{EventID: "in-none"}, 0, "",
|
||||
))
|
||||
require.NoError(t, w.WritePeriod(
|
||||
delivery.ExportArchivedEvent{EventID: "in-month"}, 0, "2026-03",
|
||||
))
|
||||
|
||||
require.Error(t, w.Rename(strings.Repeat("a", 248)+".db"))
|
||||
|
||||
assert.Equal(t, []string{"in-none"}, archivedEventIDs(t, oldPath))
|
||||
assert.Equal(t, []string{"in-month"}, archivedEventIDs(t, monthPath))
|
||||
|
||||
entries, err := os.ReadDir(dir)
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, entry := range entries {
|
||||
assert.True(t, strings.HasPrefix(entry.Name(), "archive-old"),
|
||||
"%s is not under the old name", entry.Name())
|
||||
}
|
||||
|
||||
assert.Equal(t, oldPath, w.Path())
|
||||
}
|
||||
|
||||
@@ -204,10 +204,11 @@ func TestNewTargetConfigForm(t *testing.T) {
|
||||
|
||||
form, err = delivery.NewTargetConfigForm(&database.Target{
|
||||
Type: database.TargetTypeDatabase,
|
||||
Config: `{"expiry":"720h"}`,
|
||||
Config: `{"expiry":"720h","rotation":"daily"}`,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "720h", form.Expiry)
|
||||
assert.Equal(t, rotationDaily, form.Rotation)
|
||||
|
||||
form, err = delivery.NewTargetConfigForm(&database.Target{
|
||||
Type: database.TargetTypeLog,
|
||||
@@ -216,8 +217,9 @@ func TestNewTargetConfigForm(t *testing.T) {
|
||||
assert.Empty(t, form.URL)
|
||||
}
|
||||
|
||||
// A keep-forever archive target must pre-fill as an empty field, so
|
||||
// saving the form back unchanged stores the same empty config.
|
||||
// A keep-forever archive target yields an empty expiry, so the edit
|
||||
// form starts on never; saving it unchanged stores never, which means
|
||||
// the same as an empty expiry.
|
||||
func TestNewTargetConfigForm_DatabaseNeverIsBlank(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -247,6 +249,10 @@ func TestNewTargetConfigForm_UnreadableConfigErrors(t *testing.T) {
|
||||
Type: database.TargetTypeDatabase,
|
||||
Config: `{"expiry":"soon"}`,
|
||||
},
|
||||
{
|
||||
Type: database.TargetTypeDatabase,
|
||||
Config: weeklyConfig,
|
||||
},
|
||||
{Type: database.TargetType("nope")},
|
||||
}
|
||||
|
||||
|
||||
@@ -234,7 +234,7 @@ func (c *httpCore) handleRetry(
|
||||
database.DeliveryStatusRetrying,
|
||||
)
|
||||
|
||||
backoff := calcBackoff(attemptNum)
|
||||
backoff := Backoff(attemptNum)
|
||||
|
||||
retryTask := *task
|
||||
retryTask.AttemptNum = attemptNum + 1
|
||||
@@ -301,7 +301,7 @@ func (c *httpCore) remainingBackoff(
|
||||
return 0
|
||||
}
|
||||
|
||||
backoff := calcBackoff(attemptNum)
|
||||
backoff := Backoff(attemptNum)
|
||||
elapsed := time.Since(lastResult.CreatedAt)
|
||||
remaining := backoff - elapsed
|
||||
|
||||
@@ -326,12 +326,14 @@ func (c *httpCore) backoffElapsed(
|
||||
return true
|
||||
}
|
||||
|
||||
backoff := calcBackoff(attemptNum)
|
||||
backoff := Backoff(attemptNum)
|
||||
|
||||
return time.Since(lastResult.CreatedAt) >= backoff
|
||||
}
|
||||
|
||||
func calcBackoff(attemptNum int) time.Duration {
|
||||
// Backoff is how long an http or slack target with retries waits after
|
||||
// a delivery's failed attempt attemptNum before trying it again.
|
||||
func Backoff(attemptNum int) time.Duration {
|
||||
shift := max(attemptNum-1, 0)
|
||||
shift = min(shift, maxBackoffShift)
|
||||
|
||||
|
||||
@@ -573,6 +573,8 @@ func TestRecoverPending_TargetDeleted(t *testing.T) {
|
||||
|
||||
s := newISetup(t)
|
||||
|
||||
iCreateWebhook(t, s.MainDB, s.WebhookID, "pending-recovery")
|
||||
|
||||
deliveryID := tSeedDeletedTarget(
|
||||
t, s, "gone-while-pending", "http://example.com/hook",
|
||||
database.DeliveryStatusPending,
|
||||
@@ -612,6 +614,8 @@ func TestRecoverPending_TargetDeleted_LeavesAnOwnedDeliveryAlone(
|
||||
|
||||
s := newISetup(t)
|
||||
|
||||
iCreateWebhook(t, s.MainDB, s.WebhookID, "owned-recovery")
|
||||
|
||||
deliveryID := tSeedDeletedTarget(
|
||||
t, s, "gone-but-owned", "http://example.com/hook",
|
||||
database.DeliveryStatusPending,
|
||||
|
||||
@@ -179,6 +179,27 @@ func TestDoHTTPRequest_TransportErrorMasksURL(t *testing.T) {
|
||||
)
|
||||
}
|
||||
|
||||
// TestDoHTTPRequest_UnparsableURLIsMasked is the same for an HTTP
|
||||
// target URL that no request can be built from.
|
||||
func TestDoHTTPRequest_UnparsableURLIsMasked(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
e := testEngine(t, 1)
|
||||
|
||||
statusCode, _, _, reqErr := e.ExportDoHTTPRequest(
|
||||
context.TODO(),
|
||||
&delivery.HTTPTargetConfig{
|
||||
URL: "https://hooks.example.com" + maskSecretPath + "\n",
|
||||
},
|
||||
&database.Event{},
|
||||
)
|
||||
require.Error(t, reqErr)
|
||||
assert.Zero(t, statusCode)
|
||||
|
||||
assertNoCredential(t, reqErr.Error())
|
||||
assert.Contains(t, reqErr.Error(), "invalid control character")
|
||||
}
|
||||
|
||||
// TestValidateTargetURL_UnparsableURLIsMasked proves the SSRF
|
||||
// validator's error does not carry the submitted URL, which
|
||||
// the handler both logs and shows.
|
||||
|
||||
@@ -117,8 +117,8 @@ func readFirstBootSecrets(
|
||||
}
|
||||
|
||||
// bootAtDebug starts and stops the real application graph against
|
||||
// dataDir with DEBUG=true, and returns everything it wrote to standard
|
||||
// output.
|
||||
// dataDir with DEBUG=true and nothing else set, and returns everything
|
||||
// it wrote to standard output.
|
||||
//
|
||||
// config.New reads DEBUG from the environment exactly as the binary
|
||||
// does, internal/logger builds the handler it builds in production,
|
||||
@@ -128,6 +128,7 @@ func readFirstBootSecrets(
|
||||
func bootAtDebug(t *testing.T, dataDir string) string {
|
||||
t.Helper()
|
||||
|
||||
config.ClearEnvForTest(t)
|
||||
t.Setenv("DEBUG", "true")
|
||||
t.Setenv("DATA_DIR", dataDir)
|
||||
|
||||
|
||||
@@ -111,9 +111,9 @@ func (l *Logger) LogMode(gormlogger.LogLevel) gormlogger.Interface {
|
||||
//
|
||||
// One GORM path does not consult this: (*gorm.DB).Scan records the
|
||||
// statement through gorm's own traceRecorder, which does not implement
|
||||
// this interface. No production code path calls it; its one caller is
|
||||
// internal/database/database_test.go:91, whose SELECT 1 binds nothing.
|
||||
// scan_guard_test.go fails if a non-test file calls it.
|
||||
// this interface. No production code path calls it; only tests do, and
|
||||
// what a test binds is fixture data. scan_guard_test.go fails if a
|
||||
// non-test file calls it.
|
||||
// (*gorm.DB).Pluck, Row and Raw all run through the normal callback
|
||||
// processor and are filtered.
|
||||
func (l *Logger) ParamsFilter(
|
||||
|
||||
@@ -14,18 +14,16 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// minNonTestFiles guards the walk below against passing because it
|
||||
// found nothing to look at. The tree held 60 non-test .go files when
|
||||
// this was written.
|
||||
const minNonTestFiles = 40
|
||||
|
||||
// isRowProducer reports whether name is a method that returns a
|
||||
// database/sql row handle. GORM's Row and Rows return *sql.Row and
|
||||
// *sql.Rows, so Scan on the result of one of them is database/sql's
|
||||
// Scan and never (*gorm.DB).Scan.
|
||||
// isRowProducer reports whether name is GORM's Row or database/sql's
|
||||
// QueryRow or QueryRowContext, which return a *sql.Row whose Scan is
|
||||
// database/sql's and not (*gorm.DB).Scan. GORM's Rows is not listed:
|
||||
// it also returns an error, so Scan is never called on its result
|
||||
// directly. It matches the method name only and resolves no types, so
|
||||
// a repo-local method with one of these names that returns *gorm.DB
|
||||
// gets past it: Scan on that method's result is not reported.
|
||||
func isRowProducer(name string) bool {
|
||||
switch name {
|
||||
case "Row", "Rows", "QueryRow", "QueryRowContext":
|
||||
case "Row", "QueryRow", "QueryRowContext":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
@@ -50,9 +48,14 @@ func receiverIsRowHandle(x ast.Expr) bool {
|
||||
}
|
||||
|
||||
// unguardedScans returns the position of every Scan call in file whose
|
||||
// receiver is not a row handle. It fails closed: a receiver it cannot
|
||||
// resolve syntactically — a local variable, a struct field — is
|
||||
// reported rather than assumed safe.
|
||||
// receiver is not a call to a row producer. It fails closed: any other
|
||||
// receiver — a local variable, a struct field, a call to any other
|
||||
// method — is reported rather than assumed safe.
|
||||
//
|
||||
// It sees only calls written x.Scan(...). A method value, f := db.Scan
|
||||
// followed by f(&v), is out of scope: Scan is never the called
|
||||
// expression there, and nobody writes a query that way by accident,
|
||||
// which is the mistake this check exists to catch.
|
||||
func unguardedScans(
|
||||
fset *token.FileSet, file *ast.File,
|
||||
) []token.Position {
|
||||
@@ -111,15 +114,15 @@ func skipDir(name string) bool {
|
||||
}
|
||||
}
|
||||
|
||||
// walkNonTestGo parses every non-test .go file under root and returns
|
||||
// how many it parsed along with every unguarded Scan it found.
|
||||
func walkNonTestGo(t *testing.T, root string) (int, []string) {
|
||||
// walkNonTestGo parses every non-test .go file under root. It returns
|
||||
// the directories, relative to root, it parsed a file in, along with
|
||||
// every unguarded Scan it found.
|
||||
func walkNonTestGo(t *testing.T, root string) (map[string]bool, []string) {
|
||||
t.Helper()
|
||||
|
||||
var (
|
||||
parsed int
|
||||
hits []string
|
||||
)
|
||||
walked := map[string]bool{}
|
||||
|
||||
var hits []string
|
||||
|
||||
fset := token.NewFileSet()
|
||||
|
||||
@@ -147,7 +150,12 @@ func walkNonTestGo(t *testing.T, root string) (int, []string) {
|
||||
return err
|
||||
}
|
||||
|
||||
parsed++
|
||||
dir, err := filepath.Rel(root, filepath.Dir(path))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
walked[dir] = true
|
||||
|
||||
for _, pos := range unguardedScans(fset, file) {
|
||||
hits = append(hits, relPosition(root, pos))
|
||||
@@ -157,7 +165,7 @@ func walkNonTestGo(t *testing.T, root string) (int, []string) {
|
||||
},
|
||||
))
|
||||
|
||||
return parsed, hits
|
||||
return walked, hits
|
||||
}
|
||||
|
||||
// isNonTestGo reports whether a file name is Go source this check
|
||||
@@ -189,19 +197,39 @@ func relPosition(root string, pos token.Position) string {
|
||||
// logged with its values interpolated. The package comment states the
|
||||
// limit; this fails when someone adds a call site anyway.
|
||||
//
|
||||
// The current tree has one caller, internal/database/database_test.go,
|
||||
// which this check does not govern: it is test-only and its SELECT 1
|
||||
// binds nothing.
|
||||
// Test files are not governed: what a test binds is fixture data.
|
||||
func TestGormScanIsNeverCalledOutsideTests(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
parsed, offenders := walkNonTestGo(t, moduleRoot(t))
|
||||
root := moduleRoot(t)
|
||||
walked, offenders := walkNonTestGo(t, root)
|
||||
|
||||
// The module's packages are static, templates, and every directory
|
||||
// directly under cmd and internal. Each holds non-test code, so one
|
||||
// the walk parsed nothing in was skipped, and a Scan there would
|
||||
// pass unseen.
|
||||
packages := []string{"static", "templates"}
|
||||
|
||||
for _, parent := range []string{"cmd", "internal"} {
|
||||
entries, err := os.ReadDir(filepath.Join(root, parent))
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
|
||||
packages = append(packages, filepath.Join(parent, entry.Name()))
|
||||
}
|
||||
}
|
||||
|
||||
for _, dir := range packages {
|
||||
require.True(
|
||||
t, walked[dir],
|
||||
"the walk parsed no non-test .go file in %s", dir,
|
||||
)
|
||||
}
|
||||
|
||||
require.GreaterOrEqual(
|
||||
t, parsed, minNonTestFiles,
|
||||
"parsed %d non-test .go files, so this check found "+
|
||||
"nothing to look at", parsed,
|
||||
)
|
||||
require.Empty(
|
||||
t, offenders,
|
||||
"Scan called on a receiver this check cannot show is a "+
|
||||
@@ -222,18 +250,51 @@ type scanGuardCase struct {
|
||||
want int
|
||||
}
|
||||
|
||||
// scanGuardCases covers each receiver form unguardedScans names, plus
|
||||
// each row producer isRowProducer lets through. Each body is valid Go
|
||||
// inside plantedFile.
|
||||
func scanGuardCases() []scanGuardCase {
|
||||
return []scanGuardCase{
|
||||
{"gorm chain", `db.DB().Raw("SELECT 1").Scan(&v)`, 1},
|
||||
{"gorm receiver", `gdb.Scan(&v)`, 1},
|
||||
{"gorm via variable", "q := gdb.Raw(\"x\")\nq.Scan(&v)", 1},
|
||||
{"gorm model chain", `gdb.Model(&x).Scan(&v)`, 1},
|
||||
{"sql row", `gdb.Raw("SELECT 1").Row().Scan(&v)`, 0},
|
||||
{"sql rows", `gdb.Raw("SELECT 1").Rows().Scan(&v)`, 0},
|
||||
{"local variable", "q := gdb.Raw(\"SELECT 1\")\n\tq.Scan(&v)", 1},
|
||||
{"struct field", `s.db.Scan(&v)`, 1},
|
||||
{"gorm chain", `gdb.Raw("SELECT 1").Scan(&v)`, 1},
|
||||
{
|
||||
"sql rows in a variable",
|
||||
"rows, _ := gdb.Raw(\"SELECT 1\").Rows()\n\trows.Scan(&v)",
|
||||
1,
|
||||
},
|
||||
{"gorm Row", `gdb.Raw("SELECT 1").Row().Scan(&v)`, 0},
|
||||
{"sql QueryRow", `sqlDB.QueryRow("SELECT 1").Scan(&v)`, 0},
|
||||
{
|
||||
"sql QueryRowContext",
|
||||
`sqlDB.QueryRowContext(ctx, "SELECT 1").Scan(&v)`,
|
||||
0,
|
||||
},
|
||||
{"unrelated call", `gdb.Find(&v)`, 0},
|
||||
}
|
||||
}
|
||||
|
||||
// plantedFile wraps one case body in a function that declares every
|
||||
// name the bodies use, so each body is the Go it stands for. The result
|
||||
// is parsed, never compiled.
|
||||
const plantedFile = `package p
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
type store struct{ db *gorm.DB }
|
||||
|
||||
func f(ctx context.Context, gdb *gorm.DB, sqlDB *sql.DB, s store) {
|
||||
var v int
|
||||
|
||||
%s
|
||||
}
|
||||
`
|
||||
|
||||
// TestScanGuard_ReportsPlantedCalls proves the check fires. Without it
|
||||
// a detector that matched nothing would satisfy the walk above no
|
||||
// matter what the tree contained.
|
||||
@@ -245,9 +306,7 @@ func TestScanGuard_ReportsPlantedCalls(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fset := token.NewFileSet()
|
||||
src := fmt.Sprintf(
|
||||
"package p\n\nfunc f() {\n\t%s\n}\n", tc.body,
|
||||
)
|
||||
src := fmt.Sprintf(plantedFile, tc.body)
|
||||
|
||||
file, err := parser.ParseFile(
|
||||
fset, tc.name+".go", src, 0,
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
package handlers
|
||||
|
||||
const (
|
||||
// archiveExpiryNever is the archive expiry that keeps archived
|
||||
// events forever. A stored empty expiry means the same.
|
||||
archiveExpiryNever = "never"
|
||||
|
||||
// tmplKeyArchiveExpiryChoices is the template data key for the
|
||||
// entries of a page's archive expiry select.
|
||||
tmplKeyArchiveExpiryChoices = "ArchiveExpiryChoices"
|
||||
)
|
||||
|
||||
// archiveChoice is one entry of a database target's archive expiry
|
||||
// or archive rotation select: the value stored, the label shown, and
|
||||
// whether the select starts on it.
|
||||
type archiveChoice struct {
|
||||
Value string
|
||||
Label string
|
||||
Selected bool
|
||||
}
|
||||
|
||||
// archiveExpiryChoices lists the archive expiries offered by the new
|
||||
// webhook page, the add target form and the target edit form.
|
||||
func archiveExpiryChoices() []archiveChoice {
|
||||
return []archiveChoice{
|
||||
{Value: archiveExpiryNever, Label: archiveExpiryNever},
|
||||
{Value: "1h", Label: "1h"},
|
||||
{Value: "12h", Label: "12h"},
|
||||
{Value: "24h", Label: "24h"},
|
||||
{Value: "720h", Label: "30d"},
|
||||
{Value: "2160h", Label: "90d"},
|
||||
{Value: "8760h", Label: "365d"},
|
||||
}
|
||||
}
|
||||
|
||||
// archiveExpiryOptions returns the choices with expiry selected; an
|
||||
// empty expiry selects never. An expiry that is not one of the
|
||||
// choices comes first as its own selected entry, so saving the form
|
||||
// unchanged keeps it.
|
||||
func archiveExpiryOptions(expiry string) []archiveChoice {
|
||||
if expiry == "" {
|
||||
expiry = archiveExpiryNever
|
||||
}
|
||||
|
||||
options := archiveExpiryChoices()
|
||||
|
||||
for i := range options {
|
||||
if options[i].Value == expiry {
|
||||
options[i].Selected = true
|
||||
|
||||
return options
|
||||
}
|
||||
}
|
||||
|
||||
own := archiveChoice{Value: expiry, Label: expiry, Selected: true}
|
||||
|
||||
return append([]archiveChoice{own}, options...)
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
package handlers_test
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
)
|
||||
|
||||
// expiryNever is the archive expiry that keeps archived events
|
||||
// forever.
|
||||
const expiryNever = "never"
|
||||
|
||||
// matched returns what the one group of pattern matched in page, at
|
||||
// each match.
|
||||
func matched(pattern, page string) []string {
|
||||
matches := regexp.MustCompile(pattern).FindAllStringSubmatch(page, -1)
|
||||
groups := make([]string, 0, len(matches))
|
||||
|
||||
for _, m := range matches {
|
||||
groups = append(groups, m[1])
|
||||
}
|
||||
|
||||
return groups
|
||||
}
|
||||
|
||||
// expiryShown returns the archive expiries the webhook page's target
|
||||
// list shows.
|
||||
func expiryShown(
|
||||
t *testing.T, env *sourceTestEnv, webhookID string,
|
||||
) []string {
|
||||
t.Helper()
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
env.handlers.HandleSourceDetail().ServeHTTP(w, getRequest(
|
||||
t, "/hook/"+webhookID, env.cookies,
|
||||
map[string]string{sourceIDParam: webhookID},
|
||||
))
|
||||
require.Equal(t, http.StatusOK, w.Code)
|
||||
|
||||
return matched(
|
||||
`Archive Expiry:</span>\s*<span>([^<]*)</span>`, w.Body.String(),
|
||||
)
|
||||
}
|
||||
|
||||
// expirySelected returns the target edit page and the expiries its
|
||||
// expiry select starts on.
|
||||
func expirySelected(
|
||||
t *testing.T, env *sourceTestEnv, webhookID, targetID string,
|
||||
) (string, []string) {
|
||||
t.Helper()
|
||||
|
||||
page := targetEditPage(t, env, webhookID, targetID)
|
||||
|
||||
return page, selectedIn(page, "expiry")
|
||||
}
|
||||
|
||||
// targetEditPage returns a target's edit page.
|
||||
func targetEditPage(
|
||||
t *testing.T, env *sourceTestEnv, webhookID, targetID string,
|
||||
) string {
|
||||
t.Helper()
|
||||
|
||||
w := serveTarget(
|
||||
env, http.MethodGet,
|
||||
"/hook/"+webhookID+"/targets/"+targetID+"/edit", nil,
|
||||
)
|
||||
require.Equal(t, http.StatusOK, w.Code)
|
||||
|
||||
return w.Body.String()
|
||||
}
|
||||
|
||||
// selectedIn returns the values the select named name on page starts
|
||||
// on.
|
||||
func selectedIn(page, name string) []string {
|
||||
_, rest, _ := strings.Cut(page, `<select id="`+name+`" name="`+name+`"`)
|
||||
options, _, _ := strings.Cut(rest, "</select>")
|
||||
|
||||
return matched(`<option value="([^"]*)" selected>`, options)
|
||||
}
|
||||
|
||||
// TestArchiveExpiryChoices adds a database target with each archive
|
||||
// expiry the forms offer, and checks that it is stored as chosen,
|
||||
// shown in plain units in the target list, and that the target edit
|
||||
// form starts on it.
|
||||
func TestArchiveExpiryChoices(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupSourceTest(t)
|
||||
|
||||
choices := []struct{ value, shown string }{
|
||||
{expiryNever, expiryNever},
|
||||
{"1h", "1 hour"},
|
||||
{"12h", "12 hours"},
|
||||
{"24h", "1 day"},
|
||||
{"720h", "30 days"},
|
||||
{"2160h", "90 days"},
|
||||
{"8760h", "365 days"},
|
||||
}
|
||||
|
||||
for _, choice := range choices {
|
||||
t.Run(choice.value, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
webhook := seedWebhookWithRetention(t, env.db, 30)
|
||||
|
||||
form := url.Values{}
|
||||
form.Set("name", "archive")
|
||||
form.Set("type", string(database.TargetTypeDatabase))
|
||||
form.Set("expiry", choice.value)
|
||||
|
||||
w := serveTarget(
|
||||
env, http.MethodPost, "/hook/"+webhook.ID+"/targets", form,
|
||||
)
|
||||
require.Equal(t, http.StatusSeeOther, w.Code, w.Body.String())
|
||||
|
||||
targets := targetsForWebhook(t, env.db, webhook.ID)
|
||||
require.Len(t, targets, 1)
|
||||
assert.JSONEq(
|
||||
t, `{"expiry":"`+choice.value+`"}`, targets[0].Config,
|
||||
)
|
||||
|
||||
assert.Equal(
|
||||
t, []string{choice.shown},
|
||||
expiryShown(t, env, webhook.ID),
|
||||
)
|
||||
|
||||
_, selected := expirySelected(t, env, webhook.ID, targets[0].ID)
|
||||
assert.Equal(t, []string{choice.value}, selected)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestArchiveExpiryEditStartsOnStoredValue checks the edit form of a
|
||||
// database target whose stored expiry is empty, which selects never,
|
||||
// and of one whose expiry is not one of the choices, which is listed
|
||||
// first as its own selected entry and saved unchanged.
|
||||
func TestArchiveExpiryEditStartsOnStoredValue(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupSourceTest(t)
|
||||
|
||||
webhook := seedWebhookWithRetention(t, env.db, 30)
|
||||
empty := seedConfiguredTarget(
|
||||
t, env.db, webhook.ID, database.TargetTypeDatabase, "",
|
||||
)
|
||||
|
||||
_, selected := expirySelected(t, env, webhook.ID, empty.ID)
|
||||
assert.Equal(t, []string{expiryNever}, selected)
|
||||
|
||||
webhook = seedWebhookWithRetention(t, env.db, 30)
|
||||
unlisted := seedConfiguredTarget(
|
||||
t, env.db, webhook.ID, database.TargetTypeDatabase,
|
||||
`{"expiry":"36h"}`,
|
||||
)
|
||||
|
||||
assert.Equal(t, []string{"36 hours"}, expiryShown(t, env, webhook.ID))
|
||||
|
||||
page, selected := expirySelected(t, env, webhook.ID, unlisted.ID)
|
||||
assert.Equal(t, []string{"36h"}, selected)
|
||||
assert.Regexp(
|
||||
t,
|
||||
`<select id="expiry" name="expiry" class="input">\s*`+
|
||||
`<option value="36h" selected>36h</option>\s*`+
|
||||
`<option value="never">never</option>`,
|
||||
page,
|
||||
)
|
||||
assert.Contains(t, page, `<option value="8760h">365d</option>`)
|
||||
|
||||
form := url.Values{}
|
||||
form.Set("name", unlisted.Name)
|
||||
form.Set("expiry", "36h")
|
||||
|
||||
w := submitTargetEdit(env, webhook.ID, unlisted.ID, form)
|
||||
require.Equal(t, http.StatusSeeOther, w.Code, w.Body.String())
|
||||
assert.JSONEq(
|
||||
t, `{"expiry":"36h"}`, storedTarget(t, env, unlisted.ID).Config,
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package handlers
|
||||
|
||||
const (
|
||||
// archiveRotationNone is the archive rotation that keeps a
|
||||
// database target's archive in one file. A stored empty rotation
|
||||
// means the same.
|
||||
archiveRotationNone = "none"
|
||||
|
||||
// tmplKeyArchiveRotationChoices is the template data key for the
|
||||
// entries of a page's archive rotation select.
|
||||
tmplKeyArchiveRotationChoices = "ArchiveRotationChoices"
|
||||
)
|
||||
|
||||
// archiveRotationChoices lists the archive rotations offered by the
|
||||
// new webhook page, the add target form and the target edit form.
|
||||
func archiveRotationChoices() []archiveChoice {
|
||||
return []archiveChoice{
|
||||
{Value: archiveRotationNone, Label: archiveRotationNone},
|
||||
{Value: "monthly", Label: "monthly"},
|
||||
{Value: "daily", Label: "daily"},
|
||||
{Value: "hourly", Label: "hourly"},
|
||||
}
|
||||
}
|
||||
|
||||
// archiveRotationOptions returns the choices with rotation selected;
|
||||
// an empty rotation, or one that is not a choice, selects none. A
|
||||
// stored rotation is always a choice: the forms refuse any other.
|
||||
func archiveRotationOptions(rotation string) []archiveChoice {
|
||||
options := archiveRotationChoices()
|
||||
|
||||
for i := range options {
|
||||
if options[i].Value == rotation {
|
||||
options[i].Selected = true
|
||||
|
||||
return options
|
||||
}
|
||||
}
|
||||
|
||||
options[0].Selected = true
|
||||
|
||||
return options
|
||||
}
|
||||
@@ -0,0 +1,229 @@
|
||||
package handlers_test
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
"sneak.berlin/go/webhooker/internal/delivery"
|
||||
)
|
||||
|
||||
// rotationNone is the archive rotation that keeps one file.
|
||||
const rotationNone = "none"
|
||||
|
||||
// rotationShown returns the archive rotations the webhook page's
|
||||
// target list shows.
|
||||
func rotationShown(
|
||||
t *testing.T, env *sourceTestEnv, webhookID string,
|
||||
) []string {
|
||||
t.Helper()
|
||||
|
||||
return matched(
|
||||
`Archive Rotation:</span>\s*<span>([^<]*)</span>`,
|
||||
renderedPage(t, env, webhookID),
|
||||
)
|
||||
}
|
||||
|
||||
// TestArchiveRotationChoices adds a database target with each archive
|
||||
// rotation the forms offer, and checks that it is stored as chosen,
|
||||
// shown in the target list, and that the target edit form starts on
|
||||
// it. It then edits the target to hourly, keeping its expiry.
|
||||
func TestArchiveRotationChoices(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupSourceTest(t)
|
||||
|
||||
for _, rotation := range []string{rotationNone, "monthly", "daily", "hourly"} {
|
||||
t.Run(rotation, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
webhook := seedWebhookWithRetention(t, env.db, 30)
|
||||
|
||||
form := url.Values{}
|
||||
form.Set("name", "archive")
|
||||
form.Set("type", string(database.TargetTypeDatabase))
|
||||
form.Set("expiry", "720h")
|
||||
form.Set("rotation", rotation)
|
||||
|
||||
w := serveTarget(
|
||||
env, http.MethodPost, "/hook/"+webhook.ID+"/targets", form,
|
||||
)
|
||||
require.Equal(t, http.StatusSeeOther, w.Code, w.Body.String())
|
||||
|
||||
targets := targetsForWebhook(t, env.db, webhook.ID)
|
||||
require.Len(t, targets, 1)
|
||||
assert.JSONEq(t,
|
||||
`{"expiry":"720h","rotation":"`+rotation+`"}`,
|
||||
targets[0].Config,
|
||||
)
|
||||
|
||||
assert.Equal(t,
|
||||
[]string{rotation}, rotationShown(t, env, webhook.ID))
|
||||
|
||||
page := targetEditPage(t, env, webhook.ID, targets[0].ID)
|
||||
assert.Equal(t, []string{rotation}, selectedIn(page, "rotation"))
|
||||
|
||||
form = url.Values{}
|
||||
form.Set("name", "archive")
|
||||
form.Set("expiry", "720h")
|
||||
form.Set("rotation", "hourly")
|
||||
|
||||
w = submitTargetEdit(env, webhook.ID, targets[0].ID, form)
|
||||
require.Equal(t, http.StatusSeeOther, w.Code, w.Body.String())
|
||||
assert.JSONEq(t,
|
||||
`{"expiry":"720h","rotation":"hourly"}`,
|
||||
storedTarget(t, env, targets[0].ID).Config,
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestArchiveRotationEditStartsOnNone checks the edit form of a
|
||||
// database target with no rotation stored starts on none.
|
||||
func TestArchiveRotationEditStartsOnNone(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupSourceTest(t)
|
||||
webhook := seedWebhookWithRetention(t, env.db, 30)
|
||||
target := seedConfiguredTarget(
|
||||
t, env.db, webhook.ID, database.TargetTypeDatabase, "",
|
||||
)
|
||||
|
||||
page := targetEditPage(t, env, webhook.ID, target.ID)
|
||||
assert.Equal(t, []string{rotationNone}, selectedIn(page, "rotation"))
|
||||
assert.Equal(t, []string{rotationNone}, rotationShown(t, env, webhook.ID))
|
||||
}
|
||||
|
||||
// TestArchiveRotationRefused proves a rotation that is not one of the
|
||||
// four is refused on the add target form and the target edit form, and
|
||||
// that nothing is stored.
|
||||
func TestArchiveRotationRefused(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupSourceTest(t)
|
||||
webhook := seedWebhookWithRetention(t, env.db, 30)
|
||||
|
||||
form := url.Values{}
|
||||
form.Set("name", "archive")
|
||||
form.Set("type", string(database.TargetTypeDatabase))
|
||||
form.Set("rotation", "weekly")
|
||||
|
||||
w := serveTarget(
|
||||
env, http.MethodPost, "/hook/"+webhook.ID+"/targets", form,
|
||||
)
|
||||
assert.Equal(t, http.StatusBadRequest, w.Code)
|
||||
assert.Contains(t, w.Body.String(), "Invalid archive rotation")
|
||||
assert.Empty(t, targetsForWebhook(t, env.db, webhook.ID))
|
||||
|
||||
target := seedConfiguredTarget(
|
||||
t, env.db, webhook.ID, database.TargetTypeDatabase,
|
||||
`{"rotation":"daily"}`,
|
||||
)
|
||||
|
||||
form.Del("type")
|
||||
|
||||
w = submitTargetEdit(env, webhook.ID, target.ID, form)
|
||||
assert.Equal(t, http.StatusBadRequest, w.Code)
|
||||
assert.Contains(t, w.Body.String(), "Invalid archive rotation")
|
||||
assert.JSONEq(t,
|
||||
`{"rotation":"daily"}`, storedTarget(t, env, target.ID).Config,
|
||||
)
|
||||
}
|
||||
|
||||
// TestHandleSourceCreateSubmit_ArchiveRotation proves the new webhook
|
||||
// page's archive rotation is stored on the archive target it creates.
|
||||
func TestHandleSourceCreateSubmit_ArchiveRotation(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupSourceTest(t)
|
||||
|
||||
form := url.Values{}
|
||||
form.Set("name", "rotated")
|
||||
form.Set("archive", "on")
|
||||
form.Set("archive_expiry", "720h")
|
||||
form.Set("archive_rotation", "daily")
|
||||
|
||||
w := submitCreateForm(env, form)
|
||||
require.Equal(t, http.StatusSeeOther, w.Code, w.Body.String())
|
||||
|
||||
var webhook database.Webhook
|
||||
|
||||
require.NoError(t, env.db.DB().
|
||||
Where("name = ?", "rotated").First(&webhook).Error)
|
||||
|
||||
targets := targetsForWebhook(t, env.db, webhook.ID)
|
||||
require.Len(t, targets, 1)
|
||||
assert.JSONEq(t,
|
||||
`{"expiry":"720h","rotation":"daily"}`, targets[0].Config,
|
||||
)
|
||||
}
|
||||
|
||||
// TestArchiveFileView_Rotated describes a daily target's archive files
|
||||
// at two times. On a day that has a file, the view names that file;
|
||||
// on the next, before any event, it names the file the next event
|
||||
// will go to, not created yet. Both times the size is of every file
|
||||
// together and the last write the latest of them.
|
||||
func TestArchiveFileView_Rotated(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupSourceTest(t)
|
||||
webhook := seedWebhookWithRetention(t, env.db, 30)
|
||||
target := seedConfiguredTarget(
|
||||
t, env.db, webhook.ID, database.TargetTypeDatabase,
|
||||
`{"rotation":"daily"}`,
|
||||
)
|
||||
|
||||
path := delivery.ArchivePath(env.dbMgr, &webhook, target)
|
||||
stem := strings.TrimSuffix(path, ".db")
|
||||
written := time.Date(2026, 10, 2, 9, 0, 0, 0, time.UTC)
|
||||
|
||||
for i, day := range []string{"2026-10-01", "2026-10-02"} {
|
||||
file := stem + "-" + day + ".db"
|
||||
require.NoError(t, os.WriteFile(file, make([]byte, 1000), 0o600))
|
||||
|
||||
at := written.Add(time.Duration(i-1) * 24 * time.Hour)
|
||||
require.NoError(t, os.Chtimes(file, at, at))
|
||||
}
|
||||
|
||||
view := env.handlers.ArchiveFileViewForTest(
|
||||
&webhook, target, time.Date(2026, 10, 2, 23, 0, 0, 0, time.UTC),
|
||||
)
|
||||
assert.Equal(t, filepath.Base(stem)+"-2026-10-02.db", view.Name)
|
||||
assert.Empty(t, view.Note)
|
||||
assert.Equal(t, 2, view.Files)
|
||||
assert.Equal(t, "2.0 kB", view.Size)
|
||||
assert.Equal(t, "2026-10-02 09:00:00 UTC", view.WrittenUTC)
|
||||
|
||||
view = env.handlers.ArchiveFileViewForTest(
|
||||
&webhook, target, time.Date(2026, 10, 3, 0, 0, 0, 0, time.UTC),
|
||||
)
|
||||
assert.Equal(t, filepath.Base(stem)+"-2026-10-03.db", view.Name)
|
||||
assert.Equal(t, "not created yet", view.Note)
|
||||
assert.Equal(t, 2, view.Files)
|
||||
assert.Equal(t, "2.0 kB", view.Size)
|
||||
|
||||
page := targetList(t, renderedPage(t, env, webhook.ID))
|
||||
assert.Contains(t, page, "Archive Size: 2.0 kB in 2 files")
|
||||
}
|
||||
|
||||
// renderedPage returns the webhook page.
|
||||
func renderedPage(t *testing.T, env *sourceTestEnv, webhookID string) string {
|
||||
t.Helper()
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
env.handlers.HandleSourceDetail().ServeHTTP(w, getRequest(
|
||||
t, "/hook/"+webhookID, env.cookies,
|
||||
map[string]string{sourceIDParam: webhookID},
|
||||
))
|
||||
require.Equal(t, http.StatusOK, w.Code)
|
||||
|
||||
return w.Body.String()
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
package handlers_test
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/gorm/clause"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
)
|
||||
|
||||
// TestDeliveryAttempts_ReadInTheTargetTypesOwnTerms proves, on the
|
||||
// event's page and in the event log, that an http or slack attempt
|
||||
// shows its status as before, while a database or log attempt, which
|
||||
// sends no HTTP request, says what it did and shows no status.
|
||||
func TestDeliveryAttempts_ReadInTheTargetTypesOwnTerms(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cases := []struct {
|
||||
targetType database.TargetType
|
||||
success bool
|
||||
statusCode int
|
||||
errText string
|
||||
outcome string
|
||||
status string // "" when the attempt must show no status
|
||||
}{
|
||||
{
|
||||
database.TargetTypeHTTP, false, 0, "",
|
||||
"failure", "Status: — (no response)",
|
||||
},
|
||||
{
|
||||
database.TargetTypeSlack, true, http.StatusOK, "",
|
||||
"success", "Status: 200",
|
||||
},
|
||||
{database.TargetTypeDatabase, true, 0, "", "archived", ""},
|
||||
{
|
||||
database.TargetTypeDatabase, false, 0,
|
||||
"opening archive database: disk full", "failure", "",
|
||||
},
|
||||
{database.TargetTypeLog, true, 0, "", "written to the log", ""},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(string(tc.targetType), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newRecentEventsFixture(t)
|
||||
target := seedTarget(t, f.db, f.webhook.ID, tc.targetType)
|
||||
event := f.event(t, contentTypeJSON, "{}", time.Now())
|
||||
dlv := f.delivery(
|
||||
t, event, target.ID, database.DeliveryStatusDelivered,
|
||||
)
|
||||
|
||||
require.NoError(t, f.webhookDB.Omit(clause.Associations).Create(
|
||||
&database.DeliveryResult{
|
||||
DeliveryID: dlv.ID,
|
||||
AttemptNum: 1,
|
||||
Success: tc.success,
|
||||
StatusCode: tc.statusCode,
|
||||
Error: tc.errText,
|
||||
},
|
||||
).Error)
|
||||
|
||||
w := serveEventPage(t, f.h, f.sess, f.webhook.ID, event.ID)
|
||||
require.Equal(t, http.StatusOK, w.Code)
|
||||
|
||||
pages := []string{
|
||||
w.Body.String(),
|
||||
renderSourceLogsPage(t, f.h, f.sess, f.webhook.ID),
|
||||
}
|
||||
|
||||
for _, page := range pages {
|
||||
assert.Contains(t, page, ">"+tc.outcome+"</span>")
|
||||
|
||||
if tc.errText != "" {
|
||||
assert.Contains(t, page, "Error: "+tc.errText)
|
||||
}
|
||||
|
||||
if tc.status == "" {
|
||||
assert.NotContains(t, page, "Status:")
|
||||
} else {
|
||||
assert.Contains(t, page, tc.status)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,6 @@ package handlers
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/go-chi/chi"
|
||||
"gorm.io/gorm"
|
||||
@@ -21,7 +20,9 @@ const (
|
||||
// replayTargetDeleted reports a target that once existed and has
|
||||
// since been deleted. Deletes are soft and deliveries carry no
|
||||
// foreign key to the target row, so the history survives its
|
||||
// target and this is the ordinary case for an old event.
|
||||
// target and this is the ordinary case for an old event. The
|
||||
// event log shows no Replay button for such a delivery, so only
|
||||
// a page loaded before the delete reaches this.
|
||||
replayTargetDeleted noticeCode = "replay-target-deleted"
|
||||
|
||||
// replayTargetMissing reports a target id that names no row at
|
||||
@@ -281,6 +282,7 @@ func createReplayDelivery(
|
||||
EventID: event.ID,
|
||||
TargetID: target.ID,
|
||||
Status: database.DeliveryStatusPending,
|
||||
Replay: true,
|
||||
}
|
||||
|
||||
err := webhookDB.Transaction(func(tx *gorm.DB) error {
|
||||
@@ -327,24 +329,15 @@ func replayBody(body string) *string {
|
||||
}
|
||||
|
||||
// redirectToEventLog redirects a replay or resubmit back to the event
|
||||
// log it was triggered from, carrying the outcome as its notice and
|
||||
// the page number the form submitted.
|
||||
// log it was triggered from, carrying the outcome as its notice.
|
||||
func redirectToEventLog(
|
||||
w http.ResponseWriter,
|
||||
r *http.Request,
|
||||
webhook database.Webhook,
|
||||
code noticeCode,
|
||||
) {
|
||||
dest := withNotice("/hook/"+webhook.ID+"/events", code)
|
||||
|
||||
// The page is read from the form rather than the query string:
|
||||
// this is a POST, and its query string is what logs and Referer
|
||||
// headers record.
|
||||
if page := pageOrFirst(
|
||||
r.PostFormValue("page"),
|
||||
); page > 1 {
|
||||
dest += "&page=" + strconv.Itoa(page)
|
||||
}
|
||||
|
||||
http.Redirect(w, r, dest, http.StatusSeeOther)
|
||||
http.Redirect(
|
||||
w, r, withNotice("/hook/"+webhook.ID+"/events", code),
|
||||
http.StatusSeeOther,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package handlers_test
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -513,7 +514,11 @@ func TestHandleSourceLogs_RendersReplayControlAndBanner(t *testing.T) {
|
||||
)
|
||||
|
||||
assert.Contains(t, refused, "alert-error")
|
||||
assert.Contains(t, refused, "has been deleted")
|
||||
assert.Contains(
|
||||
t, refused,
|
||||
"has been deleted. Use Resubmit to send the event "+
|
||||
"to the webhook",
|
||||
)
|
||||
|
||||
// An outcome code nobody issued renders no banner at all.
|
||||
unknown := renderSourceLogsPageWithQuery(
|
||||
@@ -524,3 +529,58 @@ func TestHandleSourceLogs_RendersReplayControlAndBanner(t *testing.T) {
|
||||
assert.NotContains(t, unknown, "alert-success")
|
||||
assert.NotContains(t, unknown, "made-up")
|
||||
}
|
||||
|
||||
// TestHandleDeliveryReplay_LabelsTheReplay proves a delivery created
|
||||
// by Replay is labelled as a replay in the event's summary line in the
|
||||
// event log, and in the list of the event's deliveries there and on
|
||||
// the event's page, while the delivery it repeats is not.
|
||||
func TestHandleDeliveryReplay_LabelsTheReplay(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var (
|
||||
h *handlers.Handlers
|
||||
sess *session.Session
|
||||
db *database.Database
|
||||
dbMgr *database.WebhookDBManager
|
||||
)
|
||||
|
||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
||||
app.RequireStart()
|
||||
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
wh := seedWebhook(t, db)
|
||||
tgt := seedConfiguredTarget(
|
||||
t, db, wh.ID, database.TargetTypeHTTP,
|
||||
`{"url":"`+replayTargetURL+`"}`,
|
||||
)
|
||||
|
||||
event, original := seedFailedDelivery(t, dbMgr, wh.ID, tgt.ID)
|
||||
|
||||
w := postReplay(t, h, sess, wh.ID, original.ID)
|
||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||
|
||||
eventLog := renderSourceLogsPage(t, h, sess, wh.ID)
|
||||
|
||||
assert.Contains(t, eventLog, tgt.Name+": failed")
|
||||
assert.Contains(t, eventLog, tgt.Name+" (replay): pending")
|
||||
|
||||
w = serveEventPage(t, h, sess, wh.ID, event.ID)
|
||||
require.Equal(t, http.StatusOK, w.Code)
|
||||
|
||||
// In each delivery list a row names the target, then the label if
|
||||
// it is a replay, then its status: the replay is still pending, the
|
||||
// original failed.
|
||||
replayRow := tgt.Name + `</span> ` +
|
||||
`<span class="text-xs text-gray-500">replay</span> ` +
|
||||
`<span class="text-xs text-gray-400">pending</span>`
|
||||
originalRow := tgt.Name + `</span> ` +
|
||||
`<span class="text-xs text-red-600">failed</span>`
|
||||
|
||||
for _, page := range []string{eventLog, w.Body.String()} {
|
||||
page = strings.Join(strings.Fields(page), " ")
|
||||
|
||||
assert.Contains(t, page, replayRow)
|
||||
assert.Contains(t, page, originalRow)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"sneak.berlin/go/webhooker/internal/delivery"
|
||||
)
|
||||
|
||||
@@ -24,8 +27,8 @@ const maxRenderedResponseBytes = 4096
|
||||
// bytes rather than characters, and they make SQLite do the
|
||||
// cut, so an oversized stored response never becomes a Go
|
||||
// string at all.
|
||||
const deliveryResultColumns = "delivery_id, attempt_num, success, " +
|
||||
"status_code, error, duration, " +
|
||||
const deliveryResultColumns = "delivery_id, attempt_num, created_at, " +
|
||||
"success, status_code, error, duration, " +
|
||||
"substr(cast(response_body as blob), 1, ?) AS response_body, " +
|
||||
"length(cast(response_body as blob)) AS response_bytes"
|
||||
|
||||
@@ -45,6 +48,11 @@ type DeliveryResultView struct {
|
||||
AttemptNum int
|
||||
Success bool
|
||||
|
||||
// Ran is how long ago the attempt was recorded, and RanUTC the
|
||||
// full timestamp the page shows on hover.
|
||||
Ran string
|
||||
RanUTC string
|
||||
|
||||
// StatusCode is 0 when the attempt never got a response,
|
||||
// which is why the page asks HasStatusCode rather than
|
||||
// printing the number.
|
||||
@@ -100,6 +108,7 @@ func (v DeliveryResultView) HasStatusCode() bool {
|
||||
type deliveryResultRow struct {
|
||||
DeliveryID string
|
||||
AttemptNum int
|
||||
CreatedAt time.Time
|
||||
Success bool
|
||||
StatusCode int
|
||||
Error string
|
||||
@@ -157,6 +166,8 @@ func (r *deliveryResultRow) view(
|
||||
return DeliveryResultView{
|
||||
AttemptNum: r.AttemptNum,
|
||||
Success: r.Success,
|
||||
Ran: humanize.Time(r.CreatedAt),
|
||||
RanUTC: r.CreatedAt.UTC().Format(time.DateTime) + " UTC",
|
||||
StatusCode: r.StatusCode,
|
||||
Error: redactor.Redact(r.Error),
|
||||
DurationMS: r.Duration,
|
||||
|
||||
@@ -435,9 +435,7 @@ func TestHandleSourceLogs_BoundsRenderedAttempts(t *testing.T) {
|
||||
}).Error)
|
||||
}
|
||||
|
||||
views := h.LoadEventLogViewsForTest(
|
||||
httptest.NewRecorder(), *wh, 1,
|
||||
)
|
||||
views := h.LoadEventLogViewsForTest(httptest.NewRecorder(), *wh)
|
||||
require.Len(t, views, 1)
|
||||
require.Len(t, views[0].Deliveries, 1)
|
||||
|
||||
@@ -489,9 +487,7 @@ func TestHandleSourceLogs_BoundsOversizeResponse(t *testing.T) {
|
||||
stored := strings.Repeat("A", responseCap*4) + tail
|
||||
seedFailedDeliveryWithResponse(t, dbMgr, wh.ID, tgt.ID, stored)
|
||||
|
||||
views := h.LoadEventLogViewsForTest(
|
||||
httptest.NewRecorder(), *wh, 1,
|
||||
)
|
||||
views := h.LoadEventLogViewsForTest(httptest.NewRecorder(), *wh)
|
||||
require.Len(t, views, 1)
|
||||
require.Len(t, views[0].Deliveries, 1)
|
||||
require.Len(t, views[0].Deliveries[0].Results, 1)
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
package handlers_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/go-chi/chi"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/gorm"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
)
|
||||
|
||||
// TestHandleEntrypointToggle_DoesNotUndoAnEdit proves that a toggle
|
||||
// which loaded the entrypoint before an edit of its description was
|
||||
// saved does not write the old description back over the edit. The
|
||||
// edit is submitted from a callback on the toggle's own read of the
|
||||
// entrypoint, so it is saved after that read and before the toggle
|
||||
// writes.
|
||||
func TestHandleEntrypointToggle_DoesNotUndoAnEdit(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupSourceTest(t)
|
||||
wh := seedWebhookWithRetention(t, env.db, 30)
|
||||
ep := seedEntrypoint(t, env.db, wh.ID)
|
||||
require.True(t, ep.Active)
|
||||
|
||||
router := chi.NewRouter()
|
||||
router.Post(
|
||||
"/hook/{sourceID}/entrypoints/{entrypointID}/edit",
|
||||
env.handlers.HandleEntrypointEdit(),
|
||||
)
|
||||
router.Post(
|
||||
"/hook/{sourceID}/entrypoints/{entrypointID}/toggle",
|
||||
env.handlers.HandleEntrypointToggle(),
|
||||
)
|
||||
|
||||
// post submits one of the entrypoint's forms as the test user and
|
||||
// returns the response's status code.
|
||||
post := func(action string, form url.Values) int {
|
||||
req := httptest.NewRequestWithContext(
|
||||
context.Background(), http.MethodPost,
|
||||
"/hook/"+wh.ID+"/entrypoints/"+ep.ID+"/"+action,
|
||||
strings.NewReader(form.Encode()),
|
||||
)
|
||||
req.Header.Set(
|
||||
"Content-Type", "application/x-www-form-urlencoded",
|
||||
)
|
||||
|
||||
for _, c := range env.cookies {
|
||||
req.AddCookie(c)
|
||||
}
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
return w.Code
|
||||
}
|
||||
|
||||
var (
|
||||
edited bool
|
||||
editCode int
|
||||
)
|
||||
|
||||
require.NoError(t, env.db.DB().Callback().Query().
|
||||
After("gorm:query").
|
||||
Register("test:edit_after_toggle_read", func(tx *gorm.DB) {
|
||||
// Only the first read of an entrypoint, the toggle's,
|
||||
// submits the edit.
|
||||
if tx.Statement.Table != "entrypoints" || edited {
|
||||
return
|
||||
}
|
||||
|
||||
edited = true
|
||||
editCode = post(
|
||||
"edit", url.Values{"description": {"Billing sender"}},
|
||||
)
|
||||
}),
|
||||
)
|
||||
|
||||
require.Equal(t, http.StatusSeeOther, post("toggle", nil))
|
||||
require.Equal(t, http.StatusSeeOther, editCode)
|
||||
|
||||
var stored database.Entrypoint
|
||||
|
||||
require.NoError(
|
||||
t, env.db.DB().First(&stored, "id = ?", ep.ID).Error,
|
||||
)
|
||||
assert.False(t, stored.Active)
|
||||
assert.Equal(t, "Billing sender", stored.Description)
|
||||
}
|
||||
@@ -1,6 +1,11 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"gorm.io/gorm"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
)
|
||||
|
||||
@@ -11,6 +16,21 @@ type EntrypointView struct {
|
||||
Path string
|
||||
Description string
|
||||
Active bool
|
||||
|
||||
// Events is how many events arrived on the entrypoint's URL within
|
||||
// the webhook's retention period. LastEvent is when the newest
|
||||
// event ever to arrive on it did, relative, and LastEventUTC the
|
||||
// full time; both are empty when none ever did.
|
||||
Events int64
|
||||
LastEvent string
|
||||
LastEventUTC string
|
||||
}
|
||||
|
||||
// entrypointEvents is one entrypoint's count read by
|
||||
// addEntrypointEvents.
|
||||
type entrypointEvents struct {
|
||||
EntrypointID string
|
||||
Events int64
|
||||
}
|
||||
|
||||
// NewEntrypointViews projects entrypoints for rendering.
|
||||
@@ -32,3 +52,60 @@ func NewEntrypointViews(
|
||||
|
||||
return views
|
||||
}
|
||||
|
||||
// addEntrypointEvents fills in each view's event figures from the
|
||||
// webhook's event database: when the last event arrived on its URL,
|
||||
// from its EntrypointTotals row, and how many events arrived on it
|
||||
// since the webhook's retention cutoff, counted in one query over the
|
||||
// events' entrypoint_id index. Resubmitted copies did not arrive on
|
||||
// the URL and are left out of both.
|
||||
func addEntrypointEvents(
|
||||
webhookDB *gorm.DB,
|
||||
webhook *database.Webhook,
|
||||
views []EntrypointView,
|
||||
now time.Time,
|
||||
) error {
|
||||
ids := make([]string, len(views))
|
||||
byID := make(map[string]*EntrypointView, len(views))
|
||||
|
||||
for i := range views {
|
||||
ids[i] = views[i].ID
|
||||
byID[views[i].ID] = &views[i]
|
||||
}
|
||||
|
||||
var totals []database.EntrypointTotals
|
||||
|
||||
err := webhookDB.Where("entrypoint_id IN ?", ids).Find(&totals).Error
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading entrypoint totals: %w", err)
|
||||
}
|
||||
|
||||
query := webhookDB.Model(&database.Event{}).
|
||||
Select("entrypoint_id, count(*) AS events").
|
||||
Where("entrypoint_id IN ? AND resubmitted_from_id IS NULL", ids)
|
||||
|
||||
cutoff, finite := webhook.RetentionCutoff(now)
|
||||
if finite {
|
||||
query = query.Where("created_at >= ?", cutoff)
|
||||
}
|
||||
|
||||
var counts []entrypointEvents
|
||||
|
||||
err = query.Group("entrypoint_id").Find(&counts).Error
|
||||
if err != nil {
|
||||
return fmt.Errorf("counting events by entrypoint: %w", err)
|
||||
}
|
||||
|
||||
for _, row := range totals {
|
||||
view := byID[row.EntrypointID]
|
||||
view.LastEvent = humanize.Time(row.LastEventAt)
|
||||
view.LastEventUTC =
|
||||
row.LastEventAt.UTC().Format(time.DateTime) + " UTC"
|
||||
}
|
||||
|
||||
for _, row := range counts {
|
||||
byID[row.EntrypointID].Events = row.Events
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,197 @@
|
||||
package handlers_test
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/gorm/clause"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
"sneak.berlin/go/webhooker/internal/handlers"
|
||||
"sneak.berlin/go/webhooker/internal/logger"
|
||||
"sneak.berlin/go/webhooker/internal/session"
|
||||
)
|
||||
|
||||
// entrypointRow returns the part of a rendered webhook page from an
|
||||
// entrypoint's URL to the next entrypoint's, which holds its figures.
|
||||
func entrypointRow(t *testing.T, page, entrypointID string) string {
|
||||
t.Helper()
|
||||
|
||||
_, row, found := strings.Cut(page, `id="entrypoint-url-`+entrypointID+`"`)
|
||||
require.True(t, found)
|
||||
|
||||
row, _, _ = strings.Cut(row, `id="entrypoint-url-`)
|
||||
|
||||
return row
|
||||
}
|
||||
|
||||
// lastEventShown matches an entrypoint row's last event arriving at at.
|
||||
func lastEventShown(at time.Time) string {
|
||||
return `Last Event:</span>\s*<span title="` +
|
||||
at.UTC().Format(time.DateTime) + ` UTC">[^<]+</span>`
|
||||
}
|
||||
|
||||
// eventsShown matches an entrypoint row's count of n events.
|
||||
func eventsShown(n int) string {
|
||||
return `Events Within Retention:</span>\s*<span>` +
|
||||
strconv.Itoa(n) + `</span>`
|
||||
}
|
||||
|
||||
// TestHandleSourceDetail_ShowsEntrypointEvents proves each entrypoint
|
||||
// on the webhook page shows its own figures: how many events arrived
|
||||
// through it within the webhook's retention period, leaving out one
|
||||
// older than that, and when the newest arrived, or "never" for an
|
||||
// entrypoint with none.
|
||||
func TestHandleSourceDetail_ShowsEntrypointEvents(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var (
|
||||
h *handlers.Handlers
|
||||
sess *session.Session
|
||||
db *database.Database
|
||||
dbMgr *database.WebhookDBManager
|
||||
)
|
||||
|
||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
||||
app.RequireStart()
|
||||
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
wh := &database.Webhook{
|
||||
UserID: deleteTestUserID, Name: "figures", RetentionDays: 7,
|
||||
}
|
||||
require.NoError(t, db.DB().Omit(clause.Associations).Create(wh).Error)
|
||||
|
||||
webhookDB, err := dbMgr.GetDB(wh.ID)
|
||||
require.NoError(t, err)
|
||||
|
||||
entrypoint := func() *database.Entrypoint {
|
||||
ep := &database.Entrypoint{
|
||||
WebhookID: wh.ID, Path: uuid.New().String(), Active: true,
|
||||
}
|
||||
require.NoError(t,
|
||||
db.DB().Omit(clause.Associations).Create(ep).Error)
|
||||
|
||||
return ep
|
||||
}
|
||||
|
||||
// event stores an event that arrived on ep's URL age ago and
|
||||
// records it as ep's last event, as the receiver does.
|
||||
event := func(ep *database.Entrypoint, age time.Duration) time.Time {
|
||||
e := &database.Event{
|
||||
WebhookID: wh.ID,
|
||||
EntrypointID: ep.ID,
|
||||
Method: http.MethodPost,
|
||||
}
|
||||
e.CreatedAt = time.Now().Add(-age)
|
||||
require.NoError(t,
|
||||
webhookDB.Omit(clause.Associations).Create(e).Error)
|
||||
require.NoError(t, database.AddEntrypointTotals(webhookDB,
|
||||
database.EntrypointTotals{
|
||||
EntrypointID: ep.ID, LastEventAt: e.CreatedAt,
|
||||
}))
|
||||
|
||||
return e.CreatedAt
|
||||
}
|
||||
|
||||
busy, quiet, unused := entrypoint(), entrypoint(), entrypoint()
|
||||
|
||||
event(busy, 8*24*time.Hour) // older than the 7 days kept
|
||||
event(busy, 3*time.Hour)
|
||||
busyLast := event(busy, time.Hour)
|
||||
quietLast := event(quiet, 2*24*time.Hour)
|
||||
|
||||
body := renderSourceDetailPage(t, h, sess, wh.ID)
|
||||
|
||||
assert.Regexp(t, lastEventShown(busyLast), entrypointRow(t, body, busy.ID))
|
||||
assert.Regexp(t, eventsShown(2), entrypointRow(t, body, busy.ID))
|
||||
assert.Regexp(t, lastEventShown(quietLast), entrypointRow(t, body, quiet.ID))
|
||||
assert.Regexp(t, eventsShown(1), entrypointRow(t, body, quiet.ID))
|
||||
assert.Regexp(t, `Last Event:</span>\s*<span>never</span>`,
|
||||
entrypointRow(t, body, unused.ID))
|
||||
assert.Regexp(t, eventsShown(0), entrypointRow(t, body, unused.ID))
|
||||
}
|
||||
|
||||
// TestHandleSourceDetail_EntrypointLastEventSurvivesRetention checks
|
||||
// that once retention has removed every event that arrived on an
|
||||
// entrypoint's URL, the entrypoint still shows when the last one
|
||||
// arrived rather than "never".
|
||||
func TestHandleSourceDetail_EntrypointLastEventSurvivesRetention(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
var (
|
||||
h *handlers.Handlers
|
||||
sess *session.Session
|
||||
db *database.Database
|
||||
dbMgr *database.WebhookDBManager
|
||||
log *logger.Logger
|
||||
)
|
||||
|
||||
app := newTestApp(t, &h, &sess, &db, &dbMgr, &log)
|
||||
app.RequireStart()
|
||||
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
wh := &database.Webhook{
|
||||
UserID: deleteTestUserID, Name: "swept", RetentionDays: 1,
|
||||
}
|
||||
require.NoError(t, db.DB().Omit(clause.Associations).Create(wh).Error)
|
||||
|
||||
ep := seedEntrypoint(t, db, wh.ID)
|
||||
webhookDB, events := receiveEvents(t, h, dbMgr, wh.ID, ep.Path, 1)
|
||||
arrived := events[0].CreatedAt
|
||||
|
||||
statsAge(t, webhookDB, events[0].ID, time.Now().Add(-50*time.Hour))
|
||||
statsPrune(t, db, dbMgr, log, webhookDB)
|
||||
require.Empty(t, listEvents(t, webhookDB))
|
||||
|
||||
row := entrypointRow(t, renderSourceDetailPage(t, h, sess, wh.ID), ep.ID)
|
||||
assert.Regexp(t, lastEventShown(arrived), row)
|
||||
assert.Regexp(t, eventsShown(0), row)
|
||||
}
|
||||
|
||||
// TestHandleSourceDetail_ResubmitLeavesEntrypointFigures checks that a
|
||||
// resubmitted copy, which did not arrive on the entrypoint's URL,
|
||||
// changes neither the entrypoint's last event nor its count.
|
||||
func TestHandleSourceDetail_ResubmitLeavesEntrypointFigures(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
var (
|
||||
h *handlers.Handlers
|
||||
sess *session.Session
|
||||
db *database.Database
|
||||
dbMgr *database.WebhookDBManager
|
||||
)
|
||||
|
||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
||||
app.RequireStart()
|
||||
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
wh := seedWebhook(t, db)
|
||||
ep := seedEntrypoint(t, db, wh.ID)
|
||||
webhookDB, events := receiveEvents(t, h, dbMgr, wh.ID, ep.Path, 1)
|
||||
arrived := events[0].CreatedAt
|
||||
|
||||
require.Equal(t, http.StatusSeeOther,
|
||||
postResubmit(t, h, sess, wh.ID, events[0].ID).Code)
|
||||
require.Len(t, listEvents(t, webhookDB), 2)
|
||||
|
||||
var totals database.EntrypointTotals
|
||||
|
||||
require.NoError(t, webhookDB.Take(&totals).Error)
|
||||
assert.True(t, arrived.Equal(totals.LastEventAt))
|
||||
|
||||
row := entrypointRow(t, renderSourceDetailPage(t, h, sess, wh.ID), ep.ID)
|
||||
assert.Regexp(t, lastEventShown(arrived), row)
|
||||
assert.Regexp(t, eventsShown(1), row)
|
||||
}
|
||||
@@ -15,16 +15,19 @@ import (
|
||||
// eventBodyQuery reads one event's stored body as bytes. The cast
|
||||
// to blob is what makes the driver hand back the stored bytes
|
||||
// rather than a string conversion, so Content-Length taken from
|
||||
// the result matches what goes on the wire. The soft-delete
|
||||
// predicate is spelled out because Raw bypasses GORM's default
|
||||
// scope, and it is what stops a reaped event still being
|
||||
// downloadable.
|
||||
// the result matches what goes on the wire. The retention reaper
|
||||
// deletes event rows outright, so a reaped event is simply gone
|
||||
// and the query finds no row. The deleted_at predicate repeats
|
||||
// the soft-delete scope GORM adds to its own queries, which Raw
|
||||
// bypasses; nothing soft-deletes an event, so today it excludes
|
||||
// nothing.
|
||||
const eventBodyQuery = "SELECT cast(body as blob) " +
|
||||
"FROM events WHERE id = ? AND webhook_id = ? AND deleted_at IS NULL"
|
||||
|
||||
// HandleEventBodyDownload serves one event's stored body in
|
||||
// full, which the event log page cannot: it caps each rendered
|
||||
// body at maxRenderedBodyBytes.
|
||||
// HandleEventBodyDownload serves one event's stored body byte
|
||||
// for byte, which the pages do not: they show it as escaped
|
||||
// text, cut at maxRenderedBodyBytes in the lists of events, and
|
||||
// leave a binary one out.
|
||||
//
|
||||
// The bytes are attacker-supplied — anyone who can reach the
|
||||
// public receiver chooses them — and this route hands them back
|
||||
|
||||
@@ -405,10 +405,11 @@ func TestHandleEventBodyDownload_UnknownEvent404s(t *testing.T) {
|
||||
// route. The body is read in one query before any header is
|
||||
// written, so a reaped event cannot produce a partial download:
|
||||
// it is a clean 404 with no Content-Length and no
|
||||
// Content-Disposition. Both removals the codebase performs are
|
||||
// covered — the reaper hard-deletes, and a soft-deleted row is
|
||||
// excluded by the query's own deleted_at predicate rather than
|
||||
// by GORM's default scope, which Raw bypasses.
|
||||
// Content-Disposition. The reaper deletes event rows outright,
|
||||
// which is the "hard deleted" case. The "soft deleted" case
|
||||
// covers a row no code produces today: it only pins the query's
|
||||
// own deleted_at predicate, the soft-delete condition Raw would
|
||||
// otherwise skip.
|
||||
func TestHandleEventBodyDownload_ReapedEvent404s(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// maxRenderedBodyBytes is the most of one event's body that the
|
||||
// recent events on a webhook's page and the event log show; a larger
|
||||
// body is cut there and shown whole only on the event's own page.
|
||||
// Bodies come from the unauthenticated receiver under its 1 MB cap,
|
||||
// and renderTemplate buffers a whole page before writing it, so a list
|
||||
// of events cannot show every body whole.
|
||||
const maxRenderedBodyBytes = 32 << 10
|
||||
|
||||
// maxInlineBodyLines is the most lines a body is shown at its full
|
||||
// height with. A body with more lines, or larger than
|
||||
// maxRenderedBodyBytes, is shown in a box of fixed height that
|
||||
// scrolls, so that it does not make the page huge.
|
||||
const maxInlineBodyLines = 200
|
||||
|
||||
// maxIndentDepth is how deeply a JSON body's objects and arrays may
|
||||
// nest for it to be indented at all; a deeper one is shown as received.
|
||||
// Each level indents every line inside it two more spaces, so 10 KB of
|
||||
// nested brackets would indent to some 50 MB; within this depth a body
|
||||
// grows at most 35 times.
|
||||
const maxIndentDepth = 16
|
||||
|
||||
// A JSON body is shown pretty-printed only when that makes it at most
|
||||
// maxIndentGrowth times its size plus indentAllowance bytes, and
|
||||
// otherwise as received, so that indenting does not undo
|
||||
// maxRenderedBodyBytes. The allowance keeps a small nested body
|
||||
// pretty-printed.
|
||||
const (
|
||||
maxIndentGrowth = 4
|
||||
indentAllowance = 1 << 10
|
||||
)
|
||||
|
||||
// jsonIndent is the indent of a pretty-printed JSON body.
|
||||
const jsonIndent = " "
|
||||
|
||||
// BodyView is an event's body as the pages show it. newBodyView
|
||||
// decides it and templates/event_body.html shows it, the same way in
|
||||
// the recent events on a webhook's page, in the event log and on the
|
||||
// event's own page.
|
||||
type BodyView struct {
|
||||
// EventURL is the event's own page. The stored body downloads
|
||||
// from EventURL/body.
|
||||
EventURL string
|
||||
|
||||
// Text is the body as shown, pretty-printed when it is JSON.
|
||||
Text string
|
||||
|
||||
// Size is the stored body's size in bytes, and ShownBytes how
|
||||
// many of them Text holds when Cut.
|
||||
Size int64
|
||||
ShownBytes int
|
||||
|
||||
// Cut reports that Text is only the start of the body.
|
||||
Cut bool
|
||||
|
||||
// Binary reports a body that is not text. It is not shown.
|
||||
Binary bool
|
||||
|
||||
// Scroll reports a body to show in a box that scrolls.
|
||||
Scroll bool
|
||||
}
|
||||
|
||||
// newBodyView decides how to show an event's body. body is the
|
||||
// stored body, or its first maxRenderedBodyBytes when only those were
|
||||
// read, and size is the stored body's size.
|
||||
func newBodyView(eventURL string, body []byte, size int64) BodyView {
|
||||
v := BodyView{EventURL: eventURL, Size: size}
|
||||
|
||||
if size > int64(len(body)) {
|
||||
v.Cut = true
|
||||
body = trimPartialRune(body)
|
||||
v.ShownBytes = len(body)
|
||||
}
|
||||
|
||||
// html/template shows invalid UTF-8 as replacement characters,
|
||||
// and a browser shows a control character other than tab, line
|
||||
// feed and carriage return as a box or not at all, so a body
|
||||
// holding either is not text.
|
||||
isControl := func(r rune) bool {
|
||||
return unicode.IsControl(r) && r != '\t' && r != '\n' && r != '\r'
|
||||
}
|
||||
|
||||
if !utf8.Valid(body) || bytes.IndexFunc(body, isControl) >= 0 {
|
||||
v.Binary = true
|
||||
|
||||
return v
|
||||
}
|
||||
|
||||
// A cut JSON document is no longer valid JSON.
|
||||
if !v.Cut {
|
||||
body = indentJSON(body)
|
||||
}
|
||||
|
||||
// The page shows a carriage return, a line feed, or the two
|
||||
// together as one line break. A final one ends the last line
|
||||
// rather than starting another.
|
||||
text := bytes.TrimSuffix(body, []byte("\n"))
|
||||
text = bytes.TrimSuffix(text, []byte("\r"))
|
||||
breaks := bytes.Count(text, []byte("\n")) + bytes.Count(text, []byte("\r")) -
|
||||
bytes.Count(text, []byte("\r\n"))
|
||||
lines := breaks + 1
|
||||
|
||||
v.Text = string(body)
|
||||
v.Scroll = lines > maxInlineBodyLines || size > maxRenderedBodyBytes
|
||||
|
||||
return v
|
||||
}
|
||||
|
||||
// indentJSON returns body pretty-printed when it is a JSON document,
|
||||
// and unchanged when it is not, nests deeper than maxIndentDepth, or
|
||||
// would grow past maxIndentGrowth times its size plus indentAllowance
|
||||
// bytes.
|
||||
func indentJSON(body []byte) []byte {
|
||||
if !json.Valid(body) || !indentFits(body) {
|
||||
return body
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
|
||||
err := json.Indent(&out, body, "", jsonIndent)
|
||||
if err != nil || out.Len() > maxIndentGrowth*len(body)+indentAllowance {
|
||||
return body
|
||||
}
|
||||
|
||||
return out.Bytes()
|
||||
}
|
||||
|
||||
// indentFits reports whether the objects and arrays of the JSON
|
||||
// document body nest at most maxIndentDepth deep.
|
||||
func indentFits(body []byte) bool {
|
||||
depth := 0
|
||||
|
||||
dec := json.NewDecoder(bytes.NewReader(body))
|
||||
|
||||
// A number too large for a float64 is still valid JSON.
|
||||
dec.UseNumber()
|
||||
|
||||
for {
|
||||
tok, err := dec.Token()
|
||||
if errors.Is(err, io.EOF) {
|
||||
return true
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
switch tok {
|
||||
case json.Delim('{'), json.Delim('['):
|
||||
depth++
|
||||
if depth > maxIndentDepth {
|
||||
return false
|
||||
}
|
||||
case json.Delim('}'), json.Delim(']'):
|
||||
depth--
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
package handlers_test
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"sneak.berlin/go/webhooker/internal/handlers"
|
||||
)
|
||||
|
||||
// bodyView is how the pages would show body, stored whole.
|
||||
func bodyView(body string) handlers.BodyView {
|
||||
return handlers.NewBodyViewForTest([]byte(body), int64(len(body)))
|
||||
}
|
||||
|
||||
// lines is n lines of text, without a newline after the last.
|
||||
func lines(n int) string {
|
||||
return strings.TrimSuffix(strings.Repeat("line\n", n), "\n")
|
||||
}
|
||||
|
||||
// TestNewBodyView_FormatsValidJSON proves a JSON body is shown
|
||||
// pretty-printed, whatever its content type, with its keys in
|
||||
// the order they arrived.
|
||||
func TestNewBodyView_FormatsValidJSON(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
v := bodyView(`{"b":1,"a":[true,null,"x"],"c":{}}`)
|
||||
|
||||
assert.Equal(t, []string{
|
||||
`{`,
|
||||
` "b": 1,`,
|
||||
` "a": [`,
|
||||
` true,`,
|
||||
` null,`,
|
||||
` "x"`,
|
||||
` ],`,
|
||||
` "c": {}`,
|
||||
`}`,
|
||||
}, strings.Split(v.Text, "\n"))
|
||||
assert.False(t, v.Scroll)
|
||||
}
|
||||
|
||||
// TestNewBodyView_FormatsNestedJSON proves a small document with a
|
||||
// few levels of nesting is pretty-printed.
|
||||
func TestNewBodyView_FormatsNestedJSON(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
v := bodyView(`{"data":[[1,2,3],[4,5,6]]}`)
|
||||
|
||||
assert.Equal(t, []string{
|
||||
`{`,
|
||||
` "data": [`,
|
||||
` [`,
|
||||
` 1,`,
|
||||
` 2,`,
|
||||
` 3`,
|
||||
` ],`,
|
||||
` [`,
|
||||
` 4,`,
|
||||
` 5,`,
|
||||
` 6`,
|
||||
` ]`,
|
||||
` ]`,
|
||||
`}`,
|
||||
}, strings.Split(v.Text, "\n"))
|
||||
}
|
||||
|
||||
// TestNewBodyView_InvalidJSONAsReceived proves a body that is not
|
||||
// a JSON document is shown exactly as it arrived.
|
||||
func TestNewBodyView_InvalidJSONAsReceived(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, body := range []string{
|
||||
`{"a":1,`,
|
||||
`{"a":1} {"b":2}`,
|
||||
"plain text\n indented",
|
||||
} {
|
||||
assert.Equal(t, body, bodyView(body).Text)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewBodyView_DeepJSONAsReceived proves a JSON body nested
|
||||
// more than 16 levels deep is shown as it arrived. 10 KB of nested
|
||||
// arrays would indent to some 50 MB.
|
||||
func TestNewBodyView_DeepJSONAsReceived(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
nested := func(depth int) string {
|
||||
return strings.Repeat("[", depth) + "1" + strings.Repeat("]", depth)
|
||||
}
|
||||
|
||||
assert.NotEqual(t, nested(16), bodyView(nested(16)).Text)
|
||||
assert.Equal(t, nested(17), bodyView(nested(17)).Text)
|
||||
|
||||
body := strings.Repeat("[", 5000) + strings.Repeat("]", 5000)
|
||||
|
||||
assert.Equal(t, body, bodyView(body).Text)
|
||||
}
|
||||
|
||||
// TestNewBodyView_GrowingJSONAsReceived proves a JSON body that
|
||||
// pretty-printing would make more than four times its size plus 1 KiB
|
||||
// is shown as it arrived, however shallow: each short element eight
|
||||
// levels deep gets a line indented sixteen spaces.
|
||||
func TestNewBodyView_GrowingJSONAsReceived(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
numbers := func(n int) string {
|
||||
return strings.Repeat("[", 8) +
|
||||
strings.TrimSuffix(strings.Repeat("1,", n), ",") +
|
||||
strings.Repeat("]", 8)
|
||||
}
|
||||
|
||||
assert.NotEqual(t, numbers(10), bodyView(numbers(10)).Text)
|
||||
assert.Equal(t, numbers(1000), bodyView(numbers(1000)).Text)
|
||||
}
|
||||
|
||||
// TestNewBodyView_ScrollsPast200Lines proves a body is shown at
|
||||
// its full height up to 200 lines and in the scrolling box past
|
||||
// them, counting the lines after formatting.
|
||||
func TestNewBodyView_ScrollsPast200Lines(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
assert.False(t, bodyView(lines(200)).Scroll)
|
||||
assert.True(t, bodyView(lines(201)).Scroll)
|
||||
|
||||
// A final newline ends the last line rather than starting another.
|
||||
assert.False(t, bodyView(lines(200)+"\n").Scroll)
|
||||
assert.True(t, bodyView(lines(201)+"\n").Scroll)
|
||||
|
||||
// The page shows a carriage return, a line feed, or the two
|
||||
// together as one line break.
|
||||
assert.True(t, bodyView(strings.Repeat("line\r", 400)).Scroll)
|
||||
assert.False(t, bodyView(strings.Repeat("line\r\n", 200)).Scroll)
|
||||
|
||||
// One line as received, 201 once formatted: the brackets and
|
||||
// 199 elements.
|
||||
numbers := "[" + strings.TrimSuffix(strings.Repeat("1,", 199), ",") + "]"
|
||||
|
||||
assert.NotContains(t, numbers, "\n")
|
||||
assert.True(t, bodyView(numbers).Scroll)
|
||||
}
|
||||
|
||||
// TestNewBodyView_LargeBodyScrolls proves a body larger than the
|
||||
// cap of the lists of events is shown in the scrolling box
|
||||
// however few lines it has, on the event's own page as in the
|
||||
// lists.
|
||||
func TestNewBodyView_LargeBodyScrolls(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
assert.False(t, bodyView(strings.Repeat("x", bodyCap)).Scroll)
|
||||
assert.True(t, bodyView(strings.Repeat("x", bodyCap+1)).Scroll)
|
||||
}
|
||||
|
||||
// TestNewBodyView_BinaryNotShown proves a body that is not text
|
||||
// is never shown: one that is not valid UTF-8, or that holds a
|
||||
// control character other than tab, line feed and carriage return.
|
||||
func TestNewBodyView_BinaryNotShown(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, body := range []string{
|
||||
"\xff\xfe\xfd",
|
||||
"a\x00b",
|
||||
// A small protobuf message: valid UTF-8, but control bytes.
|
||||
"\x08\x01\x12\x03abc",
|
||||
"\x1b[31mred\x1b[0m",
|
||||
"a\x7fb",
|
||||
} {
|
||||
v := bodyView(body)
|
||||
|
||||
assert.True(t, v.Binary, "%q", body)
|
||||
assert.Empty(t, v.Text)
|
||||
}
|
||||
|
||||
assert.False(t, bodyView("snow "+snowman).Binary)
|
||||
assert.False(t, bodyView("a\tb\r\nc\n").Binary)
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
)
|
||||
|
||||
// HandleEventDetail shows one event on its own page: its details,
|
||||
// its whole body and every delivery of it. The page reads the
|
||||
// event's body whole, which the receiver caps at 1 MB.
|
||||
func (h *Handlers) HandleEventDetail() http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
webhook, ok := h.ownedWebhook(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
if !h.dbMgr.DBExists(webhook.ID) {
|
||||
h.renderError(w, r, http.StatusNotFound)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
webhookDB, err := h.dbMgr.GetDB(webhook.ID)
|
||||
if err != nil {
|
||||
h.serverError(w, r, "failed to get webhook database", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
var rows []eventLogRow
|
||||
|
||||
err = webhookDB.Model(&database.Event{}).
|
||||
Select(eventColumns).
|
||||
Where(
|
||||
"id = ? AND webhook_id = ?",
|
||||
chi.URLParam(r, "eventID"), webhook.ID,
|
||||
).
|
||||
Limit(1).
|
||||
Find(&rows).Error
|
||||
if err != nil {
|
||||
h.serverError(w, r, "failed to load event", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if len(rows) == 0 {
|
||||
h.renderError(w, r, http.StatusNotFound)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
targets, err := h.loadTargetMap(webhook.ID)
|
||||
if err != nil {
|
||||
h.serverError(w, r, "failed to load targets", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
views, ok := h.eventLogViews(
|
||||
w, r, webhookDB, webhook.ID, rows, targets,
|
||||
)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
h.renderTemplate(w, r, "event_detail.html", map[string]any{
|
||||
tmplKeyWebhook: &webhook,
|
||||
"Event": views[0],
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package handlers_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/gorm/clause"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
"sneak.berlin/go/webhooker/internal/handlers"
|
||||
"sneak.berlin/go/webhooker/internal/session"
|
||||
)
|
||||
|
||||
// serveEventPage runs the real event page handler as the test user
|
||||
// for the given webhook and event ids.
|
||||
func serveEventPage(
|
||||
t *testing.T,
|
||||
h *handlers.Handlers,
|
||||
sess *session.Session,
|
||||
webhookID, eventID string,
|
||||
) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
|
||||
req := httptest.NewRequestWithContext(
|
||||
context.Background(),
|
||||
http.MethodGet,
|
||||
"/hook/"+webhookID+"/events/"+eventID,
|
||||
nil,
|
||||
)
|
||||
|
||||
for _, c := range authenticatedCookies(
|
||||
t, sess, deleteTestUserID, deleteTestUsername,
|
||||
) {
|
||||
req.AddCookie(c)
|
||||
}
|
||||
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add(paramSourceID, webhookID)
|
||||
rctx.URLParams.Add(paramEventID, eventID)
|
||||
|
||||
req = req.WithContext(
|
||||
context.WithValue(req.Context(), chi.RouteCtxKey, rctx),
|
||||
)
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
h.HandleEventDetail().ServeHTTP(w, req)
|
||||
|
||||
return w
|
||||
}
|
||||
|
||||
// TestHandleEventDetail_ShowsEventWholeWithDeliveries proves the
|
||||
// event's page shows its details, its whole body even past the cap
|
||||
// of the lists of events, pretty-printed and in the scrolling box,
|
||||
// and each delivery with its status and attempts.
|
||||
func TestHandleEventDetail_ShowsEventWholeWithDeliveries(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newRecentEventsFixture(t)
|
||||
target := seedTarget(t, f.db, f.webhook.ID, database.TargetTypeHTTP)
|
||||
|
||||
const sentinel = "TAIL-SENTINEL-5b2e"
|
||||
|
||||
body := `{"pad":"` + strings.Repeat("x", 2*bodyCap) +
|
||||
`","tail":"` + sentinel + `"}`
|
||||
event := f.event(t, contentTypeJSON, body, time.Now())
|
||||
f.attempt(t, f.delivery(
|
||||
t, event, target.ID, database.DeliveryStatusFailed,
|
||||
), http.StatusBadGateway, time.Second)
|
||||
|
||||
w := serveEventPage(t, f.h, f.sess, f.webhook.ID, event.ID)
|
||||
require.Equal(t, http.StatusOK, w.Code)
|
||||
|
||||
page := w.Body.String()
|
||||
|
||||
assert.Contains(t, page, event.ID)
|
||||
assert.Contains(t, page, contentTypeJSON)
|
||||
assert.Contains(t, page, strconv.Itoa(len(body))+" bytes")
|
||||
assert.Contains(t, page, "{\n "pad": "xxx")
|
||||
assert.Contains(t, page, ""tail": ""+sentinel+""\n}")
|
||||
assert.Contains(t, page, `style="max-height: 32rem; overflow-y: auto"`)
|
||||
assert.NotContains(t, page, "Showing the first")
|
||||
assert.Contains(t, page, target.Name)
|
||||
assert.Contains(t, page, ">failed</span>")
|
||||
assert.Contains(t, page, "Status: 502")
|
||||
}
|
||||
|
||||
// TestHandleEventDetail_ResubmitLinks proves a resubmitted copy's
|
||||
// page links to its original's page, and the original's page says
|
||||
// it was resubmitted.
|
||||
func TestHandleEventDetail_ResubmitLinks(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newRecentEventsFixture(t)
|
||||
original := f.event(t, contentTypeJSON, "{}", time.Now())
|
||||
|
||||
cp := &database.Event{
|
||||
WebhookID: f.webhook.ID,
|
||||
Method: http.MethodPost,
|
||||
Body: "{}",
|
||||
ContentType: contentTypeJSON,
|
||||
ResubmittedFromID: &original.ID,
|
||||
}
|
||||
require.NoError(t, f.webhookDB.Omit(clause.Associations).Create(cp).Error)
|
||||
|
||||
w := serveEventPage(t, f.h, f.sess, f.webhook.ID, cp.ID)
|
||||
require.Equal(t, http.StatusOK, w.Code)
|
||||
assert.Contains(
|
||||
t, w.Body.String(),
|
||||
`href="/hook/`+f.webhook.ID+`/events/`+original.ID+`"`,
|
||||
)
|
||||
|
||||
w = serveEventPage(t, f.h, f.sess, f.webhook.ID, original.ID)
|
||||
require.Equal(t, http.StatusOK, w.Code)
|
||||
assert.Contains(t, w.Body.String(), "as 1 new event<")
|
||||
}
|
||||
|
||||
// TestHandleEventDetail_UnknownEventNotFound proves the page is a
|
||||
// 404 for an event that does not exist and for one that belongs to
|
||||
// another webhook.
|
||||
func TestHandleEventDetail_UnknownEventNotFound(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var (
|
||||
h *handlers.Handlers
|
||||
sess *session.Session
|
||||
db *database.Database
|
||||
dbMgr *database.WebhookDBManager
|
||||
)
|
||||
|
||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
||||
app.RequireStart()
|
||||
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
mine := seedWebhook(t, db)
|
||||
theirs := seedWebhook(t, db)
|
||||
|
||||
seedEventWithBody(t, dbMgr, mine.ID, "{}")
|
||||
elsewhere := seedEventWithBody(t, dbMgr, theirs.ID, "{}")
|
||||
|
||||
for _, id := range []string{"no-such-event", elsewhere.ID} {
|
||||
w := serveEventPage(t, h, sess, mine.ID, id)
|
||||
assert.Equal(t, http.StatusNotFound, w.Code, id)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package handlers_test
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
)
|
||||
|
||||
// TestEventLog_TimesCarryTheirZone proves that the event log shows
|
||||
// when an event arrived, and that it and the event's page show when
|
||||
// each delivery was created and each attempt recorded: each as how
|
||||
// long ago, with the full UTC time on hover.
|
||||
func TestEventLog_TimesCarryTheirZone(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newRecentEventsFixture(t)
|
||||
target := seedTarget(t, f.db, f.webhook.ID, database.TargetTypeHTTP)
|
||||
|
||||
now := time.Now().UTC().Truncate(time.Second)
|
||||
receivedAt := now.Add(-3 * time.Hour)
|
||||
createdAt := now.Add(-90 * time.Minute)
|
||||
ranAt := now.Add(-30 * time.Minute)
|
||||
|
||||
event := f.event(t, contentTypeJSON, "{}", receivedAt)
|
||||
dlv := f.deliveryQueuedAt(
|
||||
t, event, target.ID, database.DeliveryStatusDelivered, createdAt,
|
||||
)
|
||||
f.attempt(t, dlv, http.StatusOK, ranAt.Sub(createdAt))
|
||||
|
||||
w := serveEventPage(t, f.h, f.sess, f.webhook.ID, event.ID)
|
||||
require.Equal(t, http.StatusOK, w.Code)
|
||||
|
||||
eventPage := w.Body.String()
|
||||
eventLog := renderSourceLogsPage(t, f.h, f.sess, f.webhook.ID)
|
||||
|
||||
received := receivedAt.Format(time.DateTime)
|
||||
assert.Contains(t, eventLog, `title="`+received+` UTC">3 hours ago</span>`)
|
||||
assert.NotContains(t, eventLog, received+"</span>",
|
||||
"an event's time must not be written without its zone")
|
||||
assert.Contains(t, eventPage, received+" UTC")
|
||||
|
||||
for _, page := range []string{eventLog, eventPage} {
|
||||
assert.Contains(t, page, `title="`+createdAt.Format(time.DateTime)+
|
||||
` UTC">created 1 hour ago</span>`)
|
||||
assert.Contains(t, page, `title="`+ranAt.Format(time.DateTime)+
|
||||
` UTC">30 minutes ago</span>`)
|
||||
}
|
||||
}
|
||||
@@ -3,15 +3,9 @@ package handlers
|
||||
import (
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// maxRenderedBodyBytes caps how many bytes of a stored event
|
||||
// body reach the event log page. Bodies come from the
|
||||
// unauthenticated receiver under the 1 MB ingest cap and
|
||||
// renderTemplate buffers a whole page before writing it, so
|
||||
// an uncapped page of paginationPerPage events is tens of
|
||||
// megabytes of resident memory per concurrent viewer.
|
||||
const maxRenderedBodyBytes = 8192
|
||||
"github.com/dustin/go-humanize"
|
||||
)
|
||||
|
||||
// eventLogColumns is the event log's projection. The casts to
|
||||
// blob are load-bearing: they make substr and length count
|
||||
@@ -24,27 +18,27 @@ const eventLogColumns = "id, created_at, method, content_type, " +
|
||||
"substr(cast(body as blob), 1, ?) AS body, " +
|
||||
"length(cast(body as blob)) AS body_bytes"
|
||||
|
||||
// eventColumns is eventLogColumns for the event's own page, which
|
||||
// shows the whole body.
|
||||
const eventColumns = "id, created_at, method, content_type, " +
|
||||
"resubmitted_from_id, " +
|
||||
"cast(body as blob) AS body, " +
|
||||
"length(cast(body as blob)) AS body_bytes"
|
||||
|
||||
// EventLogView is the display-safe projection of an event for
|
||||
// the event log page, alongside DeliveryView and TargetView.
|
||||
// It carries a capped body plus the true stored size, so the
|
||||
// page can mark a body as truncated without ever holding the
|
||||
// whole thing.
|
||||
// the event log page and the event's own page, alongside
|
||||
// DeliveryView and TargetView.
|
||||
type EventLogView struct {
|
||||
ID string
|
||||
CreatedAt time.Time
|
||||
Method string
|
||||
ContentType string
|
||||
|
||||
// Body holds at most maxRenderedBodyBytes bytes of the
|
||||
// stored body.
|
||||
Body string
|
||||
// Received is how long ago the event arrived, and ReceivedUTC
|
||||
// the full timestamp.
|
||||
Received string
|
||||
ReceivedUTC string
|
||||
|
||||
// BodyBytes is the true size of the stored body.
|
||||
BodyBytes int64
|
||||
|
||||
// BodyTruncated reports that the stored body was larger
|
||||
// than the cap, so the page owes the reader a marker.
|
||||
BodyTruncated bool
|
||||
Body BodyView
|
||||
|
||||
// ResubmittedFromID names the event this one was copied
|
||||
// from, empty for an event that arrived on the receiver.
|
||||
@@ -65,16 +59,10 @@ func (v EventLogView) ResubmittedFrom() bool {
|
||||
return v.ResubmittedFromID != ""
|
||||
}
|
||||
|
||||
// BodyShownBytes is how many body bytes the page is actually
|
||||
// rendering, which the truncation marker reports beside the
|
||||
// true size.
|
||||
func (v EventLogView) BodyShownBytes() int {
|
||||
return len(v.Body)
|
||||
}
|
||||
|
||||
// eventLogRow is one row of the event log projection. Its
|
||||
// body column arrives already cut to the cap by SQLite, with
|
||||
// the true size beside it.
|
||||
// eventLogRow is one row of the event log projection, or of
|
||||
// eventColumns. In the event log its body column arrives
|
||||
// already cut to the cap by SQLite, with the true size beside
|
||||
// it.
|
||||
type eventLogRow struct {
|
||||
ID string
|
||||
CreatedAt time.Time
|
||||
@@ -85,31 +73,23 @@ type eventLogRow struct {
|
||||
BodyBytes int64
|
||||
}
|
||||
|
||||
// view projects a loaded row for rendering.
|
||||
func (r *eventLogRow) view() EventLogView {
|
||||
body := r.Body
|
||||
truncated := r.BodyBytes > int64(len(body))
|
||||
|
||||
// Only a cut body can have been left mid-sequence by
|
||||
// this query. A whole body is passed through exactly as
|
||||
// stored, however malformed.
|
||||
if truncated {
|
||||
body = trimPartialRune(body)
|
||||
}
|
||||
|
||||
// view projects a loaded row of the webhook's events for
|
||||
// rendering.
|
||||
func (r *eventLogRow) view(webhookID string) EventLogView {
|
||||
var from string
|
||||
if r.ResubmittedFromID != nil {
|
||||
from = *r.ResubmittedFromID
|
||||
}
|
||||
|
||||
return EventLogView{
|
||||
ID: r.ID,
|
||||
CreatedAt: r.CreatedAt,
|
||||
Method: r.Method,
|
||||
ContentType: r.ContentType,
|
||||
Body: string(body),
|
||||
BodyBytes: r.BodyBytes,
|
||||
BodyTruncated: truncated,
|
||||
ID: r.ID,
|
||||
Method: r.Method,
|
||||
ContentType: r.ContentType,
|
||||
Received: humanize.Time(r.CreatedAt),
|
||||
ReceivedUTC: r.CreatedAt.UTC().Format(time.DateTime) + " UTC",
|
||||
Body: newBodyView(
|
||||
"/hook/"+webhookID+"/events/"+r.ID, r.Body, r.BodyBytes,
|
||||
),
|
||||
ResubmittedFromID: from,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ import (
|
||||
"sneak.berlin/go/webhooker/internal/session"
|
||||
)
|
||||
|
||||
// bodyCap is the number of body bytes the event log page is
|
||||
// bodyCap is the number of body bytes the lists of events are
|
||||
// allowed to render for one event.
|
||||
const bodyCap = handlers.MaxRenderedBodyBytesForTest
|
||||
|
||||
@@ -75,9 +75,7 @@ func seedAndProject(
|
||||
wh := seedWebhook(t, db)
|
||||
seedEventWithBody(t, dbMgr, wh.ID, body)
|
||||
|
||||
views := h.LoadEventLogViewsForTest(
|
||||
httptest.NewRecorder(), *wh, 1,
|
||||
)
|
||||
views := h.LoadEventLogViewsForTest(httptest.NewRecorder(), *wh)
|
||||
require.Len(t, views, 1)
|
||||
|
||||
return views[0]
|
||||
@@ -85,7 +83,7 @@ func seedAndProject(
|
||||
|
||||
// TestHandleSourceLogs_BoundsOversizeBody proves the rendered
|
||||
// page is bounded by the cap rather than by the stored payload:
|
||||
// the body here is 64 times the cap, and the ingest path would
|
||||
// the body here is 16 times the cap, and the ingest path would
|
||||
// accept twice as much again.
|
||||
func TestHandleSourceLogs_BoundsOversizeBody(t *testing.T) {
|
||||
t.Parallel()
|
||||
@@ -123,7 +121,7 @@ func TestHandleSourceLogs_BoundsOversizeBody(t *testing.T) {
|
||||
// The marker states the true stored size, not the cut one.
|
||||
assert.Contains(
|
||||
t, page,
|
||||
"showing "+strconv.Itoa(bodyCap)+
|
||||
"Showing the first "+strconv.Itoa(bodyCap)+
|
||||
" of "+strconv.Itoa(storedBytes)+" bytes",
|
||||
)
|
||||
}
|
||||
@@ -152,34 +150,35 @@ func TestHandleSourceLogs_SmallBodyRendersWhole(t *testing.T) {
|
||||
page := renderSourceLogsPage(t, h, sess, wh.ID)
|
||||
|
||||
assert.Contains(t, page, ""kept"")
|
||||
assert.NotContains(t, page, "Body truncated for display")
|
||||
assert.NotContains(t, page, "Showing the first")
|
||||
}
|
||||
|
||||
// TestEventLogView_CutMidRune proves a multi-byte rune severed
|
||||
// by the byte-wise cut is dropped rather than surfaced as a
|
||||
// mojibake tail.
|
||||
// mojibake tail, which would also make the text look binary.
|
||||
func TestEventLogView_CutMidRune(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
body := strings.Repeat(snowman, 4096)
|
||||
body := strings.Repeat(snowman, bodyCap)
|
||||
view := seedAndProject(t, body)
|
||||
|
||||
// bodyCap bytes hold bodyCap/3 whole snowmen and two bytes
|
||||
// of the next one; those two are dropped.
|
||||
whole := bodyCap / len(snowman)
|
||||
|
||||
assert.True(t, view.BodyTruncated)
|
||||
assert.Equal(t, int64(len(body)), view.BodyBytes)
|
||||
assert.Equal(t, strings.Repeat(snowman, whole), view.Body)
|
||||
assert.True(t, utf8.ValidString(view.Body))
|
||||
assert.LessOrEqual(t, len(view.Body), bodyCap)
|
||||
assert.True(t, view.Body.Cut)
|
||||
assert.False(t, view.Body.Binary)
|
||||
assert.Equal(t, int64(len(body)), view.Body.Size)
|
||||
assert.Equal(t, strings.Repeat(snowman, whole), view.Body.Text)
|
||||
assert.True(t, utf8.ValidString(view.Body.Text))
|
||||
assert.Equal(t, len(view.Body.Text), view.Body.ShownBytes)
|
||||
assert.LessOrEqual(t, view.Body.ShownBytes, bodyCap)
|
||||
}
|
||||
|
||||
// TestEventLogView_BinaryBodyLeftAsStored proves a binary
|
||||
// payload is passed through byte for byte. Its tail is invalid
|
||||
// UTF-8 however the cut falls, so repairing it would misreport
|
||||
// what the sender delivered.
|
||||
func TestEventLogView_BinaryBodyLeftAsStored(t *testing.T) {
|
||||
// TestEventLogView_BinaryBodyNotShown proves a body that is not
|
||||
// text is left out rather than shown as replacement characters,
|
||||
// whether it is cut or not.
|
||||
func TestEventLogView_BinaryBodyNotShown(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
raw := make([]byte, bodyCap+808)
|
||||
@@ -188,12 +187,21 @@ func TestEventLogView_BinaryBodyLeftAsStored(t *testing.T) {
|
||||
raw[i] = 0x80 | byte(i%0x40)
|
||||
}
|
||||
|
||||
view := seedAndProject(t, string(raw))
|
||||
for name, body := range map[string][]byte{
|
||||
"cut": raw,
|
||||
"whole": raw[:2048],
|
||||
"NUL": []byte("text\x00text"),
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
assert.True(t, view.BodyTruncated)
|
||||
assert.Equal(t, int64(len(raw)), view.BodyBytes)
|
||||
assert.Equal(t, string(raw[:bodyCap]), view.Body)
|
||||
assert.False(t, utf8.ValidString(view.Body))
|
||||
view := seedAndProject(t, string(body))
|
||||
|
||||
assert.True(t, view.Body.Binary)
|
||||
assert.Empty(t, view.Body.Text)
|
||||
assert.Equal(t, int64(len(body)), view.Body.Size)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestTrimPartialRune covers the distinction the cut repair
|
||||
|
||||
@@ -145,8 +145,9 @@ func (h *Handlers) resubmitEvent(
|
||||
// per-webhook database files — a sibling webhook's event is not in the
|
||||
// database being queried at all — and is there so the scoping survives
|
||||
// any future change that puts more than one webhook's events in one
|
||||
// file. Going through Model applies GORM's soft-delete scope, which is
|
||||
// what stops a reaped event being resubmitted.
|
||||
// file. A reaped event is not found because the retention reaper
|
||||
// deletes its row outright rather than marking it deleted; see
|
||||
// deleteEvents in internal/database/retention.go.
|
||||
func loadResubmitSource(
|
||||
webhookDB *gorm.DB,
|
||||
webhookID, eventID string,
|
||||
|
||||
@@ -19,10 +19,16 @@ func (s *Handlers) SetLogForTest(log *slog.Logger) {
|
||||
s.log = log
|
||||
}
|
||||
|
||||
// MaxRenderedBodyBytesForTest exposes the event log's body cap
|
||||
// to the handlers_test package.
|
||||
// MaxRenderedBodyBytesForTest exposes the body cap of the lists
|
||||
// of events to the handlers_test package.
|
||||
const MaxRenderedBodyBytesForTest = maxRenderedBodyBytes
|
||||
|
||||
// NewBodyViewForTest exposes newBodyView for use in the
|
||||
// handlers_test package.
|
||||
func NewBodyViewForTest(body []byte, size int64) BodyView {
|
||||
return newBodyView("/hook/w/events/e", body, size)
|
||||
}
|
||||
|
||||
// MaxRenderedResponseBytesForTest exposes the event log's
|
||||
// delivery response cap to the handlers_test package.
|
||||
const MaxRenderedResponseBytesForTest = maxRenderedResponseBytes
|
||||
@@ -36,11 +42,14 @@ const MaxRenderedAttemptsForTest = maxRenderedAttempts
|
||||
// the handlers enforce rather than a number copied beside it.
|
||||
const MaxTargetRetriesForTest = maxTargetRetries
|
||||
|
||||
// PageOrFirstForTest exposes pageOrFirst for use in the handlers_test
|
||||
// package.
|
||||
func PageOrFirstForTest(s string) int {
|
||||
return pageOrFirst(s)
|
||||
}
|
||||
// EventDBLeftMsgForTest and SidecarLeftMsgForTest expose the two
|
||||
// messages the webhook delete handler logs when a file of the event
|
||||
// database is left on disk, so a test checking that one is absent
|
||||
// checks for the handler's own wording.
|
||||
const (
|
||||
EventDBLeftMsgForTest = eventDBLeftMsg
|
||||
SidecarLeftMsgForTest = sidecarLeftMsg
|
||||
)
|
||||
|
||||
// DummyVerificationsForTest reports how many equivalent-cost
|
||||
// verifications were charged for usernames that do not exist. It
|
||||
@@ -64,10 +73,9 @@ func TrimPartialRuneForTest(b []byte) []byte {
|
||||
func (s *Handlers) LoadEventLogViewsForTest(
|
||||
w http.ResponseWriter,
|
||||
webhook database.Webhook,
|
||||
page int,
|
||||
) []EventLogView {
|
||||
views, _, _ := s.loadEventsWithDeliveries(
|
||||
w, newRequestForTest(), webhook, nil, page,
|
||||
w, newRequestForTest(), webhook, nil,
|
||||
)
|
||||
|
||||
return views
|
||||
@@ -128,22 +136,20 @@ func (s *Handlers) RenderTemplateForTest(
|
||||
// BuildSlackTargetConfigForTest exposes
|
||||
// buildSlackTargetConfig for use in the handlers_test package.
|
||||
func (s *Handlers) BuildSlackTargetConfigForTest(
|
||||
w http.ResponseWriter,
|
||||
r *http.Request,
|
||||
ctx context.Context,
|
||||
targetURL string,
|
||||
) (string, error) {
|
||||
return s.buildSlackTargetConfig(w, r, targetURL)
|
||||
) (string, string, error) {
|
||||
return s.buildSlackTargetConfig(ctx, targetURL)
|
||||
}
|
||||
|
||||
// BuildHTTPTargetConfigForTest exposes buildHTTPTargetConfig
|
||||
// for use in the handlers_test package, taking the form fields
|
||||
// an HTTP target's configuration is built from.
|
||||
func (s *Handlers) BuildHTTPTargetConfigForTest(
|
||||
w http.ResponseWriter,
|
||||
r *http.Request,
|
||||
ctx context.Context,
|
||||
targetURL, headers, timeout string,
|
||||
) (string, error) {
|
||||
return s.buildHTTPTargetConfig(w, r, targetFormInput{
|
||||
) (string, string, error) {
|
||||
return s.buildHTTPTargetConfig(ctx, targetFormInput{
|
||||
URL: targetURL,
|
||||
Headers: headers,
|
||||
Timeout: timeout,
|
||||
@@ -153,9 +159,17 @@ func (s *Handlers) BuildHTTPTargetConfigForTest(
|
||||
// BuildDatabaseTargetConfigForTest exposes
|
||||
// buildDatabaseTargetConfig for use in the handlers_test
|
||||
// package.
|
||||
func (s *Handlers) BuildDatabaseTargetConfigForTest(
|
||||
w http.ResponseWriter,
|
||||
expiry string,
|
||||
) (string, error) {
|
||||
return s.buildDatabaseTargetConfig(w, newRequestForTest(), expiry)
|
||||
func BuildDatabaseTargetConfigForTest(
|
||||
expiry, rotation string,
|
||||
) (string, string, error) {
|
||||
return buildDatabaseTargetConfig(expiry, rotation)
|
||||
}
|
||||
|
||||
// ArchiveFileViewForTest exposes archiveFileView, which describes a
|
||||
// database target's archive files as the target list shows them at
|
||||
// now.
|
||||
func (s *Handlers) ArchiveFileViewForTest(
|
||||
webhook *database.Webhook, target *database.Target, now time.Time,
|
||||
) *ArchiveFileView {
|
||||
return s.archiveFileView(webhook, target, now)
|
||||
}
|
||||
|
||||
@@ -30,10 +30,9 @@ import (
|
||||
const (
|
||||
// maxBodyShift is the bit shift for 1 MB body limit.
|
||||
maxBodyShift = 20
|
||||
// recentEventLimit is the number of recent events to show.
|
||||
// recentEventLimit is the number of most recent events that a
|
||||
// webhook's page and its event log show.
|
||||
recentEventLimit = 50
|
||||
// paginationPerPage is the number of items per page.
|
||||
paginationPerPage = 25
|
||||
|
||||
// tmplKeyError is the template data key for an error message.
|
||||
tmplKeyError = "Error"
|
||||
@@ -57,19 +56,20 @@ var errVerificationBusy = errors.New(
|
||||
type HandlersParams struct {
|
||||
fx.In
|
||||
|
||||
Logger *logger.Logger
|
||||
Globals *globals.Globals
|
||||
Config *config.Config
|
||||
Database *database.Database
|
||||
WebhookDBMgr *database.WebhookDBManager
|
||||
Healthcheck *healthcheck.Healthcheck
|
||||
Session *session.Session
|
||||
Middleware *middleware.Middleware
|
||||
Notifier delivery.Notifier
|
||||
Archives delivery.Archives
|
||||
SSRFGuard *delivery.Guard
|
||||
Metrics *metrics.Set
|
||||
Registry *prometheus.Registry
|
||||
Logger *logger.Logger
|
||||
Globals *globals.Globals
|
||||
Config *config.Config
|
||||
Database *database.Database
|
||||
WebhookDBMgr *database.WebhookDBManager
|
||||
Healthcheck *healthcheck.Healthcheck
|
||||
Session *session.Session
|
||||
Middleware *middleware.Middleware
|
||||
Notifier delivery.Notifier
|
||||
Archives delivery.Archives
|
||||
CircuitBreakers delivery.CircuitBreakers
|
||||
SSRFGuard *delivery.Guard
|
||||
Metrics *metrics.Set
|
||||
Registry *prometheus.Registry
|
||||
}
|
||||
|
||||
// Handlers provides HTTP handler methods for all application
|
||||
@@ -84,6 +84,7 @@ type Handlers struct {
|
||||
mw *middleware.Middleware
|
||||
notifier delivery.Notifier
|
||||
archives delivery.Archives
|
||||
breakers delivery.CircuitBreakers
|
||||
mtr *metrics.Set
|
||||
templates map[string]*template.Template
|
||||
|
||||
@@ -97,7 +98,10 @@ type Handlers struct {
|
||||
// names through the archive rename, the save and any move back.
|
||||
// Interleaved, one could rename an archive between another's
|
||||
// rename and save, leaving the file named for one edit and the
|
||||
// stored names from the other.
|
||||
// stored names from the other. An archive download holds it while
|
||||
// it reads the stored names and lists the files they give, and
|
||||
// again for each file while it finds the file under the names
|
||||
// stored then and opens it.
|
||||
renameMu sync.Mutex
|
||||
|
||||
// dummyVerifications counts the equivalent-cost verifications
|
||||
@@ -109,22 +113,25 @@ type Handlers struct {
|
||||
// parsePageTemplate parses a page-specific template set from the
|
||||
// embedded FS. Each page template is combined with the shared
|
||||
// base, htmlheader, navbar and notice templates, and with any further
|
||||
// files the page includes. The page file must be listed first so that
|
||||
// its root action ({{template "base" .}}) becomes the template set's
|
||||
// entry point.
|
||||
// files the page includes. The set is named after the page file, so
|
||||
// the page's root action ({{template "base" .}}) is its entry point.
|
||||
//
|
||||
// The page file is parsed last because a later definition of a name
|
||||
// replaces an earlier one: the page's {{define "title"}} must replace
|
||||
// the {{block "title"}} fallback in htmlheader.html.
|
||||
func parsePageTemplate(
|
||||
pageFile string, included ...string,
|
||||
) *template.Template {
|
||||
files := append([]string{
|
||||
pageFile,
|
||||
"base.html",
|
||||
"htmlheader.html",
|
||||
"navbar.html",
|
||||
"notice.html",
|
||||
}, included...)
|
||||
files = append(files, pageFile)
|
||||
|
||||
return template.Must(
|
||||
template.ParseFS(templates.Templates, files...),
|
||||
template.New(pageFile).ParseFS(templates.Templates, files...),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -144,21 +151,31 @@ func New(
|
||||
s.mw = params.Middleware
|
||||
s.notifier = params.Notifier
|
||||
s.archives = params.Archives
|
||||
s.breakers = params.CircuitBreakers
|
||||
s.mtr = params.Metrics
|
||||
s.ssrf = params.SSRFGuard
|
||||
|
||||
// Parse all page templates once at startup
|
||||
s.templates = map[string]*template.Template{
|
||||
"login.html": parsePageTemplate("login.html"),
|
||||
"profile.html": parsePageTemplate("profile.html"),
|
||||
"settings.html": parsePageTemplate("settings.html"),
|
||||
"sources_list.html": parsePageTemplate("sources_list.html"),
|
||||
"sources_new.html": parsePageTemplate("sources_new.html"),
|
||||
"source_detail.html": parsePageTemplate("source_detail.html", "webhook_stats.html"),
|
||||
"source_edit.html": parsePageTemplate("source_edit.html"),
|
||||
"source_logs.html": parsePageTemplate("source_logs.html"),
|
||||
"target_edit.html": parsePageTemplate("target_edit.html"),
|
||||
"error.html": parsePageTemplate("error.html"),
|
||||
"login.html": parsePageTemplate("login.html"),
|
||||
"profile.html": parsePageTemplate("profile.html"),
|
||||
"settings.html": parsePageTemplate("settings.html"),
|
||||
"sources_list.html": parsePageTemplate("sources_list.html"),
|
||||
"sources_new.html": parsePageTemplate("sources_new.html"),
|
||||
"source_detail.html": parsePageTemplate(
|
||||
"source_detail.html", "webhook_stats.html", "event_body.html",
|
||||
),
|
||||
"source_edit.html": parsePageTemplate("source_edit.html"),
|
||||
"source_logs.html": parsePageTemplate(
|
||||
"source_logs.html", "event_body.html", "delivery_row.html",
|
||||
"delivery_attempts.html",
|
||||
),
|
||||
"event_detail.html": parsePageTemplate(
|
||||
"event_detail.html", "event_body.html", "delivery_row.html",
|
||||
"delivery_attempts.html",
|
||||
),
|
||||
"target_edit.html": parsePageTemplate("target_edit.html"),
|
||||
"error.html": parsePageTemplate("error.html"),
|
||||
}
|
||||
|
||||
lc.Append(fx.Hook{
|
||||
@@ -385,7 +402,7 @@ func (s *Handlers) pageData(
|
||||
// partial body and the status before a mid-render error can be
|
||||
// reported, leaving no way to serve a 500. Buffering makes a page's
|
||||
// rendered size resident memory per concurrent viewer, so every page
|
||||
// owes it a bound: the event log caps each stored body at
|
||||
// owes it a bound: the lists of events cap each stored body at
|
||||
// maxRenderedBodyBytes for exactly this reason.
|
||||
func (s *Handlers) executeTemplate(
|
||||
w http.ResponseWriter,
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -181,6 +182,40 @@ func (r *recordingArchives) Renames() []archiveRename {
|
||||
return out
|
||||
}
|
||||
|
||||
// testCircuitBreakers is a delivery.CircuitBreakers that reports, for
|
||||
// each target, the circuit state and cooldown a test gave it with Set,
|
||||
// and a closed breaker for any other target.
|
||||
type testCircuitBreakers struct {
|
||||
mu sync.Mutex
|
||||
states map[string]delivery.CircuitState
|
||||
cooldowns map[string]time.Duration
|
||||
}
|
||||
|
||||
// Set makes the target's breaker read as state, with cooldown left.
|
||||
func (b *testCircuitBreakers) Set(
|
||||
targetID string, state delivery.CircuitState, cooldown time.Duration,
|
||||
) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
|
||||
if b.states == nil {
|
||||
b.states = map[string]delivery.CircuitState{}
|
||||
b.cooldowns = map[string]time.Duration{}
|
||||
}
|
||||
|
||||
b.states[targetID] = state
|
||||
b.cooldowns[targetID] = cooldown
|
||||
}
|
||||
|
||||
func (b *testCircuitBreakers) StateAndCooldown(
|
||||
targetID string,
|
||||
) (delivery.CircuitState, time.Duration) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
|
||||
return b.states[targetID], b.cooldowns[targetID]
|
||||
}
|
||||
|
||||
// newTestApp returns an app whose RequireStart fails the test when
|
||||
// starting takes longer than fx's default start timeout of 15s. That
|
||||
// limit catches a start that hangs, not a busy host: measured with make
|
||||
@@ -231,6 +266,12 @@ func newTestAppWithConfig(
|
||||
func(r *recordingArchives) delivery.Archives {
|
||||
return r
|
||||
},
|
||||
func() *testCircuitBreakers {
|
||||
return &testCircuitBreakers{}
|
||||
},
|
||||
func(b *testCircuitBreakers) delivery.CircuitBreakers {
|
||||
return b
|
||||
},
|
||||
metrics.NewRegistry,
|
||||
metrics.New,
|
||||
middleware.New,
|
||||
@@ -314,16 +355,12 @@ func TestBuildSlackTargetConfig_AcceptsPublicURL(t *testing.T) {
|
||||
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
req := httptest.NewRequestWithContext(
|
||||
context.Background(), http.MethodPost, "/", nil)
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
cfg, err := h.BuildSlackTargetConfigForTest(
|
||||
w, req, "http://93.184.216.34/services/T00/B00/xxx",
|
||||
cfg, errMsg, err := h.BuildSlackTargetConfigForTest(
|
||||
t.Context(), "http://93.184.216.34/services/T00/B00/xxx",
|
||||
)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, http.StatusOK, w.Code)
|
||||
assert.Empty(t, errMsg)
|
||||
assert.Contains(t, cfg, "webhookUrl")
|
||||
}
|
||||
|
||||
@@ -337,17 +374,13 @@ func TestBuildSlackTargetConfig_RejectsReservedURL(t *testing.T) {
|
||||
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
req := httptest.NewRequestWithContext(
|
||||
context.Background(), http.MethodPost, "/", nil)
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
cfg, err := h.BuildSlackTargetConfigForTest(
|
||||
w, req, "http://169.254.169.254/latest/meta-data/",
|
||||
cfg, errMsg, err := h.BuildSlackTargetConfigForTest(
|
||||
t.Context(), "http://169.254.169.254/latest/meta-data/",
|
||||
)
|
||||
|
||||
require.Error(t, err)
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, errMsg, "Invalid target URL")
|
||||
assert.Empty(t, cfg)
|
||||
assert.Equal(t, http.StatusBadRequest, w.Code)
|
||||
}
|
||||
|
||||
func TestRenderTemplate(t *testing.T) {
|
||||
@@ -444,30 +477,37 @@ func TestRenderTemplateMidRenderErrorSendsNoPartialBody(t *testing.T) {
|
||||
func TestBuildDatabaseTargetConfig_Valid(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var h *handlers.Handlers
|
||||
|
||||
app := newTestApp(t, &h)
|
||||
app.RequireStart()
|
||||
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
// Empty expiry: the keep-forever default, empty config.
|
||||
w := httptest.NewRecorder()
|
||||
cfg, err := h.BuildDatabaseTargetConfigForTest(w, "")
|
||||
// Empty expiry and rotation: the keep-forever, one-file default,
|
||||
// empty config.
|
||||
cfg, errMsg, err := handlers.BuildDatabaseTargetConfigForTest("", "")
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, errMsg)
|
||||
assert.Empty(t, cfg)
|
||||
|
||||
// Explicit never is stored as config.
|
||||
w = httptest.NewRecorder()
|
||||
cfg, err = h.BuildDatabaseTargetConfigForTest(w, "never")
|
||||
cfg, errMsg, err = handlers.BuildDatabaseTargetConfigForTest("never", "")
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, errMsg)
|
||||
assert.JSONEq(t, `{"expiry":"never"}`, cfg)
|
||||
|
||||
// A positive duration is stored as config.
|
||||
w = httptest.NewRecorder()
|
||||
cfg, err = h.BuildDatabaseTargetConfigForTest(w, "720h")
|
||||
cfg, errMsg, err = handlers.BuildDatabaseTargetConfigForTest("720h", "")
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, errMsg)
|
||||
assert.JSONEq(t, `{"expiry":"720h"}`, cfg)
|
||||
|
||||
// A rotation is stored as config, with or without an expiry.
|
||||
cfg, errMsg, err = handlers.BuildDatabaseTargetConfigForTest("", "daily")
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, errMsg)
|
||||
assert.JSONEq(t, `{"rotation":"daily"}`, cfg)
|
||||
|
||||
cfg, errMsg, err = handlers.BuildDatabaseTargetConfigForTest(
|
||||
"720h", "hourly",
|
||||
)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, errMsg)
|
||||
assert.JSONEq(t, `{"expiry":"720h","rotation":"hourly"}`, cfg)
|
||||
}
|
||||
|
||||
func TestBuildDatabaseTargetConfig_RejectsBadExpiry(
|
||||
@@ -475,22 +515,31 @@ func TestBuildDatabaseTargetConfig_RejectsBadExpiry(
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
var h *handlers.Handlers
|
||||
|
||||
app := newTestApp(t, &h)
|
||||
app.RequireStart()
|
||||
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
for _, bad := range []string{"nonsense", "7d", "-5h"} {
|
||||
w := httptest.NewRecorder()
|
||||
cfg, err := h.BuildDatabaseTargetConfigForTest(w, bad)
|
||||
cfg, errMsg, err := handlers.BuildDatabaseTargetConfigForTest(bad, "")
|
||||
|
||||
require.Error(t, err, "expiry %q", bad)
|
||||
assert.Empty(t, cfg)
|
||||
assert.Equal(
|
||||
t, http.StatusBadRequest, w.Code,
|
||||
"expiry %q should be rejected with 400", bad,
|
||||
require.NoError(t, err)
|
||||
assert.Contains(
|
||||
t, errMsg, "Invalid archive expiry",
|
||||
"expiry %q should be refused", bad,
|
||||
)
|
||||
assert.Empty(t, cfg)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildDatabaseTargetConfig_RejectsBadRotation(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
for _, bad := range []string{"weekly", "Daily", " none"} {
|
||||
cfg, errMsg, err := handlers.BuildDatabaseTargetConfigForTest("", bad)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Contains(
|
||||
t, errMsg, "Invalid archive rotation",
|
||||
"rotation %q should be refused", bad,
|
||||
)
|
||||
assert.Empty(t, cfg)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ const (
|
||||
webhookSaved noticeCode = "webhook-saved"
|
||||
webhookDeleted noticeCode = "webhook-deleted"
|
||||
entrypointAdded noticeCode = "entrypoint-added"
|
||||
entrypointSaved noticeCode = "entrypoint-saved"
|
||||
entrypointDeleted noticeCode = "entrypoint-deleted"
|
||||
entrypointActivated noticeCode = "entrypoint-activated"
|
||||
entrypointDeactivated noticeCode = "entrypoint-deactivated"
|
||||
@@ -48,6 +49,7 @@ func noticeFor(r *http.Request) *notice {
|
||||
webhookSaved: {Text: "Webhook saved."},
|
||||
webhookDeleted: {Text: "Webhook deleted."},
|
||||
entrypointAdded: {Text: "Entrypoint added."},
|
||||
entrypointSaved: {Text: "Entrypoint description saved."},
|
||||
entrypointDeleted: {Text: "Entrypoint deleted."},
|
||||
entrypointActivated: {Text: "Entrypoint activated."},
|
||||
entrypointDeactivated: {Text: "Entrypoint deactivated."},
|
||||
@@ -64,7 +66,8 @@ func noticeFor(r *http.Request) *notice {
|
||||
},
|
||||
replayTargetDeleted: {
|
||||
Text: "Not replayed: the target this delivery was for " +
|
||||
"has been deleted. Recreate the target, then replay.",
|
||||
"has been deleted. Use Resubmit to send the event " +
|
||||
"to the webhook's currently active targets.",
|
||||
Failed: true,
|
||||
},
|
||||
replayTargetMissing: {
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
package handlers_test
|
||||
|
||||
import (
|
||||
"html/template"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
"sneak.berlin/go/webhooker/internal/handlers"
|
||||
"sneak.berlin/go/webhooker/internal/session"
|
||||
"sneak.berlin/go/webhooker/templates"
|
||||
)
|
||||
|
||||
// TestEveryPageRendersItsOwnTitle renders each page template and checks
|
||||
// the browser tab title is the one the page declares, not the
|
||||
// "Webhooker" fallback in htmlheader.html. A page that fails to render
|
||||
// shows the error page's title instead, and fails here too.
|
||||
func TestEveryPageRendersItsOwnTitle(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var h *handlers.Handlers
|
||||
|
||||
var sess *session.Session
|
||||
|
||||
app := newTestApp(t, &h, &sess)
|
||||
app.RequireStart()
|
||||
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
// A pointer, as in the handlers: some pages call
|
||||
// Webhook.RetentionLabel, a pointer method.
|
||||
webhook := &database.Webhook{Name: "orders", RetentionDays: 14}
|
||||
webhook.ID = testWebhookID
|
||||
|
||||
pages := []struct {
|
||||
page string
|
||||
data map[string]any
|
||||
title string
|
||||
}{
|
||||
{"login.html", map[string]any{}, "Login - Webhooker"},
|
||||
{"profile.html", map[string]any{}, "Profile - Webhooker"},
|
||||
{"settings.html", map[string]any{}, "Settings - Webhooker"},
|
||||
{"sources_list.html", map[string]any{}, "Webhooks - Webhooker"},
|
||||
{"sources_new.html", map[string]any{}, "New Webhook - Webhooker"},
|
||||
{
|
||||
"source_detail.html",
|
||||
map[string]any{dataKeyWebhook: webhook},
|
||||
"orders - Webhooker",
|
||||
},
|
||||
{
|
||||
"source_edit.html",
|
||||
map[string]any{dataKeyWebhook: webhook},
|
||||
"Edit orders - Webhooker",
|
||||
},
|
||||
{
|
||||
"source_logs.html",
|
||||
map[string]any{
|
||||
dataKeyWebhook: webhook,
|
||||
dataKeyEvents: []handlers.EventLogView{},
|
||||
"TotalEvents": int64(0),
|
||||
},
|
||||
"Full Event Log - orders - Webhooker",
|
||||
},
|
||||
{
|
||||
"event_detail.html",
|
||||
map[string]any{dataKeyWebhook: webhook},
|
||||
"Event - orders - Webhooker",
|
||||
},
|
||||
{
|
||||
"target_edit.html",
|
||||
map[string]any{
|
||||
dataKeyWebhook: webhook,
|
||||
"Target": map[string]any{"Name": "alerts", "Type": "slack"},
|
||||
},
|
||||
"Edit alerts - Webhooker",
|
||||
},
|
||||
{
|
||||
"error.html",
|
||||
map[string]any{"StatusText": http.StatusText(http.StatusNotFound)},
|
||||
"Not Found - Webhooker",
|
||||
},
|
||||
}
|
||||
|
||||
for _, p := range pages {
|
||||
body := renderPage(t, h, sess, p.page, p.data)
|
||||
|
||||
_, afterOpen, _ := strings.Cut(body, "<title>")
|
||||
title, _, _ := strings.Cut(afterOpen, "</title>")
|
||||
|
||||
assert.Equal(t, p.title, title, p.page)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTitleFallbackIsWebhooker checks the title htmlheader.html gives a
|
||||
// page that declares none. Every page declares one, so it is checked on
|
||||
// htmlheader.html alone.
|
||||
func TestTitleFallbackIsWebhooker(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
header := template.Must(
|
||||
template.ParseFS(templates.Templates, "htmlheader.html"),
|
||||
)
|
||||
|
||||
var buf strings.Builder
|
||||
|
||||
require.NoError(t, header.ExecuteTemplate(&buf, "htmlheader", nil))
|
||||
assert.Contains(t, buf.String(), "<title>Webhooker</title>")
|
||||
}
|
||||
@@ -12,11 +12,12 @@ import (
|
||||
)
|
||||
|
||||
// recentEventColumns is the recent events list's projection. It
|
||||
// leaves out the body, for the reason maxRenderedBodyBytes gives,
|
||||
// and reads its size from body_bytes, recorded when the event was
|
||||
// reads the body cut to maxRenderedBodyBytes, as eventLogColumns
|
||||
// does, and its size from body_bytes, recorded when the event was
|
||||
// stored.
|
||||
const recentEventColumns = "id, created_at, method, content_type, " +
|
||||
"resubmitted_from_id, body_bytes"
|
||||
"resubmitted_from_id, body_bytes, " +
|
||||
"substr(cast(body as blob), 1, ?) AS body"
|
||||
|
||||
// recentAttemptColumns is the part of a recorded attempt the list
|
||||
// uses. The event log's deliveryResultColumns also reads response
|
||||
@@ -50,6 +51,9 @@ type RecentEventView struct {
|
||||
// unless the webhook has exactly one HTTP target.
|
||||
Status string
|
||||
StatusClass string
|
||||
|
||||
// Body is what the row shows when it is expanded.
|
||||
Body BodyView
|
||||
}
|
||||
|
||||
// recentEventRow is one row of recentEventColumns.
|
||||
@@ -60,6 +64,7 @@ type recentEventRow struct {
|
||||
ContentType string
|
||||
ResubmittedFromID *string
|
||||
BodyBytes uint64
|
||||
Body []byte
|
||||
}
|
||||
|
||||
// recentAttemptRow is one row of recentAttemptColumns. CreatedAt is
|
||||
@@ -100,7 +105,7 @@ func loadRecentEvents(
|
||||
var rows []recentEventRow
|
||||
|
||||
err := webhookDB.Model(&database.Event{}).
|
||||
Select(recentEventColumns).
|
||||
Select(recentEventColumns, maxRenderedBodyBytes).
|
||||
Where("webhook_id = ?", webhookID).
|
||||
Order("created_at DESC").
|
||||
Limit(recentEventLimit).
|
||||
@@ -145,7 +150,7 @@ func loadRecentEvents(
|
||||
views := make([]RecentEventView, len(rows))
|
||||
for i := range rows {
|
||||
views[i] = rows[i].view(
|
||||
byEvent[rows[i].ID], attempts, statusTargetID,
|
||||
webhookID, byEvent[rows[i].ID], attempts, statusTargetID,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -182,14 +187,20 @@ func loadRecentAttempts(
|
||||
return byDelivery, nil
|
||||
}
|
||||
|
||||
// view projects a loaded row for rendering. deliveries is the
|
||||
// event's deliveries, oldest first, and attempts their recorded
|
||||
// attempts keyed by delivery ID.
|
||||
// view projects a loaded row of the webhook's events for
|
||||
// rendering. deliveries is the event's deliveries, oldest first,
|
||||
// and attempts their recorded attempts keyed by delivery ID.
|
||||
func (r *recentEventRow) view(
|
||||
webhookID string,
|
||||
deliveries []database.Delivery,
|
||||
attempts map[string][]recentAttemptRow,
|
||||
statusTargetID string,
|
||||
) RecentEventView {
|
||||
//nolint:gosec // body_bytes is at most the receiver's 1 MB cap
|
||||
body := newBodyView(
|
||||
"/hook/"+webhookID+"/events/"+r.ID, r.Body, int64(r.BodyBytes),
|
||||
)
|
||||
|
||||
v := RecentEventView{
|
||||
Method: r.Method,
|
||||
ContentType: r.ContentType,
|
||||
@@ -197,6 +208,7 @@ func (r *recentEventRow) view(
|
||||
ReceivedUTC: r.CreatedAt.UTC().Format(time.DateTime) + " UTC",
|
||||
Size: humanize.Bytes(r.BodyBytes),
|
||||
ProcessingTime: processingTime(deliveries, attempts),
|
||||
Body: body,
|
||||
}
|
||||
|
||||
if r.ResubmittedFromID != nil {
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -301,6 +302,73 @@ func TestHandleSourceDetail_NoStatusWithoutSingleHTTPTarget(
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleSourceDetail_RecentEventsLinkAndExpand proves each row
|
||||
// links to its event's own page and expands to show its body, and
|
||||
// that only the newest row starts expanded.
|
||||
func TestHandleSourceDetail_RecentEventsLinkAndExpand(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newRecentEventsFixture(t)
|
||||
now := time.Now()
|
||||
|
||||
older := f.event(
|
||||
t, contentTypeJSON, `{"which":"older"}`, now.Add(-time.Minute),
|
||||
)
|
||||
newer := f.event(t, contentTypeJSON, `{"which":"newer"}`, now)
|
||||
|
||||
body := f.render(t)
|
||||
|
||||
for _, e := range []*database.Event{older, newer} {
|
||||
assert.Contains(
|
||||
t, body, `href="/hook/`+f.webhook.ID+`/events/`+e.ID+`"`,
|
||||
)
|
||||
}
|
||||
|
||||
assert.Equal(t, 2, strings.Count(body, `<div x-show="open" x-cloak class="mt-3">`))
|
||||
assert.Equal(t, 1, strings.Count(body, " data-open>"))
|
||||
|
||||
open := strings.Index(body, " data-open>")
|
||||
newerBody := strings.Index(body, ""which": "newer"")
|
||||
olderBody := strings.Index(body, ""which": "older"")
|
||||
|
||||
assert.Less(t, open, newerBody, "the newest row is not the open one")
|
||||
assert.Less(t, newerBody, olderBody)
|
||||
}
|
||||
|
||||
// TestHandleSourceDetail_RecentEventBodyCut proves a body up to
|
||||
// the cap is shown whole and pretty-printed, and a larger one only
|
||||
// its first bodyCap bytes, as received, with links to the whole
|
||||
// body on the event's page and to the download.
|
||||
func TestHandleSourceDetail_RecentEventBodyCut(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newRecentEventsFixture(t)
|
||||
now := time.Now()
|
||||
|
||||
// A JSON document of n bytes.
|
||||
document := func(n int) string {
|
||||
return `{"pad":"` + strings.Repeat("x", n-len(`{"pad":""}`)) + `"}`
|
||||
}
|
||||
|
||||
whole := f.event(
|
||||
t, contentTypeJSON, document(bodyCap), now.Add(-time.Minute),
|
||||
)
|
||||
cut := f.event(t, contentTypeJSON, document(bodyCap+1), now)
|
||||
|
||||
body := f.render(t)
|
||||
eventURL := `href="/hook/` + f.webhook.ID + `/events/`
|
||||
|
||||
assert.Equal(t, 1, strings.Count(body, "{\n "pad": "))
|
||||
assert.Contains(t, body, "{"pad":"xxx")
|
||||
assert.Contains(
|
||||
t, body,
|
||||
"Showing the first "+strconv.Itoa(bodyCap)+" of "+
|
||||
strconv.Itoa(bodyCap+1)+" bytes, unformatted.",
|
||||
)
|
||||
assert.Contains(t, body, eventURL+cut.ID+`/body"`)
|
||||
assert.NotContains(t, body, eventURL+whole.ID+`/body"`)
|
||||
}
|
||||
|
||||
// TestHandleWebhook_RecordsBodySize proves the receiver records the
|
||||
// body's size in bytes, not characters, with the event it stores.
|
||||
func TestHandleWebhook_RecordsBodySize(t *testing.T) {
|
||||
|
||||
@@ -77,7 +77,8 @@ func settingRows(cfg *config.Config) []settingRow {
|
||||
{
|
||||
"RETENTION_SWEEP_INTERVAL",
|
||||
"How often the retention reaper and archive sweeper run " +
|
||||
"(Go duration, must be positive)",
|
||||
"(Go duration, must be positive). A value that does " +
|
||||
"not parse, or is zero or negative, fails startup",
|
||||
cfg.RetentionSweepInterval.String(),
|
||||
},
|
||||
{
|
||||
|
||||
@@ -0,0 +1,209 @@
|
||||
package handlers_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"html"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/gorm"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
)
|
||||
|
||||
// submitCreateForm posts the new webhook form and returns the
|
||||
// recorder.
|
||||
func submitCreateForm(
|
||||
env *sourceTestEnv, form url.Values,
|
||||
) *httptest.ResponseRecorder {
|
||||
req := formRequest("/hooks/new", env.cookies, form, nil)
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
env.handlers.HandleSourceCreateSubmit().ServeHTTP(w, req)
|
||||
|
||||
return w
|
||||
}
|
||||
|
||||
// assertNothingCreated checks that the main database holds no webhook,
|
||||
// entrypoint or target.
|
||||
func assertNothingCreated(t *testing.T, db *database.Database) {
|
||||
t.Helper()
|
||||
|
||||
for _, model := range []any{
|
||||
&database.Webhook{}, &database.Entrypoint{}, &database.Target{},
|
||||
} {
|
||||
var count int64
|
||||
|
||||
require.NoError(t, db.DB().Model(model).Count(&count).Error)
|
||||
assert.Zerof(t, count, "%T rows were created", model)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleSourceCreateSubmit_CreatesRequestedTargets submits the new
|
||||
// webhook form with the HTTP target URL filled in or empty, and with
|
||||
// the archive checkbox off or on with each pruning choice. The webhook
|
||||
// gets an HTTP target only for a URL and a database target only for a
|
||||
// checked archive. The pruning choice is always submitted, as the
|
||||
// browser submits it while it is hidden, and is ignored when archive
|
||||
// is off.
|
||||
func TestHandleSourceCreateSubmit_CreatesRequestedTargets(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupSourceTest(t)
|
||||
|
||||
// Each value the archive pruning choice submits, after an empty
|
||||
// one that stands for the archive checkbox left off.
|
||||
expiries := []string{
|
||||
"", "never", "1h", "12h", "24h", "720h", "2160h", "8760h",
|
||||
}
|
||||
|
||||
for _, httpURL := range []string{"", editOriginalURL} {
|
||||
for _, expiry := range expiries {
|
||||
name := "url=" + httpURL + " archive=" + expiry
|
||||
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
form := url.Values{}
|
||||
form.Set("name", name)
|
||||
form.Set("http_url", httpURL)
|
||||
form.Set("archive_expiry", "720h")
|
||||
|
||||
if expiry != "" {
|
||||
form.Set("archive", "on")
|
||||
form.Set("archive_expiry", expiry)
|
||||
}
|
||||
|
||||
w := submitCreateForm(env, form)
|
||||
require.Equal(t, http.StatusSeeOther, w.Code, w.Body.String())
|
||||
|
||||
var webhook database.Webhook
|
||||
|
||||
require.NoError(t, env.db.DB().
|
||||
Where("name = ?", name).First(&webhook).Error)
|
||||
|
||||
byType := map[database.TargetType]database.Target{}
|
||||
for _, target := range targetsForWebhook(t, env.db, webhook.ID) {
|
||||
byType[target.Type] = target
|
||||
}
|
||||
|
||||
wantCount := 0
|
||||
|
||||
if httpURL != "" {
|
||||
wantCount++
|
||||
|
||||
assert.Equal(t, "HTTP", byType[database.TargetTypeHTTP].Name)
|
||||
assert.JSONEq(t, `{"url":"`+httpURL+`"}`,
|
||||
byType[database.TargetTypeHTTP].Config)
|
||||
}
|
||||
|
||||
if expiry != "" {
|
||||
wantCount++
|
||||
|
||||
assert.Equal(t, "Archive",
|
||||
byType[database.TargetTypeDatabase].Name)
|
||||
assert.JSONEq(t, `{"expiry":"`+expiry+`"}`,
|
||||
byType[database.TargetTypeDatabase].Config)
|
||||
}
|
||||
|
||||
assert.Len(t, byType, wantCount)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleSourceCreateSubmit_RefusedFormKeepsEveryValue refuses the
|
||||
// new webhook form for an invalid HTTP target URL and for an invalid
|
||||
// retention, each with archive on. Nothing is created, and the form
|
||||
// comes back with the reason and every value entered: name,
|
||||
// description, retention, URL, the checked archive box and the pruning
|
||||
// and rotation choices.
|
||||
func TestHandleSourceCreateSubmit_RefusedFormKeepsEveryValue(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
const badURL = "Invalid target URL"
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
retention string
|
||||
httpURL string
|
||||
reason string
|
||||
}{
|
||||
{"blocked url", "7", editBlockedURL, badURL},
|
||||
{"unsupported scheme", "7", "ftp://93.184.216.34/hook", badURL},
|
||||
{"bad retention", "-5", editOriginalURL, "Retention must be"},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupSourceTest(t)
|
||||
|
||||
form := url.Values{}
|
||||
form.Set("name", "kept name")
|
||||
form.Set("description", "kept description")
|
||||
form.Set("retention_days", tc.retention)
|
||||
form.Set("http_url", tc.httpURL)
|
||||
form.Set("archive", "on")
|
||||
form.Set("archive_expiry", "2160h")
|
||||
form.Set("archive_rotation", "hourly")
|
||||
|
||||
w := submitCreateForm(env, form)
|
||||
require.Equal(t, http.StatusBadRequest, w.Code)
|
||||
|
||||
page := w.Body.String()
|
||||
assert.Contains(t, page, tc.reason)
|
||||
assert.Contains(t, page, `value="kept name"`)
|
||||
assert.Contains(t, page, `>kept description</textarea>`)
|
||||
assert.Contains(t, page, `value="`+tc.retention+`"`)
|
||||
assert.Contains(t, page,
|
||||
`value="`+html.EscapeString(tc.httpURL)+`"`)
|
||||
assert.Contains(t, page, `name="archive" value="on" checked`)
|
||||
assert.Contains(t, page, `x-data="collapsible" data-open`)
|
||||
assert.Contains(t, page, `<option value="2160h" selected>`)
|
||||
assert.Contains(t, page, `<option value="hourly" selected>`)
|
||||
|
||||
assertNothingCreated(t, env.db)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// errInjectedTargetCreate is the failure a test makes the insert of a
|
||||
// target report.
|
||||
var errInjectedTargetCreate = errors.New("injected target create failure")
|
||||
|
||||
// TestHandleSourceCreateSubmit_FailedTargetInsertCreatesNothing makes
|
||||
// inserting a target fail after the webhook and its entrypoint were
|
||||
// inserted, and checks that neither is left behind.
|
||||
func TestHandleSourceCreateSubmit_FailedTargetInsertCreatesNothing(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupSourceTest(t)
|
||||
|
||||
require.NoError(t, env.db.DB().Callback().Create().
|
||||
Before("gorm:create").
|
||||
Register("test:fail_target_create", func(tx *gorm.DB) {
|
||||
if tx.Statement.Table == "targets" {
|
||||
_ = tx.AddError(errInjectedTargetCreate)
|
||||
}
|
||||
}),
|
||||
)
|
||||
|
||||
form := url.Values{}
|
||||
form.Set("name", "rolled back")
|
||||
form.Set("archive", "on")
|
||||
form.Set("archive_expiry", "never")
|
||||
|
||||
w := submitCreateForm(env, form)
|
||||
require.Equal(t, http.StatusInternalServerError, w.Code)
|
||||
|
||||
assertNothingCreated(t, env.db)
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user