Compare commits
1
Commits
next
..
8328016bec
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8328016bec |
+23
-84
@@ -1,84 +1,23 @@
|
|||||||
# .dockerignore does NOT use .gitignore semantics. Docker matches with
|
# .ci-fingerprint is deliberately NOT excluded: it is the CI cache barrier
|
||||||
# moby/patternmatcher: filepath.Match plus `**`, so `*` does not cross
|
# that keeps the check stages from replaying a cached pass. See the lint
|
||||||
# `/` and an unprefixed pattern is anchored at the context root. Every
|
# stage of the Dockerfile.
|
||||||
# depth-independent pattern therefore needs `**/`, or `config/.env` and
|
.git/
|
||||||
# `certs/server.key` still ship while this file reads as solved. Only
|
bin/
|
||||||
# genuinely root-anchored entries go unprefixed. Never transplant these
|
# Third-party browser assets are fetched and hash-verified inside the build by
|
||||||
# into .gitignore, where `**/` is wrong.
|
# script/fetch-assets. Excluding any host copy keeps a developer's working tree
|
||||||
#
|
# from supplying the bytes that get shipped. The script and its
|
||||||
# Matching is case-sensitive, so secrets use character ranges rather
|
# static/vendor.sha256 manifest stay in the context.
|
||||||
# than an ALL-CAPS twin, which would still miss `Server.Key`.
|
static/js/alpine.min.js
|
||||||
#
|
*.md
|
||||||
# Extend with this repo's own host-built artifacts, written anchored:
|
LICENSE
|
||||||
# `/myapp`, never `**/myapp`, which also matches `cmd/myapp/` and
|
.editorconfig
|
||||||
# deletes the package directory from the context.
|
.env
|
||||||
|
.env.*
|
||||||
# .git is sent without its config. Without a VERSION build argument the
|
*.db
|
||||||
# stage that compiles runs `git describe --tags --always` on .git, which
|
*.sqlite
|
||||||
# does not need .git/config; that file can hold a credential, such as a
|
*.sqlite3
|
||||||
# password in a remote URL or the token the CI checkout step stores there.
|
.DS_Store
|
||||||
# Each submodule keeps a config with the same exposure in its git directory
|
.idea/
|
||||||
# under .git/modules/, nested again for a submodule's own submodules, or in
|
.vscode/
|
||||||
# its own .git directory when it keeps one.
|
tmp/
|
||||||
# KNOWN GAP: a submodule whose name has a `config` segment (`config`,
|
temp/
|
||||||
# `deploy/config`, `config/lib`) loses its whole git directory, because
|
|
||||||
# `**/.git/modules/**/config` also matches that segment's directory
|
|
||||||
# under .git/modules/. Go's version stamping then fails the build;
|
|
||||||
# nothing leaks. Name such a submodule without that segment:
|
|
||||||
# `git submodule add --name`.
|
|
||||||
**/.git/config
|
|
||||||
**/.git/modules/**/config
|
|
||||||
|
|
||||||
# Agent scratch: one full checkout of the repo per in-flight agent.
|
|
||||||
# Anchored because it occurs once where agents run at the repo root.
|
|
||||||
# KNOWN GAP: a repo running agents in subdirectories still ships
|
|
||||||
# `services/api/.claude/` and must add its own anchored entry.
|
|
||||||
.claude
|
|
||||||
|
|
||||||
# Environment files. `*.env` covers bare `.env` and the `prod.env`
|
|
||||||
# convention. Re-include a committed template with a negation if the
|
|
||||||
# build needs one: `!docs/example.env`.
|
|
||||||
**/*.[eE][nN][vV]
|
|
||||||
**/.[eE][nN][vV].*
|
|
||||||
**/.[eE][nN][vV][rR][cC]
|
|
||||||
|
|
||||||
# Private keys and the bundles carrying them. Public certificates
|
|
||||||
# (*.crt, *.cer) are deliberately absent: they are legitimate inputs.
|
|
||||||
**/*.[pP][eE][mM]
|
|
||||||
**/*.[kK][eE][yY]
|
|
||||||
**/*.[pP]12
|
|
||||||
**/*.[pP][fF][xX]
|
|
||||||
**/[iI][dD]_[rR][sS][aA]
|
|
||||||
**/[iI][dD]_[dD][sS][aA]
|
|
||||||
**/[iI][dD]_[eE][cC][dD][sS][aA]
|
|
||||||
**/[iI][dD]_[eE][cC][dD][sS][aA]_[sS][kK]
|
|
||||||
**/[iI][dD]_[eE][dD]25519
|
|
||||||
**/[iI][dD]_[eE][dD]25519_[sS][kK]
|
|
||||||
|
|
||||||
# Dependencies: restored inside the image, never copied in.
|
|
||||||
**/node_modules
|
|
||||||
|
|
||||||
# OS metadata.
|
|
||||||
**/.DS_Store
|
|
||||||
**/Thumbs.db
|
|
||||||
|
|
||||||
# Editor state: never a build input, and it churns COPY.
|
|
||||||
**/*.swp
|
|
||||||
**/*.swo
|
|
||||||
**/*~
|
|
||||||
**/*.bak
|
|
||||||
**/.idea
|
|
||||||
**/.vscode
|
|
||||||
**/*.sublime-*
|
|
||||||
|
|
||||||
# This repository's own host-built artifacts: the binary `make build`
|
|
||||||
# writes, and the Alpine.js file `make assets` extracts from 3p/ (the
|
|
||||||
# build extracts its own).
|
|
||||||
/bin
|
|
||||||
/static/js/alpine.min.js
|
|
||||||
|
|
||||||
# SQLite databases, which hold the session key and webhook payloads, at
|
|
||||||
# any depth.
|
|
||||||
**/*.db
|
|
||||||
**/*.sqlite
|
|
||||||
**/*.sqlite3
|
|
||||||
|
|||||||
@@ -10,6 +10,3 @@ insert_final_newline = true
|
|||||||
|
|
||||||
[Makefile]
|
[Makefile]
|
||||||
indent_style = tab
|
indent_style = tab
|
||||||
|
|
||||||
[*.go]
|
|
||||||
indent_style = tab
|
|
||||||
|
|||||||
@@ -1,9 +1,43 @@
|
|||||||
name: check
|
name: check
|
||||||
on: [push]
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- '**'
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
check:
|
check:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
# actions/checkout v4.2.2, 2026-02-22
|
- name: Checkout
|
||||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
|
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 2024-10-23
|
||||||
- run: script/cibuild
|
with:
|
||||||
|
# The fingerprint step below needs history to find the last commit
|
||||||
|
# that touched the Docker build context, and the superseded-status
|
||||||
|
# step needs it to walk ancestors (it aborts on a shallow clone).
|
||||||
|
fetch-depth: 0
|
||||||
|
|
||||||
|
- name: Mark superseded run statuses
|
||||||
|
# Gitea cancels the in-flight run when another commit is pushed to the
|
||||||
|
# same branch and records the cancellation as `failure`, so a commit
|
||||||
|
# that was never tested reads as a test result. The script rewrites
|
||||||
|
# those statuses to say what happened. See its header for why the
|
||||||
|
# state stays `failure` and not `skipped`.
|
||||||
|
env:
|
||||||
|
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||||
|
run: script/ci-mark-superseded
|
||||||
|
|
||||||
|
- name: Fingerprint the build context
|
||||||
|
# `.dockerignore` keeps docs out of the build context, so a docs-only
|
||||||
|
# commit legitimately replays the whole image from cache and stays
|
||||||
|
# cheap. Every other commit writes a new fingerprint into the context,
|
||||||
|
# which invalidates the `COPY . .` layer of both check stages: a
|
||||||
|
# commit that was never linted, formatted-checked, tested and built
|
||||||
|
# cannot report success from cache.
|
||||||
|
run: |
|
||||||
|
set -eu
|
||||||
|
fp="$(git log -1 --format=%H -- . ':!*.md' ':!LICENSE' ':!.editorconfig')"
|
||||||
|
printf '%s\n' "${fp:-$GITHUB_SHA}" > .ci-fingerprint
|
||||||
|
|
||||||
|
- name: Build Docker image (runs make check)
|
||||||
|
run: script/cibuild
|
||||||
|
|||||||
+22
-53
@@ -1,53 +1,3 @@
|
|||||||
# OS
|
|
||||||
.DS_Store
|
|
||||||
Thumbs.db
|
|
||||||
|
|
||||||
# Editors
|
|
||||||
*.swp
|
|
||||||
*.swo
|
|
||||||
*~
|
|
||||||
*.bak
|
|
||||||
.idea/
|
|
||||||
.vscode/
|
|
||||||
*.sublime-*
|
|
||||||
|
|
||||||
# Agent scratch (worktrees of this repo, created and destroyed by
|
|
||||||
# in-flight tooling). Unanchored: .gitignore patterns already match at
|
|
||||||
# every depth, so no prefix is wanted here. This is not a .dockerignore
|
|
||||||
# entry and must not be given a `**/` prefix on the way into one.
|
|
||||||
.claude/
|
|
||||||
|
|
||||||
# Node
|
|
||||||
node_modules/
|
|
||||||
|
|
||||||
# Secrets. Unanchored like every entry above, so each matches at every
|
|
||||||
# depth. Matching is case-sensitive on Linux, so names use character
|
|
||||||
# ranges rather than a lowercase form that misses `Server.Key`.
|
|
||||||
|
|
||||||
# Environment files. `*.env` covers bare `.env` and the `prod.env`
|
|
||||||
# convention. Only the templates `example.env` and `sample.env` are
|
|
||||||
# re-included below. A repository that commits any other template adds
|
|
||||||
# its own negation after these lines, for example `!.env.example`.
|
|
||||||
*.[eE][nN][vV]
|
|
||||||
.[eE][nN][vV].*
|
|
||||||
.[eE][nN][vV][rR][cC]
|
|
||||||
!example.env
|
|
||||||
!sample.env
|
|
||||||
|
|
||||||
# Private keys and the bundles carrying them.
|
|
||||||
*.[pP][eE][mM]
|
|
||||||
*.[kK][eE][yY]
|
|
||||||
*.[pP]12
|
|
||||||
*.[pP][fF][xX]
|
|
||||||
[iI][dD]_[rR][sS][aA]
|
|
||||||
[iI][dD]_[dD][sS][aA]
|
|
||||||
[iI][dD]_[eE][cC][dD][sS][aA]
|
|
||||||
[iI][dD]_[eE][cC][dD][sS][aA]_[sS][kK]
|
|
||||||
[iI][dD]_[eE][dD]25519
|
|
||||||
[iI][dD]_[eE][dD]25519_[sS][kK]
|
|
||||||
|
|
||||||
# This repository's own entries, after the shared content above.
|
|
||||||
|
|
||||||
# Binaries
|
# Binaries
|
||||||
*.exe
|
*.exe
|
||||||
*.dll
|
*.dll
|
||||||
@@ -65,6 +15,21 @@ bin/
|
|||||||
# Go vendor directory
|
# Go vendor directory
|
||||||
vendor/
|
vendor/
|
||||||
|
|
||||||
|
# IDE specific files
|
||||||
|
.idea/
|
||||||
|
*.swp
|
||||||
|
*.swo
|
||||||
|
*~
|
||||||
|
.vscode/
|
||||||
|
|
||||||
|
# OS specific files
|
||||||
|
.DS_Store
|
||||||
|
Thumbs.db
|
||||||
|
|
||||||
|
# Environment and config files
|
||||||
|
.env
|
||||||
|
.env.local
|
||||||
|
|
||||||
# Data directory (SQLite databases)
|
# Data directory (SQLite databases)
|
||||||
data/
|
data/
|
||||||
*.db
|
*.db
|
||||||
@@ -78,6 +43,10 @@ data/
|
|||||||
tmp/
|
tmp/
|
||||||
temp/
|
temp/
|
||||||
|
|
||||||
# Alpine.js, extracted by `make assets` from its tarball in 3p/, which is
|
# CI cache barrier, written into the build context by the check workflow
|
||||||
# what is committed.
|
.ci-fingerprint
|
||||||
/static/js/alpine.min.js
|
|
||||||
|
# Third-party browser assets, fetched and hash-verified by
|
||||||
|
# script/fetch-assets against static/vendor.sha256. Not committed:
|
||||||
|
# REPO_POLICIES.md forbids minified bundles in version control.
|
||||||
|
/static/js/alpine.min.js
|
||||||
+2
-73
@@ -10,21 +10,14 @@ run:
|
|||||||
|
|
||||||
linters:
|
linters:
|
||||||
default: all
|
default: all
|
||||||
enable:
|
|
||||||
# Successor to the deprecated gomodguard. Named explicitly, rather than
|
|
||||||
# left to `default: all`, because it carries the module policy below.
|
|
||||||
- gomodguard_v2
|
|
||||||
disable:
|
disable:
|
||||||
# Genuinely incompatible with project patterns
|
# Genuinely incompatible with project patterns
|
||||||
- exhaustruct # Requires all struct fields
|
- exhaustruct # Requires all struct fields
|
||||||
- exhaustruct_v5 # Requires all struct fields (successor to exhaustruct)
|
- depguard # Dependency allow/block lists
|
||||||
- godot # Requires comments to end with periods
|
- godot # Requires comments to end with periods
|
||||||
|
- wsl # Deprecated, replaced by wsl_v5
|
||||||
- wrapcheck # Too verbose for internal packages
|
- wrapcheck # Too verbose for internal packages
|
||||||
- varnamelen # Short names like db, id are idiomatic Go
|
- varnamelen # Short names like db, id are idiomatic Go
|
||||||
# Deprecated: the warning is attached to the old name, so it is
|
|
||||||
# silenced by disabling that name, not by enabling the successor.
|
|
||||||
- wsl # Deprecated, replaced by wsl_v5
|
|
||||||
- gomodguard # Deprecated, replaced by gomodguard_v2
|
|
||||||
settings:
|
settings:
|
||||||
lll:
|
lll:
|
||||||
line-length: 88
|
line-length: 88
|
||||||
@@ -35,70 +28,6 @@ linters:
|
|||||||
max-complexity: 15
|
max-complexity: 15
|
||||||
dupl:
|
dupl:
|
||||||
threshold: 100
|
threshold: 100
|
||||||
depguard:
|
|
||||||
# Test-support code must not be compiled into the shipped binary. A
|
|
||||||
# test-support package exists to hand a test privileges the program
|
|
||||||
# itself must never have, so a file that is not a test must not import
|
|
||||||
# one. Test files, and the files inside a package whose directory name
|
|
||||||
# ends in `test`, are where that code belongs, and are exempt.
|
|
||||||
#
|
|
||||||
# The deny list below is the one part of this file a repository is
|
|
||||||
# expected to extend, and the only part it may. depguard matches an
|
|
||||||
# import path against a list of prefixes, so it cannot be told "any path
|
|
||||||
# whose last segment ends in test"; a repository's own test-support
|
|
||||||
# packages have to be named here one at a time, by full import path,
|
|
||||||
# under a module path that differs from repository to repository. Add
|
|
||||||
# them; change nothing else.
|
|
||||||
rules:
|
|
||||||
test-support:
|
|
||||||
list-mode: lax
|
|
||||||
files:
|
|
||||||
- "$all"
|
|
||||||
- "!$test"
|
|
||||||
- "!**/*test/**"
|
|
||||||
deny:
|
|
||||||
- pkg: net/http/httptest
|
|
||||||
desc: >-
|
|
||||||
Test-support code belongs in test files and in packages whose
|
|
||||||
directory name ends in test, not in the shipped binary.
|
|
||||||
- pkg: sneak.berlin/go/webhooker/internal/config/configtest
|
|
||||||
desc: test support; a file that is not a test must not import it
|
|
||||||
- pkg: sneak.berlin/go/webhooker/internal/database/databasetest
|
|
||||||
desc: test support; a file that is not a test must not import it
|
|
||||||
- pkg: sneak.berlin/go/webhooker/internal/middleware/middlewaretest
|
|
||||||
desc: test support; a file that is not a test must not import it
|
|
||||||
# Only decisions already recorded in the Go package defaults are
|
|
||||||
# listed here. Every entry matches the module path exactly.
|
|
||||||
gomodguard_v2:
|
|
||||||
blocked:
|
|
||||||
- module: github.com/rs/zerolog
|
|
||||||
recommendations:
|
|
||||||
- log/slog
|
|
||||||
reason: "Structured logging is stdlib log/slog."
|
|
||||||
# One entry per pre-fork module path, because the later releases
|
|
||||||
# are separate paths. A prefix match would be shorter but would
|
|
||||||
# also reach github.com/go-redis/redismock, the test double for
|
|
||||||
# the successor these entries recommend.
|
|
||||||
- module: github.com/go-redis/redis
|
|
||||||
recommendations:
|
|
||||||
- github.com/redis/go-redis/v9
|
|
||||||
reason: "Pre-fork module; use the maintained go-redis v9."
|
|
||||||
- module: github.com/go-redis/redis/v7
|
|
||||||
recommendations:
|
|
||||||
- github.com/redis/go-redis/v9
|
|
||||||
reason: "Pre-fork module; use the maintained go-redis v9."
|
|
||||||
- module: github.com/go-redis/redis/v8
|
|
||||||
recommendations:
|
|
||||||
- github.com/redis/go-redis/v9
|
|
||||||
reason: "Pre-fork module; use the maintained go-redis v9."
|
|
||||||
- module: github.com/sergi/go-diff
|
|
||||||
recommendations:
|
|
||||||
- github.com/aymanbagabas/go-udiff
|
|
||||||
reason: "No unified diff output; use go-udiff."
|
|
||||||
- module: github.com/hexops/gotextdiff
|
|
||||||
recommendations:
|
|
||||||
- github.com/aymanbagabas/go-udiff
|
|
||||||
reason: "Unmaintained fork; use go-udiff."
|
|
||||||
|
|
||||||
issues:
|
issues:
|
||||||
max-issues-per-linter: 0
|
max-issues-per-linter: 0
|
||||||
|
|||||||
@@ -1,2 +0,0 @@
|
|||||||
node_modules/
|
|
||||||
yarn.lock
|
|
||||||
@@ -1,4 +0,0 @@
|
|||||||
{
|
|
||||||
"tabWidth": 4,
|
|
||||||
"proseWrap": "always"
|
|
||||||
}
|
|
||||||
@@ -1,3 +0,0 @@
|
|||||||
# Install into node_modules/: the Dockerfile's lint and Markdown stages run
|
|
||||||
# ESLint and prettier from node_modules/.bin.
|
|
||||||
nodeLinker: node-modules
|
|
||||||
Binary file not shown.
+58
-202
@@ -1,82 +1,10 @@
|
|||||||
# Stylesheet stages. static/css/tailwind.css is generated, by this pinned
|
# Lint stage
|
||||||
# tailwindcss, from static/css/input.css and the files its @source lines
|
# golangci/golangci-lint:v2.12.2 (Debian-based), 2026-08-07
|
||||||
# name. `make css` (script/css) writes it out from the css-output stage.
|
# Using Debian-based image because mattn/go-sqlite3 (CGO) does not
|
||||||
# The css-check stage fails when the committed file differs from what is
|
# compile on Alpine musl (off64_t is a glibc type).
|
||||||
# generated; `make check` runs it, and so does the build stage below.
|
FROM golangci/golangci-lint:v2.12.2@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS lint
|
||||||
#
|
|
||||||
# tailwindcss v4.2.1 standalone CLI, released 2026-02-23: one binary per
|
|
||||||
# architecture, each pinned by its sha256 from the release's sha256sums.txt.
|
|
||||||
# debian:bookworm-slim, 2026-10-02: the binary needs glibc.
|
|
||||||
FROM debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251 AS tailwind-amd64
|
|
||||||
ADD --checksum=sha256:39e8d4e24b3c83b0a6e69e100a972fbc75d5fef8dce47b3ddac3cf92dea81fe3 --chmod=755 \
|
|
||||||
https://github.com/tailwindlabs/tailwindcss/releases/download/v4.2.1/tailwindcss-linux-x64 /usr/local/bin/tailwindcss
|
|
||||||
|
|
||||||
FROM debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251 AS tailwind-arm64
|
RUN apt-get update && apt-get install -y --no-install-recommends make && rm -rf /var/lib/apt/lists/*
|
||||||
ADD --checksum=sha256:d87e6486bb3f70b04ef1dcaacc4ee6548a5a15fbf521b31bc24d2c774f68a951 --chmod=755 \
|
|
||||||
https://github.com/tailwindlabs/tailwindcss/releases/download/v4.2.1/tailwindcss-linux-arm64 /usr/local/bin/tailwindcss
|
|
||||||
|
|
||||||
# TARGETARCH, set by docker, is the architecture being built for.
|
|
||||||
FROM tailwind-${TARGETARCH} AS css
|
|
||||||
WORKDIR /src
|
|
||||||
COPY . .
|
|
||||||
RUN tailwindcss -i static/css/input.css -o /out/tailwind.css --minify
|
|
||||||
|
|
||||||
FROM scratch AS css-output
|
|
||||||
COPY --from=css /out/tailwind.css /
|
|
||||||
|
|
||||||
# Both files are split after each "}", one rule per line, so that when they
|
|
||||||
# differ the diff shows the rules that differ.
|
|
||||||
FROM css AS css-check
|
|
||||||
RUN sed 's/}/}\n/g' static/css/tailwind.css > /tmp/committed.css \
|
|
||||||
&& sed 's/}/}\n/g' /out/tailwind.css > /tmp/generated.css \
|
|
||||||
&& diff -U0 /tmp/committed.css /tmp/generated.css || { \
|
|
||||||
echo "static/css/tailwind.css is not what make css generates; run make css" >&2; \
|
|
||||||
exit 1; \
|
|
||||||
}
|
|
||||||
|
|
||||||
# JavaScript lint stages: ESLint, at the version package.json and yarn.lock
|
|
||||||
# pin, checks static/js/ against eslint.config.mjs. js-deps installs it, and
|
|
||||||
# prettier for the Markdown stages below. The lint phase below runs js-lint.
|
|
||||||
#
|
|
||||||
# The image's own corepack runs the yarn that package.json's packageManager
|
|
||||||
# field names, yarn 4.18.1 (released 2026-09-24), and checks it against the
|
|
||||||
# hash there. The image also ships yarn 1, which `corepack enable yarn`
|
|
||||||
# replaces.
|
|
||||||
# node:24.21.0-alpine (LTS), 2026-09-18
|
|
||||||
FROM node:24.21.0-alpine@sha256:ebfe2f90462722a7a4de65e91990e97fe0d401c70e0e762c5b53302f905ec1c1 AS js-deps
|
|
||||||
WORKDIR /src
|
|
||||||
COPY package.json yarn.lock .yarnrc.yml ./
|
|
||||||
RUN corepack enable yarn && yarn install --immutable --mode=skip-build
|
|
||||||
|
|
||||||
FROM js-deps AS js-lint
|
|
||||||
COPY . .
|
|
||||||
RUN --network=none node_modules/.bin/eslint static/js
|
|
||||||
|
|
||||||
# Markdown stages: prettier, at the version package.json and yarn.lock pin,
|
|
||||||
# formats every Markdown file in the tree with the settings in .prettierrc.
|
|
||||||
# `make fmt` (script/fmt) writes the formatted files out from markdown-output.
|
|
||||||
# markdown-check fails on any file prettier would change; `make fmt-check`
|
|
||||||
# runs it, and so does the build stage below.
|
|
||||||
FROM js-deps AS markdown
|
|
||||||
COPY . .
|
|
||||||
RUN --network=none node_modules/.bin/prettier --write '**/*.md' \
|
|
||||||
&& mkdir /out \
|
|
||||||
&& find . -name '*.md' ! -path './node_modules/*' -exec cp -p --parents {} /out \;
|
|
||||||
|
|
||||||
FROM scratch AS markdown-output
|
|
||||||
COPY --from=markdown /out /
|
|
||||||
|
|
||||||
FROM js-deps AS markdown-check
|
|
||||||
COPY . .
|
|
||||||
RUN --network=none node_modules/.bin/prettier --check '**/*.md'
|
|
||||||
|
|
||||||
# Lint phase: the Go formatting check and golangci-lint over the Go code,
|
|
||||||
# and ESLint over static/js/ through the copy from js-lint at the end.
|
|
||||||
# `make lint` (script/lint) builds this stage alone; the build stage below
|
|
||||||
# depends on it.
|
|
||||||
#
|
|
||||||
# golangci/golangci-lint:v2.14.0 (Debian-based), 2026-09-24
|
|
||||||
FROM golangci/golangci-lint:v2.14.0@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint
|
|
||||||
|
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
|
|
||||||
@@ -84,102 +12,34 @@ WORKDIR /src
|
|||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
RUN go mod download
|
RUN go mod download
|
||||||
|
|
||||||
|
# Copy source code. In CI the context also carries .ci-fingerprint, whose
|
||||||
|
# value changes with every commit that touches the build context (see
|
||||||
|
# .gitea/workflows/check.yml). That invalidates this layer, so the checks
|
||||||
|
# below cannot report success by replaying a cached pass. Do not add it to
|
||||||
|
# .dockerignore.
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
# gofmt and golangci-lint are invoked directly rather than through `make
|
# Run formatting check and linter. golangci-lint is invoked directly rather
|
||||||
# fmt-check` and `make lint`, which are themselves docker builds and would
|
# than through `make lint`: this stage is already the pinned linter image, and
|
||||||
# need a docker daemon inside this one. The Markdown half of `make
|
# script/lint is a wrapper that builds Dockerfile.lint, so calling it here
|
||||||
# fmt-check` is the markdown-check stage above.
|
# would need a docker daemon inside the build. Keep these steps in step with
|
||||||
RUN if [ -n "$(gofmt -s -l .)" ]; then echo "gofmt needed on:"; gofmt -s -l .; exit 1; fi
|
# Dockerfile.lint, including --network=none (see its header for why).
|
||||||
|
RUN make fmt-check
|
||||||
# static/static.go embeds the Alpine.js file this extracts from 3p/; without
|
|
||||||
# it the static package does not compile and cannot be linted.
|
|
||||||
RUN script/assets
|
|
||||||
|
|
||||||
# The golangci-lint steps run with --network=none. `golangci-lint config
|
|
||||||
# verify` is documented as fetching its JSON schema over HTTPS; this pinned
|
|
||||||
# image resolves the schema without any network, and --network=none enforces
|
|
||||||
# that. It also proves no linter reaches out at analysis time.
|
|
||||||
#
|
|
||||||
# `run` silently ignores config keys it does not recognize, so a typo would
|
|
||||||
# disable a setting without a word. `config verify` is what catches that.
|
|
||||||
RUN --network=none golangci-lint config verify --config .golangci.yml
|
RUN --network=none golangci-lint config verify --config .golangci.yml
|
||||||
# --build-tags browser also lints the browser test, which is built only with
|
RUN --network=none golangci-lint run --config .golangci.yml ./...
|
||||||
# that tag (make test-browser).
|
|
||||||
RUN --network=none golangci-lint run --config .golangci.yml --build-tags browser ./...
|
|
||||||
|
|
||||||
# Nothing is wanted from js-lint; the copy is what makes this phase run it.
|
|
||||||
COPY --from=js-lint /src/yarn.lock /dev/null
|
|
||||||
|
|
||||||
# Test phase. -race needs cgo and so a C compiler, which the Debian Go image
|
|
||||||
# ships and the alpine one does not. `make test` (script/test) builds this
|
|
||||||
# stage alone; the build stage below depends on it.
|
|
||||||
#
|
|
||||||
# golang:1.26.1-bookworm (Debian-based), 2026-03-17
|
|
||||||
FROM golang:1.26.1-bookworm@sha256:4465644228bc2857a954b092167e12aa59c006a3492282a6c820bf4755fd64a4 AS test
|
|
||||||
|
|
||||||
WORKDIR /src
|
|
||||||
|
|
||||||
COPY go.mod go.sum ./
|
|
||||||
RUN go mod download
|
|
||||||
|
|
||||||
COPY . .
|
|
||||||
|
|
||||||
# static/static.go embeds the Alpine.js file this extracts from 3p/.
|
|
||||||
RUN script/assets
|
|
||||||
|
|
||||||
# -timeout applies to each package on its own, so 90s has only to clear the
|
|
||||||
# slowest one. -p 4 -parallel 8 keep the run under 2 GB of memory: at most
|
|
||||||
# four test binaries build or run at once, each with at most eight parallel
|
|
||||||
# tests. Under -race every test binary and every link costs a few hundred MB,
|
|
||||||
# so the defaults (one per core) add up to several GB on a many-core host.
|
|
||||||
#
|
|
||||||
# The first run has no -v: go test then prints one result line per package,
|
|
||||||
# with its coverage, and for a package that fails, everything its tests
|
|
||||||
# wrote. Verbose output from the whole suite passes the 2 MiB at which the
|
|
||||||
# Docker build cuts off a step's log, so on a failure only the tests that
|
|
||||||
# failed run again, with -v. go test reports a failed test as a line starting
|
|
||||||
# "--- FAIL: TestName" (a failed subtest's line is indented, and reruns with
|
|
||||||
# its parent) and a failed package as "FAIL<tab>package/path<tab>...". A
|
|
||||||
# failure that names no test, such as a build error or a timeout, is already
|
|
||||||
# shown in full, so there is nothing to rerun. The step fails after the rerun
|
|
||||||
# whatever its result: the first run already showed the suite is broken.
|
|
||||||
#
|
|
||||||
# TMPDIR, where the tests keep their SQLite databases, is a tmpfs: SQLite
|
|
||||||
# waits for the disk at every commit, and on a busy host that waiting was
|
|
||||||
# about 40% of the slowest package's run time. GOTMPDIR keeps go's own
|
|
||||||
# build files, the test binaries among them, on disk.
|
|
||||||
#
|
|
||||||
# bash with pipefail, so that the first run's status is go test's, not tee's.
|
|
||||||
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
|
|
||||||
RUN --mount=type=tmpfs,target=/tmp/tests,size=512m \
|
|
||||||
export TMPDIR=/tmp/tests GOTMPDIR=/tmp; \
|
|
||||||
go test -race -cover -p 4 -parallel 8 -timeout 90s ./... 2>&1 | tee /tmp/go-test.log && exit 0; \
|
|
||||||
tests="$(awk '/^--- FAIL: / { print $3 }' /tmp/go-test.log | paste -s -d '|' -)"; \
|
|
||||||
packages="$(awk '/^FAIL\t/ { print $2 }' /tmp/go-test.log)"; \
|
|
||||||
if [ -n "$tests" ]; then \
|
|
||||||
echo "--- Rerunning the failed tests with -v for details ---"; \
|
|
||||||
go test -race -v -p 4 -parallel 8 -timeout 90s -run "^($tests)\$" $packages; \
|
|
||||||
fi; \
|
|
||||||
exit 1
|
|
||||||
|
|
||||||
# Build stage
|
# Build stage
|
||||||
# golang:1.26.1-bookworm (Debian-based), 2026-03-17
|
# golang:1.26.1-bookworm (Debian-based), 2026-03-17
|
||||||
# The image ships git and make, which the version step below uses.
|
# Using Debian-based image because gorm.io/driver/sqlite pulls in
|
||||||
|
# mattn/go-sqlite3 (CGO), which does not compile on Alpine musl.
|
||||||
FROM golang:1.26.1-bookworm@sha256:4465644228bc2857a954b092167e12aa59c006a3492282a6c820bf4755fd64a4 AS builder
|
FROM golang:1.26.1-bookworm@sha256:4465644228bc2857a954b092167e12aa59c006a3492282a6c820bf4755fd64a4 AS builder
|
||||||
|
|
||||||
# Nothing is wanted from the lint and test phases or from the stylesheet and
|
# Depend on lint stage passing
|
||||||
# Markdown checks; the copies are what make BuildKit build them first, so
|
|
||||||
# this stage cannot run unless they all passed.
|
|
||||||
COPY --from=lint /src/go.sum /dev/null
|
COPY --from=lint /src/go.sum /dev/null
|
||||||
COPY --from=test /src/go.sum /dev/null
|
|
||||||
COPY --from=css-check /out/tailwind.css /dev/null
|
|
||||||
COPY --from=markdown-check /src/yarn.lock /dev/null
|
|
||||||
|
|
||||||
# A build context sent as a tar archive keeps its files' owners, and git
|
# jq is a runtime dependency of script/ci-mark-superseded, which the test
|
||||||
# refuses to read a checkout owned by another user. Trust this one
|
# suite executes.
|
||||||
# whoever owns it.
|
RUN apt-get update && apt-get install -y --no-install-recommends make curl ca-certificates jq && rm -rf /var/lib/apt/lists/*
|
||||||
RUN git config --system --add safe.directory /build
|
|
||||||
|
|
||||||
WORKDIR /build
|
WORKDIR /build
|
||||||
|
|
||||||
@@ -187,38 +47,48 @@ WORKDIR /build
|
|||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
RUN go mod download
|
RUN go mod download
|
||||||
|
|
||||||
|
# Copy source code, including the .ci-fingerprint cache barrier described in
|
||||||
|
# the lint stage above.
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
# Version stamped into the binary: the VERSION build arg when one is
|
# Fetch the third-party browser assets the UI serves. They are not committed
|
||||||
# given, otherwise what script/version derives from the .git the build
|
# (REPO_POLICIES.md forbids minified bundles in version control) and
|
||||||
# context carries, so any `docker build .` of a clone stamps its commit.
|
# .dockerignore keeps any host copy out of the build context, so this step is
|
||||||
# With neither, as from a source tarball, it is "unknown".
|
# the only way they enter the image. Each download is checked against a
|
||||||
ARG VERSION
|
# hardcoded sha256 and the build fails on mismatch; make test re-checks the
|
||||||
|
# hashes against the bytes go:embed actually put in the binary.
|
||||||
|
RUN script/fetch-assets
|
||||||
|
|
||||||
# A context that carries .git must not stamp an empty version, "dev" or
|
# Run tests and build
|
||||||
# "unknown": that means git is missing here or could not read the
|
RUN make test
|
||||||
# checkout, and the image could not be traced back to its commit.
|
|
||||||
RUN version="$(make version VERSION="$VERSION")"; \
|
# Version stamped into the binary. .dockerignore excludes .git/, so
|
||||||
if [ -e .git ]; then \
|
# nothing in this stage can derive it: script/docker resolves it on the
|
||||||
case "$version" in ""|dev|unknown) \
|
# host and passes it in. The default is what a bare `docker build .`
|
||||||
echo "version is '$version' although .git is present" >&2; \
|
# with no --build-arg gets, and it names no tag the tree may not be at.
|
||||||
exit 1 ;; \
|
#
|
||||||
esac; \
|
# Declared here, below the test and asset steps, so a changed version
|
||||||
fi
|
# does not invalidate their cached layers.
|
||||||
|
ARG VERSION=unknown
|
||||||
|
|
||||||
# Builds through the Makefile's build target, which runs script/assets
|
|
||||||
# (Alpine.js, extracted from its tarball in 3p/) first. The target builds
|
|
||||||
# with cgo off, so the binary is statically linked and runs on the Alpine
|
|
||||||
# runtime image below as it is.
|
|
||||||
RUN make build VERSION="$VERSION"
|
RUN make build VERSION="$VERSION"
|
||||||
|
|
||||||
|
# Rebuild with static linking for Alpine runtime.
|
||||||
|
# make build already verified compilation.
|
||||||
|
# The CGO binary from `make build` is dynamically linked against glibc,
|
||||||
|
# which doesn't exist on Alpine (musl). Rebuild with static linking so
|
||||||
|
# the binary runs on Alpine without glibc.
|
||||||
|
#
|
||||||
|
# The static flags go in through GO_LDFLAGS rather than a -ldflags of
|
||||||
|
# their own: the build target composes them with the -X that stamps the
|
||||||
|
# version, so this relink cannot silently drop the stamp.
|
||||||
|
RUN CGO_ENABLED=1 make build VERSION="$VERSION" GO_LDFLAGS='-extldflags "-static"'
|
||||||
|
|
||||||
# Runtime stage
|
# Runtime stage
|
||||||
# alpine:3.21, 2026-03-17
|
# alpine:3.21, 2026-03-17
|
||||||
FROM alpine:3.21@sha256:c3f8e73fdb79deaebaa2037150150191b9dcbfba68b4a46d70103204c53f4709
|
FROM alpine:3.21@sha256:c3f8e73fdb79deaebaa2037150150191b9dcbfba68b4a46d70103204c53f4709
|
||||||
|
|
||||||
# su-exec 0.2-r3 (Alpine 3.21), 2026-09-29: the entrypoint runs the app
|
RUN apk --no-cache add ca-certificates
|
||||||
# as webhooker with it.
|
|
||||||
RUN apk --no-cache add ca-certificates su-exec=0.2-r3
|
|
||||||
|
|
||||||
# Create non-root user
|
# Create non-root user
|
||||||
RUN addgroup -g 1000 -S webhooker && \
|
RUN addgroup -g 1000 -S webhooker && \
|
||||||
@@ -229,17 +99,13 @@ WORKDIR /app
|
|||||||
# Copy binary from builder
|
# Copy binary from builder
|
||||||
COPY --from=builder /build/bin/webhooker /app/webhooker
|
COPY --from=builder /build/bin/webhooker /app/webhooker
|
||||||
|
|
||||||
# Not under /app, which belongs to webhooker: this script runs as root.
|
|
||||||
COPY deploy/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
|
|
||||||
|
|
||||||
# Create data directory for all SQLite databases (main app DB +
|
# Create data directory for all SQLite databases (main app DB +
|
||||||
# per-webhook event DBs). DATA_DIR defaults to /var/lib/webhooker.
|
# per-webhook event DBs). DATA_DIR defaults to /var/lib/webhooker.
|
||||||
RUN mkdir -p /var/lib/webhooker
|
RUN mkdir -p /var/lib/webhooker
|
||||||
|
|
||||||
RUN chown -R webhooker:webhooker /app /var/lib/webhooker
|
RUN chown -R webhooker:webhooker /app /var/lib/webhooker
|
||||||
|
|
||||||
# No USER: the entrypoint starts as root to make the data directory
|
USER webhooker
|
||||||
# webhooker's, then runs the app as webhooker.
|
|
||||||
|
|
||||||
EXPOSE 8080
|
EXPOSE 8080
|
||||||
|
|
||||||
@@ -255,17 +121,7 @@ EXPOSE 8080
|
|||||||
# port.
|
# port.
|
||||||
ENV BIND_ADDRESS=0.0.0.0
|
ENV BIND_ADDRESS=0.0.0.0
|
||||||
|
|
||||||
# The probe requests the container's own address, not loopback, so a
|
|
||||||
# listener that answers only on loopback (an empty or loopback
|
|
||||||
# BIND_ADDRESS) fails it. On a network with IPv4, hostname -i prints the
|
|
||||||
# container's IPv4 address, which 0.0.0.0 and :: both answer. On an
|
|
||||||
# IPv6-only network it prints none, and the probe requests the IPv6
|
|
||||||
# address /etc/hosts lists under the container's hostname instead, which
|
|
||||||
# only :: answers.
|
|
||||||
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
|
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
|
||||||
CMD addr="$(hostname -i 2>/dev/null | cut -d ' ' -f 1)"; \
|
CMD wget --no-verbose --tries=1 --spider http://localhost:8080/.well-known/healthcheck || exit 1
|
||||||
[ -n "$addr" ] || addr="[$(getent hosts "$(hostname)" | cut -d ' ' -f 1)]"; \
|
|
||||||
wget --no-verbose --tries=1 --spider "http://$addr:8080/.well-known/healthcheck" || exit 1
|
|
||||||
|
|
||||||
ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
|
|
||||||
CMD ["/app/webhooker"]
|
CMD ["/app/webhooker"]
|
||||||
|
|||||||
@@ -1,29 +0,0 @@
|
|||||||
# Browser test image, built by script/test-browser (make test-browser). It
|
|
||||||
# runs the test in internal/server that loads the pages in a headless
|
|
||||||
# browser under the real Content-Security-Policy. That test is built only
|
|
||||||
# with the browser build tag, so make test leaves it out. Here the browser
|
|
||||||
# comes from a digest-pinned image, and if it is missing the test fails.
|
|
||||||
|
|
||||||
# golang:1.26.1-bookworm, 2026-03-17: the builder stage's image in Dockerfile.
|
|
||||||
FROM golang:1.26.1-bookworm@sha256:4465644228bc2857a954b092167e12aa59c006a3492282a6c820bf4755fd64a4 AS build
|
|
||||||
|
|
||||||
WORKDIR /src
|
|
||||||
|
|
||||||
COPY go.mod go.sum ./
|
|
||||||
RUN go mod download
|
|
||||||
|
|
||||||
COPY . .
|
|
||||||
|
|
||||||
# The test binary embeds the templates and static files, so the browser
|
|
||||||
# stage needs nothing else. -p 4 keeps the compile's memory down, as in
|
|
||||||
# the test phase of Dockerfile.
|
|
||||||
RUN make assets && go test -c -p 4 -tags browser -o /browser.test ./internal/server
|
|
||||||
|
|
||||||
# chromedp/headless-shell:151.0.7922.109 (Debian trixie), 2026-08-11. The
|
|
||||||
# browser is on PATH as headless-shell, where the test's browser library
|
|
||||||
# looks for it.
|
|
||||||
FROM chromedp/headless-shell:151.0.7922.109@sha256:2d349b544a1ea6b5b5fd7c0fe99215ff662339c57407ee2e8c0a11af93516b04 AS browser
|
|
||||||
|
|
||||||
COPY --from=build /browser.test /browser.test
|
|
||||||
|
|
||||||
RUN /browser.test -test.v -test.timeout 90s -test.run '^TestAlpineRunsUnderTheSecurityPolicy$'
|
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
# Lint-only image, built by script/lint. golangci-lint is never installed on
|
||||||
|
# the host: the repo is COPYed into the pinned image and linted as a build
|
||||||
|
# step, so a successful build IS a clean lint. This works even when the docker
|
||||||
|
# daemon is remote and bind mounts are impossible.
|
||||||
|
#
|
||||||
|
# script/lint passes --no-cache-filter=lint. Without it an unchanged tree
|
||||||
|
# replays the lint stage from cache and the build succeeds in under a second
|
||||||
|
# having run no linter at all. Do not drop that flag.
|
||||||
|
#
|
||||||
|
# The lint steps run with --network=none. `golangci-lint config verify` is
|
||||||
|
# documented as fetching its JSON schema over HTTPS, which would make linting
|
||||||
|
# depend on an unpinned remote artifact; this pinned image resolves the schema
|
||||||
|
# without any network, and --network=none enforces that rather than trusting
|
||||||
|
# it. It also proves no linter reaches out at analysis time. If a future image
|
||||||
|
# bump makes either step need the network, this build fails loudly instead of
|
||||||
|
# quietly acquiring an unpinned dependency.
|
||||||
|
|
||||||
|
# golangci/golangci-lint:v2.12.2 (Debian-based), 2026-08-07
|
||||||
|
# Using Debian-based image because mattn/go-sqlite3 (CGO) does not
|
||||||
|
# compile on Alpine musl (off64_t is a glibc type).
|
||||||
|
FROM golangci/golangci-lint:v2.12.2@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS deps
|
||||||
|
|
||||||
|
WORKDIR /src
|
||||||
|
|
||||||
|
# Copy go mod files first for better layer caching. This stage is cacheable;
|
||||||
|
# only the lint stage below is forced to re-execute.
|
||||||
|
COPY go.mod go.sum ./
|
||||||
|
RUN go mod download
|
||||||
|
|
||||||
|
FROM deps AS lint
|
||||||
|
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
# `run` silently ignores config keys it does not recognize, so a typo would
|
||||||
|
# disable a setting without a word. `config verify` is what catches that.
|
||||||
|
RUN --network=none golangci-lint config verify --config .golangci.yml
|
||||||
|
RUN --network=none golangci-lint run --config .golangci.yml ./...
|
||||||
@@ -1,21 +1,26 @@
|
|||||||
.PHONY: bootstrap setup assets test test-browser lint fmt fmt-check check build run dev deps docker clean hooks css css-check version
|
.PHONY: bootstrap setup assets test lint fmt fmt-check check build run dev deps docker clean hooks css version
|
||||||
|
|
||||||
# Default target
|
# Default target
|
||||||
.DEFAULT_GOAL := check
|
.DEFAULT_GOAL := check
|
||||||
|
|
||||||
# Version stamped into the binary. Derived from git by script/version;
|
# Version stamped into the binary. Derived from git by script/version;
|
||||||
# override it (`make build VERSION=v1.2.3`) to stamp a given value, which is
|
# override it (`make build VERSION=v1.2.3`) where git metadata is
|
||||||
# how the Dockerfile passes its build arg in.
|
# unavailable, which is how the Dockerfile passes its build arg in.
|
||||||
VERSION ?= $(shell script/version)
|
VERSION ?= $(shell script/version)
|
||||||
|
|
||||||
# An empty override (`make build VERSION=`, or the Dockerfile's `make build
|
# An empty override (`make build VERSION=`, or a `--build-arg VERSION=`
|
||||||
# VERSION="$VERSION"` when no VERSION build arg was given) means unset,
|
# landing on the Dockerfile's `make build VERSION="$VERSION"`) means unset,
|
||||||
# exactly as it does in script/version -- stamping "" would leave the binary
|
# exactly as it does in script/version -- stamping "" would leave the binary
|
||||||
# reporting no version and the footer back on its "dev" fallback. `override`
|
# reporting no version and the footer back on its "dev" fallback. `override`
|
||||||
# is required: a plain assignment loses to the command-line definition it
|
# is required: a plain assignment loses to the command-line definition it
|
||||||
# exists to correct.
|
# exists to correct.
|
||||||
override VERSION := $(or $(strip $(VERSION)),$(shell script/version))
|
override VERSION := $(or $(strip $(VERSION)),$(shell script/version))
|
||||||
|
|
||||||
|
# Extra linker flags for the build target. The static relink in the
|
||||||
|
# Dockerfile adds -extldflags here rather than passing its own -ldflags,
|
||||||
|
# so composing flags cannot drop the version stamp.
|
||||||
|
GO_LDFLAGS ?=
|
||||||
|
|
||||||
bootstrap:
|
bootstrap:
|
||||||
@script/bootstrap
|
@script/bootstrap
|
||||||
|
|
||||||
@@ -23,14 +28,11 @@ setup:
|
|||||||
@script/setup
|
@script/setup
|
||||||
|
|
||||||
assets:
|
assets:
|
||||||
@script/assets
|
@script/fetch-assets
|
||||||
|
|
||||||
test:
|
test:
|
||||||
@script/test
|
@script/test
|
||||||
|
|
||||||
test-browser:
|
|
||||||
@script/test-browser
|
|
||||||
|
|
||||||
lint:
|
lint:
|
||||||
@script/lint
|
@script/lint
|
||||||
|
|
||||||
@@ -43,17 +45,13 @@ fmt-check:
|
|||||||
check:
|
check:
|
||||||
@script/check
|
@script/check
|
||||||
|
|
||||||
# Built with cgo off, -trimpath and -s -w, as the Go Dockerfile in
|
build:
|
||||||
# REPO_POLICIES.md does: the binary carries no build paths, symbol table or
|
go build -ldflags '$(strip -X main.version=$(VERSION) $(GO_LDFLAGS))' -o bin/webhooker ./cmd/webhooker
|
||||||
# debug information, and on Linux, the image's included, it is statically
|
|
||||||
# linked.
|
|
||||||
build: assets
|
|
||||||
CGO_ENABLED=0 go build -trimpath -ldflags '-s -w -X main.version=$(VERSION)' -o bin/webhooker ./cmd/webhooker
|
|
||||||
|
|
||||||
run: build
|
run: build
|
||||||
./bin/webhooker
|
./bin/webhooker
|
||||||
|
|
||||||
dev: assets
|
dev:
|
||||||
go run ./cmd/webhooker
|
go run ./cmd/webhooker
|
||||||
|
|
||||||
deps:
|
deps:
|
||||||
@@ -73,7 +71,4 @@ hooks:
|
|||||||
@script/install-precommit
|
@script/install-precommit
|
||||||
|
|
||||||
css:
|
css:
|
||||||
@script/css
|
tailwindcss -i static/css/input.css -o static/css/tailwind.css --minify
|
||||||
|
|
||||||
css-check:
|
|
||||||
@script/css-check
|
|
||||||
|
|||||||
+88
-351
@@ -1,6 +1,6 @@
|
|||||||
---
|
---
|
||||||
title: Repository Policies
|
title: Repository Policies
|
||||||
last_modified: 2026-10-04
|
last_modified: 2026-08-07
|
||||||
---
|
---
|
||||||
|
|
||||||
This document covers repository structure, tooling, and workflow standards. Code
|
This document covers repository structure, tooling, and workflow standards. Code
|
||||||
@@ -60,28 +60,17 @@ style conventions are in separate documents:
|
|||||||
prerequisite since nvm requires bash. yarn is then pinned via
|
prerequisite since nvm requires bash. yarn is then pinned via
|
||||||
`corepack prepare yarn@<version> --activate`. Never install "latest" or "lts";
|
`corepack prepare yarn@<version> --activate`. Never install "latest" or "lts";
|
||||||
always exact versions. `script/cibuild` runs the CI build: it changes to the
|
always exact versions. `script/cibuild` runs the CI build: it changes to the
|
||||||
repo root, runs `script/bootstrap`, runs `script/check`, and builds the image
|
repo root and runs `docker build .`; the Gitea workflow calls it. Four further
|
||||||
with the version; the Gitea workflow calls it. **`script/cibuild` runs
|
scripts are our own extensions to the standard: `script/check` runs
|
||||||
`script/bootstrap` first**, because the workflow checks out the repo and runs
|
`script/test`, `script/lint`, and `script/fmt-check`; `script/precommit` is
|
||||||
nothing else, while `script/fmt-check` runs the formatter on the host: on a
|
what the git pre-commit hook runs, and it calls `script/check`;
|
||||||
pristine checkout with nothing installed the run dies there, after the
|
`script/install-precommit` installs the git pre-commit hook (the `make hooks`
|
||||||
containerised gates have passed. **The bootstrap alone is not enough**:
|
target shims to it); and `script/projectname` (literally that filename) simply
|
||||||
`script/bootstrap` installs node and yarn under nvm and leaves neither on the
|
outputs the project's name. Scripts that need the name call
|
||||||
`PATH` of the shell that called it, so a bare `yarn` still exits 127. The host
|
`script/projectname` — e.g. `script/docker` assembles its image tag from it —
|
||||||
entrypoints that need yarn — `script/fmt` and `script/fmt-check` — therefore
|
so those scripts stay byte-identical across all repos. Repo-type-specific
|
||||||
source nvm for the pinned node version before invoking it, exactly as
|
pre-commit extras (e.g. `go mod tidy` verification in Go repos) belong in
|
||||||
`script/bootstrap`'s own install step does. A runner carrying nothing but
|
`script/precommit`, not in the hook itself. Model scripts are at
|
||||||
docker and git then gets through `script/check`. Four further scripts are our
|
|
||||||
own extensions to the standard: `script/check` runs `script/test`,
|
|
||||||
`script/lint` and `script/fmt-check`; `script/precommit` is what the git
|
|
||||||
pre-commit hook runs, and it calls `script/check`; `script/install-precommit`
|
|
||||||
installs the git pre-commit hook (the `make hooks` target shims to it); and
|
|
||||||
`script/projectname` (literally that filename) simply outputs the project's
|
|
||||||
name. Scripts that need the name call `script/projectname` — e.g.
|
|
||||||
`script/docker` assembles its image tag from it — so those scripts stay
|
|
||||||
byte-identical across all repos. Repo-type-specific pre-commit extras (e.g.
|
|
||||||
`go mod tidy` verification in Go repos) belong in `script/precommit`, not in
|
|
||||||
the hook itself. Model scripts are at
|
|
||||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/script/<name>`. The README
|
`https://git.eeqj.de/sneak/prompts/raw/branch/main/script/<name>`. The README
|
||||||
must document the provided scripts in an **Entrypoints** section (see the
|
must document the provided scripts in an **Entrypoints** section (see the
|
||||||
README requirements below).
|
README requirements below).
|
||||||
@@ -100,198 +89,87 @@ style conventions are in separate documents:
|
|||||||
contributor should be able to understand the entire development workflow by
|
contributor should be able to understand the entire development workflow by
|
||||||
reading the Makefile.
|
reading the Makefile.
|
||||||
|
|
||||||
- Every repo should have a `Dockerfile`, and it carries the repo's gates: a
|
- Every repo should have a `Dockerfile`. All Dockerfiles must run `make check`
|
||||||
`lint` phase and a `test` phase, with the final stage depending on both so the
|
as a build step so the build fails if the branch is not green. For non-server
|
||||||
image cannot be built unless they pass. For non-server repos the final stage
|
repos, the Dockerfile should bring up a development environment and run
|
||||||
brings up a development environment; for server repos it is the runtime image.
|
`make check`. For server repos, `make check` should run as an early build
|
||||||
The gate phases and the build stage start from their pinned base images and
|
stage before the final image is assembled. Dockerfiles install development
|
||||||
install what those images lack either inline, as the canonical Go `Dockerfile`
|
prerequisites by running `script/bootstrap` rather than duplicating installs
|
||||||
below does for `git`, or by running `script/bootstrap`, as the `prompts`
|
inline; COPY `script/` and the dependency manifests (`package.json` +
|
||||||
repo's own `Dockerfile` does for its yarn packages. The development
|
`yarn.lock`, `go.mod` + `go.sum`, etc.) before running it so the bootstrap
|
||||||
environment stage installs development prerequisites by running
|
layer stays cached until dependencies change.
|
||||||
`script/bootstrap` rather than duplicating its installs inline. A stage that
|
|
||||||
runs `script/bootstrap` COPYs `script/` and the dependency manifests
|
|
||||||
(`package.json` + `yarn.lock`, `go.mod` + `go.sum`, etc.) before running it.
|
|
||||||
|
|
||||||
- **Linting and testing run in Docker, as phases of the `Dockerfile`.** There is
|
- **Dockerfiles must use a separate lint stage for fail-fast feedback.** Go
|
||||||
no separate lint file. `script/lint` and `script/test` each build one phase
|
repos use a multistage build where linting runs in an independent stage based
|
||||||
and nothing else:
|
on the `golangci/golangci-lint` image (pinned by hash). This stage runs
|
||||||
|
`make fmt-check` and `make lint` before the full build begins. The build stage
|
||||||
|
then declares an explicit dependency on the lint stage via
|
||||||
|
`COPY --from=lint /src/go.sum /dev/null`, which forces BuildKit to complete
|
||||||
|
linting before proceeding to compilation and tests. This ensures lint failures
|
||||||
|
surface in seconds rather than minutes, without blocking on dependency
|
||||||
|
download or compilation in the build stage.
|
||||||
|
|
||||||
```sh
|
The standard pattern for a Go repo Dockerfile is:
|
||||||
docker build --no-cache --target lint -t "$(script/projectname)-lint" .
|
|
||||||
docker build --no-cache --target test -t "$(script/projectname)-test" .
|
|
||||||
```
|
|
||||||
|
|
||||||
**A stage that is not the last one in the file is built only when the final
|
|
||||||
stage's chain depends on it, or when `--target` names it.** That is why the
|
|
||||||
two gates are always invoked by name here, and why the final stage carries a
|
|
||||||
`COPY --from=` of a harmless file from each of them: without that edge a
|
|
||||||
plain `docker build .` builds the last stage alone and exits 0 having linted
|
|
||||||
and tested nothing.
|
|
||||||
|
|
||||||
**Every `docker build` in `script/` is tagged**, here and in
|
|
||||||
`script/cibuild` and `script/docker`. An untagged build leaves a dangling
|
|
||||||
image behind on every invocation, on every developer host and every CI
|
|
||||||
runner; a tagged one replaces the previous image.
|
|
||||||
|
|
||||||
Inside a phase the tool is invoked directly — `golangci-lint`, `go test`,
|
|
||||||
`eslint`, `prettier` — never through `make lint` or `script/test`, which are
|
|
||||||
themselves a `docker build` and would recurse into a daemon that does not
|
|
||||||
exist in a build step. Formatting is the exception and stays on the host:
|
|
||||||
`script/fmt` writes the working tree, and `script/fmt-check` is its
|
|
||||||
read-only twin.
|
|
||||||
|
|
||||||
**No lint verdict may come from a host invocation of the linter.** On a
|
|
||||||
shared host golangci-lint reads a result cache keyed on file content rather
|
|
||||||
than location, so a second checkout of the same content is served the first
|
|
||||||
one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs
|
|
||||||
exit non-zero with `parallel golangci-lint is running` — a status a caller
|
|
||||||
cannot tell from real findings. Both have produced wrong verdicts in this
|
|
||||||
org, in both directions. A container has its own cache, its own `TMPDIR` and
|
|
||||||
a digest-pinned binary, so neither is reachable.
|
|
||||||
|
|
||||||
- **Any build that runs checks is built with `--no-cache`.** Docker invalidates
|
|
||||||
a `COPY` layer only when the copied content changes, so on an unchanged tree
|
|
||||||
the check `RUN` is served from cache, nothing executes, and the build still
|
|
||||||
exits 0. Every `docker build` in `script/` therefore passes `--no-cache`:
|
|
||||||
`script/lint`, `script/test`, `script/cibuild` and `script/docker` are the
|
|
||||||
four, and there is no fifth — `script/check` runs the two gate phases and
|
|
||||||
`script/fmt-check`, and builds no image of its own. A bare `docker build .` is
|
|
||||||
not evidence that anything ran: a sub-second build reporting success is a
|
|
||||||
cache hit, not a result. Never invalidate by pruning — `docker builder prune`
|
|
||||||
and friends destroy a build cache shared with every other build on the host.
|
|
||||||
When a check is added or changed, prove it works by planting a defect it must
|
|
||||||
catch and watching the run fail on it, then revert the defect. A green run
|
|
||||||
alone shows neither that the check ran nor that it covers what it should.
|
|
||||||
|
|
||||||
- **The gate phases are separate stages, and the build stage depends on both.**
|
|
||||||
The lint phase is based on the `golangci/golangci-lint` image (pinned by
|
|
||||||
hash), so lint failures surface in seconds rather than after a full compile,
|
|
||||||
and the test phase is based on the Debian Go image. The canonical Go repo
|
|
||||||
`Dockerfile`:
|
|
||||||
|
|
||||||
```dockerfile
|
```dockerfile
|
||||||
# Lint phase
|
# Lint stage — fast feedback on formatting and lint issues
|
||||||
# golangci/golangci-lint:v2.x.x, YYYY-MM-DD
|
# golangci/golangci-lint:v2.x.x, YYYY-MM-DD
|
||||||
FROM golangci/golangci-lint@sha256:... AS lint
|
FROM golangci/golangci-lint@sha256:... AS lint
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
RUN go mod download
|
RUN go mod download
|
||||||
COPY . .
|
COPY . .
|
||||||
RUN golangci-lint run --config .golangci.yml ./...
|
RUN make fmt-check
|
||||||
|
RUN make lint
|
||||||
|
|
||||||
# Test phase. -race needs cgo and so a C compiler, which the Debian Go
|
# Build stage
|
||||||
# image ships and the alpine one does not.
|
|
||||||
# golang:1.x, YYYY-MM-DD
|
|
||||||
FROM golang@sha256:... AS test
|
|
||||||
WORKDIR /src
|
|
||||||
COPY go.mod go.sum ./
|
|
||||||
RUN go mod download
|
|
||||||
COPY . .
|
|
||||||
RUN go test -timeout 90s -race -cover ./... || \
|
|
||||||
{ echo "--- Rerunning with -v for details ---"; \
|
|
||||||
go test -timeout 90s -race -v ./...; exit 1; }
|
|
||||||
|
|
||||||
# Build stage. Nothing is wanted from either phase above; the copies
|
|
||||||
# are what make BuildKit build them first, so this stage cannot run
|
|
||||||
# unless lint and test passed.
|
|
||||||
# golang:1.x-alpine, YYYY-MM-DD
|
# golang:1.x-alpine, YYYY-MM-DD
|
||||||
FROM golang@sha256:... AS builder
|
FROM golang@sha256:... AS builder
|
||||||
COPY --from=lint /src/go.sum /dev/null
|
|
||||||
COPY --from=test /src/go.sum /dev/null
|
|
||||||
RUN apk add --no-cache git
|
|
||||||
# A tar-stream context keeps the sender's file owners, which git refuses.
|
|
||||||
RUN git config --system --add safe.directory /src
|
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
|
|
||||||
|
# Force BuildKit to run the lint stage before proceeding
|
||||||
|
COPY --from=lint /src/go.sum /dev/null
|
||||||
|
|
||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
RUN go mod download
|
RUN go mod download
|
||||||
COPY . .
|
COPY . .
|
||||||
|
RUN make test
|
||||||
|
|
||||||
# The VERSION build arg when one is given, otherwise
|
ARG VERSION=dev
|
||||||
# `git describe --tags --always` on the .git in the build context. With
|
RUN CGO_ENABLED=0 go build -trimpath \
|
||||||
# .git present, a version that is still empty, dev or unknown fails the
|
-ldflags="-s -w -X main.Version=${VERSION}" \
|
||||||
# build: git is missing or could not read the checkout.
|
-o /app ./cmd/app/
|
||||||
ARG VERSION
|
|
||||||
RUN VERSION="${VERSION:-$(git describe --tags --always)}"; \
|
|
||||||
if [ -e .git ]; then \
|
|
||||||
case "$VERSION" in ""|dev|unknown) \
|
|
||||||
echo "version is '$VERSION' although .git is present" >&2; \
|
|
||||||
exit 1 ;; \
|
|
||||||
esac; \
|
|
||||||
fi; \
|
|
||||||
CGO_ENABLED=0 go build -trimpath \
|
|
||||||
-ldflags="-s -w -X main.Version=${VERSION}" \
|
|
||||||
-o /app ./cmd/app/
|
|
||||||
|
|
||||||
# Runtime stage, and the last one
|
# Runtime stage
|
||||||
FROM alpine@sha256:...
|
FROM alpine@sha256:...
|
||||||
COPY --from=builder /app /usr/local/bin/app
|
COPY --from=builder /app /usr/local/bin/app
|
||||||
ENTRYPOINT ["app"]
|
ENTRYPOINT ["app"]
|
||||||
```
|
```
|
||||||
|
|
||||||
Key points:
|
Key points:
|
||||||
- The lint phase uses the `golangci/golangci-lint` image directly (it has
|
- The lint stage uses the `golangci/golangci-lint` image directly (it
|
||||||
both Go and the linter), so nothing needs installing.
|
includes both Go and the linter), so there is no need to install the
|
||||||
- `COPY --from=<phase> /src/go.sum /dev/null` is a no-op copy whose only
|
linter separately.
|
||||||
purpose is the ordering edge. BuildKit runs stages in parallel by default,
|
- `COPY --from=lint /src/go.sum /dev/null` is a no-op file copy that creates
|
||||||
and a stage nothing depends on is not built at all, so without these two
|
a stage dependency. BuildKit runs stages in parallel by default; without
|
||||||
lines a red gate would not fail the build.
|
this line, the build stage would not wait for lint to finish and a lint
|
||||||
- Keep the runtime stage last, and if you add a stage after it, give it the
|
failure might not fail the overall build.
|
||||||
same two copies. A plain `docker build .` builds the last stage's chain
|
|
||||||
and nothing else.
|
|
||||||
- If the project uses `//go:embed` directives that reference build artifacts
|
- If the project uses `//go:embed` directives that reference build artifacts
|
||||||
(e.g. a web frontend compiled in a separate stage), the lint phase must
|
(e.g. a web frontend compiled in a separate stage), the lint stage must
|
||||||
create placeholder files so the embed directives resolve. Example:
|
create placeholder files so the embed directives resolve. Example:
|
||||||
`RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`.
|
`RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`.
|
||||||
- If the project requires CGO or system libraries for linting, install them
|
The lint stage should not depend on the actual build output — it exists to
|
||||||
in the lint phase. The `golangci/golangci-lint` image is Debian-based and
|
fail fast.
|
||||||
has no `apk`, so install with `apt-get` under the Debian package name
|
- If the project requires CGO or system libraries for linting (e.g.
|
||||||
(`libvips-dev`, where alpine says `vips-dev`), and delete the package
|
`vips-dev`), install them in the lint stage with `apk add`.
|
||||||
lists in the same `RUN`, so the layer does not keep them:
|
- The build stage runs `make test` after compilation setup. Tests run in the
|
||||||
|
build stage, not the lint stage, because they may require compiled
|
||||||
```dockerfile
|
artifacts or heavier dependencies.
|
||||||
RUN apt-get update \
|
|
||||||
&& apt-get install -y --no-install-recommends libvips-dev \
|
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
|
||||||
```
|
|
||||||
|
|
||||||
- `.dockerignore` lets `.git` into the build context. It keeps out every git
|
|
||||||
`config` at any depth (`**/.git/config`, `**/.git/modules/**/config`): the
|
|
||||||
repository's own, each submodule's under `.git/modules/`, and that of a
|
|
||||||
submodule keeping its own `.git` directory. `git describe` does not need
|
|
||||||
them, and each can hold a credential: a password in a remote URL, or the
|
|
||||||
token the CI checkout step stores there. A submodule whose name has a
|
|
||||||
`config` segment (`config`, `deploy/config`, `config/lib`) loses its whole
|
|
||||||
git directory to `**/.git/modules/**/config`, and Go's version stamping
|
|
||||||
then fails the build: give it a name without that segment
|
|
||||||
(`git submodule add --name`). The stage that compiles has `git` (the
|
|
||||||
Debian Go image has it; an alpine one needs `apk add --no-cache git`) and
|
|
||||||
takes the version from the `VERSION` build argument when one is given,
|
|
||||||
otherwise from `git describe --tags --always`. That gives the tag on a
|
|
||||||
tagged commit; on a later commit, the tag, the number of commits since it
|
|
||||||
and the short commit (`v1.2.3-4-gabc1234`); and the short commit when no
|
|
||||||
tag is reachable. The stage that compiles also marks its working directory
|
|
||||||
safe for git (`git config --system --add safe.directory /src`): a context
|
|
||||||
sent as a tar stream keeps the sender's file owners, and git refuses a
|
|
||||||
checkout owned by another user, so the version would come out empty.
|
|
||||||
`ARG VERSION` has no default, and the build fails if the context carries
|
|
||||||
`.git` and the version still comes out empty, `dev` or `unknown`. A plain
|
|
||||||
`docker build .` with no build arguments must succeed; a Dockerfile that
|
|
||||||
refuses an empty build argument drops that refusal and keeps the argument.
|
|
||||||
|
|
||||||
- Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that
|
- Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that
|
||||||
runs `script/cibuild` on push, and checks out the repo as its only other step.
|
runs `script/cibuild` (which runs `docker build .`) on push. Since the
|
||||||
That script bootstraps, runs the gate phases, and then builds the image, so a
|
Dockerfile already runs `make check`, a successful build implies all checks
|
||||||
successful run means every check passed; a bare `docker build .` does not
|
pass.
|
||||||
carry the same guarantee, because its gate phases may come from the cache. The
|
|
||||||
image build is uncached and so runs the gate phases a second time. That is the
|
|
||||||
price of the rule above, and it is worth paying: the image that ships is built
|
|
||||||
from a run of its own gates rather than from a cache entry. A separate
|
|
||||||
workflow limited to `main` by a `branches` list under `on: push` cannot be
|
|
||||||
checked by review: to try a change to it, add the feature branch to that list
|
|
||||||
and push, then remove the branch from the list again before merging. Keep any
|
|
||||||
job in it that publishes behind `if: github.ref_name == 'main'`, so the run
|
|
||||||
from the feature branch publishes nothing.
|
|
||||||
|
|
||||||
- Use platform-standard formatters: `black` for Python, `prettier` for
|
- Use platform-standard formatters: `black` for Python, `prettier` for
|
||||||
JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with
|
JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with
|
||||||
@@ -315,17 +193,15 @@ style conventions are in separate documents:
|
|||||||
suite that exceeds it fails. Under 20 seconds is the target. A suite between
|
suite that exceeds it fails. Under 20 seconds is the target. A suite between
|
||||||
20 and 60 seconds is still green, but the overage must be filed as an
|
20 and 60 seconds is still green, but the overage must be filed as an
|
||||||
improvement bug against that repo. Add a 90-second timeout to the test
|
improvement bug against that repo. Add a 90-second timeout to the test
|
||||||
invocation (`go test -timeout 90s`). The backstop deliberately sits above the
|
invocation in the Makefile (`go test -timeout 90s`). The backstop deliberately
|
||||||
hard cap so that it catches a genuinely hung test rather than a merely slow
|
sits above the hard cap so that it catches a genuinely hung test rather than a
|
||||||
one.
|
merely slow one.
|
||||||
|
|
||||||
- **The test command should use the conditional verbose rerun pattern.** Run
|
- **`make test` should use the conditional verbose rerun pattern.** Run tests
|
||||||
tests without `-v` (verbose) first. If tests fail, automatically rerun with
|
without `-v` (verbose) first. If tests fail, automatically rerun with `-v` to
|
||||||
`-v` to show full output. This keeps CI logs and `docker build` output clean
|
show full output. This keeps CI logs and `docker build` output clean on
|
||||||
on success (just package/suite summaries) while providing full diagnostic
|
success (just package/suite summaries) while providing full diagnostic detail
|
||||||
detail on failure (every test case, every assertion). The command lives in the
|
on failure (every test case, every assertion). The general shell pattern:
|
||||||
`test` phase of the `Dockerfile`, since `script/test` builds that phase; the
|
|
||||||
Makefile form below is the same pattern for any repo-local invocation:
|
|
||||||
|
|
||||||
```makefile
|
```makefile
|
||||||
test:
|
test:
|
||||||
@@ -338,26 +214,11 @@ style conventions are in separate documents:
|
|||||||
|
|
||||||
```makefile
|
```makefile
|
||||||
test:
|
test:
|
||||||
@go test -count=1 -timeout 90s -race -cover ./... || \
|
@go test -timeout 90s -race -cover ./... || \
|
||||||
{ echo "--- Rerunning with -v for details ---"; \
|
{ echo "--- Rerunning with -v for details ---"; \
|
||||||
go test -count=1 -timeout 90s -race -v ./...; exit 1; }
|
go test -timeout 90s -race -v ./...; exit 1; }
|
||||||
```
|
```
|
||||||
|
|
||||||
`-count=1` is required on both invocations: it defeats Go's test _result_
|
|
||||||
cache, so neither run can report a stored pass in place of running the
|
|
||||||
tests. It leaves the build cache alone, so it costs the runtime of the suite
|
|
||||||
and no recompilation.
|
|
||||||
|
|
||||||
That cache is Go's own, separate from Docker's layer cache. Go stores a
|
|
||||||
passing result in its cache directory (`GOCACHE`), and when the same tests
|
|
||||||
run again on unchanged code it prints that result, marked `(cached)`,
|
|
||||||
without running them. That matters on a developer's machine, where this
|
|
||||||
target runs and the directory lasts from one run to the next. The `test`
|
|
||||||
phase of the `Dockerfile` needs no `-count=1`: its base image holds no
|
|
||||||
result for this repo's tests and nothing before its `go test` step runs a
|
|
||||||
test, so there is nothing to replay. `--no-cache` (above) is what makes that
|
|
||||||
step run on an unchanged tree.
|
|
||||||
|
|
||||||
Python example:
|
Python example:
|
||||||
|
|
||||||
```makefile
|
```makefile
|
||||||
@@ -383,84 +244,10 @@ style conventions are in separate documents:
|
|||||||
must be in `.gitignore`. No exceptions.
|
must be in `.gitignore`. No exceptions.
|
||||||
|
|
||||||
- `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`),
|
- `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`),
|
||||||
editor files (`.swp`, `*~`), in-repo agent scratch directories (`.claude/`),
|
editor files (`.swp`, `*~`), language build artifacts, and `node_modules/`.
|
||||||
language build artifacts, and `node_modules/`. Fetch the standard `.gitignore`
|
Fetch the standard `.gitignore` from
|
||||||
from `https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when
|
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when setting up
|
||||||
setting up a new repo. These patterns are written to `.gitignore`'s own
|
a new repo.
|
||||||
semantics, in which an unanchored pattern already matches at every depth; they
|
|
||||||
are not a `.dockerignore` and must not be transplanted into one unmodified.
|
|
||||||
|
|
||||||
- **`.dockerignore` does not use `.gitignore` semantics, and copying patterns
|
|
||||||
across unmodified leaves secrets in the build context.** Docker matches with
|
|
||||||
`moby/patternmatcher`: `filepath.Match` semantics plus a `**` extension, so
|
|
||||||
`*` does not cross `/` and a pattern without a leading `**/` is anchored at
|
|
||||||
the build-context root. A `.dockerignore` listing `.env`, `*.pem` and `*.key`
|
|
||||||
therefore excludes only the copies at the repository root, while `config/.env`
|
|
||||||
and `certs/server.key` still reach the context and can land in an image layer
|
|
||||||
— which is more dangerous than a short file with no secret patterns at all,
|
|
||||||
because it reads as solved and stops anyone looking. Give every
|
|
||||||
depth-independent pattern the `**/` prefix and leave only genuinely
|
|
||||||
root-anchored entries unprefixed: `.claude`, and the repo's own host-built
|
|
||||||
binary, written `/myapp` and never `**/myapp`, which would also match
|
|
||||||
`cmd/myapp/` and delete the package directory from the context. Matching is
|
|
||||||
case-sensitive, and an ALL-CAPS twin per pattern still misses `Server.Key`, so
|
|
||||||
secret names use character ranges — `**/*.[kK][eE][yY]`, `**/*.[pP][eE][mM]`,
|
|
||||||
and likewise for `.envrc` and the extensionless SSH keys. Where such a pattern
|
|
||||||
also catches something the build needs, re-include it with a negation
|
|
||||||
(`!docs/example.env`); deleting the pattern reopens the exposure for every
|
|
||||||
other file it covers. Fetch the standard `.dockerignore` from
|
|
||||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.dockerignore` and extend
|
|
||||||
it with the repo's own artifacts.
|
|
||||||
|
|
||||||
- **In-repo agent scratch belongs in both files, written to each file's own
|
|
||||||
semantics.** `.claude/` holds one worktree per in-flight agent — an entire
|
|
||||||
additional checkout of the repo — so under `COPY . .` the build context
|
|
||||||
inflates by a multiple of the repo and another session's unreviewed work can
|
|
||||||
be copied into an image layer. In `.gitignore` the entry is `.claude/`,
|
|
||||||
unanchored. In `.dockerignore` it is `.claude`, anchored and with **no** `**/`
|
|
||||||
prefix, because the prefixed form would also delete any nested directory of
|
|
||||||
that name from the build. Anchoring carries a known gap that the canonical
|
|
||||||
`.dockerignore` states in its own comment, since consuming repos receive the
|
|
||||||
file and not the tracker: the directory is created in the agent's working
|
|
||||||
directory, so a repo running agents in subdirectories still ships
|
|
||||||
`services/api/.claude/` and must add its own anchored entry there.
|
|
||||||
|
|
||||||
- **A plain `docker build .` of a clone stamps the version that
|
|
||||||
`git describe --tags --always` gives**, derived from the `.git` in the build
|
|
||||||
context as the canonical `Dockerfile` above shows. Without its failure check,
|
|
||||||
a missing `git` or an unreadable checkout would leave `-X main.Version=` empty
|
|
||||||
and the build would still exit 0. `script/docker` and `script/cibuild` pass
|
|
||||||
the version they compute on the host; it takes precedence. They do this
|
|
||||||
byte-identically across repos:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
# Own line: a failing command substitution inside an argument does not
|
|
||||||
# trip `set -e`, so the inline form degrades to an empty constant.
|
|
||||||
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
|
|
||||||
[ -n "$version" ] || version="unknown"
|
|
||||||
docker build --no-cache \
|
|
||||||
--build-arg VERSION="$version" \
|
|
||||||
-t "$(script/projectname)" .
|
|
||||||
```
|
|
||||||
|
|
||||||
`--always` makes an untagged repo yield an abbreviated commit hash rather
|
|
||||||
than failing, and the `[ -n "$version" ]` line is the single place the
|
|
||||||
fallback is applied — a live check that fires on a build from an export with
|
|
||||||
no `.git` and on a repository with no commits yet. Do not fold it into the
|
|
||||||
substitution as `|| echo unknown`, which makes the guard unreachable. The
|
|
||||||
Dockerfile's side is `ARG VERSION` in the stage that compiles, declared
|
|
||||||
there because `ARG` is stage-scoped; passing `VERSION` to a repo whose
|
|
||||||
Dockerfile declares no such `ARG` is ignored and costs nothing, which is why
|
|
||||||
the scripts stay byte-identical. One consequence for CI: the standard
|
|
||||||
checkout action clones shallow and fetches no tags, so a repo that embeds a
|
|
||||||
tag-derived version must set `fetch-depth: 0` on its checkout step.
|
|
||||||
|
|
||||||
- **Verify `.dockerignore` by enumerating the image, not by reading the
|
|
||||||
patterns.** Plant files at the root _and_ at least two directories deep, build
|
|
||||||
a probe image that does `COPY . .`, and list what actually landed
|
|
||||||
(`docker run --rm --entrypoint find IMAGE /app`). The `transferring context`
|
|
||||||
size is not a substitute: a nested secret is a few bytes, and BuildKit
|
|
||||||
transfers only the delta from the previous build.
|
|
||||||
|
|
||||||
- **No build artifacts in version control.** Code-derived data (compiled
|
- **No build artifacts in version control.** Code-derived data (compiled
|
||||||
bundles, minified output, generated assets) must never be committed to the
|
bundles, minified output, generated assets) must never be committed to the
|
||||||
@@ -476,56 +263,12 @@ style conventions are in separate documents:
|
|||||||
- Make all changes on a feature branch. You can do whatever you want on a
|
- Make all changes on a feature branch. You can do whatever you want on a
|
||||||
feature branch.
|
feature branch.
|
||||||
|
|
||||||
- `.golangci.yml` is standardized. The vendored copy in a consuming repo must
|
- `.golangci.yml` is standardized and must _NEVER_ be modified by an agent, only
|
||||||
_NEVER_ be modified by an agent: fetch it from
|
manually by the user. Fetch from
|
||||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml` and keep it
|
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml`. The
|
||||||
byte-identical, so that no repo can quietly loosen its own linting. Linter
|
canonical golangci-lint version is v2.12.2 (released 2026-05-06), installed
|
||||||
configuration changes are made to the canonical copy in the `prompts` repo and
|
commit-pinned via
|
||||||
reach consuming repos by re-vendoring; an agent may open a PR against
|
`go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@c0d3ddc9cf3faa61a4e378e879ece580256d76e5`.
|
||||||
canonical, which only the user merges. One list is exempt from byte-identity,
|
|
||||||
because it cannot be written once for every repo: the `deny` list of the
|
|
||||||
`test-support` depguard rule, where a repo names its own test-support packages
|
|
||||||
by full import path. A repo adds entries there and changes nothing else, and a
|
|
||||||
re-vendor carries its entries forward. The canonical golangci-lint version is
|
|
||||||
v2.14.0 (released 2026-09-24), pinned as the digest of the lint phase's base
|
|
||||||
image
|
|
||||||
(`golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f`,
|
|
||||||
which reports `2.14.0 built with go1.27.0 from 114493f9`). A module's `go`
|
|
||||||
directive must not name a newer Go minor version than the one golangci-lint
|
|
||||||
was built with, or golangci-lint refuses to lint it: this release lints
|
|
||||||
`go 1.27.1` but not `go 1.28`. That digest is the only pin, since no repo
|
|
||||||
installs golangci-lint on the host. A repo sets the lint phase digest to the
|
|
||||||
one named here and re-vendors `.golangci.yml` in the same commit, whichever of
|
|
||||||
the two prompted the change: the canonical copy can name linters that an older
|
|
||||||
golangci-lint rejects, and a newer golangci-lint can add linters that
|
|
||||||
`default: all` switches on until the canonical copy disables them.
|
|
||||||
|
|
||||||
- **`script/bootstrap` installs a pinned tool by comparing versions, never by
|
|
||||||
testing presence.** An `if ! command -v <tool>; then install; fi` guard tests
|
|
||||||
`PATH` only, so on an already-provisioned machine the pin is inert and a
|
|
||||||
version bump is a silent no-op — while the Dockerfile, installing into a clean
|
|
||||||
image, gets the pinned version, so a local `make check` and `make docker` can
|
|
||||||
disagree about what the tool even is. The canonical form:
|
|
||||||
- compares the installed version against the pin over the **whole** version
|
|
||||||
token; a parser that stops at the first `-` reports `2.12.2` for a host
|
|
||||||
running `2.12.2-rc1` and skips the install;
|
|
||||||
- treats absent, non-zero, empty or unrecognised `--version` output as a
|
|
||||||
mismatch, so the failure direction is a redundant install and never a
|
|
||||||
skipped one;
|
|
||||||
- after installing, re-resolves the binary the way callers do — `hash -r`,
|
|
||||||
then through `PATH`, not through the directory the installer wrote to —
|
|
||||||
and fails naming the resolved path, since an install that a shadowing
|
|
||||||
binary hides succeeds while changing nothing any caller sees;
|
|
||||||
- is actually called, and prints the version on both success paths: a
|
|
||||||
function defined and never invoked has the same exit status and the same
|
|
||||||
empty output as one that worked.
|
|
||||||
|
|
||||||
Keep it POSIX sh: no arrays, no `[[`, no `grep -P`.
|
|
||||||
|
|
||||||
A Go tool a repo needs on the host is installed with `go install` pinned to
|
|
||||||
a commit hash (`go install <package>@<commit hash>`). It is never tracked as
|
|
||||||
a `go.mod` tool dependency or through a `tools.go` file, either of which
|
|
||||||
pulls the tool's own dependencies into the repo's `go.mod` and `go.sum`.
|
|
||||||
|
|
||||||
- When pinning images or packages by hash, add a comment above the reference
|
- When pinning images or packages by hash, add a comment above the reference
|
||||||
with the version and date (YYYY-MM-DD).
|
with the version and date (YYYY-MM-DD).
|
||||||
@@ -639,14 +382,12 @@ style conventions are in separate documents:
|
|||||||
settings.
|
settings.
|
||||||
|
|
||||||
- Avoid putting files in the repo root unless necessary. Root should contain
|
- Avoid putting files in the repo root unless necessary. Root should contain
|
||||||
only project-level config files (`README.md`, `AGENTS.md`, `Makefile`,
|
only project-level config files (`README.md`, `Makefile`, `Dockerfile`,
|
||||||
`Dockerfile`, `LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`,
|
`LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, and
|
||||||
and language-specific config). Everything else goes in a subdirectory.
|
language-specific config). Everything else goes in a subdirectory. Canonical
|
||||||
Canonical subdirectory names:
|
subdirectory names:
|
||||||
- `bin/` — executable scripts and tools
|
- `bin/` — executable scripts and tools
|
||||||
- `cmd/` — Go command entrypoints; thin only: one `main.go` per binary whose
|
- `cmd/` — Go command entrypoints
|
||||||
body is a single call into `internal/` or `pkg/`, no project logic in
|
|
||||||
`cmd/`
|
|
||||||
- `configs/` — configuration templates and examples
|
- `configs/` — configuration templates and examples
|
||||||
- `deploy/` — deployment manifests (k8s, compose, terraform)
|
- `deploy/` — deployment manifests (k8s, compose, terraform)
|
||||||
- `docs/` — documentation and markdown (README.md stays in root)
|
- `docs/` — documentation and markdown (README.md stays in root)
|
||||||
@@ -673,7 +414,3 @@ style conventions are in separate documents:
|
|||||||
- Go: `go.mod`, `go.sum`, `.golangci.yml`
|
- Go: `go.mod`, `go.sum`, `.golangci.yml`
|
||||||
- JS: `package.json`, `yarn.lock`, `.prettierrc`, `.prettierignore`
|
- JS: `package.json`, `yarn.lock`, `.prettierrc`, `.prettierignore`
|
||||||
- Python: `pyproject.toml`
|
- Python: `pyproject.toml`
|
||||||
|
|
||||||
- Guidance for coding agents lives in one `AGENTS.md` at the repository root. It
|
|
||||||
is never committed under a file or directory named after one agent tool, such
|
|
||||||
as `CLAUDE.md` or `.claude/`, and never split into separate memory files.
|
|
||||||
|
|||||||
@@ -2,71 +2,424 @@
|
|||||||
|
|
||||||
One issue per unit of work, one branch and one PR per issue:
|
One issue per unit of work, one branch and one PR per issue:
|
||||||
|
|
||||||
- ensure a tracked issue exists with a definition of done
|
* ensure a tracked issue exists with a definition of done
|
||||||
- branch from `next` (never from `main`)
|
* branch from `next` (never from `main`)
|
||||||
- do the work; open a PR based on `next` (never on `main`)
|
* do the work; open a PR based on `next` (never on `main`)
|
||||||
- pass an independent review, then the manager squash-merges into `next`
|
* pass an independent review, then the manager squash-merges into `next`
|
||||||
- push; nothing stays local-only
|
* push; nothing stays local-only
|
||||||
|
|
||||||
`next` is the branch for the next milestone and must stay green and mergeable to
|
`next` is the branch for the next milestone and must stay green and
|
||||||
`main` without notice. One `next` -> `main` PR accumulates the milestone;
|
mergeable to `main` without notice. One `next` -> `main` PR accumulates
|
||||||
releases are cut from `main` separately.
|
the milestone; releases are cut from `main` separately.
|
||||||
|
|
||||||
The tracker (https://git.eeqj.de/sneak/webhooker/issues) is the record of what
|
Issue branches do NOT touch this file — the manager maintains it on
|
||||||
is done and what is under way. This file keeps no list of either.
|
`next`. Every branch editing `TODO.md` conflicts with every other
|
||||||
|
(#112).
|
||||||
|
|
||||||
# Status
|
# Status
|
||||||
|
|
||||||
What is still open for 1.0.0 is in its milestone
|
The milestone (https://git.eeqj.de/sneak/webhooker/milestone/9) is the
|
||||||
(https://git.eeqj.de/sneak/webhooker/milestone/9). `v1.0.0` is not tagged, and
|
authoritative list, and the only place to read a count or a state of
|
||||||
only @sneak tags it.
|
play from. This file records where the project is, not what is in
|
||||||
|
flight: a sentence whose truth depends on a branch being unmerged is
|
||||||
|
wrong the moment it merges, and this file has been wrong that way
|
||||||
|
before.
|
||||||
|
|
||||||
Delivery is at-least-once by design, not by accident: a send whose result row
|
The durability defect that held the tag has landed
|
||||||
does not land is attempted again, so a receiver can see a duplicate. That is
|
(https://git.eeqj.de/sneak/webhooker/issues/256, commit `8d64259`).
|
||||||
deliberate — the alternative is a silent lost delivery — and the README says so
|
Every SQLite handle opens with WAL journaling and a busy timeout, a
|
||||||
under Rationale. It is not a defect to re-file.
|
bookkeeping write that fails leaves its delivery in a recoverable
|
||||||
|
state rather than a lying one, and recovery skips a delivery that
|
||||||
|
already has a successful result row. Final pre-tag verification
|
||||||
|
exercised it and confirmed it holds. Whatever the milestone still
|
||||||
|
shows open is what remains before `v1.0.0`.
|
||||||
|
|
||||||
|
Delivery is at-least-once by design, not by accident: a send whose
|
||||||
|
result row does not land is attempted again, so a receiver can see a
|
||||||
|
duplicate. That is deliberate — the alternative is a silent lost
|
||||||
|
delivery — and the README says so under Rationale. It is not a defect
|
||||||
|
to re-file.
|
||||||
|
|
||||||
|
One caveat on reading a green check: a docs-only commit deliberately
|
||||||
|
replays from the layer cache
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/119), so a green status on
|
||||||
|
such a commit evidences a replay rather than an executed run. A code
|
||||||
|
commit invalidates the `COPY` layer and genuinely executes.
|
||||||
|
|
||||||
# Next Step
|
# Next Step
|
||||||
|
|
||||||
@sneak runs webhooker in limited production, then tags `v1.0.0`. Merging `next`
|
Clear the rest of the open 1.0.0 milestone
|
||||||
into `main` is a separate act from tagging and waits on neither of those: `next`
|
(https://git.eeqj.de/sneak/webhooker/milestone/9) and tag `v1.0.0`.
|
||||||
is kept mergeable at all times, which is the point of the branch.
|
Merging `next` into `main` is a separate act from tagging and waits on
|
||||||
|
neither of those: `next` is kept mergeable at all times, which is the
|
||||||
|
point of the branch.
|
||||||
|
|
||||||
|
# Completed Steps
|
||||||
|
|
||||||
|
- 2026-08-24 Bind the plaintext HTTP listener deliberately, via
|
||||||
|
`BIND_ADDRESS` defaulting to `127.0.0.1`, and document the
|
||||||
|
reverse-proxy deployment. A hostname, an empty value or a value
|
||||||
|
carrying a port is a startup error, and the `Dockerfile` sets
|
||||||
|
`0.0.0.0` because a loopback bind inside a container is unreachable
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/268). The same commit
|
||||||
|
removed the shutdown race: `httpServer` is built in the constructor
|
||||||
|
rather than assigned from the serving goroutine, which orders the
|
||||||
|
write before every fx hook and rules out the nil dereference a
|
||||||
|
SIGTERM arriving first would have caused, and `sentryEnabled` is an
|
||||||
|
`atomic.Bool` (https://git.eeqj.de/sneak/webhooker/issues/226)
|
||||||
|
- 2026-08-24 Remove inbound request signature verification. The
|
||||||
|
entrypoint UUID is the authentication secret, so the per-entrypoint
|
||||||
|
shared secret, the `internal/signature` package, the receiver check,
|
||||||
|
the model fields and the forms are all gone. This reverses the
|
||||||
|
feature that landed earlier in the same milestone
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/67,
|
||||||
|
https://git.eeqj.de/sneak/webhooker/issues/279)
|
||||||
|
- 2026-08-24 Stamp the build version into the binary and render it in
|
||||||
|
the UI footer. `script/version` is the single source — `$VERSION`,
|
||||||
|
else `git describe --tags --always --dirty`, else `unknown` — so a
|
||||||
|
`make build` binary and a `make docker` image from one checkout
|
||||||
|
report the same thing, and nothing in it varies between two builds
|
||||||
|
of the same commit, which the release gate's byte-identical
|
||||||
|
assertion would catch
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/253)
|
||||||
|
- 2026-08-24 Derive cookie `Secure` and CSRF strictness from the
|
||||||
|
request transport rather than from `WEBHOOKER_ENVIRONMENT`. Behind a
|
||||||
|
real TLS proxy with the environment left at its `dev` default, the
|
||||||
|
session cookie silently lost `Secure` while the CSRF cookie on the
|
||||||
|
same response kept it. `X-Forwarded-Proto` is now matched
|
||||||
|
case-insensitively on its first comma-separated element, so `HTTPS`
|
||||||
|
and `https, http` no longer fall to the relaxed CSRF path
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/269)
|
||||||
|
- 2026-08-24 Roll back a failed webhook deletion instead of committing
|
||||||
|
it. A failing delete committed whatever had already succeeded,
|
||||||
|
hard-deleted the per-webhook event database anyway, and redirected as
|
||||||
|
though it had worked — orphaned config plus permanently destroyed
|
||||||
|
history, reported as success. All three delete positions now roll
|
||||||
|
back with the event database intact
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/262)
|
||||||
|
- 2026-08-24 Name a deleted target on its historical deliveries, marked
|
||||||
|
`(deleted)`, rather than leaving the event log unable to say where a
|
||||||
|
delivery went. A deleted target's credentials stay masked exactly as
|
||||||
|
a live one's, and it cannot become deliverable again through the
|
||||||
|
receiver, resubmit, replay, the edit form or the toggle
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/211)
|
||||||
|
- 2026-08-24 Bound both request-controlled `/metrics` label dimensions,
|
||||||
|
so the unauthenticated receiver is no longer a memory-exhaustion
|
||||||
|
vector: `handler` carries the chi route pattern, and `method` folds
|
||||||
|
anything chi cannot route onto a single `(unmatched)` sentinel. Both
|
||||||
|
were reproduced before the fix — 300 random method tokens took the
|
||||||
|
series count from 106 to 7,631, and path flooding reached 62,532 —
|
||||||
|
and a label audit across a live scrape found no third unbounded
|
||||||
|
dimension (https://git.eeqj.de/sneak/webhooker/issues/254,
|
||||||
|
https://git.eeqj.de/sneak/webhooker/issues/261)
|
||||||
|
- 2026-08-24 Validate `max_retries` on both target forms. `abc`, `2.7`
|
||||||
|
and `-5` silently became 0 — fire-and-forget — including on the edit
|
||||||
|
path, where it destroyed a working value, and `999999999` stored
|
||||||
|
verbatim. The ceiling of 20 is the `max` both templates already
|
||||||
|
declared (https://git.eeqj.de/sneak/webhooker/issues/221)
|
||||||
|
- 2026-08-24 Resubmit a stored event as a new undelivered event, so a
|
||||||
|
backend under development can be tested against real captured
|
||||||
|
traffic. Per-delivery replay cannot serve that: it re-sends one
|
||||||
|
finished delivery to its own original target, and a target created
|
||||||
|
for a dev backend has no prior delivery to replay. Resubmit
|
||||||
|
re-injects the stored event at the top of the receiver path and fans
|
||||||
|
it out to whatever targets are active now
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/250)
|
||||||
|
- 2026-08-20 Take an exclusive lock on `DATA_DIR` at startup, so two
|
||||||
|
instances on one directory cannot both deliver
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/201)
|
||||||
|
- 2026-08-20 Shut down the app when the HTTP listener fails. The
|
||||||
|
`OnStart` hook returned as soon as the serving goroutine was
|
||||||
|
spawned, so a failed listen left fx reporting RUNNING and a live
|
||||||
|
process with nothing bound — invisible to systemd and Docker restart
|
||||||
|
policies (https://git.eeqj.de/sneak/webhooker/issues/200)
|
||||||
|
- 2026-08-20 Stop target credentials leaking into the per-webhook event
|
||||||
|
databases (https://git.eeqj.de/sneak/webhooker/issues/206), log SQL
|
||||||
|
with placeholders rather than bound values
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/207), and fail loudly on
|
||||||
|
half-set metrics auth credentials
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/205)
|
||||||
|
- 2026-08-20 Read queue depths with `Find`, not `Scan`. `Scan` swaps
|
||||||
|
GORM's own trace recorder in for the logging adapter, and that
|
||||||
|
recorder does not implement `gorm.ParamsFilter`, so those statements
|
||||||
|
logged their bound values interpolated and bypassed the suppression
|
||||||
|
above. The two units gated green against a `next` that lacked the
|
||||||
|
other, and `next` went red when both landed
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/234)
|
||||||
|
- 2026-08-20 Render per-attempt delivery detail in the event log
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/202) and add replay of a
|
||||||
|
terminally failed delivery
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/203)
|
||||||
|
- 2026-08-20 Expose delivery metrics on `/metrics`
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/209) and document the
|
||||||
|
backup, restore and upgrade procedures
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/210)
|
||||||
|
- 2026-08-20 Add a `webhooker resetpw` subcommand and a bootstrap
|
||||||
|
banner. The admin bootstrap password was printed once among roughly
|
||||||
|
45 fx lines, and under `docker run -d` went to container logs subject
|
||||||
|
to rotation; there was no reset path at all, so recovery meant
|
||||||
|
hand-deleting the users row, documented nowhere. The password is read
|
||||||
|
from stdin or generated, never from argv where `/proc` would publish
|
||||||
|
it (https://git.eeqj.de/sneak/webhooker/issues/208)
|
||||||
|
- 2026-08-20 Add `ALLOWED_EGRESS_CIDRS`, an allowlist-only escape hatch
|
||||||
|
for the SSRF guard, so a self-hosted proxy can forward into the
|
||||||
|
operator's own network. The guard's always-blocked set cannot be
|
||||||
|
reopened by configuration
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/204)
|
||||||
|
- 2026-08-20 Harden operator-set target headers, which were carried
|
||||||
|
unsafely across a redirect
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/233)
|
||||||
|
- 2026-08-20 Add a target edit form with headers and timeout fields
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/127)
|
||||||
|
- 2026-08-18 Raise `script/test`'s per-package timeout from 30s to 90s,
|
||||||
|
matching the org-wide backstop. `go test` applies `-timeout` per
|
||||||
|
package, and `internal/handlers` had grown past the old budget: a
|
||||||
|
cache-defeated build failed outright at `GOMAXPROCS=4`, and every run
|
||||||
|
under deliberate host load breached 30s. The measurement table lives
|
||||||
|
in the script (#194)
|
||||||
|
- 2026-08-18 Re-sync `REPO_POLICIES.md` from `prompts`. The local copy
|
||||||
|
was stale and still mandated a 20s test target with a 30s timeout,
|
||||||
|
which the org replaced with a 60s cap and a 90s backstop. A synced
|
||||||
|
copy is not a source; reading it as one nearly produced a PR against
|
||||||
|
`prompts` proposing a change already merged there (#196)
|
||||||
|
- 2026-08-18 Report handler panics through the logger and answer 500.
|
||||||
|
chi v1.5.5's `Recoverer` scans for a `panic(0x` frame the runtime no
|
||||||
|
longer emits, then indexes `pkg[-1:]`, so it panicked inside its own
|
||||||
|
stack printer before writing a byte: the recovery never ran, the
|
||||||
|
client got a dropped connection instead of a 500, and the original
|
||||||
|
panic was lost. A local middleware replaces it, bounded by
|
||||||
|
`MaxPanicLogLineBytes` (#187)
|
||||||
|
- 2026-08-18 Route GORM's logger through `slog` and bound it. Every
|
||||||
|
`gorm.Open` left `logger.Default` in place at `Warn` with
|
||||||
|
`IgnoreRecordNotFoundError` false, so **every record-not-found
|
||||||
|
printed the fully interpolated SQL to stdout** — including the
|
||||||
|
client-chosen path on `/webhook/{uuid}` and the submitted username on
|
||||||
|
the login form, at no level the operator set and outside
|
||||||
|
`internal/logger` entirely. Three call sites, not the two the issue
|
||||||
|
named (#178)
|
||||||
|
- 2026-08-18 Bound every `slog` line against client-chosen text. Eight
|
||||||
|
sites reachable unauthenticated, found by reading every `slog` call in
|
||||||
|
the tree rather than only the one reported; the budget moved to a
|
||||||
|
shared `internal/logfield` so no second truncation exists. `DEBUG`
|
||||||
|
being off by default is not a bound and is not treated as one (#176)
|
||||||
|
- 2026-08-18 Stop a slow host turning a login-guard test into a
|
||||||
|
segfault. A non-fatal `assert` on an acquire result was dereferenced
|
||||||
|
on the next line, so one timing miss killed the whole
|
||||||
|
`internal/middleware` binary and reddened CI for unrelated PRs. The
|
||||||
|
fix also removed a real production race — `acquire` could shed a
|
||||||
|
request with a slot standing free, because Go picks uniformly among
|
||||||
|
ready `select` cases (#186)
|
||||||
|
- 2026-08-18 Send the chi route pattern to Sentry rather than the
|
||||||
|
concrete path. The receiver's path carries the entrypoint capability
|
||||||
|
token, so every Sentry event from `/webhook/{uuid}` shipped a live
|
||||||
|
credential to a third party. Request `Data`, `QueryString`, `Cookies`
|
||||||
|
and `Env` are dropped and headers reduced to an allowlist (#179)
|
||||||
|
- 2026-08-18 Read form fields from the POST body only. `r.FormValue`
|
||||||
|
merges the query string, so a login could be driven by URL parameters
|
||||||
|
— putting the password somewhere that lands in access logs, proxy
|
||||||
|
logs and browser history (#160)
|
||||||
|
- 2026-08-18 Verify login credentials before spending rate-limit
|
||||||
|
budget, so a flood of wrong passwords cannot lock out the account it
|
||||||
|
is guessing at. The manager took this decision rather than stall the
|
||||||
|
queue; it is flagged on the issue for reversal (#150)
|
||||||
|
- 2026-08-18 Run all linting in Docker via `Dockerfile.lint`. Host lint
|
||||||
|
was wrong in both directions from version skew and shared caches.
|
||||||
|
`script/lint` asserts the summary line, because `--no-cache-filter`
|
||||||
|
silently ignores a stage name it does not match — the flag that makes
|
||||||
|
the gate meaningful fails open (#109)
|
||||||
|
- 2026-08-18 Serve an event's full stored body over HTTP. The list
|
||||||
|
query truncates for rendering, and that truncated value was the only
|
||||||
|
way to read a body, so the full payload was unreachable (#157)
|
||||||
|
- 2026-08-18 Bound the access log line against client-chosen text.
|
||||||
|
`internal/logfield` budgets by *encoded* bytes, not runes, so a
|
||||||
|
handler's JSON escaping cannot multiply a field past its allowance
|
||||||
|
(#146)
|
||||||
|
- 2026-08-18 Mark superseded CI commits `failure` rather than
|
||||||
|
`skipped`. A skipped run rolls up green, so a commit that was never
|
||||||
|
tested reported success (#152)
|
||||||
|
- 2026-08-18 Set `fx.StopTimeout` inside the container stop grace, so
|
||||||
|
shutdown hooks are bounded by a deadline the orchestrator will
|
||||||
|
actually honour rather than being killed mid-flush (#134)
|
||||||
|
- 2026-08-17 Bucket IPv6 rate-limit keys by `/64`. A single allocation
|
||||||
|
hands out 2^64 addresses, so per-address keying let one client mint
|
||||||
|
unlimited buckets. Manager decision, recorded on the issue (#125)
|
||||||
|
- 2026-08-17 Correct release-blocking README and startup-warning
|
||||||
|
inaccuracies, including claims about behaviour the code does not have
|
||||||
|
(#151)
|
||||||
|
- 2026-08-17 Fetch and verify Alpine.js at build time against
|
||||||
|
`static/vendor.sha256` instead of committing the minified blob, so
|
||||||
|
the dependency is pinned by hash rather than by trust (#145)
|
||||||
|
- 2026-08-17 Bound the event log's rendered bodies in the query itself,
|
||||||
|
so a large stored payload cannot be read into memory just to be
|
||||||
|
truncated for display (#135)
|
||||||
|
- 2026-08-17 Mask the `http` target's destination URL in the UI: it can
|
||||||
|
carry a bearer credential in its path or query, and was rendered
|
||||||
|
verbatim. Manager decision to mask unconditionally (#115)
|
||||||
|
- 2026-08-14 Bound shutdown hooks by their stop context, so a hook that
|
||||||
|
hangs cannot hold the process past its grace period (#102)
|
||||||
|
- 2026-08-14 Render templates via a buffer rather than the
|
||||||
|
`ResponseWriter`, so a template error part-way through cannot commit
|
||||||
|
a 200 and then fail — the response is written only once it is whole
|
||||||
|
(#123)
|
||||||
|
- 2026-08-14 Align the session codec's max-age with the 7-day absolute
|
||||||
|
cap. The codec accepted cookies the session layer considered expired,
|
||||||
|
so the cap was enforced in one place and not the other (#108)
|
||||||
|
- 2026-08-12 Warn when `TRUSTED_PROXIES` is empty in production, where
|
||||||
|
the safe default silently discards forwarded headers and every client
|
||||||
|
rate-limits as the proxy's address (#149)
|
||||||
|
- 2026-08-12 Bound the receiver rate limit per client IP across the
|
||||||
|
whole `/webhook/*` route. The existing limiter keyed on the request
|
||||||
|
path and `/webhook/{uuid}` matches any single segment, so a client
|
||||||
|
that invented a fresh path per request minted a fresh bucket per
|
||||||
|
request: the limit on the only unauthenticated endpoint bounded
|
||||||
|
nothing in aggregate, and every request still cost an entrypoint
|
||||||
|
lookup before it 404ed. An outer limiter keyed on the client address
|
||||||
|
alone now bounds that, chained in front of the unchanged
|
||||||
|
per-entrypoint limiter (#139)
|
||||||
|
- 2026-08-12 Correct release-blocking documentation inaccuracies: the
|
||||||
|
README promised manual redelivery in the present tense in three
|
||||||
|
places when nothing implements it (the same false claim also sat in
|
||||||
|
the doc comment that was its source text), the env table omitted
|
||||||
|
`RETENTION_SWEEP_INTERVAL`, and `TODO.md` itself omitted five landed
|
||||||
|
units (#141)
|
||||||
|
- 2026-08-12 Make the CI gate execute the checks it reports on. The
|
||||||
|
workflow now writes a build-context fingerprint before calling
|
||||||
|
`script/cibuild`, so a code commit invalidates the `COPY` layer of
|
||||||
|
the lint and builder stages while a docs-only commit still replays
|
||||||
|
from cache; a superseding run also rewrites the `failure` status
|
||||||
|
Gitea leaves on commits it cancelled and never tested. Verified by
|
||||||
|
pushing a deliberately broken test and watching CI go red (#119)
|
||||||
|
- 2026-08-12 Require a positive `RETENTION_SWEEP_INTERVAL`: a
|
||||||
|
non-positive value reached `time.NewTicker` in both the retention
|
||||||
|
reaper and the archive sweeper, panicking two goroutines with no
|
||||||
|
recover after startup had already reported success (#140)
|
||||||
|
- 2026-08-12 Bound the `X-Forwarded-For` scan's allocation to the hop
|
||||||
|
cap: the reverse walk cuts entries with `strings.LastIndexByte`
|
||||||
|
instead of joining and splitting, so a 1 MB header allocates 16 bytes
|
||||||
|
rather than 1.6 MB per request on the unauthenticated receiver.
|
||||||
|
Semantics proven unchanged by differential testing against the
|
||||||
|
previous implementation (#133)
|
||||||
|
- 2026-08-12 Cap the `X-Forwarded-For` hop walk at 64 entries, so an
|
||||||
|
attacker-supplied chain cannot burn unbounded CPU in the rate-limit
|
||||||
|
key function; running off the end falls back to the peer address
|
||||||
|
(#124)
|
||||||
|
- 2026-08-12 Gate forwarded-header trust behind a `TRUSTED_PROXIES` CIDR
|
||||||
|
list: all three rate limiters key on the connection's own address
|
||||||
|
unless the direct peer is a configured proxy, in which case
|
||||||
|
`X-Forwarded-For` is walked right to left for the first non-proxy hop.
|
||||||
|
Default trusts nothing, and a set-but-unparseable value aborts
|
||||||
|
startup. Before this, any client could mint a fresh bucket or drain
|
||||||
|
another's by rotating a spoofed header (#88)
|
||||||
|
- 2026-08-11 Web UI cleanup: nav terminology unified on Webhooks, the
|
||||||
|
Profile settings placeholder removed, a progressive-enhancement copy
|
||||||
|
button for the entrypoint URL, and retention form copy that states the
|
||||||
|
actual policy (deletion by the reaper, 0 retains forever) (#57)
|
||||||
|
- 2026-08-11 Mask the webhook credential in delivery errors and logs:
|
||||||
|
Go embeds the request URL in `*url.Error`, so every transport failure
|
||||||
|
persisted the full Slack webhook URL into the per-webhook event
|
||||||
|
database via `DeliveryResult.Error`, a field a future REST API would
|
||||||
|
have served. `maskURLError` drops path, query and userinfo while
|
||||||
|
preserving the wrapped cause, so `errors.Is`/`As` and `Timeout()`
|
||||||
|
still work and DNS, TLS and timeout failures still read differently
|
||||||
|
(#118)
|
||||||
|
- 2026-08-11 Rate-limit the public webhook receiver endpoint
|
||||||
|
(`RECEIVER_RATE_LIMIT`, default 120/min), keyed on client IP plus
|
||||||
|
entrypoint path so one entrypoint cannot exhaust another's budget;
|
||||||
|
over-limit requests get 429 with `Retry-After`. It was the one
|
||||||
|
unauthenticated, internet-facing endpoint with no limit at all (#64)
|
||||||
|
- 2026-08-11 Enforce the body size limit before CSRF parses the form:
|
||||||
|
`MaxBodySize` is now first in all four form-parsing route groups, so
|
||||||
|
an oversized request is rejected with 413 instead of being read in
|
||||||
|
full by the CSRF middleware before any cap applied (#90)
|
||||||
|
- 2026-08-11 Mask target config on the source detail page, which
|
||||||
|
rendered the stored blob verbatim and so exposed the Slack
|
||||||
|
incoming-webhook URL — a bearer credential that cannot be revoked
|
||||||
|
per-holder. Config reaches the template only as a `TargetView` of
|
||||||
|
labelled fields, and header values are rendered as a count (#113)
|
||||||
|
- 2026-08-11 Allow `retention_days` of 0 to mean retain forever, via a
|
||||||
|
sentinel written in `BeforeSave` so the GORM column default cannot
|
||||||
|
win the race. Also bounds the reaper's cutoff arithmetic: day counts
|
||||||
|
above 106751 overflowed `time.Duration` and wrapped the cutoff into
|
||||||
|
the future, where every row matched and the sweep deleted everything
|
||||||
|
(#79)
|
||||||
|
- 2026-08-09 Inactivity-based session timeout: sliding idle expiry
|
||||||
|
(`SESSION_IDLE_TIMEOUT`, default `24h`) refreshed on authenticated
|
||||||
|
requests, with the 7-day absolute cap kept as an independent
|
||||||
|
backstop that activity never extends (#66)
|
||||||
|
- 2026-08-09 Restart recovery and the 60s retry sweep terminally fail an
|
||||||
|
orphaned `retrying` delivery whose target type no longer supports
|
||||||
|
retries, recording a `DeliveryResult` with the reason instead of
|
||||||
|
leaving the delivery stuck forever (#82)
|
||||||
|
- 2026-08-09 Root the delivery engine's worker pool and the retention
|
||||||
|
reaper's sweep loop at `context.Background()` rather than the fx
|
||||||
|
`OnStart` hook context (#97), which carries fx's 15s start timeout and
|
||||||
|
killed both roughly fifteen seconds after boot: the proxy silently
|
||||||
|
stopped delivering webhooks entirely, and the reaper never ran a
|
||||||
|
single sweep under its default one-hour interval
|
||||||
|
- 2026-08-09 Archive writer lifecycle (#89): deleting a webhook (or its
|
||||||
|
last `database` target) evicts the cached archive writer and closes
|
||||||
|
its handle while deliberately leaving `archive-{webhookID}.db` on
|
||||||
|
disk, and a new `ArchiveSweeper` prunes idle archives on the existing
|
||||||
|
`RETENTION_SWEEP_INTERVAL` without ever creating an archive file
|
||||||
|
- 2026-08-09 Configuration parsing fails loudly on set-but-unparseable
|
||||||
|
environment values: `envInt` removed in favour of `envPositiveInt`
|
||||||
|
plus a `PORT` range check, `envBool` now parses with
|
||||||
|
`strconv.ParseBool`, and defaults apply only to unset variables (#80)
|
||||||
|
- 2026-08-07 Automatic event retention cleanup based on
|
||||||
|
`retention_days`, deleting expired events, deliveries, and delivery
|
||||||
|
results from each per-webhook event database (#63)
|
||||||
|
- 2026-08-07 Update golangci-lint to v2.12.2 (Docker image digest in
|
||||||
|
`Dockerfile`, release-archive sha256 pins in `script/bootstrap`),
|
||||||
|
adopt the canonical `.golangci.yml` (v2 `linters.settings` layout so
|
||||||
|
`lll`/`funlen`/`cyclop`/`dupl` thresholds actually apply), and fix
|
||||||
|
all newly surfaced lint findings
|
||||||
|
- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints,
|
||||||
|
Makefile shims, README Entrypoints section
|
||||||
|
- 2026-03-25 pin golangci-lint Docker image for linting (#55)
|
||||||
|
- 2026-03-18 CSRF middleware detects TLS per-request, fixing login over
|
||||||
|
plain HTTP and behind reverse proxies (#54)
|
||||||
|
- 2026-03-17 root path redirects based on auth state (#52)
|
||||||
|
- 2026-03-17 CSRF protection, SSRF prevention for HTTP delivery targets
|
||||||
|
with DNS rebinding defense, and per-IP login rate limiting (#42)
|
||||||
|
- 2026-03-17 Slack target type for incoming webhook notifications (#47)
|
||||||
|
- 2026-03-17 Dockerfile absolute paths and static linking (#49);
|
||||||
|
absolute dev DATA_DIR default and clarified env docs (#46)
|
||||||
|
- 2026-03-05 security headers middleware, session regeneration on
|
||||||
|
login, request body size limits (#41)
|
||||||
|
- 2026-03-04 tests for delivery, middleware, and session packages
|
||||||
|
(#32); removed globals.Buildarch (#31)
|
||||||
|
- 2026-03-04 1.0 MVP merge: Webhook/Entrypoint/Target rename, core
|
||||||
|
delivery engine with bounded worker pool and circuit breaker,
|
||||||
|
parallel fan-out, per-webhook event databases, management UI (#16)
|
||||||
|
- 2026-03-01 repo brought to REPO_POLICIES standards; TODO.md folded
|
||||||
|
into README (#6)
|
||||||
|
|
||||||
# Future Steps
|
# Future Steps
|
||||||
|
|
||||||
- Per-webhook rate limits, set in the web UI and enforced in the receiver
|
- Delivery status and retry management UI. Replay of a terminally
|
||||||
handler, on top of the receiver's existing limits, which `RECEIVER_RATE_LIMIT`
|
failed delivery and per-attempt detail already landed
|
||||||
sets for the whole server (see Rate Limiting in the README)
|
(https://git.eeqj.de/sneak/webhooker/issues/203,
|
||||||
- API key authentication for programmatic access (the `APIKey` model exists, but
|
https://git.eeqj.de/sneak/webhooker/issues/202)
|
||||||
nothing creates a key and no middleware checks one)
|
- Per-webhook rate limiting in the receiver handler (per-webhook config
|
||||||
|
plus handler enforcement; global limits must not apply to receiver
|
||||||
|
endpoints)
|
||||||
|
- API key authentication for programmatic access (APIKey model exists;
|
||||||
|
Bearer token middleware does not)
|
||||||
- REST API v1
|
- REST API v1
|
||||||
- CRUD for webhooks, entrypoints, targets
|
- CRUD for webhooks, entrypoints, targets
|
||||||
- event viewing and filtering endpoints
|
- event viewing and filtering endpoints
|
||||||
- event redelivery endpoint
|
- event redelivery endpoint
|
||||||
- OpenAPI specification
|
- OpenAPI specification
|
||||||
- Analytics in the web UI: response times summarised per webhook or per target,
|
- Analytics dashboard: success rates, response times, volume
|
||||||
and figures for all webhooks together. The webhook page already shows event,
|
- A remember-me option at login
|
||||||
delivery and failure counts and failure percentages, the target list counts
|
- Password reset flow for a forgotten password over the web. The
|
||||||
each target's deliveries, the event log and each event's page show every
|
authenticated password *change* flow already landed, and a lost
|
||||||
attempt's duration, and `/metrics` carries server-wide counts and delivery
|
password is recoverable from the console with `webhooker resetpw`
|
||||||
durations
|
|
||||||
- A remember-me option at login. Every session already survives a browser
|
|
||||||
restart: it lasts up to 7 days, and ends sooner after `SESSION_IDLE_TIMEOUT`
|
|
||||||
(default 24 hours) without activity
|
|
||||||
- Password reset flow for a forgotten password over the web. The authenticated
|
|
||||||
password _change_ flow already landed, and a lost password is recoverable from
|
|
||||||
the console with `webhooker resetpw`
|
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/208)
|
(https://git.eeqj.de/sneak/webhooker/issues/208)
|
||||||
- Later, nice to have
|
- Later, nice to have
|
||||||
- email delivery target type
|
- email delivery target type
|
||||||
- SNS and S3 delivery targets
|
- SNS and S3 delivery targets
|
||||||
- data transformations (e.g. a configurable Slack message format; the Slack
|
- data transformations (e.g. webhook to Slack message formatting)
|
||||||
target sends one fixed format today)
|
- JSONL file delivery with periodic S3 upload
|
||||||
- JSONL file delivery with periodic S3 upload (the `database` target already
|
- webhook event search and filtering
|
||||||
archives events to SQLite files, rotated hourly, daily or monthly if set,
|
- multi-user with role-based access control
|
||||||
and the webhook page offers them as one gzipped JSON download)
|
|
||||||
- webhook event search, and filtering beyond the event log's failed and
|
|
||||||
pending views
|
|
||||||
- multi-user with role-based access control (each webhook already belongs to
|
|
||||||
one account, but there is only the account made at first boot and no way
|
|
||||||
to add another)
|
|
||||||
|
|||||||
+17
-44
@@ -4,13 +4,10 @@ package main
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"log/slog"
|
|
||||||
"os"
|
"os"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"go.uber.org/fx/fxevent"
|
|
||||||
"sneak.berlin/go/webhooker/internal/archive"
|
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/datadir"
|
"sneak.berlin/go/webhooker/internal/datadir"
|
||||||
@@ -19,7 +16,6 @@ import (
|
|||||||
"sneak.berlin/go/webhooker/internal/handlers"
|
"sneak.berlin/go/webhooker/internal/handlers"
|
||||||
"sneak.berlin/go/webhooker/internal/healthcheck"
|
"sneak.berlin/go/webhooker/internal/healthcheck"
|
||||||
"sneak.berlin/go/webhooker/internal/logger"
|
"sneak.berlin/go/webhooker/internal/logger"
|
||||||
"sneak.berlin/go/webhooker/internal/metrics"
|
|
||||||
"sneak.berlin/go/webhooker/internal/middleware"
|
"sneak.berlin/go/webhooker/internal/middleware"
|
||||||
"sneak.berlin/go/webhooker/internal/resetpw"
|
"sneak.berlin/go/webhooker/internal/resetpw"
|
||||||
"sneak.berlin/go/webhooker/internal/server"
|
"sneak.berlin/go/webhooker/internal/server"
|
||||||
@@ -40,19 +36,17 @@ import (
|
|||||||
// hook that used the whole budget would exhaust it at that instant,
|
// hook that used the whole budget would exhaust it at that instant,
|
||||||
// and fx would skip every hook after the server — the delivery
|
// and fx would skip every hook after the server — the delivery
|
||||||
// engine, the healthcheck, the webhook DB manager and the database
|
// engine, the healthcheck, the webhook DB manager and the database
|
||||||
// close. That hook is the HTTP drain plus the Sentry flush that
|
// close. That hook is the 3s HTTP drain plus the Sentry flush that
|
||||||
// follows it in the same hook, and each is clamped to the stop
|
// follows it in the same hook, so the flush is clamped to the stop
|
||||||
// context's remaining time less server.TailHookReserve rather than
|
// context's remaining time less server.TailHookReserve rather than
|
||||||
// running for its own fixed 3s and 2s; the reserve is what the tail
|
// running for its own fixed 2s; the reserve is what the tail hooks
|
||||||
// hooks live on, and they are microsecond-scale in normal operation.
|
// live on, and they are microsecond-scale in normal operation.
|
||||||
// TestStopTimeout_LeavesHeadroomForTailHooks pins the arithmetic
|
// TestStopTimeout_LeavesHeadroomForTailHooks pins the arithmetic
|
||||||
// across every drain length and every amount of budget the hooks
|
// across every drain length.
|
||||||
// before the server may already have spent.
|
|
||||||
//
|
//
|
||||||
// This does not make the database close unconditional: the
|
// This does not make the database close unconditional: the
|
||||||
// archive.Sweeper and RetentionReaper hooks run before the server.
|
// ArchiveSweeper and RetentionReaper hooks run before the server
|
||||||
// What they spend comes out of the drain first, but past 3s it comes
|
// and can still consume the whole budget on their own.
|
||||||
// out of the reserve, and they can consume the whole budget.
|
|
||||||
const stopTimeout = 5 * time.Second
|
const stopTimeout = 5 * time.Second
|
||||||
|
|
||||||
// exitUsage is the status for a command line this binary cannot make
|
// exitUsage is the status for a command line this binary cannot make
|
||||||
@@ -173,25 +167,9 @@ func run(stderr io.Writer) int {
|
|||||||
func newApp() *fx.App {
|
func newApp() *fx.App {
|
||||||
return fx.New(
|
return fx.New(
|
||||||
fx.StopTimeout(stopTimeout),
|
fx.StopTimeout(stopTimeout),
|
||||||
// fx's own events go through the service's logger, not fx's
|
|
||||||
// console logger on standard error. The exception is a failure
|
|
||||||
// before this logger is built, such as an invalid configuration
|
|
||||||
// value, which fx's console logger still prints there. fx holds
|
|
||||||
// its events back until this logger is built and then replays
|
|
||||||
// them, so it takes the configuration, which sets the level
|
|
||||||
// DEBUG=true asks for: without it the replay would run at INFO
|
|
||||||
// and drop every record of how the graph was built.
|
|
||||||
fx.WithLogger(
|
|
||||||
func(l *logger.Logger, _ *config.Config) fxevent.Logger {
|
|
||||||
return logger.NewFxLogger(l.Get())
|
|
||||||
},
|
|
||||||
),
|
|
||||||
fx.Provide(
|
fx.Provide(
|
||||||
globals.New,
|
globals.New,
|
||||||
logger.New,
|
logger.New,
|
||||||
// The plain logger the session, the middleware, the
|
|
||||||
// webhook database manager and the archive writers take.
|
|
||||||
func(l *logger.Logger) *slog.Logger { return l.Get() },
|
|
||||||
config.New,
|
config.New,
|
||||||
database.New,
|
database.New,
|
||||||
database.NewWebhookDBManager,
|
database.NewWebhookDBManager,
|
||||||
@@ -199,26 +177,21 @@ func newApp() *fx.App {
|
|||||||
healthcheck.New,
|
healthcheck.New,
|
||||||
session.New,
|
session.New,
|
||||||
handlers.New,
|
handlers.New,
|
||||||
// The registry /metrics serves, and the delivery
|
|
||||||
// collectors registered on it.
|
|
||||||
metrics.NewRegistry,
|
|
||||||
metrics.New,
|
|
||||||
middleware.New,
|
middleware.New,
|
||||||
// The one SSRF guard both target-creation validation
|
// The one SSRF guard both target-creation validation
|
||||||
// and the delivery dialer consult, so they cannot
|
// and the delivery dialer consult, so they cannot
|
||||||
// disagree about a destination.
|
// disagree about a destination.
|
||||||
delivery.NewGuard,
|
delivery.NewGuard,
|
||||||
delivery.New,
|
delivery.New,
|
||||||
archive.NewWriters,
|
delivery.NewArchiveSweeper,
|
||||||
archive.NewSweeper,
|
// Wire *delivery.Engine as delivery.Notifier so the
|
||||||
// Wire *archive.Writers as archive.Archives so deleting
|
// webhook handler can notify the engine of new deliveries.
|
||||||
// or renaming a webhook or target reaches its archive
|
func(e *delivery.Engine) delivery.Notifier { return e },
|
||||||
// files.
|
// Wire *delivery.Engine as delivery.WebhookEvictor so
|
||||||
func(w *archive.Writers) archive.Archives { return w },
|
// deleting a webhook releases its archive writer.
|
||||||
// Wire *delivery.Engine as delivery.CircuitBreakers so
|
func(e *delivery.Engine) delivery.WebhookEvictor {
|
||||||
// the pages can show a target whose deliveries are
|
return e
|
||||||
// paused.
|
},
|
||||||
func(e *delivery.Engine) delivery.CircuitBreakers { return e },
|
|
||||||
server.New,
|
server.New,
|
||||||
),
|
),
|
||||||
fx.Invoke(
|
fx.Invoke(
|
||||||
@@ -226,7 +199,7 @@ func newApp() *fx.App {
|
|||||||
*server.Server,
|
*server.Server,
|
||||||
*delivery.Engine,
|
*delivery.Engine,
|
||||||
*database.RetentionReaper,
|
*database.RetentionReaper,
|
||||||
*archive.Sweeper,
|
*delivery.ArchiveSweeper,
|
||||||
) {
|
) {
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
|
|||||||
+9
-129
@@ -2,19 +2,12 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"encoding/json"
|
|
||||||
"io"
|
|
||||||
"log/slog"
|
|
||||||
"net"
|
|
||||||
"os"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"sneak.berlin/go/webhooker/internal/config/configtest"
|
|
||||||
"sneak.berlin/go/webhooker/internal/datadir"
|
"sneak.berlin/go/webhooker/internal/datadir"
|
||||||
"sneak.berlin/go/webhooker/internal/resetpw"
|
"sneak.berlin/go/webhooker/internal/resetpw"
|
||||||
"sneak.berlin/go/webhooker/internal/server"
|
"sneak.berlin/go/webhooker/internal/server"
|
||||||
@@ -37,7 +30,6 @@ const dockerStopGrace = 10 * time.Second
|
|||||||
// fx.New applies options before it executes invokes, so the timeout
|
// fx.New applies options before it executes invokes, so the timeout
|
||||||
// is set whether or not the graph itself can be constructed here.
|
// is set whether or not the graph itself can be constructed here.
|
||||||
func TestNewApp_StopTimeout(t *testing.T) {
|
func TestNewApp_StopTimeout(t *testing.T) {
|
||||||
configtest.ClearEnv(t)
|
|
||||||
t.Setenv("DATA_DIR", t.TempDir())
|
t.Setenv("DATA_DIR", t.TempDir())
|
||||||
|
|
||||||
got := newApp().StopTimeout()
|
got := newApp().StopTimeout()
|
||||||
@@ -46,100 +38,6 @@ func TestNewApp_StopTimeout(t *testing.T) {
|
|||||||
require.Less(t, got, dockerStopGrace)
|
require.Less(t, got, dockerStopGrace)
|
||||||
}
|
}
|
||||||
|
|
||||||
// freePort returns a loopback TCP port that was free a moment ago, by
|
|
||||||
// taking one and releasing it.
|
|
||||||
func freePort(t *testing.T) int {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var listenCfg net.ListenConfig
|
|
||||||
|
|
||||||
l, err := listenCfg.Listen(t.Context(), "tcp", "127.0.0.1:0")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
addr, ok := l.Addr().(*net.TCPAddr)
|
|
||||||
require.True(t, ok, "listener is not TCP")
|
|
||||||
require.NoError(t, l.Close())
|
|
||||||
|
|
||||||
return addr.Port
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestNewApp_SendsFxEventsToTheLogger starts and stops the app main
|
|
||||||
// runs, with DEBUG=true, and reads back what reached the service's
|
|
||||||
// logger. fx's own events must arrive there as structured records:
|
|
||||||
// the start at INFO, and at DEBUG the records of how the graph was
|
|
||||||
// built.
|
|
||||||
//
|
|
||||||
// fx holds its events back until its logger is built and then replays
|
|
||||||
// them all at once, so the earliest of them arriving shows the replay
|
|
||||||
// ran at DEBUG: that globals.New was provided, which fx records before
|
|
||||||
// anything is built, and the run of logger.New, which happens before
|
|
||||||
// the configuration sets the level.
|
|
||||||
func TestNewApp_SendsFxEventsToTheLogger(t *testing.T) {
|
|
||||||
configtest.ClearEnv(t)
|
|
||||||
t.Setenv("DATA_DIR", t.TempDir())
|
|
||||||
t.Setenv("PORT", strconv.Itoa(freePort(t)))
|
|
||||||
t.Setenv("DEBUG", "true")
|
|
||||||
|
|
||||||
// internal/logger writes to whatever os.Stdout is when it builds
|
|
||||||
// its handler. A file is not a terminal, so that handler is the
|
|
||||||
// JSON one the service uses in production.
|
|
||||||
out, err := os.CreateTemp(t.TempDir(), "stdout")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
stdout := os.Stdout
|
|
||||||
os.Stdout = out
|
|
||||||
|
|
||||||
t.Cleanup(func() {
|
|
||||||
os.Stdout = stdout
|
|
||||||
_ = out.Close()
|
|
||||||
})
|
|
||||||
|
|
||||||
app := newApp()
|
|
||||||
require.NoError(t, app.Start(t.Context()))
|
|
||||||
require.NoError(t, app.Stop(t.Context()))
|
|
||||||
|
|
||||||
_, err = out.Seek(0, io.SeekStart)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
written, err := io.ReadAll(out)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
type record struct {
|
|
||||||
Level string `json:"level"`
|
|
||||||
Msg string `json:"msg"`
|
|
||||||
Name string `json:"name"`
|
|
||||||
Constructor string `json:"constructor"`
|
|
||||||
}
|
|
||||||
|
|
||||||
var records []record
|
|
||||||
|
|
||||||
for line := range strings.Lines(string(written)) {
|
|
||||||
var r record
|
|
||||||
|
|
||||||
// The first-boot banner is plain text, not a record.
|
|
||||||
if json.Unmarshal([]byte(line), &r) == nil {
|
|
||||||
records = append(records, r)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const pkg = "sneak.berlin/go/webhooker/internal/"
|
|
||||||
|
|
||||||
info := slog.LevelInfo.String()
|
|
||||||
debug := slog.LevelDebug.String()
|
|
||||||
|
|
||||||
assert.Contains(t, records, record{Level: info, Msg: "started"})
|
|
||||||
assert.Contains(t, records, record{
|
|
||||||
Level: debug, Msg: "provided", Constructor: pkg + "globals.New()",
|
|
||||||
})
|
|
||||||
assert.Contains(t, records, record{
|
|
||||||
Level: debug, Msg: "run", Name: pkg + "logger.New()",
|
|
||||||
})
|
|
||||||
assert.Contains(t, records, record{Level: debug, Msg: "invoking"})
|
|
||||||
assert.Contains(t, records, record{
|
|
||||||
Level: debug, Msg: "initialized custom fxevent.Logger",
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRunRefusesLockedDataDir pins what an operator's second start
|
// TestRunRefusesLockedDataDir pins what an operator's second start
|
||||||
// does. The entry point must refuse before it builds the fx graph —
|
// does. The entry point must refuse before it builds the fx graph —
|
||||||
// nothing may open a database in a DATA_DIR another process holds —
|
// nothing may open a database in a DATA_DIR another process holds —
|
||||||
@@ -252,40 +150,22 @@ const tailHeadroom = 2 * time.Second
|
|||||||
// can produce, since a shorter drain leaves the flush more room and
|
// can produce, since a shorter drain leaves the flush more room and
|
||||||
// the worst case is not necessarily at either extreme.
|
// the worst case is not necessarily at either extreme.
|
||||||
//
|
//
|
||||||
// Nor does the hook start on a full budget: the archive.Sweeper and
|
// Shrinking either budget, or unbounding the flush again, must fail
|
||||||
// RetentionReaper hooks run before it, and whatever they spent is
|
// here rather than silently recreating a hook that swallows the
|
||||||
// gone. The outer sweep walks every amount they can spend. Once they
|
// whole sequence.
|
||||||
// have eaten into the headroom themselves, the hook must spend
|
|
||||||
// nothing of what is left. A drain that starts on the full budget
|
|
||||||
// must still get all of ShutdownTimeout, so a smaller stopTimeout
|
|
||||||
// cannot silently shorten every drain.
|
|
||||||
//
|
|
||||||
// Shrinking either budget, or unbounding the drain or the flush
|
|
||||||
// again, must fail here rather than silently recreating a hook that
|
|
||||||
// swallows the whole sequence.
|
|
||||||
func TestStopTimeout_LeavesHeadroomForTailHooks(t *testing.T) {
|
func TestStopTimeout_LeavesHeadroomForTailHooks(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
require.Less(t, server.ShutdownTimeout, stopTimeout)
|
require.Less(t, server.ShutdownTimeout, stopTimeout)
|
||||||
require.Equal(
|
|
||||||
t, server.ShutdownTimeout, server.DrainBudget(stopTimeout),
|
|
||||||
"a drain that starts on the full stop budget is cut short",
|
|
||||||
)
|
|
||||||
|
|
||||||
const step = 10 * time.Millisecond
|
const step = 10 * time.Millisecond
|
||||||
|
|
||||||
for spent := time.Duration(0); spent <= stopTimeout; spent += step {
|
for drain := time.Duration(0); drain <= server.ShutdownTimeout; drain += step {
|
||||||
remaining := stopTimeout - spent
|
hook := drain + server.SentryFlushBudget(stopTimeout-drain)
|
||||||
longest := max(server.DrainBudget(remaining), 0)
|
|
||||||
|
|
||||||
for drain := time.Duration(0); drain <= longest; drain += step {
|
require.LessOrEqual(
|
||||||
hook := drain + server.SentryFlushBudget(remaining-drain)
|
t, hook+tailHeadroom, stopTimeout,
|
||||||
|
"a %s drain leaves the tail hooks short", drain,
|
||||||
require.GreaterOrEqual(
|
)
|
||||||
t, remaining-hook, min(remaining, tailHeadroom),
|
|
||||||
"a %s drain after %s of earlier hooks leaves "+
|
|
||||||
"the tail hooks short", drain, spent,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,22 +0,0 @@
|
|||||||
#!/bin/sh
|
|
||||||
# deploy/docker-entrypoint.sh: the image's ENTRYPOINT. A bind-mounted
|
|
||||||
# data directory keeps its owner from the host, often root, and the app
|
|
||||||
# could not write to it. Started as root, this creates DATA_DIR if
|
|
||||||
# needed, gives it and everything in it to webhooker, sets its mode, and
|
|
||||||
# runs the command as webhooker, so the app never runs as root. Started
|
|
||||||
# as another user, it only runs the command.
|
|
||||||
set -eu
|
|
||||||
|
|
||||||
main() {
|
|
||||||
if [ "$(id -u)" != 0 ]; then
|
|
||||||
exec "$@"
|
|
||||||
fi
|
|
||||||
|
|
||||||
dir="${DATA_DIR:-/var/lib/webhooker}"
|
|
||||||
mkdir -p "$dir"
|
|
||||||
find "$dir" ! -user webhooker -exec chown -h webhooker:webhooker {} +
|
|
||||||
chmod 750 "$dir"
|
|
||||||
exec su-exec webhooker "$@"
|
|
||||||
}
|
|
||||||
|
|
||||||
main "$@"
|
|
||||||
@@ -1,18 +0,0 @@
|
|||||||
// ESLint configuration for static/js/. script/lint and the image build run
|
|
||||||
// ESLint in the Dockerfile's js-lint stage, never on the host.
|
|
||||||
//
|
|
||||||
// The rules are the ones the JavaScript styleguide linked from
|
|
||||||
// REPO_POLICIES.md states that a linter can check: const for everything,
|
|
||||||
// let only for a variable that is reassigned, never var.
|
|
||||||
export default [
|
|
||||||
// Alpine.js, extracted from 3p/ by make assets; not ours to lint.
|
|
||||||
{ ignores: ["static/js/alpine.min.js"] },
|
|
||||||
{
|
|
||||||
// The pages load static/js/app.js as a classic script, not a module.
|
|
||||||
languageOptions: { sourceType: "script" },
|
|
||||||
rules: {
|
|
||||||
"no-var": "error",
|
|
||||||
"prefer-const": "error",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
];
|
|
||||||
@@ -4,9 +4,6 @@ go 1.26.1
|
|||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/99designs/basicauth-go v0.0.0-20230316000542-bf6f9cbbf0f8
|
github.com/99designs/basicauth-go v0.0.0-20230316000542-bf6f9cbbf0f8
|
||||||
github.com/chromedp/cdproto v0.0.0-20260922220944-a19bff23514f
|
|
||||||
github.com/chromedp/chromedp v0.16.0
|
|
||||||
github.com/dustin/go-humanize v1.0.1
|
|
||||||
github.com/getsentry/sentry-go v0.25.0
|
github.com/getsentry/sentry-go v0.25.0
|
||||||
github.com/go-chi/chi v1.5.5
|
github.com/go-chi/chi v1.5.5
|
||||||
github.com/go-chi/cors v1.2.1
|
github.com/go-chi/cors v1.2.1
|
||||||
@@ -20,8 +17,9 @@ require (
|
|||||||
github.com/prometheus/client_model v0.5.0
|
github.com/prometheus/client_model v0.5.0
|
||||||
github.com/slok/go-http-metrics v0.11.0
|
github.com/slok/go-http-metrics v0.11.0
|
||||||
github.com/stretchr/testify v1.11.1
|
github.com/stretchr/testify v1.11.1
|
||||||
go.uber.org/fx v1.24.0
|
go.uber.org/fx v1.20.1
|
||||||
golang.org/x/crypto v0.38.0
|
golang.org/x/crypto v0.38.0
|
||||||
|
gopkg.in/yaml.v3 v3.0.1
|
||||||
gorm.io/driver/sqlite v1.5.4
|
gorm.io/driver/sqlite v1.5.4
|
||||||
gorm.io/gorm v1.25.5
|
gorm.io/gorm v1.25.5
|
||||||
modernc.org/sqlite v1.28.0
|
modernc.org/sqlite v1.28.0
|
||||||
@@ -30,17 +28,14 @@ require (
|
|||||||
require (
|
require (
|
||||||
github.com/beorn7/perks v1.0.1 // indirect
|
github.com/beorn7/perks v1.0.1 // indirect
|
||||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||||
github.com/chromedp/sysutil v1.1.0 // indirect
|
|
||||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||||
github.com/go-json-experiment/json v0.0.0-20260820222146-c27c302e5fc3 // indirect
|
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||||
github.com/gobwas/httphead v0.1.0 // indirect
|
|
||||||
github.com/gobwas/pool v0.2.1 // indirect
|
|
||||||
github.com/gobwas/ws v1.4.0 // indirect
|
|
||||||
github.com/gorilla/securecookie v1.1.2 // indirect
|
github.com/gorilla/securecookie v1.1.2 // indirect
|
||||||
github.com/jinzhu/inflection v1.0.0 // indirect
|
github.com/jinzhu/inflection v1.0.0 // indirect
|
||||||
github.com/jinzhu/now v1.1.5 // indirect
|
github.com/jinzhu/now v1.1.5 // indirect
|
||||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
|
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
|
||||||
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
|
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
|
||||||
|
github.com/kr/text v0.2.0 // indirect
|
||||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||||
github.com/mattn/go-sqlite3 v1.14.17 // indirect
|
github.com/mattn/go-sqlite3 v1.14.17 // indirect
|
||||||
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
|
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
|
||||||
@@ -49,16 +44,16 @@ require (
|
|||||||
github.com/prometheus/procfs v0.12.0 // indirect
|
github.com/prometheus/procfs v0.12.0 // indirect
|
||||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||||
github.com/zeebo/xxh3 v1.0.2 // indirect
|
github.com/zeebo/xxh3 v1.0.2 // indirect
|
||||||
go.uber.org/dig v1.19.0 // indirect
|
go.uber.org/atomic v1.9.0 // indirect
|
||||||
go.uber.org/multierr v1.10.0 // indirect
|
go.uber.org/dig v1.17.0 // indirect
|
||||||
go.uber.org/zap v1.26.0 // indirect
|
go.uber.org/multierr v1.9.0 // indirect
|
||||||
|
go.uber.org/zap v1.23.0 // indirect
|
||||||
golang.org/x/mod v0.17.0 // indirect
|
golang.org/x/mod v0.17.0 // indirect
|
||||||
golang.org/x/sync v0.14.0 // indirect
|
golang.org/x/sync v0.14.0 // indirect
|
||||||
golang.org/x/sys v0.47.0 // indirect
|
golang.org/x/sys v0.37.0 // indirect
|
||||||
golang.org/x/text v0.25.0 // indirect
|
golang.org/x/text v0.25.0 // indirect
|
||||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
|
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
|
||||||
google.golang.org/protobuf v1.31.0 // indirect
|
google.golang.org/protobuf v1.31.0 // indirect
|
||||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
|
||||||
lukechampine.com/uint128 v1.2.0 // indirect
|
lukechampine.com/uint128 v1.2.0 // indirect
|
||||||
modernc.org/cc/v3 v3.40.0 // indirect
|
modernc.org/cc/v3 v3.40.0 // indirect
|
||||||
modernc.org/ccgo/v3 v3.16.13 // indirect
|
modernc.org/ccgo/v3 v3.16.13 // indirect
|
||||||
|
|||||||
@@ -1,15 +1,14 @@
|
|||||||
github.com/99designs/basicauth-go v0.0.0-20230316000542-bf6f9cbbf0f8 h1:nMpu1t4amK3vJWBibQ5X/Nv0aXL+b69TQf2uK5PH7Go=
|
github.com/99designs/basicauth-go v0.0.0-20230316000542-bf6f9cbbf0f8 h1:nMpu1t4amK3vJWBibQ5X/Nv0aXL+b69TQf2uK5PH7Go=
|
||||||
github.com/99designs/basicauth-go v0.0.0-20230316000542-bf6f9cbbf0f8/go.mod h1:3cARGAK9CfW3HoxCy1a0G4TKrdiKke8ftOMEOHyySYs=
|
github.com/99designs/basicauth-go v0.0.0-20230316000542-bf6f9cbbf0f8/go.mod h1:3cARGAK9CfW3HoxCy1a0G4TKrdiKke8ftOMEOHyySYs=
|
||||||
|
github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
|
||||||
|
github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
|
||||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||||
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
|
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
|
||||||
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||||
github.com/chromedp/cdproto v0.0.0-20260922220944-a19bff23514f h1:8PK9FM4bE0C8GMoWBW5lVsef3U7sPICjDg6JqngyYhk=
|
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||||
github.com/chromedp/cdproto v0.0.0-20260922220944-a19bff23514f/go.mod h1:3v4FIp5njIUyPDvqXsxEOxnB34lijG0up98/5kM1KaE=
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/chromedp/chromedp v0.16.0 h1:rOO4deOm4CbZgBCa8mD9g2rDyIoNs0BkgvNrlbp5ouk=
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/chromedp/chromedp v0.16.0/go.mod h1:rbuGKFT1vMcFcFqKfPIO1GpX/N+2s8onm2qMxZLbU5U=
|
|
||||||
github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM=
|
|
||||||
github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8=
|
|
||||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
|
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
|
||||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||||
@@ -24,14 +23,6 @@ github.com/go-chi/httprate v0.15.0 h1:j54xcWV9KGmPf/X4H32/aTH+wBlrvxL7P+SdnRqxh5
|
|||||||
github.com/go-chi/httprate v0.15.0/go.mod h1:rzGHhVrsBn3IMLYDOZQsSU4fJNWcjui4fWKJcCId1R4=
|
github.com/go-chi/httprate v0.15.0/go.mod h1:rzGHhVrsBn3IMLYDOZQsSU4fJNWcjui4fWKJcCId1R4=
|
||||||
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
|
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
|
||||||
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
|
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
|
||||||
github.com/go-json-experiment/json v0.0.0-20260820222146-c27c302e5fc3 h1:UADEEmDKgfXbtnGJZ97beY5XLo9ZechG1nlU4KnRrkE=
|
|
||||||
github.com/go-json-experiment/json v0.0.0-20260820222146-c27c302e5fc3/go.mod h1:tphK2c80bpPhMOI4v6bIc2xWywPfbqi1Z06+RcrMkDg=
|
|
||||||
github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
|
|
||||||
github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
|
|
||||||
github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
|
|
||||||
github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
|
|
||||||
github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs=
|
|
||||||
github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc=
|
|
||||||
github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw=
|
github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw=
|
||||||
github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0=
|
github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0=
|
||||||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||||
@@ -64,20 +55,17 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
|||||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||||
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo=
|
|
||||||
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=
|
|
||||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||||
github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
|
github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
|
||||||
github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
|
github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
|
||||||
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg=
|
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg=
|
||||||
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k=
|
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k=
|
||||||
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw=
|
|
||||||
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
|
|
||||||
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
|
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
|
||||||
github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
|
github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
|
||||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||||
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
||||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk=
|
github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk=
|
||||||
@@ -94,24 +82,28 @@ github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjR
|
|||||||
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
|
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
|
||||||
github.com/slok/go-http-metrics v0.11.0 h1:ABJUpekCZSkQT1wQrFvS4kGbhea/w6ndFJaWJeh3zL0=
|
github.com/slok/go-http-metrics v0.11.0 h1:ABJUpekCZSkQT1wQrFvS4kGbhea/w6ndFJaWJeh3zL0=
|
||||||
github.com/slok/go-http-metrics v0.11.0/go.mod h1:ZGKeYG1ET6TEJpQx18BqAJAvxw9jBAZXCHU7bWQqqAc=
|
github.com/slok/go-http-metrics v0.11.0/go.mod h1:ZGKeYG1ET6TEJpQx18BqAJAvxw9jBAZXCHU7bWQqqAc=
|
||||||
|
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
|
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
|
||||||
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
||||||
|
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||||
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
||||||
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||||
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
|
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
|
||||||
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
|
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
|
||||||
go.uber.org/dig v1.19.0 h1:BACLhebsYdpQ7IROQ1AGPjrXcP5dF80U3gKoFzbaq/4=
|
go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
|
||||||
go.uber.org/dig v1.19.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE=
|
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||||
go.uber.org/fx v1.24.0 h1:wE8mruvpg2kiiL1Vqd0CC+tr0/24XIB10Iwp2lLWzkg=
|
go.uber.org/dig v1.17.0 h1:5Chju+tUvcC+N7N6EV08BJz41UZuO3BmHcN4A287ZLI=
|
||||||
go.uber.org/fx v1.24.0/go.mod h1:AmDeGyS+ZARGKM4tlH4FY2Jr63VjbEDJHtqXTGP5hbo=
|
go.uber.org/dig v1.17.0/go.mod h1:rTxpf7l5I0eBTlE6/9RL+lDybC7WFwY2QH55ZSjy1mU=
|
||||||
go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
|
go.uber.org/fx v1.20.1 h1:zVwVQGS8zYvhh9Xxcu4w1M6ESyeMzebzj2NbSayZ4Mk=
|
||||||
go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=
|
go.uber.org/fx v1.20.1/go.mod h1:iSYNbHf2y55acNCwCXKx7LbWb5WG1Bnue5RDXz1OREg=
|
||||||
go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
|
go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
|
||||||
go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
|
||||||
go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
|
go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
|
||||||
go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
|
go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
|
||||||
|
go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY=
|
||||||
|
go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY=
|
||||||
golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8=
|
golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8=
|
||||||
golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw=
|
golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw=
|
||||||
golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
|
golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
|
||||||
@@ -119,8 +111,8 @@ golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
|||||||
golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ=
|
golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ=
|
||||||
golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
|
golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
|
||||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
|
||||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||||
golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4=
|
golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4=
|
||||||
golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA=
|
golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA=
|
||||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=
|
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=
|
||||||
|
|||||||
@@ -1,446 +0,0 @@
|
|||||||
// Package archive manages the SQLite archive files the database target
|
|
||||||
// writes: how they are named and rotated, the writers that open, write,
|
|
||||||
// close and reopen them, pruning the rows past a target's expiry,
|
|
||||||
// renaming a target's files and moving them back when that fails, the
|
|
||||||
// gzipped JSON export, the idle sweep, and the locks that keep a rename
|
|
||||||
// apart from archive writes, from other edits and from exports.
|
|
||||||
package archive
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"log/slog"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"go.uber.org/fx"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// archiveNameMaxLen is how many characters of a webhook or target
|
|
||||||
// name an archive file name keeps.
|
|
||||||
const archiveNameMaxLen = 40
|
|
||||||
|
|
||||||
// Archives is how the handlers keep the database targets' archive
|
|
||||||
// files in step with the configuration. Deleting a webhook or a
|
|
||||||
// target releases the cached archive writers, whose open file
|
|
||||||
// handles would otherwise outlive them; renaming one renames the
|
|
||||||
// archive files, which are named for the webhook and the target
|
|
||||||
// (see FileName). RenameLock returns the lock every rename runs under
|
|
||||||
// (see Writers.RenameLock).
|
|
||||||
//
|
|
||||||
// It is an interface so that the handlers' tests can stand in for
|
|
||||||
// Writers.
|
|
||||||
//
|
|
||||||
// Neither eviction deletes an archive file. Both are idempotent
|
|
||||||
// and are no-ops for a webhook or target with no writer.
|
|
||||||
type Archives interface {
|
|
||||||
RenameLock() *sync.Mutex
|
|
||||||
EvictWebhook(webhookID string)
|
|
||||||
EvictTarget(targetID string)
|
|
||||||
Rename(targetID, webhookName, targetName string) error
|
|
||||||
}
|
|
||||||
|
|
||||||
// WritersParams holds the fx dependencies for Writers.
|
|
||||||
type WritersParams struct {
|
|
||||||
fx.In
|
|
||||||
|
|
||||||
Database *database.Database
|
|
||||||
DBManager *database.WebhookDBManager
|
|
||||||
Logger *slog.Logger
|
|
||||||
}
|
|
||||||
|
|
||||||
// Writers holds the archive writer of each database target, and the
|
|
||||||
// lock every rename of archive files runs under. The database target
|
|
||||||
// writes an event through it, the handlers rename and evict through
|
|
||||||
// it, the archive sweeper prunes through it, and the delivery engine
|
|
||||||
// evicts every writer when it stops. See archiveWriter for the
|
|
||||||
// close/reopen, auto-recreate, and expiry semantics.
|
|
||||||
type Writers struct {
|
|
||||||
db *database.Database
|
|
||||||
dbMgr *database.WebhookDBManager
|
|
||||||
log *slog.Logger
|
|
||||||
|
|
||||||
// writers holds one archive writer per database target, keyed
|
|
||||||
// by target ID.
|
|
||||||
mu sync.Mutex
|
|
||||||
writers map[string]*archiveWriter
|
|
||||||
|
|
||||||
// renameMu makes the webhook edit, the target edit and target
|
|
||||||
// creation run one at a time, each held from loading the stored
|
|
||||||
// names through the archive rename, the save and any move back.
|
|
||||||
// Interleaved, one could rename an archive between another's
|
|
||||||
// rename and save, leaving the file named for one edit and the
|
|
||||||
// stored names from the other. An archive download holds it while
|
|
||||||
// it reads the stored names and lists the files they give, and
|
|
||||||
// again for each file while it finds the file under the names
|
|
||||||
// stored then and opens it. RenameLock returns it.
|
|
||||||
renameMu sync.Mutex
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewWriters builds the archive writers. It opens no archive file:
|
|
||||||
// each target's writer is made on its first use.
|
|
||||||
func NewWriters(params WritersParams) *Writers {
|
|
||||||
return &Writers{
|
|
||||||
db: params.Database,
|
|
||||||
dbMgr: params.DBManager,
|
|
||||||
log: params.Logger,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// RenameLock implements Archives. It returns the lock every rename of
|
|
||||||
// archive files runs under: see renameMu for who holds it and for how
|
|
||||||
// long. Archive writes and Rename itself do not take it.
|
|
||||||
func (ws *Writers) RenameLock() *sync.Mutex {
|
|
||||||
return &ws.renameMu
|
|
||||||
}
|
|
||||||
|
|
||||||
// FileName returns the file name of a database target's
|
|
||||||
// archive: archive-WEBHOOKNAME-TARGETNAME-TARGETID.db, with both
|
|
||||||
// names passed through archiveNamePart. The target ID keeps the
|
|
||||||
// name unique when two targets' names come out the same.
|
|
||||||
func FileName(webhookName, targetName, targetID string) string {
|
|
||||||
return "archive-" + archiveNamePart(webhookName) + "-" +
|
|
||||||
archiveNamePart(targetName) + "-" + targetID + ".db"
|
|
||||||
}
|
|
||||||
|
|
||||||
// archiveNamePart makes a webhook or target name safe to put in a
|
|
||||||
// file name. It is lowercased; ASCII letters and digits are kept,
|
|
||||||
// every other run of characters becomes a single "-", and no "-" is
|
|
||||||
// left at either end. It is cut to archiveNameMaxLen characters, and
|
|
||||||
// a name with nothing left is "unnamed".
|
|
||||||
func archiveNamePart(name string) string {
|
|
||||||
var b strings.Builder
|
|
||||||
|
|
||||||
dash := false
|
|
||||||
|
|
||||||
for _, r := range strings.ToLower(name) {
|
|
||||||
if (r < 'a' || r > 'z') && (r < '0' || r > '9') {
|
|
||||||
dash = b.Len() > 0
|
|
||||||
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if dash {
|
|
||||||
b.WriteByte('-')
|
|
||||||
|
|
||||||
dash = false
|
|
||||||
}
|
|
||||||
|
|
||||||
b.WriteRune(r)
|
|
||||||
}
|
|
||||||
|
|
||||||
part := b.String()
|
|
||||||
if len(part) > archiveNameMaxLen {
|
|
||||||
part = strings.TrimRight(part[:archiveNameMaxLen], "-")
|
|
||||||
}
|
|
||||||
|
|
||||||
if part == "" {
|
|
||||||
return "unnamed"
|
|
||||||
}
|
|
||||||
|
|
||||||
return part
|
|
||||||
}
|
|
||||||
|
|
||||||
// Write writes the full event of a delivery to a database target as
|
|
||||||
// a row into the target's archive database, honouring the target's
|
|
||||||
// ArchiveExpiry and ArchiveRotation. With rotation, the event goes to
|
|
||||||
// the file for the period of its receive time.
|
|
||||||
func (ws *Writers) Write(d *database.Delivery) error {
|
|
||||||
webhookID := d.Event.WebhookID
|
|
||||||
if webhookID == "" {
|
|
||||||
return errArchiveMissingWebhookID
|
|
||||||
}
|
|
||||||
|
|
||||||
// An event whose stored row was gone before its delivery ran has
|
|
||||||
// no receive time (see delivery's Engine.hydrateEvent), and goes
|
|
||||||
// to the file for now.
|
|
||||||
receivedAt := d.Event.CreatedAt
|
|
||||||
if receivedAt.IsZero() {
|
|
||||||
receivedAt = time.Now()
|
|
||||||
}
|
|
||||||
|
|
||||||
w, err := ws.writerFor(d.TargetID)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
row := archivedEvent{
|
|
||||||
EventID: d.Event.ID,
|
|
||||||
WebhookID: webhookID,
|
|
||||||
EntrypointID: d.Event.EntrypointID,
|
|
||||||
Method: d.Event.Method,
|
|
||||||
RawQuery: d.Event.RawQuery,
|
|
||||||
Headers: d.Event.Headers,
|
|
||||||
Body: d.Event.Body,
|
|
||||||
ContentType: d.Event.ContentType,
|
|
||||||
}
|
|
||||||
|
|
||||||
return w.write(
|
|
||||||
row, d.Target.ArchiveExpiry,
|
|
||||||
archivePeriod(d.Target.ArchiveRotation, receivedAt),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Rename implements Archives. It renames every one of a database
|
|
||||||
// target's archive files to FileName(webhookName, targetName,
|
|
||||||
// targetID), each keeping the period in its name. It goes through the
|
|
||||||
// target's writer, so the move holds the lock that the target's
|
|
||||||
// archive writes and the idle sweep take, and later writes use the
|
|
||||||
// new name. It never replaces a file: if one already has a new name,
|
|
||||||
// the error is ErrNameTaken.
|
|
||||||
//
|
|
||||||
// The writer is created if there is none, and it stays cached. The
|
|
||||||
// handlers rename before they save the new name, so until the save
|
|
||||||
// the main database still has the old one; a delivery in that window
|
|
||||||
// must find this writer rather than build one from the old name.
|
|
||||||
func (ws *Writers) Rename(
|
|
||||||
targetID, webhookName, targetName string,
|
|
||||||
) error {
|
|
||||||
w, err := ws.writerFor(targetID)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return w.rename(FileName(webhookName, targetName, targetID))
|
|
||||||
}
|
|
||||||
|
|
||||||
// EvictTarget implements Archives. It drops a database target's
|
|
||||||
// archive writer from the registry and closes its handle, so a
|
|
||||||
// deleted target does not leave a writer (and an open archive handle
|
|
||||||
// within its debounce window) alive for the process lifetime.
|
|
||||||
//
|
|
||||||
// The map entry is removed under the registry lock, which is
|
|
||||||
// then released before the handle is closed under the writer's
|
|
||||||
// own lock: that ordering keeps the registry available to other
|
|
||||||
// targets while an in-flight write on this one drains, and
|
|
||||||
// closing under the writer's lock means eviction can never race
|
|
||||||
// a write.
|
|
||||||
//
|
|
||||||
// Eviction is idempotent and silent for a target with no writer,
|
|
||||||
// which is the common case: only a database target that has
|
|
||||||
// received an event or been renamed has one. It never deletes the
|
|
||||||
// archive file.
|
|
||||||
func (ws *Writers) EvictTarget(targetID string) {
|
|
||||||
ws.mu.Lock()
|
|
||||||
|
|
||||||
w, ok := ws.writers[targetID]
|
|
||||||
if ok {
|
|
||||||
delete(ws.writers, targetID)
|
|
||||||
}
|
|
||||||
|
|
||||||
ws.mu.Unlock()
|
|
||||||
|
|
||||||
if !ok {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
w.evict()
|
|
||||||
|
|
||||||
ws.log.Info(
|
|
||||||
"evicted archive writer",
|
|
||||||
"target_id", targetID,
|
|
||||||
"path", w.path,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// EvictWebhook implements Archives. It evicts, exactly as EvictTarget
|
|
||||||
// does, the writer of every database target of a webhook. The archive
|
|
||||||
// files themselves are left on disk — they are long-term storage the
|
|
||||||
// operator owns.
|
|
||||||
func (ws *Writers) EvictWebhook(webhookID string) {
|
|
||||||
ws.mu.Lock()
|
|
||||||
|
|
||||||
var gone []*archiveWriter
|
|
||||||
|
|
||||||
for targetID, w := range ws.writers {
|
|
||||||
if w.webhookID == webhookID {
|
|
||||||
delete(ws.writers, targetID)
|
|
||||||
|
|
||||||
gone = append(gone, w)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
ws.mu.Unlock()
|
|
||||||
|
|
||||||
for _, w := range gone {
|
|
||||||
w.evict()
|
|
||||||
|
|
||||||
ws.log.Info(
|
|
||||||
"evicted archive writer",
|
|
||||||
"webhook_id", webhookID,
|
|
||||||
"path", w.path,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// EvictAll evicts every cached archive writer, exactly as EvictTarget
|
|
||||||
// does for one target. The delivery engine calls it at shutdown, once
|
|
||||||
// its workers have returned. Closing the last handle on an archive
|
|
||||||
// moves the contents of its -wal into the .db and removes the
|
|
||||||
// -wal, so a clean stop leaves each archive as a single file.
|
|
||||||
func (ws *Writers) EvictAll() {
|
|
||||||
ws.mu.Lock()
|
|
||||||
|
|
||||||
writers := ws.writers
|
|
||||||
ws.writers = nil
|
|
||||||
|
|
||||||
ws.mu.Unlock()
|
|
||||||
|
|
||||||
for _, w := range writers {
|
|
||||||
w.evict()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// writerFor returns the archive writer for a database target,
|
|
||||||
// creating and caching it on first use. Each target has one writer
|
|
||||||
// so its close/reopen debounce state is shared across concurrent
|
|
||||||
// deliveries, and so a rename and the idle sweep take the same lock
|
|
||||||
// as its writes.
|
|
||||||
func (ws *Writers) writerFor(
|
|
||||||
targetID string,
|
|
||||||
) (*archiveWriter, error) {
|
|
||||||
ws.mu.Lock()
|
|
||||||
defer ws.mu.Unlock()
|
|
||||||
|
|
||||||
w, ok := ws.writers[targetID]
|
|
||||||
if !ok {
|
|
||||||
var err error
|
|
||||||
|
|
||||||
w, err = ws.newWriter(targetID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if ws.writers == nil {
|
|
||||||
ws.writers = make(map[string]*archiveWriter)
|
|
||||||
}
|
|
||||||
|
|
||||||
ws.writers[targetID] = w
|
|
||||||
}
|
|
||||||
|
|
||||||
// A delivery claims the entry: even if the idle sweep created
|
|
||||||
// it moments ago, it now belongs to the registry proper and
|
|
||||||
// the sweep must leave it in place when it finishes.
|
|
||||||
w.sweepOwned = false
|
|
||||||
|
|
||||||
return w, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// sweepWriterFor returns the archive writer the idle sweep should
|
|
||||||
// prune a target's archive through, together with whether the sweep
|
|
||||||
// itself created the registry entry.
|
|
||||||
//
|
|
||||||
// The sweep must route its prune through the registered writer so
|
|
||||||
// the writer's mutex orders it against concurrent writes, but it
|
|
||||||
// must never leave a registry entry behind: a sweep that ran
|
|
||||||
// concurrently with the target's deletion would otherwise
|
|
||||||
// re-create an entry that nothing will ever evict again, which is
|
|
||||||
// exactly the leak eviction exists to prevent. An entry the sweep
|
|
||||||
// creates is therefore marked sweep-owned and handed back to
|
|
||||||
// releaseSweepWriter when the sweep is done.
|
|
||||||
func (ws *Writers) sweepWriterFor(
|
|
||||||
targetID string,
|
|
||||||
) (*archiveWriter, bool, error) {
|
|
||||||
ws.mu.Lock()
|
|
||||||
defer ws.mu.Unlock()
|
|
||||||
|
|
||||||
w, ok := ws.writers[targetID]
|
|
||||||
if ok {
|
|
||||||
return w, false, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
w, err := ws.newWriter(targetID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, false, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if ws.writers == nil {
|
|
||||||
ws.writers = make(map[string]*archiveWriter)
|
|
||||||
}
|
|
||||||
|
|
||||||
w.sweepOwned = true
|
|
||||||
ws.writers[targetID] = w
|
|
||||||
|
|
||||||
return w, true, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// releaseSweepWriter drops a registry entry that the idle sweep
|
|
||||||
// created, so a sweep leaves the registry exactly as it found it.
|
|
||||||
//
|
|
||||||
// The entry is removed only if it is still the very writer the
|
|
||||||
// sweep installed and no delivery has claimed it in the meantime
|
|
||||||
// (writerFor clears sweepOwned when it hands a writer to the
|
|
||||||
// write path). Both conditions are evaluated under the registry
|
|
||||||
// lock, so an eviction that raced the sweep — which removes the
|
|
||||||
// entry outright — simply finds nothing left to do here, and a
|
|
||||||
// delivery that adopted the writer keeps a registered, evictable
|
|
||||||
// one.
|
|
||||||
func (ws *Writers) releaseSweepWriter(
|
|
||||||
targetID string, w *archiveWriter,
|
|
||||||
) {
|
|
||||||
ws.mu.Lock()
|
|
||||||
defer ws.mu.Unlock()
|
|
||||||
|
|
||||||
cur, ok := ws.writers[targetID]
|
|
||||||
if !ok || cur != w || !cur.sweepOwned {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
delete(ws.writers, targetID)
|
|
||||||
}
|
|
||||||
|
|
||||||
// newWriter builds the writer for a database target's archive. Its
|
|
||||||
// path is the one Path gives for the webhook and the target as
|
|
||||||
// the main database names them now; from then on only rename changes
|
|
||||||
// the name the writer uses. It does not touch the archive files.
|
|
||||||
func (ws *Writers) newWriter(
|
|
||||||
targetID string,
|
|
||||||
) (*archiveWriter, error) {
|
|
||||||
if ws.dbMgr == nil {
|
|
||||||
return nil, errArchiveNoDataDir
|
|
||||||
}
|
|
||||||
|
|
||||||
var target database.Target
|
|
||||||
|
|
||||||
err := ws.db.DB().
|
|
||||||
Preload("Webhook").
|
|
||||||
First(&target, "id = ?", targetID).Error
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf(
|
|
||||||
"loading database target %s: %w", targetID, err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
w := newArchiveWriter(
|
|
||||||
Path(ws.dbMgr, &target.Webhook, &target),
|
|
||||||
ws.log,
|
|
||||||
)
|
|
||||||
w.webhookID = target.WebhookID
|
|
||||||
|
|
||||||
return w, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// sweepArchive prunes one database target's archive of rows older
|
|
||||||
// than expiry, without requiring a write. A missing archive file is
|
|
||||||
// left missing (see sweepExpired), so a sweep never creates an
|
|
||||||
// archive for a target that has never received an event.
|
|
||||||
//
|
|
||||||
// It also never leaves a registry entry behind: an entry it had
|
|
||||||
// to create to reach the writer's mutex is released again once
|
|
||||||
// the prune is done, so a sweep racing a target deletion cannot
|
|
||||||
// resurrect the writer the eviction just dropped.
|
|
||||||
func (ws *Writers) sweepArchive(
|
|
||||||
targetID string, expiry time.Duration,
|
|
||||||
) error {
|
|
||||||
w, created, err := ws.sweepWriterFor(targetID)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if created {
|
|
||||||
defer ws.releaseSweepWriter(targetID, w)
|
|
||||||
}
|
|
||||||
|
|
||||||
return w.sweepExpired(expiry)
|
|
||||||
}
|
|
||||||
@@ -1,757 +0,0 @@
|
|||||||
package archive_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"database/sql"
|
|
||||||
"fmt"
|
|
||||||
"io/fs"
|
|
||||||
"log/slog"
|
|
||||||
"net/http"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/google/uuid"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"gorm.io/driver/sqlite"
|
|
||||||
"gorm.io/gorm"
|
|
||||||
_ "modernc.org/sqlite" // Pure Go SQLite driver.
|
|
||||||
"sneak.berlin/go/webhooker/internal/archive"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/gormlog"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
// testContentType is the event content type used in tests.
|
|
||||||
testContentType = "application/json"
|
|
||||||
|
|
||||||
// eventQuery is the query string an archived event carries.
|
|
||||||
eventQuery = "a=1&b=2"
|
|
||||||
)
|
|
||||||
|
|
||||||
func archiveTestLogger() *slog.Logger {
|
|
||||||
return slog.New(slog.NewTextHandler(
|
|
||||||
os.Stderr,
|
|
||||||
&slog.HandlerOptions{Level: slog.LevelDebug},
|
|
||||||
))
|
|
||||||
}
|
|
||||||
|
|
||||||
// openArchiveDBForRead opens an archive file read-only so a
|
|
||||||
// test can inspect the rows the writer persisted.
|
|
||||||
func openArchiveDBForRead(
|
|
||||||
t *testing.T, path string,
|
|
||||||
) *gorm.DB {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
sqlDB, err := sql.Open(
|
|
||||||
"sqlite",
|
|
||||||
fmt.Sprintf("file:%s?mode=ro", path),
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
t.Cleanup(func() { _ = sqlDB.Close() })
|
|
||||||
|
|
||||||
gdb, err := gorm.Open(
|
|
||||||
sqlite.Dialector{Conn: sqlDB},
|
|
||||||
&gorm.Config{Logger: gormlog.New(slog.New(slog.DiscardHandler))},
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
return gdb
|
|
||||||
}
|
|
||||||
|
|
||||||
// archiveFileSuffixes returns the archive file itself and the
|
|
||||||
// SQLite sidecars that accompany an open database. A test that
|
|
||||||
// asserts no archive was created has to check all of them.
|
|
||||||
func archiveFileSuffixes() []string {
|
|
||||||
return []string{"", "-wal", "-shm"}
|
|
||||||
}
|
|
||||||
|
|
||||||
// removeArchiveFiles simulates an operator moving the archive
|
|
||||||
// away by deleting the SQLite file and its sidecar files.
|
|
||||||
func removeArchiveFiles(t *testing.T, path string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
for _, suffix := range []string{
|
|
||||||
"", "-wal", "-shm", "-journal",
|
|
||||||
} {
|
|
||||||
err := os.Remove(path + suffix)
|
|
||||||
if err != nil && !os.IsNotExist(err) {
|
|
||||||
t.Fatalf("removing %s%s: %v", path, suffix, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// assertStoredTime asserts that text, a timestamp as SQLite holds it,
|
|
||||||
// is in the form GORM stores created_at in: the layout of
|
|
||||||
// time.Time.String, with no " m=+…" monotonic clock reading after it.
|
|
||||||
func assertStoredTime(t *testing.T, text string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
_, err := time.Parse(
|
|
||||||
"2006-01-02 15:04:05.999999999 -0700 MST", text,
|
|
||||||
)
|
|
||||||
assert.NoError(t, err, "stored as %q", text)
|
|
||||||
}
|
|
||||||
|
|
||||||
// newEvent returns an event received now with the given body, as the
|
|
||||||
// webhook handler stores it.
|
|
||||||
func newEvent(body string) database.Event {
|
|
||||||
return database.Event{
|
|
||||||
BaseModel: database.BaseModel{
|
|
||||||
ID: uuid.New().String(),
|
|
||||||
CreatedAt: time.Now(),
|
|
||||||
},
|
|
||||||
WebhookID: uuid.New().String(),
|
|
||||||
EntrypointID: uuid.New().String(),
|
|
||||||
Method: http.MethodPost,
|
|
||||||
Headers: `{"Content-Type":["application/json"]}`,
|
|
||||||
Body: body,
|
|
||||||
ContentType: testContentType,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// databaseTargetDelivery returns a delivery of event to a database
|
|
||||||
// target, as the database target hands it to Writers.Write.
|
|
||||||
func databaseTargetDelivery(
|
|
||||||
event database.Event, tgt *database.Target,
|
|
||||||
) *database.Delivery {
|
|
||||||
return &database.Delivery{
|
|
||||||
EventID: event.ID,
|
|
||||||
TargetID: tgt.ID,
|
|
||||||
Event: event,
|
|
||||||
Target: database.Target{
|
|
||||||
Name: tgt.Name,
|
|
||||||
Type: database.TargetTypeDatabase,
|
|
||||||
ArchiveExpiry: tgt.ArchiveExpiry,
|
|
||||||
ArchiveRotation: tgt.ArchiveRotation,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestArchiveWriter_WritesRow(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "archive-wh.db")
|
|
||||||
w := archive.NewExportArchiveWriter(
|
|
||||||
path, archiveTestLogger(), 0,
|
|
||||||
)
|
|
||||||
|
|
||||||
row := archive.ExportArchivedEvent{
|
|
||||||
EventID: "ev-1",
|
|
||||||
WebhookID: "wh-1",
|
|
||||||
EntrypointID: "ep-1",
|
|
||||||
Method: "POST",
|
|
||||||
Headers: `{"X":"Y"}`,
|
|
||||||
Body: `{"hello":"world"}`,
|
|
||||||
ContentType: "application/json",
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NoError(t, w.Write(row, 0))
|
|
||||||
assert.FileExists(t, path)
|
|
||||||
|
|
||||||
var got []archive.ExportArchivedEvent
|
|
||||||
|
|
||||||
require.NoError(t, w.DB().Find(&got).Error)
|
|
||||||
require.Len(t, got, 1)
|
|
||||||
assert.Equal(t, "ev-1", got[0].EventID)
|
|
||||||
assert.Equal(t, "wh-1", got[0].WebhookID)
|
|
||||||
assert.Equal(t, "ep-1", got[0].EntrypointID)
|
|
||||||
assert.Equal(t, row.Method, got[0].Method)
|
|
||||||
assert.Equal(t, row.ContentType, got[0].ContentType)
|
|
||||||
assert.JSONEq(t, `{"hello":"world"}`, got[0].Body)
|
|
||||||
assert.False(t, got[0].ArchivedAt.IsZero())
|
|
||||||
|
|
||||||
var archivedAt string
|
|
||||||
|
|
||||||
require.NoError(t, w.DB().Raw(
|
|
||||||
"SELECT CAST(archived_at AS TEXT) FROM archived_events",
|
|
||||||
).Row().Scan(&archivedAt))
|
|
||||||
assertStoredTime(t, archivedAt)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestArchiveWriter_RecreatesAfterRemoval(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "archive-wh.db")
|
|
||||||
w := archive.NewExportArchiveWriter(
|
|
||||||
path, archiveTestLogger(), 0,
|
|
||||||
)
|
|
||||||
|
|
||||||
require.NoError(t, w.Write(
|
|
||||||
archive.ExportArchivedEvent{EventID: "a"}, 0,
|
|
||||||
))
|
|
||||||
assert.FileExists(t, path)
|
|
||||||
|
|
||||||
// The operator moves the archive away while the handle is
|
|
||||||
// still open.
|
|
||||||
removeArchiveFiles(t, path)
|
|
||||||
require.NoFileExists(t, path)
|
|
||||||
|
|
||||||
// The next write recreates the file with a fresh schema and
|
|
||||||
// only the new row.
|
|
||||||
require.NoError(t, w.Write(
|
|
||||||
archive.ExportArchivedEvent{EventID: "b"}, 0,
|
|
||||||
))
|
|
||||||
assert.FileExists(t, path)
|
|
||||||
|
|
||||||
var got []archive.ExportArchivedEvent
|
|
||||||
|
|
||||||
require.NoError(t, w.DB().Find(&got).Error)
|
|
||||||
require.Len(t, got, 1)
|
|
||||||
assert.Equal(t, "b", got[0].EventID)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveHandleIsWAL closes the last gap in the durability
|
|
||||||
// evidence: the main and per-webhook tiers each assert their journal
|
|
||||||
// mode on a live handle, and the archive tier gets its settings from
|
|
||||||
// the same code path but nothing checked the running file.
|
|
||||||
func TestArchiveHandleIsWAL(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
w := archive.NewExportArchiveWriter(
|
|
||||||
filepath.Join(t.TempDir(), "archive-wal.db"),
|
|
||||||
archiveTestLogger(), 0,
|
|
||||||
)
|
|
||||||
|
|
||||||
require.NoError(t, w.Open(0))
|
|
||||||
|
|
||||||
var mode string
|
|
||||||
|
|
||||||
row := w.DB().Raw("pragma journal_mode").Row()
|
|
||||||
require.NoError(t, row.Scan(&mode))
|
|
||||||
assert.Equal(t, "wal", strings.ToLower(mode))
|
|
||||||
|
|
||||||
var busy string
|
|
||||||
|
|
||||||
row = w.DB().Raw("pragma busy_timeout").Row()
|
|
||||||
require.NoError(t, row.Scan(&busy))
|
|
||||||
assert.Equal(t, "10000", busy)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestStatArchive proves Stat finds no file before the first
|
|
||||||
// write; after a write still held in the -wal, counts the -wal in the
|
|
||||||
// size and takes its later time as the last write; and finds no file
|
|
||||||
// again once the file has been moved away.
|
|
||||||
func TestStatArchive(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "archive-wh.db")
|
|
||||||
|
|
||||||
_, err := archive.Stat(path)
|
|
||||||
require.ErrorIs(t, err, fs.ErrNotExist)
|
|
||||||
|
|
||||||
// With the clock stopped, the reopen debounce never passes, so
|
|
||||||
// the handle stays open after the write.
|
|
||||||
stopped := time.Now()
|
|
||||||
w := archive.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
|
||||||
w.SetNow(func() time.Time { return stopped })
|
|
||||||
|
|
||||||
require.NoError(t, w.Write(archive.ExportArchivedEvent{EventID: "a"}, 0))
|
|
||||||
|
|
||||||
written := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
|
||||||
earlier := written.Add(-time.Hour)
|
|
||||||
require.NoError(t, os.Chtimes(path, earlier, earlier))
|
|
||||||
require.NoError(t, os.Chtimes(path+"-wal", written, written))
|
|
||||||
|
|
||||||
file, err := os.Stat(path)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
wal, err := os.Stat(path + "-wal")
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Positive(t, wal.Size())
|
|
||||||
|
|
||||||
got, err := archive.Stat(path)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, 1, got.Files)
|
|
||||||
assert.Equal(t, file.Size()+wal.Size(), got.Size)
|
|
||||||
assert.True(t, written.Equal(got.Written), got.Written)
|
|
||||||
|
|
||||||
removeArchiveFiles(t, path)
|
|
||||||
|
|
||||||
_, err = archive.Stat(path)
|
|
||||||
require.ErrorIs(t, err, fs.ErrNotExist)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestArchiveWriter_ReopenDebounce(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const debounce = 2 * time.Second
|
|
||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "archive-wh.db")
|
|
||||||
w := archive.NewExportArchiveWriter(
|
|
||||||
path, archiveTestLogger(), debounce,
|
|
||||||
)
|
|
||||||
|
|
||||||
// The writer measures its reopen debounce on this clock, which
|
|
||||||
// only the test moves, so how long the host takes between
|
|
||||||
// writes cannot change the result.
|
|
||||||
now := time.Now()
|
|
||||||
|
|
||||||
w.SetNow(func() time.Time { return now })
|
|
||||||
|
|
||||||
require.NoError(t, w.Write(
|
|
||||||
archive.ExportArchivedEvent{EventID: "a"}, 0,
|
|
||||||
))
|
|
||||||
require.NoError(t, w.Write(
|
|
||||||
archive.ExportArchivedEvent{EventID: "b"}, 0,
|
|
||||||
))
|
|
||||||
|
|
||||||
// Two writes inside the debounce window trigger only the
|
|
||||||
// initial open — no extra close/reopen.
|
|
||||||
assert.Equal(t, 1, w.Reopens())
|
|
||||||
|
|
||||||
now = now.Add(debounce)
|
|
||||||
|
|
||||||
require.NoError(t, w.Write(
|
|
||||||
archive.ExportArchivedEvent{EventID: "c"}, 0,
|
|
||||||
))
|
|
||||||
|
|
||||||
// A write after the window elapses closes and reopens once.
|
|
||||||
assert.Equal(t, 2, w.Reopens())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestArchiveWriter_ExpiryPrune(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "archive-wh.db")
|
|
||||||
w := archive.NewExportArchiveWriter(
|
|
||||||
path, archiveTestLogger(), 0,
|
|
||||||
)
|
|
||||||
|
|
||||||
require.NoError(t, w.Open(0))
|
|
||||||
|
|
||||||
old := archive.ExportArchivedEvent{
|
|
||||||
EventID: "old",
|
|
||||||
ArchivedAt: time.Now().Add(-2 * time.Hour),
|
|
||||||
}
|
|
||||||
fresh := archive.ExportArchivedEvent{
|
|
||||||
EventID: "fresh",
|
|
||||||
ArchivedAt: time.Now(),
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NoError(t, w.DB().Create(&old).Error)
|
|
||||||
require.NoError(t, w.DB().Create(&fresh).Error)
|
|
||||||
|
|
||||||
// Reopening with a one-hour expiry prunes the old row.
|
|
||||||
require.NoError(t, w.Reopen(time.Hour))
|
|
||||||
|
|
||||||
var got []archive.ExportArchivedEvent
|
|
||||||
|
|
||||||
require.NoError(t, w.DB().Find(&got).Error)
|
|
||||||
require.Len(t, got, 1)
|
|
||||||
assert.Equal(t, "fresh", got[0].EventID)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseArchiveExpiry(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
cases := []struct {
|
|
||||||
name string
|
|
||||||
in string
|
|
||||||
want time.Duration
|
|
||||||
wantErr bool
|
|
||||||
}{
|
|
||||||
{"empty", "", 0, false},
|
|
||||||
{"explicit never", "never", 0, false},
|
|
||||||
{"hour", "1h", time.Hour, false},
|
|
||||||
{"thirty days", "720h", 720 * time.Hour, false},
|
|
||||||
{"minutes", "30m", 30 * time.Minute, false},
|
|
||||||
{"unparseable", "nonsense", 0, true},
|
|
||||||
{"days are not a Go unit", "7d", 0, true},
|
|
||||||
{"zero duration", "0s", 0, true},
|
|
||||||
{"bare zero", "0", 0, true},
|
|
||||||
{"negative duration", "-5h", 0, true},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range cases {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
got, err := archive.ParseExpiry(tc.in)
|
|
||||||
if tc.wantErr {
|
|
||||||
require.Error(t, err)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, tc.want, got)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveFileName pins the archive file name and the rules
|
|
||||||
// that make a webhook or target name safe to put in it.
|
|
||||||
func TestArchiveFileName(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const id = "3f2a1c9e-8d4b-4c1a-9e2f-0a1b2c3d4e5f"
|
|
||||||
|
|
||||||
cases := []struct {
|
|
||||||
name string
|
|
||||||
webhook string
|
|
||||||
target string
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
"plain names", "orders", "archive",
|
|
||||||
"archive-orders-archive-" + id + ".db",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"lowercased", "Orders", "Main Archive",
|
|
||||||
"archive-orders-main-archive-" + id + ".db",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"a run of other characters is one dash",
|
|
||||||
`a /\..b`, "c__--d",
|
|
||||||
"archive-a-b-c-d-" + id + ".db",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"no dash at either end", " --orders!! ", "(archive)",
|
|
||||||
"archive-orders-archive-" + id + ".db",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"path separators", "../../etc/passwd", "a/b",
|
|
||||||
"archive-etc-passwd-a-b-" + id + ".db",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"letters outside ASCII are dropped",
|
|
||||||
"Bestellungen Größe", "café",
|
|
||||||
"archive-bestellungen-gr-e-caf-" + id + ".db",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"nothing left is unnamed", "", "!!!",
|
|
||||||
"archive-unnamed-unnamed-" + id + ".db",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cut to 40 characters", strings.Repeat("a", 50), "x",
|
|
||||||
"archive-" + strings.Repeat("a", 40) + "-x-" + id + ".db",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"no dash left by the cut",
|
|
||||||
strings.Repeat("a", 39) + " b", "x",
|
|
||||||
"archive-" + strings.Repeat("a", 39) + "-x-" + id + ".db",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range cases {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
assert.Equal(
|
|
||||||
t, tc.want,
|
|
||||||
archive.FileName(tc.webhook, tc.target, id),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestWrite_EachTargetHasItsOwnArchive proves two
|
|
||||||
// database targets of one webhook archive into separate files.
|
|
||||||
func TestWrite_EachTargetHasItsOwnArchive(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
|
||||||
first := env.seedDatabaseTarget(t, 0, "")
|
|
||||||
second := env.addDatabaseTarget(t, first.WebhookID, 0, "")
|
|
||||||
|
|
||||||
event := newEvent(`{"n":1}`)
|
|
||||||
|
|
||||||
for _, tgt := range []*database.Target{first, second} {
|
|
||||||
require.NoError(t, env.writers.Write(
|
|
||||||
databaseTargetDelivery(event, tgt),
|
|
||||||
))
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NotEqual(
|
|
||||||
t, env.archivePath(first), env.archivePath(second),
|
|
||||||
)
|
|
||||||
assert.Equal(
|
|
||||||
t, []string{event.ID},
|
|
||||||
archivedEventIDs(t, env.archivePath(first)),
|
|
||||||
)
|
|
||||||
assert.Equal(
|
|
||||||
t, []string{event.ID},
|
|
||||||
archivedEventIDs(t, env.archivePath(second)),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRename_MovesTheFile proves a rename moves the archive, rows
|
|
||||||
// and all, and that later writes go to the new name.
|
|
||||||
func TestRename_MovesTheFile(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
|
||||||
tgt := env.seedDatabaseTarget(t, 0, "")
|
|
||||||
oldPath := env.archivePath(tgt)
|
|
||||||
|
|
||||||
first := newEvent(`{"n":1}`)
|
|
||||||
require.NoError(t, env.writers.Write(databaseTargetDelivery(first, tgt)))
|
|
||||||
require.FileExists(t, oldPath)
|
|
||||||
|
|
||||||
require.NoError(
|
|
||||||
t, env.writers.Rename(tgt.ID, "Orders", "Long Term"),
|
|
||||||
)
|
|
||||||
|
|
||||||
newPath := filepath.Join(
|
|
||||||
env.dataDir, "archive-orders-long-term-"+tgt.ID+".db",
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.NoFileExists(t, oldPath)
|
|
||||||
assert.Equal(t, []string{first.ID}, archivedEventIDs(t, newPath))
|
|
||||||
|
|
||||||
second := newEvent(`{"n":2}`)
|
|
||||||
require.NoError(t, env.writers.Write(databaseTargetDelivery(second, tgt)))
|
|
||||||
|
|
||||||
assert.ElementsMatch(
|
|
||||||
t, []string{first.ID, second.ID},
|
|
||||||
archivedEventIDs(t, newPath),
|
|
||||||
)
|
|
||||||
assert.NoFileExists(
|
|
||||||
t, oldPath, "a write after the rename must use the new name",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRename_NeverReplacesAFile plants a file at the new name, once
|
|
||||||
// the .db alone, once a lone -wal and once a lone -shm, and proves
|
|
||||||
// each time that the rename is refused, the planted file survives,
|
|
||||||
// and the archive keeps its name and its rows.
|
|
||||||
func TestRename_NeverReplacesAFile(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, suffix := range archiveFileSuffixes() {
|
|
||||||
t.Run("planted .db"+suffix, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
|
||||||
tgt := env.seedDatabaseTarget(t, 0, "")
|
|
||||||
oldPath := env.archivePath(tgt)
|
|
||||||
|
|
||||||
first := newEvent(`{"n":1}`)
|
|
||||||
require.NoError(
|
|
||||||
t, env.writers.Write(databaseTargetDelivery(first, tgt)),
|
|
||||||
)
|
|
||||||
|
|
||||||
newPath := filepath.Join(
|
|
||||||
env.dataDir, "archive-orders-long-term-"+tgt.ID+".db",
|
|
||||||
)
|
|
||||||
plantedPath := newPath + suffix
|
|
||||||
require.NoError(
|
|
||||||
t, os.WriteFile(plantedPath, []byte("planted"), 0o600),
|
|
||||||
)
|
|
||||||
|
|
||||||
require.ErrorIs(
|
|
||||||
t, env.writers.Rename(tgt.ID, "Orders", "Long Term"),
|
|
||||||
archive.ErrNameTaken,
|
|
||||||
)
|
|
||||||
|
|
||||||
//nolint:gosec // reads the file the test planted under t.TempDir()
|
|
||||||
planted, err := os.ReadFile(plantedPath)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, "planted", string(planted))
|
|
||||||
|
|
||||||
second := newEvent(`{"n":2}`)
|
|
||||||
require.NoError(
|
|
||||||
t, env.writers.Write(databaseTargetDelivery(second, tgt)),
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.ElementsMatch(
|
|
||||||
t, []string{first.ID, second.ID},
|
|
||||||
archivedEventIDs(t, oldPath),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRename_BeforeTheNameIsSaved covers the order the handlers
|
|
||||||
// use: they rename before they save the new name, so a delivery in
|
|
||||||
// between must write under the new name although the main database
|
|
||||||
// still has the old one. It also shows that renaming an archive that
|
|
||||||
// does not exist yet is not an error.
|
|
||||||
func TestRename_BeforeTheNameIsSaved(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
|
||||||
tgt := env.seedDatabaseTarget(t, 0, "")
|
|
||||||
|
|
||||||
require.NoError(
|
|
||||||
t, env.writers.Rename(tgt.ID, "Orders", "Archive"),
|
|
||||||
)
|
|
||||||
|
|
||||||
event := newEvent(`{"n":1}`)
|
|
||||||
require.NoError(t, env.writers.Write(databaseTargetDelivery(event, tgt)))
|
|
||||||
|
|
||||||
assert.FileExists(
|
|
||||||
t,
|
|
||||||
filepath.Join(
|
|
||||||
env.dataDir, "archive-orders-archive-"+tgt.ID+".db",
|
|
||||||
),
|
|
||||||
)
|
|
||||||
assert.NoFileExists(t, env.archivePath(tgt))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveWriter_RenameMovesSidecars proves a rename carries
|
|
||||||
// the -wal and -shm a crash can leave beside an archive no handle
|
|
||||||
// has opened since. SQLite finds them by name, so a -wal left
|
|
||||||
// behind would lose the transactions it holds.
|
|
||||||
func TestArchiveWriter_RenameMovesSidecars(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
oldPath := filepath.Join(dir, "archive-old.db")
|
|
||||||
newPath := filepath.Join(dir, "archive-new.db")
|
|
||||||
|
|
||||||
for _, suffix := range archiveFileSuffixes() {
|
|
||||||
require.NoError(
|
|
||||||
t, os.WriteFile(oldPath+suffix, []byte(suffix), 0o600),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
w := archive.NewExportArchiveWriter(
|
|
||||||
oldPath, archiveTestLogger(), 0,
|
|
||||||
)
|
|
||||||
|
|
||||||
require.NoError(t, w.Rename("archive-new.db"))
|
|
||||||
|
|
||||||
for _, suffix := range archiveFileSuffixes() {
|
|
||||||
assert.NoFileExists(t, oldPath+suffix)
|
|
||||||
assert.FileExists(t, newPath+suffix)
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equal(t, newPath, w.Path())
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveWriter_RenameMovesBackOnFailure makes the -wal fail to
|
|
||||||
// move after the .db has moved, and proves the .db is moved back, so
|
|
||||||
// the archive is never split across two names. The new name is 255
|
|
||||||
// bytes, the longest a file name may be, so the .db can take it but
|
|
||||||
// the -wal, four bytes longer, cannot.
|
|
||||||
func TestArchiveWriter_RenameMovesBackOnFailure(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
oldPath := filepath.Join(dir, "archive-old.db")
|
|
||||||
newName := strings.Repeat("a", 252) + ".db"
|
|
||||||
|
|
||||||
for _, suffix := range archiveFileSuffixes() {
|
|
||||||
require.NoError(
|
|
||||||
t, os.WriteFile(oldPath+suffix, []byte(suffix), 0o600),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
w := archive.NewExportArchiveWriter(
|
|
||||||
oldPath, archiveTestLogger(), 0,
|
|
||||||
)
|
|
||||||
|
|
||||||
require.Error(t, w.Rename(newName))
|
|
||||||
|
|
||||||
for _, suffix := range archiveFileSuffixes() {
|
|
||||||
assert.FileExists(t, oldPath+suffix)
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.NoFileExists(t, filepath.Join(dir, newName))
|
|
||||||
assert.Equal(t, oldPath, w.Path())
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveWriter_RenameMovesBackEveryFile renames a target with a
|
|
||||||
// file without a period and a file for a month, and proves that when
|
|
||||||
// the month's file fails to move, the file already moved is moved back:
|
|
||||||
// both files are under the old name with their rows, and nothing is
|
|
||||||
// under the new name. The new name is 251 bytes, so the file without a
|
|
||||||
// period, with its -wal and -shm, can take it, but the month's file,
|
|
||||||
// eight bytes longer, cannot.
|
|
||||||
func TestArchiveWriter_RenameMovesBackEveryFile(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
oldPath := filepath.Join(dir, "archive-old.db")
|
|
||||||
monthPath := filepath.Join(dir, "archive-old-2026-03.db")
|
|
||||||
w := archive.NewExportArchiveWriter(oldPath, archiveTestLogger(), 0)
|
|
||||||
|
|
||||||
require.NoError(t, w.WritePeriod(
|
|
||||||
archive.ExportArchivedEvent{EventID: "in-none"}, 0, "",
|
|
||||||
))
|
|
||||||
require.NoError(t, w.WritePeriod(
|
|
||||||
archive.ExportArchivedEvent{EventID: "in-month"}, 0, "2026-03",
|
|
||||||
))
|
|
||||||
|
|
||||||
require.Error(t, w.Rename(strings.Repeat("a", 248)+".db"))
|
|
||||||
|
|
||||||
assert.Equal(t, []string{"in-none"}, archivedEventIDs(t, oldPath))
|
|
||||||
assert.Equal(t, []string{"in-month"}, archivedEventIDs(t, monthPath))
|
|
||||||
|
|
||||||
entries, err := os.ReadDir(dir)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
for _, entry := range entries {
|
|
||||||
assert.True(t, strings.HasPrefix(entry.Name(), "archive-old"),
|
|
||||||
"%s is not under the old name", entry.Name())
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equal(t, oldPath, w.Path())
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRenameLock_HeldByOneEditAtATime holds the rename lock, as an edit
|
|
||||||
// does from loading the stored names until it has saved them. While it
|
|
||||||
// is held, the edit's rename and an archive write still finish, since
|
|
||||||
// neither takes it, and the event ends up under the new name whichever
|
|
||||||
// ran first. A second edit waits for the lock until the first lets go.
|
|
||||||
func TestRenameLock_HeldByOneEditAtATime(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
|
||||||
tgt := env.seedDatabaseTarget(t, 0, "")
|
|
||||||
event := newEvent(`{"n":1}`)
|
|
||||||
|
|
||||||
env.writers.RenameLock().Lock()
|
|
||||||
|
|
||||||
finished := make(chan error, 2)
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
finished <- env.writers.Rename(tgt.ID, "Orders", "Long Term")
|
|
||||||
}()
|
|
||||||
go func() {
|
|
||||||
finished <- env.writers.Write(databaseTargetDelivery(event, tgt))
|
|
||||||
}()
|
|
||||||
|
|
||||||
for range 2 {
|
|
||||||
select {
|
|
||||||
case err := <-finished:
|
|
||||||
require.NoError(t, err)
|
|
||||||
case <-time.After(10 * time.Second):
|
|
||||||
t.Fatal("a rename or a write waited for the rename lock")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
newPath := filepath.Join(
|
|
||||||
env.dataDir, "archive-orders-long-term-"+tgt.ID+".db",
|
|
||||||
)
|
|
||||||
assert.Equal(t, []string{event.ID}, archivedEventIDs(t, newPath))
|
|
||||||
|
|
||||||
secondEdit := make(chan struct{})
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
env.writers.RenameLock().Lock()
|
|
||||||
close(secondEdit)
|
|
||||||
env.writers.RenameLock().Unlock()
|
|
||||||
}()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-secondEdit:
|
|
||||||
t.Fatal("a second edit took the rename lock while it was held")
|
|
||||||
case <-time.After(100 * time.Millisecond):
|
|
||||||
}
|
|
||||||
|
|
||||||
env.writers.RenameLock().Unlock()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-secondEdit:
|
|
||||||
case <-time.After(10 * time.Second):
|
|
||||||
t.Fatal("a second edit did not get the rename lock once released")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,388 +0,0 @@
|
|||||||
package archive
|
|
||||||
|
|
||||||
import (
|
|
||||||
"compress/gzip"
|
|
||||||
"context"
|
|
||||||
"database/sql"
|
|
||||||
"encoding/base64"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"io/fs"
|
|
||||||
"log/slog"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
"unicode/utf8"
|
|
||||||
|
|
||||||
"gorm.io/driver/sqlite"
|
|
||||||
"gorm.io/gorm"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/gormlog"
|
|
||||||
)
|
|
||||||
|
|
||||||
// archiveTableQuery counts the archive's table: 0 when the archive
|
|
||||||
// writer has created the file but not yet the table in it.
|
|
||||||
const archiveTableQuery = "SELECT count(*) FROM sqlite_master " +
|
|
||||||
"WHERE type = 'table' AND name = 'archived_events'"
|
|
||||||
|
|
||||||
// Path returns where a database target's archive file is: in
|
|
||||||
// the data directory, beside the webhook's event database, under the
|
|
||||||
// name FileName gives it.
|
|
||||||
func Path(
|
|
||||||
dbMgr *database.WebhookDBManager,
|
|
||||||
webhook *database.Webhook,
|
|
||||||
target *database.Target,
|
|
||||||
) string {
|
|
||||||
return filepath.Join(
|
|
||||||
filepath.Dir(dbMgr.DBPath(webhook.ID)),
|
|
||||||
FileName(webhook.Name, target.Name, target.ID),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExportFileName returns the name a database target's archive
|
|
||||||
// downloads under:
|
|
||||||
// archive-WEBHOOKNAME-TARGETNAME-YYYYMMDDTHHMMSSZ.json.gz, the names
|
|
||||||
// made safe as in FileName and the time in UTC.
|
|
||||||
func ExportFileName(
|
|
||||||
webhookName, targetName string, at time.Time,
|
|
||||||
) string {
|
|
||||||
return "archive-" + archiveNamePart(webhookName) + "-" +
|
|
||||||
archiveNamePart(targetName) + "-" +
|
|
||||||
at.UTC().Format("20060102T150405Z") + ".json.gz"
|
|
||||||
}
|
|
||||||
|
|
||||||
// Export is a database target's archive listed for download. It
|
|
||||||
// opens one of the target's files at a time, only when its rows are
|
|
||||||
// about to be written out, and closes it before it opens the next, so
|
|
||||||
// an export holds at most one file open however many the target has.
|
|
||||||
//
|
|
||||||
// Each file is read on its own connection inside one read-only
|
|
||||||
// transaction, so its rows are written out as the file stood when it
|
|
||||||
// was opened. Archives are in WAL mode, where a reader works from a
|
|
||||||
// snapshot and never blocks a writer: archive writes go on while a file
|
|
||||||
// is open, and the export does not see them. SQLite cannot checkpoint
|
|
||||||
// a -wal past an open snapshot, so the open file's -wal grows until the
|
|
||||||
// export has written that file out.
|
|
||||||
type Export struct {
|
|
||||||
// periods are the periods of the target's files when the export
|
|
||||||
// was listed, "" for the file without one, in the order
|
|
||||||
// archiveFiles lists them.
|
|
||||||
periods []string
|
|
||||||
|
|
||||||
// lock is held while currentPath is called and a file is opened,
|
|
||||||
// so that a rename, which holds it too, cannot move the file in
|
|
||||||
// between.
|
|
||||||
lock sync.Locker
|
|
||||||
|
|
||||||
// currentPath returns the path Path gives the target under
|
|
||||||
// the names stored for it now, which a rename may have changed since
|
|
||||||
// the export was listed.
|
|
||||||
currentPath func() (string, error)
|
|
||||||
|
|
||||||
log *slog.Logger
|
|
||||||
}
|
|
||||||
|
|
||||||
// exportFile is one archive file opened for an export.
|
|
||||||
type exportFile struct {
|
|
||||||
db *sql.DB
|
|
||||||
tx *gorm.DB
|
|
||||||
|
|
||||||
// period is the period in the file's name, "" for none.
|
|
||||||
period string
|
|
||||||
|
|
||||||
// empty is true for a file without the archive's table yet.
|
|
||||||
empty bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// exportedName is how an export names its webhook and its target.
|
|
||||||
type exportedName struct {
|
|
||||||
ID string `json:"id"`
|
|
||||||
Name string `json:"name"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewExport lists a database target's archive files for export,
|
|
||||||
// given the path Path gives it (see archiveFiles). It opens none
|
|
||||||
// of them. Its caller holds lock, which every rename of the target's
|
|
||||||
// files runs under (Writers.RenameLock returns it), from reading the names
|
|
||||||
// path is made of until it returns, so the files it lists are the ones
|
|
||||||
// those names give.
|
|
||||||
//
|
|
||||||
// WriteGzipJSON, called without lock held, finds each file again by its
|
|
||||||
// period under the path currentPath gives, holding lock while it does
|
|
||||||
// and while it opens the file, so a rename during the export loses no
|
|
||||||
// file. A file that is gone by then, emptied by the sweep or moved
|
|
||||||
// away, is skipped. The export never creates a file: with no files, it
|
|
||||||
// has no rows.
|
|
||||||
func NewExport(
|
|
||||||
path string,
|
|
||||||
lock sync.Locker,
|
|
||||||
currentPath func() (string, error),
|
|
||||||
log *slog.Logger,
|
|
||||||
) (*Export, error) {
|
|
||||||
files, err := archiveFiles(path)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
x := &Export{lock: lock, currentPath: currentPath, log: log}
|
|
||||||
|
|
||||||
for _, file := range files {
|
|
||||||
x.periods = append(x.periods, file.period)
|
|
||||||
}
|
|
||||||
|
|
||||||
return x, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// openExportFile opens one archive file for an export and takes its
|
|
||||||
// snapshot. The transaction lasts as long as ctx does.
|
|
||||||
func openExportFile(
|
|
||||||
ctx context.Context, file archiveFile, log *slog.Logger,
|
|
||||||
) (*exportFile, error) {
|
|
||||||
db, err := database.OpenSQLite(file.path, archiveModeExisting)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("opening archive %s: %w", file.path, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
gdb, err := gorm.Open(
|
|
||||||
sqlite.Dialector{Conn: db}, &gorm.Config{
|
|
||||||
// Never leave this at GORM's default. See
|
|
||||||
// internal/gormlog.
|
|
||||||
Logger: gormlog.New(log),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
_ = db.Close()
|
|
||||||
|
|
||||||
return nil, fmt.Errorf("opening archive %s: %w", file.path, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ReadOnly makes the driver begin a deferred transaction in place
|
|
||||||
// of the BEGIN IMMEDIATE the connection string asks for, so the
|
|
||||||
// export never takes the archive's write lock.
|
|
||||||
tx := gdb.WithContext(ctx).Begin(&sql.TxOptions{ReadOnly: true})
|
|
||||||
if tx.Error != nil {
|
|
||||||
_ = db.Close()
|
|
||||||
|
|
||||||
return nil, fmt.Errorf(
|
|
||||||
"reading archive %s: %w", file.path, tx.Error,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The transaction's first read is what takes the snapshot.
|
|
||||||
var tables int
|
|
||||||
|
|
||||||
err = tx.Raw(archiveTableQuery).Row().Scan(&tables)
|
|
||||||
if err != nil {
|
|
||||||
_ = tx.Rollback()
|
|
||||||
_ = db.Close()
|
|
||||||
|
|
||||||
return nil, fmt.Errorf("reading archive %s: %w", file.path, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return &exportFile{
|
|
||||||
db: db, tx: tx, period: file.period, empty: tables == 0,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// WriteGzipJSON writes the export to w as one gzipped JSON object:
|
|
||||||
// webhook and target, each an id and a name; exported_at; and
|
|
||||||
// archived_events, one object per archived row, keyed by column name,
|
|
||||||
// the files in the order archiveFiles lists them. A row from a file
|
|
||||||
// named for a period has "period" beside its columns. A body that is
|
|
||||||
// not valid UTF-8 cannot be a JSON string, so it is written in base64,
|
|
||||||
// with "body_encoding": "base64" beside it.
|
|
||||||
//
|
|
||||||
// Each row is written out before the next is read, so neither the
|
|
||||||
// archive nor its JSON is ever held in memory whole, and each file is
|
|
||||||
// closed once its rows are written, before the next is opened. When it
|
|
||||||
// returns, no file is open. After an error the gzip stream is left
|
|
||||||
// unfinished, so what was written does not decompress as a whole file.
|
|
||||||
func (x *Export) WriteGzipJSON(
|
|
||||||
ctx context.Context,
|
|
||||||
w io.Writer,
|
|
||||||
webhook *database.Webhook,
|
|
||||||
target *database.Target,
|
|
||||||
exportedAt time.Time,
|
|
||||||
) error {
|
|
||||||
head, err := json.Marshal(map[string]any{
|
|
||||||
"webhook": exportedName{ID: webhook.ID, Name: webhook.Name},
|
|
||||||
"target": exportedName{ID: target.ID, Name: target.Name},
|
|
||||||
"exported_at": exportedAt.UTC(),
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("encoding archive export: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
zw := gzip.NewWriter(w)
|
|
||||||
|
|
||||||
err = x.writeJSON(ctx, zw, head)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("writing archive export: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return zw.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
// openFile finds the target's archive file for period under the path
|
|
||||||
// currentPath gives now, and opens it for the export, holding x.lock
|
|
||||||
// for both. For a file that is gone, the error wraps fs.ErrNotExist.
|
|
||||||
func (x *Export) openFile(
|
|
||||||
ctx context.Context, period string,
|
|
||||||
) (*exportFile, error) {
|
|
||||||
x.lock.Lock()
|
|
||||||
defer x.lock.Unlock()
|
|
||||||
|
|
||||||
path, err := x.currentPath()
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("finding archive file: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
file := archiveFile{path: archivePeriodPath(path, period), period: period}
|
|
||||||
|
|
||||||
_, err = os.Stat(file.path)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return openExportFile(ctx, file, x.log)
|
|
||||||
}
|
|
||||||
|
|
||||||
// close ends the file's transaction and closes its connection.
|
|
||||||
func (f *exportFile) close() error {
|
|
||||||
_ = f.tx.Rollback()
|
|
||||||
|
|
||||||
return f.db.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeJSON writes head with archived_events added as its last key,
|
|
||||||
// the rows going into it one at a time.
|
|
||||||
func (x *Export) writeJSON(
|
|
||||||
ctx context.Context, w io.Writer, head []byte,
|
|
||||||
) error {
|
|
||||||
// head goes out without its closing brace, so that
|
|
||||||
// archived_events can follow it.
|
|
||||||
_, err := w.Write(head[:len(head)-1])
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = io.WriteString(w, `,"archived_events":[`)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = x.writeRows(ctx, w)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = io.WriteString(w, "\n]}\n")
|
|
||||||
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeRows writes the archived rows of each file to w, one per line,
|
|
||||||
// separated by commas, opening each file in turn and closing it once
|
|
||||||
// its rows are written.
|
|
||||||
func (x *Export) writeRows(ctx context.Context, w io.Writer) error {
|
|
||||||
sep := "\n"
|
|
||||||
|
|
||||||
for _, period := range x.periods {
|
|
||||||
f, err := x.openFile(ctx, period)
|
|
||||||
if errors.Is(err, fs.ErrNotExist) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
sep, err = f.writeRows(ctx, w, sep)
|
|
||||||
|
|
||||||
err = errors.Join(err, f.close())
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeRows writes the file's archived rows to w, oldest first, the
|
|
||||||
// first after sep and each other after ",\n". It returns what goes
|
|
||||||
// before the next row: sep again when the file had no rows.
|
|
||||||
func (f *exportFile) writeRows(
|
|
||||||
ctx context.Context, w io.Writer, sep string,
|
|
||||||
) (string, error) {
|
|
||||||
if f.empty {
|
|
||||||
return sep, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
rows, err := f.tx.WithContext(ctx).
|
|
||||||
Model(&archivedEvent{}).Order("id").Rows()
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() { _ = rows.Close() }()
|
|
||||||
|
|
||||||
for ; rows.Next(); sep = ",\n" {
|
|
||||||
var ev archivedEvent
|
|
||||||
|
|
||||||
err = f.tx.ScanRows(rows, &ev)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = io.WriteString(w, sep)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = writeRow(w, &ev, f.period)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return sep, rows.Err()
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeRow writes an archived row to w as a JSON object keyed by
|
|
||||||
// column name, its body in base64 when it is not valid UTF-8, with
|
|
||||||
// the period of its file beside them unless that is "".
|
|
||||||
func writeRow(w io.Writer, ev *archivedEvent, period string) error {
|
|
||||||
row := map[string]any{
|
|
||||||
"id": ev.ID,
|
|
||||||
"event_id": ev.EventID,
|
|
||||||
"webhook_id": ev.WebhookID,
|
|
||||||
"entrypoint_id": ev.EntrypointID,
|
|
||||||
"method": ev.Method,
|
|
||||||
"raw_query": ev.RawQuery,
|
|
||||||
"headers": ev.Headers,
|
|
||||||
"body": ev.Body,
|
|
||||||
"content_type": ev.ContentType,
|
|
||||||
"archived_at": ev.ArchivedAt.UTC(),
|
|
||||||
}
|
|
||||||
|
|
||||||
if !utf8.ValidString(ev.Body) {
|
|
||||||
row["body"] = base64.StdEncoding.EncodeToString([]byte(ev.Body))
|
|
||||||
row["body_encoding"] = "base64"
|
|
||||||
}
|
|
||||||
|
|
||||||
if period != "" {
|
|
||||||
row["period"] = period
|
|
||||||
}
|
|
||||||
|
|
||||||
line, err := json.Marshal(row)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = w.Write(line)
|
|
||||||
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
@@ -1,573 +0,0 @@
|
|||||||
package archive_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bufio"
|
|
||||||
"bytes"
|
|
||||||
"compress/gzip"
|
|
||||||
"crypto/rand"
|
|
||||||
"encoding/base64"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"runtime"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/webhooker/internal/archive"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The webhook and the target the export tests' archives belong to.
|
|
||||||
const (
|
|
||||||
exportWebhookID = "wh-export"
|
|
||||||
exportWebhookName = "Orders (EU)"
|
|
||||||
exportTargetID = "tgt-export"
|
|
||||||
exportTargetName = "Long-term archive"
|
|
||||||
)
|
|
||||||
|
|
||||||
// binaryBody is a body that is not valid UTF-8.
|
|
||||||
const binaryBody = "\xff\xfe\x00\x01binary\x80"
|
|
||||||
|
|
||||||
// writeExportTo writes export to w as the archive of the export tests'
|
|
||||||
// webhook and target, exported at 2026-10-02T12:03:04Z.
|
|
||||||
func writeExportTo(
|
|
||||||
t *testing.T, export *archive.Export, w io.Writer,
|
|
||||||
) error {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
return export.WriteGzipJSON(
|
|
||||||
t.Context(), w,
|
|
||||||
&database.Webhook{
|
|
||||||
BaseModel: database.BaseModel{ID: exportWebhookID},
|
|
||||||
Name: exportWebhookName,
|
|
||||||
},
|
|
||||||
&database.Target{
|
|
||||||
BaseModel: database.BaseModel{ID: exportTargetID},
|
|
||||||
Name: exportTargetName,
|
|
||||||
},
|
|
||||||
time.Date(2026, 10, 2, 12, 3, 4, 0, time.UTC),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// listExport lists the archive at path for export, as the archive of a
|
|
||||||
// target whose names do not change.
|
|
||||||
func listExport(t *testing.T, path string) *archive.Export {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
return newExport(t, path, &sync.Mutex{}, func() (string, error) {
|
|
||||||
return path, nil
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// newExport lists the archive at path for export, to find each file
|
|
||||||
// again under the path currentPath gives, holding lock while it does.
|
|
||||||
// Nothing else takes lock while it lists, so it does not hold lock.
|
|
||||||
func newExport(
|
|
||||||
t *testing.T,
|
|
||||||
path string,
|
|
||||||
lock sync.Locker,
|
|
||||||
currentPath func() (string, error),
|
|
||||||
) *archive.Export {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
export, err := archive.NewExport(
|
|
||||||
path, lock, currentPath, archiveTestLogger(),
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
return export
|
|
||||||
}
|
|
||||||
|
|
||||||
// exportArchive runs a whole export of the archive at path and returns
|
|
||||||
// its JSON, decompressed and parsed.
|
|
||||||
func exportArchive(t *testing.T, path string) map[string]any {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
return writeExport(t, listExport(t, path))
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeExport writes an opened export and returns its JSON,
|
|
||||||
// decompressed and parsed. Reading to the end makes the gzip reader
|
|
||||||
// check that the stream was finished.
|
|
||||||
func writeExport(
|
|
||||||
t *testing.T, export *archive.Export,
|
|
||||||
) map[string]any {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
|
|
||||||
require.NoError(t, writeExportTo(t, export, &buf))
|
|
||||||
|
|
||||||
zr, err := gzip.NewReader(&buf)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
raw, err := io.ReadAll(zr)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
var got map[string]any
|
|
||||||
|
|
||||||
require.NoError(t, json.Unmarshal(raw, &got))
|
|
||||||
|
|
||||||
return got
|
|
||||||
}
|
|
||||||
|
|
||||||
// exportedEvents returns an export's archived_events.
|
|
||||||
func exportedEvents(t *testing.T, got map[string]any) []map[string]any {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
list, ok := got["archived_events"].([]any)
|
|
||||||
require.True(t, ok, "archived_events must be an array: %v", got)
|
|
||||||
|
|
||||||
events := make([]map[string]any, len(list))
|
|
||||||
|
|
||||||
for i, v := range list {
|
|
||||||
events[i], ok = v.(map[string]any)
|
|
||||||
require.True(t, ok, "an archived event must be an object: %v", v)
|
|
||||||
}
|
|
||||||
|
|
||||||
return events
|
|
||||||
}
|
|
||||||
|
|
||||||
// exportedEventIDs returns the event_id of each of an export's
|
|
||||||
// archived_events.
|
|
||||||
func exportedEventIDs(t *testing.T, got map[string]any) []string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
events := exportedEvents(t, got)
|
|
||||||
ids := make([]string, 0, len(events))
|
|
||||||
|
|
||||||
for _, ev := range events {
|
|
||||||
ids = append(ids, fmt.Sprint(ev["event_id"]))
|
|
||||||
}
|
|
||||||
|
|
||||||
return ids
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveExport_MatchesStoredRows proves an export holds the
|
|
||||||
// webhook, the target, the time, and every column of every stored
|
|
||||||
// row: a body that is valid UTF-8 as a string, and one that is not in
|
|
||||||
// base64, marked as such.
|
|
||||||
func TestArchiveExport_MatchesStoredRows(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "archive.db")
|
|
||||||
w := archive.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
|
||||||
bodies := []string{`{"order":1}`, "plain text", "", binaryBody}
|
|
||||||
|
|
||||||
for i, body := range bodies {
|
|
||||||
require.NoError(t, w.Write(archive.ExportArchivedEvent{
|
|
||||||
EventID: fmt.Sprintf("ev-%d", i),
|
|
||||||
WebhookID: exportWebhookID,
|
|
||||||
EntrypointID: "ep-1",
|
|
||||||
Method: "POST",
|
|
||||||
RawQuery: eventQuery,
|
|
||||||
Headers: `{"X-Test":["yes"]}`,
|
|
||||||
Body: body,
|
|
||||||
ContentType: testContentType,
|
|
||||||
}, 0))
|
|
||||||
}
|
|
||||||
|
|
||||||
var stored []archive.ExportArchivedEvent
|
|
||||||
|
|
||||||
require.NoError(t, openArchiveDBForRead(t, path).
|
|
||||||
Order("id").Find(&stored).Error)
|
|
||||||
|
|
||||||
got := exportArchive(t, path)
|
|
||||||
|
|
||||||
assert.Equal(t,
|
|
||||||
map[string]any{"id": exportWebhookID, "name": exportWebhookName},
|
|
||||||
got["webhook"],
|
|
||||||
)
|
|
||||||
assert.Equal(t,
|
|
||||||
map[string]any{"id": exportTargetID, "name": exportTargetName},
|
|
||||||
got["target"],
|
|
||||||
)
|
|
||||||
assert.Equal(t, "2026-10-02T12:03:04Z", got["exported_at"])
|
|
||||||
|
|
||||||
events := exportedEvents(t, got)
|
|
||||||
require.Len(t, events, len(bodies))
|
|
||||||
|
|
||||||
for i, row := range stored {
|
|
||||||
assertExportedRow(t, row, events[i])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// assertExportedRow checks that ev, from an export, holds every column
|
|
||||||
// of the stored row.
|
|
||||||
func assertExportedRow(
|
|
||||||
t *testing.T, row archive.ExportArchivedEvent, ev map[string]any,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
archivedAt, err := time.Parse(
|
|
||||||
time.RFC3339Nano, fmt.Sprint(ev["archived_at"]),
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.True(t, archivedAt.Equal(row.ArchivedAt))
|
|
||||||
|
|
||||||
assert.EqualValues(t, row.ID, ev["id"])
|
|
||||||
assert.Equal(t, row.EventID, ev["event_id"])
|
|
||||||
assert.Equal(t, row.WebhookID, ev["webhook_id"])
|
|
||||||
assert.Equal(t, row.EntrypointID, ev["entrypoint_id"])
|
|
||||||
assert.Equal(t, row.Method, ev["method"])
|
|
||||||
assert.Equal(t, row.RawQuery, ev["raw_query"])
|
|
||||||
assert.Equal(t, row.Headers, ev["headers"])
|
|
||||||
assert.Equal(t, row.ContentType, ev["content_type"])
|
|
||||||
|
|
||||||
if row.Body != binaryBody {
|
|
||||||
assert.Equal(t, row.Body, ev["body"])
|
|
||||||
assert.Len(t, ev, 10, "the ten columns and nothing else: %v", ev)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
body, err := base64.StdEncoding.DecodeString(fmt.Sprint(ev["body"]))
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, binaryBody, string(body))
|
|
||||||
assert.Equal(t, "base64", ev["body_encoding"])
|
|
||||||
assert.Len(t, ev, 11, "the ten columns and body_encoding: %v", ev)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveExport_Empty proves an archive with nothing in it exports
|
|
||||||
// as an empty archived_events: no file, which the export must not
|
|
||||||
// create; a file the archive writer has not yet put its table in; and
|
|
||||||
// a table with no rows.
|
|
||||||
func TestArchiveExport_Empty(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
missing := filepath.Join(dir, "missing.db")
|
|
||||||
noTable := filepath.Join(dir, "no-table.db")
|
|
||||||
noRows := filepath.Join(dir, "no-rows.db")
|
|
||||||
|
|
||||||
require.NoError(t, os.WriteFile(noTable, nil, 0o600))
|
|
||||||
require.NoError(t,
|
|
||||||
archive.NewExportArchiveWriter(noRows, archiveTestLogger(), 0).
|
|
||||||
Open(0),
|
|
||||||
)
|
|
||||||
|
|
||||||
for _, path := range []string{missing, noTable, noRows} {
|
|
||||||
assert.Empty(t, exportedEvents(t, exportArchive(t, path)), path)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, suffix := range archiveFileSuffixes() {
|
|
||||||
assert.NoFileExists(t, missing+suffix)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// unlockHook is a sync.Locker that runs fn each time it is unlocked. An
|
|
||||||
// export unlocks its lock right after it opens a file.
|
|
||||||
type unlockHook struct {
|
|
||||||
sync.Mutex
|
|
||||||
|
|
||||||
fn func()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u *unlockHook) Unlock() {
|
|
||||||
u.Mutex.Unlock()
|
|
||||||
u.fn()
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveExport_ReadsOneSnapshot proves an export writes a file
|
|
||||||
// out as it was when the export opened it, and holds up no archive
|
|
||||||
// write: a row written after the export was listed but before the file
|
|
||||||
// was opened is in the export, and one written while the file is open
|
|
||||||
// is stored, and is not. A write held up for the whole busy timeout
|
|
||||||
// would fail.
|
|
||||||
func TestArchiveExport_ReadsOneSnapshot(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "archive.db")
|
|
||||||
w := archive.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
|
||||||
|
|
||||||
require.NoError(t, w.Write(archive.ExportArchivedEvent{EventID: "listed"}, 0))
|
|
||||||
|
|
||||||
opened := &unlockHook{fn: func() {
|
|
||||||
require.NoError(t, w.Write(archive.ExportArchivedEvent{EventID: "during"}, 0))
|
|
||||||
}}
|
|
||||||
export := newExport(t, path, opened, func() (string, error) {
|
|
||||||
return path, nil
|
|
||||||
})
|
|
||||||
|
|
||||||
require.NoError(t, w.Write(archive.ExportArchivedEvent{EventID: "before-open"}, 0))
|
|
||||||
|
|
||||||
assert.Equal(t,
|
|
||||||
[]string{"listed", "before-open"},
|
|
||||||
exportedEventIDs(t, writeExport(t, export)),
|
|
||||||
)
|
|
||||||
|
|
||||||
var stored int64
|
|
||||||
|
|
||||||
require.NoError(t, openArchiveDBForRead(t, path).
|
|
||||||
Model(&archive.ExportArchivedEvent{}).Count(&stored).Error)
|
|
||||||
assert.Equal(t, int64(3), stored)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveExport_FindsFilesAfterRename proves that renaming the
|
|
||||||
// archive after an export has listed it, as renaming its webhook or
|
|
||||||
// target does, loses no file: the export finds each file again by its
|
|
||||||
// period under the new name. A file moved away by then is skipped.
|
|
||||||
func TestArchiveExport_FindsFilesAfterRename(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
path := filepath.Join(dir, "archive-old.db")
|
|
||||||
w := archive.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
|
||||||
|
|
||||||
for _, period := range []string{"", dayPeriod, hourPeriod} {
|
|
||||||
require.NoError(t, w.WritePeriod(
|
|
||||||
archive.ExportArchivedEvent{EventID: "in-" + period}, 0, period,
|
|
||||||
))
|
|
||||||
}
|
|
||||||
|
|
||||||
current := path
|
|
||||||
export := newExport(t, path, &sync.Mutex{}, func() (string, error) {
|
|
||||||
return current, nil
|
|
||||||
})
|
|
||||||
|
|
||||||
require.NoError(t, w.Rename("archive-new.db"))
|
|
||||||
|
|
||||||
current = filepath.Join(dir, "archive-new.db")
|
|
||||||
|
|
||||||
removeArchiveFiles(t, periodPath(current, dayPeriod))
|
|
||||||
|
|
||||||
assert.Equal(t,
|
|
||||||
[]string{"in-", "in-" + hourPeriod},
|
|
||||||
exportedEventIDs(t, writeExport(t, export)),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveExport_EveryFileOldestFirst writes a row to a target's
|
|
||||||
// file without a period and to its files for a month, an hour and a
|
|
||||||
// day, and proves the export holds every row: the file without a
|
|
||||||
// period first, then the others oldest period first, each row from a
|
|
||||||
// file named for a period carrying that period. A file made after the
|
|
||||||
// export was listed is not in it.
|
|
||||||
func TestArchiveExport_EveryFileOldestFirst(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "archive-wh.db")
|
|
||||||
w := archive.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
|
||||||
|
|
||||||
// Written in an order that is not the export's.
|
|
||||||
for _, period := range []string{nextDayPeriod, "", hourPeriod, "2026-03"} {
|
|
||||||
require.NoError(t, w.WritePeriod(
|
|
||||||
archive.ExportArchivedEvent{EventID: "in-" + period}, 0, period,
|
|
||||||
))
|
|
||||||
}
|
|
||||||
|
|
||||||
export := listExport(t, path)
|
|
||||||
|
|
||||||
require.NoError(t, w.WritePeriod(
|
|
||||||
archive.ExportArchivedEvent{EventID: "later"}, 0, "2026-03-06",
|
|
||||||
))
|
|
||||||
|
|
||||||
events := exportedEvents(t, writeExport(t, export))
|
|
||||||
ids := make([]string, 0, len(events))
|
|
||||||
periods := make([]any, 0, len(events))
|
|
||||||
|
|
||||||
for _, ev := range events {
|
|
||||||
ids = append(ids, fmt.Sprint(ev["event_id"]))
|
|
||||||
periods = append(periods, ev["period"])
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equal(t,
|
|
||||||
[]string{"in-", "in-2026-03", "in-" + hourPeriod, "in-" + nextDayPeriod},
|
|
||||||
ids,
|
|
||||||
)
|
|
||||||
assert.Equal(t,
|
|
||||||
[]any{nil, "2026-03", hourPeriod, nextDayPeriod}, periods,
|
|
||||||
)
|
|
||||||
assert.NotContains(t, events[0], "period",
|
|
||||||
"a row from the file without a period has no period")
|
|
||||||
}
|
|
||||||
|
|
||||||
// openFilesPeak is an io.Writer that discards what it is given and
|
|
||||||
// records the most archive files in dir the process had open at any
|
|
||||||
// write, as /proc/self/fd lists the files a process has open.
|
|
||||||
type openFilesPeak struct {
|
|
||||||
dir string
|
|
||||||
max int
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *openFilesPeak) Write(b []byte) (int, error) {
|
|
||||||
fds, err := os.ReadDir("/proc/self/fd")
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
open := map[string]bool{}
|
|
||||||
|
|
||||||
for _, fd := range fds {
|
|
||||||
file, err := os.Readlink(filepath.Join("/proc/self/fd", fd.Name()))
|
|
||||||
if err == nil && filepath.Dir(file) == p.dir &&
|
|
||||||
strings.HasSuffix(file, ".db") {
|
|
||||||
open[file] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
p.max = max(p.max, len(open))
|
|
||||||
|
|
||||||
return len(b), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveExport_OneFileOpenAtATime exports a target with a file for
|
|
||||||
// each of 24 hours and proves the export never had more than one of
|
|
||||||
// them open, and had one open while it wrote. Each file holds a row of
|
|
||||||
// 48 KiB of random base64, which gzip shrinks little, so the export
|
|
||||||
// writes output while it reads each file.
|
|
||||||
func TestArchiveExport_OneFileOpenAtATime(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
if runtime.GOOS != "linux" {
|
|
||||||
t.Skip("only Linux lists a process's open files in /proc/self/fd")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Readlink gives each open file's path with no symbolic link in it.
|
|
||||||
dir, err := filepath.EvalSymlinks(t.TempDir())
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
path := filepath.Join(dir, "archive-wh.db")
|
|
||||||
w := archive.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
|
||||||
random := make([]byte, 36<<10)
|
|
||||||
|
|
||||||
for hour := range 24 {
|
|
||||||
_, _ = rand.Read(random)
|
|
||||||
|
|
||||||
require.NoError(t, w.WritePeriod(archive.ExportArchivedEvent{
|
|
||||||
Body: base64.StdEncoding.EncodeToString(random),
|
|
||||||
}, 0, fmt.Sprintf("2026-10-01-%02d", hour)))
|
|
||||||
}
|
|
||||||
|
|
||||||
// The writer's own handle on the last file is not the export's.
|
|
||||||
w.Evict()
|
|
||||||
|
|
||||||
// Through a buffer, the open files are listed once per 8 KiB of
|
|
||||||
// output, a few times for each file, rather than at each of gzip's
|
|
||||||
// small writes, which takes far longer.
|
|
||||||
peak := &openFilesPeak{dir: dir}
|
|
||||||
buffered := bufio.NewWriterSize(peak, 8<<10)
|
|
||||||
|
|
||||||
require.NoError(t, writeExportTo(t, listExport(t, path), buffered))
|
|
||||||
require.NoError(t, buffered.Flush())
|
|
||||||
assert.Equal(t, 1, peak.max)
|
|
||||||
}
|
|
||||||
|
|
||||||
// heapPeak is an io.Writer that discards what it is given and records
|
|
||||||
// the largest heap it saw at a write. It collects garbage twice before
|
|
||||||
// each reading, so the heap it reads is what is still held. Once is not
|
|
||||||
// enough: the libraries the export calls (regexp, under GORM's table
|
|
||||||
// names, and encoding/json among them) cache buffers in a sync.Pool,
|
|
||||||
// which keeps them through one collection, so after one the reading
|
|
||||||
// counts however many happen to be cached. That varies from run to run
|
|
||||||
// by about as much as the limit in TestArchiveExport_Streams.
|
|
||||||
type heapPeak struct {
|
|
||||||
max uint64
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *heapPeak) Write(b []byte) (int, error) {
|
|
||||||
var m runtime.MemStats
|
|
||||||
|
|
||||||
runtime.GC()
|
|
||||||
runtime.GC()
|
|
||||||
runtime.ReadMemStats(&m)
|
|
||||||
p.max = max(p.max, m.HeapAlloc)
|
|
||||||
|
|
||||||
return len(b), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// exportHeapGrowth exports an archive of rows random bodies, each
|
|
||||||
// bodySize bytes of base64, and returns how far the heap rose above
|
|
||||||
// where it stood when the export began, at its highest.
|
|
||||||
func exportHeapGrowth(t *testing.T, rows, bodySize int) uint64 {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "archive.db")
|
|
||||||
w := archive.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
|
||||||
|
|
||||||
// Base64 makes four characters of every three bytes.
|
|
||||||
random := make([]byte, bodySize/4*3)
|
|
||||||
|
|
||||||
for range rows {
|
|
||||||
_, _ = rand.Read(random)
|
|
||||||
|
|
||||||
require.NoError(t, w.Write(archive.ExportArchivedEvent{
|
|
||||||
Body: base64.StdEncoding.EncodeToString(random),
|
|
||||||
}, 0))
|
|
||||||
}
|
|
||||||
|
|
||||||
export := listExport(t, path)
|
|
||||||
|
|
||||||
// Twice, for the reason heapPeak gives.
|
|
||||||
runtime.GC()
|
|
||||||
runtime.GC()
|
|
||||||
|
|
||||||
var start runtime.MemStats
|
|
||||||
|
|
||||||
runtime.ReadMemStats(&start)
|
|
||||||
|
|
||||||
// Through a buffer, the heap is read once per 8 KiB of output
|
|
||||||
// rather than at each of gzip's small writes, which takes far
|
|
||||||
// longer.
|
|
||||||
peak := &heapPeak{max: start.HeapAlloc}
|
|
||||||
buffered := bufio.NewWriterSize(peak, 8<<10)
|
|
||||||
|
|
||||||
require.NoError(t, writeExportTo(t, export, buffered))
|
|
||||||
require.NoError(t, buffered.Flush())
|
|
||||||
|
|
||||||
return peak.max - start.HeapAlloc
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveExport_Streams proves an export holds neither the archive
|
|
||||||
// nor its output in memory whole: exporting 384 KiB more of archive
|
|
||||||
// raises the heap's peak by less than half of that. The export's own
|
|
||||||
// memory, mostly gzip's compressor, is the same for both archives, so
|
|
||||||
// it cancels out. The bodies are random bytes in base64, which gzip
|
|
||||||
// shrinks by only a quarter, so an export that read every row before
|
|
||||||
// writing, or built the JSON or the gzipped file before writing it,
|
|
||||||
// would raise the peak by at least three quarters of the difference.
|
|
||||||
//
|
|
||||||
// The smaller archive has two rows so that its export, too, writes
|
|
||||||
// out more than the 8 KiB buffer in exportHeapGrowth before it ends:
|
|
||||||
// the heap must be read while the export's own memory is held.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // It measures the heap, which tests share.
|
|
||||||
func TestArchiveExport_Streams(t *testing.T) {
|
|
||||||
const (
|
|
||||||
bodySize = 16 << 10
|
|
||||||
smallRows = 2
|
|
||||||
largeRows = smallRows + 24
|
|
||||||
limit = (largeRows - smallRows) * bodySize / 2
|
|
||||||
)
|
|
||||||
|
|
||||||
small := exportHeapGrowth(t, smallRows, bodySize)
|
|
||||||
large := exportHeapGrowth(t, largeRows, bodySize)
|
|
||||||
|
|
||||||
assert.Less(t, large, small+limit,
|
|
||||||
"the heap rose by %d for %d rows and by %d for %d rows",
|
|
||||||
small, smallRows, large, largeRows,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveExportFileName proves the download is named for the
|
|
||||||
// webhook and the target, with the names made safe as for the archive
|
|
||||||
// file, and the export time in UTC.
|
|
||||||
func TestArchiveExportFileName(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
cest := time.FixedZone("CEST", int((2 * time.Hour).Seconds()))
|
|
||||||
|
|
||||||
assert.Equal(t,
|
|
||||||
"archive-orders-eu-long-term-archive-20261002T120304Z.json.gz",
|
|
||||||
archive.ExportFileName(
|
|
||||||
exportWebhookName, exportTargetName,
|
|
||||||
time.Date(2026, 10, 2, 14, 3, 4, 0, cest),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
@@ -1,280 +0,0 @@
|
|||||||
package archive
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"log/slog"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"go.uber.org/fx"
|
|
||||||
"gorm.io/gorm"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ErrExportArchiveWriterEvicted exposes the sentinel returned by
|
|
||||||
// an evicted archive writer. It carries the Err prefix rather
|
|
||||||
// than this file's usual Export one because it is a sentinel
|
|
||||||
// error.
|
|
||||||
var ErrExportArchiveWriterEvicted = errArchiveWriterEvicted
|
|
||||||
|
|
||||||
// ExportArchivedEvent aliases the archive row type so black-box
|
|
||||||
// tests can construct and read archive rows.
|
|
||||||
type ExportArchivedEvent = archivedEvent
|
|
||||||
|
|
||||||
// ExportArchiveWriter wraps an archiveWriter so black-box tests
|
|
||||||
// can exercise the archive file mechanics.
|
|
||||||
type ExportArchiveWriter struct {
|
|
||||||
w *archiveWriter
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewExportArchiveWriter builds an archive writer for tests,
|
|
||||||
// optionally overriding the reopen debounce (a non-positive
|
|
||||||
// debounce keeps the production default).
|
|
||||||
func NewExportArchiveWriter(
|
|
||||||
path string, log *slog.Logger, debounce time.Duration,
|
|
||||||
) *ExportArchiveWriter {
|
|
||||||
w := newArchiveWriter(path, log)
|
|
||||||
if debounce > 0 {
|
|
||||||
w.debounce = debounce
|
|
||||||
}
|
|
||||||
|
|
||||||
return &ExportArchiveWriter{w: w}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Write archives a row through the writer, into the file named
|
|
||||||
// without a period.
|
|
||||||
func (e *ExportArchiveWriter) Write(
|
|
||||||
row ExportArchivedEvent, expiry time.Duration,
|
|
||||||
) error {
|
|
||||||
return e.w.write(row, expiry, "")
|
|
||||||
}
|
|
||||||
|
|
||||||
// WritePeriod archives a row through the writer, into the file for
|
|
||||||
// period.
|
|
||||||
func (e *ExportArchiveWriter) WritePeriod(
|
|
||||||
row ExportArchivedEvent, expiry time.Duration, period string,
|
|
||||||
) error {
|
|
||||||
return e.w.write(row, expiry, period)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Open opens the archive file, pruning when expiry is positive.
|
|
||||||
func (e *ExportArchiveWriter) Open(expiry time.Duration) error {
|
|
||||||
return e.w.open(e.w.path, expiry)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Reopen closes and reopens the archive file.
|
|
||||||
func (e *ExportArchiveWriter) Reopen(
|
|
||||||
expiry time.Duration,
|
|
||||||
) error {
|
|
||||||
return e.w.reopen(e.w.path, expiry)
|
|
||||||
}
|
|
||||||
|
|
||||||
// SetNow replaces the clock the writer measures its reopen
|
|
||||||
// debounce on.
|
|
||||||
func (e *ExportArchiveWriter) SetNow(now func() time.Time) {
|
|
||||||
e.w.now = now
|
|
||||||
}
|
|
||||||
|
|
||||||
// Reopens reports how many times the file has been opened.
|
|
||||||
func (e *ExportArchiveWriter) Reopens() int {
|
|
||||||
return e.w.reopens
|
|
||||||
}
|
|
||||||
|
|
||||||
// DB returns the writer's current open handle for row
|
|
||||||
// inspection in tests.
|
|
||||||
func (e *ExportArchiveWriter) DB() *gorm.DB {
|
|
||||||
return e.w.db
|
|
||||||
}
|
|
||||||
|
|
||||||
// Path returns the archive file the writer owns.
|
|
||||||
func (e *ExportArchiveWriter) Path() string {
|
|
||||||
return e.w.path
|
|
||||||
}
|
|
||||||
|
|
||||||
// OpenExisting opens the archive without permitting creation,
|
|
||||||
// the way the idle sweep does.
|
|
||||||
func (e *ExportArchiveWriter) OpenExisting(
|
|
||||||
expiry time.Duration,
|
|
||||||
) error {
|
|
||||||
return e.w.openMode(e.w.path, archiveModeExisting, expiry)
|
|
||||||
}
|
|
||||||
|
|
||||||
// SweepExpired runs an idle sweep of the archive.
|
|
||||||
func (e *ExportArchiveWriter) SweepExpired(
|
|
||||||
expiry time.Duration,
|
|
||||||
) error {
|
|
||||||
return e.w.sweepExpired(expiry)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Evict marks the writer evicted and closes its handle, exactly
|
|
||||||
// as leaving the registry does.
|
|
||||||
func (e *ExportArchiveWriter) Evict() {
|
|
||||||
e.w.evict()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Rename gives the archive file a new name in the same directory,
|
|
||||||
// as a rename of the webhook or target does.
|
|
||||||
func (e *ExportArchiveWriter) Rename(name string) error {
|
|
||||||
return e.w.rename(name)
|
|
||||||
}
|
|
||||||
|
|
||||||
// HandleOpen reports whether the writer currently holds an open
|
|
||||||
// archive handle.
|
|
||||||
func (e *ExportArchiveWriter) HandleOpen() bool {
|
|
||||||
e.w.mu.Lock()
|
|
||||||
defer e.w.mu.Unlock()
|
|
||||||
|
|
||||||
return e.w.db != nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Same reports whether both wrappers refer to the very same
|
|
||||||
// underlying archive writer, so a test can prove a registry entry
|
|
||||||
// is the writer it was handed rather than a replacement.
|
|
||||||
func (e *ExportArchiveWriter) Same(
|
|
||||||
other *ExportArchiveWriter,
|
|
||||||
) bool {
|
|
||||||
return other != nil && e.w == other.w
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExportArchiveWriterFor returns the archive writer the registry
|
|
||||||
// currently caches for a database target, or nil when none is
|
|
||||||
// cached. It never creates one, so a test can hold a reference to
|
|
||||||
// the very writer an eviction is about to detach.
|
|
||||||
func (ws *Writers) ExportArchiveWriterFor(
|
|
||||||
targetID string,
|
|
||||||
) *ExportArchiveWriter {
|
|
||||||
ws.mu.Lock()
|
|
||||||
defer ws.mu.Unlock()
|
|
||||||
|
|
||||||
w, ok := ws.writers[targetID]
|
|
||||||
if !ok {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return &ExportArchiveWriter{w: w}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExportHasArchiveWriter reports whether the registry currently
|
|
||||||
// caches an archive writer for a target.
|
|
||||||
func (ws *Writers) ExportHasArchiveWriter(
|
|
||||||
targetID string,
|
|
||||||
) bool {
|
|
||||||
ws.mu.Lock()
|
|
||||||
defer ws.mu.Unlock()
|
|
||||||
|
|
||||||
_, ok := ws.writers[targetID]
|
|
||||||
|
|
||||||
return ok
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExportArchiveHandleOpen reports whether the cached archive
|
|
||||||
// writer for a target holds an open database handle. It
|
|
||||||
// returns false when no writer is cached.
|
|
||||||
func (ws *Writers) ExportArchiveHandleOpen(
|
|
||||||
targetID string,
|
|
||||||
) bool {
|
|
||||||
ws.mu.Lock()
|
|
||||||
w, ok := ws.writers[targetID]
|
|
||||||
ws.mu.Unlock()
|
|
||||||
|
|
||||||
if !ok {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
w.mu.Lock()
|
|
||||||
defer w.mu.Unlock()
|
|
||||||
|
|
||||||
return w.db != nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExportEnsureArchiveWriter creates (if needed) and returns the
|
|
||||||
// archive file path of the cached writer for a target, so a
|
|
||||||
// test can prime the registry the way a delivery would.
|
|
||||||
func (ws *Writers) ExportEnsureArchiveWriter(
|
|
||||||
targetID string,
|
|
||||||
) (string, error) {
|
|
||||||
w, err := ws.writerFor(targetID)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
return w.path, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExportSweepWriterFor takes a target's registry writer exactly
|
|
||||||
// as the idle sweep does, reporting whether the sweep had to
|
|
||||||
// create the entry. It lets a test drive the registry through the
|
|
||||||
// sweep's own entry point instead of choreographing goroutines.
|
|
||||||
func (ws *Writers) ExportSweepWriterFor(
|
|
||||||
targetID string,
|
|
||||||
) (*ExportArchiveWriter, bool, error) {
|
|
||||||
w, created, err := ws.sweepWriterFor(targetID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, false, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return &ExportArchiveWriter{w: w}, created, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExportReleaseSweepWriter releases a sweep-created registry entry
|
|
||||||
// exactly as a finished sweep does.
|
|
||||||
func (ws *Writers) ExportReleaseSweepWriter(
|
|
||||||
targetID string, w *ExportArchiveWriter,
|
|
||||||
) {
|
|
||||||
ws.releaseSweepWriter(targetID, w.w)
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewTestSweeper builds a Sweeper backed by the given main
|
|
||||||
// database and archive writers, without the fx lifecycle.
|
|
||||||
// Intended for tests.
|
|
||||||
func NewTestSweeper(
|
|
||||||
db *database.Database,
|
|
||||||
writers *Writers,
|
|
||||||
log *slog.Logger,
|
|
||||||
) *Sweeper {
|
|
||||||
return &Sweeper{
|
|
||||||
db: db,
|
|
||||||
writers: writers,
|
|
||||||
log: log,
|
|
||||||
interval: time.Hour,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExportSweep runs a single archive sweep synchronously for
|
|
||||||
// tests.
|
|
||||||
func (s *Sweeper) ExportSweep(ctx context.Context) {
|
|
||||||
s.sweep(ctx)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExportStart starts the sweeper's background loop for tests.
|
|
||||||
func (s *Sweeper) ExportStart() {
|
|
||||||
s.start()
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExportRegisterHooks registers the sweeper's real fx lifecycle
|
|
||||||
// hooks on a lifecycle supplied by a test, so a test can drive
|
|
||||||
// the exact OnStart/OnStop functions the application runs and
|
|
||||||
// hand OnStart the kind of context fx actually supplies.
|
|
||||||
func (s *Sweeper) ExportRegisterHooks(lc fx.Lifecycle) {
|
|
||||||
s.registerHooks(lc)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExportStop stops the sweeper's background loop for tests.
|
|
||||||
func (s *Sweeper) ExportStop(ctx context.Context) error {
|
|
||||||
return s.stop(ctx)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExportWedgeLoop adds a goroutine to the sweeper's WaitGroup
|
|
||||||
// that never observes cancellation and returns only when release
|
|
||||||
// is closed. It stands in for a prune stuck on a locked archive.
|
|
||||||
func (s *Sweeper) ExportWedgeLoop(
|
|
||||||
release <-chan struct{},
|
|
||||||
) {
|
|
||||||
s.wg.Go(func() {
|
|
||||||
<-release
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExportSetInterval overrides the sweep interval for tests.
|
|
||||||
func (s *Sweeper) ExportSetInterval(d time.Duration) {
|
|
||||||
s.interval = d
|
|
||||||
}
|
|
||||||
@@ -1,167 +0,0 @@
|
|||||||
package archive
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The archive rotations: how often a database target starts a new
|
|
||||||
// archive file. Every rotation but none puts the period of an event's
|
|
||||||
// receive time, in UTC, in the name of the file the event goes to,
|
|
||||||
// written in the layout beside it.
|
|
||||||
const (
|
|
||||||
archiveRotationNone = "none"
|
|
||||||
archiveRotationMonthly = "monthly"
|
|
||||||
archiveRotationDaily = "daily"
|
|
||||||
archiveRotationHourly = "hourly"
|
|
||||||
|
|
||||||
archiveMonthLayout = "2006-01"
|
|
||||||
archiveDayLayout = "2006-01-02"
|
|
||||||
archiveHourLayout = "2006-01-02-15"
|
|
||||||
)
|
|
||||||
|
|
||||||
// errArchiveRotationUnknown is returned for a rotation that is not one
|
|
||||||
// of the four.
|
|
||||||
var errArchiveRotationUnknown = errors.New(
|
|
||||||
"rotation must be none, monthly, daily or hourly",
|
|
||||||
)
|
|
||||||
|
|
||||||
// archiveFile is one of a database target's archive files, and the
|
|
||||||
// period in its name: "" for the file named without one.
|
|
||||||
type archiveFile struct {
|
|
||||||
path string
|
|
||||||
period string
|
|
||||||
}
|
|
||||||
|
|
||||||
// ValidateRotation checks a user-supplied archive rotation for
|
|
||||||
// a database target: empty or none (both meaning one file), monthly,
|
|
||||||
// daily or hourly.
|
|
||||||
func ValidateRotation(rotation string) error {
|
|
||||||
switch rotation {
|
|
||||||
case "", archiveRotationNone, archiveRotationMonthly,
|
|
||||||
archiveRotationDaily, archiveRotationHourly:
|
|
||||||
return nil
|
|
||||||
default:
|
|
||||||
return fmt.Errorf("%w: %q", errArchiveRotationUnknown, rotation)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// archivePeriod returns the period, in UTC, that a rotation puts an
|
|
||||||
// event received at receivedAt in: "2026-10" for monthly,
|
|
||||||
// "2026-10-01" for daily, "2026-10-01-19" for hourly, and "" for none
|
|
||||||
// or an empty rotation.
|
|
||||||
func archivePeriod(rotation string, receivedAt time.Time) string {
|
|
||||||
switch rotation {
|
|
||||||
case archiveRotationMonthly:
|
|
||||||
return receivedAt.UTC().Format(archiveMonthLayout)
|
|
||||||
case archiveRotationDaily:
|
|
||||||
return receivedAt.UTC().Format(archiveDayLayout)
|
|
||||||
case archiveRotationHourly:
|
|
||||||
return receivedAt.UTC().Format(archiveHourLayout)
|
|
||||||
default:
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// archivePeriodPath returns the path of a database target's archive
|
|
||||||
// file for a period: path, as Path gives it, with "-" and the
|
|
||||||
// period put before its ".db". The period "" gives path itself.
|
|
||||||
func archivePeriodPath(path, period string) string {
|
|
||||||
if period == "" {
|
|
||||||
return path
|
|
||||||
}
|
|
||||||
|
|
||||||
return strings.TrimSuffix(path, ".db") + "-" + period + ".db"
|
|
||||||
}
|
|
||||||
|
|
||||||
// PathAt returns the archive file a database target writes an
|
|
||||||
// event received at receivedAt to: Path's file, with the
|
|
||||||
// period in its name when the target rotates.
|
|
||||||
func PathAt(
|
|
||||||
dbMgr *database.WebhookDBManager,
|
|
||||||
webhook *database.Webhook,
|
|
||||||
target *database.Target,
|
|
||||||
receivedAt time.Time,
|
|
||||||
) string {
|
|
||||||
return archivePeriodPath(
|
|
||||||
Path(dbMgr, webhook, target),
|
|
||||||
archivePeriod(target.ArchiveRotation, receivedAt),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// archiveFiles lists the database target's archive files that exist,
|
|
||||||
// given the path Path gives it: the file at path, then each
|
|
||||||
// file archivePeriodPath names for path and a period, oldest period
|
|
||||||
// first. Which rotation wrote a file does not matter, so the files of
|
|
||||||
// an earlier rotation setting are listed too.
|
|
||||||
func archiveFiles(path string) ([]archiveFile, error) {
|
|
||||||
dir := filepath.Dir(path)
|
|
||||||
|
|
||||||
entries, err := os.ReadDir(dir)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("listing archive files: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
stem := strings.TrimSuffix(filepath.Base(path), ".db")
|
|
||||||
|
|
||||||
var files []archiveFile
|
|
||||||
|
|
||||||
for _, entry := range entries {
|
|
||||||
period, ok := archiveFilePeriod(stem, entry.Name())
|
|
||||||
if ok {
|
|
||||||
files = append(files, archiveFile{
|
|
||||||
path: filepath.Join(dir, entry.Name()),
|
|
||||||
period: period,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A month sorts before the days and hours in it.
|
|
||||||
slices.SortFunc(files, func(a, b archiveFile) int {
|
|
||||||
return strings.Compare(a.period, b.period)
|
|
||||||
})
|
|
||||||
|
|
||||||
return files, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// archiveFilePeriod reports whether name is the name of an archive
|
|
||||||
// file of the target whose file name without a period is stem+".db",
|
|
||||||
// and the period in it.
|
|
||||||
func archiveFilePeriod(stem, name string) (string, bool) {
|
|
||||||
rest, ok := strings.CutPrefix(name, stem)
|
|
||||||
if !ok {
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
|
|
||||||
rest, ok = strings.CutSuffix(rest, ".db")
|
|
||||||
if !ok {
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
|
|
||||||
if rest == "" {
|
|
||||||
return "", true
|
|
||||||
}
|
|
||||||
|
|
||||||
period, ok := strings.CutPrefix(rest, "-")
|
|
||||||
if !ok {
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, layout := range []string{
|
|
||||||
archiveMonthLayout, archiveDayLayout, archiveHourLayout,
|
|
||||||
} {
|
|
||||||
_, err := time.Parse(layout, period)
|
|
||||||
if err == nil {
|
|
||||||
return period, true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
@@ -1,375 +0,0 @@
|
|||||||
package archive_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/webhooker/internal/archive"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database/databasetest"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The archive rotations.
|
|
||||||
const (
|
|
||||||
rotationNone = "none"
|
|
||||||
rotationMonthly = "monthly"
|
|
||||||
rotationDaily = "daily"
|
|
||||||
rotationHourly = "hourly"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The periods the tests archive into most: two days, and an hour of
|
|
||||||
// the first.
|
|
||||||
const (
|
|
||||||
dayPeriod = "2026-03-04"
|
|
||||||
nextDayPeriod = "2026-03-05"
|
|
||||||
hourPeriod = "2026-03-04-05"
|
|
||||||
)
|
|
||||||
|
|
||||||
// periodPath returns the archive file for a period of the target whose
|
|
||||||
// file without a period is path.
|
|
||||||
func periodPath(path, period string) string {
|
|
||||||
return strings.TrimSuffix(path, ".db") + "-" + period + ".db"
|
|
||||||
}
|
|
||||||
|
|
||||||
// deliverReceivedAt archives, as a delivery to a database target
|
|
||||||
// does, an event whose receive time is receivedAt, and returns the
|
|
||||||
// event's id. The receive time is what decides a rotated archive's
|
|
||||||
// file, so setting it is how these tests move the clock across a
|
|
||||||
// period boundary.
|
|
||||||
func (env *archiveEnv) deliverReceivedAt(
|
|
||||||
t *testing.T, tgt *database.Target, receivedAt time.Time,
|
|
||||||
) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
event := newEvent(`{"n":1}`)
|
|
||||||
event.CreatedAt = receivedAt
|
|
||||||
|
|
||||||
require.NoError(t, env.writers.Write(databaseTargetDelivery(event, tgt)))
|
|
||||||
|
|
||||||
return event.ID
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestWrite_RotatesAtEachPeriodBoundary writes, for each
|
|
||||||
// rotation, an event received in the last second of a period and one
|
|
||||||
// received in the first second of the next, and checks each lands in
|
|
||||||
// the file named for its own period, in UTC. Rotation none keeps both
|
|
||||||
// in the one file.
|
|
||||||
func TestWrite_RotatesAtEachPeriodBoundary(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
berlin := time.FixedZone("CEST", 2*60*60)
|
|
||||||
|
|
||||||
cases := []struct {
|
|
||||||
name string
|
|
||||||
rotation string
|
|
||||||
before, after time.Time
|
|
||||||
// periods are the periods of before and after.
|
|
||||||
periods [2]string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
rotationMonthly, rotationMonthly,
|
|
||||||
time.Date(2026, 1, 31, 23, 59, 59, 0, time.UTC),
|
|
||||||
time.Date(2026, 2, 1, 0, 0, 0, 0, time.UTC),
|
|
||||||
[2]string{"2026-01", "2026-02"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
rotationDaily, rotationDaily,
|
|
||||||
time.Date(2026, 3, 4, 23, 59, 59, 0, time.UTC),
|
|
||||||
time.Date(2026, 3, 5, 0, 0, 0, 0, time.UTC),
|
|
||||||
[2]string{dayPeriod, nextDayPeriod},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
// The same instants, received in a zone two hours ahead
|
|
||||||
// of UTC, where they fall on 5 March: the period is UTC's.
|
|
||||||
"daily in another zone", rotationDaily,
|
|
||||||
time.Date(2026, 3, 5, 1, 59, 59, 0, berlin),
|
|
||||||
time.Date(2026, 3, 5, 2, 0, 0, 0, berlin),
|
|
||||||
[2]string{dayPeriod, nextDayPeriod},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
rotationHourly, rotationHourly,
|
|
||||||
time.Date(2026, 3, 4, 5, 59, 59, 0, time.UTC),
|
|
||||||
time.Date(2026, 3, 4, 6, 0, 0, 0, time.UTC),
|
|
||||||
[2]string{hourPeriod, "2026-03-04-06"},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range cases {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
|
||||||
tgt := env.seedDatabaseTarget(t, 0, tc.rotation)
|
|
||||||
path := env.archivePath(tgt)
|
|
||||||
|
|
||||||
first := env.deliverReceivedAt(t, tgt, tc.before)
|
|
||||||
second := env.deliverReceivedAt(t, tgt, tc.after)
|
|
||||||
|
|
||||||
assert.Equal(t, []string{first},
|
|
||||||
archivedEventIDs(t, periodPath(path, tc.periods[0])))
|
|
||||||
assert.Equal(t, []string{second},
|
|
||||||
archivedEventIDs(t, periodPath(path, tc.periods[1])))
|
|
||||||
assert.NoFileExists(t, path,
|
|
||||||
"a rotated target never writes the file without a period")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Run(rotationNone, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
|
||||||
tgt := env.seedDatabaseTarget(t, 0, rotationNone)
|
|
||||||
|
|
||||||
first := env.deliverReceivedAt(t, tgt, cases[0].before)
|
|
||||||
second := env.deliverReceivedAt(t, tgt, cases[0].after)
|
|
||||||
|
|
||||||
assert.ElementsMatch(t, []string{first, second},
|
|
||||||
archivedEventIDs(t, env.archivePath(tgt)))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestWrite_EventWithoutReceiveTime proves an event whose
|
|
||||||
// receive time is not known goes to the file for the time it is
|
|
||||||
// archived, rather than to one for the year 1.
|
|
||||||
func TestWrite_EventWithoutReceiveTime(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
|
||||||
tgt := env.seedDatabaseTarget(t, 0, rotationMonthly)
|
|
||||||
path := env.archivePath(tgt)
|
|
||||||
|
|
||||||
before := time.Now().UTC().Format("2006-01")
|
|
||||||
id := env.deliverReceivedAt(t, tgt, time.Time{})
|
|
||||||
|
|
||||||
file := periodPath(path, time.Now().UTC().Format("2006-01"))
|
|
||||||
|
|
||||||
_, err := os.Stat(file)
|
|
||||||
if err != nil {
|
|
||||||
// The month turned during the delivery.
|
|
||||||
file = periodPath(path, before)
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equal(t, []string{id}, archivedEventIDs(t, file))
|
|
||||||
assert.NoFileExists(t, periodPath(path, "0001-01"))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestWrite_RotationChangeKeepsOldFiles changes a target's
|
|
||||||
// rotation from none to daily between two events, and checks the
|
|
||||||
// second goes to the daily file while the first stays where it was.
|
|
||||||
func TestWrite_RotationChangeKeepsOldFiles(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
|
||||||
tgt := env.seedDatabaseTarget(t, 0, rotationNone)
|
|
||||||
path := env.archivePath(tgt)
|
|
||||||
at := time.Date(2026, 3, 4, 12, 0, 0, 0, time.UTC)
|
|
||||||
|
|
||||||
first := env.deliverReceivedAt(t, tgt, at)
|
|
||||||
|
|
||||||
tgt.ArchiveRotation = rotationDaily
|
|
||||||
second := env.deliverReceivedAt(t, tgt, at)
|
|
||||||
|
|
||||||
assert.Equal(t, []string{first}, archivedEventIDs(t, path))
|
|
||||||
assert.Equal(t, []string{second},
|
|
||||||
archivedEventIDs(t, periodPath(path, dayPeriod)))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveSweep_PrunesEveryFile gives a daily target three files:
|
|
||||||
// the file without a period, left from before it rotated, and two
|
|
||||||
// daily files. Each holds a row older than the expiry, and one daily
|
|
||||||
// file also a newer row. The sweep prunes the old row from every file,
|
|
||||||
// deletes the daily file it leaves empty, and keeps the file without a
|
|
||||||
// period although it is empty too.
|
|
||||||
func TestArchiveSweep_PrunesEveryFile(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
|
||||||
tgt := env.seedDatabaseTarget(t, time.Hour, rotationDaily)
|
|
||||||
path := env.archivePath(tgt)
|
|
||||||
emptied := periodPath(path, dayPeriod)
|
|
||||||
kept := periodPath(path, nextDayPeriod)
|
|
||||||
|
|
||||||
now := time.Now()
|
|
||||||
old := now.Add(-48 * time.Hour)
|
|
||||||
|
|
||||||
seedArchiveFile(t, path, tgt.WebhookID, old)
|
|
||||||
seedArchiveFile(t, emptied, tgt.WebhookID, old)
|
|
||||||
seedArchiveFile(t, kept, tgt.WebhookID, old, now.Add(-time.Minute))
|
|
||||||
|
|
||||||
env.sweeper.ExportSweep(t.Context())
|
|
||||||
|
|
||||||
assert.Empty(t, archivedEventIDs(t, path))
|
|
||||||
assert.Equal(t, []string{sweepRowNew}, archivedEventIDs(t, kept))
|
|
||||||
|
|
||||||
for _, suffix := range archiveFileSuffixes() {
|
|
||||||
assert.NoFileExists(t, emptied+suffix)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRename_MovesEveryFile renames a daily target that also has a
|
|
||||||
// file without a period, and checks every file moves to the new name
|
|
||||||
// with its period, rows and all, and that a later write uses the new
|
|
||||||
// name.
|
|
||||||
func TestRename_MovesEveryFile(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
|
||||||
tgt := env.seedDatabaseTarget(t, 0, rotationNone)
|
|
||||||
oldPath := env.archivePath(tgt)
|
|
||||||
day := time.Date(2026, 3, 4, 12, 0, 0, 0, time.UTC)
|
|
||||||
|
|
||||||
unrotated := env.deliverReceivedAt(t, tgt, day)
|
|
||||||
|
|
||||||
tgt.ArchiveRotation = rotationDaily
|
|
||||||
first := env.deliverReceivedAt(t, tgt, day)
|
|
||||||
second := env.deliverReceivedAt(t, tgt, day.Add(24*time.Hour))
|
|
||||||
|
|
||||||
require.NoError(t, env.writers.Rename(tgt.ID, "Orders", "Long Term"))
|
|
||||||
|
|
||||||
newPath := filepath.Join(
|
|
||||||
env.dataDir, "archive-orders-long-term-"+tgt.ID+".db",
|
|
||||||
)
|
|
||||||
|
|
||||||
for _, old := range []string{
|
|
||||||
oldPath,
|
|
||||||
periodPath(oldPath, dayPeriod),
|
|
||||||
periodPath(oldPath, nextDayPeriod),
|
|
||||||
} {
|
|
||||||
assert.NoFileExists(t, old)
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equal(t, []string{unrotated}, archivedEventIDs(t, newPath))
|
|
||||||
assert.Equal(t, []string{first},
|
|
||||||
archivedEventIDs(t, periodPath(newPath, dayPeriod)))
|
|
||||||
assert.Equal(t, []string{second},
|
|
||||||
archivedEventIDs(t, periodPath(newPath, nextDayPeriod)))
|
|
||||||
|
|
||||||
third := env.deliverReceivedAt(t, tgt, day.Add(48*time.Hour))
|
|
||||||
assert.Equal(t, []string{third},
|
|
||||||
archivedEventIDs(t, periodPath(newPath, "2026-03-06")))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRename_NeverReplacesARotatedFile plants a file at the new name
|
|
||||||
// of a target's daily file, and proves the rename is refused and moves
|
|
||||||
// none of the target's files.
|
|
||||||
func TestRename_NeverReplacesARotatedFile(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
|
||||||
tgt := env.seedDatabaseTarget(t, 0, rotationDaily)
|
|
||||||
oldPath := env.archivePath(tgt)
|
|
||||||
day := time.Date(2026, 3, 4, 12, 0, 0, 0, time.UTC)
|
|
||||||
|
|
||||||
env.deliverReceivedAt(t, tgt, day)
|
|
||||||
env.deliverReceivedAt(t, tgt, day.Add(24*time.Hour))
|
|
||||||
|
|
||||||
newPath := filepath.Join(
|
|
||||||
env.dataDir, "archive-orders-long-term-"+tgt.ID+".db",
|
|
||||||
)
|
|
||||||
planted := periodPath(newPath, nextDayPeriod)
|
|
||||||
require.NoError(t, os.WriteFile(planted, []byte("planted"), 0o600))
|
|
||||||
|
|
||||||
require.ErrorIs(
|
|
||||||
t, env.writers.Rename(tgt.ID, "Orders", "Long Term"),
|
|
||||||
archive.ErrNameTaken,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.FileExists(t, periodPath(oldPath, dayPeriod))
|
|
||||||
assert.FileExists(t, periodPath(oldPath, nextDayPeriod))
|
|
||||||
assert.NoFileExists(t, periodPath(newPath, dayPeriod))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestStatArchive_EveryFile proves Stat counts and adds up every
|
|
||||||
// one of a target's files, takes the latest write of any of them, and
|
|
||||||
// leaves out files whose names only look like the target's.
|
|
||||||
func TestStatArchive_EveryFile(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
path := filepath.Join(dir, "archive-wh.db")
|
|
||||||
files := []string{
|
|
||||||
path, periodPath(path, "2026-03"), periodPath(path, hourPeriod),
|
|
||||||
}
|
|
||||||
written := time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC)
|
|
||||||
|
|
||||||
var size int64
|
|
||||||
|
|
||||||
for i, file := range files {
|
|
||||||
require.NoError(t, os.WriteFile(file, make([]byte, 100*(i+1)), 0o600))
|
|
||||||
|
|
||||||
size += int64(100 * (i + 1))
|
|
||||||
at := written.Add(-time.Duration(i) * time.Hour)
|
|
||||||
require.NoError(t, os.Chtimes(file, at, at))
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, other := range []string{
|
|
||||||
"archive-wh-2026-13.db", "archive-wh-2026-3.db",
|
|
||||||
"archive-wh-other.db", "archive-wh-2026-03.json",
|
|
||||||
"archive-whx.db",
|
|
||||||
} {
|
|
||||||
require.NoError(t,
|
|
||||||
os.WriteFile(filepath.Join(dir, other), []byte("x"), 0o600))
|
|
||||||
}
|
|
||||||
|
|
||||||
got, err := archive.Stat(path)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, len(files), got.Files)
|
|
||||||
assert.Equal(t, size, got.Size)
|
|
||||||
assert.True(t, written.Equal(got.Written), got.Written)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchivePathAt names the file each rotation writes an event to.
|
|
||||||
func TestArchivePathAt(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dataDir := t.TempDir()
|
|
||||||
dbMgr := databasetest.NewWebhookDBManager(t, dataDir)
|
|
||||||
webhook := &database.Webhook{
|
|
||||||
BaseModel: database.BaseModel{ID: "wh-id"}, Name: "Orders",
|
|
||||||
}
|
|
||||||
at := time.Date(2026, 10, 1, 19, 30, 0, 0, time.UTC)
|
|
||||||
|
|
||||||
cases := map[string]string{
|
|
||||||
"": "",
|
|
||||||
rotationNone: "",
|
|
||||||
rotationMonthly: "-2026-10",
|
|
||||||
rotationDaily: "-2026-10-01",
|
|
||||||
rotationHourly: "-2026-10-01-19",
|
|
||||||
}
|
|
||||||
|
|
||||||
for rotation, period := range cases {
|
|
||||||
target := &database.Target{
|
|
||||||
BaseModel: database.BaseModel{ID: "tgt-id"},
|
|
||||||
Name: "Archive",
|
|
||||||
ArchiveRotation: rotation,
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equal(t,
|
|
||||||
filepath.Join(
|
|
||||||
dataDir, "archive-orders-archive-tgt-id"+period+".db",
|
|
||||||
),
|
|
||||||
archive.PathAt(dbMgr, webhook, target, at), rotation,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestValidateArchiveRotation accepts the four rotations, and empty,
|
|
||||||
// and refuses anything else.
|
|
||||||
func TestValidateArchiveRotation(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, ok := range []string{
|
|
||||||
"", rotationNone, rotationMonthly, rotationDaily, rotationHourly,
|
|
||||||
} {
|
|
||||||
require.NoError(t, archive.ValidateRotation(ok), ok)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, bad := range []string{"weekly", "Daily", "hourly "} {
|
|
||||||
require.Error(t, archive.ValidateRotation(bad), bad)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,638 +0,0 @@
|
|||||||
package archive
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io/fs"
|
|
||||||
"log/slog"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"gorm.io/driver/sqlite"
|
|
||||||
"gorm.io/gorm"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/db/migrations"
|
|
||||||
"sneak.berlin/go/webhooker/internal/gormlog"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ExpiryNever is the archive expiry, as the target forms take it,
|
|
||||||
// that disables pruning so archived rows are kept forever. It is
|
|
||||||
// stored as an ArchiveExpiry of 0.
|
|
||||||
const ExpiryNever = "never"
|
|
||||||
|
|
||||||
// archiveReopenDebounce bounds how often an archive file is
|
|
||||||
// closed and reopened. After each write the handle is closed
|
|
||||||
// and reopened so an operator can move the file away for
|
|
||||||
// offline archiving, but never more than once per this window.
|
|
||||||
const archiveReopenDebounce = time.Second
|
|
||||||
|
|
||||||
const (
|
|
||||||
// archiveModeCreate is the SQLite URI mode used by the write
|
|
||||||
// path: open the archive file, creating it if missing, so a
|
|
||||||
// first write (or a write after the operator moved the file
|
|
||||||
// away) recreates it.
|
|
||||||
archiveModeCreate = database.SQLiteModeCreate
|
|
||||||
|
|
||||||
// archiveModeExisting is the SQLite URI mode used by the idle
|
|
||||||
// sweep: open read-write but never create. A sweep must never
|
|
||||||
// conjure an empty archive file for a webhook that has a
|
|
||||||
// database target but has never received an event.
|
|
||||||
archiveModeExisting = database.SQLiteModeExisting
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
// errArchiveMissingWebhookID is returned when an event to
|
|
||||||
// archive has no webhook id to record in its archive row.
|
|
||||||
errArchiveMissingWebhookID = errors.New(
|
|
||||||
"cannot archive event without a webhook id",
|
|
||||||
)
|
|
||||||
|
|
||||||
// errArchiveNoDataDir is returned when the database target
|
|
||||||
// has no webhook database manager and so cannot locate the
|
|
||||||
// data directory for archive files.
|
|
||||||
errArchiveNoDataDir = errors.New(
|
|
||||||
"database target has no data directory",
|
|
||||||
)
|
|
||||||
|
|
||||||
// errArchiveExpiryNotPositive is returned when a
|
|
||||||
// user-supplied archive expiry parses as a duration but is
|
|
||||||
// zero or negative; "never" is the way to disable pruning.
|
|
||||||
errArchiveExpiryNotPositive = errors.New(
|
|
||||||
"expiry must be a positive duration or \"never\"",
|
|
||||||
)
|
|
||||||
|
|
||||||
// errArchiveWriterEvicted is returned when a writer that has
|
|
||||||
// been evicted (its target or its webhook was deleted) is used
|
|
||||||
// again. An evicted writer is no longer in the registry, so
|
|
||||||
// reopening its file would leak a handle nothing owns.
|
|
||||||
errArchiveWriterEvicted = errors.New(
|
|
||||||
"archive writer has been evicted",
|
|
||||||
)
|
|
||||||
|
|
||||||
// ErrNameTaken is returned when an archive cannot be
|
|
||||||
// renamed because a file already has the new name. That file may
|
|
||||||
// be an archive with rows of its own, so it is never replaced.
|
|
||||||
ErrNameTaken = errors.New(
|
|
||||||
"a file already has the archive's new name",
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
// archivedEvent is one fully captured webhook event stored in a
|
|
||||||
// database target's archive for long-term retention. It is a
|
|
||||||
// self-contained copy — independent of the per-webhook event
|
|
||||||
// database, which may prune events under its own retention.
|
|
||||||
type archivedEvent struct {
|
|
||||||
ID uint `gorm:"primaryKey;autoIncrement"`
|
|
||||||
EventID string `gorm:"index"`
|
|
||||||
WebhookID string
|
|
||||||
EntrypointID string
|
|
||||||
Method string
|
|
||||||
RawQuery string
|
|
||||||
Headers string
|
|
||||||
Body string
|
|
||||||
ContentType string
|
|
||||||
|
|
||||||
// ArchivedAt is when the row was archived and is the age
|
|
||||||
// basis for expiry pruning.
|
|
||||||
ArchivedAt time.Time `gorm:"index"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// ParseExpiry reads a user-supplied archive expiry for a database
|
|
||||||
// target, as the target forms submit it, into the ArchiveExpiry the
|
|
||||||
// target stores. Empty and "never" both mean keep forever, which is
|
|
||||||
// 0; anything else must be a positive Go duration such as "720h". Any
|
|
||||||
// other value is an error, so a bad expiry is refused at the form
|
|
||||||
// rather than stored.
|
|
||||||
func ParseExpiry(expiry string) (time.Duration, error) {
|
|
||||||
if expiry == "" || expiry == ExpiryNever {
|
|
||||||
return 0, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
dur, err := time.ParseDuration(expiry)
|
|
||||||
if err != nil {
|
|
||||||
return 0, fmt.Errorf(
|
|
||||||
"expiry must be %q or a Go duration "+
|
|
||||||
"such as \"720h\": %w",
|
|
||||||
ExpiryNever, err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
if dur <= 0 {
|
|
||||||
return 0, fmt.Errorf(
|
|
||||||
"%w: %q", errArchiveExpiryNotPositive, expiry,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return dur, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// archiveWriter owns one database target's archive SQLite files.
|
|
||||||
// It serialises writes, and after each write closes and reopens
|
|
||||||
// the file (debounced to at most once per debounce window) so
|
|
||||||
// an operator can move the file away for offline archiving. The
|
|
||||||
// next write recreates a moved or removed file, because the
|
|
||||||
// file is opened create-if-missing and its schema is migrated
|
|
||||||
// on every open.
|
|
||||||
type archiveWriter struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
|
|
||||||
// path is the target's archive file as Path names it. A
|
|
||||||
// target that rotates writes to the files archivePeriodPath names
|
|
||||||
// for path and a period instead.
|
|
||||||
path string
|
|
||||||
|
|
||||||
// current is the file db is open on.
|
|
||||||
current string
|
|
||||||
|
|
||||||
log *slog.Logger
|
|
||||||
debounce time.Duration
|
|
||||||
db *gorm.DB
|
|
||||||
lastReopen time.Time
|
|
||||||
reopens int
|
|
||||||
|
|
||||||
// now is the clock the reopen debounce is measured on. It is
|
|
||||||
// time.Now outside tests.
|
|
||||||
now func() time.Time
|
|
||||||
|
|
||||||
// evicted marks a writer that has been removed from the
|
|
||||||
// registry. Its handle is closed and it must never open the
|
|
||||||
// file again: nothing holds it any more, so a reopen would
|
|
||||||
// leak the handle for the process lifetime.
|
|
||||||
evicted bool
|
|
||||||
|
|
||||||
// webhookID is the webhook the archive's target belongs to,
|
|
||||||
// so deleting the webhook can find its writers. It is set
|
|
||||||
// when the writer is created and never changes.
|
|
||||||
webhookID string
|
|
||||||
|
|
||||||
// sweepOwned marks a registry entry that the idle sweep
|
|
||||||
// created because no writer was cached for the target. The
|
|
||||||
// sweep removes such an entry again when it is done, so a
|
|
||||||
// sweep can never leave — or resurrect — a registry entry
|
|
||||||
// for a target that has been deleted. A delivery that adopts
|
|
||||||
// the writer clears the flag, handing the entry to the
|
|
||||||
// registry proper.
|
|
||||||
//
|
|
||||||
// Unlike every other field here it is guarded by
|
|
||||||
// Writers.mu, not by this writer's mu: it describes the
|
|
||||||
// registry entry rather than the file.
|
|
||||||
sweepOwned bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// newArchiveWriter builds an archiveWriter for a file path with
|
|
||||||
// the default reopen debounce.
|
|
||||||
func newArchiveWriter(
|
|
||||||
path string, log *slog.Logger,
|
|
||||||
) *archiveWriter {
|
|
||||||
return &archiveWriter{
|
|
||||||
path: path,
|
|
||||||
log: log,
|
|
||||||
debounce: archiveReopenDebounce,
|
|
||||||
now: time.Now,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// write appends the event as a row to the archive file for period
|
|
||||||
// (see archivePeriodPath), then applies the debounced close/reopen.
|
|
||||||
// When period names a different file from the one open, the open one
|
|
||||||
// is closed first. It recreates the archive file if it was moved or
|
|
||||||
// removed since the last open. A positive expiry prunes rows older
|
|
||||||
// than it on each (re)open.
|
|
||||||
func (w *archiveWriter) write(
|
|
||||||
row archivedEvent, expiry time.Duration, period string,
|
|
||||||
) error {
|
|
||||||
w.mu.Lock()
|
|
||||||
defer w.mu.Unlock()
|
|
||||||
|
|
||||||
if w.evicted {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"%w: %s", errArchiveWriterEvicted, w.path,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
file := archivePeriodPath(w.path, period)
|
|
||||||
|
|
||||||
if w.db == nil || w.current != file || !fileExists(file) {
|
|
||||||
err := w.reopen(file, expiry)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// GORM's clock, the one that stamps created_at in the other
|
|
||||||
// databases, so this is stored in the same form. A bare time.Now()
|
|
||||||
// carries a monotonic clock reading, which the driver would store
|
|
||||||
// as an " m=+…" suffix.
|
|
||||||
row.ArchivedAt = w.db.NowFunc()
|
|
||||||
|
|
||||||
err := w.db.Create(&row).Error
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"archiving event to %s: %w", file, err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
if w.now().Sub(w.lastReopen) >= w.debounce {
|
|
||||||
return w.reopen(file, expiry)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// open opens (creating if missing) an archive file, migrates
|
|
||||||
// its schema, records the reopen time, and prunes expired rows
|
|
||||||
// when expiry is positive.
|
|
||||||
func (w *archiveWriter) open(file string, expiry time.Duration) error {
|
|
||||||
return w.openMode(file, archiveModeCreate, expiry)
|
|
||||||
}
|
|
||||||
|
|
||||||
// openMode opens an archive file with the given SQLite URI
|
|
||||||
// mode, migrates its schema, records the reopen time, and
|
|
||||||
// prunes expired rows when expiry is positive. The write path
|
|
||||||
// passes archiveModeCreate so a missing file is recreated; the
|
|
||||||
// idle sweep passes archiveModeExisting so a missing file is an
|
|
||||||
// error rather than a newly conjured empty archive.
|
|
||||||
func (w *archiveWriter) openMode(
|
|
||||||
file, mode string, expiry time.Duration,
|
|
||||||
) error {
|
|
||||||
// Opened through database.OpenSQLite so an archive file carries
|
|
||||||
// the same WAL journaling, busy timeout, immediate-transaction
|
|
||||||
// locking, and pool bounds as every other database file. See
|
|
||||||
// internal/database/sqlite_open.go.
|
|
||||||
sqlDB, err := database.OpenSQLite(file, mode)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"opening archive database %s: %w", file, err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
gdb, err := gorm.Open(
|
|
||||||
sqlite.Dialector{Conn: sqlDB}, &gorm.Config{
|
|
||||||
// Never leave this at GORM's default. See
|
|
||||||
// internal/gormlog.
|
|
||||||
Logger: gormlog.New(w.log),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
_ = sqlDB.Close()
|
|
||||||
|
|
||||||
return fmt.Errorf(
|
|
||||||
"connecting to archive database %s: %w",
|
|
||||||
file, err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = migrations.Apply(gdb, migrations.Archive)
|
|
||||||
if err != nil {
|
|
||||||
_ = sqlDB.Close()
|
|
||||||
|
|
||||||
return fmt.Errorf(
|
|
||||||
"migrating archive database %s: %w", file, err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
w.db = gdb
|
|
||||||
w.current = file
|
|
||||||
w.lastReopen = w.now()
|
|
||||||
w.reopens++
|
|
||||||
|
|
||||||
if expiry > 0 {
|
|
||||||
w.prune(expiry)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// reopen closes any open handle and opens file afresh. The
|
|
||||||
// fresh open recreates the file if it was moved away.
|
|
||||||
func (w *archiveWriter) reopen(file string, expiry time.Duration) error {
|
|
||||||
w.close()
|
|
||||||
|
|
||||||
return w.open(file, expiry)
|
|
||||||
}
|
|
||||||
|
|
||||||
// close closes the underlying handle, if any.
|
|
||||||
func (w *archiveWriter) close() {
|
|
||||||
if w.db == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
sqlDB, err := w.db.DB()
|
|
||||||
if err == nil {
|
|
||||||
_ = sqlDB.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
w.db = nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// sweepExpired prunes the target's archive files, which may have
|
|
||||||
// gone idle, with no write to trigger the usual on-reopen prune. It
|
|
||||||
// lists the files under the writer's own mutex, then takes the mutex
|
|
||||||
// again for one file at a time, so a write waits for at most one
|
|
||||||
// file's prune, and each prune is ordered against concurrent writes
|
|
||||||
// rather than reaching around them to the file.
|
|
||||||
//
|
|
||||||
// It never creates an archive file: it prunes only the files
|
|
||||||
// archiveFiles lists, skips one that is gone by the time it is
|
|
||||||
// reached (moved away, or renamed since the listing), and opens each
|
|
||||||
// with archiveModeExisting so SQLite itself refuses to create one if
|
|
||||||
// the file disappears between the check and the open. A file named
|
|
||||||
// for a period that the prune leaves empty is deleted.
|
|
||||||
//
|
|
||||||
// The archive is left CLOSED afterwards. An idle archive holding
|
|
||||||
// no handle is what keeps the operator's move-the-file-away
|
|
||||||
// workflow working; the next write reopens (and recreates) the
|
|
||||||
// file as it always has.
|
|
||||||
func (w *archiveWriter) sweepExpired(expiry time.Duration) error {
|
|
||||||
w.mu.Lock()
|
|
||||||
files, err := archiveFiles(w.path)
|
|
||||||
w.mu.Unlock()
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
var errs []error
|
|
||||||
|
|
||||||
for _, file := range files {
|
|
||||||
err = w.sweepFile(file, expiry)
|
|
||||||
if errors.Is(err, errArchiveWriterEvicted) {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
errs = append(errs, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return errors.Join(errs...)
|
|
||||||
}
|
|
||||||
|
|
||||||
// sweepFile prunes one of the target's archive files for sweepExpired,
|
|
||||||
// holding w.mu while it does. It skips a file that is gone, and deletes
|
|
||||||
// the file, with its -wal and -shm, when it is named for a period and
|
|
||||||
// the prune leaves it empty.
|
|
||||||
func (w *archiveWriter) sweepFile(
|
|
||||||
file archiveFile, expiry time.Duration,
|
|
||||||
) error {
|
|
||||||
w.mu.Lock()
|
|
||||||
defer w.mu.Unlock()
|
|
||||||
|
|
||||||
if w.evicted {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"%w: %s", errArchiveWriterEvicted, w.path,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !fileExists(file.path) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Drop any live handle first so the prune runs against a
|
|
||||||
// freshly opened file, matching the write path's semantics.
|
|
||||||
w.close()
|
|
||||||
|
|
||||||
err := w.openMode(file.path, archiveModeExisting, expiry)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if file.period == "" {
|
|
||||||
w.close()
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var rows int64
|
|
||||||
|
|
||||||
err = w.db.Model(&archivedEvent{}).Count(&rows).Error
|
|
||||||
|
|
||||||
w.close()
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"counting rows in archive %s: %w", file.path, err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
if rows > 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, suffix := range []string{"", "-wal", "-shm"} {
|
|
||||||
err = os.Remove(file.path + suffix)
|
|
||||||
if err != nil && !errors.Is(err, fs.ErrNotExist) {
|
|
||||||
return fmt.Errorf("deleting empty archive file: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
w.log.Info("deleted empty archive file", "path", file.path)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// rename gives every one of the target's archive files the new
|
|
||||||
// name, keeping the period in the name of each (see
|
|
||||||
// archivePeriodPath), and the writer uses the files under that name
|
|
||||||
// from now on. The handle is closed first, which folds the -wal into
|
|
||||||
// the .db; any -wal or -shm still beside a file (left by a crash) is
|
|
||||||
// moved with it, because SQLite finds them by name. A target with no
|
|
||||||
// files is not an error: the operator may have moved them away, and
|
|
||||||
// the next write creates its file under the new name.
|
|
||||||
//
|
|
||||||
// If a file already has one of the new names, nothing is moved and
|
|
||||||
// the error is ErrNameTaken. If one file fails to move, those
|
|
||||||
// already moved are moved back before the error is returned, so the
|
|
||||||
// archive is never split across two names.
|
|
||||||
func (w *archiveWriter) rename(name string) error {
|
|
||||||
w.mu.Lock()
|
|
||||||
defer w.mu.Unlock()
|
|
||||||
|
|
||||||
if w.evicted {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"%w: %s", errArchiveWriterEvicted, w.path,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
path := filepath.Join(filepath.Dir(w.path), name)
|
|
||||||
if path == w.path {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
files, err := archiveFiles(w.path)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// from[i] moves to to[i].
|
|
||||||
var from, to []string
|
|
||||||
|
|
||||||
for _, file := range files {
|
|
||||||
renamed := archivePeriodPath(path, file.period)
|
|
||||||
|
|
||||||
for _, suffix := range []string{"", "-wal", "-shm"} {
|
|
||||||
from = append(from, file.path+suffix)
|
|
||||||
to = append(to, renamed+suffix)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, taken := range to {
|
|
||||||
if fileExists(taken) {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"%w: %s", ErrNameTaken, filepath.Base(taken),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
w.close()
|
|
||||||
|
|
||||||
for i := range from {
|
|
||||||
err = os.Rename(from[i], to[i])
|
|
||||||
if err == nil || errors.Is(err, fs.ErrNotExist) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
for j := range i {
|
|
||||||
backErr := os.Rename(to[j], from[j])
|
|
||||||
if backErr != nil && !errors.Is(backErr, fs.ErrNotExist) {
|
|
||||||
w.log.Error(
|
|
||||||
"failed to move archive file back",
|
|
||||||
"from", to[j],
|
|
||||||
"to", from[j],
|
|
||||||
"error", backErr,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return fmt.Errorf(
|
|
||||||
"renaming archive %s to %s: %w", from[i], to[i], err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
w.path = path
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// evict closes the writer's handle and marks it unusable. It is
|
|
||||||
// called when the writer leaves the registry, because its target
|
|
||||||
// or its webhook was deleted, or at shutdown. The archive FILE is
|
|
||||||
// deliberately left on disk: it is long-term storage an operator
|
|
||||||
// may still want.
|
|
||||||
func (w *archiveWriter) evict() {
|
|
||||||
w.mu.Lock()
|
|
||||||
defer w.mu.Unlock()
|
|
||||||
|
|
||||||
w.evicted = true
|
|
||||||
|
|
||||||
w.close()
|
|
||||||
}
|
|
||||||
|
|
||||||
// prune deletes archived rows older than expiry, measured from
|
|
||||||
// each row's archived time. It runs on every (re)open, so a
|
|
||||||
// steadily written archive is swept by its own write traffic. An
|
|
||||||
// archive that goes idle receives no further reopens, which is
|
|
||||||
// why Sweeper exists to drive sweepExpired on a timer.
|
|
||||||
// Failures are logged, not fatal: a prune error must not stop
|
|
||||||
// archiving.
|
|
||||||
func (w *archiveWriter) prune(expiry time.Duration) {
|
|
||||||
cutoff := time.Now().Add(-expiry)
|
|
||||||
|
|
||||||
res := w.db.Where("archived_at < ?", cutoff).
|
|
||||||
Delete(&archivedEvent{})
|
|
||||||
if res.Error != nil {
|
|
||||||
w.log.Error(
|
|
||||||
"failed to prune expired archive rows",
|
|
||||||
"path", w.current,
|
|
||||||
"error", res.Error,
|
|
||||||
)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if res.RowsAffected > 0 {
|
|
||||||
w.log.Info(
|
|
||||||
"pruned expired archive rows",
|
|
||||||
"path", w.current,
|
|
||||||
"rows_deleted", res.RowsAffected,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// FileInfo is what the metadata of a database target's archive
|
|
||||||
// files says about them.
|
|
||||||
type FileInfo struct {
|
|
||||||
// Files counts the files.
|
|
||||||
Files int
|
|
||||||
|
|
||||||
// Size is the bytes on disk of the files and their -wal together.
|
|
||||||
Size int64
|
|
||||||
|
|
||||||
// Written is when a file or a -wal was last modified, whichever is
|
|
||||||
// latest: a write lands in the -wal first.
|
|
||||||
Written time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stat reads the metadata of a database target's archive
|
|
||||||
// files, given the path Path gives it (see archiveFiles), and
|
|
||||||
// of their -wal, without opening them. With no files, which is so
|
|
||||||
// before the first write and after the operator moved them away, the
|
|
||||||
// error wraps fs.ErrNotExist.
|
|
||||||
func Stat(path string) (FileInfo, error) {
|
|
||||||
files, err := archiveFiles(path)
|
|
||||||
if err != nil {
|
|
||||||
return FileInfo{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
var info FileInfo
|
|
||||||
|
|
||||||
for _, file := range files {
|
|
||||||
db, err := os.Stat(file.path)
|
|
||||||
if errors.Is(err, fs.ErrNotExist) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return FileInfo{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
info.Files++
|
|
||||||
info.Size += db.Size()
|
|
||||||
|
|
||||||
if db.ModTime().After(info.Written) {
|
|
||||||
info.Written = db.ModTime()
|
|
||||||
}
|
|
||||||
|
|
||||||
wal, err := os.Stat(file.path + "-wal")
|
|
||||||
if errors.Is(err, fs.ErrNotExist) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return FileInfo{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
info.Size += wal.Size()
|
|
||||||
|
|
||||||
if wal.ModTime().After(info.Written) {
|
|
||||||
info.Written = wal.ModTime()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if info.Files == 0 {
|
|
||||||
return FileInfo{}, fmt.Errorf(
|
|
||||||
"no archive file for %s: %w", path, fs.ErrNotExist,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return info, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// fileExists reports whether a path currently exists.
|
|
||||||
func fileExists(path string) bool {
|
|
||||||
_, err := os.Stat(path)
|
|
||||||
|
|
||||||
return err == nil
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,387 @@
|
|||||||
|
package ciscript_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"maps"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"gopkg.in/yaml.v3"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// supersededDesc is the description script/ci-mark-superseded
|
||||||
|
// writes, and the one an earlier revision of it wrote alongside a
|
||||||
|
// `skipped` state.
|
||||||
|
supersededDesc = "Superseded by a newer commit; never tested"
|
||||||
|
|
||||||
|
// liveContext is the commit-status context Gitea uses for this
|
||||||
|
// repository's runs, as seen in its API. The script derives it from
|
||||||
|
// the workflow and job names rather than hardcoding it; the
|
||||||
|
// derivation is checked against this value below.
|
||||||
|
liveContext = "check / check (push)"
|
||||||
|
|
||||||
|
scriptPath = "../../script/ci-mark-superseded"
|
||||||
|
workflow = "../../.gitea/workflows/check.yml"
|
||||||
|
|
||||||
|
// failure is the only state that neither folds into a combined
|
||||||
|
// `success` (as `skipped` does) nor blocks the commit forever (as
|
||||||
|
// `pending` does).
|
||||||
|
failure = "failure"
|
||||||
|
)
|
||||||
|
|
||||||
|
// repo is a throwaway git history: parent is the commit a run would be
|
||||||
|
// cancelled on, head the commit that superseded it.
|
||||||
|
type repo struct {
|
||||||
|
dir string
|
||||||
|
head string
|
||||||
|
parent string
|
||||||
|
}
|
||||||
|
|
||||||
|
// scriptEnv is the run identity the Gitea runner exports and the script
|
||||||
|
// builds its context string from.
|
||||||
|
type scriptEnv struct {
|
||||||
|
workflow string
|
||||||
|
job string
|
||||||
|
event string
|
||||||
|
}
|
||||||
|
|
||||||
|
func defaultEnv() scriptEnv {
|
||||||
|
return scriptEnv{workflow: "check", job: "check", event: "push"}
|
||||||
|
}
|
||||||
|
|
||||||
|
func cancelled() commitStatus {
|
||||||
|
return commitStatus{
|
||||||
|
Context: liveContext,
|
||||||
|
Status: failure,
|
||||||
|
Description: "Has been cancelled",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func running() commitStatus {
|
||||||
|
return commitStatus{
|
||||||
|
Context: liveContext,
|
||||||
|
Status: "pending",
|
||||||
|
Description: "Has started running",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMarkSuperseded(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
cases := map[string]struct {
|
||||||
|
parent commitStatus
|
||||||
|
wantMark bool
|
||||||
|
}{
|
||||||
|
"a cancelled run is marked": {
|
||||||
|
parent: cancelled(),
|
||||||
|
wantMark: true,
|
||||||
|
},
|
||||||
|
"a laundered skipped status is marked": {
|
||||||
|
parent: commitStatus{
|
||||||
|
Context: liveContext,
|
||||||
|
Status: "skipped",
|
||||||
|
Description: supersededDesc,
|
||||||
|
},
|
||||||
|
wantMark: true,
|
||||||
|
},
|
||||||
|
"a genuine failure is left alone": {
|
||||||
|
parent: commitStatus{
|
||||||
|
Context: liveContext,
|
||||||
|
Status: failure,
|
||||||
|
Description: "Failing after 3m1s",
|
||||||
|
},
|
||||||
|
wantMark: false,
|
||||||
|
},
|
||||||
|
"a passing run is left alone": {
|
||||||
|
parent: commitStatus{
|
||||||
|
Context: liveContext,
|
||||||
|
Status: "success",
|
||||||
|
Description: "Successful in 2m52s",
|
||||||
|
},
|
||||||
|
wantMark: false,
|
||||||
|
},
|
||||||
|
"another context is left alone": {
|
||||||
|
parent: commitStatus{
|
||||||
|
Context: "other / other (push)",
|
||||||
|
Status: failure,
|
||||||
|
Description: "Has been cancelled",
|
||||||
|
},
|
||||||
|
wantMark: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, tc := range cases {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
requireTools(t)
|
||||||
|
|
||||||
|
history := newRepo(t)
|
||||||
|
fake, api := newFakeGitea(t)
|
||||||
|
fake.setStatus(history.head, running())
|
||||||
|
fake.setStatus(history.parent, tc.parent)
|
||||||
|
|
||||||
|
out, err := runScript(t, history, api, defaultEnv())
|
||||||
|
require.NoError(t, err, out)
|
||||||
|
|
||||||
|
posted := fake.postedFor(history.parent)
|
||||||
|
if !tc.wantMark {
|
||||||
|
require.Empty(t, posted)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
require.Equal(t, []postedStatus{{
|
||||||
|
Context: liveContext,
|
||||||
|
// Not `skipped`: Gitea's combined status folds
|
||||||
|
// that into `success`, which is what made a
|
||||||
|
// never-tested commit read green.
|
||||||
|
State: failure,
|
||||||
|
Description: supersededDesc,
|
||||||
|
}}, posted)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A second run must not rewrite what the first one wrote, or every
|
||||||
|
// later push would post a duplicate status.
|
||||||
|
func TestMarkSupersededIsIdempotent(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
requireTools(t)
|
||||||
|
|
||||||
|
history := newRepo(t)
|
||||||
|
fake, api := newFakeGitea(t)
|
||||||
|
fake.setStatus(history.head, running())
|
||||||
|
fake.setStatus(history.parent, cancelled())
|
||||||
|
|
||||||
|
for range 2 {
|
||||||
|
out, err := runScript(t, history, api, defaultEnv())
|
||||||
|
require.NoError(t, err, out)
|
||||||
|
}
|
||||||
|
|
||||||
|
require.Len(t, fake.postedFor(history.parent), 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Renaming the workflow or the job changes the context string Gitea
|
||||||
|
// uses. The script must say so instead of quietly matching nothing.
|
||||||
|
func TestMarkSupersededRejectsAnUnknownContext(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
requireTools(t)
|
||||||
|
|
||||||
|
history := newRepo(t)
|
||||||
|
fake, api := newFakeGitea(t)
|
||||||
|
fake.setStatus(history.head, running())
|
||||||
|
fake.setStatus(history.parent, cancelled())
|
||||||
|
|
||||||
|
env := defaultEnv()
|
||||||
|
env.job = "renamed"
|
||||||
|
|
||||||
|
out, err := runScript(t, history, api, env)
|
||||||
|
require.Error(t, err)
|
||||||
|
require.Contains(t, out, "renamed")
|
||||||
|
require.Contains(t, out, liveContext)
|
||||||
|
require.Empty(t, fake.postedFor(history.parent))
|
||||||
|
}
|
||||||
|
|
||||||
|
// ANCESTOR_LIMIT is a documented knob. A value that is set but unusable
|
||||||
|
// must abort: handing it to git and discarding the exit status left the
|
||||||
|
// walk empty and the step green, marking nothing.
|
||||||
|
func TestMarkSupersededRejectsAnUnparseableAncestorLimit(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
requireTools(t)
|
||||||
|
|
||||||
|
history := newRepo(t)
|
||||||
|
fake, api := newFakeGitea(t)
|
||||||
|
fake.setStatus(history.head, running())
|
||||||
|
fake.setStatus(history.parent, cancelled())
|
||||||
|
|
||||||
|
out, err := runScript(
|
||||||
|
t, history, api, defaultEnv(), "ANCESTOR_LIMIT=twenty",
|
||||||
|
)
|
||||||
|
require.Error(t, err)
|
||||||
|
require.Contains(t, out, "ANCESTOR_LIMIT")
|
||||||
|
require.Contains(t, out, "twenty")
|
||||||
|
require.Empty(t, fake.postedFor(history.parent))
|
||||||
|
}
|
||||||
|
|
||||||
|
// A status read that fails is not the same as a commit with nothing to
|
||||||
|
// do. Losing curl's exit status through a pipe made the two identical
|
||||||
|
// and left a laundered commit laundered with no signal.
|
||||||
|
func TestMarkSupersededFailsOnAnUnreadableAncestorStatus(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
requireTools(t)
|
||||||
|
|
||||||
|
history := newRepo(t)
|
||||||
|
fake, api := newFakeGitea(t)
|
||||||
|
fake.setStatus(history.head, running())
|
||||||
|
fake.setStatus(history.parent, cancelled())
|
||||||
|
fake.failStatusRead(history.parent)
|
||||||
|
|
||||||
|
out, err := runScript(t, history, api, defaultEnv())
|
||||||
|
require.Error(t, err)
|
||||||
|
require.Contains(t, out, history.parent)
|
||||||
|
require.Contains(t, out, "cannot read commit statuses")
|
||||||
|
require.Empty(t, fake.postedFor(history.parent))
|
||||||
|
}
|
||||||
|
|
||||||
|
// A shallow clone cannot resolve the parent, so it is indistinguishable
|
||||||
|
// from a root commit to rev-parse and the walk would exit 0 having
|
||||||
|
// marked nothing. It must abort instead: dropping `fetch-depth: 0` from
|
||||||
|
// the checkout step is one edit, and a silent no-op there restores the
|
||||||
|
// false-green bug this script exists to prevent.
|
||||||
|
func TestMarkSupersededRejectsAShallowRepository(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
requireTools(t)
|
||||||
|
|
||||||
|
history := shallowClone(t, newRepo(t))
|
||||||
|
fake, api := newFakeGitea(t)
|
||||||
|
fake.setStatus(history.head, running())
|
||||||
|
fake.setStatus(history.parent, cancelled())
|
||||||
|
|
||||||
|
out, err := runScript(t, history, api, defaultEnv())
|
||||||
|
require.Error(t, err)
|
||||||
|
require.Contains(t, out, "shallow repository")
|
||||||
|
require.Empty(t, fake.postedFor(history.parent))
|
||||||
|
require.Empty(t, fake.postedFor(history.head))
|
||||||
|
}
|
||||||
|
|
||||||
|
// shallowClone returns the same history as a depth-1 clone. The `file://`
|
||||||
|
// URL is required: git ignores --depth for a plain local path.
|
||||||
|
func shallowClone(t *testing.T, history repo) repo {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
|
||||||
|
//nolint:gosec // fixed argv, arguments are test-local paths
|
||||||
|
cmd := exec.CommandContext(t.Context(), "git", "clone", "-q",
|
||||||
|
"--depth=1", "file://"+history.dir, dir)
|
||||||
|
|
||||||
|
out, err := cmd.CombinedOutput()
|
||||||
|
require.NoError(t, err, string(out))
|
||||||
|
|
||||||
|
return repo{dir: dir, head: history.head, parent: history.parent}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The derived context must equal the one Gitea actually uses, which is
|
||||||
|
// built from the same workflow and job names.
|
||||||
|
func TestDerivedContextMatchesGitea(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
requireTools(t)
|
||||||
|
|
||||||
|
name, job := workflowIdentity(t)
|
||||||
|
|
||||||
|
history := newRepo(t)
|
||||||
|
fake, api := newFakeGitea(t)
|
||||||
|
fake.setStatus(history.head, running())
|
||||||
|
fake.setStatus(history.parent, cancelled())
|
||||||
|
|
||||||
|
out, err := runScript(t, history, api, scriptEnv{
|
||||||
|
workflow: name,
|
||||||
|
job: job,
|
||||||
|
event: "push",
|
||||||
|
})
|
||||||
|
require.NoError(t, err, out)
|
||||||
|
|
||||||
|
posted := fake.postedFor(history.parent)
|
||||||
|
require.Len(t, posted, 1)
|
||||||
|
require.Equal(t, liveContext, posted[0].Context)
|
||||||
|
}
|
||||||
|
|
||||||
|
// workflowIdentity reads the workflow name and its single job id out of
|
||||||
|
// the checked-in workflow file.
|
||||||
|
func workflowIdentity(t *testing.T) (string, string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
raw, err := os.ReadFile(workflow)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
var parsed struct {
|
||||||
|
Name string `yaml:"name"`
|
||||||
|
Jobs map[string]any `yaml:"jobs"`
|
||||||
|
}
|
||||||
|
|
||||||
|
require.NoError(t, yaml.Unmarshal(raw, &parsed))
|
||||||
|
|
||||||
|
jobs := slices.Collect(maps.Keys(parsed.Jobs))
|
||||||
|
require.Len(t, jobs, 1)
|
||||||
|
|
||||||
|
return parsed.Name, jobs[0]
|
||||||
|
}
|
||||||
|
|
||||||
|
func runScript(
|
||||||
|
t *testing.T, history repo, api string, env scriptEnv,
|
||||||
|
extra ...string,
|
||||||
|
) (string, error) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
script, err := filepath.Abs(scriptPath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
//nolint:gosec // fixed argv, repo-local script under test
|
||||||
|
cmd := exec.CommandContext(t.Context(), "sh", script)
|
||||||
|
cmd.Dir = history.dir
|
||||||
|
cmd.Env = append(os.Environ(),
|
||||||
|
"GITHUB_API_URL="+api,
|
||||||
|
"GITHUB_REPOSITORY=sneak/webhooker",
|
||||||
|
"GITHUB_SHA="+history.head,
|
||||||
|
"GITHUB_WORKFLOW="+env.workflow,
|
||||||
|
"GITHUB_JOB="+env.job,
|
||||||
|
"GITHUB_EVENT_NAME="+env.event,
|
||||||
|
"GITEA_TOKEN=test-token",
|
||||||
|
)
|
||||||
|
cmd.Env = append(cmd.Env, extra...)
|
||||||
|
|
||||||
|
out, err := cmd.CombinedOutput()
|
||||||
|
|
||||||
|
return string(out), err
|
||||||
|
}
|
||||||
|
|
||||||
|
func newRepo(t *testing.T) repo {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
|
||||||
|
git := func(args ...string) string {
|
||||||
|
//nolint:gosec // fixed argv, arguments are test constants
|
||||||
|
cmd := exec.CommandContext(t.Context(), "git", args...)
|
||||||
|
cmd.Dir = dir
|
||||||
|
|
||||||
|
out, err := cmd.CombinedOutput()
|
||||||
|
require.NoError(t, err, string(out))
|
||||||
|
|
||||||
|
return strings.TrimSpace(string(out))
|
||||||
|
}
|
||||||
|
|
||||||
|
commit := func(message string) string {
|
||||||
|
git(
|
||||||
|
"-c", "user.email=ci@example.invalid",
|
||||||
|
"-c", "user.name=ci",
|
||||||
|
"-c", "commit.gpgsign=false",
|
||||||
|
"commit", "-q", "--allow-empty", "-m", message,
|
||||||
|
)
|
||||||
|
|
||||||
|
return git("rev-parse", "HEAD")
|
||||||
|
}
|
||||||
|
|
||||||
|
git("init", "-q", "-b", "main")
|
||||||
|
|
||||||
|
parent := commit("parent")
|
||||||
|
head := commit("head")
|
||||||
|
|
||||||
|
return repo{dir: dir, head: head, parent: parent}
|
||||||
|
}
|
||||||
|
|
||||||
|
func requireTools(t *testing.T) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
for _, tool := range []string{"sh", "git", "curl", "jq"} {
|
||||||
|
_, err := exec.LookPath(tool)
|
||||||
|
if err != nil {
|
||||||
|
t.Skipf("%s is not installed: %v", tool, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
// Package ciscript holds the tests for the repository's CI shell
|
||||||
|
// scripts in script/. It carries no runtime code: the scripts run on
|
||||||
|
// the CI runner, not inside the binary, but their behaviour still has
|
||||||
|
// to be verified by the test suite.
|
||||||
|
//
|
||||||
|
// The scripts under test are outside the Go build graph, so `go test`'s
|
||||||
|
// result cache serves a stale PASS when only a script changed: run the
|
||||||
|
// container build, or GOFLAGS=-count=1, to trust a result here after
|
||||||
|
// editing script/.
|
||||||
|
package ciscript
|
||||||
@@ -0,0 +1,162 @@
|
|||||||
|
package ciscript_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// commitStatus is the part of an entry in Gitea's combined-status
|
||||||
|
// response that script/ci-mark-superseded reads.
|
||||||
|
type commitStatus struct {
|
||||||
|
Context string `json:"context"`
|
||||||
|
Status string `json:"status"`
|
||||||
|
Description string `json:"description"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// postedStatus is the part of a create-status request body the script
|
||||||
|
// writes.
|
||||||
|
type postedStatus struct {
|
||||||
|
Context string `json:"context"`
|
||||||
|
State string `json:"state"`
|
||||||
|
Description string `json:"description"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// fakeGitea serves the two endpoints the script talks to. Like Gitea,
|
||||||
|
// the newest status for a context replaces the previous one, so a
|
||||||
|
// second run of the script sees what the first one wrote.
|
||||||
|
type fakeGitea struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
statuses map[string][]commitStatus
|
||||||
|
posted map[string][]postedStatus
|
||||||
|
// failRead is a commit whose combined-status read answers HTTP
|
||||||
|
// 500, standing in for a status API that is down.
|
||||||
|
failRead string
|
||||||
|
}
|
||||||
|
|
||||||
|
// newFakeGitea returns the fake and the base URL to hand the script as
|
||||||
|
// GITHUB_API_URL.
|
||||||
|
func newFakeGitea(t *testing.T) (*fakeGitea, string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
fake := &fakeGitea{
|
||||||
|
mu: sync.Mutex{},
|
||||||
|
statuses: map[string][]commitStatus{},
|
||||||
|
posted: map[string][]postedStatus{},
|
||||||
|
failRead: "",
|
||||||
|
}
|
||||||
|
|
||||||
|
srv := httptest.NewServer(fake.routes())
|
||||||
|
t.Cleanup(srv.Close)
|
||||||
|
|
||||||
|
return fake, srv.URL
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeGitea) routes() http.Handler {
|
||||||
|
mux := http.NewServeMux()
|
||||||
|
|
||||||
|
mux.HandleFunc(
|
||||||
|
"GET /repos/{owner}/{repo}/commits/{sha}/status",
|
||||||
|
f.handleCombined,
|
||||||
|
)
|
||||||
|
mux.HandleFunc(
|
||||||
|
"POST /repos/{owner}/{repo}/statuses/{sha}",
|
||||||
|
f.handleCreate,
|
||||||
|
)
|
||||||
|
|
||||||
|
return mux
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeGitea) handleCombined(
|
||||||
|
w http.ResponseWriter, r *http.Request,
|
||||||
|
) {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
|
||||||
|
sha := r.PathValue("sha")
|
||||||
|
if f.failRead != "" && f.failRead == sha {
|
||||||
|
http.Error(w, "boom", http.StatusInternalServerError)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
body := struct {
|
||||||
|
Statuses []commitStatus `json:"statuses"`
|
||||||
|
}{Statuses: f.statuses[sha]}
|
||||||
|
|
||||||
|
payload, err := json.Marshal(body)
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
|
||||||
|
_, _ = w.Write(payload)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeGitea) handleCreate(w http.ResponseWriter, r *http.Request) {
|
||||||
|
var got postedStatus
|
||||||
|
|
||||||
|
err := json.NewDecoder(r.Body).Decode(&got)
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
sha := r.PathValue("sha")
|
||||||
|
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
|
||||||
|
f.posted[sha] = append(f.posted[sha], got)
|
||||||
|
f.replaceLocked(sha, commitStatus{
|
||||||
|
Context: got.Context,
|
||||||
|
Status: got.State,
|
||||||
|
Description: got.Description,
|
||||||
|
})
|
||||||
|
|
||||||
|
w.WriteHeader(http.StatusCreated)
|
||||||
|
}
|
||||||
|
|
||||||
|
// failStatusRead makes the combined-status read for one commit answer
|
||||||
|
// HTTP 500.
|
||||||
|
func (f *fakeGitea) failStatusRead(sha string) {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
|
||||||
|
f.failRead = sha
|
||||||
|
}
|
||||||
|
|
||||||
|
// setStatus gives a commit its latest status for a context.
|
||||||
|
func (f *fakeGitea) setStatus(sha string, status commitStatus) {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
|
||||||
|
f.replaceLocked(sha, status)
|
||||||
|
}
|
||||||
|
|
||||||
|
// postedFor returns the statuses the script created for a commit.
|
||||||
|
func (f *fakeGitea) postedFor(sha string) []postedStatus {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
|
||||||
|
return append([]postedStatus(nil), f.posted[sha]...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// replaceLocked requires f.mu.
|
||||||
|
func (f *fakeGitea) replaceLocked(sha string, status commitStatus) {
|
||||||
|
for i, existing := range f.statuses[sha] {
|
||||||
|
if existing.Context == status.Context {
|
||||||
|
f.statuses[sha][i] = status
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
f.statuses[sha] = append(f.statuses[sha], status)
|
||||||
|
}
|
||||||
@@ -1,37 +0,0 @@
|
|||||||
package config_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
|
||||||
"sneak.berlin/go/webhooker/internal/config/configtest"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestClearEnv_RemovesAddedVariables pins that a variable set
|
|
||||||
// after the clear other than through t.Setenv, as a test's .env file
|
|
||||||
// sets one, is gone once the test ends, so it cannot reach the tests
|
|
||||||
// that run after it.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // ClearEnv uses t.Setenv.
|
|
||||||
func TestClearEnv_RemovesAddedVariables(t *testing.T) {
|
|
||||||
// The outer clear keeps a value of the key exported in the shell
|
|
||||||
// from making it a variable the inner clear has to put back.
|
|
||||||
configtest.ClearEnv(t)
|
|
||||||
|
|
||||||
t.Run("loads a .env file after the clear", func(t *testing.T) {
|
|
||||||
configtest.ClearEnv(t)
|
|
||||||
|
|
||||||
path := writeDotEnv(t, dotEnvKey+"=from-dot-env\n")
|
|
||||||
require.NoError(t, config.LoadDotEnvFileForTest(path))
|
|
||||||
require.Equal(t, "from-dot-env", os.Getenv(dotEnvKey))
|
|
||||||
})
|
|
||||||
|
|
||||||
_, present := os.LookupEnv(dotEnvKey)
|
|
||||||
assert.False(
|
|
||||||
t, present,
|
|
||||||
"a variable set after the clear must not outlive the test",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
+94
-101
@@ -68,10 +68,6 @@ const (
|
|||||||
// authenticated activity before it expires.
|
// authenticated activity before it expires.
|
||||||
defaultSessionIdleTimeout = 24 * time.Hour
|
defaultSessionIdleTimeout = 24 * time.Hour
|
||||||
|
|
||||||
// defaultHSTSMaxAge is the max-age of the Strict-Transport-Security
|
|
||||||
// header when HSTS_MAX_AGE is unset: ten years.
|
|
||||||
defaultHSTSMaxAge = 87600 * time.Hour
|
|
||||||
|
|
||||||
// defaultReceiverRateLimit is the default number of requests
|
// defaultReceiverRateLimit is the default number of requests
|
||||||
// per minute each client IP may send to a single webhook
|
// per minute each client IP may send to a single webhook
|
||||||
// receiver entrypoint. Generous for legitimate webhook
|
// receiver entrypoint. Generous for legitimate webhook
|
||||||
@@ -79,12 +75,8 @@ const (
|
|||||||
// internet-exposed endpoint.
|
// internet-exposed endpoint.
|
||||||
defaultReceiverRateLimit = 120
|
defaultReceiverRateLimit = 120
|
||||||
|
|
||||||
// defaultTrustedProxies is TRUSTED_PROXIES when it is unset: the
|
// maxPort is the highest valid TCP port number. The lower
|
||||||
// RFC 1918 private ranges, which a reverse proxy reaching the
|
// bound (at least 1) is enforced by envPositiveInt.
|
||||||
// process over a Docker network or a private LAN connects from.
|
|
||||||
defaultTrustedProxies = "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16"
|
|
||||||
|
|
||||||
// maxPort is the highest valid TCP port number.
|
|
||||||
maxPort = 65535
|
maxPort = 65535
|
||||||
|
|
||||||
// mappedV4Offset is the number of leading bits an IPv4-mapped
|
// mappedV4Offset is the number of leading bits an IPv4-mapped
|
||||||
@@ -108,7 +100,7 @@ var ErrInvalidEnvironment = errors.New("invalid environment")
|
|||||||
var ErrNonPositiveValue = errors.New("value must be positive")
|
var ErrNonPositiveValue = errors.New("value must be positive")
|
||||||
|
|
||||||
// ErrInvalidPort is returned when an environment variable holding a
|
// ErrInvalidPort is returned when an environment variable holding a
|
||||||
// TCP port number is set to a number outside 1 to 65535.
|
// TCP port number is set above the valid port range.
|
||||||
var ErrInvalidPort = errors.New("invalid port")
|
var ErrInvalidPort = errors.New("invalid port")
|
||||||
|
|
||||||
// ErrInvalidCIDR is returned when an environment variable holding a
|
// ErrInvalidCIDR is returned when an environment variable holding a
|
||||||
@@ -124,10 +116,6 @@ var ErrInvalidBindAddress = errors.New("invalid bind address")
|
|||||||
// the Sentry SDK cannot parse as a DSN.
|
// the Sentry SDK cannot parse as a DSN.
|
||||||
var ErrInvalidSentryDSN = errors.New("invalid Sentry DSN")
|
var ErrInvalidSentryDSN = errors.New("invalid Sentry DSN")
|
||||||
|
|
||||||
// ErrInvalidHSTSMaxAge is returned when HSTS_MAX_AGE is set to a
|
|
||||||
// negative duration or one that is not a whole number of seconds.
|
|
||||||
var ErrInvalidHSTSMaxAge = errors.New("invalid HSTS max-age")
|
|
||||||
|
|
||||||
// ErrDotEnvUnreadable is returned when the optional .env file exists
|
// ErrDotEnvUnreadable is returned when the optional .env file exists
|
||||||
// but cannot be read or parsed. A file that is not there is not an
|
// but cannot be read or parsed. A file that is not there is not an
|
||||||
// error; a file that is there and broken is.
|
// error; a file that is there and broken is.
|
||||||
@@ -156,6 +144,7 @@ type ConfigParams struct {
|
|||||||
type Config struct {
|
type Config struct {
|
||||||
DataDir string
|
DataDir string
|
||||||
Debug bool
|
Debug bool
|
||||||
|
MaintenanceMode bool
|
||||||
Environment string
|
Environment string
|
||||||
MetricsPassword string
|
MetricsPassword string
|
||||||
MetricsUsername string
|
MetricsUsername string
|
||||||
@@ -177,24 +166,19 @@ type Config struct {
|
|||||||
// which a session expires. Non-positive disables idle expiry.
|
// which a session expires. Non-positive disables idle expiry.
|
||||||
SessionIdleTimeout time.Duration
|
SessionIdleTimeout time.Duration
|
||||||
|
|
||||||
// HSTSMaxAge is the max-age of the Strict-Transport-Security
|
|
||||||
// header. Zero means the header is not sent at all.
|
|
||||||
HSTSMaxAge time.Duration
|
|
||||||
|
|
||||||
// ReceiverRateLimit is the number of requests per minute each
|
// ReceiverRateLimit is the number of requests per minute each
|
||||||
// client IP may send to a single webhook receiver entrypoint.
|
// client IP may send to a single webhook receiver entrypoint.
|
||||||
ReceiverRateLimit int
|
ReceiverRateLimit int
|
||||||
|
|
||||||
// TrustedProxies is the set of networks whose members are
|
// TrustedProxies is the set of networks whose members are
|
||||||
// allowed to speak for the client with X-Forwarded-For, the
|
// allowed to speak for the client with X-Forwarded-For, the
|
||||||
// only forwarded header read. Unless TRUSTED_PROXIES is set it
|
// only forwarded header read. It is empty unless
|
||||||
// is the RFC 1918 private ranges (defaultTrustedProxies); a set
|
// TRUSTED_PROXIES is set, and empty means no peer is
|
||||||
// value replaces them. If any client can reach the process, or
|
// trusted: forwarded headers are then ignored entirely and
|
||||||
// the proxy in front of it, from an RFC 1918 source address
|
// clients are identified by the connection's own address.
|
||||||
// (directly, or through anything that can rewrite source
|
// Members can choose their own rate-limit key, so this must
|
||||||
// addresses, such as NAT or a published container port), it
|
// name proxy hosts only, never a block that also covers
|
||||||
// must be set to the proxy's address alone, or every rate limit
|
// clients.
|
||||||
// can be bypassed by those clients.
|
|
||||||
TrustedProxies []netip.Prefix
|
TrustedProxies []netip.Prefix
|
||||||
|
|
||||||
// AllowedEgressCIDRs is the set of networks a delivery target
|
// AllowedEgressCIDRs is the set of networks a delivery target
|
||||||
@@ -206,13 +190,11 @@ type Config struct {
|
|||||||
// otherwise refuse. The guard itself is always on: there is no
|
// otherwise refuse. The guard itself is always on: there is no
|
||||||
// setting that disables SSRF protection, and delivery's
|
// setting that disables SSRF protection, and delivery's
|
||||||
// alwaysBlockedNetworks stays blocked no matter what is listed
|
// alwaysBlockedNetworks stays blocked no matter what is listed
|
||||||
// here. That set is link-local, the unspecified addresses
|
// here. That set is link-local plus the cloud metadata
|
||||||
// 0.0.0.0 and ::, and the cloud metadata endpoints outside
|
// endpoints outside it that disclose credentials or user data
|
||||||
// link-local that disclose credentials or user data at a
|
// at a provider-fixed address; it is not exhaustive of every
|
||||||
// provider-fixed, non-public address; it is not exhaustive of
|
// cloud's metadata address. See alwaysBlockedNetworks for the
|
||||||
// every cloud's metadata address. See
|
// authoritative list and the criterion it is built from.
|
||||||
// alwaysBlockedNetworks for the authoritative list and why
|
|
||||||
// each entry is on it.
|
|
||||||
AllowedEgressCIDRs []netip.Prefix
|
AllowedEgressCIDRs []netip.Prefix
|
||||||
|
|
||||||
params *ConfigParams
|
params *ConfigParams
|
||||||
@@ -374,27 +356,17 @@ func envPositiveInt(
|
|||||||
// envPort returns the value of the named environment variable parsed
|
// envPort returns the value of the named environment variable parsed
|
||||||
// as a TCP port number. Returns defaultValue if not set. A set value
|
// as a TCP port number. Returns defaultValue if not set. A set value
|
||||||
// that is unparseable, below 1, or above maxPort is a hard error
|
// that is unparseable, below 1, or above maxPort is a hard error
|
||||||
// naming the key and the bad value; every out-of-range value wraps
|
// naming the key and the bad value.
|
||||||
// ErrInvalidPort, including one too large or too small for an int.
|
|
||||||
func envPort(key string, defaultValue int) (int, error) {
|
func envPort(key string, defaultValue int) (int, error) {
|
||||||
v := os.Getenv(key)
|
port, err := envPositiveInt(key, defaultValue)
|
||||||
if v == "" {
|
if err != nil {
|
||||||
return defaultValue, nil
|
return 0, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// strconv.ErrRange means a number too large or too small for an
|
if port > maxPort {
|
||||||
// int, which is outside the port range as well.
|
|
||||||
port, err := strconv.Atoi(v)
|
|
||||||
if err != nil && !errors.Is(err, strconv.ErrRange) {
|
|
||||||
return 0, fmt.Errorf(
|
return 0, fmt.Errorf(
|
||||||
"invalid integer for %s: %q: %w", key, v, err,
|
"%w: %s must be at most %d, got %d",
|
||||||
)
|
ErrInvalidPort, key, maxPort, port,
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil || port < 1 || port > maxPort {
|
|
||||||
return 0, fmt.Errorf(
|
|
||||||
"%w: %s must be from 1 to %d, got %q",
|
|
||||||
ErrInvalidPort, key, maxPort, v,
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -487,15 +459,14 @@ func parseCIDR(entry string) (netip.Prefix, error) {
|
|||||||
|
|
||||||
// envPrefixList returns the value of the named environment variable
|
// envPrefixList returns the value of the named environment variable
|
||||||
// parsed as a comma-separated list of CIDR blocks (bare addresses
|
// parsed as a comma-separated list of CIDR blocks (bare addresses
|
||||||
// allowed). An unset, empty, or blank value is read as defaultValue
|
// allowed). An unset, empty, or blank value yields an empty list. A
|
||||||
// instead. A set value containing an unparseable entry is a hard
|
// set value containing an unparseable entry is a hard error naming
|
||||||
// error naming the key and the bad entry, so startup fails loudly
|
// the key and the bad entry, so startup fails loudly rather than
|
||||||
// rather than silently running with a list the operator did not
|
// silently running with a list the operator did not intend.
|
||||||
// intend.
|
func envPrefixList(key string) ([]netip.Prefix, error) {
|
||||||
func envPrefixList(key, defaultValue string) ([]netip.Prefix, error) {
|
|
||||||
v := strings.TrimSpace(os.Getenv(key))
|
v := strings.TrimSpace(os.Getenv(key))
|
||||||
if v == "" {
|
if v == "" {
|
||||||
v = defaultValue
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
var prefixes []netip.Prefix
|
var prefixes []netip.Prefix
|
||||||
@@ -614,14 +585,12 @@ func resolveMetricsAuth() (string, string, error) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// resolveEnvironment reads WEBHOOKER_ENVIRONMENT, defaulting to prod
|
// resolveEnvironment reads WEBHOOKER_ENVIRONMENT, defaulting to
|
||||||
// when it is unset so a deployment that forgets the variable is not
|
// dev, and rejects unrecognised values.
|
||||||
// silently permissive; dev must be set explicitly. It rejects
|
|
||||||
// unrecognised values.
|
|
||||||
func resolveEnvironment() (string, error) {
|
func resolveEnvironment() (string, error) {
|
||||||
environment := os.Getenv("WEBHOOKER_ENVIRONMENT")
|
environment := os.Getenv("WEBHOOKER_ENVIRONMENT")
|
||||||
if environment == "" {
|
if environment == "" {
|
||||||
environment = EnvironmentProd
|
environment = EnvironmentDev
|
||||||
}
|
}
|
||||||
|
|
||||||
if environment != EnvironmentDev &&
|
if environment != EnvironmentDev &&
|
||||||
@@ -636,27 +605,6 @@ func resolveEnvironment() (string, error) {
|
|||||||
return environment, nil
|
return environment, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// resolveHSTSMaxAge reads HSTS_MAX_AGE. Zero is allowed and turns the
|
|
||||||
// Strict-Transport-Security header off. A fraction of a second is an
|
|
||||||
// error rather than rounded, because the header carries whole seconds
|
|
||||||
// and would otherwise say something other than what was set.
|
|
||||||
func resolveHSTSMaxAge() (time.Duration, error) {
|
|
||||||
maxAge, err := envDuration("HSTS_MAX_AGE", defaultHSTSMaxAge)
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if maxAge < 0 || maxAge%time.Second != 0 {
|
|
||||||
return 0, fmt.Errorf(
|
|
||||||
"%w: HSTS_MAX_AGE must be zero or a positive whole "+
|
|
||||||
"number of seconds, got %s",
|
|
||||||
ErrInvalidHSTSMaxAge, maxAge,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return maxAge, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// resolveListener reads the two variables that describe the HTTP
|
// resolveListener reads the two variables that describe the HTTP
|
||||||
// listener: which port it claims and which address it claims it on.
|
// listener: which port it claims and which address it claims it on.
|
||||||
// They are read together because neither is meaningful alone, and
|
// They are read together because neither is meaningful alone, and
|
||||||
@@ -699,6 +647,11 @@ func loadFromEnv() (*Config, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
maintenanceMode, err := envBool("MAINTENANCE_MODE", false)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
retentionSweepInterval, err := envPositiveDuration(
|
retentionSweepInterval, err := envPositiveDuration(
|
||||||
"RETENTION_SWEEP_INTERVAL",
|
"RETENTION_SWEEP_INTERVAL",
|
||||||
defaultRetentionSweepInterval,
|
defaultRetentionSweepInterval,
|
||||||
@@ -717,11 +670,6 @@ func loadFromEnv() (*Config, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
hstsMaxAge, err := resolveHSTSMaxAge()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
receiverRateLimit, err := envPositiveInt(
|
receiverRateLimit, err := envPositiveInt(
|
||||||
"RECEIVER_RATE_LIMIT",
|
"RECEIVER_RATE_LIMIT",
|
||||||
defaultReceiverRateLimit,
|
defaultReceiverRateLimit,
|
||||||
@@ -730,12 +678,12 @@ func loadFromEnv() (*Config, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
trustedProxies, err := envPrefixList("TRUSTED_PROXIES", defaultTrustedProxies)
|
trustedProxies, err := envPrefixList("TRUSTED_PROXIES")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
allowedEgressCIDRs, err := envPrefixList("ALLOWED_EGRESS_CIDRS", "")
|
allowedEgressCIDRs, err := envPrefixList("ALLOWED_EGRESS_CIDRS")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -753,6 +701,7 @@ func loadFromEnv() (*Config, error) {
|
|||||||
return &Config{
|
return &Config{
|
||||||
DataDir: DataDir(),
|
DataDir: DataDir(),
|
||||||
Debug: debug,
|
Debug: debug,
|
||||||
|
MaintenanceMode: maintenanceMode,
|
||||||
Environment: environment,
|
Environment: environment,
|
||||||
MetricsUsername: metricsUsername,
|
MetricsUsername: metricsUsername,
|
||||||
MetricsPassword: metricsPassword,
|
MetricsPassword: metricsPassword,
|
||||||
@@ -761,7 +710,6 @@ func loadFromEnv() (*Config, error) {
|
|||||||
SentryDSN: sentryDSN,
|
SentryDSN: sentryDSN,
|
||||||
RetentionSweepInterval: retentionSweepInterval,
|
RetentionSweepInterval: retentionSweepInterval,
|
||||||
SessionIdleTimeout: sessionIdleTimeout,
|
SessionIdleTimeout: sessionIdleTimeout,
|
||||||
HSTSMaxAge: hstsMaxAge,
|
|
||||||
ReceiverRateLimit: receiverRateLimit,
|
ReceiverRateLimit: receiverRateLimit,
|
||||||
TrustedProxies: trustedProxies,
|
TrustedProxies: trustedProxies,
|
||||||
AllowedEgressCIDRs: allowedEgressCIDRs,
|
AllowedEgressCIDRs: allowedEgressCIDRs,
|
||||||
@@ -796,19 +744,63 @@ func (c *Config) warnEgressAllowlist(log *slog.Logger) {
|
|||||||
|
|
||||||
log.Warn(
|
log.Warn(
|
||||||
"ALLOWED_EGRESS_CIDRS lets delivery targets reach these "+
|
"ALLOWED_EGRESS_CIDRS lets delivery targets reach these "+
|
||||||
"otherwise-blocked networks. Anyone who can create a "+
|
"otherwise-blocked private/reserved networks. Anyone "+
|
||||||
"delivery target can now make this process issue "+
|
"who can create a delivery target can now make this "+
|
||||||
"requests into them, and read back the response. Only "+
|
"process issue requests into them, and read back the "+
|
||||||
"the addresses the README lists as blocked "+
|
"response. Link-local and the known cloud instance "+
|
||||||
"unconditionally stay blocked regardless of what is "+
|
"metadata endpoints outside it stay blocked "+
|
||||||
"listed here; a public cloud metadata address such as "+
|
"regardless of what is listed here.",
|
||||||
"168.63.129.16 is reachable once it, or a block "+
|
|
||||||
"covering it, is listed.",
|
|
||||||
"allowedEgressCIDRs",
|
"allowedEgressCIDRs",
|
||||||
strings.Join(PrefixStrings(c.AllowedEgressCIDRs), ","),
|
strings.Join(PrefixStrings(c.AllowedEgressCIDRs), ","),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// warnSharedRateLimitBucket logs a startup warning whenever
|
||||||
|
// TRUSTED_PROXIES is empty, in any environment.
|
||||||
|
//
|
||||||
|
// With no trusted proxies every rate limiter keys on the connecting
|
||||||
|
// peer's address. Whether that is harmless or dangerous depends on
|
||||||
|
// what is in front of the process, which this code cannot observe:
|
||||||
|
// with nothing in front, the peer is the client and the limits are
|
||||||
|
// per-client as intended; behind a reverse proxy the peer is the proxy
|
||||||
|
// for every request, so all clients share one bucket per limiter.
|
||||||
|
//
|
||||||
|
// The login endpoint no longer spends budget on arrival — it verifies
|
||||||
|
// credentials first and charges only failures — so a shared bucket
|
||||||
|
// cannot deny the operator a correct password. What it does collapse
|
||||||
|
// is the failure counting: one client's wrong passwords throttle
|
||||||
|
// everyone else's wrong passwords, and the receiver's limits become
|
||||||
|
// service-wide ceilings.
|
||||||
|
//
|
||||||
|
// The warning is deliberately not gated on WEBHOOKER_ENVIRONMENT. That
|
||||||
|
// variable defaults to dev, so gating on it would silence the warning
|
||||||
|
// for exactly the operator who forgot to configure the deployment —
|
||||||
|
// the case it exists to catch.
|
||||||
|
//
|
||||||
|
// The default of trusting nobody is deliberate — trusting forwarded
|
||||||
|
// headers from arbitrary peers lets any client choose its own bucket —
|
||||||
|
// so this warns rather than failing startup or changing the key.
|
||||||
|
func (c *Config) warnSharedRateLimitBucket(log *slog.Logger) {
|
||||||
|
if len(c.TrustedProxies) > 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Warn(
|
||||||
|
"TRUSTED_PROXIES is empty: every rate limit keys on the "+
|
||||||
|
"connecting peer's address. With nothing proxying to "+
|
||||||
|
"this process that is the client itself and the limits "+
|
||||||
|
"are per-client as intended. Behind a reverse proxy the "+
|
||||||
|
"peer is the proxy on every request, so all clients "+
|
||||||
|
"share one bucket per limit: the receiver limits become "+
|
||||||
|
"service-wide ceilings, and one client's failed logins "+
|
||||||
|
"throttle every other client's failed logins — a "+
|
||||||
|
"correct password still gets in. If anything proxies to "+
|
||||||
|
"this process, set TRUSTED_PROXIES to its address.",
|
||||||
|
"environment", c.Environment,
|
||||||
|
"trustedProxies", len(c.TrustedProxies),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
// New creates a Config by reading environment variables.
|
// New creates a Config by reading environment variables.
|
||||||
//
|
//
|
||||||
//nolint:revive // lc parameter is required by fx even if unused.
|
//nolint:revive // lc parameter is required by fx even if unused.
|
||||||
@@ -840,13 +832,13 @@ func New(lc fx.Lifecycle, params ConfigParams) (*Config, error) {
|
|||||||
// host can reach the admin UI.
|
// host can reach the admin UI.
|
||||||
"bindAddress", s.BindAddress,
|
"bindAddress", s.BindAddress,
|
||||||
"debug", s.Debug,
|
"debug", s.Debug,
|
||||||
|
"maintenanceMode", s.MaintenanceMode,
|
||||||
"dataDir", s.DataDir,
|
"dataDir", s.DataDir,
|
||||||
"retentionSweepInterval", s.RetentionSweepInterval.String(),
|
"retentionSweepInterval", s.RetentionSweepInterval.String(),
|
||||||
// Logged because a perfectly valid non-positive value here
|
// Logged because a perfectly valid non-positive value here
|
||||||
// disables idle expiry entirely, and that is worth showing
|
// disables idle expiry entirely, and that is worth showing
|
||||||
// back to the operator.
|
// back to the operator.
|
||||||
"sessionIdleTimeout", s.SessionIdleTimeout.String(),
|
"sessionIdleTimeout", s.SessionIdleTimeout.String(),
|
||||||
"hstsMaxAge", s.HSTSMaxAge.String(),
|
|
||||||
"receiverRateLimit", s.ReceiverRateLimit,
|
"receiverRateLimit", s.ReceiverRateLimit,
|
||||||
"trustedProxies", len(s.TrustedProxies),
|
"trustedProxies", len(s.TrustedProxies),
|
||||||
"allowedEgressCIDRs", len(s.AllowedEgressCIDRs),
|
"allowedEgressCIDRs", len(s.AllowedEgressCIDRs),
|
||||||
@@ -854,6 +846,7 @@ func New(lc fx.Lifecycle, params ConfigParams) (*Config, error) {
|
|||||||
"hasMetricsAuth", s.MetricsAuthEnabled(),
|
"hasMetricsAuth", s.MetricsAuthEnabled(),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
s.warnSharedRateLimitBucket(log)
|
||||||
s.warnEgressAllowlist(log)
|
s.warnEgressAllowlist(log)
|
||||||
|
|
||||||
return s, nil
|
return s, nil
|
||||||
|
|||||||
+160
-130
@@ -3,6 +3,7 @@ package config_test
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
|
"os"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -11,7 +12,6 @@ import (
|
|||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"go.uber.org/fx/fxtest"
|
"go.uber.org/fx/fxtest"
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/config/configtest"
|
|
||||||
"sneak.berlin/go/webhooker/internal/globals"
|
"sneak.berlin/go/webhooker/internal/globals"
|
||||||
"sneak.berlin/go/webhooker/internal/logger"
|
"sneak.berlin/go/webhooker/internal/logger"
|
||||||
)
|
)
|
||||||
@@ -44,9 +44,9 @@ func TestEnvironmentConfig(t *testing.T) {
|
|||||||
isProd bool
|
isProd bool
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "default is prod",
|
name: "default is dev",
|
||||||
isDev: false,
|
isDev: true,
|
||||||
isProd: true,
|
isProd: false,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "explicit dev",
|
name: "explicit dev",
|
||||||
@@ -71,12 +71,14 @@ func TestEnvironmentConfig(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
configtest.ClearEnv(t)
|
|
||||||
|
|
||||||
if tt.envValue != "" {
|
if tt.envValue != "" {
|
||||||
t.Setenv(
|
t.Setenv(
|
||||||
"WEBHOOKER_ENVIRONMENT", tt.envValue,
|
"WEBHOOKER_ENVIRONMENT", tt.envValue,
|
||||||
)
|
)
|
||||||
|
} else {
|
||||||
|
require.NoError(t, os.Unsetenv(
|
||||||
|
"WEBHOOKER_ENVIRONMENT",
|
||||||
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
for k, v := range tt.envVars {
|
for k, v := range tt.envVars {
|
||||||
@@ -122,11 +124,6 @@ func testEnvironmentConfigSuccess(
|
|||||||
|
|
||||||
app := fxtest.New(
|
app := fxtest.New(
|
||||||
t,
|
t,
|
||||||
// fx's own log is discarded, not sent to t.Logf: a hook still
|
|
||||||
// running after a start or stop timeout would write there after
|
|
||||||
// the test has returned. The same holds for every fxtest.New
|
|
||||||
// below.
|
|
||||||
fx.NopLogger,
|
|
||||||
fx.Provide(
|
fx.Provide(
|
||||||
globals.New,
|
globals.New,
|
||||||
logger.New,
|
logger.New,
|
||||||
@@ -197,11 +194,14 @@ func TestRetentionSweepInterval(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
configtest.ClearEnv(t)
|
|
||||||
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
||||||
|
|
||||||
if tt.set {
|
if tt.set {
|
||||||
t.Setenv("RETENTION_SWEEP_INTERVAL", tt.value)
|
t.Setenv("RETENTION_SWEEP_INTERVAL", tt.value)
|
||||||
|
} else {
|
||||||
|
require.NoError(t, os.Unsetenv(
|
||||||
|
"RETENTION_SWEEP_INTERVAL",
|
||||||
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
if tt.expectError {
|
if tt.expectError {
|
||||||
@@ -272,7 +272,6 @@ func testRetentionSweepIntervalSuccess(
|
|||||||
|
|
||||||
app := fxtest.New(
|
app := fxtest.New(
|
||||||
t,
|
t,
|
||||||
fx.NopLogger,
|
|
||||||
fx.Provide(
|
fx.Provide(
|
||||||
globals.New,
|
globals.New,
|
||||||
logger.New,
|
logger.New,
|
||||||
@@ -336,11 +335,14 @@ func TestSessionIdleTimeout(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
configtest.ClearEnv(t)
|
|
||||||
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
||||||
|
|
||||||
if tt.set {
|
if tt.set {
|
||||||
t.Setenv("SESSION_IDLE_TIMEOUT", tt.value)
|
t.Setenv("SESSION_IDLE_TIMEOUT", tt.value)
|
||||||
|
} else {
|
||||||
|
require.NoError(t, os.Unsetenv(
|
||||||
|
"SESSION_IDLE_TIMEOUT",
|
||||||
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
if tt.expectError {
|
if tt.expectError {
|
||||||
@@ -362,7 +364,6 @@ func testSessionIdleTimeoutSuccess(
|
|||||||
|
|
||||||
app := fxtest.New(
|
app := fxtest.New(
|
||||||
t,
|
t,
|
||||||
fx.NopLogger,
|
|
||||||
fx.Provide(
|
fx.Provide(
|
||||||
globals.New,
|
globals.New,
|
||||||
logger.New,
|
logger.New,
|
||||||
@@ -379,83 +380,6 @@ func testSessionIdleTimeoutSuccess(
|
|||||||
assert.Equal(t, expected, cfg.SessionIdleTimeout)
|
assert.Equal(t, expected, cfg.SessionIdleTimeout)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHSTSMaxAge(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
set bool
|
|
||||||
value string
|
|
||||||
expectError bool
|
|
||||||
// sentinel, when set, must be wrapped by the startup
|
|
||||||
// error; every error case must additionally name the
|
|
||||||
// variable in its message.
|
|
||||||
sentinel error
|
|
||||||
expected time.Duration
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: caseUnsetUsesDefault,
|
|
||||||
set: false,
|
|
||||||
expected: 87600 * time.Hour,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: caseValidValueParsed,
|
|
||||||
set: true,
|
|
||||||
value: "720h",
|
|
||||||
expected: 720 * time.Hour,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
// Zero turns the header off; it is not an error.
|
|
||||||
name: "zero is accepted",
|
|
||||||
set: true,
|
|
||||||
value: "0",
|
|
||||||
expected: 0,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: caseUnparseableFails,
|
|
||||||
set: true,
|
|
||||||
value: "10 years",
|
|
||||||
expectError: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
// A whole number of seconds, so only the sign is wrong.
|
|
||||||
name: "negative whole seconds fail startup",
|
|
||||||
set: true,
|
|
||||||
value: "-1s",
|
|
||||||
expectError: true,
|
|
||||||
sentinel: config.ErrInvalidHSTSMaxAge,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "fraction of a second fails startup",
|
|
||||||
set: true,
|
|
||||||
value: "1500ms",
|
|
||||||
expectError: true,
|
|
||||||
sentinel: config.ErrInvalidHSTSMaxAge,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
|
||||||
// is incompatible with parallel subtests.
|
|
||||||
configtest.ClearEnv(t)
|
|
||||||
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
|
||||||
|
|
||||||
if tt.set {
|
|
||||||
t.Setenv("HSTS_MAX_AGE", tt.value)
|
|
||||||
}
|
|
||||||
|
|
||||||
if tt.expectError {
|
|
||||||
expectStartupErrorFor(t, "HSTS_MAX_AGE", tt.sentinel)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg, err := buildConfig(t)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, tt.expected, cfg.HSTSMaxAge)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDefaultDataDir(t *testing.T) {
|
func TestDefaultDataDir(t *testing.T) {
|
||||||
for _, env := range []string{"", "dev", "prod"} {
|
for _, env := range []string{"", "dev", "prod"} {
|
||||||
name := env
|
name := env
|
||||||
@@ -466,17 +390,20 @@ func TestDefaultDataDir(t *testing.T) {
|
|||||||
t.Run("env="+name, func(t *testing.T) {
|
t.Run("env="+name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
configtest.ClearEnv(t)
|
|
||||||
|
|
||||||
if env != "" {
|
if env != "" {
|
||||||
t.Setenv("WEBHOOKER_ENVIRONMENT", env)
|
t.Setenv("WEBHOOKER_ENVIRONMENT", env)
|
||||||
|
} else {
|
||||||
|
require.NoError(t, os.Unsetenv(
|
||||||
|
"WEBHOOKER_ENVIRONMENT",
|
||||||
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
require.NoError(t, os.Unsetenv("DATA_DIR"))
|
||||||
|
|
||||||
var cfg *config.Config
|
var cfg *config.Config
|
||||||
|
|
||||||
app := fxtest.New(
|
app := fxtest.New(
|
||||||
t,
|
t,
|
||||||
fx.NopLogger,
|
|
||||||
fx.Provide(
|
fx.Provide(
|
||||||
globals.New,
|
globals.New,
|
||||||
logger.New,
|
logger.New,
|
||||||
@@ -511,9 +438,9 @@ func TestDataDirHelper(t *testing.T) {
|
|||||||
t.Run(name, func(t *testing.T) {
|
t.Run(name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
configtest.ClearEnv(t)
|
if set == "" {
|
||||||
|
require.NoError(t, os.Unsetenv("DATA_DIR"))
|
||||||
if set != "" {
|
} else {
|
||||||
t.Setenv("DATA_DIR", set)
|
t.Setenv("DATA_DIR", set)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -576,11 +503,14 @@ func TestReceiverRateLimit(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
configtest.ClearEnv(t)
|
|
||||||
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
||||||
|
|
||||||
if tt.set {
|
if tt.set {
|
||||||
t.Setenv("RECEIVER_RATE_LIMIT", tt.value)
|
t.Setenv("RECEIVER_RATE_LIMIT", tt.value)
|
||||||
|
} else {
|
||||||
|
require.NoError(t, os.Unsetenv(
|
||||||
|
"RECEIVER_RATE_LIMIT",
|
||||||
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
if tt.expectError {
|
if tt.expectError {
|
||||||
@@ -604,7 +534,6 @@ func testReceiverRateLimitSuccess(
|
|||||||
|
|
||||||
app := fxtest.New(
|
app := fxtest.New(
|
||||||
t,
|
t,
|
||||||
fx.NopLogger,
|
|
||||||
fx.Provide(
|
fx.Provide(
|
||||||
globals.New,
|
globals.New,
|
||||||
logger.New,
|
logger.New,
|
||||||
@@ -622,11 +551,6 @@ func testReceiverRateLimitSuccess(
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestTrustedProxies(t *testing.T) {
|
func TestTrustedProxies(t *testing.T) {
|
||||||
// Unset, the RFC 1918 private ranges are trusted, so a reverse
|
|
||||||
// proxy on a Docker network or a private LAN is covered without
|
|
||||||
// configuration.
|
|
||||||
defaultProxies := []string{cidrPrivateV4, "172.16.0.0/12", "192.168.0.0/16"}
|
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
set bool
|
set bool
|
||||||
@@ -635,21 +559,18 @@ func TestTrustedProxies(t *testing.T) {
|
|||||||
expected []string
|
expected []string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
|
// The default must be "trust nobody": an empty list
|
||||||
|
// means forwarded headers are ignored, never that
|
||||||
|
// every peer may speak for the client.
|
||||||
name: caseUnsetUsesDefault,
|
name: caseUnsetUsesDefault,
|
||||||
set: false,
|
set: false,
|
||||||
expected: defaultProxies,
|
expected: []string{},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "blank value uses default",
|
name: "blank value trusts nothing",
|
||||||
set: true,
|
set: true,
|
||||||
value: " ",
|
value: " ",
|
||||||
expected: defaultProxies,
|
expected: []string{},
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "set value replaces the default entirely",
|
|
||||||
set: true,
|
|
||||||
value: "203.0.113.7",
|
|
||||||
expected: []string{"203.0.113.7/32"},
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: caseValidValueParsed,
|
name: caseValidValueParsed,
|
||||||
@@ -692,11 +613,12 @@ func TestTrustedProxies(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
configtest.ClearEnv(t)
|
|
||||||
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
||||||
|
|
||||||
if tt.set {
|
if tt.set {
|
||||||
t.Setenv("TRUSTED_PROXIES", tt.value)
|
t.Setenv("TRUSTED_PROXIES", tt.value)
|
||||||
|
} else {
|
||||||
|
require.NoError(t, os.Unsetenv("TRUSTED_PROXIES"))
|
||||||
}
|
}
|
||||||
|
|
||||||
if tt.expectError {
|
if tt.expectError {
|
||||||
@@ -720,7 +642,6 @@ func testTrustedProxiesSuccess(
|
|||||||
|
|
||||||
app := fxtest.New(
|
app := fxtest.New(
|
||||||
t,
|
t,
|
||||||
fx.NopLogger,
|
|
||||||
fx.Provide(
|
fx.Provide(
|
||||||
globals.New,
|
globals.New,
|
||||||
logger.New,
|
logger.New,
|
||||||
@@ -803,11 +724,14 @@ func TestAllowedEgressCIDRs(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
configtest.ClearEnv(t)
|
|
||||||
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
||||||
|
|
||||||
if tt.set {
|
if tt.set {
|
||||||
t.Setenv("ALLOWED_EGRESS_CIDRS", tt.value)
|
t.Setenv("ALLOWED_EGRESS_CIDRS", tt.value)
|
||||||
|
} else {
|
||||||
|
require.NoError(
|
||||||
|
t, os.Unsetenv("ALLOWED_EGRESS_CIDRS"),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
if tt.expectError {
|
if tt.expectError {
|
||||||
@@ -831,7 +755,6 @@ func testAllowedEgressCIDRsSuccess(
|
|||||||
|
|
||||||
app := fxtest.New(
|
app := fxtest.New(
|
||||||
t,
|
t,
|
||||||
fx.NopLogger,
|
|
||||||
fx.Provide(
|
fx.Provide(
|
||||||
globals.New,
|
globals.New,
|
||||||
logger.New,
|
logger.New,
|
||||||
@@ -875,10 +798,13 @@ func TestEgressAllowlistWarning(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
configtest.ClearEnv(t)
|
|
||||||
t.Setenv("WEBHOOKER_ENVIRONMENT", config.EnvironmentDev)
|
t.Setenv("WEBHOOKER_ENVIRONMENT", config.EnvironmentDev)
|
||||||
|
|
||||||
if tt.allowed != "" {
|
if tt.allowed == "" {
|
||||||
|
require.NoError(
|
||||||
|
t, os.Unsetenv("ALLOWED_EGRESS_CIDRS"),
|
||||||
|
)
|
||||||
|
} else {
|
||||||
t.Setenv("ALLOWED_EGRESS_CIDRS", tt.allowed)
|
t.Setenv("ALLOWED_EGRESS_CIDRS", tt.allowed)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -908,13 +834,112 @@ func TestEgressAllowlistWarning(t *testing.T) {
|
|||||||
// to be able to read back which networks are open.
|
// to be able to read back which networks are open.
|
||||||
assert.Contains(t, logged, "10.0.0.0/8")
|
assert.Contains(t, logged, "10.0.0.0/8")
|
||||||
assert.Contains(t, logged, "127.0.0.0/8")
|
assert.Contains(t, logged, "127.0.0.0/8")
|
||||||
// What stays shut is the whole unconditional set, not
|
// What stays shut. Asserted on the clause naming the
|
||||||
// link-local alone; a public metadata address is not in
|
// wider set rather than on "Link-local" alone, so the
|
||||||
// it, so a listed block covering it opens it.
|
// string cannot narrow back to link-local only while
|
||||||
assert.Contains(t, logged, "blocked unconditionally")
|
// the always-blocked set covers ULA, CGNAT and two
|
||||||
assert.Contains(t, logged, "168.63.129.16 is reachable")
|
// public metadata addresses as well.
|
||||||
// The listed blocks need not be private or reserved.
|
assert.Contains(t, logged, "metadata endpoints outside it")
|
||||||
assert.NotContains(t, logged, "private/reserved")
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSharedRateLimitBucketWarning covers the startup warning that
|
||||||
|
// tells an operator a deployment behind a reverse proxy shares one
|
||||||
|
// rate-limit bucket between every client, which turns the receiver
|
||||||
|
// limits into service-wide ceilings and collapses login failure
|
||||||
|
// counting. It must fire whenever TRUSTED_PROXIES is empty,
|
||||||
|
// in any environment: WEBHOOKER_ENVIRONMENT defaults to dev, so gating
|
||||||
|
// on it would silence the warning for exactly the operator who never
|
||||||
|
// configured the deployment. It stays quiet once proxies are named.
|
||||||
|
func TestSharedRateLimitBucketWarning(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
environment string
|
||||||
|
trustedProxies string
|
||||||
|
expectWarning bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "prod without trusted proxies warns",
|
||||||
|
environment: config.EnvironmentProd,
|
||||||
|
expectWarning: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "prod with trusted proxies is quiet",
|
||||||
|
environment: config.EnvironmentProd,
|
||||||
|
trustedProxies: cidrPrivateV4,
|
||||||
|
expectWarning: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// The default environment. An internet-exposed
|
||||||
|
// deployment whose operator never set
|
||||||
|
// WEBHOOKER_ENVIRONMENT lands here and has exactly
|
||||||
|
// the exposure the warning announces.
|
||||||
|
name: "dev without trusted proxies warns",
|
||||||
|
environment: config.EnvironmentDev,
|
||||||
|
expectWarning: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "dev with trusted proxies is quiet",
|
||||||
|
environment: config.EnvironmentDev,
|
||||||
|
trustedProxies: cidrPrivateV4,
|
||||||
|
expectWarning: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
|
// is incompatible with parallel subtests.
|
||||||
|
t.Setenv("WEBHOOKER_ENVIRONMENT", tt.environment)
|
||||||
|
|
||||||
|
if tt.trustedProxies == "" {
|
||||||
|
require.NoError(
|
||||||
|
t, os.Unsetenv("TRUSTED_PROXIES"),
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
t.Setenv("TRUSTED_PROXIES", tt.trustedProxies)
|
||||||
|
}
|
||||||
|
|
||||||
|
var buf bytes.Buffer
|
||||||
|
|
||||||
|
log := slog.New(slog.NewJSONHandler(
|
||||||
|
&buf, &slog.HandlerOptions{
|
||||||
|
Level: slog.LevelDebug,
|
||||||
|
},
|
||||||
|
))
|
||||||
|
|
||||||
|
require.NoError(
|
||||||
|
t,
|
||||||
|
config.WarnSharedRateLimitBucketForTest(log),
|
||||||
|
)
|
||||||
|
|
||||||
|
if !tt.expectWarning {
|
||||||
|
assert.Empty(t, buf.String())
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
logged := buf.String()
|
||||||
|
|
||||||
|
assert.Contains(t, logged, `"level":"WARN"`)
|
||||||
|
assert.Contains(t, logged, "TRUSTED_PROXIES")
|
||||||
|
assert.Contains(t, logged, "share one bucket")
|
||||||
|
assert.Contains(
|
||||||
|
t, logged, "throttle every other client's failed logins",
|
||||||
|
)
|
||||||
|
// The warning must not claim a lockout the login
|
||||||
|
// endpoint no longer permits: credentials are verified
|
||||||
|
// before any budget is spent.
|
||||||
|
assert.Contains(
|
||||||
|
t, logged, "a correct password still gets in",
|
||||||
|
)
|
||||||
|
// The text must stay accurate for a developer with
|
||||||
|
// nothing in front of the process, where an empty
|
||||||
|
// list costs nothing.
|
||||||
|
assert.Contains(
|
||||||
|
t, logged, "nothing proxying to this process",
|
||||||
|
)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1011,14 +1036,20 @@ func TestMetricsAuthConfig(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
configtest.ClearEnv(t)
|
|
||||||
|
|
||||||
if tt.username.set {
|
if tt.username.set {
|
||||||
t.Setenv("METRICS_USERNAME", tt.username.value)
|
t.Setenv("METRICS_USERNAME", tt.username.value)
|
||||||
|
} else {
|
||||||
|
require.NoError(
|
||||||
|
t, os.Unsetenv("METRICS_USERNAME"),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
if tt.password.set {
|
if tt.password.set {
|
||||||
t.Setenv("METRICS_PASSWORD", tt.password.value)
|
t.Setenv("METRICS_PASSWORD", tt.password.value)
|
||||||
|
} else {
|
||||||
|
require.NoError(
|
||||||
|
t, os.Unsetenv("METRICS_PASSWORD"),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
if tt.expectError {
|
if tt.expectError {
|
||||||
@@ -1066,7 +1097,6 @@ func assertMetricsAuthAccepted(t *testing.T, expectAuth bool) {
|
|||||||
|
|
||||||
app := fxtest.New(
|
app := fxtest.New(
|
||||||
t,
|
t,
|
||||||
fx.NopLogger,
|
|
||||||
fx.Provide(globals.New, logger.New, config.New),
|
fx.Provide(globals.New, logger.New, config.New),
|
||||||
fx.Populate(&cfg),
|
fx.Populate(&cfg),
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,52 +0,0 @@
|
|||||||
// Package configtest holds test support for code that reads the
|
|
||||||
// process environment.
|
|
||||||
package configtest
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ClearEnv unsets every variable in the process environment
|
|
||||||
// for the rest of the test, so a test sees only the variables it sets
|
|
||||||
// itself, not whatever the developer's shell exports. When the test
|
|
||||||
// ends it leaves the environment exactly as it found it: each variable
|
|
||||||
// it unset is put back, and any variable added since is removed.
|
|
||||||
func ClearEnv(t *testing.T) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
present := make(map[string]bool)
|
|
||||||
|
|
||||||
for _, entry := range os.Environ() {
|
|
||||||
key, _, _ := strings.Cut(entry, "=")
|
|
||||||
present[key] = true
|
|
||||||
|
|
||||||
// t.Setenv registers the restore; the Unsetenv after it is
|
|
||||||
// what makes the key absent, since a key set to the empty
|
|
||||||
// string is still present, and godotenv will not overwrite a
|
|
||||||
// present key.
|
|
||||||
t.Setenv(key, "")
|
|
||||||
|
|
||||||
err := os.Unsetenv(key)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unsetting %s: %v", key, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A variable the test adds other than through t.Setenv, as loading
|
|
||||||
// a .env file does, has no restore of its own.
|
|
||||||
t.Cleanup(func() {
|
|
||||||
for _, entry := range os.Environ() {
|
|
||||||
key, _, _ := strings.Cut(entry, "=")
|
|
||||||
if present[key] {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
err := os.Unsetenv(key)
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("unsetting %s: %v", key, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
@@ -8,7 +8,6 @@ import (
|
|||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/config/configtest"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// dotEnvKey is a throwaway variable name the .env tests write and
|
// dotEnvKey is a throwaway variable name the .env tests write and
|
||||||
@@ -23,6 +22,17 @@ const malformedDotEnv = "PORT 19615\n" +
|
|||||||
"this is not = valid ! syntax\n" +
|
"this is not = valid ! syntax\n" +
|
||||||
"\"unclosed\n"
|
"\"unclosed\n"
|
||||||
|
|
||||||
|
// unsetDotEnvKey makes dotEnvKey genuinely absent for the duration of
|
||||||
|
// the test and restores it afterwards. t.Setenv registers the restore;
|
||||||
|
// the Unsetenv that follows is what the test actually needs, because a
|
||||||
|
// variable set to the empty string is still present in os.Environ and
|
||||||
|
// godotenv would refuse to overwrite it.
|
||||||
|
func unsetDotEnvKey(t *testing.T) {
|
||||||
|
t.Helper()
|
||||||
|
t.Setenv(dotEnvKey, "placeholder")
|
||||||
|
require.NoError(t, os.Unsetenv(dotEnvKey))
|
||||||
|
}
|
||||||
|
|
||||||
// writeDotEnv writes contents to a .env file in a fresh temporary
|
// writeDotEnv writes contents to a .env file in a fresh temporary
|
||||||
// directory and returns its path.
|
// directory and returns its path.
|
||||||
func writeDotEnv(t *testing.T, contents string) string {
|
func writeDotEnv(t *testing.T, contents string) string {
|
||||||
@@ -40,9 +50,9 @@ func writeDotEnv(t *testing.T, contents string) string {
|
|||||||
// normally rather than be refused for a file it was never meant to
|
// normally rather than be refused for a file it was never meant to
|
||||||
// have.
|
// have.
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // ClearEnv uses t.Setenv.
|
//nolint:paralleltest // unsetDotEnvKey uses t.Setenv.
|
||||||
func TestLoadDotEnv_MissingFileIsFine(t *testing.T) {
|
func TestLoadDotEnv_MissingFileIsFine(t *testing.T) {
|
||||||
configtest.ClearEnv(t)
|
unsetDotEnvKey(t)
|
||||||
|
|
||||||
absent := filepath.Join(t.TempDir(), config.DotEnvPath)
|
absent := filepath.Join(t.TempDir(), config.DotEnvPath)
|
||||||
require.NoError(t, config.LoadDotEnvFileForTest(absent))
|
require.NoError(t, config.LoadDotEnvFileForTest(absent))
|
||||||
@@ -55,9 +65,9 @@ func TestLoadDotEnv_MissingFileIsFine(t *testing.T) {
|
|||||||
// reaches the environment, which is the whole reason the file is read
|
// reaches the environment, which is the whole reason the file is read
|
||||||
// at all.
|
// at all.
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // ClearEnv uses t.Setenv.
|
//nolint:paralleltest // unsetDotEnvKey uses t.Setenv.
|
||||||
func TestLoadDotEnv_AppliesValues(t *testing.T) {
|
func TestLoadDotEnv_AppliesValues(t *testing.T) {
|
||||||
configtest.ClearEnv(t)
|
unsetDotEnvKey(t)
|
||||||
|
|
||||||
path := writeDotEnv(t, "# a comment\n"+dotEnvKey+"=from-dot-env\n")
|
path := writeDotEnv(t, "# a comment\n"+dotEnvKey+"=from-dot-env\n")
|
||||||
|
|
||||||
@@ -83,9 +93,9 @@ func TestLoadDotEnv_RealEnvironmentWins(t *testing.T) {
|
|||||||
// reverts to its default; the process used to start that way with no
|
// reverts to its default; the process used to start that way with no
|
||||||
// log line naming the file at all.
|
// log line naming the file at all.
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // ClearEnv uses t.Setenv.
|
//nolint:paralleltest // unsetDotEnvKey uses t.Setenv.
|
||||||
func TestLoadDotEnv_MalformedFileAborts(t *testing.T) {
|
func TestLoadDotEnv_MalformedFileAborts(t *testing.T) {
|
||||||
configtest.ClearEnv(t)
|
unsetDotEnvKey(t)
|
||||||
|
|
||||||
path := writeDotEnv(
|
path := writeDotEnv(
|
||||||
t, malformedDotEnv+dotEnvKey+"=from-dot-env\n",
|
t, malformedDotEnv+dotEnvKey+"=from-dot-env\n",
|
||||||
@@ -133,7 +143,7 @@ func TestLoadDotEnv_UnreadableFileAborts(t *testing.T) {
|
|||||||
//
|
//
|
||||||
//nolint:paralleltest // t.Chdir moves the whole process.
|
//nolint:paralleltest // t.Chdir moves the whole process.
|
||||||
func TestLoadDotEnv_ReadsTheWorkingDirectory(t *testing.T) {
|
func TestLoadDotEnv_ReadsTheWorkingDirectory(t *testing.T) {
|
||||||
configtest.ClearEnv(t)
|
unsetDotEnvKey(t)
|
||||||
|
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
require.NoError(t, os.WriteFile(
|
require.NoError(t, os.WriteFile(
|
||||||
|
|||||||
+103
-82
@@ -1,13 +1,13 @@
|
|||||||
package config_test
|
package config_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"os"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/config/configtest"
|
|
||||||
"sneak.berlin/go/webhooker/internal/globals"
|
"sneak.berlin/go/webhooker/internal/globals"
|
||||||
"sneak.berlin/go/webhooker/internal/logger"
|
"sneak.berlin/go/webhooker/internal/logger"
|
||||||
)
|
)
|
||||||
@@ -18,9 +18,10 @@ const testEnvKey = "WEBHOOKER_TEST_VALUE"
|
|||||||
|
|
||||||
// Real configuration variables exercised by the config.New tests.
|
// Real configuration variables exercised by the config.New tests.
|
||||||
const (
|
const (
|
||||||
envKeyPort = "PORT"
|
envKeyPort = "PORT"
|
||||||
envKeyDebug = "DEBUG"
|
envKeyDebug = "DEBUG"
|
||||||
envKeyBindAddress = "BIND_ADDRESS"
|
envKeyMaintenanceMode = "MAINTENANCE_MODE"
|
||||||
|
envKeyBindAddress = "BIND_ADDRESS"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Sample BIND_ADDRESS values used by the tables below.
|
// Sample BIND_ADDRESS values used by the tables below.
|
||||||
@@ -121,10 +122,10 @@ func TestEnvBool(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
configtest.ClearEnv(t)
|
|
||||||
|
|
||||||
if tt.set {
|
if tt.set {
|
||||||
t.Setenv(testEnvKey, tt.value)
|
t.Setenv(testEnvKey, tt.value)
|
||||||
|
} else {
|
||||||
|
require.NoError(t, os.Unsetenv(testEnvKey))
|
||||||
}
|
}
|
||||||
|
|
||||||
got, err := config.EnvBoolForTest(
|
got, err := config.EnvBoolForTest(
|
||||||
@@ -145,62 +146,17 @@ func TestEnvBool(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// envIntCase is one row of the envPositiveInt and envPort tables.
|
|
||||||
type envIntCase struct {
|
|
||||||
name string
|
|
||||||
set bool
|
|
||||||
value string
|
|
||||||
expectError bool
|
|
||||||
errIs error
|
|
||||||
expected int
|
|
||||||
}
|
|
||||||
|
|
||||||
// runEnvIntCases runs each row through parse, which is
|
|
||||||
// envPositiveInt or envPort, with testEnvKey set to the row's value
|
|
||||||
// or left unset.
|
|
||||||
func runEnvIntCases(
|
|
||||||
t *testing.T,
|
|
||||||
parse func(key string, defaultValue int) (int, error),
|
|
||||||
defaultValue int,
|
|
||||||
tests []envIntCase,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
|
||||||
// is incompatible with parallel subtests.
|
|
||||||
configtest.ClearEnv(t)
|
|
||||||
|
|
||||||
if tt.set {
|
|
||||||
t.Setenv(testEnvKey, tt.value)
|
|
||||||
}
|
|
||||||
|
|
||||||
got, err := parse(testEnvKey, defaultValue)
|
|
||||||
|
|
||||||
if tt.expectError {
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.Contains(t, err.Error(), testEnvKey)
|
|
||||||
assert.Contains(t, err.Error(), tt.value)
|
|
||||||
|
|
||||||
if tt.errIs != nil {
|
|
||||||
require.ErrorIs(t, err, tt.errIs)
|
|
||||||
}
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, tt.expected, got)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
//nolint:paralleltest // runEnvIntCases uses t.Setenv.
|
|
||||||
func TestEnvPositiveInt(t *testing.T) {
|
func TestEnvPositiveInt(t *testing.T) {
|
||||||
const defaultValue = 7
|
const defaultValue = 7
|
||||||
|
|
||||||
runEnvIntCases(t, config.EnvPositiveIntForTest, defaultValue, []envIntCase{
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
set bool
|
||||||
|
value string
|
||||||
|
expectError bool
|
||||||
|
errIs error
|
||||||
|
expected int
|
||||||
|
}{
|
||||||
{
|
{
|
||||||
name: "unset returns the default integer",
|
name: "unset returns the default integer",
|
||||||
expected: defaultValue,
|
expected: defaultValue,
|
||||||
@@ -237,14 +193,51 @@ func TestEnvPositiveInt(t *testing.T) {
|
|||||||
expectError: true,
|
expectError: true,
|
||||||
errIs: config.ErrNonPositiveValue,
|
errIs: config.ErrNonPositiveValue,
|
||||||
},
|
},
|
||||||
})
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
|
// is incompatible with parallel subtests.
|
||||||
|
if tt.set {
|
||||||
|
t.Setenv(testEnvKey, tt.value)
|
||||||
|
} else {
|
||||||
|
require.NoError(t, os.Unsetenv(testEnvKey))
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err := config.EnvPositiveIntForTest(
|
||||||
|
testEnvKey, defaultValue,
|
||||||
|
)
|
||||||
|
|
||||||
|
if tt.expectError {
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Contains(t, err.Error(), testEnvKey)
|
||||||
|
assert.Contains(t, err.Error(), tt.value)
|
||||||
|
|
||||||
|
if tt.errIs != nil {
|
||||||
|
require.ErrorIs(t, err, tt.errIs)
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, tt.expected, got)
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
//nolint:paralleltest // runEnvIntCases uses t.Setenv.
|
|
||||||
func TestEnvPort(t *testing.T) {
|
func TestEnvPort(t *testing.T) {
|
||||||
const defaultValue = 8080
|
const defaultValue = 8080
|
||||||
|
|
||||||
runEnvIntCases(t, config.EnvPortForTest, defaultValue, []envIntCase{
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
set bool
|
||||||
|
value string
|
||||||
|
expectError bool
|
||||||
|
errIs error
|
||||||
|
expected int
|
||||||
|
}{
|
||||||
{
|
{
|
||||||
name: "unset returns the default port",
|
name: "unset returns the default port",
|
||||||
expected: defaultValue,
|
expected: defaultValue,
|
||||||
@@ -272,14 +265,7 @@ func TestEnvPort(t *testing.T) {
|
|||||||
set: true,
|
set: true,
|
||||||
value: "0",
|
value: "0",
|
||||||
expectError: true,
|
expectError: true,
|
||||||
errIs: config.ErrInvalidPort,
|
errIs: config.ErrNonPositiveValue,
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "negative is rejected",
|
|
||||||
set: true,
|
|
||||||
value: "-1",
|
|
||||||
expectError: true,
|
|
||||||
errIs: config.ErrInvalidPort,
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "above the port range is rejected",
|
name: "above the port range is rejected",
|
||||||
@@ -288,14 +274,37 @@ func TestEnvPort(t *testing.T) {
|
|||||||
expectError: true,
|
expectError: true,
|
||||||
errIs: config.ErrInvalidPort,
|
errIs: config.ErrInvalidPort,
|
||||||
},
|
},
|
||||||
{
|
}
|
||||||
name: "too large for an int is rejected",
|
|
||||||
set: true,
|
for _, tt := range tests {
|
||||||
value: "99999999999999999999",
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
expectError: true,
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
errIs: config.ErrInvalidPort,
|
// is incompatible with parallel subtests.
|
||||||
},
|
if tt.set {
|
||||||
})
|
t.Setenv(testEnvKey, tt.value)
|
||||||
|
} else {
|
||||||
|
require.NoError(t, os.Unsetenv(testEnvKey))
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err := config.EnvPortForTest(
|
||||||
|
testEnvKey, defaultValue,
|
||||||
|
)
|
||||||
|
|
||||||
|
if tt.expectError {
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Contains(t, err.Error(), testEnvKey)
|
||||||
|
|
||||||
|
if tt.errIs != nil {
|
||||||
|
require.ErrorIs(t, err, tt.errIs)
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, tt.expected, got)
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestEnvBindAddress covers BIND_ADDRESS parsing.
|
// TestEnvBindAddress covers BIND_ADDRESS parsing.
|
||||||
@@ -311,10 +320,10 @@ func TestEnvBindAddress(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
configtest.ClearEnv(t)
|
|
||||||
|
|
||||||
if tt.set {
|
if tt.set {
|
||||||
t.Setenv(testEnvKey, tt.value)
|
t.Setenv(testEnvKey, tt.value)
|
||||||
|
} else {
|
||||||
|
require.NoError(t, os.Unsetenv(testEnvKey))
|
||||||
}
|
}
|
||||||
|
|
||||||
got, err := config.EnvBindAddressForTest(
|
got, err := config.EnvBindAddressForTest(
|
||||||
@@ -477,7 +486,6 @@ func TestNewRejectsBadEnvValues(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
configtest.ClearEnv(t)
|
|
||||||
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
||||||
t.Setenv(tt.key, tt.value)
|
t.Setenv(tt.key, tt.value)
|
||||||
|
|
||||||
@@ -596,6 +604,12 @@ func flagEnvValueCases() []badEnvValueCase {
|
|||||||
value: "ture",
|
value: "ture",
|
||||||
expectError: true,
|
expectError: true,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "unparseable MAINTENANCE_MODE aborts startup",
|
||||||
|
key: envKeyMaintenanceMode,
|
||||||
|
value: "sometimes",
|
||||||
|
expectError: true,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -639,15 +653,22 @@ func sentryEnvValueCases() []badEnvValueCase {
|
|||||||
// break the legitimate unset case: absent variables still get their
|
// break the legitimate unset case: absent variables still get their
|
||||||
// documented defaults.
|
// documented defaults.
|
||||||
func TestNewUsesDefaultsWhenUnset(t *testing.T) {
|
func TestNewUsesDefaultsWhenUnset(t *testing.T) {
|
||||||
configtest.ClearEnv(t)
|
|
||||||
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
||||||
|
|
||||||
|
for _, key := range []string{
|
||||||
|
envKeyPort, envKeyDebug, envKeyMaintenanceMode,
|
||||||
|
envKeyBindAddress, envKeySentryDSN,
|
||||||
|
} {
|
||||||
|
require.NoError(t, os.Unsetenv(key))
|
||||||
|
}
|
||||||
|
|
||||||
cfg, err := buildConfig(t)
|
cfg, err := buildConfig(t)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.NotNil(t, cfg)
|
require.NotNil(t, cfg)
|
||||||
|
|
||||||
assert.Equal(t, 8080, cfg.Port)
|
assert.Equal(t, 8080, cfg.Port)
|
||||||
assert.False(t, cfg.Debug)
|
assert.False(t, cfg.Debug)
|
||||||
|
assert.False(t, cfg.MaintenanceMode)
|
||||||
|
|
||||||
// Loopback, not the wildcard: the default must not publish the
|
// Loopback, not the wildcard: the default must not publish the
|
||||||
// cleartext admin UI and the unauthenticated receiver on every
|
// cleartext admin UI and the unauthenticated receiver on every
|
||||||
|
|||||||
@@ -6,6 +6,21 @@ import "log/slog"
|
|||||||
// the external config_test package so each helper can be covered by
|
// the external config_test package so each helper can be covered by
|
||||||
// its own table-driven test without weakening the package API.
|
// its own table-driven test without weakening the package API.
|
||||||
|
|
||||||
|
// WarnSharedRateLimitBucketForTest loads a Config from the current
|
||||||
|
// environment and emits its startup warnings to log. The real logger
|
||||||
|
// writes to stdout, so this lets the warning's firing condition be
|
||||||
|
// asserted against a handler the test controls.
|
||||||
|
func WarnSharedRateLimitBucketForTest(log *slog.Logger) error {
|
||||||
|
c, err := loadFromEnv()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
c.warnSharedRateLimitBucket(log)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// WarnEgressAllowlistForTest loads a Config from the current
|
// WarnEgressAllowlistForTest loads a Config from the current
|
||||||
// environment and emits its egress-allowlist startup warning to
|
// environment and emits its egress-allowlist startup warning to
|
||||||
// log, so a test can assert both that the warning fires only when
|
// log, so a test can assert both that the warning fires only when
|
||||||
|
|||||||
@@ -1,12 +1,12 @@
|
|||||||
package config_test
|
package config_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"os"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/config/configtest"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// envKeySentryDSN is the variable envSentryDSN reads in production.
|
// envKeySentryDSN is the variable envSentryDSN reads in production.
|
||||||
@@ -101,10 +101,10 @@ func TestEnvSentryDSN(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
configtest.ClearEnv(t)
|
|
||||||
|
|
||||||
if tt.set {
|
if tt.set {
|
||||||
t.Setenv(envKeySentryDSN, tt.value)
|
t.Setenv(envKeySentryDSN, tt.value)
|
||||||
|
} else {
|
||||||
|
require.NoError(t, os.Unsetenv(envKeySentryDSN))
|
||||||
}
|
}
|
||||||
|
|
||||||
got, err := config.EnvSentryDSNForTest(envKeySentryDSN)
|
got, err := config.EnvSentryDSNForTest(envKeySentryDSN)
|
||||||
|
|||||||
@@ -3,9 +3,6 @@ package database_test
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"log/slog"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
@@ -86,60 +83,3 @@ func TestFirstBoot_PrintsTheAdminPasswordAsABanner(t *testing.T) {
|
|||||||
t, ok, "the printed password must open the seeded account",
|
t, ok, "the printed password must open the seeded account",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestNewDatabase_IsLoggedWithItsPath is the log half of
|
|
||||||
// https://git.eeqj.de/sneak/webhooker/issues/359. A DATA_DIR that is
|
|
||||||
// unexpectedly empty boots exactly like a first start, so the start
|
|
||||||
// that creates the database must say so, and where. Opening that
|
|
||||||
// database again must not.
|
|
||||||
func TestNewDatabase_IsLoggedWithItsPath(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
|
|
||||||
open := func() string {
|
|
||||||
var out bytes.Buffer
|
|
||||||
|
|
||||||
db, err := database.Open(dir, slog.New(slog.NewTextHandler(&out, nil)))
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, db.Close())
|
|
||||||
|
|
||||||
return out.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
const created = `level=WARN msg="created a new, empty database"`
|
|
||||||
|
|
||||||
first := open()
|
|
||||||
second := open()
|
|
||||||
|
|
||||||
assert.Contains(
|
|
||||||
t, first,
|
|
||||||
created+" path="+filepath.Join(dir, database.MainDBFileName),
|
|
||||||
)
|
|
||||||
assert.NotContains(
|
|
||||||
t, second, created, "an existing database is not new",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestZeroLengthDatabase_IsLoggedAsNew covers what
|
|
||||||
// https://git.eeqj.de/sneak/webhooker/issues/290 found: SQLite opens a
|
|
||||||
// zero-length file as an empty database, so a start on one is a first
|
|
||||||
// start, and it must say so exactly as a start with no file does.
|
|
||||||
func TestZeroLengthDatabase_IsLoggedAsNew(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
path := filepath.Join(dir, database.MainDBFileName)
|
|
||||||
require.NoError(t, os.WriteFile(path, nil, database.SQLiteFilePerm))
|
|
||||||
|
|
||||||
var out bytes.Buffer
|
|
||||||
|
|
||||||
db, err := database.Open(dir, slog.New(slog.NewTextHandler(&out, nil)))
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, db.Close())
|
|
||||||
|
|
||||||
assert.Contains(
|
|
||||||
t, out.String(),
|
|
||||||
`level=WARN msg="created a new, empty database" path=`+path,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -17,13 +17,12 @@ import (
|
|||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
"sneak.berlin/go/webhooker/internal/banner"
|
"sneak.berlin/go/webhooker/internal/banner"
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/datadir"
|
|
||||||
"sneak.berlin/go/webhooker/internal/db/migrations"
|
|
||||||
"sneak.berlin/go/webhooker/internal/gormlog"
|
"sneak.berlin/go/webhooker/internal/gormlog"
|
||||||
"sneak.berlin/go/webhooker/internal/logger"
|
"sneak.berlin/go/webhooker/internal/logger"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
dataDirPerm = 0750
|
||||||
randomPasswordLen = 16
|
randomPasswordLen = 16
|
||||||
sessionKeyLen = 32
|
sessionKeyLen = 32
|
||||||
)
|
)
|
||||||
@@ -186,9 +185,7 @@ func (d *Database) connect() error {
|
|||||||
// caller's decision.
|
// caller's decision.
|
||||||
func (d *Database) connectTo(dataDir string) error {
|
func (d *Database) connectTo(dataDir string) error {
|
||||||
// Ensure the data directory exists before opening the database.
|
// Ensure the data directory exists before opening the database.
|
||||||
// datadir.DirPerm is the single source of the directory mode; this
|
err := os.MkdirAll(dataDir, dataDirPerm)
|
||||||
// package creates the directory too, since either may run first.
|
|
||||||
err := os.MkdirAll(dataDir, datadir.DirPerm)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf(
|
return fmt.Errorf(
|
||||||
"creating data directory %s: %w",
|
"creating data directory %s: %w",
|
||||||
@@ -200,11 +197,6 @@ func (d *Database) connectTo(dataDir string) error {
|
|||||||
// Construct the main application database path inside DATA_DIR.
|
// Construct the main application database path inside DATA_DIR.
|
||||||
dbPath := filepath.Join(dataDir, MainDBFileName)
|
dbPath := filepath.Join(dataDir, MainDBFileName)
|
||||||
|
|
||||||
// Checked before opening, which creates the file. A DATA_DIR that
|
|
||||||
// is unexpectedly empty -- its volume not mounted, say -- looks
|
|
||||||
// exactly like a first start, so a new database is a warning.
|
|
||||||
created := missingOrEmpty(dbPath)
|
|
||||||
|
|
||||||
// Opened through OpenSQLite so this handle carries the same WAL
|
// Opened through OpenSQLite so this handle carries the same WAL
|
||||||
// journaling, busy timeout, immediate-transaction locking, and pool
|
// journaling, busy timeout, immediate-transaction locking, and pool
|
||||||
// bounds as every other database file. See sqlite_open.go.
|
// bounds as every other database file. See sqlite_open.go.
|
||||||
@@ -212,15 +204,13 @@ func (d *Database) connectTo(dataDir string) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
d.log.Error(
|
d.log.Error(
|
||||||
"failed to open database",
|
"failed to open database",
|
||||||
"path", dbPath,
|
|
||||||
"error", err,
|
"error", err,
|
||||||
)
|
)
|
||||||
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Then use it with GORM. Its errors are SQLite's alone and name no
|
// Then use it with GORM
|
||||||
// file, so the path is added to them here.
|
|
||||||
db, err := gorm.Open(sqlite.Dialector{
|
db, err := gorm.Open(sqlite.Dialector{
|
||||||
Conn: sqlDB,
|
Conn: sqlDB,
|
||||||
}, &gorm.Config{
|
}, &gorm.Config{
|
||||||
@@ -230,41 +220,22 @@ func (d *Database) connectTo(dataDir string) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
d.log.Error(
|
d.log.Error(
|
||||||
"failed to connect to database",
|
"failed to connect to database",
|
||||||
"path", dbPath,
|
|
||||||
"error", err,
|
"error", err,
|
||||||
)
|
)
|
||||||
|
|
||||||
return fmt.Errorf("connecting to %s: %w", dbPath, err)
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
d.db = db
|
d.db = db
|
||||||
|
d.log.Info("connected to database", "path", dbPath)
|
||||||
|
|
||||||
if created {
|
// Run migrations
|
||||||
d.log.Warn("created a new, empty database", "path", dbPath)
|
return d.migrate()
|
||||||
} else {
|
|
||||||
d.log.Info("connected to database", "path", dbPath)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = d.migrate()
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("migrating %s: %w", dbPath, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = parseModels(
|
|
||||||
db, &Setting{}, &User{}, &APIKey{},
|
|
||||||
&Webhook{}, &Entrypoint{}, &Target{},
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("connecting to %s: %w", dbPath, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// migrate applies the main database's SQL files. See
|
|
||||||
// internal/db/migrations.
|
|
||||||
func (d *Database) migrate() error {
|
func (d *Database) migrate() error {
|
||||||
err := migrations.Apply(d.db, migrations.Main)
|
// Run GORM auto-migrations
|
||||||
|
err := d.Migrate()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
d.log.Error(
|
d.log.Error(
|
||||||
"failed to run database migrations",
|
"failed to run database migrations",
|
||||||
@@ -279,21 +250,6 @@ func (d *Database) migrate() error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// parseModels has GORM parse each model for db before the handle is
|
|
||||||
// shared. GORM parses a model the first time a handle uses it, and two
|
|
||||||
// goroutines doing that at once for models that refer to each other,
|
|
||||||
// as Webhook and Target do, race inside GORM.
|
|
||||||
func parseModels(db *gorm.DB, models ...any) error {
|
|
||||||
for _, model := range models {
|
|
||||||
err := (&gorm.Statement{DB: db}).Parse(model)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("parsing model %T: %w", model, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ensureAdminUser creates the bootstrap admin account when the
|
// ensureAdminUser creates the bootstrap admin account when the
|
||||||
// deployment has no users at all.
|
// deployment has no users at all.
|
||||||
func (d *Database) ensureAdminUser() error {
|
func (d *Database) ensureAdminUser() error {
|
||||||
|
|||||||
@@ -1,15 +1,9 @@
|
|||||||
package database_test
|
package database_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"context"
|
"context"
|
||||||
"log/slog"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"go.uber.org/fx/fxtest"
|
"go.uber.org/fx/fxtest"
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
@@ -106,22 +100,3 @@ func TestDatabaseConnection(t *testing.T) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestOpen_UnreadableDatabaseIsNamed pins
|
|
||||||
// https://git.eeqj.de/sneak/webhooker/issues/459: when SQLite cannot
|
|
||||||
// read webhooker.db, the error that stops the server and `webhooker
|
|
||||||
// resetpw` names the file, not only SQLite's own message.
|
|
||||||
func TestOpen_UnreadableDatabaseIsNamed(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
path := filepath.Join(dir, database.MainDBFileName)
|
|
||||||
require.NoError(t, os.WriteFile(
|
|
||||||
path, bytes.Repeat([]byte("junk"), 1024), database.SQLiteFilePerm,
|
|
||||||
))
|
|
||||||
|
|
||||||
_, err := database.Open(dir, slog.New(slog.DiscardHandler))
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.Contains(t, err.Error(), path)
|
|
||||||
assert.Contains(t, err.Error(), "file is not a database")
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,55 +0,0 @@
|
|||||||
// Package databasetest builds a WebhookDBManager for tests in other
|
|
||||||
// packages.
|
|
||||||
package databasetest
|
|
||||||
|
|
||||||
import (
|
|
||||||
"log/slog"
|
|
||||||
"os"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"go.uber.org/fx/fxtest"
|
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// NewWebhookDBManager creates a WebhookDBManager backed by the given
|
|
||||||
// data directory, logging at DEBUG to standard error.
|
|
||||||
func NewWebhookDBManager(
|
|
||||||
t *testing.T, dataDir string,
|
|
||||||
) *database.WebhookDBManager {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
return NewWebhookDBManagerWithLogger(
|
|
||||||
t,
|
|
||||||
dataDir,
|
|
||||||
slog.New(slog.NewTextHandler(
|
|
||||||
os.Stderr,
|
|
||||||
&slog.HandlerOptions{Level: slog.LevelDebug},
|
|
||||||
)),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewWebhookDBManagerWithLogger is NewWebhookDBManager with the
|
|
||||||
// logger supplied by the caller. The per-webhook databases this manager
|
|
||||||
// opens hand that logger to gormlog, so a test that needs to see the SQL
|
|
||||||
// the service emits can capture it.
|
|
||||||
//
|
|
||||||
// It is built through database.NewWebhookDBManager on a lifecycle that
|
|
||||||
// is never started, so nothing closes its databases but the caller.
|
|
||||||
func NewWebhookDBManagerWithLogger(
|
|
||||||
t *testing.T, dataDir string, log *slog.Logger,
|
|
||||||
) *database.WebhookDBManager {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
mgr, err := database.NewWebhookDBManager(
|
|
||||||
fxtest.NewLifecycle(t),
|
|
||||||
database.WebhookDBManagerParams{
|
|
||||||
Config: &config.Config{DataDir: dataDir},
|
|
||||||
Logger: log,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
return mgr
|
|
||||||
}
|
|
||||||
@@ -2,6 +2,7 @@ package database
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
"gorm.io/gorm/clause"
|
"gorm.io/gorm/clause"
|
||||||
@@ -17,14 +18,14 @@ const omitAssociationsCallback = "webhooker:omit_associations"
|
|||||||
//
|
//
|
||||||
// A per-webhook database holds the event tier only, but Delivery
|
// A per-webhook database holds the event tier only, but Delivery
|
||||||
// declares belongs-to Event and Target and the delivery engine fills
|
// declares belongs-to Event and Target and the delivery engine fills
|
||||||
// both in memory before writing. Without this guard GORM upserts those
|
// both in memory before writing. Without this guard GORM upserts
|
||||||
// parent rows here on the delivery and retry write paths, and the
|
// those parent rows here on the delivery and retry write paths,
|
||||||
// target, whose config holds destination URLs and bearer credentials,
|
// copying targets.config, which holds destination URLs and bearer
|
||||||
// goes to a targets table this file does not have, which fails the
|
// credentials, into the file most likely to be backed up or handed
|
||||||
// write. Registering the guard on the connection
|
// to someone else. Registering the guard on the connection covers
|
||||||
// covers every write path, including writes inside a transaction and
|
// every write path, including writes inside a transaction and write
|
||||||
// write paths added later. Every event-tier row this file holds is
|
// paths added later. Every event-tier row this file holds is written
|
||||||
// written explicitly, so nothing depends on the automatic save.
|
// explicitly, so nothing depends on the automatic save.
|
||||||
func omitAssociations(db *gorm.DB) error {
|
func omitAssociations(db *gorm.DB) error {
|
||||||
omit := func(tx *gorm.DB) {
|
omit := func(tx *gorm.DB) {
|
||||||
tx.Statement.Omits = append(
|
tx.Statement.Omits = append(
|
||||||
@@ -52,3 +53,107 @@ func omitAssociations(db *gorm.DB) error {
|
|||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// eventDBSweptVersion is the PRAGMA user_version purgeTargetRows
|
||||||
|
// stamps into a per-webhook database once it has removed any leaked
|
||||||
|
// target rows *and* the VACUUM that removes their bytes has returned.
|
||||||
|
// Nothing else in the tree uses user_version, so 0 means "not swept
|
||||||
|
// by this build".
|
||||||
|
//
|
||||||
|
// The stamp, not the DELETE, is what records that a file is done. A
|
||||||
|
// DELETE commits on its own, so a sweep that is interrupted or whose
|
||||||
|
// VACUUM fails leaves a file whose rows are gone but whose credential
|
||||||
|
// bytes are still in the free pages -- indistinguishable, by row
|
||||||
|
// count, from a file that never leaked. Both leave the stamp unset,
|
||||||
|
// so the next open sweeps again.
|
||||||
|
const eventDBSweptVersion = 1
|
||||||
|
|
||||||
|
// purgeTargetRows deletes target rows that an earlier build's
|
||||||
|
// association upsert wrote into a per-webhook database, and rewrites
|
||||||
|
// the file so their bytes are gone with them. AutoMigrate creates a
|
||||||
|
// targets table in every one of these files because Delivery declares
|
||||||
|
// a belongs-to Target, but nothing in the event tier may put rows in
|
||||||
|
// it. The rows it did put there are junk, not history: they carry an
|
||||||
|
// empty webhook_id, and delivery rows resolve their target against
|
||||||
|
// the main database, so nothing here refers to them.
|
||||||
|
//
|
||||||
|
// The DELETE only unlinks the rows: modernc.org/sqlite leaves
|
||||||
|
// secure_delete at SQLite's default of off, so the credential bytes
|
||||||
|
// stay readable in the file's free pages and a backup of a swept file
|
||||||
|
// would still hand them over. VACUUM rewrites the file without them.
|
||||||
|
//
|
||||||
|
// This runs before every migration and is gated on
|
||||||
|
// eventDBSweptVersion, so a file pays for the rewrite once, on the
|
||||||
|
// first open that finds it unstamped, and every open after that is a
|
||||||
|
// PRAGMA read. A file this build created is stamped before its
|
||||||
|
// targets table exists, so it never vacuums at all. A failure here
|
||||||
|
// fails the open with the stamp left unset, so the sweep is retried
|
||||||
|
// rather than skipped -- a webhook whose file cannot be swept stays
|
||||||
|
// unusable instead of quietly serving from a file that still holds
|
||||||
|
// recoverable credentials.
|
||||||
|
func purgeTargetRows(
|
||||||
|
db *gorm.DB, log *slog.Logger, webhookID string,
|
||||||
|
) error {
|
||||||
|
var version int
|
||||||
|
|
||||||
|
// Row().Scan, not (*gorm.DB).Scan: see internal/gormlog.
|
||||||
|
err := db.Raw("PRAGMA user_version").Row().Scan(&version)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"reading sweep marker of webhook database %s: %w",
|
||||||
|
webhookID, err,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
if version >= eventDBSweptVersion {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var purged int64
|
||||||
|
|
||||||
|
if db.Migrator().HasTable("targets") {
|
||||||
|
res := db.Exec("DELETE FROM targets")
|
||||||
|
if res.Error != nil {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"purging target rows from webhook database %s: %w",
|
||||||
|
webhookID, res.Error,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
purged = res.RowsAffected
|
||||||
|
|
||||||
|
// Unconditional: a zero row count here does not mean there is
|
||||||
|
// nothing to remove, only that no *live* row is left. See
|
||||||
|
// eventDBSweptVersion.
|
||||||
|
err = db.Exec("VACUUM").Error
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"purged %d leaked target rows from webhook database "+
|
||||||
|
"%s but vacuuming it failed, so the deleted "+
|
||||||
|
"target credentials are still recoverable from "+
|
||||||
|
"the file; it stays marked unswept and the next "+
|
||||||
|
"open retries: %w",
|
||||||
|
purged, webhookID, err,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
err = db.Exec(fmt.Sprintf(
|
||||||
|
"PRAGMA user_version = %d", eventDBSweptVersion,
|
||||||
|
)).Error
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"marking webhook database %s swept: %w", webhookID, err,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
if purged > 0 {
|
||||||
|
log.Warn(
|
||||||
|
"purged leaked target rows from per-webhook database",
|
||||||
|
"webhook_id", webhookID,
|
||||||
|
"rows", purged,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,15 +1,391 @@
|
|||||||
package database_test
|
package database_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
|
"database/sql"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
_ "modernc.org/sqlite"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/database/databasetest"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// testDataDirPerm is the mode the test data directory is created
|
||||||
|
// with.
|
||||||
|
const testDataDirPerm = 0o750
|
||||||
|
|
||||||
|
// eventDBDataDir returns a data directory that a WebhookDBManager
|
||||||
|
// can be pointed at.
|
||||||
|
func eventDBDataDir(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
dir := filepath.Join(t.TempDir(), "events")
|
||||||
|
require.NoError(t, os.MkdirAll(dir, testDataDirPerm))
|
||||||
|
|
||||||
|
return dir
|
||||||
|
}
|
||||||
|
|
||||||
|
// openRawEventDB opens the per-webhook database file directly,
|
||||||
|
// without the manager, so a test can put a file on disk in a state
|
||||||
|
// the manager has to cope with, or inspect one afterwards.
|
||||||
|
func openRawEventDB(
|
||||||
|
t *testing.T, dataDir, webhookID string,
|
||||||
|
) *sql.DB {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
path := filepath.Join(
|
||||||
|
dataDir, fmt.Sprintf("events-%s.db", webhookID),
|
||||||
|
)
|
||||||
|
|
||||||
|
sqlDB, err := sql.Open(
|
||||||
|
"sqlite",
|
||||||
|
fmt.Sprintf("file:%s?mode=rwc", path),
|
||||||
|
)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
t.Cleanup(func() { _ = sqlDB.Close() })
|
||||||
|
|
||||||
|
return sqlDB
|
||||||
|
}
|
||||||
|
|
||||||
|
// eventDBFileBytes reads a per-webhook database file off disk, so a
|
||||||
|
// test can assert on what the file itself still holds rather than on
|
||||||
|
// what a query returns.
|
||||||
|
func eventDBFileBytes(t *testing.T, dataDir, webhookID string) []byte {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
//nolint:gosec // reads a file the test just created under t.TempDir()
|
||||||
|
raw, err := os.ReadFile(filepath.Join(
|
||||||
|
dataDir, fmt.Sprintf("events-%s.db", webhookID),
|
||||||
|
))
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
|
||||||
|
// eventDBUserVersion returns the PRAGMA user_version of a per-webhook
|
||||||
|
// database file, which is the marker purgeTargetRows stamps once it
|
||||||
|
// has swept and vacuumed.
|
||||||
|
func eventDBUserVersion(t *testing.T, sqlDB *sql.DB) int {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
var version int
|
||||||
|
|
||||||
|
require.NoError(t, sqlDB.QueryRowContext(
|
||||||
|
t.Context(), "PRAGMA user_version",
|
||||||
|
).Scan(&version))
|
||||||
|
|
||||||
|
return version
|
||||||
|
}
|
||||||
|
|
||||||
|
// clearEventDBSweptMarker resets the sweep marker to 0, which is what
|
||||||
|
// a file written by a build without the sweep looks like. Tests that
|
||||||
|
// seed a leaked row have to create the file through the manager to
|
||||||
|
// get the real targets table shape, and that stamps it.
|
||||||
|
func clearEventDBSweptMarker(t *testing.T, sqlDB *sql.DB) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
_, err := sqlDB.ExecContext(t.Context(), "PRAGMA user_version = 0")
|
||||||
|
require.NoError(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// countTargetRows returns the number of rows in the targets table of
|
||||||
|
// a per-webhook database file, or -1 if the table does not exist.
|
||||||
|
func countTargetRows(t *testing.T, sqlDB *sql.DB) int {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
var tables int
|
||||||
|
|
||||||
|
require.NoError(t, sqlDB.QueryRowContext(
|
||||||
|
t.Context(),
|
||||||
|
"SELECT count(*) FROM sqlite_master "+
|
||||||
|
"WHERE type = 'table' AND name = 'targets'",
|
||||||
|
).Scan(&tables))
|
||||||
|
|
||||||
|
if tables == 0 {
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
|
||||||
|
var rows int
|
||||||
|
|
||||||
|
require.NoError(t, sqlDB.QueryRowContext(
|
||||||
|
t.Context(), "SELECT count(*) FROM targets",
|
||||||
|
).Scan(&rows))
|
||||||
|
|
||||||
|
return rows
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestOpenPurgesLeakedTargetRows covers the sweep for event
|
||||||
|
// databases written by a build that let GORM upsert target rows
|
||||||
|
// into them: opening the database clears them, and opening it again
|
||||||
|
// is a no-op.
|
||||||
|
func TestOpenPurgesLeakedTargetRows(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
dataDir := eventDBDataDir(t)
|
||||||
|
webhookID := uuid.New().String()
|
||||||
|
|
||||||
|
// Create the file the way the application does, so the targets
|
||||||
|
// table has exactly the shape AutoMigrate gives it, then write
|
||||||
|
// a leaked row into it the way the association upsert did.
|
||||||
|
initial := database.NewTestWebhookDBManager(dataDir)
|
||||||
|
|
||||||
|
_, err := initial.GetDB(webhookID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, initial.CloseAll())
|
||||||
|
|
||||||
|
seed := openRawEventDB(t, dataDir, webhookID)
|
||||||
|
|
||||||
|
_, err = seed.ExecContext(
|
||||||
|
t.Context(),
|
||||||
|
"INSERT INTO targets "+
|
||||||
|
"(id, webhook_id, name, type, config) "+
|
||||||
|
"VALUES (?, '', ?, ?, ?)",
|
||||||
|
uuid.New().String(),
|
||||||
|
"leaked-target",
|
||||||
|
"slack",
|
||||||
|
`{"webhookUrl":"https://hooks.example/T000/B000/secret"}`,
|
||||||
|
)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, 1, countTargetRows(t, seed))
|
||||||
|
clearEventDBSweptMarker(t, seed)
|
||||||
|
require.NoError(t, seed.Close())
|
||||||
|
|
||||||
|
mgr := database.NewTestWebhookDBManager(dataDir)
|
||||||
|
|
||||||
|
_, err = mgr.GetDB(webhookID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, mgr.CloseAll())
|
||||||
|
|
||||||
|
check := openRawEventDB(t, dataDir, webhookID)
|
||||||
|
assert.Zero(t, countTargetRows(t, check))
|
||||||
|
assert.Equal(
|
||||||
|
t, 1, eventDBUserVersion(t, check),
|
||||||
|
"a completed sweep must mark the file so later opens skip it",
|
||||||
|
)
|
||||||
|
require.NoError(t, check.Close())
|
||||||
|
|
||||||
|
// Idempotent: a second open leaves it at zero and does not
|
||||||
|
// error.
|
||||||
|
again := database.NewTestWebhookDBManager(dataDir)
|
||||||
|
|
||||||
|
_, err = again.GetDB(webhookID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, again.CloseAll())
|
||||||
|
|
||||||
|
recheck := openRawEventDB(t, dataDir, webhookID)
|
||||||
|
assert.Zero(t, countTargetRows(t, recheck))
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestOpenPurgeRemovesCredentialBytes covers the sweep at the level
|
||||||
|
// that matters for a backup handed to someone else: the leaked
|
||||||
|
// credential must be gone from the raw bytes of the file, not merely
|
||||||
|
// unreachable by query. A bare DELETE unlinks the row and leaves the
|
||||||
|
// bytes readable in the free pages, so this fails without the VACUUM
|
||||||
|
// in purgeTargetRows.
|
||||||
|
func TestOpenPurgeRemovesCredentialBytes(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
dataDir := eventDBDataDir(t)
|
||||||
|
webhookID := uuid.New().String()
|
||||||
|
credential := "T00000000/B00000000/" + uuid.New().String()
|
||||||
|
|
||||||
|
initial := database.NewTestWebhookDBManager(dataDir)
|
||||||
|
|
||||||
|
_, err := initial.GetDB(webhookID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, initial.CloseAll())
|
||||||
|
|
||||||
|
seed := openRawEventDB(t, dataDir, webhookID)
|
||||||
|
|
||||||
|
_, err = seed.ExecContext(
|
||||||
|
t.Context(),
|
||||||
|
"INSERT INTO targets "+
|
||||||
|
"(id, webhook_id, name, type, config) "+
|
||||||
|
"VALUES (?, '', ?, ?, ?)",
|
||||||
|
uuid.New().String(),
|
||||||
|
"leaked-target",
|
||||||
|
"slack",
|
||||||
|
fmt.Sprintf(
|
||||||
|
`{"webhookUrl":"https://hooks.example/%s"}`, credential,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
require.NoError(t, err)
|
||||||
|
clearEventDBSweptMarker(t, seed)
|
||||||
|
require.NoError(t, seed.Close())
|
||||||
|
|
||||||
|
// The seed has to be in the file for its absence later to mean
|
||||||
|
// anything.
|
||||||
|
require.True(
|
||||||
|
t,
|
||||||
|
bytes.Contains(
|
||||||
|
eventDBFileBytes(t, dataDir, webhookID),
|
||||||
|
[]byte(credential),
|
||||||
|
),
|
||||||
|
"seeded credential is not in the file, so this test proves nothing",
|
||||||
|
)
|
||||||
|
|
||||||
|
mgr := database.NewTestWebhookDBManager(dataDir)
|
||||||
|
|
||||||
|
_, err = mgr.GetDB(webhookID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, mgr.CloseAll())
|
||||||
|
|
||||||
|
assert.NotContains(
|
||||||
|
t,
|
||||||
|
string(eventDBFileBytes(t, dataDir, webhookID)),
|
||||||
|
credential,
|
||||||
|
"leaked credential is still recoverable from the raw file",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestOpenRevacuumsAfterIncompleteSweep covers the case a row count
|
||||||
|
// cannot see: the rows are already deleted but the file was never
|
||||||
|
// vacuumed, because an earlier sweep died between the two or its
|
||||||
|
// VACUUM failed. The credential bytes are still recoverable, and the
|
||||||
|
// unset marker is the only thing that says so, so the next open must
|
||||||
|
// vacuum rather than conclude from the empty table that there is
|
||||||
|
// nothing to do.
|
||||||
|
func TestOpenRevacuumsAfterIncompleteSweep(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
dataDir := eventDBDataDir(t)
|
||||||
|
webhookID := uuid.New().String()
|
||||||
|
credential := "T00000000/B00000000/" + uuid.New().String()
|
||||||
|
|
||||||
|
initial := database.NewTestWebhookDBManager(dataDir)
|
||||||
|
|
||||||
|
_, err := initial.GetDB(webhookID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, initial.CloseAll())
|
||||||
|
|
||||||
|
seed := openRawEventDB(t, dataDir, webhookID)
|
||||||
|
|
||||||
|
_, err = seed.ExecContext(
|
||||||
|
t.Context(),
|
||||||
|
"INSERT INTO targets "+
|
||||||
|
"(id, webhook_id, name, type, config) "+
|
||||||
|
"VALUES (?, '', ?, ?, ?)",
|
||||||
|
uuid.New().String(),
|
||||||
|
"leaked-target",
|
||||||
|
"slack",
|
||||||
|
fmt.Sprintf(
|
||||||
|
`{"webhookUrl":"https://hooks.example/%s"}`, credential,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// Exactly the state an interrupted sweep leaves: rows gone,
|
||||||
|
// marker unset, bytes still in the free pages.
|
||||||
|
_, err = seed.ExecContext(t.Context(), "DELETE FROM targets")
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Zero(t, countTargetRows(t, seed))
|
||||||
|
clearEventDBSweptMarker(t, seed)
|
||||||
|
require.NoError(t, seed.Close())
|
||||||
|
|
||||||
|
require.True(
|
||||||
|
t,
|
||||||
|
bytes.Contains(
|
||||||
|
eventDBFileBytes(t, dataDir, webhookID),
|
||||||
|
[]byte(credential),
|
||||||
|
),
|
||||||
|
"the deleted row's bytes must still be in the file, or this "+
|
||||||
|
"test proves nothing",
|
||||||
|
)
|
||||||
|
|
||||||
|
mgr := database.NewTestWebhookDBManager(dataDir)
|
||||||
|
|
||||||
|
_, err = mgr.GetDB(webhookID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, mgr.CloseAll())
|
||||||
|
|
||||||
|
assert.NotContains(
|
||||||
|
t,
|
||||||
|
string(eventDBFileBytes(t, dataDir, webhookID)),
|
||||||
|
credential,
|
||||||
|
"an interrupted sweep was not retried, so the credential is "+
|
||||||
|
"still recoverable from the raw file",
|
||||||
|
)
|
||||||
|
|
||||||
|
check := openRawEventDB(t, dataDir, webhookID)
|
||||||
|
assert.Equal(t, 1, eventDBUserVersion(t, check))
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestOpenSkipsSweptDatabase covers the other half of the marker: a
|
||||||
|
// file this build created is marked without ever being vacuumed, and
|
||||||
|
// a marked file is not swept again.
|
||||||
|
func TestOpenSkipsSweptDatabase(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
dataDir := eventDBDataDir(t)
|
||||||
|
webhookID := uuid.New().String()
|
||||||
|
|
||||||
|
mgr := database.NewTestWebhookDBManager(dataDir)
|
||||||
|
|
||||||
|
_, err := mgr.GetDB(webhookID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, mgr.CloseAll())
|
||||||
|
|
||||||
|
marked := openRawEventDB(t, dataDir, webhookID)
|
||||||
|
assert.Equal(t, 1, eventDBUserVersion(t, marked))
|
||||||
|
|
||||||
|
// A marked file is left alone, so a row written into it survives
|
||||||
|
// a reopen. Nothing writes target rows any more; this stands in
|
||||||
|
// for the sweep having run.
|
||||||
|
_, err = marked.ExecContext(
|
||||||
|
t.Context(),
|
||||||
|
"INSERT INTO targets "+
|
||||||
|
"(id, webhook_id, name, type, config) "+
|
||||||
|
"VALUES (?, '', ?, ?, ?)",
|
||||||
|
uuid.New().String(), "sentinel", "slack", `{}`,
|
||||||
|
)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, marked.Close())
|
||||||
|
|
||||||
|
again := database.NewTestWebhookDBManager(dataDir)
|
||||||
|
|
||||||
|
_, err = again.GetDB(webhookID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, again.CloseAll())
|
||||||
|
|
||||||
|
check := openRawEventDB(t, dataDir, webhookID)
|
||||||
|
assert.Equal(
|
||||||
|
t, 1, countTargetRows(t, check),
|
||||||
|
"a marked file must not be swept again",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestOpenSucceedsWithoutTargetsTable covers an existing event
|
||||||
|
// database that never grew a targets table. The sweep must not fail
|
||||||
|
// startup on it.
|
||||||
|
func TestOpenSucceedsWithoutTargetsTable(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
dataDir := eventDBDataDir(t)
|
||||||
|
webhookID := uuid.New().String()
|
||||||
|
|
||||||
|
seed := openRawEventDB(t, dataDir, webhookID)
|
||||||
|
|
||||||
|
_, err := seed.ExecContext(
|
||||||
|
t.Context(),
|
||||||
|
"CREATE TABLE events (id text PRIMARY KEY)",
|
||||||
|
)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, seed.Close())
|
||||||
|
|
||||||
|
mgr := database.NewTestWebhookDBManager(dataDir)
|
||||||
|
|
||||||
|
db, err := mgr.GetDB(webhookID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.NotNil(t, db)
|
||||||
|
require.NoError(t, mgr.CloseAll())
|
||||||
|
}
|
||||||
|
|
||||||
// TestEventDBCreateOmitsAssociations covers the connection-level
|
// TestEventDBCreateOmitsAssociations covers the connection-level
|
||||||
// guard directly: a Delivery carrying its Event and Target in
|
// guard directly: a Delivery carrying its Event and Target in
|
||||||
// memory, written through the manager's handle, must store only the
|
// memory, written through the manager's handle, must store only the
|
||||||
@@ -17,19 +393,19 @@ import (
|
|||||||
func TestEventDBCreateOmitsAssociations(t *testing.T) {
|
func TestEventDBCreateOmitsAssociations(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
dataDir := eventDBDataDir(t)
|
||||||
webhookID := uuid.New().String()
|
webhookID := uuid.New().String()
|
||||||
|
|
||||||
mgr := databasetest.NewWebhookDBManager(t, t.TempDir())
|
mgr := database.NewTestWebhookDBManager(dataDir)
|
||||||
|
|
||||||
db, err := mgr.GetDB(webhookID)
|
db, err := mgr.GetDB(webhookID)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
target := database.Target{
|
target := database.Target{
|
||||||
WebhookID: webhookID,
|
WebhookID: webhookID,
|
||||||
Name: "leaky-target",
|
Name: "leaky-target",
|
||||||
Type: database.TargetTypeSlack,
|
Type: database.TargetTypeSlack,
|
||||||
URL: "https://hooks.example/secret",
|
Config: `{"webhookUrl":"https://hooks.example/secret"}`,
|
||||||
MaxAttempts: 1,
|
|
||||||
}
|
}
|
||||||
target.ID = uuid.New().String()
|
target.ID = uuid.New().String()
|
||||||
|
|
||||||
@@ -55,9 +431,8 @@ func TestEventDBCreateOmitsAssociations(t *testing.T) {
|
|||||||
require.NoError(t, db.Model(d).
|
require.NoError(t, db.Model(d).
|
||||||
Update("status", database.DeliveryStatusDelivered).
|
Update("status", database.DeliveryStatusDelivered).
|
||||||
Error)
|
Error)
|
||||||
|
|
||||||
// The file has no targets table, so the create and update above
|
|
||||||
// would have failed had GORM tried to save the target.
|
|
||||||
assert.False(t, db.Migrator().HasTable(&database.Target{}))
|
|
||||||
require.NoError(t, mgr.CloseAll())
|
require.NoError(t, mgr.CloseAll())
|
||||||
|
|
||||||
|
check := openRawEventDB(t, dataDir, webhookID)
|
||||||
|
assert.Zero(t, countTargetRows(t, check))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,33 +2,39 @@ package database_test
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"gorm.io/gorm"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TestWebhookDBManager_OpenAddsEventTierIndexes verifies that a new
|
// indexedColumn names a secondary index by the model and struct field
|
||||||
// per-webhook database has these indexes.
|
// GORM derives the index name from.
|
||||||
|
type indexedColumn struct {
|
||||||
|
model any
|
||||||
|
field string
|
||||||
|
}
|
||||||
|
|
||||||
|
// eventTierIndexes are the columns the background work reads by: the
|
||||||
|
// recovery and sweep queries (status), the event log (event_id and
|
||||||
|
// delivery_id) and retention (created_at).
|
||||||
|
var eventTierIndexes = []indexedColumn{
|
||||||
|
{&database.Delivery{}, "Status"},
|
||||||
|
{&database.Delivery{}, "EventID"},
|
||||||
|
{&database.DeliveryResult{}, "DeliveryID"},
|
||||||
|
{&database.Event{}, "CreatedAt"},
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestWebhookDBManager_OpenAddsEventTierIndexes verifies that opening a
|
||||||
|
// per-webhook database that predates these indexes creates them, so the
|
||||||
|
// queries above stop scanning whole tables. It stands in for an older
|
||||||
|
// database file by dropping the indexes AutoMigrate just created, then
|
||||||
|
// reopening the same file.
|
||||||
func TestWebhookDBManager_OpenAddsEventTierIndexes(t *testing.T) {
|
func TestWebhookDBManager_OpenAddsEventTierIndexes(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
indexes := []struct {
|
|
||||||
model any
|
|
||||||
name string
|
|
||||||
}{
|
|
||||||
{&database.Delivery{}, "idx_deliveries_status"},
|
|
||||||
{&database.Delivery{}, "idx_deliveries_event_id"},
|
|
||||||
{&database.DeliveryResult{}, "idx_delivery_results_delivery_id"},
|
|
||||||
{&database.Event{}, "idx_events_deleted_at_created_at"},
|
|
||||||
{&database.Event{}, "idx_events_created_at"},
|
|
||||||
}
|
|
||||||
|
|
||||||
mgr, lc := setupTestWebhookDBManager(t)
|
mgr, lc := setupTestWebhookDBManager(t)
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
require.NoError(t, lc.Start(ctx))
|
require.NoError(t, lc.Start(ctx))
|
||||||
@@ -40,290 +46,27 @@ func TestWebhookDBManager_OpenAddsEventTierIndexes(t *testing.T) {
|
|||||||
db, err := mgr.GetDB(webhookID)
|
db, err := mgr.GetDB(webhookID)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
for _, ix := range indexes {
|
// A fresh database has them.
|
||||||
assert.True(t, db.Migrator().HasIndex(ix.model, ix.name),
|
for _, ix := range eventTierIndexes {
|
||||||
"a new database should have %s", ix.name)
|
require.True(t, db.Migrator().HasIndex(ix.model, ix.field))
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// TestEventTierQueriesUseTheirIndexes verifies that the statements the
|
// Stand in for a database file created before the indexes existed.
|
||||||
// indexes are for use them. GORM builds each statement in a dry run as
|
for _, ix := range eventTierIndexes {
|
||||||
// the code named above it does, soft-delete condition included, and
|
require.NoError(t, db.Migrator().DropIndex(ix.model, ix.field))
|
||||||
// SQLite, which keeps no statistics on these tables, must plan to seek
|
require.False(t, db.Migrator().HasIndex(ix.model, ix.field))
|
||||||
// on each index listed by the columns in parentheses.
|
}
|
||||||
func TestEventTierQueriesUseTheirIndexes(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
mgr, lc := setupTestWebhookDBManager(t)
|
// Drop the cached connection so the next open reopens the file and
|
||||||
ctx := context.Background()
|
// runs AutoMigrate against it, as a restart would.
|
||||||
require.NoError(t, lc.Start(ctx))
|
require.NoError(t, mgr.CloseAll())
|
||||||
|
|
||||||
defer func() { require.NoError(t, lc.Stop(ctx)) }()
|
db, err = mgr.GetDB(webhookID)
|
||||||
|
|
||||||
db, err := mgr.GetDB(uuid.New().String())
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
dry := db.Session(&gorm.Session{DryRun: true})
|
for _, ix := range eventTierIndexes {
|
||||||
ids := []string{
|
assert.True(t, db.Migrator().HasIndex(ix.model, ix.field),
|
||||||
uuid.New().String(), uuid.New().String(), uuid.New().String(),
|
"opening the existing database should create the index on %s",
|
||||||
}
|
ix.field)
|
||||||
cutoff := time.Now()
|
|
||||||
|
|
||||||
var (
|
|
||||||
deliveries []database.Delivery
|
|
||||||
results []database.DeliveryResult
|
|
||||||
depths []struct{ Depth int }
|
|
||||||
removed []database.TargetTotals
|
|
||||||
)
|
|
||||||
|
|
||||||
byStatus := "idx_deliveries_status (status=? AND deleted_at=?)"
|
|
||||||
byEvent := "idx_deliveries_event_id (event_id=? AND deleted_at=?)"
|
|
||||||
|
|
||||||
// The delivery engine: the recovery routine, which reads the
|
|
||||||
// retrying deliveries and, on a tick of the sweep, the pending ones
|
|
||||||
// untouched for a while; and the queue depth count.
|
|
||||||
assertPlanUses(t, db, dry.Where(
|
|
||||||
"status = ?", database.DeliveryStatusRetrying,
|
|
||||||
).Find(&deliveries), byStatus)
|
|
||||||
assertPlanUses(t, db, dry.Where(
|
|
||||||
"status = ? AND updated_at < ?",
|
|
||||||
database.DeliveryStatusPending, cutoff,
|
|
||||||
).Limit(500).Find(&deliveries), byStatus)
|
|
||||||
assertPlanUses(t, db, dry.Model(&database.Delivery{}).
|
|
||||||
Select("target_id", "status", "count(*) as depth").
|
|
||||||
Where("status IN ?", []database.DeliveryStatus{
|
|
||||||
database.DeliveryStatusPending,
|
|
||||||
database.DeliveryStatusRetrying,
|
|
||||||
}).Group("target_id, status").Find(&depths), byStatus)
|
|
||||||
|
|
||||||
// The event log: each event's deliveries, then their attempts
|
|
||||||
// (loadEventsWithDeliveries, loadDeliveryResults).
|
|
||||||
assertPlanUses(t, db, dry.Where("event_id = ?", ids[0]).
|
|
||||||
Find(&deliveries), byEvent)
|
|
||||||
assertPlanUses(t, db, dry.Where("delivery_id IN ?", ids).
|
|
||||||
Order("attempt_num ASC").Find(&results),
|
|
||||||
"idx_delivery_results_delivery_id (delivery_id=? AND deleted_at=?)")
|
|
||||||
|
|
||||||
// Retention (reapExpired, deleteEvents): one batch of expired
|
|
||||||
// events, then their attempts, deliveries and the events.
|
|
||||||
var expired []string
|
|
||||||
|
|
||||||
assertPlanUses(t, db, dry.Unscoped().Model(&database.Event{}).
|
|
||||||
Where("created_at < ?", cutoff).
|
|
||||||
Limit(database.ExportReapBatchSize).Pluck("id", &expired),
|
|
||||||
"idx_events_created_at (created_at<?)")
|
|
||||||
assertPlanUses(t, db, dry.Unscoped().Where(
|
|
||||||
"delivery_id IN (?)", dry.Unscoped().Model(&database.Delivery{}).
|
|
||||||
Select("id").Where("event_id IN ?", ids),
|
|
||||||
).Delete(&database.DeliveryResult{}),
|
|
||||||
"idx_delivery_results_delivery_id (delivery_id=?)",
|
|
||||||
"idx_deliveries_event_id (event_id=?)")
|
|
||||||
assertPlanUses(t, db, dry.Unscoped().Model(&database.Delivery{}).
|
|
||||||
Select("target_id, count(*) AS deliveries_removed, "+
|
|
||||||
"count(CASE WHEN status = ? THEN 1 END) AS failed_removed",
|
|
||||||
database.DeliveryStatusFailed).
|
|
||||||
Where("event_id IN ?", ids).Group("target_id").Find(&removed),
|
|
||||||
"idx_deliveries_event_id (event_id=?)")
|
|
||||||
assertPlanUses(t, db, dry.Unscoped().Where("event_id IN ?", ids).
|
|
||||||
Delete(&database.Delivery{}), "idx_deliveries_event_id (event_id=?)")
|
|
||||||
assertPlanUses(t, db, dry.Unscoped().Where("id IN ?", ids).
|
|
||||||
Delete(&database.Event{}), "sqlite_autoindex_events_1 (id=?)")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEventLogFiltersUseTheStatusIndex does the same for the event log's
|
|
||||||
// Failed and Pending lists, of the newest events with a delivery in
|
|
||||||
// given statuses, and for their counts (eventsWithStatus and
|
|
||||||
// countEventsWithStatus in the handlers). The lists must also reach
|
|
||||||
// the events table only by ID: from the matching deliveries, then from
|
|
||||||
// the newest of those events.
|
|
||||||
func TestEventLogFiltersUseTheStatusIndex(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
mgr, lc := setupTestWebhookDBManager(t)
|
|
||||||
ctx := context.Background()
|
|
||||||
require.NoError(t, lc.Start(ctx))
|
|
||||||
|
|
||||||
defer func() { require.NoError(t, lc.Stop(ctx)) }()
|
|
||||||
|
|
||||||
webhookID := uuid.New().String()
|
|
||||||
|
|
||||||
db, err := mgr.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
dry := db.Session(&gorm.Session{DryRun: true})
|
|
||||||
byStatus := "idx_deliveries_status (status=? AND deleted_at=?)"
|
|
||||||
pending := []database.DeliveryStatus{
|
|
||||||
database.DeliveryStatusPending,
|
|
||||||
database.DeliveryStatusRetrying,
|
|
||||||
}
|
|
||||||
|
|
||||||
var (
|
|
||||||
rows []struct{ ID string }
|
|
||||||
count int64
|
|
||||||
)
|
|
||||||
|
|
||||||
matching := dry.Model(&database.Delivery{}).
|
|
||||||
Distinct("event_id").Where("status IN ?", pending)
|
|
||||||
newest := dry.Table("(?) AS matching", matching).
|
|
||||||
Joins("CROSS JOIN events ON events.id = matching.event_id").
|
|
||||||
Where(
|
|
||||||
"events.webhook_id = ? AND events.deleted_at IS NULL",
|
|
||||||
webhookID,
|
|
||||||
).
|
|
||||||
Order("events.created_at DESC").Limit(50).
|
|
||||||
Select("events.id AS event_id")
|
|
||||||
|
|
||||||
// Each step of the plan is printed in braces, so these name the
|
|
||||||
// lookup that follows each scan.
|
|
||||||
byID := "{SEARCH events USING INDEX sqlite_autoindex_events_1 (id=?)}"
|
|
||||||
|
|
||||||
assertPlanUses(t, db, dry.Table("(?) AS newest", newest).
|
|
||||||
Joins("CROSS JOIN events ON events.id = newest.event_id").
|
|
||||||
Select("id").Order("created_at DESC").Limit(50).Find(&rows),
|
|
||||||
byStatus, "{SCAN matching} "+byID, "{SCAN newest} "+byID)
|
|
||||||
assertPlanUses(t, db, dry.Model(&database.Delivery{}).
|
|
||||||
Distinct("event_id").Where("status IN ?", pending).Count(&count),
|
|
||||||
byStatus)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestStatisticsQueriesUseTheirIndexes does the same for the webhook
|
|
||||||
// page's statistics (readEventStats in the handlers): deliveries in
|
|
||||||
// progress, each target's deliveries finished since a time, which must
|
|
||||||
// come from the index alone, and events received since a time.
|
|
||||||
func TestStatisticsQueriesUseTheirIndexes(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
mgr, lc := setupTestWebhookDBManager(t)
|
|
||||||
ctx := context.Background()
|
|
||||||
require.NoError(t, lc.Start(ctx))
|
|
||||||
|
|
||||||
defer func() { require.NoError(t, lc.Stop(ctx)) }()
|
|
||||||
|
|
||||||
db, err := mgr.GetDB(uuid.New().String())
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
dry := db.Session(&gorm.Session{DryRun: true})
|
|
||||||
since := time.Now()
|
|
||||||
|
|
||||||
var (
|
|
||||||
count int64
|
|
||||||
byTarget []struct{ TargetID string }
|
|
||||||
)
|
|
||||||
|
|
||||||
assertPlanUses(t, db, dry.Model(&database.Delivery{}).
|
|
||||||
Where("status IN ?", []database.DeliveryStatus{
|
|
||||||
database.DeliveryStatusPending,
|
|
||||||
database.DeliveryStatusRetrying,
|
|
||||||
}).Count(&count),
|
|
||||||
"idx_deliveries_status (status=? AND deleted_at=?)")
|
|
||||||
assertPlanUses(t, db, dry.Model(&database.Delivery{}).
|
|
||||||
Select("target_id, "+
|
|
||||||
"count(CASE WHEN status = ? THEN 1 END) AS delivered, "+
|
|
||||||
"count(CASE WHEN status = ? THEN 1 END) AS failed",
|
|
||||||
database.DeliveryStatusDelivered,
|
|
||||||
database.DeliveryStatusFailed).
|
|
||||||
Where("status IN ? AND finished_at >= ?",
|
|
||||||
[]database.DeliveryStatus{
|
|
||||||
database.DeliveryStatusDelivered,
|
|
||||||
database.DeliveryStatusFailed,
|
|
||||||
}, since).
|
|
||||||
Group("target_id").Find(&byTarget),
|
|
||||||
"COVERING INDEX idx_deliveries_status "+
|
|
||||||
"(status=? AND deleted_at=? AND finished_at>?)")
|
|
||||||
assertPlanUses(t, db, dry.Model(&database.Event{}).
|
|
||||||
Where("created_at >= ?", since).Count(&count),
|
|
||||||
"idx_events_deleted_at_created_at "+
|
|
||||||
"(deleted_at=? AND created_at>?)")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestResubmitCountUsesItsIndex does the same for the event log's count
|
|
||||||
// of the events resubmitted from each of a page's events (resubmitCounts
|
|
||||||
// in the handlers). It passes a full page of 25 ids: with an index on
|
|
||||||
// resubmitted_from_id alone, SQLite uses it for three ids and turns to
|
|
||||||
// the deleted_at index from five.
|
|
||||||
func TestResubmitCountUsesItsIndex(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
mgr, lc := setupTestWebhookDBManager(t)
|
|
||||||
ctx := context.Background()
|
|
||||||
require.NoError(t, lc.Start(ctx))
|
|
||||||
|
|
||||||
defer func() { require.NoError(t, lc.Stop(ctx)) }()
|
|
||||||
|
|
||||||
db, err := mgr.GetDB(uuid.New().String())
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
dry := db.Session(&gorm.Session{DryRun: true})
|
|
||||||
|
|
||||||
page := make([]string, 25)
|
|
||||||
for i := range page {
|
|
||||||
page[i] = uuid.New().String()
|
|
||||||
}
|
|
||||||
|
|
||||||
var counts []struct{ Total int }
|
|
||||||
|
|
||||||
assertPlanUses(t, db, dry.Model(&database.Event{}).
|
|
||||||
Select("resubmitted_from_id, count(*) AS total").
|
|
||||||
Where("resubmitted_from_id IN ?", page).
|
|
||||||
Group("resubmitted_from_id").Find(&counts),
|
|
||||||
"idx_events_resubmitted_from_id "+
|
|
||||||
"(resubmitted_from_id=? AND deleted_at=?)")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEntrypointEventsUseTheirIndex does the same for the webhook
|
|
||||||
// page's count, for each entrypoint, of the events that arrived on its
|
|
||||||
// URL since the retention cutoff (addEntrypointEvents in the
|
|
||||||
// handlers), which must come from the index alone. It passes 25
|
|
||||||
// entrypoints, as TestResubmitCountUsesItsIndex passes 25 events.
|
|
||||||
func TestEntrypointEventsUseTheirIndex(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
mgr, lc := setupTestWebhookDBManager(t)
|
|
||||||
ctx := context.Background()
|
|
||||||
require.NoError(t, lc.Start(ctx))
|
|
||||||
|
|
||||||
defer func() { require.NoError(t, lc.Stop(ctx)) }()
|
|
||||||
|
|
||||||
db, err := mgr.GetDB(uuid.New().String())
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
dry := db.Session(&gorm.Session{DryRun: true})
|
|
||||||
|
|
||||||
entrypoints := make([]string, 25)
|
|
||||||
for i := range entrypoints {
|
|
||||||
entrypoints[i] = uuid.New().String()
|
|
||||||
}
|
|
||||||
|
|
||||||
var rows []struct{ Events int }
|
|
||||||
|
|
||||||
assertPlanUses(t, db, dry.Model(&database.Event{}).
|
|
||||||
Select("entrypoint_id, count(*) AS events").
|
|
||||||
Where("entrypoint_id IN ? AND resubmitted_from_id IS NULL",
|
|
||||||
entrypoints).
|
|
||||||
Where("created_at >= ?", time.Now()).
|
|
||||||
Group("entrypoint_id").Find(&rows),
|
|
||||||
"COVERING INDEX idx_events_entrypoint_id "+
|
|
||||||
"(entrypoint_id=? AND deleted_at=? AND "+
|
|
||||||
"resubmitted_from_id=? AND created_at>?)")
|
|
||||||
}
|
|
||||||
|
|
||||||
// assertPlanUses asserts that SQLite's plan for a statement GORM built
|
|
||||||
// in a dry run, run with the same SQL and arguments GORM would send,
|
|
||||||
// names each of the given indexes.
|
|
||||||
func assertPlanUses(
|
|
||||||
t *testing.T, db, built *gorm.DB, indexes ...string,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var plan []struct{ Detail string }
|
|
||||||
|
|
||||||
require.NoError(t, db.Raw(
|
|
||||||
"EXPLAIN QUERY PLAN "+built.Statement.SQL.String(),
|
|
||||||
built.Statement.Vars...,
|
|
||||||
).Scan(&plan).Error)
|
|
||||||
|
|
||||||
for _, index := range indexes {
|
|
||||||
assert.Contains(t, fmt.Sprint(plan), index,
|
|
||||||
built.Statement.SQL.String())
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"os"
|
"os"
|
||||||
"testing"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
@@ -29,10 +28,6 @@ func NewTestRetentionReaper(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExportReapBatchSize exposes how many expired events one retention
|
|
||||||
// transaction deletes.
|
|
||||||
const ExportReapBatchSize = reapBatchSize
|
|
||||||
|
|
||||||
// ExportSweep runs a single retention sweep synchronously for tests.
|
// ExportSweep runs a single retention sweep synchronously for tests.
|
||||||
func (r *RetentionReaper) ExportSweep(ctx context.Context) {
|
func (r *RetentionReaper) ExportSweep(ctx context.Context) {
|
||||||
r.sweep(ctx)
|
r.sweep(ctx)
|
||||||
@@ -84,14 +79,3 @@ func (d *Database) ExportSetBannerOut(w io.Writer) {
|
|||||||
func DummyPasswordHashForTest() string {
|
func DummyPasswordHashForTest() string {
|
||||||
return dummyPasswordHash()
|
return dummyPasswordHash()
|
||||||
}
|
}
|
||||||
|
|
||||||
// HashAtShippedCostForTest makes HashPassword hash at the shipped
|
|
||||||
// memory cost until t ends. t must not run in parallel with other
|
|
||||||
// tests, which would hash at that cost alongside it.
|
|
||||||
func HashAtShippedCostForTest(t *testing.T) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
hashAtShippedCostInTest = true
|
|
||||||
|
|
||||||
t.Cleanup(func() { hashAtShippedCostInTest = false })
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -15,7 +15,6 @@ type APIKey struct {
|
|||||||
Description string `json:"description"`
|
Description string `json:"description"`
|
||||||
LastUsedAt *time.Time `json:"lastUsedAt,omitempty"`
|
LastUsedAt *time.Time `json:"lastUsedAt,omitempty"`
|
||||||
|
|
||||||
// Relations. No model marshals the record it belongs to:
|
// Relations
|
||||||
// User.APIKeys leads back here, and the JSON could loop.
|
User User `json:"user,omitzero"`
|
||||||
User User `json:"-"`
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,11 +1,5 @@
|
|||||||
package database
|
package database
|
||||||
|
|
||||||
import (
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"gorm.io/gorm"
|
|
||||||
)
|
|
||||||
|
|
||||||
// DeliveryStatus represents the status of a delivery
|
// DeliveryStatus represents the status of a delivery
|
||||||
type DeliveryStatus string
|
type DeliveryStatus string
|
||||||
|
|
||||||
@@ -35,35 +29,15 @@ func (s DeliveryStatus) Terminal() bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Delivery represents a delivery attempt for an event to a target
|
// Delivery represents a delivery attempt for an event to a target
|
||||||
//
|
|
||||||
//nolint:lll // a struct tag cannot wrap
|
|
||||||
type Delivery struct {
|
type Delivery struct {
|
||||||
BaseModel
|
BaseModel
|
||||||
|
|
||||||
EventID string `gorm:"type:uuid;not null;index:idx_deliveries_event_id,priority:1" json:"eventId"`
|
EventID string `gorm:"type:uuid;not null;index" json:"eventId"`
|
||||||
TargetID string `gorm:"type:uuid;not null;index:idx_deliveries_status,priority:4" json:"targetId"`
|
TargetID string `gorm:"type:uuid;not null" json:"targetId"`
|
||||||
Status DeliveryStatus `gorm:"not null;default:'pending';index:idx_deliveries_status,priority:1" json:"status"`
|
Status DeliveryStatus `gorm:"not null;default:'pending';index" json:"status"`
|
||||||
|
|
||||||
// DeletedAt repeats the BaseModel field only to be the second column
|
// Relations
|
||||||
// of the event_id and status indexes, for the reason DeliveryResult
|
Event Event `json:"event,omitzero"`
|
||||||
// gives.
|
Target Target `json:"target,omitzero"`
|
||||||
DeletedAt gorm.DeletedAt `gorm:"index:idx_deliveries_event_id,priority:2;index:idx_deliveries_status,priority:2" json:"deletedAt,omitzero"`
|
|
||||||
|
|
||||||
// FinishedAt is when the delivery became delivered or failed, and
|
|
||||||
// nil while it is pending or retrying. It and then TargetID end the
|
|
||||||
// status index, so the webhook page counts each target's deliveries
|
|
||||||
// that finished in a recent window by reading just that window from
|
|
||||||
// the index.
|
|
||||||
FinishedAt *time.Time `gorm:"index:idx_deliveries_status,priority:3" json:"finishedAt,omitempty"`
|
|
||||||
|
|
||||||
// Replay is set on a delivery created by the event log's Replay
|
|
||||||
// action, so the pages can tell it from the delivery it repeats.
|
|
||||||
Replay bool `gorm:"not null;default:false" json:"replay"`
|
|
||||||
|
|
||||||
// Relations. No model marshals the record it belongs to:
|
|
||||||
// Event.Deliveries and Target.Deliveries lead back here, and the
|
|
||||||
// JSON could loop.
|
|
||||||
Event Event `json:"-"`
|
|
||||||
Target Target `json:"-"`
|
|
||||||
DeliveryResults []DeliveryResult `json:"deliveryResults,omitempty"`
|
DeliveryResults []DeliveryResult `json:"deliveryResults,omitempty"`
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,21 +1,10 @@
|
|||||||
package database
|
package database
|
||||||
|
|
||||||
import "gorm.io/gorm"
|
|
||||||
|
|
||||||
// DeliveryResult represents the result of a delivery attempt
|
// DeliveryResult represents the result of a delivery attempt
|
||||||
//
|
|
||||||
//nolint:lll // a struct tag cannot wrap
|
|
||||||
type DeliveryResult struct {
|
type DeliveryResult struct {
|
||||||
BaseModel
|
BaseModel
|
||||||
|
|
||||||
// DeliveryID and DeletedAt make up one index, in that order.
|
DeliveryID string `gorm:"type:uuid;not null;index" json:"deliveryId"`
|
||||||
// DeletedAt repeats the BaseModel field only to join it: GORM adds
|
|
||||||
// "deleted_at IS NULL" to almost every query, and where a column is
|
|
||||||
// matched against several values SQLite otherwise reads through the
|
|
||||||
// deleted_at index, which every live row matches.
|
|
||||||
DeliveryID string `gorm:"type:uuid;not null;index:idx_delivery_results_delivery_id,priority:1" json:"deliveryId"`
|
|
||||||
DeletedAt gorm.DeletedAt `gorm:"index:idx_delivery_results_delivery_id,priority:2" json:"deletedAt,omitzero"`
|
|
||||||
|
|
||||||
AttemptNum int `gorm:"not null" json:"attemptNum"`
|
AttemptNum int `gorm:"not null" json:"attemptNum"`
|
||||||
Success bool `json:"success"`
|
Success bool `json:"success"`
|
||||||
StatusCode int `json:"statusCode,omitempty"`
|
StatusCode int `json:"statusCode,omitempty"`
|
||||||
@@ -23,7 +12,6 @@ type DeliveryResult struct {
|
|||||||
Error string `json:"error,omitempty"`
|
Error string `json:"error,omitempty"`
|
||||||
Duration int64 `json:"durationMs"` // Duration in milliseconds
|
Duration int64 `json:"durationMs"` // Duration in milliseconds
|
||||||
|
|
||||||
// Relations. No model marshals the record it belongs to:
|
// Relations
|
||||||
// Delivery.DeliveryResults leads back here, and the JSON could loop.
|
Delivery Delivery `json:"delivery,omitzero"`
|
||||||
Delivery Delivery `json:"-"`
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,7 +15,6 @@ type Entrypoint struct {
|
|||||||
Description string `json:"description"`
|
Description string `json:"description"`
|
||||||
Active bool `gorm:"default:true" json:"active"`
|
Active bool `gorm:"default:true" json:"active"`
|
||||||
|
|
||||||
// Relations. No model marshals the record it belongs to:
|
// Relations
|
||||||
// Webhook.Entrypoints leads back here, and the JSON could loop.
|
Webhook Webhook `json:"webhook,omitzero"`
|
||||||
Webhook Webhook `json:"-"`
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,58 +1,36 @@
|
|||||||
package database
|
package database
|
||||||
|
|
||||||
import (
|
import "time"
|
||||||
"time"
|
|
||||||
|
|
||||||
"gorm.io/gorm"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Event represents a captured webhook event
|
// Event represents a captured webhook event
|
||||||
//
|
|
||||||
//nolint:lll // a struct tag cannot wrap
|
|
||||||
type Event struct {
|
type Event struct {
|
||||||
BaseModel
|
BaseModel
|
||||||
|
|
||||||
// CreatedAt and DeletedAt repeat the BaseModel fields only to index
|
// CreatedAt overrides BaseModel.CreatedAt only to add an index:
|
||||||
// them for retention, which finds events by age. Its lookups carry
|
// retention deletes events by age, so events.created_at is queried
|
||||||
// GORM's "deleted_at IS NULL" (see DeliveryResult) and compare
|
// on every sweep. The other tables keep the unindexed BaseModel
|
||||||
// created_at with <, so their index has deleted_at first: SQLite
|
// field.
|
||||||
// narrows by a < only on the last column it uses. Its final delete
|
CreatedAt time.Time `gorm:"index" json:"createdAt"`
|
||||||
// has no deleted_at condition and uses the index on created_at
|
|
||||||
// alone. The other tables keep the unindexed BaseModel created_at.
|
|
||||||
// DeletedAt is also the second column of the resubmitted_from_id
|
|
||||||
// index, for the reason DeliveryResult gives. The entrypoint_id
|
|
||||||
// index, for the webhook page's entrypoint list, has it second too,
|
|
||||||
// resubmitted_from_id third, and created_at last, which the list
|
|
||||||
// compares with a range.
|
|
||||||
CreatedAt time.Time `gorm:"index;index:idx_events_deleted_at_created_at,priority:2;index:idx_events_entrypoint_id,priority:4" json:"createdAt"`
|
|
||||||
DeletedAt gorm.DeletedAt `gorm:"index:idx_events_deleted_at_created_at,priority:1;index:idx_events_resubmitted_from_id,priority:2;index:idx_events_entrypoint_id,priority:2" json:"deletedAt,omitzero"`
|
|
||||||
|
|
||||||
WebhookID string `gorm:"type:uuid;not null" json:"webhookId"`
|
WebhookID string `gorm:"type:uuid;not null" json:"webhookId"`
|
||||||
EntrypointID string `gorm:"type:uuid;not null;index:idx_events_entrypoint_id,priority:1" json:"entrypointId"`
|
EntrypointID string `gorm:"type:uuid;not null" json:"entrypointId"`
|
||||||
|
|
||||||
// Request data. RawQuery is the receiving request's query string
|
// Request data
|
||||||
// as sent, without the leading "?".
|
|
||||||
Method string `gorm:"not null" json:"method"`
|
Method string `gorm:"not null" json:"method"`
|
||||||
RawQuery string `gorm:"type:text" json:"rawQuery"`
|
|
||||||
Headers string `gorm:"type:text" json:"headers"` // JSON
|
Headers string `gorm:"type:text" json:"headers"` // JSON
|
||||||
Body string `gorm:"type:text" json:"body"`
|
Body string `gorm:"type:text" json:"body"`
|
||||||
ContentType string `json:"contentType"`
|
ContentType string `json:"contentType"`
|
||||||
|
|
||||||
// BodyBytes is the size of Body in bytes, recorded when the event
|
|
||||||
// is stored, so that the recent events list, which reads only the
|
|
||||||
// start of each body, knows the whole body's size.
|
|
||||||
BodyBytes int64 `gorm:"not null" json:"bodyBytes"`
|
|
||||||
|
|
||||||
// ResubmittedFromID names the event this one was copied from by
|
// ResubmittedFromID names the event this one was copied from by
|
||||||
// an operator resubmit. It is nil for an event that arrived on
|
// an operator resubmit. It is nil for an event that arrived on
|
||||||
// the receiver. It is not a foreign key: the source event can be
|
// the receiver, which is every event created before the column
|
||||||
|
// existed. It is not a foreign key: the source event can be
|
||||||
// reaped by retention while its copies remain, and the id is
|
// reaped by retention while its copies remain, and the id is
|
||||||
// kept as the record of where the copy came from either way.
|
// kept as the record of where the copy came from either way.
|
||||||
ResubmittedFromID *string `gorm:"type:uuid;index:idx_events_resubmitted_from_id,priority:1;index:idx_events_entrypoint_id,priority:3" json:"resubmittedFromId,omitempty"`
|
ResubmittedFromID *string `gorm:"type:uuid;index" json:"resubmittedFromId,omitempty"`
|
||||||
|
|
||||||
// Relations. No model marshals the record it belongs to, so
|
// Relations
|
||||||
// Webhook and Entrypoint are left out of the JSON.
|
Webhook Webhook `json:"webhook,omitzero"`
|
||||||
Webhook Webhook `json:"-"`
|
Entrypoint Entrypoint `json:"entrypoint,omitzero"`
|
||||||
Entrypoint Entrypoint `json:"-"`
|
|
||||||
Deliveries []Delivery `json:"deliveries,omitempty"`
|
Deliveries []Delivery `json:"deliveries,omitempty"`
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,127 +0,0 @@
|
|||||||
package database_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/google/uuid"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestPreloadedModelsMarshalWithoutTheirParent pins that a child's
|
|
||||||
// reference to the record it belongs to is left out of the JSON, so a
|
|
||||||
// webhook and its targets cannot marshal each other in a loop, and that
|
|
||||||
// GORM still preloads that reference, since it ignores json tags.
|
|
||||||
func TestPreloadedModelsMarshalWithoutTheirParent(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := startedTestDB(t)
|
|
||||||
|
|
||||||
stored := database.Webhook{
|
|
||||||
UserID: uuid.New().String(),
|
|
||||||
Name: testWebhookName,
|
|
||||||
Entrypoints: []database.Entrypoint{{Path: uuid.New().String()}},
|
|
||||||
Targets: []database.Target{{
|
|
||||||
Name: "log",
|
|
||||||
Type: database.TargetTypeLog,
|
|
||||||
MaxAttempts: 1,
|
|
||||||
}},
|
|
||||||
}
|
|
||||||
require.NoError(t, db.Create(&stored).Error)
|
|
||||||
|
|
||||||
entrypointID := stored.Entrypoints[0].ID
|
|
||||||
targetID := stored.Targets[0].ID
|
|
||||||
|
|
||||||
var webhook database.Webhook
|
|
||||||
|
|
||||||
require.NoError(t, db.
|
|
||||||
Preload("Entrypoints.Webhook").
|
|
||||||
Preload("Targets.Webhook").
|
|
||||||
First(&webhook, "id = ?", stored.ID).Error)
|
|
||||||
|
|
||||||
require.Len(t, webhook.Entrypoints, 1)
|
|
||||||
require.Len(t, webhook.Targets, 1)
|
|
||||||
assert.Equal(t, stored.ID, webhook.Entrypoints[0].Webhook.ID)
|
|
||||||
assert.Equal(t, stored.ID, webhook.Targets[0].Webhook.ID)
|
|
||||||
|
|
||||||
encoded := marshalModel(t, webhook)
|
|
||||||
|
|
||||||
assert.Contains(t, encoded, entrypointID)
|
|
||||||
assert.Contains(t, encoded, targetID)
|
|
||||||
|
|
||||||
// Each child holds the parent's id as its webhookId, so the parent
|
|
||||||
// is looked for by its own id field.
|
|
||||||
parentIDField := `"id":"` + stored.ID + `"`
|
|
||||||
|
|
||||||
assert.NotContains(t, marshalModel(t, webhook.Entrypoints[0]), parentIDField)
|
|
||||||
assert.NotContains(t, marshalModel(t, webhook.Targets[0]), parentIDField)
|
|
||||||
|
|
||||||
var target database.Target
|
|
||||||
|
|
||||||
require.NoError(t, db.
|
|
||||||
Preload("Webhook").
|
|
||||||
First(&target, "id = ?", targetID).Error)
|
|
||||||
|
|
||||||
assert.Equal(t, stored.ID, target.Webhook.ID)
|
|
||||||
|
|
||||||
encoded = marshalModel(t, target)
|
|
||||||
|
|
||||||
assert.Contains(t, encoded, stored.ID)
|
|
||||||
assert.NotContains(t, encoded, parentIDField)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestModelsMarshalWithoutTheirParent covers the other references to a
|
|
||||||
// parent: each model is built with its parent set, and the parent's id
|
|
||||||
// must not appear in the JSON.
|
|
||||||
func TestModelsMarshalWithoutTheirParent(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
parent := database.BaseModel{ID: uuid.New().String()}
|
|
||||||
|
|
||||||
cases := []struct {
|
|
||||||
name string
|
|
||||||
model any
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "Webhook.User",
|
|
||||||
model: database.Webhook{User: database.User{BaseModel: parent}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "APIKey.User",
|
|
||||||
model: database.APIKey{User: database.User{BaseModel: parent}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "Delivery.Event",
|
|
||||||
model: database.Delivery{Event: database.Event{BaseModel: parent}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "Delivery.Target",
|
|
||||||
model: database.Delivery{Target: database.Target{BaseModel: parent}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "DeliveryResult.Delivery",
|
|
||||||
model: database.DeliveryResult{
|
|
||||||
Delivery: database.Delivery{BaseModel: parent},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "Event.Webhook",
|
|
||||||
model: database.Event{Webhook: database.Webhook{BaseModel: parent}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "Event.Entrypoint",
|
|
||||||
model: database.Event{
|
|
||||||
Entrypoint: database.Entrypoint{BaseModel: parent},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range cases {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
assert.NotContains(t, marshalModel(t, tc.model), parent.ID)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -29,10 +29,8 @@ func marshalModel(t *testing.T, v any) string {
|
|||||||
// these tags the first handler that marshals a model serialises the
|
// these tags the first handler that marshals a model serialises the
|
||||||
// secret with it. Each field below is a live credential:
|
// secret with it. Each field below is a live credential:
|
||||||
//
|
//
|
||||||
// - Target.URL is an incoming-webhook URL whose path segments are
|
// - Target.Config holds an incoming-webhook URL whose path segments
|
||||||
// the bearer token.
|
// are the bearer token.
|
||||||
// - Target.Headers holds header values, routinely authorization
|
|
||||||
// tokens.
|
|
||||||
// - APIKey.Key is a bearer token outright.
|
// - APIKey.Key is a bearer token outright.
|
||||||
// - Setting.Value holds the session encryption key.
|
// - Setting.Value holds the session encryption key.
|
||||||
// - User.Password holds the Argon2 hash, and was already tagged.
|
// - User.Password holds the Argon2 hash, and was already tagged.
|
||||||
@@ -46,21 +44,11 @@ func TestModelsDoNotMarshalTheirSecrets(t *testing.T) {
|
|||||||
model any
|
model any
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "target url",
|
name: "target config",
|
||||||
model: database.Target{
|
model: database.Target{
|
||||||
Name: keptField,
|
Name: keptField,
|
||||||
Type: database.TargetTypeSlack,
|
Type: database.TargetTypeSlack,
|
||||||
URL: "https://h/s/" + marker,
|
Config: `{"webhookUrl":"https://h/s/` + marker + `"}`,
|
||||||
MaxAttempts: 1,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "target headers",
|
|
||||||
model: database.Target{
|
|
||||||
Name: keptField,
|
|
||||||
Type: database.TargetTypeHTTP,
|
|
||||||
Headers: map[string]string{"Authorization": marker},
|
|
||||||
MaxAttempts: 1,
|
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -98,10 +86,10 @@ func TestModelsDoNotMarshalTheirSecrets(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestWebhookMarshalsNoTargetURL covers the nested case: a webhook
|
// TestWebhookMarshalsNoTargetConfig covers the nested case: a webhook
|
||||||
// marshalled with its targets preloaded must not carry the credential
|
// marshalled with its targets preloaded must not carry the credential
|
||||||
// through the association either.
|
// through the association either.
|
||||||
func TestWebhookMarshalsNoTargetURL(t *testing.T) {
|
func TestWebhookMarshalsNoTargetConfig(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const marker = "QQNESTEDMARKERQQ"
|
const marker = "QQNESTEDMARKERQQ"
|
||||||
@@ -109,9 +97,8 @@ func TestWebhookMarshalsNoTargetURL(t *testing.T) {
|
|||||||
encoded := marshalModel(t, database.Webhook{
|
encoded := marshalModel(t, database.Webhook{
|
||||||
Name: keptField,
|
Name: keptField,
|
||||||
Targets: []database.Target{{
|
Targets: []database.Target{{
|
||||||
Name: "slack",
|
Name: "slack",
|
||||||
URL: "https://h/s/" + marker,
|
Config: `{"webhookUrl":"https://h/s/` + marker + `"}`,
|
||||||
MaxAttempts: 1,
|
|
||||||
}},
|
}},
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,5 @@
|
|||||||
package database
|
package database
|
||||||
|
|
||||||
import "time"
|
|
||||||
|
|
||||||
// TargetType represents the type of delivery target
|
// TargetType represents the type of delivery target
|
||||||
type TargetType string
|
type TargetType string
|
||||||
|
|
||||||
@@ -13,9 +11,7 @@ const (
|
|||||||
TargetTypeSlack TargetType = "slack"
|
TargetTypeSlack TargetType = "slack"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Target represents a delivery target for a webhook. Its settings
|
// Target represents a delivery target for a webhook
|
||||||
// are the fields from URL to ArchiveRotation; each type uses some of
|
|
||||||
// them, and the ones a type does not use stay empty.
|
|
||||||
type Target struct {
|
type Target struct {
|
||||||
BaseModel
|
BaseModel
|
||||||
|
|
||||||
@@ -24,45 +20,21 @@ type Target struct {
|
|||||||
Type TargetType `gorm:"not null" json:"type"`
|
Type TargetType `gorm:"not null" json:"type"`
|
||||||
Active bool `gorm:"default:true" json:"active"`
|
Active bool `gorm:"default:true" json:"active"`
|
||||||
|
|
||||||
// URL is where an http or slack target sends each event.
|
// Configuration fields (JSON stored based on type).
|
||||||
//
|
//
|
||||||
// json:"-" because the URL is the target's credential: a Slack
|
// json:"-" because the blob holds the target's credential — a
|
||||||
// incoming-webhook URL, or an http destination whose path segments
|
// Slack incoming-webhook URL, or an http destination whose path
|
||||||
// are the secret. delivery.TargetView is the masking barrier for
|
// segments are the secret. delivery.TargetView is the masking
|
||||||
// the HTML path; this tag is the barrier for any handler that
|
// barrier for the HTML path; this tag is the barrier for any
|
||||||
// marshals the model itself.
|
// handler that marshals the model itself.
|
||||||
URL string `json:"-"`
|
Config string `gorm:"type:text" json:"-"` // JSON configuration
|
||||||
|
|
||||||
// Headers is an http target's extra request headers, name to
|
// For HTTP targets (max_retries=0 means fire-and-forget,
|
||||||
// value, stored as JSON text. json:"-" for the same reason as URL:
|
// >0 enables retries with backoff)
|
||||||
// a header value is routinely an authorization token.
|
MaxRetries int `json:"maxRetries,omitempty"`
|
||||||
Headers map[string]string `gorm:"serializer:json" json:"-"`
|
MaxQueueSize int `json:"maxQueueSize,omitempty"`
|
||||||
|
|
||||||
// TimeoutSeconds is an http target's per-request timeout; 0 uses
|
// Relations
|
||||||
// the delivery engine's shared client timeout.
|
Webhook Webhook `json:"webhook,omitzero"`
|
||||||
TimeoutSeconds int `json:"timeoutSeconds,omitempty"`
|
|
||||||
|
|
||||||
// ForwardQuery passes each event's query string on to an http
|
|
||||||
// target, appended to URL. Off, URL is sent exactly as stored.
|
|
||||||
ForwardQuery bool `json:"forwardQuery,omitempty"`
|
|
||||||
|
|
||||||
// ArchiveExpiry is how long a database target keeps each archived
|
|
||||||
// event; 0 keeps them forever.
|
|
||||||
ArchiveExpiry time.Duration `json:"archiveExpiry,omitempty"`
|
|
||||||
|
|
||||||
// ArchiveRotation is how often a database target starts a new
|
|
||||||
// archive file: none, monthly, daily or hourly.
|
|
||||||
ArchiveRotation string `json:"archiveRotation,omitempty"`
|
|
||||||
|
|
||||||
// MaxAttempts is how many times an HTTP or Slack target attempts
|
|
||||||
// each delivery in all, at least 1. 1 is a single attempt with no
|
|
||||||
// circuit breaker; above 1, a failed attempt is retried with
|
|
||||||
// backoff behind the breaker. The table's check constraint refuses
|
|
||||||
// a target saved with less than 1, whatever its type.
|
|
||||||
MaxAttempts int `gorm:"not null;check:max_attempts >= 1" json:"maxAttempts,omitempty"`
|
|
||||||
|
|
||||||
// Relations. No model marshals the record it belongs to:
|
|
||||||
// Webhook.Targets leads back here, and the JSON could loop.
|
|
||||||
Webhook Webhook `json:"-"`
|
|
||||||
Deliveries []Delivery `json:"deliveries,omitempty"`
|
Deliveries []Delivery `json:"deliveries,omitempty"`
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,63 +0,0 @@
|
|||||||
package database_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/google/uuid"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// newHTTPTarget returns an http target that makes maxAttempts attempts
|
|
||||||
// at each delivery.
|
|
||||||
func newHTTPTarget(maxAttempts int) *database.Target {
|
|
||||||
return &database.Target{
|
|
||||||
WebhookID: uuid.New().String(),
|
|
||||||
Name: "target",
|
|
||||||
Type: database.TargetTypeHTTP,
|
|
||||||
URL: "https://example.com/",
|
|
||||||
MaxAttempts: maxAttempts,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestTargetCreate_RefusesZeroAttempts relies on the targets table
|
|
||||||
// alone: nothing in the model checks the number, so the check
|
|
||||||
// constraint is what keeps a target with 0 attempts out of the
|
|
||||||
// database.
|
|
||||||
func TestTargetCreate_RefusesZeroAttempts(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := startedTestDB(t)
|
|
||||||
|
|
||||||
err := db.Create(newHTTPTarget(0)).Error
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.Contains(t, err.Error(), "CHECK constraint failed")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTargetCreate_AcceptsOneAttempt(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := startedTestDB(t)
|
|
||||||
|
|
||||||
require.NoError(t, db.Create(newHTTPTarget(1)).Error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestTargetHeaders_StoredAsJSON reads the headers column back as
|
|
||||||
// text, to show it holds the header map as JSON.
|
|
||||||
func TestTargetHeaders_StoredAsJSON(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := startedTestDB(t)
|
|
||||||
|
|
||||||
target := newHTTPTarget(1)
|
|
||||||
target.Headers = map[string]string{"Authorization": "Bearer x"}
|
|
||||||
require.NoError(t, db.Create(target).Error)
|
|
||||||
|
|
||||||
var headers string
|
|
||||||
|
|
||||||
require.NoError(t, db.Raw(
|
|
||||||
"SELECT headers FROM targets WHERE id = ?", target.ID,
|
|
||||||
).Scan(&headers).Error)
|
|
||||||
assert.JSONEq(t, `{"Authorization":"Bearer x"}`, headers)
|
|
||||||
}
|
|
||||||
@@ -1,136 +0,0 @@
|
|||||||
package database
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"gorm.io/gorm"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The running totals in a webhook's event database keep the webhook
|
|
||||||
// page's lifetime figures right after retention has removed the rows
|
|
||||||
// they count, and let the page show them without counting every row.
|
|
||||||
// Each total changes in the transaction that writes or deletes the
|
|
||||||
// rows it counts.
|
|
||||||
|
|
||||||
// EventTotals is the single row counting a webhook's events: every
|
|
||||||
// event ever stored, how many of them retention has deleted, and when
|
|
||||||
// the newest arrived, which retention leaves as it is.
|
|
||||||
type EventTotals struct {
|
|
||||||
ID int64 `gorm:"primaryKey"`
|
|
||||||
|
|
||||||
Events int64 `gorm:"not null"`
|
|
||||||
EventsRemoved int64 `gorm:"not null"`
|
|
||||||
|
|
||||||
// LastEventAt is when the newest event arrived, or nil before the
|
|
||||||
// first.
|
|
||||||
LastEventAt *time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// TableName names the table AddEventTotals updates.
|
|
||||||
func (EventTotals) TableName() string {
|
|
||||||
return "event_totals"
|
|
||||||
}
|
|
||||||
|
|
||||||
// TargetTotals is one row per target counting its deliveries: every
|
|
||||||
// delivery ever created, how many became delivered and how many
|
|
||||||
// failed, and how many deliveries and failed deliveries retention has
|
|
||||||
// deleted. The webhook's delivery figures are these rows summed.
|
|
||||||
type TargetTotals struct {
|
|
||||||
TargetID string `gorm:"type:uuid;primaryKey"`
|
|
||||||
|
|
||||||
Deliveries int64 `gorm:"not null"`
|
|
||||||
Delivered int64 `gorm:"not null"`
|
|
||||||
Failed int64 `gorm:"not null"`
|
|
||||||
|
|
||||||
DeliveriesRemoved int64 `gorm:"not null"`
|
|
||||||
FailedRemoved int64 `gorm:"not null"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// TableName names the table AddTargetTotals updates.
|
|
||||||
func (TargetTotals) TableName() string {
|
|
||||||
return "target_totals"
|
|
||||||
}
|
|
||||||
|
|
||||||
// EntrypointTotals is one row per entrypoint, created by the first
|
|
||||||
// event that arrives on its URL: when the newest such event arrived,
|
|
||||||
// which retention leaves as it is. A resubmitted copy did not arrive
|
|
||||||
// on the URL and does not change it.
|
|
||||||
type EntrypointTotals struct {
|
|
||||||
EntrypointID string `gorm:"type:uuid;primaryKey"`
|
|
||||||
|
|
||||||
LastEventAt time.Time `gorm:"not null"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// TableName names the table AddEntrypointTotals updates.
|
|
||||||
func (EntrypointTotals) TableName() string {
|
|
||||||
return "entrypoint_totals"
|
|
||||||
}
|
|
||||||
|
|
||||||
// AddEventTotals adds each count in add to the webhook's event totals,
|
|
||||||
// and records add.LastEventAt as when the newest event arrived if it is
|
|
||||||
// set. Call it on the transaction that writes or deletes the events it
|
|
||||||
// counts.
|
|
||||||
func AddEventTotals(tx *gorm.DB, add EventTotals) error {
|
|
||||||
err := tx.Exec(
|
|
||||||
`UPDATE event_totals SET
|
|
||||||
events = events + ?,
|
|
||||||
events_removed = events_removed + ?,
|
|
||||||
last_event_at = coalesce(?, last_event_at)`,
|
|
||||||
add.Events, add.EventsRemoved, add.LastEventAt,
|
|
||||||
).Error
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("adding to event totals: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// AddTargetTotals adds each count in add to the totals of the target
|
|
||||||
// add.TargetID names, creating its row the first time. Call it on the
|
|
||||||
// transaction that writes or deletes the deliveries it counts.
|
|
||||||
func AddTargetTotals(tx *gorm.DB, add TargetTotals) error {
|
|
||||||
err := tx.Exec(
|
|
||||||
`INSERT INTO target_totals (target_id, deliveries, delivered,
|
|
||||||
failed, deliveries_removed, failed_removed)
|
|
||||||
VALUES (?, ?, ?, ?, ?, ?)
|
|
||||||
ON CONFLICT (target_id) DO UPDATE SET
|
|
||||||
deliveries = deliveries + excluded.deliveries,
|
|
||||||
delivered = delivered + excluded.delivered,
|
|
||||||
failed = failed + excluded.failed,
|
|
||||||
deliveries_removed =
|
|
||||||
deliveries_removed + excluded.deliveries_removed,
|
|
||||||
failed_removed = failed_removed + excluded.failed_removed`,
|
|
||||||
add.TargetID, add.Deliveries, add.Delivered,
|
|
||||||
add.Failed, add.DeliveriesRemoved, add.FailedRemoved,
|
|
||||||
).Error
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"adding to totals of target %s: %w", add.TargetID, err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// AddEntrypointTotals records add.LastEventAt as when the newest event
|
|
||||||
// arrived on the URL of the entrypoint add.EntrypointID names, creating
|
|
||||||
// its row the first time. Call it on the transaction that stores the
|
|
||||||
// event.
|
|
||||||
func AddEntrypointTotals(tx *gorm.DB, add EntrypointTotals) error {
|
|
||||||
err := tx.Exec(
|
|
||||||
`INSERT INTO entrypoint_totals (entrypoint_id, last_event_at)
|
|
||||||
VALUES (?, ?)
|
|
||||||
ON CONFLICT (entrypoint_id) DO UPDATE SET
|
|
||||||
last_event_at = excluded.last_event_at`,
|
|
||||||
add.EntrypointID, add.LastEventAt,
|
|
||||||
).Error
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"adding to totals of entrypoint %s: %w",
|
|
||||||
add.EntrypointID, err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,58 +1,13 @@
|
|||||||
package database
|
package database
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"gorm.io/gorm"
|
|
||||||
)
|
|
||||||
|
|
||||||
// MaxUsernameBytes is the longest username, in bytes, that a user may
|
|
||||||
// have. The same number appears in the check constraint on
|
|
||||||
// User.Username, because a struct tag cannot reference a constant.
|
|
||||||
//
|
|
||||||
// A login stores the username in the session cookie, and both
|
|
||||||
// securecookie and browsers refuse a cookie value past about 4096
|
|
||||||
// bytes. That value is the session base64-encoded twice, so it holds
|
|
||||||
// 4096 × 3/4 × 3/4 = 2304 bytes of session, and the signature,
|
|
||||||
// timestamp and the session's other values take about 270 of those: a
|
|
||||||
// username longer than about 2030 bytes can never log in. The limit is
|
|
||||||
// about half that, so the session can carry more values later without
|
|
||||||
// locking out an account whose username is already at the limit.
|
|
||||||
const MaxUsernameBytes = 1024
|
|
||||||
|
|
||||||
// ErrUsernameTooLong is returned when a user is saved with a username
|
|
||||||
// longer than MaxUsernameBytes.
|
|
||||||
var ErrUsernameTooLong = errors.New("username is too long")
|
|
||||||
|
|
||||||
// User represents a user of the webhooker service
|
// User represents a user of the webhooker service
|
||||||
//
|
|
||||||
//nolint:lll // a struct tag cannot wrap
|
|
||||||
type User struct {
|
type User struct {
|
||||||
BaseModel
|
BaseModel
|
||||||
|
|
||||||
Username string `gorm:"uniqueIndex;not null;check:length(CAST(username AS BLOB)) <= 1024" json:"username"`
|
Username string `gorm:"uniqueIndex;not null" json:"username"`
|
||||||
Password string `gorm:"not null" json:"-"` // Argon2 hashed
|
Password string `gorm:"not null" json:"-"` // Argon2 hashed
|
||||||
|
|
||||||
// Relations
|
// Relations
|
||||||
Webhooks []Webhook `json:"webhooks,omitempty"`
|
Webhooks []Webhook `json:"webhooks,omitempty"`
|
||||||
APIKeys []APIKey `json:"apiKeys,omitempty"`
|
APIKeys []APIKey `json:"apiKeys,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// BeforeSave rejects a username longer than MaxUsernameBytes when a whole
|
|
||||||
// User is created or saved, so those calls get ErrUsernameTooLong rather
|
|
||||||
// than the database's constraint error. A column update such as
|
|
||||||
// Update("username", ...) is caught only by the check constraint, as is
|
|
||||||
// any path that writes the table without this model.
|
|
||||||
func (u *User) BeforeSave(_ *gorm.DB) error {
|
|
||||||
if len(u.Username) > MaxUsernameBytes {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"%w: %d bytes, limit is %d",
|
|
||||||
ErrUsernameTooLong,
|
|
||||||
len(u.Username),
|
|
||||||
MaxUsernameBytes,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,65 +0,0 @@
|
|||||||
package database_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/google/uuid"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// usernameAtLimit is exactly MaxUsernameBytes long, built from a
|
|
||||||
// two-byte character. A check that counted characters rather than bytes
|
|
||||||
// would see half the length and let the one-byte-longer name through.
|
|
||||||
func usernameAtLimit() string {
|
|
||||||
return strings.Repeat("é", database.MaxUsernameBytes/2)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestUserCreate_RejectsOverlongUsername(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := startedTestDB(t)
|
|
||||||
|
|
||||||
err := db.Create(&database.User{
|
|
||||||
Username: usernameAtLimit() + "x",
|
|
||||||
Password: "hash",
|
|
||||||
}).Error
|
|
||||||
|
|
||||||
require.ErrorIs(t, err, database.ErrUsernameTooLong)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestUserCreate_AcceptsUsernameAtLimit(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := startedTestDB(t)
|
|
||||||
|
|
||||||
require.NoError(t, db.Create(&database.User{
|
|
||||||
Username: usernameAtLimit(),
|
|
||||||
Password: "hash",
|
|
||||||
}).Error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestUsersTable_EnforcesUsernameLimitWithoutTheModel inserts with raw
|
|
||||||
// SQL, as a path that bypassed User.BeforeSave would, so only the
|
|
||||||
// table's check constraint stands between it and an over-long
|
|
||||||
// username. Accepting the name at the limit and refusing the next byte
|
|
||||||
// also pins the constraint's number to MaxUsernameBytes.
|
|
||||||
func TestUsersTable_EnforcesUsernameLimitWithoutTheModel(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := startedTestDB(t)
|
|
||||||
|
|
||||||
insert := "INSERT INTO users (id, username, password) VALUES (?, ?, ?)"
|
|
||||||
|
|
||||||
require.NoError(t, db.Exec(
|
|
||||||
insert, uuid.New().String(), usernameAtLimit(), "hash",
|
|
||||||
).Error)
|
|
||||||
|
|
||||||
err := db.Exec(
|
|
||||||
insert, uuid.New().String(), usernameAtLimit()+"x", "hash",
|
|
||||||
).Error
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.Contains(t, err.Error(), "CHECK constraint failed")
|
|
||||||
}
|
|
||||||
@@ -4,13 +4,27 @@ import (
|
|||||||
"math"
|
"math"
|
||||||
"strconv"
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"gorm.io/gorm"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
// DefaultRetentionDays is the event retention period the new
|
// DefaultRetentionDays is the event retention period applied to a
|
||||||
// webhook form applies when its retention field is left blank.
|
// webhook created without an explicit retention value. It is the
|
||||||
|
// single source of truth for that policy and must stay in sync
|
||||||
|
// with the `gorm:"default:30"` column default on
|
||||||
|
// Webhook.RetentionDays below; a struct tag cannot reference a
|
||||||
|
// constant, so a test asserts the two agree.
|
||||||
DefaultRetentionDays = 30
|
DefaultRetentionDays = 30
|
||||||
|
|
||||||
|
// RetentionForeverDays is the sentinel RetentionDays value meaning
|
||||||
|
// "retain events forever". Users express that intent as 0, which
|
||||||
|
// Webhook.BeforeSave rewrites to this value: the column default
|
||||||
|
// substitutes DefaultRetentionDays for a zero value at insert
|
||||||
|
// time, so a zero can never survive a round trip to the database.
|
||||||
|
// Nothing outside this file may hardcode the number.
|
||||||
|
RetentionForeverDays = 365 * 1000
|
||||||
|
|
||||||
// MaxFiniteRetentionDays is the largest finite retention period the
|
// MaxFiniteRetentionDays is the largest finite retention period the
|
||||||
// reaper's cutoff arithmetic can represent, and therefore the
|
// reaper's cutoff arithmetic can represent, and therefore the
|
||||||
// largest one a caller may request. It is derived from that
|
// largest one a caller may request. It is derived from that
|
||||||
@@ -24,7 +38,9 @@ const (
|
|||||||
// turns the cutoff into a timestamp in the far future that matches
|
// turns the cutoff into a timestamp in the far future that matches
|
||||||
// every row in the webhook's database. That is why this bound is
|
// every row in the webhook's database. That is why this bound is
|
||||||
// enforced on input and why retentionCutoff saturates underneath
|
// enforced on input and why retentionCutoff saturates underneath
|
||||||
// it.
|
// it. Note that RetentionForeverDays deliberately sits above this
|
||||||
|
// ceiling: such webhooks are skipped before any cutoff is
|
||||||
|
// computed, and never reach the arithmetic at all.
|
||||||
MaxFiniteRetentionDays = int(
|
MaxFiniteRetentionDays = int(
|
||||||
math.MaxInt64 / int64(time.Hour) / hoursPerDay,
|
math.MaxInt64 / int64(time.Hour) / hoursPerDay,
|
||||||
)
|
)
|
||||||
@@ -32,10 +48,12 @@ const (
|
|||||||
|
|
||||||
// Webhook represents a webhook processing unit that groups entrypoints and targets
|
// Webhook represents a webhook processing unit that groups entrypoints and targets
|
||||||
//
|
//
|
||||||
// Every method below takes a pointer receiver, so handlers put a
|
// Every method below takes a pointer receiver. BeforeSave has to,
|
||||||
// *Webhook into template data: html/template cannot call a pointer
|
// because it mutates the record and GORM only invokes hooks declared
|
||||||
// method on a value held in a map, because a map element is not
|
// that way; the display helpers follow suit so the receiver kinds do
|
||||||
// addressable.
|
// not mix. Handlers therefore put a *Webhook into template data:
|
||||||
|
// html/template cannot call a pointer method on a value held in a map,
|
||||||
|
// because a map element is not addressable.
|
||||||
type Webhook struct {
|
type Webhook struct {
|
||||||
BaseModel
|
BaseModel
|
||||||
|
|
||||||
@@ -43,28 +61,47 @@ type Webhook struct {
|
|||||||
Name string `gorm:"not null" json:"name"`
|
Name string `gorm:"not null" json:"name"`
|
||||||
Description string `json:"description"`
|
Description string `json:"description"`
|
||||||
|
|
||||||
// RetentionDays is the number of days to retain events, and 0
|
// RetentionDays is the number of days to retain events. A value of
|
||||||
// means retain them forever.
|
// RetentionForeverDays means retain forever. The column default
|
||||||
RetentionDays int `json:"retentionDays"`
|
// must equal DefaultRetentionDays.
|
||||||
|
RetentionDays int `gorm:"default:30" json:"retentionDays"`
|
||||||
|
|
||||||
// Relations. No model marshals the record it belongs to:
|
// Relations
|
||||||
// User.Webhooks leads back here, and the JSON could loop.
|
User User `json:"user,omitzero"`
|
||||||
User User `json:"-"`
|
|
||||||
Entrypoints []Entrypoint `json:"entrypoints,omitempty"`
|
Entrypoints []Entrypoint `json:"entrypoints,omitempty"`
|
||||||
Targets []Target `json:"targets,omitempty"`
|
Targets []Target `json:"targets,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// retainsForever reports whether a RetentionDays value means "keep
|
// BeforeSave normalises RetentionDays on every insert and update. A
|
||||||
// events indefinitely", which 0 does. It is the single definition of
|
// non-positive value is the user's way of asking for "retain forever",
|
||||||
// that question, shared by Webhook.RetainsForever and by the reaper's
|
// which is stored as the RetentionForeverDays sentinel.
|
||||||
|
//
|
||||||
|
// This has to happen in a hook rather than at the call sites. GORM
|
||||||
|
// substitutes the column default (DefaultRetentionDays) for a zero
|
||||||
|
// value while building the insert statement, which runs after
|
||||||
|
// BeforeSave; rewriting any later than this loses that race and the
|
||||||
|
// row lands at 30 days. Living on the model also means a future call
|
||||||
|
// site — a REST API, a fixture, a migration — cannot bypass it.
|
||||||
|
func (w *Webhook) BeforeSave(_ *gorm.DB) error {
|
||||||
|
if w.RetentionDays <= 0 {
|
||||||
|
w.RetentionDays = RetentionForeverDays
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// retainsForever reports whether a stored RetentionDays value means
|
||||||
|
// "keep events indefinitely". It is the single definition of that
|
||||||
|
// question, shared by Webhook.RetainsForever and by the reaper's
|
||||||
// cutoff computation so the two cannot disagree about which webhooks
|
// cutoff computation so the two cannot disagree about which webhooks
|
||||||
// are exempt from reaping.
|
// are exempt from reaping.
|
||||||
//
|
//
|
||||||
// A negative value, which the forms refuse, is read the same way: as a
|
// It accepts the RetentionForeverDays sentinel written by BeforeSave
|
||||||
// day count it would put the cutoff in the future, where it matches
|
// and, defensively, the non-positive values that rows written before
|
||||||
// every event the webhook has.
|
// the sentinel existed may still carry.
|
||||||
func retainsForever(retentionDays int) bool {
|
func retainsForever(retentionDays int) bool {
|
||||||
return retentionDays <= 0
|
return retentionDays <= 0 ||
|
||||||
|
retentionDays >= RetentionForeverDays
|
||||||
}
|
}
|
||||||
|
|
||||||
// RetainsForever reports whether this webhook's events are kept
|
// RetainsForever reports whether this webhook's events are kept
|
||||||
@@ -73,16 +110,8 @@ func (w *Webhook) RetainsForever() bool {
|
|||||||
return retainsForever(w.RetentionDays)
|
return retainsForever(w.RetentionDays)
|
||||||
}
|
}
|
||||||
|
|
||||||
// RetentionCutoff returns the time before which this webhook's events
|
|
||||||
// have expired, as the reaper computes it, and false when the webhook
|
|
||||||
// retains them forever.
|
|
||||||
func (w *Webhook) RetentionCutoff(now time.Time) (time.Time, bool) {
|
|
||||||
return retentionCutoff(now, w.RetentionDays)
|
|
||||||
}
|
|
||||||
|
|
||||||
// RetentionLabel returns the webhook's retention policy as display
|
// RetentionLabel returns the webhook's retention policy as display
|
||||||
// text: "forever" rather than "0 days" for a webhook that keeps its
|
// text, so that no template has to know about the sentinel value.
|
||||||
// events.
|
|
||||||
func (w *Webhook) RetentionLabel() string {
|
func (w *Webhook) RetentionLabel() string {
|
||||||
if w.RetainsForever() {
|
if w.RetainsForever() {
|
||||||
return "forever"
|
return "forever"
|
||||||
|
|||||||
@@ -2,6 +2,8 @@ package database_test
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"reflect"
|
||||||
|
"strconv"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -45,7 +47,8 @@ func storedRetention(t *testing.T, db *gorm.DB, id string) int {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// newWebhookWithRetention creates a webhook through the ordinary Create
|
// newWebhookWithRetention creates a webhook through the ordinary Create
|
||||||
// path, as the new webhook form does.
|
// path, so the BeforeSave hook and the GORM column default both apply
|
||||||
|
// exactly as they do in production.
|
||||||
func newWebhookWithRetention(
|
func newWebhookWithRetention(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
db *gorm.DB,
|
db *gorm.DB,
|
||||||
@@ -64,7 +67,7 @@ func newWebhookWithRetention(
|
|||||||
return wh.ID
|
return wh.ID
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestWebhookRetentionDays_ZeroIsStoredAsZero(t *testing.T) {
|
func TestWebhookBeforeSave_ZeroBecomesForeverSentinel(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
db := startedTestDB(t)
|
db := startedTestDB(t)
|
||||||
@@ -72,10 +75,31 @@ func TestWebhookRetentionDays_ZeroIsStoredAsZero(t *testing.T) {
|
|||||||
wh := &database.Webhook{RetentionDays: 0}
|
wh := &database.Webhook{RetentionDays: 0}
|
||||||
id := newWebhookWithRetention(t, db, wh)
|
id := newWebhookWithRetention(t, db, wh)
|
||||||
|
|
||||||
assert.Equal(t, 0, storedRetention(t, db, id))
|
assert.Equal(
|
||||||
|
t,
|
||||||
|
database.RetentionForeverDays,
|
||||||
|
storedRetention(t, db, id),
|
||||||
|
"a zero retention must be stored as the sentinel, "+
|
||||||
|
"not replaced by the column default",
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestWebhookRetentionDays_DayCountIsStored(t *testing.T) {
|
func TestWebhookBeforeSave_NegativeBecomesForeverSentinel(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
db := startedTestDB(t)
|
||||||
|
|
||||||
|
wh := &database.Webhook{RetentionDays: -5}
|
||||||
|
id := newWebhookWithRetention(t, db, wh)
|
||||||
|
|
||||||
|
assert.Equal(
|
||||||
|
t,
|
||||||
|
database.RetentionForeverDays,
|
||||||
|
storedRetention(t, db, id),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWebhookBeforeSave_PositiveIsPreserved(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
db := startedTestDB(t)
|
db := startedTestDB(t)
|
||||||
@@ -86,9 +110,10 @@ func TestWebhookRetentionDays_DayCountIsStored(t *testing.T) {
|
|||||||
assert.Equal(t, 7, storedRetention(t, db, id))
|
assert.Equal(t, 7, storedRetention(t, db, id))
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestWebhookRetentionDays_UpdateToZeroIsStoredAsZero changes a day
|
// TestWebhookBeforeSave_UpdateToZeroBecomesSentinel proves the hook
|
||||||
// count to 0 through the same Save call the edit handler makes.
|
// fires on update as well as insert, via the same Save call the edit
|
||||||
func TestWebhookRetentionDays_UpdateToZeroIsStoredAsZero(t *testing.T) {
|
// handler makes.
|
||||||
|
func TestWebhookBeforeSave_UpdateToZeroBecomesSentinel(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
db := startedTestDB(t)
|
db := startedTestDB(t)
|
||||||
@@ -100,7 +125,28 @@ func TestWebhookRetentionDays_UpdateToZeroIsStoredAsZero(t *testing.T) {
|
|||||||
wh.RetentionDays = 0
|
wh.RetentionDays = 0
|
||||||
require.NoError(t, db.Omit(clause.Associations).Save(wh).Error)
|
require.NoError(t, db.Omit(clause.Associations).Save(wh).Error)
|
||||||
|
|
||||||
assert.Equal(t, 0, storedRetention(t, db, id))
|
assert.Equal(
|
||||||
|
t,
|
||||||
|
database.RetentionForeverDays,
|
||||||
|
storedRetention(t, db, id),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestWebhookRetentionColumnDefaultMatchesConstant guards the one place
|
||||||
|
// the default lives twice: a struct tag cannot reference a constant, so
|
||||||
|
// this asserts the tag and DefaultRetentionDays agree.
|
||||||
|
func TestWebhookRetentionColumnDefaultMatchesConstant(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
field, ok := reflect.TypeFor[database.Webhook]().
|
||||||
|
FieldByName("RetentionDays")
|
||||||
|
require.True(t, ok, "Webhook.RetentionDays must exist")
|
||||||
|
|
||||||
|
assert.Equal(
|
||||||
|
t,
|
||||||
|
"default:"+strconv.Itoa(database.DefaultRetentionDays),
|
||||||
|
field.Tag.Get("gorm"),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestMaxFiniteRetentionDaysIsTheOverflowCeiling asserts that the
|
// TestMaxFiniteRetentionDaysIsTheOverflowCeiling asserts that the
|
||||||
@@ -130,6 +176,14 @@ func TestMaxFiniteRetentionDaysIsTheOverflowCeiling(t *testing.T) {
|
|||||||
time.Duration(overCeiling*hoursPerDay)*time.Hour,
|
time.Duration(overCeiling*hoursPerDay)*time.Hour,
|
||||||
"one day past the ceiling must overflow",
|
"one day past the ceiling must overflow",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
assert.Less(
|
||||||
|
t,
|
||||||
|
database.MaxFiniteRetentionDays,
|
||||||
|
database.RetentionForeverDays,
|
||||||
|
"the sentinel sits above the ceiling and is only safe "+
|
||||||
|
"because retain-forever webhooks skip the arithmetic",
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestWebhookRetainsForeverAndLabel(t *testing.T) {
|
func TestWebhookRetainsForeverAndLabel(t *testing.T) {
|
||||||
@@ -141,8 +195,16 @@ func TestWebhookRetainsForeverAndLabel(t *testing.T) {
|
|||||||
forever bool
|
forever bool
|
||||||
label string
|
label string
|
||||||
}{
|
}{
|
||||||
{"zero", 0, true, testForeverLabel},
|
{
|
||||||
{"negative", -1, true, testForeverLabel},
|
"sentinel",
|
||||||
|
database.RetentionForeverDays, true, testForeverLabel,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"above sentinel",
|
||||||
|
database.RetentionForeverDays + 1, true, testForeverLabel,
|
||||||
|
},
|
||||||
|
{"legacy zero", 0, true, testForeverLabel},
|
||||||
|
{"legacy negative", -1, true, testForeverLabel},
|
||||||
{"default", database.DefaultRetentionDays, false, "30 days"},
|
{"default", database.DefaultRetentionDays, false, "30 days"},
|
||||||
{"one day", 1, false, "1 day"},
|
{"one day", 1, false, "1 day"},
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,16 @@
|
|||||||
|
package database
|
||||||
|
|
||||||
|
// Migrate runs database migrations for the main application database.
|
||||||
|
// Only configuration-tier models are stored in the main database.
|
||||||
|
// Event-tier models (Event, Delivery, DeliveryResult) live in
|
||||||
|
// per-webhook dedicated databases managed by WebhookDBManager.
|
||||||
|
func (d *Database) Migrate() error {
|
||||||
|
return d.db.AutoMigrate(
|
||||||
|
&Setting{},
|
||||||
|
&User{},
|
||||||
|
&APIKey{},
|
||||||
|
&Webhook{},
|
||||||
|
&Entrypoint{},
|
||||||
|
&Target{},
|
||||||
|
)
|
||||||
|
}
|
||||||
@@ -9,7 +9,6 @@ import (
|
|||||||
"math/big"
|
"math/big"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
|
||||||
|
|
||||||
"golang.org/x/crypto/argon2"
|
"golang.org/x/crypto/argon2"
|
||||||
)
|
)
|
||||||
@@ -64,30 +63,10 @@ func DefaultPasswordConfig() *PasswordConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// testArgon2Memory is the Argon2id memory cost, in KiB, that a test
|
// HashPassword generates an Argon2id hash of the password
|
||||||
// binary hashes with: 1 MB instead of the shipped 64 MB. Every test
|
|
||||||
// that starts a database hashes the bootstrap admin password, dozens
|
|
||||||
// of them run in parallel, and under the race detector each 64 MB hash
|
|
||||||
// holds about 150 MB. VerifyPassword reads the cost from the hash it
|
|
||||||
// checks, so verification follows.
|
|
||||||
const testArgon2Memory = 1024
|
|
||||||
|
|
||||||
// hashAtShippedCostInTest makes a test binary hash at the shipped
|
|
||||||
// memory cost. Only TestHashPassword_ShippedParameters sets it.
|
|
||||||
//
|
|
||||||
//nolint:gochecknoglobals // set by one test, see above
|
|
||||||
var hashAtShippedCostInTest bool
|
|
||||||
|
|
||||||
// HashPassword generates an Argon2id hash of the password. A binary
|
|
||||||
// built by go test hashes at testArgon2Memory; one built by go build
|
|
||||||
// always hashes at the defaults.
|
|
||||||
func HashPassword(password string) (string, error) {
|
func HashPassword(password string) (string, error) {
|
||||||
config := DefaultPasswordConfig()
|
config := DefaultPasswordConfig()
|
||||||
|
|
||||||
if testing.Testing() && !hashAtShippedCostInTest {
|
|
||||||
config.Memory = testArgon2Memory
|
|
||||||
}
|
|
||||||
|
|
||||||
// Generate a salt
|
// Generate a salt
|
||||||
salt := make([]byte, config.SaltLen)
|
salt := make([]byte, config.SaltLen)
|
||||||
|
|
||||||
|
|||||||
@@ -192,39 +192,6 @@ func TestHashPasswordUniqueness(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestHashPassword_ShippedParameters hashes and verifies through
|
|
||||||
// HashPassword at the shipped Argon2id parameters. Every other test
|
|
||||||
// hashes at the lower memory cost a test binary uses, so this is the
|
|
||||||
// one that keeps production hashing covered. One hash and one
|
|
||||||
// verification: each costs 64 MB.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // changes the hashing cost for the whole binary
|
|
||||||
func TestHashPassword_ShippedParameters(t *testing.T) {
|
|
||||||
database.HashAtShippedCostForTest(t)
|
|
||||||
|
|
||||||
password := "correct horse battery staple"
|
|
||||||
|
|
||||||
hash, err := database.HashPassword(password)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("hashing with the shipped parameters: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
const shipped = "$argon2id$v=19$m=65536,t=1,p=4$"
|
|
||||||
|
|
||||||
if !strings.HasPrefix(hash, shipped) {
|
|
||||||
t.Errorf("hash = %q, want prefix %q", hash, shipped)
|
|
||||||
}
|
|
||||||
|
|
||||||
valid, err := database.VerifyPassword(password, hash)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("VerifyPassword() error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !valid {
|
|
||||||
t.Error("VerifyPassword() returned false for correct password")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestVerifyDummyPassword_DoesRealWork covers the anti-enumeration
|
// TestVerifyDummyPassword_DoesRealWork covers the anti-enumeration
|
||||||
// path. Login charges an unknown username a verification against a
|
// path. Login charges an unknown username a verification against a
|
||||||
// dummy hash so that a nonexistent account is not answered in
|
// dummy hash so that a nonexistent account is not answered in
|
||||||
|
|||||||
+65
-124
@@ -18,19 +18,6 @@ import (
|
|||||||
// computation.
|
// computation.
|
||||||
const hoursPerDay = 24
|
const hoursPerDay = 24
|
||||||
|
|
||||||
// reapBatchSize is how many expired events one retention transaction
|
|
||||||
// deletes. A transaction holds the event database's write lock, which
|
|
||||||
// the receiver and the delivery workers wait for, so a large prune is
|
|
||||||
// split into transactions each short enough to finish well inside the
|
|
||||||
// busy timeout.
|
|
||||||
const reapBatchSize = 1000
|
|
||||||
|
|
||||||
// reapBatchPause is how long retention waits after one batch before
|
|
||||||
// starting the next. A writer waiting for the write lock checks for it
|
|
||||||
// again after at most 100 ms, so a longer pause lets it in between two
|
|
||||||
// batches instead of only after the whole prune.
|
|
||||||
const reapBatchPause = 200 * time.Millisecond
|
|
||||||
|
|
||||||
// RetentionReaperParams holds the fx dependencies for the
|
// RetentionReaperParams holds the fx dependencies for the
|
||||||
// RetentionReaper.
|
// RetentionReaper.
|
||||||
type RetentionReaperParams struct {
|
type RetentionReaperParams struct {
|
||||||
@@ -184,32 +171,32 @@ func (r *RetentionReaper) sweep(ctx context.Context) {
|
|||||||
|
|
||||||
wh := webhooks[i]
|
wh := webhooks[i]
|
||||||
|
|
||||||
// A missing database has nothing to reap. The delivery
|
// Skip retain-forever webhooks before building any query.
|
||||||
// engine's recovery routine reports a lost one at start (see
|
// RetainsForever covers both the RetentionForeverDays
|
||||||
// WebhookDBManager.GetDB).
|
// sentinel and the non-positive values that predate it: the
|
||||||
|
// sentinel is a positive number, so without this the reaper
|
||||||
|
// would compute a cutoff a thousand years in the past and
|
||||||
|
// issue a DELETE matching nothing on every single sweep.
|
||||||
|
if wh.RetainsForever() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Nothing to reap if the per-webhook database has never
|
||||||
|
// been created.
|
||||||
if !r.dbManager.DBExists(wh.ID) {
|
if !r.dbManager.DBExists(wh.ID) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
r.reapWebhook(ctx, wh.ID, wh.RetentionDays)
|
r.reapWebhook(wh.ID, wh.RetentionDays)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// reapWebhook removes every expired event (and its dependents) from a
|
// reapWebhook removes every expired event (and its dependents) from a
|
||||||
// single webhook's database, or as many as it reaches before ctx is
|
// single webhook's database.
|
||||||
// cancelled.
|
|
||||||
func (r *RetentionReaper) reapWebhook(
|
func (r *RetentionReaper) reapWebhook(
|
||||||
ctx context.Context,
|
|
||||||
webhookID string,
|
webhookID string,
|
||||||
retentionDays int,
|
retentionDays int,
|
||||||
) {
|
) {
|
||||||
// A retain-forever webhook has no cutoff, so its database is not
|
|
||||||
// even opened.
|
|
||||||
cutoff, ok := retentionCutoff(time.Now(), retentionDays)
|
|
||||||
if !ok {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
db, err := r.dbManager.GetDB(webhookID)
|
db, err := r.dbManager.GetDB(webhookID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
r.log.Error(
|
r.log.Error(
|
||||||
@@ -221,7 +208,12 @@ func (r *RetentionReaper) reapWebhook(
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
deleted, err := reapExpired(ctx, db, cutoff)
|
cutoff, ok := retentionCutoff(time.Now(), retentionDays)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
deleted, err := reapExpired(db, cutoff)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
r.log.Error(
|
r.log.Error(
|
||||||
"retention sweep: failed to reap expired events",
|
"retention sweep: failed to reap expired events",
|
||||||
@@ -244,8 +236,7 @@ func (r *RetentionReaper) reapWebhook(
|
|||||||
|
|
||||||
// retentionCutoff returns the timestamp before which a webhook's
|
// retentionCutoff returns the timestamp before which a webhook's
|
||||||
// events have expired, and whether any cutoff applies at all. It
|
// events have expired, and whether any cutoff applies at all. It
|
||||||
// reports false for a retention of 0, which means retain forever, so
|
// reports false for a retain-forever policy, so no DELETE is issued.
|
||||||
// no DELETE is issued.
|
|
||||||
//
|
//
|
||||||
// The day count is clamped to MaxFiniteRetentionDays first. This is
|
// The day count is clamped to MaxFiniteRetentionDays first. This is
|
||||||
// defense in depth rather than decoration: a time.Duration is an int64
|
// defense in depth rather than decoration: a time.Duration is an int64
|
||||||
@@ -255,7 +246,7 @@ func (r *RetentionReaper) reapWebhook(
|
|||||||
// row in the database: the sweep then deletes every event, delivery,
|
// row in the database: the sweep then deletes every event, delivery,
|
||||||
// and delivery result, including ones created seconds ago. Rejecting
|
// and delivery result, including ones created seconds ago. Rejecting
|
||||||
// out-of-range input at the form is the primary guard; saturating here
|
// out-of-range input at the form is the primary guard; saturating here
|
||||||
// means a value that reaches the row by any other path cannot turn a
|
// means an old row, a migration, or a future call site cannot turn a
|
||||||
// too-large retention into total data loss.
|
// too-large retention into total data loss.
|
||||||
func retentionCutoff(
|
func retentionCutoff(
|
||||||
now time.Time,
|
now time.Time,
|
||||||
@@ -274,107 +265,57 @@ func retentionCutoff(
|
|||||||
), true
|
), true
|
||||||
}
|
}
|
||||||
|
|
||||||
// reapExpired hard-deletes the events older than cutoff, with their
|
// reapExpired hard-deletes, in foreign-key-safe order, the delivery
|
||||||
// deliveries and delivery results, reapBatchSize events per
|
// results, deliveries, and events associated with events older than
|
||||||
// transaction with reapBatchPause between transactions, until none is
|
// cutoff. Deletes are unscoped so rows are physically removed rather
|
||||||
// left. Once ctx is cancelled it returns after the batch in hand,
|
// than soft-deleted, reclaiming disk. It returns the number of events
|
||||||
// leaving the rest to the next sweep, so stopping the app does not
|
// deleted.
|
||||||
// wait for a long prune. It returns the number of events deleted.
|
func reapExpired(db *gorm.DB, cutoff time.Time) (int64, error) {
|
||||||
func reapExpired(
|
// Fresh subqueries are built per statement to avoid reusing a
|
||||||
ctx context.Context, db *gorm.DB, cutoff time.Time,
|
// mutated builder across executions.
|
||||||
) (int64, error) {
|
expiredEventIDs := func() *gorm.DB {
|
||||||
var total int64
|
return db.Model(&Event{}).
|
||||||
|
|
||||||
for {
|
|
||||||
var eventIDs []string
|
|
||||||
|
|
||||||
err := db.Transaction(func(tx *gorm.DB) error {
|
|
||||||
err := tx.Unscoped().Model(&Event{}).
|
|
||||||
Where("created_at < ?", cutoff).
|
|
||||||
Limit(reapBatchSize).
|
|
||||||
Pluck("id", &eventIDs).Error
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("selecting expired events: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(eventIDs) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return deleteEvents(tx, eventIDs)
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return total, err
|
|
||||||
}
|
|
||||||
|
|
||||||
total += int64(len(eventIDs))
|
|
||||||
|
|
||||||
if len(eventIDs) < reapBatchSize {
|
|
||||||
return total, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return total, nil
|
|
||||||
case <-time.After(reapBatchPause):
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// deleteEvents hard-deletes the given events and, in foreign-key-safe
|
|
||||||
// order before them, their delivery results and deliveries, then adds
|
|
||||||
// what it deleted to the running totals. It runs on reapExpired's
|
|
||||||
// transaction, so the totals change exactly when the rows do. Deletes
|
|
||||||
// are unscoped so rows are physically removed rather than
|
|
||||||
// soft-deleted, reclaiming disk.
|
|
||||||
func deleteEvents(tx *gorm.DB, eventIDs []string) error {
|
|
||||||
// 1. The delivery results of the events' deliveries.
|
|
||||||
err := tx.Unscoped().
|
|
||||||
Where("delivery_id IN (?)", tx.Unscoped().Model(&Delivery{}).
|
|
||||||
Select("id").
|
Select("id").
|
||||||
Where("event_id IN ?", eventIDs)).
|
Where("created_at < ?", cutoff)
|
||||||
Delete(&DeliveryResult{}).Error
|
}
|
||||||
if err != nil {
|
expiredDeliveryIDs := func() *gorm.DB {
|
||||||
return fmt.Errorf("deleting expired delivery results: %w", err)
|
return db.Model(&Delivery{}).
|
||||||
|
Select("id").
|
||||||
|
Where("event_id IN (?)", expiredEventIDs())
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2. The events' deliveries, after counting them, and the failed
|
// 1. Delivery results whose delivery belongs to an expired event.
|
||||||
// ones among them, per target. The status is tested in the select
|
res := db.Unscoped().
|
||||||
// list rather than the WHERE clause: there, SQLite would read every
|
Where("delivery_id IN (?)", expiredDeliveryIDs()).
|
||||||
// failed delivery the webhook has through the status index,
|
Delete(&DeliveryResult{})
|
||||||
// instead of only these through the event_id index.
|
if res.Error != nil {
|
||||||
var removed []TargetTotals
|
return 0, fmt.Errorf(
|
||||||
|
"deleting expired delivery results: %w",
|
||||||
err = tx.Unscoped().Model(&Delivery{}).
|
res.Error,
|
||||||
Select("target_id, count(*) AS deliveries_removed, "+
|
)
|
||||||
"count(CASE WHEN status = ? THEN 1 END) AS failed_removed",
|
|
||||||
DeliveryStatusFailed).
|
|
||||||
Where("event_id IN ?", eventIDs).
|
|
||||||
Group("target_id").
|
|
||||||
Find(&removed).Error
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("counting expired deliveries: %w", err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
err = tx.Unscoped().
|
// 2. Deliveries belonging to an expired event.
|
||||||
Where("event_id IN ?", eventIDs).
|
del := db.Unscoped().
|
||||||
Delete(&Delivery{}).Error
|
Where("event_id IN (?)", expiredEventIDs()).
|
||||||
if err != nil {
|
Delete(&Delivery{})
|
||||||
return fmt.Errorf("deleting expired deliveries: %w", err)
|
if del.Error != nil {
|
||||||
|
return 0, fmt.Errorf(
|
||||||
|
"deleting expired deliveries: %w",
|
||||||
|
del.Error,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// 3. The events themselves.
|
// 3. The expired events themselves.
|
||||||
ev := tx.Unscoped().Where("id IN ?", eventIDs).Delete(&Event{})
|
ev := db.Unscoped().
|
||||||
|
Where("created_at < ?", cutoff).
|
||||||
|
Delete(&Event{})
|
||||||
if ev.Error != nil {
|
if ev.Error != nil {
|
||||||
return fmt.Errorf("deleting expired events: %w", ev.Error)
|
return 0, fmt.Errorf(
|
||||||
|
"deleting expired events: %w",
|
||||||
|
ev.Error,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
for i := range removed {
|
return ev.RowsAffected, nil
|
||||||
err = AddTargetTotals(tx, removed[i])
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return AddEventTotals(tx, EventTotals{EventsRemoved: ev.RowsAffected})
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -51,7 +51,7 @@ func setupRetentionTest(t *testing.T) *retentionTestEnv {
|
|||||||
|
|
||||||
mgr, err := database.NewWebhookDBManager(
|
mgr, err := database.NewWebhookDBManager(
|
||||||
lc,
|
lc,
|
||||||
database.WebhookDBManagerParams{Config: cfg, Logger: l.Get()},
|
database.WebhookDBManagerParams{Config: cfg, Logger: l},
|
||||||
)
|
)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
@@ -85,6 +85,41 @@ func createWebhook(
|
|||||||
db.Omit(clause.Associations).Create(wh).Error,
|
db.Omit(clause.Associations).Create(wh).Error,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Webhook.BeforeSave rewrites a non-positive RetentionDays to the
|
||||||
|
// retain-forever sentinel, and the column's GORM default would
|
||||||
|
// otherwise substitute 30. Force the requested value with a
|
||||||
|
// column-level update so tests can plant legacy rows that predate
|
||||||
|
// the sentinel and still carry a literal 0 or negative value.
|
||||||
|
require.NoError(
|
||||||
|
t,
|
||||||
|
db.Model(wh).
|
||||||
|
Update("retention_days", retentionDays).Error,
|
||||||
|
)
|
||||||
|
|
||||||
|
return wh.ID
|
||||||
|
}
|
||||||
|
|
||||||
|
// createWebhookNormally inserts a webhook through the ordinary Create
|
||||||
|
// path, with no column-level forcing, so Webhook.BeforeSave applies
|
||||||
|
// exactly as it does in production. Passing 0 therefore yields a row
|
||||||
|
// holding the RetentionForeverDays sentinel.
|
||||||
|
func createWebhookNormally(
|
||||||
|
t *testing.T,
|
||||||
|
db *gorm.DB,
|
||||||
|
retentionDays int,
|
||||||
|
) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
wh := &database.Webhook{
|
||||||
|
UserID: uuid.New().String(),
|
||||||
|
Name: testWebhookName,
|
||||||
|
RetentionDays: retentionDays,
|
||||||
|
}
|
||||||
|
require.NoError(
|
||||||
|
t,
|
||||||
|
db.Omit(clause.Associations).Create(wh).Error,
|
||||||
|
)
|
||||||
|
|
||||||
return wh.ID
|
return wh.ID
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -246,19 +281,19 @@ func TestRetentionReaper_ReapsExpiredKeepsRecent(t *testing.T) {
|
|||||||
assertChainPresent(t, db, recent)
|
assertChainPresent(t, db, recent)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestRetentionReaper_SkipsZeroReapsFiniteInSameSweep covers the
|
// TestRetentionReaper_SkipsSentinelReapsFiniteInSameSweep covers the
|
||||||
// end-to-end retain-forever path: a webhook created with a retention
|
// end-to-end retain-forever path: a webhook created the normal way with
|
||||||
// of 0 reads back 0, and the reaper leaves its ancient events alone
|
// a requested retention of 0 lands on the RetentionForeverDays
|
||||||
// while still reaping a finite-retention webhook in the very same
|
// sentinel, and the reaper leaves its ancient events alone while still
|
||||||
// sweep.
|
// reaping a finite-retention webhook in the very same sweep.
|
||||||
func TestRetentionReaper_SkipsZeroReapsFiniteInSameSweep(
|
func TestRetentionReaper_SkipsSentinelReapsFiniteInSameSweep(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupRetentionTest(t)
|
env := setupRetentionTest(t)
|
||||||
|
|
||||||
foreverID := createWebhook(t, env.mainDB.DB(), 0)
|
foreverID := createWebhookNormally(t, env.mainDB.DB(), 0)
|
||||||
|
|
||||||
var stored database.Webhook
|
var stored database.Webhook
|
||||||
|
|
||||||
@@ -267,12 +302,14 @@ func TestRetentionReaper_SkipsZeroReapsFiniteInSameSweep(
|
|||||||
env.mainDB.DB().Where("id = ?", foreverID).
|
env.mainDB.DB().Where("id = ?", foreverID).
|
||||||
First(&stored).Error,
|
First(&stored).Error,
|
||||||
)
|
)
|
||||||
require.Zero(
|
require.Equal(
|
||||||
t, stored.RetentionDays,
|
t,
|
||||||
"a retention of 0 must be stored as 0",
|
database.RetentionForeverDays,
|
||||||
|
stored.RetentionDays,
|
||||||
|
"a requested retention of 0 must persist as the sentinel",
|
||||||
)
|
)
|
||||||
|
|
||||||
finiteID := createWebhook(t, env.mainDB.DB(), 30)
|
finiteID := createWebhookNormally(t, env.mainDB.DB(), 30)
|
||||||
|
|
||||||
foreverDB, err := env.mgr.GetDB(foreverID)
|
foreverDB, err := env.mgr.GetDB(foreverID)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -298,10 +335,14 @@ func TestRetentionReaper_SkipsZeroReapsFiniteInSameSweep(
|
|||||||
// negative, so subtracting it moves the cutoff into the far future,
|
// negative, so subtracting it moves the cutoff into the far future,
|
||||||
// where "created_at < cutoff" matches every row: an event created a
|
// where "created_at < cutoff" matches every row: an event created a
|
||||||
// moment ago, and its delivery and delivery result, were all deleted
|
// moment ago, and its delivery and delivery result, were all deleted
|
||||||
// on the first sweep. 200000 is inside that band.
|
// on the first sweep. 200000 is inside that band and below the
|
||||||
|
// retain-forever sentinel, so it is treated as a finite policy and
|
||||||
|
// really does reach the arithmetic.
|
||||||
//
|
//
|
||||||
// The form refuses such a value; the point of the test is that a row
|
// The row is planted at the column level because such a value can no
|
||||||
// written by any other path still cannot trigger the wipe.
|
// longer be submitted through the form; the point of the test is that
|
||||||
|
// a row from an older version, or a future call site, still cannot
|
||||||
|
// trigger the wipe.
|
||||||
func TestRetentionReaper_HugeFiniteRetentionRetainsRecentEvents(
|
func TestRetentionReaper_HugeFiniteRetentionRetainsRecentEvents(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
) {
|
) {
|
||||||
@@ -317,6 +358,12 @@ func TestRetentionReaper_HugeFiniteRetentionRetainsRecentEvents(
|
|||||||
database.MaxFiniteRetentionDays,
|
database.MaxFiniteRetentionDays,
|
||||||
"the test value must exceed what the cutoff can represent",
|
"the test value must exceed what the cutoff can represent",
|
||||||
)
|
)
|
||||||
|
require.Less(
|
||||||
|
t,
|
||||||
|
overflowingRetentionDays,
|
||||||
|
database.RetentionForeverDays,
|
||||||
|
"the test value must not be rescued by the forever skip",
|
||||||
|
)
|
||||||
|
|
||||||
webhookID := createWebhook(
|
webhookID := createWebhook(
|
||||||
t, env.mainDB.DB(), overflowingRetentionDays,
|
t, env.mainDB.DB(), overflowingRetentionDays,
|
||||||
@@ -332,15 +379,14 @@ func TestRetentionReaper_HugeFiniteRetentionRetainsRecentEvents(
|
|||||||
assertChainPresent(t, db, fresh)
|
assertChainPresent(t, db, fresh)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestRetentionReaper_RetainsForeverWhenNegative checks that a negative
|
func TestRetentionReaper_RetainsForeverWhenNonPositive(t *testing.T) {
|
||||||
// retention, which the forms refuse, keeps every event rather than
|
|
||||||
// putting the cutoff in the future, where it would match them all.
|
|
||||||
func TestRetentionReaper_RetainsForeverWhenNegative(t *testing.T) {
|
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupRetentionTest(t)
|
env := setupRetentionTest(t)
|
||||||
|
|
||||||
webhookID := createWebhook(t, env.mainDB.DB(), -1)
|
// A legacy row written before the sentinel existed still carries a
|
||||||
|
// literal 0; the <= 0 guard must keep honouring it.
|
||||||
|
webhookID := createWebhook(t, env.mainDB.DB(), 0)
|
||||||
|
|
||||||
db, err := env.mgr.GetDB(webhookID)
|
db, err := env.mgr.GetDB(webhookID)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|||||||
@@ -1,173 +0,0 @@
|
|||||||
package database_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"log/slog"
|
|
||||||
"net/http"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/google/uuid"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"gorm.io/gorm"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database/databasetest"
|
|
||||||
)
|
|
||||||
|
|
||||||
// withFixedNow returns db with GORM's clock stopped at a time in UTC,
|
|
||||||
// so that the created and updated times it writes come back from
|
|
||||||
// SQLite equal to the row written, which holds them in UTC.
|
|
||||||
func withFixedNow(db *gorm.DB) (*gorm.DB, time.Time) {
|
|
||||||
at := time.Date(2026, 10, 13, 12, 30, 45, 123456789, time.UTC)
|
|
||||||
|
|
||||||
return db.Session(&gorm.Session{
|
|
||||||
NowFunc: func() time.Time { return at },
|
|
||||||
}), at
|
|
||||||
}
|
|
||||||
|
|
||||||
// newBaseModel returns a BaseModel with a new ID.
|
|
||||||
func newBaseModel() database.BaseModel {
|
|
||||||
return database.BaseModel{ID: uuid.New().String()}
|
|
||||||
}
|
|
||||||
|
|
||||||
// assertRoundTrip writes row, which must be the only row of its
|
|
||||||
// table, and reads back the same row.
|
|
||||||
func assertRoundTrip[T any](t *testing.T, db *gorm.DB, row *T) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
require.NoError(t, db.Create(row).Error)
|
|
||||||
|
|
||||||
var got T
|
|
||||||
|
|
||||||
require.NoError(t, db.First(&got).Error)
|
|
||||||
assert.Equal(t, *row, got)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestMainSchemaHoldsEveryModel writes a row of each main database
|
|
||||||
// model, every field set, into a database made from its SQL files.
|
|
||||||
func TestMainSchemaHoldsEveryModel(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
mainDB, err := database.Open(
|
|
||||||
t.TempDir(), slog.New(slog.DiscardHandler),
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
t.Cleanup(func() { _ = mainDB.Close() })
|
|
||||||
|
|
||||||
db, at := withFixedNow(mainDB.DB())
|
|
||||||
|
|
||||||
assertRoundTrip(t, db, &database.Setting{
|
|
||||||
Key: "session_key", Value: "value",
|
|
||||||
})
|
|
||||||
assertRoundTrip(t, db, &database.User{
|
|
||||||
BaseModel: newBaseModel(),
|
|
||||||
Username: "user",
|
|
||||||
Password: "password hash",
|
|
||||||
})
|
|
||||||
assertRoundTrip(t, db, &database.APIKey{
|
|
||||||
BaseModel: newBaseModel(),
|
|
||||||
UserID: uuid.New().String(),
|
|
||||||
Key: "key",
|
|
||||||
Description: "API key description",
|
|
||||||
LastUsedAt: &at,
|
|
||||||
})
|
|
||||||
assertRoundTrip(t, db, &database.Webhook{
|
|
||||||
BaseModel: newBaseModel(),
|
|
||||||
UserID: uuid.New().String(),
|
|
||||||
Name: "webhook",
|
|
||||||
Description: "webhook description",
|
|
||||||
RetentionDays: 7,
|
|
||||||
})
|
|
||||||
assertRoundTrip(t, db, &database.Entrypoint{
|
|
||||||
BaseModel: newBaseModel(),
|
|
||||||
WebhookID: uuid.New().String(),
|
|
||||||
Path: uuid.New().String(),
|
|
||||||
Description: "entrypoint description",
|
|
||||||
Active: true,
|
|
||||||
})
|
|
||||||
assertRoundTrip(t, db, &database.Target{
|
|
||||||
BaseModel: newBaseModel(),
|
|
||||||
WebhookID: uuid.New().String(),
|
|
||||||
Name: "target",
|
|
||||||
Type: database.TargetTypeHTTP,
|
|
||||||
Active: true,
|
|
||||||
URL: "https://example.com/",
|
|
||||||
Headers: map[string]string{"X-Api-Key": "key"},
|
|
||||||
TimeoutSeconds: 30,
|
|
||||||
ForwardQuery: true,
|
|
||||||
ArchiveExpiry: 720 * time.Hour,
|
|
||||||
ArchiveRotation: "daily",
|
|
||||||
MaxAttempts: 3,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEventSchemaHoldsEveryModel does the same for a webhook's event
|
|
||||||
// database, which the manager makes from its SQL files.
|
|
||||||
func TestEventSchemaHoldsEveryModel(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
mgr := databasetest.NewWebhookDBManager(t, t.TempDir())
|
|
||||||
|
|
||||||
t.Cleanup(func() { _ = mgr.CloseAll() })
|
|
||||||
|
|
||||||
events, err := mgr.GetDB(uuid.New().String())
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
db, at := withFixedNow(events)
|
|
||||||
source := uuid.New().String()
|
|
||||||
|
|
||||||
assertRoundTrip(t, db, &database.Event{
|
|
||||||
BaseModel: newBaseModel(),
|
|
||||||
WebhookID: uuid.New().String(),
|
|
||||||
EntrypointID: uuid.New().String(),
|
|
||||||
Method: http.MethodPost,
|
|
||||||
RawQuery: "a=1",
|
|
||||||
Headers: `{"X":"Y"}`,
|
|
||||||
Body: "body",
|
|
||||||
ContentType: "text/plain",
|
|
||||||
BodyBytes: 4,
|
|
||||||
ResubmittedFromID: &source,
|
|
||||||
})
|
|
||||||
assertRoundTrip(t, db, &database.Delivery{
|
|
||||||
BaseModel: newBaseModel(),
|
|
||||||
EventID: uuid.New().String(),
|
|
||||||
TargetID: uuid.New().String(),
|
|
||||||
Status: database.DeliveryStatusDelivered,
|
|
||||||
FinishedAt: &at,
|
|
||||||
Replay: true,
|
|
||||||
})
|
|
||||||
assertRoundTrip(t, db, &database.DeliveryResult{
|
|
||||||
BaseModel: newBaseModel(),
|
|
||||||
DeliveryID: uuid.New().String(),
|
|
||||||
AttemptNum: 2,
|
|
||||||
Success: true,
|
|
||||||
StatusCode: 200,
|
|
||||||
ResponseBody: "ok",
|
|
||||||
Error: "error",
|
|
||||||
Duration: 15,
|
|
||||||
})
|
|
||||||
assertRoundTrip(t, db, &database.TargetTotals{
|
|
||||||
TargetID: uuid.New().String(),
|
|
||||||
Deliveries: 5,
|
|
||||||
Delivered: 3,
|
|
||||||
Failed: 1,
|
|
||||||
DeliveriesRemoved: 2,
|
|
||||||
FailedRemoved: 1,
|
|
||||||
})
|
|
||||||
assertRoundTrip(t, db, &database.EntrypointTotals{
|
|
||||||
EntrypointID: uuid.New().String(), LastEventAt: at,
|
|
||||||
})
|
|
||||||
|
|
||||||
// A new event database has its one row of event totals already.
|
|
||||||
require.NoError(t, database.AddEventTotals(db, database.EventTotals{
|
|
||||||
Events: 3, EventsRemoved: 1, LastEventAt: &at,
|
|
||||||
}))
|
|
||||||
|
|
||||||
var totals []database.EventTotals
|
|
||||||
|
|
||||||
require.NoError(t, db.Find(&totals).Error)
|
|
||||||
assert.Equal(t, []database.EventTotals{{
|
|
||||||
ID: 1, Events: 3, EventsRemoved: 1, LastEventAt: &at,
|
|
||||||
}}, totals)
|
|
||||||
}
|
|
||||||
@@ -137,7 +137,7 @@ func TestPerWebhookEventDatabaseFilesAreOwnerOnly(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TestArchiveDatabaseFilesAreOwnerOnly covers the archive-*.db tier.
|
// TestArchiveDatabaseFilesAreOwnerOnly covers the archive-*.db tier.
|
||||||
// internal/archive builds that path and opens it through OpenSQLite,
|
// internal/delivery builds that path and opens it through OpenSQLite,
|
||||||
// the same single open path exercised here, so the mode is settled for
|
// the same single open path exercised here, so the mode is settled for
|
||||||
// all three tiers in one place.
|
// all three tiers in one place.
|
||||||
func TestArchiveDatabaseFilesAreOwnerOnly(t *testing.T) {
|
func TestArchiveDatabaseFilesAreOwnerOnly(t *testing.T) {
|
||||||
@@ -182,29 +182,6 @@ func TestOpenSQLiteTightensFilesLeftWorldReadable(t *testing.T) {
|
|||||||
requireDatabaseSetOwnerOnly(t, path)
|
requireDatabaseSetOwnerOnly(t, path)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestOpenSQLiteRefusesADirectorySidecar covers a directory in place
|
|
||||||
// of -wal or -shm. Beside a -shm directory SQLite opens the database
|
|
||||||
// read-only without a word, and every write then fails naming no file,
|
|
||||||
// so the open must stop instead, naming the directory.
|
|
||||||
func TestOpenSQLiteRefusesADirectorySidecar(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, suffix := range []string{"-wal", "-shm"} {
|
|
||||||
t.Run(suffix, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
path := filepath.Join(t.TempDir(), database.MainDBFileName)
|
|
||||||
require.NoError(t, os.Mkdir(path+suffix, 0o700))
|
|
||||||
|
|
||||||
_, err := database.OpenSQLite(
|
|
||||||
path, database.SQLiteModeCreate,
|
|
||||||
)
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.Contains(t, err.Error(), path+suffix)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestOpenSQLiteExistingModeDoesNotCreateTheFile guards the mechanism
|
// TestOpenSQLiteExistingModeDoesNotCreateTheFile guards the mechanism
|
||||||
// the fix uses: OpenSQLite now creates the database file itself, and
|
// the fix uses: OpenSQLite now creates the database file itself, and
|
||||||
// must not do so for a caller that asked for an existing database. An
|
// must not do so for a caller that asked for an existing database. An
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ import (
|
|||||||
"io/fs"
|
"io/fs"
|
||||||
"net/url"
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
"syscall"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
_ "modernc.org/sqlite" // Pure Go SQLite driver
|
_ "modernc.org/sqlite" // Pure Go SQLite driver
|
||||||
@@ -94,8 +93,7 @@ const (
|
|||||||
const SQLiteFilePerm fs.FileMode = 0o600
|
const SQLiteFilePerm fs.FileMode = 0o600
|
||||||
|
|
||||||
// reserveSQLiteFile puts path at SQLiteFilePerm before the driver ever
|
// reserveSQLiteFile puts path at SQLiteFilePerm before the driver ever
|
||||||
// touches it, and tightens any sidecar already on disk. A directory in
|
// touches it, and tightens any sidecar already on disk.
|
||||||
// place of any of them is an error naming it.
|
|
||||||
//
|
//
|
||||||
// The mode has to be settled here rather than by a chmod after opening,
|
// The mode has to be settled here rather than by a chmod after opening,
|
||||||
// because SQLite picks it: robust_open substitutes
|
// because SQLite picks it: robust_open substitutes
|
||||||
@@ -118,8 +116,9 @@ const SQLiteFilePerm fs.FileMode = 0o600
|
|||||||
// SQLite still reports the absence rather than this function
|
// SQLite still reports the absence rather than this function
|
||||||
// materializing an empty database the caller asked not to create.
|
// materializing an empty database the caller asked not to create.
|
||||||
//
|
//
|
||||||
// The mode is set on every open, whoever created the file, so a
|
// Chmod of a file that already exists is what tightens a data
|
||||||
// database or sidecar found at a wider mode is tightened too.
|
// directory an earlier build left at 0644 — including a developer's
|
||||||
|
// own scratch directory — without any migration machinery.
|
||||||
func reserveSQLiteFile(path string, create bool) error {
|
func reserveSQLiteFile(path string, create bool) error {
|
||||||
if create {
|
if create {
|
||||||
// gosec G304: the path is the database file the caller asked
|
// gosec G304: the path is the database file the caller asked
|
||||||
@@ -144,15 +143,7 @@ func reserveSQLiteFile(path string, create bool) error {
|
|||||||
for _, p := range append(
|
for _, p := range append(
|
||||||
[]string{path}, sqliteSidecarPaths(path)...,
|
[]string{path}, sqliteSidecarPaths(path)...,
|
||||||
) {
|
) {
|
||||||
// Chmod accepts a directory, and SQLite opens a database whose
|
err := os.Chmod(p, SQLiteFilePerm)
|
||||||
// -shm is one read-only, without a word: every write then
|
|
||||||
// fails naming no file.
|
|
||||||
info, err := os.Stat(p)
|
|
||||||
if err == nil && info.IsDir() {
|
|
||||||
return fmt.Errorf("securing %s: %w", p, syscall.EISDIR)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = os.Chmod(p, SQLiteFilePerm)
|
|
||||||
if err != nil && !errors.Is(err, fs.ErrNotExist) {
|
if err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||||
return fmt.Errorf("securing %s: %w", p, err)
|
return fmt.Errorf("securing %s: %w", p, err)
|
||||||
}
|
}
|
||||||
@@ -161,20 +152,6 @@ func reserveSQLiteFile(path string, create bool) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// missingOrEmpty reports whether opening path in SQLiteModeCreate
|
|
||||||
// would start a new, empty database: the file is not there, or it is
|
|
||||||
// zero-length, which SQLite opens as an empty database. A file left at
|
|
||||||
// zero length by an interrupted first start or a truncated copy holds
|
|
||||||
// as little as a missing one, and must be reported the same way.
|
|
||||||
func missingOrEmpty(path string) bool {
|
|
||||||
info, err := os.Stat(path)
|
|
||||||
if errors.Is(err, fs.ErrNotExist) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
return err == nil && info.Size() == 0
|
|
||||||
}
|
|
||||||
|
|
||||||
// sqliteSidecarPaths returns the files SQLite maintains beside a
|
// sqliteSidecarPaths returns the files SQLite maintains beside a
|
||||||
// database under WAL. They carry the same rows as the database itself,
|
// database under WAL. They carry the same rows as the database itself,
|
||||||
// so a fix that tightens only the main file has fixed nothing.
|
// so a fix that tightens only the main file has fixed nothing.
|
||||||
@@ -241,19 +218,7 @@ func SQLiteDSN(path, mode string) string {
|
|||||||
)
|
)
|
||||||
q.Add("_pragma", "journal_mode(WAL)")
|
q.Add("_pragma", "journal_mode(WAL)")
|
||||||
|
|
||||||
// SQLite reads a file: URI's path only up to the first ? or #, and
|
return "file:" + path + "?" + q.Encode()
|
||||||
// decodes %XX escapes in it, so url.URL escapes the path. OmitHost
|
|
||||||
// keeps a relative path as "file:data/x.db"; written
|
|
||||||
// "file://data/x.db", SQLite would reject "data" as the URI's
|
|
||||||
// authority.
|
|
||||||
dsn := url.URL{
|
|
||||||
Scheme: "file",
|
|
||||||
Path: path,
|
|
||||||
OmitHost: true,
|
|
||||||
RawQuery: q.Encode(),
|
|
||||||
}
|
|
||||||
|
|
||||||
return dsn.String()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// OpenSQLite opens the SQLite file at path with the service's
|
// OpenSQLite opens the SQLite file at path with the service's
|
||||||
|
|||||||
@@ -2,8 +2,6 @@ package database_test
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"database/sql"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -114,79 +112,6 @@ func TestMainDBAppliesPragmasOnALiveHandle(t *testing.T) {
|
|||||||
assert.Equal(t, "10000", busy)
|
assert.Equal(t, "10000", busy)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestOpenSQLiteOpensThePathAskedFor uses a data directory whose name
|
|
||||||
// holds ?, # and %. SQLite reads a file: URI's path only up to a ? or
|
|
||||||
// #, and decodes %41 to A, so an unescaped path sent every database in
|
|
||||||
// that directory to one file outside it.
|
|
||||||
func TestOpenSQLiteOpensThePathAskedFor(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := filepath.Join(t.TempDir(), "hooks#1?x%41")
|
|
||||||
require.NoError(t, os.Mkdir(dir, 0o700))
|
|
||||||
|
|
||||||
for _, name := range []string{
|
|
||||||
database.MainDBFileName,
|
|
||||||
"events-" + uuid.New().String() + ".db",
|
|
||||||
} {
|
|
||||||
path := filepath.Join(dir, name)
|
|
||||||
|
|
||||||
sqlDB, err := database.OpenSQLite(
|
|
||||||
path, database.SQLiteModeCreate,
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
t.Cleanup(func() { assert.NoError(t, sqlDB.Close()) })
|
|
||||||
|
|
||||||
assertOpenedFile(t, sqlDB, path)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestOpenSQLiteOpensARelativePath opens a database at a path relative
|
|
||||||
// to the working directory, which is what a relative DATA_DIR such as
|
|
||||||
// the README's DATA_DIR=../webhooker-data gives OpenSQLite.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // t.Chdir moves the whole process.
|
|
||||||
func TestOpenSQLiteOpensARelativePath(t *testing.T) {
|
|
||||||
t.Chdir(t.TempDir())
|
|
||||||
require.NoError(t, os.Mkdir("data", 0o700))
|
|
||||||
|
|
||||||
path := filepath.Join("data", database.MainDBFileName)
|
|
||||||
|
|
||||||
sqlDB, err := database.OpenSQLite(path, database.SQLiteModeCreate)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
defer func() { require.NoError(t, sqlDB.Close()) }()
|
|
||||||
|
|
||||||
assertOpenedFile(t, sqlDB, path)
|
|
||||||
}
|
|
||||||
|
|
||||||
// assertOpenedFile checks that sqlDB is the SQLite file at path.
|
|
||||||
// database_list names the file SQLite actually opened, as an absolute
|
|
||||||
// path with symlinks resolved, so the files are compared, not the names.
|
|
||||||
func assertOpenedFile(t *testing.T, sqlDB *sql.DB, path string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var (
|
|
||||||
seq int
|
|
||||||
schema, file string
|
|
||||||
)
|
|
||||||
|
|
||||||
require.NoError(t, sqlDB.
|
|
||||||
QueryRowContext(t.Context(), "pragma database_list").
|
|
||||||
Scan(&seq, &schema, &file))
|
|
||||||
|
|
||||||
asked, err := os.Stat(path)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
opened, err := os.Stat(file)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
assert.True(
|
|
||||||
t, os.SameFile(asked, opened),
|
|
||||||
"asked for %s, SQLite opened %s", path, file,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestConcurrentReaderDoesNotBlockWrites is the unit-scale form of the
|
// TestConcurrentReaderDoesNotBlockWrites is the unit-scale form of the
|
||||||
// reproduction in
|
// reproduction in
|
||||||
// https://git.eeqj.de/sneak/webhooker/issues/256: an operator's
|
// https://git.eeqj.de/sneak/webhooker/issues/256: an operator's
|
||||||
|
|||||||
@@ -0,0 +1,47 @@
|
|||||||
|
package database
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"gorm.io/gorm"
|
||||||
|
)
|
||||||
|
|
||||||
|
// NewTestDatabase creates a Database wrapper around a pre-opened *gorm.DB.
|
||||||
|
// Intended for use in tests that need a *database.Database without the
|
||||||
|
// full fx lifecycle. The caller is responsible for closing the underlying
|
||||||
|
// sql.DB connection.
|
||||||
|
func NewTestDatabase(db *gorm.DB) *Database {
|
||||||
|
return &Database{
|
||||||
|
db: db,
|
||||||
|
log: slog.New(slog.NewTextHandler(
|
||||||
|
os.Stderr,
|
||||||
|
&slog.HandlerOptions{Level: slog.LevelDebug},
|
||||||
|
)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewTestWebhookDBManager creates a WebhookDBManager backed by the given
|
||||||
|
// data directory. Intended for use in tests without the fx lifecycle.
|
||||||
|
func NewTestWebhookDBManager(dataDir string) *WebhookDBManager {
|
||||||
|
return NewTestWebhookDBManagerWithLogger(
|
||||||
|
dataDir,
|
||||||
|
slog.New(slog.NewTextHandler(
|
||||||
|
os.Stderr,
|
||||||
|
&slog.HandlerOptions{Level: slog.LevelDebug},
|
||||||
|
)),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewTestWebhookDBManagerWithLogger is NewTestWebhookDBManager with the
|
||||||
|
// logger supplied by the caller. The per-webhook databases this manager
|
||||||
|
// opens hand that logger to gormlog, so a test that needs to see the SQL
|
||||||
|
// the service emits can capture it.
|
||||||
|
func NewTestWebhookDBManagerWithLogger(
|
||||||
|
dataDir string, log *slog.Logger,
|
||||||
|
) *WebhookDBManager {
|
||||||
|
return &WebhookDBManager{
|
||||||
|
dataDir: dataDir,
|
||||||
|
log: log,
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,417 +0,0 @@
|
|||||||
package database_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"net/http"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/google/uuid"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"gorm.io/gorm"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// readEventTotals reads a webhook database's row of event totals,
|
|
||||||
// asserting that it has exactly one.
|
|
||||||
func readEventTotals(t *testing.T, db *gorm.DB) database.EventTotals {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var rows []database.EventTotals
|
|
||||||
|
|
||||||
require.NoError(t, db.Find(&rows).Error)
|
|
||||||
require.Len(t, rows, 1)
|
|
||||||
|
|
||||||
return rows[0]
|
|
||||||
}
|
|
||||||
|
|
||||||
// readTargetTotals reads a webhook database's target totals, keyed by
|
|
||||||
// target.
|
|
||||||
func readTargetTotals(
|
|
||||||
t *testing.T, db *gorm.DB,
|
|
||||||
) map[string]database.TargetTotals {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var rows []database.TargetTotals
|
|
||||||
|
|
||||||
require.NoError(t, db.Find(&rows).Error)
|
|
||||||
|
|
||||||
byTarget := make(map[string]database.TargetTotals, len(rows))
|
|
||||||
for _, row := range rows {
|
|
||||||
byTarget[row.TargetID] = row
|
|
||||||
}
|
|
||||||
|
|
||||||
return byTarget
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestWebhookDBManager_TotalsSurviveReopen verifies that a new event
|
|
||||||
// database starts with one row of zero event totals and no target
|
|
||||||
// totals, that adding to a target twice adds to the one row, and that
|
|
||||||
// opening the database again keeps everything added.
|
|
||||||
func TestWebhookDBManager_TotalsSurviveReopen(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
mgr, lc := setupTestWebhookDBManager(t)
|
|
||||||
ctx := context.Background()
|
|
||||||
require.NoError(t, lc.Start(ctx))
|
|
||||||
|
|
||||||
defer func() { require.NoError(t, lc.Stop(ctx)) }()
|
|
||||||
|
|
||||||
webhookID := uuid.New().String()
|
|
||||||
|
|
||||||
db, err := mgr.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
fresh := readEventTotals(t, db)
|
|
||||||
assert.Equal(t, database.EventTotals{ID: fresh.ID}, fresh)
|
|
||||||
assert.Empty(t, readTargetTotals(t, db))
|
|
||||||
|
|
||||||
first, second := uuid.New().String(), uuid.New().String()
|
|
||||||
|
|
||||||
require.NoError(t, database.AddEventTotals(db, database.EventTotals{
|
|
||||||
Events: 2,
|
|
||||||
}))
|
|
||||||
require.NoError(t, database.AddTargetTotals(db, database.TargetTotals{
|
|
||||||
TargetID: first, Deliveries: 2, Delivered: 1,
|
|
||||||
}))
|
|
||||||
require.NoError(t, database.AddTargetTotals(db, database.TargetTotals{
|
|
||||||
TargetID: first, Failed: 1,
|
|
||||||
}))
|
|
||||||
require.NoError(t, database.AddTargetTotals(db, database.TargetTotals{
|
|
||||||
TargetID: second, Deliveries: 1,
|
|
||||||
}))
|
|
||||||
|
|
||||||
// Drop the cached connection so the next open reopens the file,
|
|
||||||
// as a restart would.
|
|
||||||
require.NoError(t, mgr.CloseAll())
|
|
||||||
|
|
||||||
db, err = mgr.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
assert.Equal(t, database.EventTotals{ID: fresh.ID, Events: 2},
|
|
||||||
readEventTotals(t, db))
|
|
||||||
assert.Equal(t, map[string]database.TargetTotals{
|
|
||||||
first: {
|
|
||||||
TargetID: first, Deliveries: 2, Delivered: 1, Failed: 1,
|
|
||||||
},
|
|
||||||
second: {TargetID: second, Deliveries: 1},
|
|
||||||
}, readTargetTotals(t, db))
|
|
||||||
}
|
|
||||||
|
|
||||||
// seedExpiredEvents stores count events created at the given time,
|
|
||||||
// each with a delivered delivery to one target and a failed delivery
|
|
||||||
// to the other, and one attempt for each delivery.
|
|
||||||
//
|
|
||||||
// It and seedBareEvents insert 50 rows per statement, not more. The
|
|
||||||
// SQLite driver looks up each parameter's value by scanning the
|
|
||||||
// statement's arguments from the first until it reaches that
|
|
||||||
// parameter's, so the time to bind a statement grows with the square of
|
|
||||||
// its parameter count: at 500 rows, several thousand parameters, the
|
|
||||||
// seeding took most of these tests' time under -race.
|
|
||||||
func seedExpiredEvents(
|
|
||||||
t *testing.T,
|
|
||||||
db *gorm.DB,
|
|
||||||
webhookID string,
|
|
||||||
count int,
|
|
||||||
createdAt time.Time,
|
|
||||||
delivered, failed string,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
events := make([]database.Event, count)
|
|
||||||
deliveries := make([]database.Delivery, 0, 2*count)
|
|
||||||
|
|
||||||
for i := range events {
|
|
||||||
events[i] = database.Event{
|
|
||||||
WebhookID: webhookID,
|
|
||||||
EntrypointID: uuid.New().String(),
|
|
||||||
Method: http.MethodPost,
|
|
||||||
}
|
|
||||||
events[i].ID = uuid.New().String()
|
|
||||||
events[i].CreatedAt = createdAt
|
|
||||||
|
|
||||||
deliveries = append(deliveries,
|
|
||||||
database.Delivery{
|
|
||||||
EventID: events[i].ID,
|
|
||||||
TargetID: delivered,
|
|
||||||
Status: database.DeliveryStatusDelivered,
|
|
||||||
},
|
|
||||||
database.Delivery{
|
|
||||||
EventID: events[i].ID,
|
|
||||||
TargetID: failed,
|
|
||||||
Status: database.DeliveryStatusFailed,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NoError(t, db.CreateInBatches(events, 50).Error)
|
|
||||||
require.NoError(t, db.CreateInBatches(deliveries, 50).Error)
|
|
||||||
|
|
||||||
results := make([]database.DeliveryResult, len(deliveries))
|
|
||||||
for i := range deliveries {
|
|
||||||
results[i] = database.DeliveryResult{
|
|
||||||
DeliveryID: deliveries[i].ID, AttemptNum: 1,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NoError(t, db.CreateInBatches(results, 50).Error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// seedBareEvents stores count events created at the given time, with
|
|
||||||
// no deliveries.
|
|
||||||
func seedBareEvents(
|
|
||||||
t *testing.T,
|
|
||||||
db *gorm.DB,
|
|
||||||
webhookID string,
|
|
||||||
count int,
|
|
||||||
createdAt time.Time,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
events := make([]database.Event, count)
|
|
||||||
for i := range events {
|
|
||||||
events[i] = database.Event{
|
|
||||||
WebhookID: webhookID,
|
|
||||||
EntrypointID: uuid.New().String(),
|
|
||||||
Method: http.MethodPost,
|
|
||||||
}
|
|
||||||
events[i].CreatedAt = createdAt
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NoError(t, db.CreateInBatches(events, 50).Error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRetentionReaper_PrunesMoreThanOneBatch verifies that a prune
|
|
||||||
// larger than one transaction's batch removes every expired event with
|
|
||||||
// its deliveries and delivery results, keeps the recent event, and
|
|
||||||
// adds what it removed to the event and target totals, so the totals
|
|
||||||
// within retention match the rows still stored.
|
|
||||||
func TestRetentionReaper_PrunesMoreThanOneBatch(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupRetentionTest(t)
|
|
||||||
|
|
||||||
webhookID := createWebhook(t, env.mainDB.DB(), 30)
|
|
||||||
|
|
||||||
db, err := env.mgr.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
expired := database.ExportReapBatchSize + 1
|
|
||||||
delivered, failed := uuid.New().String(), uuid.New().String()
|
|
||||||
seedExpiredEvents(t, db, webhookID, expired,
|
|
||||||
time.Now().Add(-40*24*time.Hour), delivered, failed)
|
|
||||||
|
|
||||||
// One recent event, delivered to the first target.
|
|
||||||
recent := seedEventChain(t, db, webhookID, time.Now())
|
|
||||||
require.NoError(t, db.Model(&database.Delivery{}).
|
|
||||||
Where("id = ?", recent.deliveryID).
|
|
||||||
Update("target_id", delivered).Error)
|
|
||||||
|
|
||||||
// The totals storing those rows would have left.
|
|
||||||
n := int64(expired)
|
|
||||||
require.NoError(t, database.AddEventTotals(db, database.EventTotals{
|
|
||||||
Events: n + 1,
|
|
||||||
}))
|
|
||||||
require.NoError(t, database.AddTargetTotals(db, database.TargetTotals{
|
|
||||||
TargetID: delivered, Deliveries: n + 1, Delivered: n + 1,
|
|
||||||
}))
|
|
||||||
require.NoError(t, database.AddTargetTotals(db, database.TargetTotals{
|
|
||||||
TargetID: failed, Deliveries: n, Failed: n,
|
|
||||||
}))
|
|
||||||
|
|
||||||
env.reaper.ExportSweep(context.Background())
|
|
||||||
|
|
||||||
// Only the recent event's rows are left.
|
|
||||||
for _, model := range []any{
|
|
||||||
&database.Event{}, &database.Delivery{}, &database.DeliveryResult{},
|
|
||||||
} {
|
|
||||||
var count int64
|
|
||||||
|
|
||||||
require.NoError(t, db.Model(model).Count(&count).Error)
|
|
||||||
assert.Equal(t, int64(1), count, "%T rows left", model)
|
|
||||||
}
|
|
||||||
|
|
||||||
assertChainPresent(t, db, recent)
|
|
||||||
|
|
||||||
eventTotals := readEventTotals(t, db)
|
|
||||||
assert.Equal(t, database.EventTotals{
|
|
||||||
ID: eventTotals.ID, Events: n + 1, EventsRemoved: n,
|
|
||||||
}, eventTotals)
|
|
||||||
|
|
||||||
targetTotals := readTargetTotals(t, db)
|
|
||||||
assert.Equal(t, map[string]database.TargetTotals{
|
|
||||||
delivered: {
|
|
||||||
TargetID: delivered, Deliveries: n + 1, Delivered: n + 1,
|
|
||||||
DeliveriesRemoved: n,
|
|
||||||
},
|
|
||||||
failed: {
|
|
||||||
TargetID: failed, Deliveries: n, Failed: n,
|
|
||||||
DeliveriesRemoved: n, FailedRemoved: n,
|
|
||||||
},
|
|
||||||
}, targetTotals)
|
|
||||||
|
|
||||||
// A sweep with nothing left to remove changes nothing.
|
|
||||||
env.reaper.ExportSweep(context.Background())
|
|
||||||
|
|
||||||
assert.Equal(t, eventTotals, readEventTotals(t, db))
|
|
||||||
assert.Equal(t, targetTotals, readTargetTotals(t, db))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRetentionReaper_WriteDuringPruneSucceeds verifies that a prune
|
|
||||||
// of several batches lets other writers in between its batches: an
|
|
||||||
// event stored once the first batch is deleted is stored while expired
|
|
||||||
// events are still left, not only after the prune has finished.
|
|
||||||
func TestRetentionReaper_WriteDuringPruneSucceeds(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupRetentionTest(t)
|
|
||||||
|
|
||||||
webhookID := createWebhook(t, env.mainDB.DB(), 30)
|
|
||||||
|
|
||||||
db, err := env.mgr.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// Three batches of expired events, with nothing else stored: only
|
|
||||||
// the number of batches matters here.
|
|
||||||
expired := 3 * database.ExportReapBatchSize
|
|
||||||
seedBareEvents(t, db, webhookID, expired,
|
|
||||||
time.Now().Add(-40*24*time.Hour))
|
|
||||||
|
|
||||||
cutoff := time.Now().Add(-30 * 24 * time.Hour)
|
|
||||||
countExpired := func() int64 {
|
|
||||||
var count int64
|
|
||||||
|
|
||||||
require.NoError(t, db.Model(&database.Event{}).
|
|
||||||
Where("created_at < ?", cutoff).
|
|
||||||
Count(&count).Error)
|
|
||||||
|
|
||||||
return count
|
|
||||||
}
|
|
||||||
|
|
||||||
pruned := make(chan struct{})
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
defer close(pruned)
|
|
||||||
|
|
||||||
env.reaper.ExportSweep(context.Background())
|
|
||||||
}()
|
|
||||||
|
|
||||||
t.Cleanup(func() { <-pruned })
|
|
||||||
|
|
||||||
// Every stored event is expired until the write below.
|
|
||||||
require.Eventually(t, func() bool {
|
|
||||||
var count int64
|
|
||||||
|
|
||||||
err := db.Model(&database.Event{}).Count(&count).Error
|
|
||||||
|
|
||||||
return err == nil && count < int64(expired)
|
|
||||||
}, 10*time.Second, 10*time.Millisecond)
|
|
||||||
|
|
||||||
event := &database.Event{
|
|
||||||
WebhookID: webhookID,
|
|
||||||
EntrypointID: uuid.New().String(),
|
|
||||||
Method: http.MethodPost,
|
|
||||||
}
|
|
||||||
require.NoError(t, db.Create(event).Error)
|
|
||||||
|
|
||||||
assert.Positive(t, countExpired(),
|
|
||||||
"the event was stored only after the whole prune")
|
|
||||||
|
|
||||||
<-pruned
|
|
||||||
|
|
||||||
assert.Zero(t, countExpired())
|
|
||||||
|
|
||||||
var stored database.Event
|
|
||||||
|
|
||||||
require.NoError(t, db.First(&stored, "id = ?", event.ID).Error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRetentionReaper_StopDuringPruneLeavesTheRest verifies that
|
|
||||||
// stopping the reaper during a prune of several batches returns
|
|
||||||
// between two batches, well inside the stop timeout, leaving the
|
|
||||||
// remaining expired events for the next sweep, and that the totals
|
|
||||||
// match the rows left.
|
|
||||||
func TestRetentionReaper_StopDuringPruneLeavesTheRest(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupRetentionTest(t)
|
|
||||||
|
|
||||||
webhookID := createWebhook(t, env.mainDB.DB(), 30)
|
|
||||||
|
|
||||||
db, err := env.mgr.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// Two batches and one more of expired events, a few of them with a
|
|
||||||
// delivered and a failed delivery for the target totals to count.
|
|
||||||
// Most carry nothing else, to keep the test quick.
|
|
||||||
const withDeliveries = 10
|
|
||||||
|
|
||||||
expiredAt := time.Now().Add(-40 * 24 * time.Hour)
|
|
||||||
delivered, failed := uuid.New().String(), uuid.New().String()
|
|
||||||
seedExpiredEvents(t, db, webhookID, withDeliveries, expiredAt,
|
|
||||||
delivered, failed)
|
|
||||||
seedBareEvents(t, db, webhookID,
|
|
||||||
2*database.ExportReapBatchSize+1-withDeliveries, expiredAt)
|
|
||||||
|
|
||||||
n := int64(2*database.ExportReapBatchSize + 1)
|
|
||||||
require.NoError(t, database.AddEventTotals(db, database.EventTotals{
|
|
||||||
Events: n,
|
|
||||||
}))
|
|
||||||
require.NoError(t, database.AddTargetTotals(db, database.TargetTotals{
|
|
||||||
TargetID: delivered, Deliveries: withDeliveries,
|
|
||||||
Delivered: withDeliveries,
|
|
||||||
}))
|
|
||||||
require.NoError(t, database.AddTargetTotals(db, database.TargetTotals{
|
|
||||||
TargetID: failed, Deliveries: withDeliveries,
|
|
||||||
Failed: withDeliveries,
|
|
||||||
}))
|
|
||||||
|
|
||||||
env.reaper.ExportSetInterval(time.Millisecond)
|
|
||||||
env.reaper.ExportStart()
|
|
||||||
|
|
||||||
// Stop once the first batch is deleted. The stop lands in the pause
|
|
||||||
// after it, or at worst during the second batch, so at least the
|
|
||||||
// last event is left.
|
|
||||||
require.Eventually(t, func() bool {
|
|
||||||
var count int64
|
|
||||||
|
|
||||||
err := db.Model(&database.Event{}).Count(&count).Error
|
|
||||||
|
|
||||||
return err == nil && count < n
|
|
||||||
}, 10*time.Second, 10*time.Millisecond)
|
|
||||||
|
|
||||||
// The app's stop timeout.
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
require.NoError(t, env.reaper.ExportStop(ctx))
|
|
||||||
|
|
||||||
var events int64
|
|
||||||
|
|
||||||
require.NoError(t, db.Model(&database.Event{}).Count(&events).Error)
|
|
||||||
assert.Positive(t, events, "the stop waited for the whole prune")
|
|
||||||
|
|
||||||
eventTotals := readEventTotals(t, db)
|
|
||||||
assert.Equal(t, events, eventTotals.Events-eventTotals.EventsRemoved)
|
|
||||||
|
|
||||||
targetTotals := readTargetTotals(t, db)
|
|
||||||
require.Len(t, targetTotals, 2)
|
|
||||||
|
|
||||||
for target, totals := range targetTotals {
|
|
||||||
var deliveries, failures int64
|
|
||||||
|
|
||||||
require.NoError(t, db.Model(&database.Delivery{}).
|
|
||||||
Where("target_id = ?", target).
|
|
||||||
Count(&deliveries).Error)
|
|
||||||
require.NoError(t, db.Model(&database.Delivery{}).
|
|
||||||
Where("target_id = ? AND status = ?",
|
|
||||||
target, database.DeliveryStatusFailed).
|
|
||||||
Count(&failures).Error)
|
|
||||||
|
|
||||||
assert.Equal(t, deliveries,
|
|
||||||
totals.Deliveries-totals.DeliveriesRemoved, target)
|
|
||||||
assert.Equal(t, failures, totals.Failed-totals.FailedRemoved,
|
|
||||||
target)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -13,9 +13,8 @@ import (
|
|||||||
"gorm.io/driver/sqlite"
|
"gorm.io/driver/sqlite"
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/datadir"
|
|
||||||
"sneak.berlin/go/webhooker/internal/db/migrations"
|
|
||||||
"sneak.berlin/go/webhooker/internal/gormlog"
|
"sneak.berlin/go/webhooker/internal/gormlog"
|
||||||
|
"sneak.berlin/go/webhooker/internal/logger"
|
||||||
)
|
)
|
||||||
|
|
||||||
// WebhookDBManagerParams holds the fx dependencies for
|
// WebhookDBManagerParams holds the fx dependencies for
|
||||||
@@ -24,7 +23,7 @@ type WebhookDBManagerParams struct {
|
|||||||
fx.In
|
fx.In
|
||||||
|
|
||||||
Config *config.Config
|
Config *config.Config
|
||||||
Logger *slog.Logger
|
Logger *logger.Logger
|
||||||
}
|
}
|
||||||
|
|
||||||
// errInvalidCachedDBType indicates a type assertion failure
|
// errInvalidCachedDBType indicates a type assertion failure
|
||||||
@@ -33,33 +32,14 @@ var errInvalidCachedDBType = errors.New(
|
|||||||
"invalid cached database type",
|
"invalid cached database type",
|
||||||
)
|
)
|
||||||
|
|
||||||
// ErrEventDBNotRemoved is in DeleteDB's error when the event
|
|
||||||
// database file itself could not be removed: it is still on disk.
|
|
||||||
var ErrEventDBNotRemoved = errors.New(
|
|
||||||
"event database file not removed",
|
|
||||||
)
|
|
||||||
|
|
||||||
// ErrSidecarNotRemoved is in DeleteDB's error when the event
|
|
||||||
// database file was removed, so its events are gone, but its -wal
|
|
||||||
// or -shm sidecar could not be.
|
|
||||||
var ErrSidecarNotRemoved = errors.New(
|
|
||||||
"event database file removed, but a -wal or -shm sidecar was not",
|
|
||||||
)
|
|
||||||
|
|
||||||
// WebhookDBManager manages per-webhook SQLite database files
|
// WebhookDBManager manages per-webhook SQLite database files
|
||||||
// for event storage. Each webhook gets its own dedicated
|
// for event storage. Each webhook gets its own dedicated
|
||||||
// database containing Events, Deliveries, DeliveryResults and the
|
// database containing Events, Deliveries, and DeliveryResults.
|
||||||
// running totals of them (EventTotals, TargetTotals, EntrypointTotals).
|
|
||||||
// Database connections are opened lazily and cached.
|
// Database connections are opened lazily and cached.
|
||||||
type WebhookDBManager struct {
|
type WebhookDBManager struct {
|
||||||
dataDir string
|
dataDir string
|
||||||
dbs sync.Map // map[webhookID]*gorm.DB
|
dbs sync.Map // map[webhookID]*gorm.DB
|
||||||
log *slog.Logger
|
log *slog.Logger
|
||||||
|
|
||||||
// mu is held while a database is opened, deleted, or closed, so
|
|
||||||
// each file has at most one open handle. Reading an already cached
|
|
||||||
// handle does not take it.
|
|
||||||
mu sync.Mutex
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewWebhookDBManager creates a new WebhookDBManager and
|
// NewWebhookDBManager creates a new WebhookDBManager and
|
||||||
@@ -70,12 +50,11 @@ func NewWebhookDBManager(
|
|||||||
) (*WebhookDBManager, error) {
|
) (*WebhookDBManager, error) {
|
||||||
m := &WebhookDBManager{
|
m := &WebhookDBManager{
|
||||||
dataDir: params.Config.DataDir,
|
dataDir: params.Config.DataDir,
|
||||||
log: params.Logger,
|
log: params.Logger.Get(),
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create data directory if it doesn't exist. datadir.DirPerm is the
|
// Create data directory if it doesn't exist
|
||||||
// single source of the directory mode; either package may run first.
|
err := os.MkdirAll(m.dataDir, dataDirPerm)
|
||||||
err := os.MkdirAll(m.dataDir, datadir.DirPerm)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf(
|
return nil, fmt.Errorf(
|
||||||
"creating data directory %s: %w",
|
"creating data directory %s: %w",
|
||||||
@@ -98,60 +77,61 @@ func NewWebhookDBManager(
|
|||||||
return m, nil
|
return m, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetDB returns the database connection for a webhook, opening it on
|
// GetDB returns the database connection for a webhook,
|
||||||
// first use.
|
// creating the database file lazily if it doesn't exist.
|
||||||
//
|
|
||||||
// The file is made by CreateDB when the webhook is created. One that is
|
|
||||||
// missing or zero-length here means the webhook's events and pending
|
|
||||||
// deliveries are gone: an empty database is created in its place so
|
|
||||||
// the webhook keeps receiving, and that is logged as a warning naming
|
|
||||||
// the file, as a new main database is.
|
|
||||||
func (m *WebhookDBManager) GetDB(
|
func (m *WebhookDBManager) GetDB(
|
||||||
webhookID string,
|
webhookID string,
|
||||||
) (*gorm.DB, error) {
|
) (*gorm.DB, error) {
|
||||||
return m.getDB(webhookID, false)
|
// Fast path: already open
|
||||||
}
|
if val, ok := m.dbs.Load(webhookID); ok {
|
||||||
|
cachedDB, castOK := val.(*gorm.DB)
|
||||||
|
if !castOK {
|
||||||
|
return nil, fmt.Errorf(
|
||||||
|
"%w for webhook %s",
|
||||||
|
errInvalidCachedDBType,
|
||||||
|
webhookID,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
// GetDBIf is GetDB, done only when check reports true. check runs under
|
return cachedDB, nil
|
||||||
// the lock DeleteDB holds while it removes the files, so a caller can
|
}
|
||||||
// confirm the webhook still exists and open its database with no delete
|
|
||||||
// in between. The handle is nil when check reports false. check must
|
|
||||||
// not call the manager.
|
|
||||||
func (m *WebhookDBManager) GetDBIf(
|
|
||||||
webhookID string, check func() (bool, error),
|
|
||||||
) (*gorm.DB, error) {
|
|
||||||
m.mu.Lock()
|
|
||||||
defer m.mu.Unlock()
|
|
||||||
|
|
||||||
ok, err := check()
|
// Slow path: open/create the database
|
||||||
if err != nil || !ok {
|
db, err := m.openDB(webhookID)
|
||||||
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return m.getDBLocked(webhookID, false)
|
// Store it; if another goroutine beat us, close ours
|
||||||
}
|
actual, loaded := m.dbs.LoadOrStore(webhookID, db)
|
||||||
|
if loaded {
|
||||||
|
// Another goroutine created it first; close our duplicate
|
||||||
|
sqlDB, closeErr := db.DB()
|
||||||
|
if closeErr == nil {
|
||||||
|
_ = sqlDB.Close()
|
||||||
|
}
|
||||||
|
|
||||||
// asGormDB returns a value read from the cache as the database
|
existingDB, castOK := actual.(*gorm.DB)
|
||||||
// handle it is.
|
if !castOK {
|
||||||
func asGormDB(val any, webhookID string) (*gorm.DB, error) {
|
return nil, fmt.Errorf(
|
||||||
db, ok := val.(*gorm.DB)
|
"%w for webhook %s",
|
||||||
if !ok {
|
errInvalidCachedDBType,
|
||||||
return nil, fmt.Errorf(
|
webhookID,
|
||||||
"%w for webhook %s",
|
)
|
||||||
errInvalidCachedDBType,
|
}
|
||||||
webhookID,
|
|
||||||
)
|
return existingDB, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return db, nil
|
return db, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// CreateDB creates a new webhook's database file and runs
|
// CreateDB explicitly creates a new per-webhook database file
|
||||||
// migrations.
|
// and runs migrations.
|
||||||
func (m *WebhookDBManager) CreateDB(
|
func (m *WebhookDBManager) CreateDB(
|
||||||
webhookID string,
|
webhookID string,
|
||||||
) error {
|
) error {
|
||||||
_, err := m.getDB(webhookID, true)
|
_, err := m.GetDB(webhookID)
|
||||||
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -167,18 +147,10 @@ func (m *WebhookDBManager) DBExists(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// DeleteDB closes the connection and deletes the database file
|
// DeleteDB closes the connection and deletes the database file
|
||||||
// for a webhook, with its -wal and -shm sidecars. The files are
|
// for a webhook. The file is permanently removed.
|
||||||
// permanently removed. Each file is tried even when another could
|
|
||||||
// not be removed, and the error wraps ErrEventDBNotRemoved or
|
|
||||||
// ErrSidecarNotRemoved to say which was left, naming each file.
|
|
||||||
func (m *WebhookDBManager) DeleteDB(
|
func (m *WebhookDBManager) DeleteDB(
|
||||||
webhookID string,
|
webhookID string,
|
||||||
) error {
|
) error {
|
||||||
// Held until the files are gone, so GetDB cannot open the file
|
|
||||||
// again between the close and the removal.
|
|
||||||
m.mu.Lock()
|
|
||||||
defer m.mu.Unlock()
|
|
||||||
|
|
||||||
// Close and remove from cache
|
// Close and remove from cache
|
||||||
if val, ok := m.dbs.LoadAndDelete(webhookID); ok {
|
if val, ok := m.dbs.LoadAndDelete(webhookID); ok {
|
||||||
if gormDB, castOK := val.(*gorm.DB); castOK {
|
if gormDB, castOK := val.(*gorm.DB); castOK {
|
||||||
@@ -189,23 +161,16 @@ func (m *WebhookDBManager) DeleteDB(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Delete the main DB file and WAL/SHM files
|
||||||
path := m.dbPath(webhookID)
|
path := m.dbPath(webhookID)
|
||||||
|
for _, suffix := range []string{"", "-wal", "-shm"} {
|
||||||
dbErr := removeFile(path)
|
err := os.Remove(path + suffix)
|
||||||
sidecarErr := errors.Join(
|
if err != nil && !os.IsNotExist(err) {
|
||||||
removeFile(path+"-wal"),
|
return fmt.Errorf(
|
||||||
removeFile(path+"-shm"),
|
"deleting webhook database file %s%s: %w",
|
||||||
)
|
path, suffix, err,
|
||||||
|
)
|
||||||
if dbErr != nil {
|
}
|
||||||
return fmt.Errorf(
|
|
||||||
"%w: %w",
|
|
||||||
ErrEventDBNotRemoved, errors.Join(dbErr, sidecarErr),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
if sidecarErr != nil {
|
|
||||||
return fmt.Errorf("%w: %w", ErrSidecarNotRemoved, sidecarErr)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
m.log.Info(
|
m.log.Info(
|
||||||
@@ -216,25 +181,9 @@ func (m *WebhookDBManager) DeleteDB(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// removeFile removes path. A file that is already gone counts as
|
|
||||||
// removed; the error from any other failure names the file.
|
|
||||||
func removeFile(path string) error {
|
|
||||||
err := os.Remove(path)
|
|
||||||
if errors.Is(err, os.ErrNotExist) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// CloseAll closes all open per-webhook database connections.
|
// CloseAll closes all open per-webhook database connections.
|
||||||
// Called during application shutdown.
|
// Called during application shutdown.
|
||||||
func (m *WebhookDBManager) CloseAll() error {
|
func (m *WebhookDBManager) CloseAll() error {
|
||||||
// An open already under way finishes and is cached first, so it
|
|
||||||
// is closed here rather than cached after this loop has passed.
|
|
||||||
m.mu.Lock()
|
|
||||||
defer m.mu.Unlock()
|
|
||||||
|
|
||||||
var lastErr error
|
var lastErr error
|
||||||
|
|
||||||
m.dbs.Range(func(key, value any) bool {
|
m.dbs.Range(func(key, value any) bool {
|
||||||
@@ -269,54 +218,6 @@ func (m *WebhookDBManager) DBPath(
|
|||||||
return m.dbPath(webhookID)
|
return m.dbPath(webhookID)
|
||||||
}
|
}
|
||||||
|
|
||||||
// getDB is GetDB, and CreateDB when isNew is true: the webhook has just
|
|
||||||
// been created, so a missing file is expected rather than lost.
|
|
||||||
func (m *WebhookDBManager) getDB(
|
|
||||||
webhookID string, isNew bool,
|
|
||||||
) (*gorm.DB, error) {
|
|
||||||
// Fast path: already open
|
|
||||||
if val, ok := m.dbs.Load(webhookID); ok {
|
|
||||||
return asGormDB(val, webhookID)
|
|
||||||
}
|
|
||||||
|
|
||||||
m.mu.Lock()
|
|
||||||
defer m.mu.Unlock()
|
|
||||||
|
|
||||||
return m.getDBLocked(webhookID, isNew)
|
|
||||||
}
|
|
||||||
|
|
||||||
// getDBLocked is getDB's slow path, run with m.mu held. It looks in the
|
|
||||||
// cache again first: a caller that raced another one to the lock then
|
|
||||||
// gets its handle instead of opening a second one.
|
|
||||||
func (m *WebhookDBManager) getDBLocked(
|
|
||||||
webhookID string, isNew bool,
|
|
||||||
) (*gorm.DB, error) {
|
|
||||||
if val, ok := m.dbs.Load(webhookID); ok {
|
|
||||||
return asGormDB(val, webhookID)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Checked before opening, which creates the file. See GetDB.
|
|
||||||
path := m.dbPath(webhookID)
|
|
||||||
replaced := !isNew && missingOrEmpty(path)
|
|
||||||
|
|
||||||
db, err := m.openDB(webhookID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if replaced {
|
|
||||||
m.log.Warn(
|
|
||||||
"created a new, empty database",
|
|
||||||
"webhook_id", webhookID,
|
|
||||||
"path", path,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
m.dbs.Store(webhookID, db)
|
|
||||||
|
|
||||||
return db, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *WebhookDBManager) dbPath(
|
func (m *WebhookDBManager) dbPath(
|
||||||
webhookID string,
|
webhookID string,
|
||||||
) string {
|
) string {
|
||||||
@@ -371,25 +272,22 @@ func (m *WebhookDBManager) openDB(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = migrations.Apply(db, migrations.Events)
|
err = purgeTargetRows(db, m.log, webhookID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = sqlDB.Close()
|
_ = sqlDB.Close()
|
||||||
|
|
||||||
return nil, fmt.Errorf(
|
return nil, err
|
||||||
"migrating webhook database %s: %w",
|
|
||||||
webhookID, err,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
err = parseModels(
|
// Run migrations for event-tier models only
|
||||||
db, &Event{}, &Delivery{}, &DeliveryResult{},
|
err = db.AutoMigrate(
|
||||||
&EventTotals{}, &TargetTotals{}, &EntrypointTotals{},
|
&Event{}, &Delivery{}, &DeliveryResult{},
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = sqlDB.Close()
|
_ = sqlDB.Close()
|
||||||
|
|
||||||
return nil, fmt.Errorf(
|
return nil, fmt.Errorf(
|
||||||
"connecting to webhook database %s: %w",
|
"migrating webhook database %s: %w",
|
||||||
webhookID, err,
|
webhookID, err,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,14 +1,10 @@
|
|||||||
package database_test
|
package database_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"context"
|
"context"
|
||||||
"log/slog"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
@@ -18,7 +14,6 @@ import (
|
|||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/database/databasetest"
|
|
||||||
"sneak.berlin/go/webhooker/internal/globals"
|
"sneak.berlin/go/webhooker/internal/globals"
|
||||||
"sneak.berlin/go/webhooker/internal/logger"
|
"sneak.berlin/go/webhooker/internal/logger"
|
||||||
)
|
)
|
||||||
@@ -51,7 +46,7 @@ func setupTestWebhookDBManager(
|
|||||||
lc,
|
lc,
|
||||||
database.WebhookDBManagerParams{
|
database.WebhookDBManagerParams{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
Logger: l.Get(),
|
Logger: l,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -109,55 +104,6 @@ func TestWebhookDBManager_CreateAndGetDB(t *testing.T) {
|
|||||||
assert.Equal(t, `{"test": true}`, readEvent.Body)
|
assert.Equal(t, `{"test": true}`, readEvent.Body)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Many callers ask for one webhook's database at the same moment,
|
|
||||||
// before it is cached. Only one of them may open the file; the others
|
|
||||||
// must wait for its handle. openDB logs one "opened per-webhook
|
|
||||||
// database" line per open, and those lines are what is counted.
|
|
||||||
func TestWebhookDBManager_ConcurrentFirstTouchOpensOnce(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var logs bytes.Buffer
|
|
||||||
|
|
||||||
mgr := databasetest.NewWebhookDBManagerWithLogger(
|
|
||||||
t,
|
|
||||||
t.TempDir(),
|
|
||||||
slog.New(slog.NewTextHandler(&logs, nil)),
|
|
||||||
)
|
|
||||||
|
|
||||||
t.Cleanup(func() { assert.NoError(t, mgr.CloseAll()) })
|
|
||||||
|
|
||||||
webhookID := uuid.New().String()
|
|
||||||
|
|
||||||
const callers = 16
|
|
||||||
|
|
||||||
start := make(chan struct{})
|
|
||||||
handles := make([]*gorm.DB, callers)
|
|
||||||
errs := make([]error, callers)
|
|
||||||
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
|
|
||||||
for i := range callers {
|
|
||||||
wg.Go(func() {
|
|
||||||
<-start
|
|
||||||
|
|
||||||
handles[i], errs[i] = mgr.GetDB(webhookID)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
close(start)
|
|
||||||
wg.Wait()
|
|
||||||
|
|
||||||
for i := range callers {
|
|
||||||
require.NoError(t, errs[i])
|
|
||||||
assert.Same(t, handles[0], handles[i])
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equal(
|
|
||||||
t, 1,
|
|
||||||
strings.Count(logs.String(), "opened per-webhook database"),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestWebhookDBManager_DeleteDB(t *testing.T) {
|
func TestWebhookDBManager_DeleteDB(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -184,91 +130,17 @@ func TestWebhookDBManager_DeleteDB(t *testing.T) {
|
|||||||
}
|
}
|
||||||
require.NoError(t, db.Create(event).Error)
|
require.NoError(t, db.Create(event).Error)
|
||||||
|
|
||||||
// Under WAL, an open database that has been written to has both
|
|
||||||
// sidecars beside it.
|
|
||||||
dbPath := mgr.DBPath(webhookID)
|
|
||||||
require.FileExists(t, dbPath+"-wal")
|
|
||||||
require.FileExists(t, dbPath+"-shm")
|
|
||||||
|
|
||||||
// Delete the DB
|
// Delete the DB
|
||||||
require.NoError(t, mgr.DeleteDB(webhookID))
|
require.NoError(t, mgr.DeleteDB(webhookID))
|
||||||
|
|
||||||
// File should no longer exist
|
// File should no longer exist
|
||||||
assert.False(t, mgr.DBExists(webhookID))
|
assert.False(t, mgr.DBExists(webhookID))
|
||||||
|
|
||||||
// Verify the files are actually gone from disk
|
// Verify the file is actually gone from disk
|
||||||
assert.NoFileExists(t, dbPath)
|
|
||||||
assert.NoFileExists(t, dbPath+"-wal")
|
|
||||||
assert.NoFileExists(t, dbPath+"-shm")
|
|
||||||
}
|
|
||||||
|
|
||||||
// blockRemoval puts a non-empty directory at path, which os.Remove
|
|
||||||
// cannot remove whoever runs the test, root included.
|
|
||||||
func blockRemoval(t *testing.T, path string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
require.NoError(t, os.MkdirAll(filepath.Join(path, "keep"), 0o700))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestWebhookDBManager_DeleteDBKeepsDatabaseFile proves that when the
|
|
||||||
// event database file cannot be removed, the error says so, and both
|
|
||||||
// sidecars are still removed.
|
|
||||||
func TestWebhookDBManager_DeleteDBKeepsDatabaseFile(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
mgr, lc := setupTestWebhookDBManager(t)
|
|
||||||
ctx := context.Background()
|
|
||||||
require.NoError(t, lc.Start(ctx))
|
|
||||||
|
|
||||||
defer func() { require.NoError(t, lc.Stop(ctx)) }()
|
|
||||||
|
|
||||||
webhookID := uuid.New().String()
|
|
||||||
dbPath := mgr.DBPath(webhookID)
|
dbPath := mgr.DBPath(webhookID)
|
||||||
|
|
||||||
blockRemoval(t, dbPath)
|
_, err = os.Stat(dbPath)
|
||||||
require.NoError(t, os.WriteFile(dbPath+"-wal", nil, 0o600))
|
assert.True(t, os.IsNotExist(err))
|
||||||
require.NoError(t, os.WriteFile(dbPath+"-shm", nil, 0o600))
|
|
||||||
|
|
||||||
err := mgr.DeleteDB(webhookID)
|
|
||||||
|
|
||||||
require.ErrorIs(t, err, database.ErrEventDBNotRemoved)
|
|
||||||
require.NotErrorIs(t, err, database.ErrSidecarNotRemoved)
|
|
||||||
assert.Contains(t, err.Error(), dbPath)
|
|
||||||
assert.NoFileExists(t, dbPath+"-wal")
|
|
||||||
assert.NoFileExists(t, dbPath+"-shm")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestWebhookDBManager_DeleteDBKeepsSidecar proves that when the
|
|
||||||
// event database file is removed but a sidecar is not, the error
|
|
||||||
// says the database file is gone, and the other sidecar is still
|
|
||||||
// removed.
|
|
||||||
func TestWebhookDBManager_DeleteDBKeepsSidecar(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
mgr, lc := setupTestWebhookDBManager(t)
|
|
||||||
ctx := context.Background()
|
|
||||||
require.NoError(t, lc.Start(ctx))
|
|
||||||
|
|
||||||
defer func() { require.NoError(t, lc.Stop(ctx)) }()
|
|
||||||
|
|
||||||
webhookID := uuid.New().String()
|
|
||||||
dbPath := mgr.DBPath(webhookID)
|
|
||||||
|
|
||||||
require.NoError(t, mgr.CreateDB(webhookID))
|
|
||||||
// Closing removes the sidecars, so the ones below are the only
|
|
||||||
// ones there.
|
|
||||||
require.NoError(t, mgr.CloseAll())
|
|
||||||
|
|
||||||
blockRemoval(t, dbPath+"-wal")
|
|
||||||
require.NoError(t, os.WriteFile(dbPath+"-shm", nil, 0o600))
|
|
||||||
|
|
||||||
err := mgr.DeleteDB(webhookID)
|
|
||||||
|
|
||||||
require.ErrorIs(t, err, database.ErrSidecarNotRemoved)
|
|
||||||
require.NotErrorIs(t, err, database.ErrEventDBNotRemoved)
|
|
||||||
assert.Contains(t, err.Error(), dbPath+"-wal")
|
|
||||||
assert.NoFileExists(t, dbPath)
|
|
||||||
assert.NoFileExists(t, dbPath+"-shm")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestWebhookDBManager_LazyCreation(t *testing.T) {
|
func TestWebhookDBManager_LazyCreation(t *testing.T) {
|
||||||
@@ -291,76 +163,6 @@ func TestWebhookDBManager_LazyCreation(t *testing.T) {
|
|||||||
assert.True(t, mgr.DBExists(webhookID))
|
assert.True(t, mgr.DBExists(webhookID))
|
||||||
}
|
}
|
||||||
|
|
||||||
// A webhook's database is made by CreateDB along with the webhook. One
|
|
||||||
// that GetDB finds missing or zero-length has lost the webhook's events
|
|
||||||
// and pending deliveries, so the empty database made in its place is
|
|
||||||
// logged as a warning naming the file
|
|
||||||
// (https://git.eeqj.de/sneak/webhooker/issues/290). CreateDB, and
|
|
||||||
// reopening a database that is there, log no such warning.
|
|
||||||
func TestWebhookDBManager_LostDatabaseIsLogged(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const created = `level=WARN msg="created a new, empty database"`
|
|
||||||
|
|
||||||
open := func(
|
|
||||||
t *testing.T, prepare func(*database.WebhookDBManager, string),
|
|
||||||
) (string, string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var logs bytes.Buffer
|
|
||||||
|
|
||||||
mgr := databasetest.NewWebhookDBManagerWithLogger(
|
|
||||||
t,
|
|
||||||
t.TempDir(),
|
|
||||||
slog.New(slog.NewTextHandler(&logs, nil)),
|
|
||||||
)
|
|
||||||
|
|
||||||
webhookID := uuid.New().String()
|
|
||||||
prepare(mgr, webhookID)
|
|
||||||
|
|
||||||
_, err := mgr.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, mgr.CloseAll())
|
|
||||||
|
|
||||||
return logs.String(),
|
|
||||||
" webhook_id=" + webhookID + " path=" + mgr.DBPath(webhookID)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Run("missing", func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
logs, fields := open(
|
|
||||||
t, func(*database.WebhookDBManager, string) {},
|
|
||||||
)
|
|
||||||
assert.Contains(t, logs, created+fields)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("zero-length", func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
logs, fields := open(
|
|
||||||
t, func(mgr *database.WebhookDBManager, webhookID string) {
|
|
||||||
require.NoError(t, os.WriteFile(
|
|
||||||
mgr.DBPath(webhookID), nil, database.SQLiteFilePerm,
|
|
||||||
))
|
|
||||||
},
|
|
||||||
)
|
|
||||||
assert.Contains(t, logs, created+fields)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("created with the webhook, then reopened", func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
logs, _ := open(
|
|
||||||
t, func(mgr *database.WebhookDBManager, webhookID string) {
|
|
||||||
require.NoError(t, mgr.CreateDB(webhookID))
|
|
||||||
require.NoError(t, mgr.CloseAll())
|
|
||||||
},
|
|
||||||
)
|
|
||||||
assert.NotContains(t, logs, created)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestWebhookDBManager_DeliveryWorkflow(t *testing.T) {
|
func TestWebhookDBManager_DeliveryWorkflow(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -29,11 +29,9 @@ import (
|
|||||||
// process that was killed with SIGKILL blocks nothing.
|
// process that was killed with SIGKILL blocks nothing.
|
||||||
const LockFileName = "webhooker.lock"
|
const LockFileName = "webhooker.lock"
|
||||||
|
|
||||||
// DirPerm is the mode DATA_DIR is created with. It is the single
|
// dirPerm is the mode Acquire creates DATA_DIR with. It matches what
|
||||||
// source of that mode: internal/database consumes it rather than
|
// internal/database uses, since whichever runs first creates it.
|
||||||
// keeping its own copy, so the two packages that both create the
|
const dirPerm = 0o750
|
||||||
// directory cannot drift into disagreeing about its permissions.
|
|
||||||
const DirPerm = 0o750
|
|
||||||
|
|
||||||
// ErrLocked reports that another live process holds the data
|
// ErrLocked reports that another live process holds the data
|
||||||
// directory. Callers that need to know whether a deployment is running
|
// directory. Callers that need to know whether a deployment is running
|
||||||
@@ -66,7 +64,7 @@ func Acquire(dir string) (*Lock, error) {
|
|||||||
return nil, ErrNoDir
|
return nil, ErrNoDir
|
||||||
}
|
}
|
||||||
|
|
||||||
err := os.MkdirAll(dir, DirPerm)
|
err := os.MkdirAll(dir, dirPerm)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf(
|
return nil, fmt.Errorf(
|
||||||
"creating data directory %s: %w", dir, err,
|
"creating data directory %s: %w", dir, err,
|
||||||
|
|||||||
@@ -1,9 +0,0 @@
|
|||||||
-- Migration 000: the table recording which migrations this file has
|
|
||||||
-- had. Applied before the others; it records itself.
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
|
||||||
version INTEGER PRIMARY KEY,
|
|
||||||
applied_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
|
||||||
);
|
|
||||||
|
|
||||||
INSERT OR IGNORE INTO schema_migrations (version) VALUES (0);
|
|
||||||
@@ -1,18 +0,0 @@
|
|||||||
-- Migration 001: the schema of a database target's archive file. The
|
|
||||||
-- archivedEvent model in internal/archive must agree with it.
|
|
||||||
|
|
||||||
CREATE TABLE archived_events (
|
|
||||||
id integer PRIMARY KEY AUTOINCREMENT,
|
|
||||||
event_id text,
|
|
||||||
webhook_id text,
|
|
||||||
entrypoint_id text,
|
|
||||||
method text,
|
|
||||||
raw_query text,
|
|
||||||
headers text,
|
|
||||||
body text,
|
|
||||||
content_type text,
|
|
||||||
archived_at datetime
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX idx_archived_events_event_id ON archived_events (event_id);
|
|
||||||
CREATE INDEX idx_archived_events_archived_at ON archived_events (archived_at);
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
-- Migration 000: the table recording which migrations this file has
|
|
||||||
-- had. Applied before the others; it records itself.
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
|
||||||
version INTEGER PRIMARY KEY,
|
|
||||||
applied_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
|
||||||
);
|
|
||||||
|
|
||||||
INSERT OR IGNORE INTO schema_migrations (version) VALUES (0);
|
|
||||||
@@ -1,98 +0,0 @@
|
|||||||
-- Migration 001: the schema of a webhook's event database. The models
|
|
||||||
-- in internal/database must agree with it. The webhook, entrypoint and
|
|
||||||
-- target ids here name rows in the main database, which has those
|
|
||||||
-- tables; this file does not.
|
|
||||||
|
|
||||||
CREATE TABLE events (
|
|
||||||
id uuid,
|
|
||||||
created_at datetime,
|
|
||||||
updated_at datetime,
|
|
||||||
deleted_at datetime,
|
|
||||||
webhook_id uuid NOT NULL,
|
|
||||||
entrypoint_id uuid NOT NULL,
|
|
||||||
method text NOT NULL,
|
|
||||||
raw_query text,
|
|
||||||
headers text,
|
|
||||||
body text,
|
|
||||||
content_type text,
|
|
||||||
body_bytes integer NOT NULL,
|
|
||||||
resubmitted_from_id uuid,
|
|
||||||
PRIMARY KEY (id)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX idx_events_created_at ON events (created_at);
|
|
||||||
CREATE INDEX idx_events_deleted_at ON events (deleted_at);
|
|
||||||
CREATE INDEX idx_events_deleted_at_created_at
|
|
||||||
ON events (deleted_at, created_at);
|
|
||||||
CREATE INDEX idx_events_resubmitted_from_id
|
|
||||||
ON events (resubmitted_from_id, deleted_at);
|
|
||||||
CREATE INDEX idx_events_entrypoint_id
|
|
||||||
ON events (entrypoint_id, deleted_at, resubmitted_from_id, created_at);
|
|
||||||
|
|
||||||
CREATE TABLE deliveries (
|
|
||||||
id uuid,
|
|
||||||
created_at datetime,
|
|
||||||
updated_at datetime,
|
|
||||||
deleted_at datetime,
|
|
||||||
event_id uuid NOT NULL,
|
|
||||||
target_id uuid NOT NULL,
|
|
||||||
status text NOT NULL DEFAULT 'pending',
|
|
||||||
finished_at datetime,
|
|
||||||
replay numeric NOT NULL DEFAULT false,
|
|
||||||
PRIMARY KEY (id),
|
|
||||||
CONSTRAINT fk_events_deliveries
|
|
||||||
FOREIGN KEY (event_id) REFERENCES events (id)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX idx_deliveries_deleted_at ON deliveries (deleted_at);
|
|
||||||
CREATE INDEX idx_deliveries_event_id ON deliveries (event_id, deleted_at);
|
|
||||||
CREATE INDEX idx_deliveries_status
|
|
||||||
ON deliveries (status, deleted_at, finished_at, target_id);
|
|
||||||
|
|
||||||
CREATE TABLE delivery_results (
|
|
||||||
id uuid,
|
|
||||||
created_at datetime,
|
|
||||||
updated_at datetime,
|
|
||||||
deleted_at datetime,
|
|
||||||
delivery_id uuid NOT NULL,
|
|
||||||
attempt_num integer NOT NULL,
|
|
||||||
success numeric,
|
|
||||||
status_code integer,
|
|
||||||
response_body text,
|
|
||||||
error text,
|
|
||||||
duration integer,
|
|
||||||
PRIMARY KEY (id),
|
|
||||||
CONSTRAINT fk_deliveries_delivery_results
|
|
||||||
FOREIGN KEY (delivery_id) REFERENCES deliveries (id)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX idx_delivery_results_deleted_at
|
|
||||||
ON delivery_results (deleted_at);
|
|
||||||
CREATE INDEX idx_delivery_results_delivery_id
|
|
||||||
ON delivery_results (delivery_id, deleted_at);
|
|
||||||
|
|
||||||
CREATE TABLE event_totals (
|
|
||||||
id integer PRIMARY KEY AUTOINCREMENT,
|
|
||||||
events integer NOT NULL,
|
|
||||||
events_removed integer NOT NULL,
|
|
||||||
last_event_at datetime
|
|
||||||
);
|
|
||||||
|
|
||||||
-- The one row of event totals, all zero.
|
|
||||||
INSERT INTO event_totals (events, events_removed) VALUES (0, 0);
|
|
||||||
|
|
||||||
CREATE TABLE target_totals (
|
|
||||||
target_id uuid,
|
|
||||||
deliveries integer NOT NULL,
|
|
||||||
delivered integer NOT NULL,
|
|
||||||
failed integer NOT NULL,
|
|
||||||
deliveries_removed integer NOT NULL,
|
|
||||||
failed_removed integer NOT NULL,
|
|
||||||
PRIMARY KEY (target_id)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE entrypoint_totals (
|
|
||||||
entrypoint_id uuid,
|
|
||||||
last_event_at datetime NOT NULL,
|
|
||||||
PRIMARY KEY (entrypoint_id)
|
|
||||||
);
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
-- Migration 000: the table recording which migrations this file has
|
|
||||||
-- had. Applied before the others; it records itself.
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
|
||||||
version INTEGER PRIMARY KEY,
|
|
||||||
applied_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
|
||||||
);
|
|
||||||
|
|
||||||
INSERT OR IGNORE INTO schema_migrations (version) VALUES (0);
|
|
||||||
@@ -1,95 +0,0 @@
|
|||||||
-- Migration 001: the main database's schema. The models in
|
|
||||||
-- internal/database must agree with it.
|
|
||||||
|
|
||||||
CREATE TABLE settings (
|
|
||||||
"key" text,
|
|
||||||
value text NOT NULL,
|
|
||||||
PRIMARY KEY ("key")
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE users (
|
|
||||||
id uuid,
|
|
||||||
created_at datetime,
|
|
||||||
updated_at datetime,
|
|
||||||
deleted_at datetime,
|
|
||||||
username text NOT NULL,
|
|
||||||
password text NOT NULL,
|
|
||||||
PRIMARY KEY (id),
|
|
||||||
CONSTRAINT chk_users_username
|
|
||||||
CHECK (length(CAST(username AS BLOB)) <= 1024)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX idx_users_deleted_at ON users (deleted_at);
|
|
||||||
CREATE UNIQUE INDEX idx_users_username ON users (username);
|
|
||||||
|
|
||||||
CREATE TABLE api_keys (
|
|
||||||
id uuid,
|
|
||||||
created_at datetime,
|
|
||||||
updated_at datetime,
|
|
||||||
deleted_at datetime,
|
|
||||||
user_id uuid NOT NULL,
|
|
||||||
"key" text NOT NULL,
|
|
||||||
description text,
|
|
||||||
last_used_at datetime,
|
|
||||||
PRIMARY KEY (id),
|
|
||||||
CONSTRAINT fk_users_api_keys FOREIGN KEY (user_id) REFERENCES users (id)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX idx_api_keys_deleted_at ON api_keys (deleted_at);
|
|
||||||
CREATE UNIQUE INDEX idx_api_keys_key ON api_keys ("key");
|
|
||||||
|
|
||||||
CREATE TABLE webhooks (
|
|
||||||
id uuid,
|
|
||||||
created_at datetime,
|
|
||||||
updated_at datetime,
|
|
||||||
deleted_at datetime,
|
|
||||||
user_id uuid NOT NULL,
|
|
||||||
name text NOT NULL,
|
|
||||||
description text,
|
|
||||||
retention_days integer,
|
|
||||||
PRIMARY KEY (id),
|
|
||||||
CONSTRAINT fk_users_webhooks FOREIGN KEY (user_id) REFERENCES users (id)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX idx_webhooks_deleted_at ON webhooks (deleted_at);
|
|
||||||
|
|
||||||
CREATE TABLE entrypoints (
|
|
||||||
id uuid,
|
|
||||||
created_at datetime,
|
|
||||||
updated_at datetime,
|
|
||||||
deleted_at datetime,
|
|
||||||
webhook_id uuid NOT NULL,
|
|
||||||
path text NOT NULL,
|
|
||||||
description text,
|
|
||||||
active numeric DEFAULT true,
|
|
||||||
PRIMARY KEY (id),
|
|
||||||
CONSTRAINT fk_webhooks_entrypoints
|
|
||||||
FOREIGN KEY (webhook_id) REFERENCES webhooks (id)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX idx_entrypoints_deleted_at ON entrypoints (deleted_at);
|
|
||||||
CREATE UNIQUE INDEX idx_entrypoints_path ON entrypoints (path);
|
|
||||||
|
|
||||||
CREATE TABLE targets (
|
|
||||||
id uuid,
|
|
||||||
created_at datetime,
|
|
||||||
updated_at datetime,
|
|
||||||
deleted_at datetime,
|
|
||||||
webhook_id uuid NOT NULL,
|
|
||||||
name text NOT NULL,
|
|
||||||
type text NOT NULL,
|
|
||||||
active numeric DEFAULT true,
|
|
||||||
url text,
|
|
||||||
headers text,
|
|
||||||
timeout_seconds integer,
|
|
||||||
forward_query numeric,
|
|
||||||
archive_expiry integer,
|
|
||||||
archive_rotation text,
|
|
||||||
max_attempts integer NOT NULL,
|
|
||||||
PRIMARY KEY (id),
|
|
||||||
CONSTRAINT fk_webhooks_targets
|
|
||||||
FOREIGN KEY (webhook_id) REFERENCES webhooks (id),
|
|
||||||
CONSTRAINT chk_targets_max_attempts CHECK (max_attempts >= 1)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX idx_targets_deleted_at ON targets (deleted_at);
|
|
||||||
@@ -1,135 +0,0 @@
|
|||||||
// Package migrations holds the schema of each kind of SQLite file
|
|
||||||
// webhooker writes, as numbered SQL files embedded in the binary, and
|
|
||||||
// applies it when a file is opened.
|
|
||||||
//
|
|
||||||
// Each kind of file has its own directory. In each, 000_migration.sql
|
|
||||||
// creates only the schema_migrations table, and 001_schema.sql holds
|
|
||||||
// the whole schema. Before 1.0 a schema change is an edit to
|
|
||||||
// 001_schema.sql, never a new file.
|
|
||||||
package migrations
|
|
||||||
|
|
||||||
import (
|
|
||||||
"embed"
|
|
||||||
"fmt"
|
|
||||||
"io/fs"
|
|
||||||
"path"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"gorm.io/gorm"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The directories, one for each kind of database file.
|
|
||||||
const (
|
|
||||||
// Main is the main database, webhooker.db.
|
|
||||||
Main = "main"
|
|
||||||
|
|
||||||
// Events is a webhook's event database, events-{uuid}.db.
|
|
||||||
Events = "events"
|
|
||||||
|
|
||||||
// Archive is a database target's archive file.
|
|
||||||
Archive = "archive"
|
|
||||||
)
|
|
||||||
|
|
||||||
//go:embed main/*.sql events/*.sql archive/*.sql
|
|
||||||
var files embed.FS
|
|
||||||
|
|
||||||
// Apply runs each SQL file in dir, which is Main, Events or Archive,
|
|
||||||
// that the database has not had yet, in the order of the number its
|
|
||||||
// name starts with, and records that number in schema_migrations.
|
|
||||||
// 000_migration.sql creates that table and records itself, so it runs
|
|
||||||
// first, and only when the table is missing.
|
|
||||||
//
|
|
||||||
// It all runs in one transaction, so a failure leaves the file as it
|
|
||||||
// was.
|
|
||||||
func Apply(db *gorm.DB, dir string) error {
|
|
||||||
return db.Transaction(func(tx *gorm.DB) error {
|
|
||||||
var tables int
|
|
||||||
|
|
||||||
// Row().Scan, not (*gorm.DB).Scan: see internal/gormlog.
|
|
||||||
err := tx.Raw(
|
|
||||||
"SELECT count(*) FROM sqlite_master "+
|
|
||||||
"WHERE type = 'table' AND name = ?",
|
|
||||||
"schema_migrations",
|
|
||||||
).Row().Scan(&tables)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("looking for schema_migrations: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if tables == 0 {
|
|
||||||
err = run(tx, path.Join(dir, "000_migration.sql"))
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// fs.Glob lists the files in name order, which is number order.
|
|
||||||
names, err := fs.Glob(files, path.Join(dir, "*.sql"))
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("listing migrations: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, name := range names {
|
|
||||||
err = applyOnce(tx, name)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// applyOnce runs the migration file name and records its number,
|
|
||||||
// unless schema_migrations already has that number.
|
|
||||||
func applyOnce(tx *gorm.DB, name string) error {
|
|
||||||
number, _, _ := strings.Cut(path.Base(name), "_")
|
|
||||||
|
|
||||||
version, err := strconv.Atoi(number)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("numbering migration %s: %w", name, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var applied int
|
|
||||||
|
|
||||||
err = tx.Raw(
|
|
||||||
"SELECT count(*) FROM schema_migrations WHERE version = ?",
|
|
||||||
version,
|
|
||||||
).Row().Scan(&applied)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("checking migration %s: %w", name, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if applied > 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
err = run(tx, name)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = tx.Exec(
|
|
||||||
"INSERT INTO schema_migrations (version) VALUES (?)", version,
|
|
||||||
).Error
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("recording migration %s: %w", name, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// run executes every statement in the migration file name.
|
|
||||||
func run(tx *gorm.DB, name string) error {
|
|
||||||
script, err := files.ReadFile(name)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("reading migration %s: %w", name, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = tx.Exec(string(script)).Error
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("applying migration %s: %w", name, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,57 +0,0 @@
|
|||||||
package migrations_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"database/sql"
|
|
||||||
"log/slog"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"gorm.io/driver/sqlite"
|
|
||||||
"gorm.io/gorm"
|
|
||||||
_ "modernc.org/sqlite"
|
|
||||||
"sneak.berlin/go/webhooker/internal/db/migrations"
|
|
||||||
"sneak.berlin/go/webhooker/internal/gormlog"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestApplyRunsEachFileOnce applies each kind of database file's
|
|
||||||
// migrations to a new file twice. The first time records every file.
|
|
||||||
// The second finds them all recorded and runs none, which is the only
|
|
||||||
// way it can succeed: 001_schema.sql creates tables that then exist.
|
|
||||||
func TestApplyRunsEachFileOnce(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, dir := range []string{
|
|
||||||
migrations.Main, migrations.Events, migrations.Archive,
|
|
||||||
} {
|
|
||||||
t.Run(dir, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
sqlDB, err := sql.Open(
|
|
||||||
"sqlite", filepath.Join(t.TempDir(), "test.db"),
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
t.Cleanup(func() { _ = sqlDB.Close() })
|
|
||||||
|
|
||||||
db, err := gorm.Open(
|
|
||||||
sqlite.Dialector{Conn: sqlDB},
|
|
||||||
&gorm.Config{
|
|
||||||
Logger: gormlog.New(slog.New(slog.DiscardHandler)),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
require.NoError(t, migrations.Apply(db, dir))
|
|
||||||
require.NoError(t, migrations.Apply(db, dir))
|
|
||||||
|
|
||||||
var versions string
|
|
||||||
|
|
||||||
require.NoError(t, sqlDB.QueryRowContext(t.Context(),
|
|
||||||
"SELECT group_concat(version) FROM schema_migrations",
|
|
||||||
).Scan(&versions))
|
|
||||||
assert.Equal(t, "0,1", versions)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package archive
|
package delivery
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
@@ -8,62 +8,56 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"gorm.io/gorm"
|
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/lifecycle"
|
"sneak.berlin/go/webhooker/internal/lifecycle"
|
||||||
"sneak.berlin/go/webhooker/internal/logger"
|
"sneak.berlin/go/webhooker/internal/logger"
|
||||||
)
|
)
|
||||||
|
|
||||||
// SweeperParams holds the fx dependencies for the
|
// ArchiveSweeperParams holds the fx dependencies for the
|
||||||
// Sweeper.
|
// ArchiveSweeper.
|
||||||
type SweeperParams struct {
|
type ArchiveSweeperParams struct {
|
||||||
fx.In
|
fx.In
|
||||||
|
|
||||||
Config *config.Config
|
Config *config.Config
|
||||||
Database *database.Database
|
Database *database.Database
|
||||||
Writers *Writers
|
Engine *Engine
|
||||||
Logger *logger.Logger
|
Logger *logger.Logger
|
||||||
}
|
}
|
||||||
|
|
||||||
// Sweeper periodically prunes expired rows from the
|
// ArchiveSweeper periodically prunes expired rows from
|
||||||
// archive databases of database targets that carry a positive
|
// per-webhook archive databases whose database target carries a
|
||||||
// expiry.
|
// positive expiry.
|
||||||
//
|
//
|
||||||
// Without it, pruning happens only when an archive is
|
// Without it, pruning happens only when an archive is
|
||||||
// (re)opened, and archives are only ever reopened by writes: an
|
// (re)opened, and archives are only ever reopened by writes: an
|
||||||
// archive whose target has stopped receiving events would keep
|
// archive belonging to a webhook that has stopped receiving
|
||||||
// its expired rows forever. The sweep closes
|
// events would keep its expired rows forever. The sweep closes
|
||||||
// that gap without changing anything for archives whose expiry
|
// that gap without changing anything for archives whose expiry
|
||||||
// is unset or "never".
|
// is unset or "never".
|
||||||
//
|
//
|
||||||
// It reuses Config.RetentionSweepInterval rather than
|
// It reuses Config.RetentionSweepInterval rather than
|
||||||
// introducing a second interval: this is a retention sweep with
|
// introducing a second interval: this is a retention sweep with
|
||||||
// the same semantics as the event retention reaper.
|
// the same semantics as the event retention reaper.
|
||||||
type Sweeper struct {
|
type ArchiveSweeper struct {
|
||||||
db *database.Database
|
db *database.Database
|
||||||
writers *Writers
|
eng *Engine
|
||||||
log *slog.Logger
|
log *slog.Logger
|
||||||
interval time.Duration
|
interval time.Duration
|
||||||
|
cancel context.CancelFunc
|
||||||
// cancel needs no lock: fx calls the stop hook only after the
|
wg sync.WaitGroup
|
||||||
// start hook has returned, so stop never reads it while start
|
|
||||||
// is still setting it.
|
|
||||||
cancel context.CancelFunc
|
|
||||||
|
|
||||||
wg sync.WaitGroup
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewSweeper creates the archive sweeper and registers
|
// NewArchiveSweeper creates the archive sweeper and registers
|
||||||
// its fx lifecycle hooks. The background sweep loop starts on
|
// its fx lifecycle hooks. The background sweep loop starts on
|
||||||
// OnStart and stops cleanly on OnStop via context cancellation.
|
// OnStart and stops cleanly on OnStop via context cancellation.
|
||||||
func NewSweeper(
|
func NewArchiveSweeper(
|
||||||
lc fx.Lifecycle,
|
lc fx.Lifecycle,
|
||||||
params SweeperParams,
|
params ArchiveSweeperParams,
|
||||||
) *Sweeper {
|
) *ArchiveSweeper {
|
||||||
s := &Sweeper{
|
s := &ArchiveSweeper{
|
||||||
db: params.Database,
|
db: params.Database,
|
||||||
writers: params.Writers,
|
eng: params.Engine,
|
||||||
log: params.Logger.Get(),
|
log: params.Logger.Get(),
|
||||||
interval: params.Config.RetentionSweepInterval,
|
interval: params.Config.RetentionSweepInterval,
|
||||||
}
|
}
|
||||||
@@ -77,7 +71,7 @@ func NewSweeper(
|
|||||||
// lifecycle. The start hook's context is deliberately ignored
|
// lifecycle. The start hook's context is deliberately ignored
|
||||||
// (see start for why the background loop must not inherit it);
|
// (see start for why the background loop must not inherit it);
|
||||||
// the stop hook's context is honoured (see stop).
|
// the stop hook's context is honoured (see stop).
|
||||||
func (s *Sweeper) registerHooks(lc fx.Lifecycle) {
|
func (s *ArchiveSweeper) registerHooks(lc fx.Lifecycle) {
|
||||||
lc.Append(fx.Hook{
|
lc.Append(fx.Hook{
|
||||||
//nolint:contextcheck // Not passing the hook context is
|
//nolint:contextcheck // Not passing the hook context is
|
||||||
// the point: see start.
|
// the point: see start.
|
||||||
@@ -103,7 +97,7 @@ func (s *Sweeper) registerHooks(lc fx.Lifecycle) {
|
|||||||
// goroutine must outlive the startup phase, so its lifetime is
|
// goroutine must outlive the startup phase, so its lifetime is
|
||||||
// bounded by OnStop instead: stop cancels this context and waits
|
// bounded by OnStop instead: stop cancels this context and waits
|
||||||
// on the WaitGroup.
|
// on the WaitGroup.
|
||||||
func (s *Sweeper) start() {
|
func (s *ArchiveSweeper) start() {
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
s.cancel = cancel
|
s.cancel = cancel
|
||||||
|
|
||||||
@@ -121,7 +115,7 @@ func (s *Sweeper) start() {
|
|||||||
// exit, bounded by the stop hook's context: a prune wedged on a
|
// exit, bounded by the stop hook's context: a prune wedged on a
|
||||||
// locked archive must not hang the process past fx's stop
|
// locked archive must not hang the process past fx's stop
|
||||||
// timeout.
|
// timeout.
|
||||||
func (s *Sweeper) stop(ctx context.Context) error {
|
func (s *ArchiveSweeper) stop(ctx context.Context) error {
|
||||||
s.log.Info("archive sweeper stopping")
|
s.log.Info("archive sweeper stopping")
|
||||||
|
|
||||||
if s.cancel != nil {
|
if s.cancel != nil {
|
||||||
@@ -140,7 +134,7 @@ func (s *Sweeper) stop(ctx context.Context) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Sweeper) run(ctx context.Context) {
|
func (s *ArchiveSweeper) run(ctx context.Context) {
|
||||||
defer s.wg.Done()
|
defer s.wg.Done()
|
||||||
|
|
||||||
ticker := time.NewTicker(s.interval)
|
ticker := time.NewTicker(s.interval)
|
||||||
@@ -161,25 +155,17 @@ func (s *Sweeper) run(ctx context.Context) {
|
|||||||
// soft-deleted along with it, so GORM's default scope already
|
// soft-deleted along with it, so GORM's default scope already
|
||||||
// excludes them.
|
// excludes them.
|
||||||
//
|
//
|
||||||
// A failure for one target is logged and the sweep continues,
|
// A failure for one webhook is logged and the sweep continues,
|
||||||
// matching how the write path already treats a prune error as
|
// matching how the write path already treats a prune error as
|
||||||
// non-fatal.
|
// non-fatal.
|
||||||
func (s *Sweeper) sweep(ctx context.Context) {
|
func (s *ArchiveSweeper) sweep(ctx context.Context) {
|
||||||
var targets []database.Target
|
var targets []database.Target
|
||||||
|
|
||||||
err := s.db.DB().
|
err := s.db.DB().
|
||||||
WithContext(ctx).
|
|
||||||
Model(&database.Target{}).
|
Model(&database.Target{}).
|
||||||
Where("type = ?", database.TargetTypeDatabase).
|
Where("type = ?", database.TargetTypeDatabase).
|
||||||
Find(&targets).Error
|
Find(&targets).Error
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// The app stopping as a sweep starts cancels the listing.
|
|
||||||
// Stopping is not a failure, so it must not produce an
|
|
||||||
// error line.
|
|
||||||
if ctx.Err() != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
s.log.Error(
|
s.log.Error(
|
||||||
"archive sweep: failed to list database targets",
|
"archive sweep: failed to list database targets",
|
||||||
"error", err,
|
"error", err,
|
||||||
@@ -199,33 +185,44 @@ func (s *Sweeper) sweep(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// sweepTarget prunes the archive of a single database target. A
|
// sweepTarget prunes the archive of a single database target.
|
||||||
// target with an ArchiveExpiry of 0 keeps its archive forever and is
|
// A missing, empty, or "never" expiry parses as a zero duration
|
||||||
// skipped entirely, so that archive is never pruned.
|
// and is skipped entirely, so those archives keep exactly the
|
||||||
func (s *Sweeper) sweepTarget(target *database.Target) {
|
// behaviour they had before the sweep existed.
|
||||||
expiry := target.ArchiveExpiry
|
func (s *ArchiveSweeper) sweepTarget(target *database.Target) {
|
||||||
|
expiry, err := parseArchiveExpiry(target.Config)
|
||||||
|
if err != nil {
|
||||||
|
s.log.Error(
|
||||||
|
"archive sweep: invalid database target config",
|
||||||
|
"webhook_id", target.WebhookID,
|
||||||
|
"target_id", target.ID,
|
||||||
|
"error", err,
|
||||||
|
)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
if expiry <= 0 {
|
if expiry <= 0 {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if s.writers == nil {
|
if s.eng == nil || s.eng.dbTarget == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
err := s.writers.sweepArchive(target.ID, expiry)
|
err = s.eng.dbTarget.sweepWebhook(target.WebhookID, expiry)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// A writer evicted, or a target row gone, underneath the sweep
|
// A writer evicted underneath the sweep means the operator
|
||||||
// means the operator deleted the target or its webhook while
|
// deleted the webhook (or its last database target) while the
|
||||||
// the sweep was walking the target list. That is an ordinary
|
// sweep was walking the target list. That is an ordinary
|
||||||
// interleaving, not a failure, so it must not produce an
|
// interleaving, not a failure, so it must not produce an
|
||||||
// error line.
|
// error line.
|
||||||
if errors.Is(err, errArchiveWriterEvicted) ||
|
if errors.Is(err, errArchiveWriterEvicted) {
|
||||||
errors.Is(err, gorm.ErrRecordNotFound) {
|
|
||||||
s.log.Debug(
|
s.log.Debug(
|
||||||
"archive sweep: target deleted mid-sweep",
|
"archive sweep: writer evicted mid-sweep",
|
||||||
"webhook_id", target.WebhookID,
|
"webhook_id", target.WebhookID,
|
||||||
"target_id", target.ID,
|
"target_id", target.ID,
|
||||||
)
|
)
|
||||||
@@ -1,11 +1,9 @@
|
|||||||
package archive_test
|
package delivery_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"context"
|
"context"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -16,16 +14,12 @@ import (
|
|||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"go.uber.org/fx"
|
|
||||||
"gorm.io/driver/sqlite"
|
"gorm.io/driver/sqlite"
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
"gorm.io/gorm/clause"
|
"gorm.io/gorm/clause"
|
||||||
_ "modernc.org/sqlite" // Pure Go SQLite driver.
|
_ "modernc.org/sqlite" // Pure Go SQLite driver.
|
||||||
"sneak.berlin/go/webhooker/internal/archive"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/database/databasetest"
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
"sneak.berlin/go/webhooker/internal/db/migrations"
|
|
||||||
"sneak.berlin/go/webhooker/internal/gormlog"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -38,84 +32,81 @@ const (
|
|||||||
// sweepConcurrentWrites is how many deliveries the
|
// sweepConcurrentWrites is how many deliveries the
|
||||||
// concurrent write-plus-sweep test races against the sweep.
|
// concurrent write-plus-sweep test races against the sweep.
|
||||||
sweepConcurrentWrites = 20
|
sweepConcurrentWrites = 20
|
||||||
|
|
||||||
// hookStopTimeout bounds how long a lifecycle test waits for
|
|
||||||
// the sweeper's OnStop hook to return before declaring the
|
|
||||||
// shutdown hung.
|
|
||||||
hookStopTimeout = 10 * time.Second
|
|
||||||
|
|
||||||
// wedgeStopTimeout is the stop timeout a wedged-shutdown test
|
|
||||||
// hands OnStop, standing in for fx's StopTimeout. The test
|
|
||||||
// asserts only that the hook returns at all, and allows it
|
|
||||||
// hookStopTimeout — forty times this budget — to do so, so no
|
|
||||||
// assertion here races the wall clock.
|
|
||||||
wedgeStopTimeout = 250 * time.Millisecond
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// archiveTestWebhookName is the name of every webhook
|
// sweeperEnv bundles the pieces an archive sweep test drives:
|
||||||
// seedDatabaseTarget creates. It is not safe in a file name as it
|
// a main configuration database holding webhooks and targets, a
|
||||||
// stands, so every archive test goes through archiveNamePart.
|
// delivery engine owning the archive writer registry, and the
|
||||||
const archiveTestWebhookName = "Sweep Test!"
|
// data directory the archive files live in.
|
||||||
|
type sweeperEnv struct {
|
||||||
// archiveEnv bundles the pieces an archive test drives: a main
|
sweeper *delivery.ArchiveSweeper
|
||||||
// configuration database holding webhooks and targets, the archive
|
eng *delivery.Engine
|
||||||
// writers, the archive sweeper, and the data directory the archive
|
|
||||||
// files live in.
|
|
||||||
type archiveEnv struct {
|
|
||||||
sweeper *archive.Sweeper
|
|
||||||
writers *archive.Writers
|
|
||||||
mainDB *database.Database
|
mainDB *database.Database
|
||||||
dataDir string
|
dataDir string
|
||||||
}
|
}
|
||||||
|
|
||||||
func setupArchiveTest(t *testing.T) *archiveEnv {
|
func setupSweeperTest(t *testing.T) *sweeperEnv {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
dataDir := t.TempDir()
|
dataDir := t.TempDir()
|
||||||
log := archiveTestLogger()
|
log := archiveTestLogger()
|
||||||
|
|
||||||
mainDB, err := database.Open(dataDir, slog.New(slog.DiscardHandler))
|
sqlDB, err := sql.Open(
|
||||||
|
"sqlite",
|
||||||
|
fmt.Sprintf(
|
||||||
|
"file:%s?mode=rwc",
|
||||||
|
filepath.Join(dataDir, "main.db"),
|
||||||
|
),
|
||||||
|
)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
t.Cleanup(func() { _ = mainDB.Close() })
|
t.Cleanup(func() { _ = sqlDB.Close() })
|
||||||
|
|
||||||
writers := archive.NewWriters(archive.WritersParams{
|
gdb, err := gorm.Open(
|
||||||
Database: mainDB,
|
sqlite.Dialector{Conn: sqlDB}, &gorm.Config{},
|
||||||
DBManager: databasetest.NewWebhookDBManager(t, dataDir),
|
)
|
||||||
Logger: log,
|
require.NoError(t, err)
|
||||||
})
|
|
||||||
|
|
||||||
return &archiveEnv{
|
mainDB := database.NewTestDatabase(gdb)
|
||||||
sweeper: archive.NewTestSweeper(
|
require.NoError(t, mainDB.Migrate())
|
||||||
mainDB, writers, log,
|
|
||||||
|
eng := delivery.NewTestEngineWithDB(
|
||||||
|
mainDB,
|
||||||
|
database.NewTestWebhookDBManager(dataDir),
|
||||||
|
log,
|
||||||
|
&http.Client{Timeout: 5 * time.Second},
|
||||||
|
1,
|
||||||
|
)
|
||||||
|
|
||||||
|
return &sweeperEnv{
|
||||||
|
sweeper: delivery.NewTestArchiveSweeper(
|
||||||
|
mainDB, eng, log,
|
||||||
),
|
),
|
||||||
writers: writers,
|
eng: eng,
|
||||||
mainDB: mainDB,
|
mainDB: mainDB,
|
||||||
dataDir: dataDir,
|
dataDir: dataDir,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// archivePath returns where the writers keep a database target's
|
// archivePath returns where the engine keeps a webhook's
|
||||||
// archive file, for the names seedDatabaseTarget gave it.
|
// archive file.
|
||||||
func (env *archiveEnv) archivePath(tgt *database.Target) string {
|
func (env *sweeperEnv) archivePath(webhookID string) string {
|
||||||
return filepath.Join(
|
return filepath.Join(
|
||||||
env.dataDir,
|
env.dataDir, fmt.Sprintf("archive-%s.db", webhookID),
|
||||||
archive.FileName(
|
|
||||||
archiveTestWebhookName, tgt.Name, tgt.ID,
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// seedDatabaseTarget creates a webhook with one database target with
|
// seedDatabaseTarget creates a webhook with one database target
|
||||||
// the given archive expiry and rotation, and returns the target.
|
// carrying the given target config JSON, and returns the
|
||||||
func (env *archiveEnv) seedDatabaseTarget(
|
// webhook id.
|
||||||
t *testing.T, expiry time.Duration, rotation string,
|
func (env *sweeperEnv) seedDatabaseTarget(
|
||||||
) *database.Target {
|
t *testing.T, configJSON string,
|
||||||
|
) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
wh := &database.Webhook{
|
wh := &database.Webhook{
|
||||||
UserID: uuid.New().String(),
|
UserID: uuid.New().String(),
|
||||||
Name: archiveTestWebhookName,
|
Name: "sweep-test",
|
||||||
}
|
}
|
||||||
require.NoError(
|
require.NoError(
|
||||||
t,
|
t,
|
||||||
@@ -124,27 +115,12 @@ func (env *archiveEnv) seedDatabaseTarget(
|
|||||||
Create(wh).Error,
|
Create(wh).Error,
|
||||||
)
|
)
|
||||||
|
|
||||||
return env.addDatabaseTarget(t, wh.ID, expiry, rotation)
|
|
||||||
}
|
|
||||||
|
|
||||||
// addDatabaseTarget creates one more database target on an
|
|
||||||
// existing webhook and returns it.
|
|
||||||
func (env *archiveEnv) addDatabaseTarget(
|
|
||||||
t *testing.T,
|
|
||||||
webhookID string,
|
|
||||||
expiry time.Duration,
|
|
||||||
rotation string,
|
|
||||||
) *database.Target {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
tgt := &database.Target{
|
tgt := &database.Target{
|
||||||
WebhookID: webhookID,
|
WebhookID: wh.ID,
|
||||||
Name: "Archive",
|
Name: "archive",
|
||||||
Type: database.TargetTypeDatabase,
|
Type: database.TargetTypeDatabase,
|
||||||
Active: true,
|
Active: true,
|
||||||
ArchiveExpiry: expiry,
|
Config: configJSON,
|
||||||
ArchiveRotation: rotation,
|
|
||||||
MaxAttempts: 1,
|
|
||||||
}
|
}
|
||||||
require.NoError(
|
require.NoError(
|
||||||
t,
|
t,
|
||||||
@@ -153,30 +129,19 @@ func (env *archiveEnv) addDatabaseTarget(
|
|||||||
Create(tgt).Error,
|
Create(tgt).Error,
|
||||||
)
|
)
|
||||||
|
|
||||||
return tgt
|
return wh.ID
|
||||||
}
|
}
|
||||||
|
|
||||||
// seedArchiveRows creates the archive file for a target and
|
// seedArchiveRows creates the archive file for a webhook and
|
||||||
// inserts one row per supplied archived-at timestamp, returning
|
// inserts one row per supplied archived-at timestamp, returning
|
||||||
// the archive path. The handle is closed before returning, so
|
// the archive path. The handle is closed before returning, so
|
||||||
// the archive is idle exactly as it would be with no traffic.
|
// the archive is idle exactly as it would be with no traffic.
|
||||||
func (env *archiveEnv) seedArchiveRows(
|
func (env *sweeperEnv) seedArchiveRows(
|
||||||
t *testing.T, tgt *database.Target, archivedAt ...time.Time,
|
t *testing.T, webhookID string, archivedAt ...time.Time,
|
||||||
) string {
|
) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
path := env.archivePath(tgt)
|
path := env.archivePath(webhookID)
|
||||||
seedArchiveFile(t, path, tgt.WebhookID, archivedAt...)
|
|
||||||
|
|
||||||
return path
|
|
||||||
}
|
|
||||||
|
|
||||||
// seedArchiveFile creates the archive file at path and inserts one row
|
|
||||||
// per supplied archived-at timestamp, as seedArchiveRows does.
|
|
||||||
func seedArchiveFile(
|
|
||||||
t *testing.T, path, webhookID string, archivedAt ...time.Time,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
sqlDB, err := sql.Open(
|
sqlDB, err := sql.Open(
|
||||||
"sqlite", fmt.Sprintf("file:%s?mode=rwc", path),
|
"sqlite", fmt.Sprintf("file:%s?mode=rwc", path),
|
||||||
@@ -184,15 +149,16 @@ func seedArchiveFile(
|
|||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
gdb, err := gorm.Open(
|
gdb, err := gorm.Open(
|
||||||
sqlite.Dialector{Conn: sqlDB},
|
sqlite.Dialector{Conn: sqlDB}, &gorm.Config{},
|
||||||
&gorm.Config{Logger: gormlog.New(slog.New(slog.DiscardHandler))},
|
|
||||||
)
|
)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
require.NoError(t, migrations.Apply(gdb, migrations.Archive))
|
require.NoError(
|
||||||
|
t, gdb.AutoMigrate(&delivery.ExportArchivedEvent{}),
|
||||||
|
)
|
||||||
|
|
||||||
for i, at := range archivedAt {
|
for i, at := range archivedAt {
|
||||||
row := archive.ExportArchivedEvent{
|
row := delivery.ExportArchivedEvent{
|
||||||
EventID: fmt.Sprintf("ev-%d", i),
|
EventID: fmt.Sprintf("ev-%d", i),
|
||||||
WebhookID: webhookID,
|
WebhookID: webhookID,
|
||||||
Method: http.MethodPost,
|
Method: http.MethodPost,
|
||||||
@@ -203,6 +169,8 @@ func seedArchiveFile(
|
|||||||
}
|
}
|
||||||
|
|
||||||
require.NoError(t, sqlDB.Close())
|
require.NoError(t, sqlDB.Close())
|
||||||
|
|
||||||
|
return path
|
||||||
}
|
}
|
||||||
|
|
||||||
// archivedEventIDs returns the event ids currently stored in an
|
// archivedEventIDs returns the event ids currently stored in an
|
||||||
@@ -212,7 +180,7 @@ func archivedEventIDs(
|
|||||||
) []string {
|
) []string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
var rows []archive.ExportArchivedEvent
|
var rows []delivery.ExportArchivedEvent
|
||||||
|
|
||||||
rdb := openArchiveDBForRead(t, path)
|
rdb := openArchiveDBForRead(t, path)
|
||||||
require.NoError(t, rdb.Order("event_id").Find(&rows).Error)
|
require.NoError(t, rdb.Order("event_id").Find(&rows).Error)
|
||||||
@@ -240,8 +208,7 @@ func countArchivedRows(path string) (int64, error) {
|
|||||||
defer func() { _ = sqlDB.Close() }()
|
defer func() { _ = sqlDB.Close() }()
|
||||||
|
|
||||||
gdb, err := gorm.Open(
|
gdb, err := gorm.Open(
|
||||||
sqlite.Dialector{Conn: sqlDB},
|
sqlite.Dialector{Conn: sqlDB}, &gorm.Config{},
|
||||||
&gorm.Config{Logger: gormlog.New(slog.New(slog.DiscardHandler))},
|
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
@@ -249,7 +216,7 @@ func countArchivedRows(path string) (int64, error) {
|
|||||||
|
|
||||||
var count int64
|
var count int64
|
||||||
|
|
||||||
err = gdb.Model(&archive.ExportArchivedEvent{}).
|
err = gdb.Model(&delivery.ExportArchivedEvent{}).
|
||||||
Count(&count).Error
|
Count(&count).Error
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
@@ -258,55 +225,6 @@ func countArchivedRows(path string) (int64, error) {
|
|||||||
return count, nil
|
return count, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// recordingLifecycle is a minimal fx.Lifecycle that records the
|
|
||||||
// hooks a component registers, so a test can invoke the real
|
|
||||||
// OnStart/OnStop functions with a context of its choosing.
|
|
||||||
type recordingLifecycle struct {
|
|
||||||
hooks []fx.Hook
|
|
||||||
}
|
|
||||||
|
|
||||||
func (l *recordingLifecycle) Append(h fx.Hook) {
|
|
||||||
l.hooks = append(l.hooks, h)
|
|
||||||
}
|
|
||||||
|
|
||||||
// requireStopHookExpires drives hook.OnStop with a stop context
|
|
||||||
// that expires while a wedged goroutine is still running, and
|
|
||||||
// requires the hook to return the deadline error naming
|
|
||||||
// component instead of blocking on the WaitGroup forever.
|
|
||||||
func requireStopHookExpires(
|
|
||||||
t *testing.T, hook fx.Hook, component string,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
stopCtx, cancel := context.WithTimeout(
|
|
||||||
context.Background(), wedgeStopTimeout,
|
|
||||||
)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
var stopErr error
|
|
||||||
|
|
||||||
stopped := make(chan struct{})
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
defer close(stopped)
|
|
||||||
|
|
||||||
stopErr = hook.OnStop(stopCtx)
|
|
||||||
}()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-stopped:
|
|
||||||
case <-time.After(hookStopTimeout):
|
|
||||||
t.Fatal(
|
|
||||||
"OnStop did not return: it discarded the stop " +
|
|
||||||
"context and is waiting on a wedged goroutine " +
|
|
||||||
"that will never observe cancellation",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
require.ErrorIs(t, stopErr, context.DeadlineExceeded)
|
|
||||||
require.ErrorContains(t, stopErr, component)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveSweeper_LoopOutlivesStartHookContext is the
|
// TestArchiveSweeper_LoopOutlivesStartHookContext is the
|
||||||
// regression test for a sweeper that never swept. fx calls
|
// regression test for a sweeper that never swept. fx calls
|
||||||
// OnStart with a context carrying the application's start
|
// OnStart with a context carrying the application's start
|
||||||
@@ -325,13 +243,13 @@ func TestArchiveSweeper_LoopOutlivesStartHookContext(
|
|||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
env := setupSweeperTest(t)
|
||||||
|
|
||||||
tgt := env.seedDatabaseTarget(t, time.Hour, "")
|
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
|
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
path := env.seedArchiveRows(
|
path := env.seedArchiveRows(
|
||||||
t, tgt,
|
t, webhookID,
|
||||||
now.Add(-48*time.Hour),
|
now.Add(-48*time.Hour),
|
||||||
now.Add(-time.Minute),
|
now.Add(-time.Minute),
|
||||||
)
|
)
|
||||||
@@ -369,60 +287,60 @@ func TestArchiveSweeper_LoopOutlivesStartHookContext(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TestArchiveSweep_DoesNotResurrectEvictedWriter covers the
|
// TestArchiveSweep_DoesNotResurrectEvictedWriter covers the
|
||||||
// interleaving where a sweep tick has already listed a target
|
// interleaving where a sweep tick has already listed a webhook's
|
||||||
// when the target is deleted and its writer evicted. The sweep
|
// target when the webhook is deleted and its writer evicted. The
|
||||||
// must not put a writer back into the registry: nothing would
|
// sweep must not put a writer back into the registry: nothing
|
||||||
// ever evict it again, which is precisely the leak this change
|
// would ever evict it again, which is precisely the leak this
|
||||||
// exists to close.
|
// change exists to close.
|
||||||
func TestArchiveSweep_DoesNotResurrectEvictedWriter(
|
func TestArchiveSweep_DoesNotResurrectEvictedWriter(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
env := setupSweeperTest(t)
|
||||||
|
|
||||||
tgt := env.seedDatabaseTarget(t, time.Hour, "")
|
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
env.seedArchiveRows(
|
env.seedArchiveRows(
|
||||||
t, tgt, time.Now().Add(-48*time.Hour),
|
t, webhookID, time.Now().Add(-48*time.Hour),
|
||||||
)
|
)
|
||||||
|
|
||||||
// Prime the registry the way a delivery would, then evict as
|
// Prime the registry the way a delivery would, then evict as
|
||||||
// the deletion path does. The target row is deliberately left
|
// the deletion path does. The target row is deliberately left
|
||||||
// in place: this is the tick that listed the target before
|
// in place: this is the tick that listed the webhook before
|
||||||
// the deletion committed.
|
// the deletion committed.
|
||||||
_, err := env.writers.ExportEnsureArchiveWriter(tgt.ID)
|
_, err := env.eng.ExportEnsureArchiveWriter(webhookID)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
env.writers.EvictTarget(tgt.ID)
|
env.eng.EvictWebhook(webhookID)
|
||||||
require.False(t, env.writers.ExportHasArchiveWriter(tgt.ID))
|
require.False(t, env.eng.ExportHasArchiveWriter(webhookID))
|
||||||
|
|
||||||
env.sweeper.ExportSweep(context.Background())
|
env.sweeper.ExportSweep(context.Background())
|
||||||
|
|
||||||
assert.False(
|
assert.False(
|
||||||
t, env.writers.ExportHasArchiveWriter(tgt.ID),
|
t, env.eng.ExportHasArchiveWriter(webhookID),
|
||||||
"a sweep must never re-register a writer for a target "+
|
"a sweep must never re-register a writer for a webhook "+
|
||||||
"whose registry entry has already been released",
|
"whose registry entry has already been released",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestArchiveSweep_LeavesNoRegistryEntry states the same
|
// TestArchiveSweep_LeavesNoRegistryEntry states the same
|
||||||
// invariant in its general form: sweeping an archive whose
|
// invariant in its general form: sweeping an archive whose
|
||||||
// target has no cached writer must not leave one behind, so the
|
// webhook has no cached writer must not leave one behind, so the
|
||||||
// registry keeps holding only writers a delivery created and an
|
// registry keeps holding only writers a delivery created and an
|
||||||
// eviction can reach.
|
// eviction can reach.
|
||||||
func TestArchiveSweep_LeavesNoRegistryEntry(t *testing.T) {
|
func TestArchiveSweep_LeavesNoRegistryEntry(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
env := setupSweeperTest(t)
|
||||||
|
|
||||||
tgt := env.seedDatabaseTarget(t, time.Hour, "")
|
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
path := env.seedArchiveRows(
|
path := env.seedArchiveRows(
|
||||||
t, tgt,
|
t, webhookID,
|
||||||
time.Now().Add(-48*time.Hour),
|
time.Now().Add(-48*time.Hour),
|
||||||
time.Now().Add(-time.Minute),
|
time.Now().Add(-time.Minute),
|
||||||
)
|
)
|
||||||
|
|
||||||
require.False(t, env.writers.ExportHasArchiveWriter(tgt.ID))
|
require.False(t, env.eng.ExportHasArchiveWriter(webhookID))
|
||||||
|
|
||||||
env.sweeper.ExportSweep(context.Background())
|
env.sweeper.ExportSweep(context.Background())
|
||||||
|
|
||||||
@@ -431,7 +349,7 @@ func TestArchiveSweep_LeavesNoRegistryEntry(t *testing.T) {
|
|||||||
"the sweep must still prune an idle archive",
|
"the sweep must still prune an idle archive",
|
||||||
)
|
)
|
||||||
assert.False(
|
assert.False(
|
||||||
t, env.writers.ExportHasArchiveWriter(tgt.ID),
|
t, env.eng.ExportHasArchiveWriter(webhookID),
|
||||||
"the sweep must release the registry entry it created",
|
"the sweep must release the registry entry it created",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -446,29 +364,34 @@ func TestArchiveSweep_KeepsWriterAdoptedByDelivery(
|
|||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
env := setupSweeperTest(t)
|
||||||
|
|
||||||
tgt := env.seedDatabaseTarget(t, time.Hour, "")
|
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
env.seedArchiveRows(
|
env.seedArchiveRows(
|
||||||
t, tgt, time.Now().Add(-48*time.Hour),
|
t, webhookID, time.Now().Add(-48*time.Hour),
|
||||||
)
|
)
|
||||||
|
|
||||||
d := databaseTargetDelivery(newEvent(`{"n":1}`), tgt)
|
webhookDB := testWebhookDB(t)
|
||||||
|
event := seedEvent(t, webhookDB, `{"n":1}`)
|
||||||
|
event.WebhookID = webhookID
|
||||||
|
d := seedDatabaseTargetDelivery(
|
||||||
|
t, webhookDB, event, `{"expiry":"1h"}`,
|
||||||
|
)
|
||||||
|
|
||||||
env.sweeper.ExportSweep(context.Background())
|
env.sweeper.ExportSweep(context.Background())
|
||||||
require.False(t, env.writers.ExportHasArchiveWriter(tgt.ID))
|
require.False(t, env.eng.ExportHasArchiveWriter(webhookID))
|
||||||
|
|
||||||
require.NoError(t, env.writers.Write(d))
|
env.eng.ExportDeliverDatabase(webhookDB, d)
|
||||||
|
|
||||||
assert.True(
|
assert.True(
|
||||||
t, env.writers.ExportHasArchiveWriter(tgt.ID),
|
t, env.eng.ExportHasArchiveWriter(webhookID),
|
||||||
"a delivery's writer must stay registered",
|
"a delivery's writer must stay registered",
|
||||||
)
|
)
|
||||||
|
|
||||||
env.sweeper.ExportSweep(context.Background())
|
env.sweeper.ExportSweep(context.Background())
|
||||||
|
|
||||||
assert.True(
|
assert.True(
|
||||||
t, env.writers.ExportHasArchiveWriter(tgt.ID),
|
t, env.eng.ExportHasArchiveWriter(webhookID),
|
||||||
"a sweep must not drop a writer a delivery owns",
|
"a sweep must not drop a writer a delivery owns",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -500,15 +423,15 @@ func TestArchiveSweep_KeepsWriterAdoptedDuringSweep(
|
|||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
env := setupSweeperTest(t)
|
||||||
|
|
||||||
tgt := env.seedDatabaseTarget(t, time.Hour, "")
|
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
env.seedArchiveRows(
|
env.seedArchiveRows(
|
||||||
t, tgt, time.Now().Add(-48*time.Hour),
|
t, webhookID, time.Now().Add(-48*time.Hour),
|
||||||
)
|
)
|
||||||
|
|
||||||
sweepWriter, created, err := env.writers.ExportSweepWriterFor(
|
sweepWriter, created, err := env.eng.ExportSweepWriterFor(
|
||||||
tgt.ID,
|
webhookID,
|
||||||
)
|
)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.True(
|
require.True(
|
||||||
@@ -517,34 +440,39 @@ func TestArchiveSweep_KeepsWriterAdoptedDuringSweep(
|
|||||||
)
|
)
|
||||||
|
|
||||||
// The delivery lands mid-sweep and adopts the entry.
|
// The delivery lands mid-sweep and adopts the entry.
|
||||||
d := databaseTargetDelivery(newEvent(`{"n":1}`), tgt)
|
webhookDB := testWebhookDB(t)
|
||||||
|
event := seedEvent(t, webhookDB, `{"n":1}`)
|
||||||
|
event.WebhookID = webhookID
|
||||||
|
d := seedDatabaseTargetDelivery(
|
||||||
|
t, webhookDB, event, `{"expiry":"1h"}`,
|
||||||
|
)
|
||||||
|
|
||||||
require.NoError(t, env.writers.Write(d))
|
env.eng.ExportDeliverDatabase(webhookDB, d)
|
||||||
|
|
||||||
adopted := env.writers.ExportArchiveWriterFor(tgt.ID)
|
adopted := env.eng.ExportArchiveWriterFor(webhookID)
|
||||||
require.NotNil(t, adopted)
|
require.NotNil(t, adopted)
|
||||||
require.True(
|
require.True(
|
||||||
t, sweepWriter.Same(adopted),
|
t, sweepWriter.Same(adopted),
|
||||||
"the delivery must have adopted the sweep's writer",
|
"the delivery must have adopted the sweep's writer",
|
||||||
)
|
)
|
||||||
require.True(
|
require.True(
|
||||||
t, env.writers.ExportArchiveHandleOpen(tgt.ID),
|
t, env.eng.ExportArchiveHandleOpen(webhookID),
|
||||||
"the delivery leaves the archive handle open",
|
"the delivery leaves the archive handle open",
|
||||||
)
|
)
|
||||||
|
|
||||||
// The sweep finishes.
|
// The sweep finishes.
|
||||||
env.writers.ExportReleaseSweepWriter(tgt.ID, sweepWriter)
|
env.eng.ExportReleaseSweepWriter(webhookID, sweepWriter)
|
||||||
|
|
||||||
require.True(
|
require.True(
|
||||||
t, env.writers.ExportHasArchiveWriter(tgt.ID),
|
t, env.eng.ExportHasArchiveWriter(webhookID),
|
||||||
"a writer adopted by a delivery during a sweep must "+
|
"a writer adopted by a delivery during a sweep must "+
|
||||||
"stay registered, or its open handle is unreachable",
|
"stay registered, or its open handle is unreachable",
|
||||||
)
|
)
|
||||||
|
|
||||||
env.writers.EvictTarget(tgt.ID)
|
env.eng.EvictWebhook(webhookID)
|
||||||
|
|
||||||
assert.False(
|
assert.False(
|
||||||
t, env.writers.ExportHasArchiveWriter(tgt.ID),
|
t, env.eng.ExportHasArchiveWriter(webhookID),
|
||||||
"the adopted writer must still be evictable",
|
"the adopted writer must still be evictable",
|
||||||
)
|
)
|
||||||
assert.False(
|
assert.False(
|
||||||
@@ -553,29 +481,34 @@ func TestArchiveSweep_KeepsWriterAdoptedDuringSweep(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestArchiveSweep_ContinuesAfterPerTargetFailure proves a
|
// TestArchiveSweep_ContinuesAfterPerWebhookFailure proves a
|
||||||
// failure for one target does not abort the sweep for the
|
// failure for one webhook does not abort the sweep for the
|
||||||
// others: an unreadable archive has to be logged and stepped
|
// others: an unparseable expiry and an unreadable archive both
|
||||||
// over.
|
// have to be logged and stepped over.
|
||||||
func TestArchiveSweep_ContinuesAfterPerTargetFailure(
|
func TestArchiveSweep_ContinuesAfterPerWebhookFailure(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
env := setupSweeperTest(t)
|
||||||
|
|
||||||
// Seeded first so the sweep reaches it before the healthy
|
// Seeded first so the sweep reaches them before the healthy
|
||||||
// target: targets come back in insertion order.
|
// webhook: targets come back in insertion order.
|
||||||
corrupt := env.seedDatabaseTarget(t, time.Hour, "")
|
badConfigID := env.seedDatabaseTarget(t, `{"expiry":"!!!"}`)
|
||||||
|
env.seedArchiveRows(
|
||||||
|
t, badConfigID, time.Now().Add(-48*time.Hour),
|
||||||
|
)
|
||||||
|
|
||||||
|
corruptID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
require.NoError(t, os.WriteFile(
|
require.NoError(t, os.WriteFile(
|
||||||
env.archivePath(corrupt),
|
env.archivePath(corruptID),
|
||||||
[]byte("this is not a sqlite database"),
|
[]byte("this is not a sqlite database"),
|
||||||
0o600,
|
0o600,
|
||||||
))
|
))
|
||||||
|
|
||||||
healthy := env.seedDatabaseTarget(t, time.Hour, "")
|
healthyID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
healthyPath := env.seedArchiveRows(
|
healthyPath := env.seedArchiveRows(
|
||||||
t, healthy,
|
t, healthyID,
|
||||||
time.Now().Add(-48*time.Hour),
|
time.Now().Add(-48*time.Hour),
|
||||||
time.Now().Add(-time.Minute),
|
time.Now().Add(-time.Minute),
|
||||||
)
|
)
|
||||||
@@ -585,14 +518,14 @@ func TestArchiveSweep_ContinuesAfterPerTargetFailure(
|
|||||||
assert.Equal(
|
assert.Equal(
|
||||||
t, []string{sweepRowNew},
|
t, []string{sweepRowNew},
|
||||||
archivedEventIDs(t, healthyPath),
|
archivedEventIDs(t, healthyPath),
|
||||||
"a failure for an earlier target must not stop the "+
|
"a failure for an earlier webhook must not stop the "+
|
||||||
"sweep from pruning the ones after it",
|
"sweep from pruning the ones after it",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestArchiveSweep_OpenExistingDoesNotCreateFile pins the second
|
// TestArchiveSweep_OpenExistingDoesNotCreateFile pins the second
|
||||||
// of the two no-create guards. The first is the stat in
|
// of the two no-create guards. The first is the stat in
|
||||||
// sweepExpired; this one is the SQLite open mode, which is what
|
// sweepWebhook; this one is the SQLite open mode, which is what
|
||||||
// protects the window between that stat and the open. Flipping
|
// protects the window between that stat and the open. Flipping
|
||||||
// the sweep's mode to create-if-missing makes this fail.
|
// the sweep's mode to create-if-missing makes this fail.
|
||||||
func TestArchiveSweep_OpenExistingDoesNotCreateFile(
|
func TestArchiveSweep_OpenExistingDoesNotCreateFile(
|
||||||
@@ -603,7 +536,7 @@ func TestArchiveSweep_OpenExistingDoesNotCreateFile(
|
|||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
path := filepath.Join(dir, "archive-absent.db")
|
path := filepath.Join(dir, "archive-absent.db")
|
||||||
|
|
||||||
w := archive.NewExportArchiveWriter(
|
w := delivery.NewExportArchiveWriter(
|
||||||
path, archiveTestLogger(), 0,
|
path, archiveTestLogger(), 0,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -628,13 +561,13 @@ func TestArchiveSweep_OpenExistingDoesNotCreateFile(
|
|||||||
func TestArchiveSweep_PrunesIdleArchive(t *testing.T) {
|
func TestArchiveSweep_PrunesIdleArchive(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
env := setupSweeperTest(t)
|
||||||
|
|
||||||
tgt := env.seedDatabaseTarget(t, time.Hour, "")
|
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
|
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
path := env.seedArchiveRows(
|
path := env.seedArchiveRows(
|
||||||
t, tgt,
|
t, webhookID,
|
||||||
now.Add(-48*time.Hour),
|
now.Add(-48*time.Hour),
|
||||||
now.Add(-time.Minute),
|
now.Add(-time.Minute),
|
||||||
)
|
)
|
||||||
@@ -667,14 +600,14 @@ func TestArchiveSweep_PrunesIdleArchive(t *testing.T) {
|
|||||||
func TestArchiveSweep_LeavesArchiveClosed(t *testing.T) {
|
func TestArchiveSweep_LeavesArchiveClosed(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
env := setupSweeperTest(t)
|
||||||
|
|
||||||
tgt := env.seedDatabaseTarget(t, time.Hour, "")
|
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
path := env.seedArchiveRows(
|
path := env.seedArchiveRows(
|
||||||
t, tgt, time.Now().Add(-48*time.Hour),
|
t, webhookID, time.Now().Add(-48*time.Hour),
|
||||||
)
|
)
|
||||||
|
|
||||||
w := archive.NewExportArchiveWriter(
|
w := delivery.NewExportArchiveWriter(
|
||||||
path, archiveTestLogger(), 0,
|
path, archiveTestLogger(), 0,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -707,116 +640,70 @@ func TestArchiveSweep_ClosesHandleOfRegisteredWriter(
|
|||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
env := setupSweeperTest(t)
|
||||||
|
|
||||||
tgt := env.seedDatabaseTarget(t, time.Hour, "")
|
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
env.seedArchiveRows(
|
env.seedArchiveRows(
|
||||||
t, tgt, time.Now().Add(-48*time.Hour),
|
t, webhookID, time.Now().Add(-48*time.Hour),
|
||||||
)
|
)
|
||||||
|
|
||||||
d := databaseTargetDelivery(newEvent(`{"n":1}`), tgt)
|
webhookDB := testWebhookDB(t)
|
||||||
|
event := seedEvent(t, webhookDB, `{"n":1}`)
|
||||||
|
event.WebhookID = webhookID
|
||||||
|
d := seedDatabaseTargetDelivery(
|
||||||
|
t, webhookDB, event, `{"expiry":"1h"}`,
|
||||||
|
)
|
||||||
|
|
||||||
require.NoError(t, env.writers.Write(d))
|
env.eng.ExportDeliverDatabase(webhookDB, d)
|
||||||
|
|
||||||
require.True(
|
require.True(
|
||||||
t, env.writers.ExportArchiveHandleOpen(tgt.ID),
|
t, env.eng.ExportArchiveHandleOpen(webhookID),
|
||||||
"the delivery must leave the archive handle open",
|
"the delivery must leave the archive handle open",
|
||||||
)
|
)
|
||||||
|
|
||||||
env.sweeper.ExportSweep(context.Background())
|
env.sweeper.ExportSweep(context.Background())
|
||||||
|
|
||||||
require.True(
|
require.True(
|
||||||
t, env.writers.ExportHasArchiveWriter(tgt.ID),
|
t, env.eng.ExportHasArchiveWriter(webhookID),
|
||||||
"the delivery's registry entry must survive the sweep",
|
"the delivery's registry entry must survive the sweep",
|
||||||
)
|
)
|
||||||
assert.False(
|
assert.False(
|
||||||
t, env.writers.ExportArchiveHandleOpen(tgt.ID),
|
t, env.eng.ExportArchiveHandleOpen(webhookID),
|
||||||
"the sweep must leave the archive closed",
|
"the sweep must leave the archive closed",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestArchiveSweep_ClosesHandleBeforeReopening proves the sweep
|
|
||||||
// closes the handle it finds open before it reopens the file.
|
|
||||||
// TestArchiveSweep_LeavesArchiveClosed cannot see this: without the
|
|
||||||
// close, the reopen replaces the handle without closing it, the
|
|
||||||
// sweep then closes only the new one, and one connection leaks per
|
|
||||||
// archive per sweep.
|
|
||||||
func TestArchiveSweep_ClosesHandleBeforeReopening(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "archive.db")
|
|
||||||
|
|
||||||
w := archive.NewExportArchiveWriter(
|
|
||||||
path, archiveTestLogger(), 0,
|
|
||||||
)
|
|
||||||
|
|
||||||
require.NoError(t, w.Open(time.Hour))
|
|
||||||
|
|
||||||
before, err := w.DB().DB()
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
require.NoError(t, w.SweepExpired(time.Hour))
|
|
||||||
|
|
||||||
assert.Error(
|
|
||||||
t, before.PingContext(t.Context()),
|
|
||||||
"the handle open before the sweep must be closed by it",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveSweep_CancelledSweepLogsNoError proves a sweep whose
|
|
||||||
// context is already cancelled, as when the app stops just as a
|
|
||||||
// sweep starts, returns without an error line: stopping is not a
|
|
||||||
// failure.
|
|
||||||
func TestArchiveSweep_CancelledSweepLogsNoError(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
|
||||||
|
|
||||||
var errorLines bytes.Buffer
|
|
||||||
|
|
||||||
sweeper := archive.NewTestSweeper(
|
|
||||||
env.mainDB, env.writers,
|
|
||||||
slog.New(slog.NewTextHandler(
|
|
||||||
&errorLines,
|
|
||||||
&slog.HandlerOptions{Level: slog.LevelError},
|
|
||||||
)),
|
|
||||||
)
|
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
cancel()
|
|
||||||
|
|
||||||
sweeper.ExportSweep(ctx)
|
|
||||||
|
|
||||||
assert.Empty(
|
|
||||||
t, errorLines.String(),
|
|
||||||
"a cancelled sweep must not log at error level",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveSweep_NeverExpiryUntouched proves the sweep is a
|
// TestArchiveSweep_NeverExpiryUntouched proves the sweep is a
|
||||||
// no-op for the default retention policy, so an archive with an
|
// no-op for the default retention policy, so archives with no
|
||||||
// expiry of 0, which keeps it forever, is left alone.
|
// expiry (or the literal "never") behave exactly as before.
|
||||||
func TestArchiveSweep_NeverExpiryUntouched(t *testing.T) {
|
func TestArchiveSweep_NeverExpiryUntouched(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
for _, configJSON := range []string{
|
||||||
|
`{"expiry":"never"}`,
|
||||||
|
`{"expiry":""}`,
|
||||||
|
"",
|
||||||
|
} {
|
||||||
|
env := setupSweeperTest(t)
|
||||||
|
|
||||||
tgt := env.seedDatabaseTarget(t, 0, "")
|
webhookID := env.seedDatabaseTarget(t, configJSON)
|
||||||
path := env.seedArchiveRows(
|
path := env.seedArchiveRows(
|
||||||
t, tgt,
|
t, webhookID,
|
||||||
time.Now().Add(-10000*time.Hour),
|
time.Now().Add(-10000*time.Hour),
|
||||||
)
|
)
|
||||||
|
|
||||||
env.sweeper.ExportSweep(context.Background())
|
env.sweeper.ExportSweep(context.Background())
|
||||||
|
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t, []string{sweepRowOld}, archivedEventIDs(t, path),
|
t, []string{sweepRowOld}, archivedEventIDs(t, path),
|
||||||
"an expiry of 0 must keep rows forever",
|
"config %q must keep rows forever", configJSON,
|
||||||
)
|
)
|
||||||
assert.False(
|
assert.False(
|
||||||
t, env.writers.ExportHasArchiveWriter(tgt.ID),
|
t, env.eng.ExportHasArchiveWriter(webhookID),
|
||||||
"an expiry of 0 must leave no registry entry behind",
|
"config %q must leave no registry entry behind",
|
||||||
)
|
configJSON,
|
||||||
|
)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestArchiveSweep_NeverExpirySkipsBeforeOpening pins the
|
// TestArchiveSweep_NeverExpirySkipsBeforeOpening pins the
|
||||||
@@ -827,18 +714,18 @@ func TestArchiveSweep_NeverExpiryUntouched(t *testing.T) {
|
|||||||
//
|
//
|
||||||
// The spec is stronger than that — a "never" archive is skipped
|
// The spec is stronger than that — a "never" archive is skipped
|
||||||
// before any file is touched — so the archive here exists but has
|
// before any file is touched — so the archive here exists but has
|
||||||
// never been migrated. Opening it at all would apply the archive
|
// never been migrated. Opening it at all would run AutoMigrate
|
||||||
// schema and create the archive table, which is exactly what must
|
// and create the archive table, which is exactly what must not
|
||||||
// not happen.
|
// happen.
|
||||||
func TestArchiveSweep_NeverExpirySkipsBeforeOpening(
|
func TestArchiveSweep_NeverExpirySkipsBeforeOpening(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
env := setupSweeperTest(t)
|
||||||
|
|
||||||
tgt := env.seedDatabaseTarget(t, 0, "")
|
webhookID := env.seedDatabaseTarget(t, `{"expiry":"never"}`)
|
||||||
path := env.archivePath(tgt)
|
path := env.archivePath(webhookID)
|
||||||
|
|
||||||
seedUnmigratedArchive(t, path)
|
seedUnmigratedArchive(t, path)
|
||||||
require.False(t, archiveTableExists(t, path))
|
require.False(t, archiveTableExists(t, path))
|
||||||
@@ -853,7 +740,7 @@ func TestArchiveSweep_NeverExpirySkipsBeforeOpening(
|
|||||||
|
|
||||||
// seedUnmigratedArchive creates an archive file that exists but
|
// seedUnmigratedArchive creates an archive file that exists but
|
||||||
// carries no archive schema, so any open of it is observable: the
|
// carries no archive schema, so any open of it is observable: the
|
||||||
// archive table appears only if something applied the schema.
|
// archive table appears only if something ran AutoMigrate.
|
||||||
func seedUnmigratedArchive(t *testing.T, path string) {
|
func seedUnmigratedArchive(t *testing.T, path string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -877,20 +764,20 @@ func archiveTableExists(t *testing.T, path string) bool {
|
|||||||
|
|
||||||
return openArchiveDBForRead(t, path).
|
return openArchiveDBForRead(t, path).
|
||||||
Migrator().
|
Migrator().
|
||||||
HasTable(&archive.ExportArchivedEvent{})
|
HasTable(&delivery.ExportArchivedEvent{})
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestArchiveSweep_DoesNotCreateArchiveFile proves the sweep
|
// TestArchiveSweep_DoesNotCreateArchiveFile proves the sweep
|
||||||
// never conjures an archive: a database target that has never
|
// never conjures an archive: a webhook with a database target
|
||||||
// received an event must still have no archive file (nor SQLite
|
// that has never received an event must still have no archive
|
||||||
// sidecar) after a sweep, and no registry entry either.
|
// file (nor SQLite sidecar) after a sweep.
|
||||||
func TestArchiveSweep_DoesNotCreateArchiveFile(t *testing.T) {
|
func TestArchiveSweep_DoesNotCreateArchiveFile(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
env := setupSweeperTest(t)
|
||||||
|
|
||||||
tgt := env.seedDatabaseTarget(t, time.Hour, "")
|
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
path := env.archivePath(tgt)
|
path := env.archivePath(webhookID)
|
||||||
|
|
||||||
require.NoFileExists(t, path)
|
require.NoFileExists(t, path)
|
||||||
|
|
||||||
@@ -902,11 +789,6 @@ func TestArchiveSweep_DoesNotCreateArchiveFile(t *testing.T) {
|
|||||||
"the sweep must not create an archive file",
|
"the sweep must not create an archive file",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
assert.False(
|
|
||||||
t, env.writers.ExportHasArchiveWriter(tgt.ID),
|
|
||||||
"the sweep must leave no registry entry behind",
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestArchiveSweep_DoesNotCreateAfterWriterExists covers the
|
// TestArchiveSweep_DoesNotCreateAfterWriterExists covers the
|
||||||
@@ -918,11 +800,11 @@ func TestArchiveSweep_DoesNotCreateAfterWriterExists(
|
|||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
env := setupSweeperTest(t)
|
||||||
|
|
||||||
tgt := env.seedDatabaseTarget(t, time.Hour, "")
|
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
|
|
||||||
path, err := env.writers.ExportEnsureArchiveWriter(tgt.ID)
|
path, err := env.eng.ExportEnsureArchiveWriter(webhookID)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.NoFileExists(t, path)
|
require.NoFileExists(t, path)
|
||||||
|
|
||||||
@@ -937,17 +819,17 @@ func TestArchiveSweep_DoesNotCreateAfterWriterExists(
|
|||||||
func TestArchiveSweep_SkipsDeletedWebhookTargets(t *testing.T) {
|
func TestArchiveSweep_SkipsDeletedWebhookTargets(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
env := setupSweeperTest(t)
|
||||||
|
|
||||||
tgt := env.seedDatabaseTarget(t, time.Hour, "")
|
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
path := env.seedArchiveRows(
|
path := env.seedArchiveRows(
|
||||||
t, tgt, time.Now().Add(-48*time.Hour),
|
t, webhookID, time.Now().Add(-48*time.Hour),
|
||||||
)
|
)
|
||||||
|
|
||||||
require.NoError(
|
require.NoError(
|
||||||
t,
|
t,
|
||||||
env.mainDB.DB().
|
env.mainDB.DB().
|
||||||
Where("webhook_id = ?", tgt.WebhookID).
|
Where("webhook_id = ?", webhookID).
|
||||||
Delete(&database.Target{}).Error,
|
Delete(&database.Target{}).Error,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -960,23 +842,33 @@ func TestArchiveSweep_SkipsDeletedWebhookTargets(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TestArchiveSweep_ConcurrentWrites proves the sweep serialises
|
// TestArchiveSweep_ConcurrentWrites proves the sweep serialises
|
||||||
// against writes through the target's writer mutex. Run under
|
// against writes through the per-webhook writer mutex. Run
|
||||||
// -race, an unsynchronised sweep would be caught here.
|
// under -race, an unsynchronised sweep would be caught here.
|
||||||
func TestArchiveSweep_ConcurrentWrites(t *testing.T) {
|
func TestArchiveSweep_ConcurrentWrites(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
env := setupSweeperTest(t)
|
||||||
|
|
||||||
tgt := env.seedDatabaseTarget(t, time.Hour, "")
|
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
|
|
||||||
|
webhookDB := testWebhookDB(t)
|
||||||
|
|
||||||
|
// The deliveries are seeded up front, on the test's own
|
||||||
|
// goroutine: the seed helpers assert, and testify assertions
|
||||||
|
// must not run off the test goroutine.
|
||||||
deliveries := make(
|
deliveries := make(
|
||||||
[]*database.Delivery, 0, sweepConcurrentWrites,
|
[]*database.Delivery, 0, sweepConcurrentWrites,
|
||||||
)
|
)
|
||||||
|
|
||||||
for range sweepConcurrentWrites {
|
for range sweepConcurrentWrites {
|
||||||
|
event := seedEvent(t, webhookDB, `{"n":1}`)
|
||||||
|
event.WebhookID = webhookID
|
||||||
|
|
||||||
deliveries = append(
|
deliveries = append(
|
||||||
deliveries,
|
deliveries,
|
||||||
databaseTargetDelivery(newEvent(`{"n":1}`), tgt),
|
seedDatabaseTargetDelivery(
|
||||||
|
t, webhookDB, event, `{"expiry":"1h"}`,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -988,7 +880,7 @@ func TestArchiveSweep_ConcurrentWrites(t *testing.T) {
|
|||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
|
|
||||||
for _, d := range deliveries {
|
for _, d := range deliveries {
|
||||||
_ = env.writers.Write(d)
|
env.eng.ExportDeliverDatabase(webhookDB, d)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
@@ -1002,7 +894,7 @@ func TestArchiveSweep_ConcurrentWrites(t *testing.T) {
|
|||||||
|
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
|
|
||||||
assert.FileExists(t, env.archivePath(tgt))
|
assert.FileExists(t, env.archivePath(webhookID))
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestArchiveSweeper_StopsCleanly proves the background loop
|
// TestArchiveSweeper_StopsCleanly proves the background loop
|
||||||
@@ -1010,11 +902,11 @@ func TestArchiveSweep_ConcurrentWrites(t *testing.T) {
|
|||||||
func TestArchiveSweeper_StopsCleanly(t *testing.T) {
|
func TestArchiveSweeper_StopsCleanly(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
env := setupSweeperTest(t)
|
||||||
|
|
||||||
tgt := env.seedDatabaseTarget(t, time.Hour, "")
|
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
env.seedArchiveRows(
|
env.seedArchiveRows(
|
||||||
t, tgt, time.Now().Add(-48*time.Hour),
|
t, webhookID, time.Now().Add(-48*time.Hour),
|
||||||
)
|
)
|
||||||
|
|
||||||
env.sweeper.ExportSetInterval(time.Millisecond)
|
env.sweeper.ExportSetInterval(time.Millisecond)
|
||||||
@@ -1038,7 +930,7 @@ func TestArchiveSweeper_StopHookHonoursStopTimeout(
|
|||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
env := setupSweeperTest(t)
|
||||||
|
|
||||||
lc := &recordingLifecycle{}
|
lc := &recordingLifecycle{}
|
||||||
env.sweeper.ExportRegisterHooks(lc)
|
env.sweeper.ExportRegisterHooks(lc)
|
||||||
@@ -76,20 +76,12 @@ func (cb *CircuitBreaker) Allow() bool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// CooldownRemaining returns how long a delivery that Allow refused
|
// CooldownRemaining returns how much time is left before
|
||||||
// should wait before it is tried again. Closed, it returns zero.
|
// an open circuit transitions to half-open.
|
||||||
// Open, it returns what is left of the cooldown, or zero once that
|
|
||||||
// has passed. Half-open, it returns the whole cooldown: the one
|
|
||||||
// probe delivery is still in flight, and if it fails the circuit
|
|
||||||
// reopens for that long.
|
|
||||||
func (cb *CircuitBreaker) CooldownRemaining() time.Duration {
|
func (cb *CircuitBreaker) CooldownRemaining() time.Duration {
|
||||||
cb.mu.Lock()
|
cb.mu.Lock()
|
||||||
defer cb.mu.Unlock()
|
defer cb.mu.Unlock()
|
||||||
|
|
||||||
if cb.state == CircuitHalfOpen {
|
|
||||||
return cb.cooldown
|
|
||||||
}
|
|
||||||
|
|
||||||
if cb.state != CircuitOpen {
|
if cb.state != CircuitOpen {
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
@@ -102,20 +94,6 @@ func (cb *CircuitBreaker) CooldownRemaining() time.Duration {
|
|||||||
return remaining
|
return remaining
|
||||||
}
|
}
|
||||||
|
|
||||||
// StateAndCooldown returns the circuit state and, while the circuit is
|
|
||||||
// open, what is left of the cooldown, or zero once that has passed.
|
|
||||||
// Both are read under one lock, so they always agree.
|
|
||||||
func (cb *CircuitBreaker) StateAndCooldown() (CircuitState, time.Duration) {
|
|
||||||
cb.mu.Lock()
|
|
||||||
defer cb.mu.Unlock()
|
|
||||||
|
|
||||||
if cb.state != CircuitOpen {
|
|
||||||
return cb.state, 0
|
|
||||||
}
|
|
||||||
|
|
||||||
return cb.state, max(cb.cooldown-time.Since(cb.lastFailure), 0)
|
|
||||||
}
|
|
||||||
|
|
||||||
// RecordSuccess records a successful delivery and resets
|
// RecordSuccess records a successful delivery and resets
|
||||||
// the circuit breaker to closed state.
|
// the circuit breaker to closed state.
|
||||||
func (cb *CircuitBreaker) RecordSuccess() {
|
func (cb *CircuitBreaker) RecordSuccess() {
|
||||||
|
|||||||
@@ -267,7 +267,7 @@ func TestCircuitBreaker_CooldownRemaining_ClosedReturnsZero(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCircuitBreaker_CooldownRemaining_HalfOpenReturnsCooldown(
|
func TestCircuitBreaker_CooldownRemaining_HalfOpenReturnsZero(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
@@ -282,11 +282,9 @@ func TestCircuitBreaker_CooldownRemaining_HalfOpenReturnsCooldown(
|
|||||||
|
|
||||||
require.True(t, cb.Allow())
|
require.True(t, cb.Allow())
|
||||||
|
|
||||||
// The cooldown newShortCooldownCB gives the breaker.
|
assert.Equal(t, time.Duration(0),
|
||||||
assert.Equal(t, 50*time.Millisecond,
|
|
||||||
cb.CooldownRemaining(),
|
cb.CooldownRemaining(),
|
||||||
"a delivery refused while half-open should wait "+
|
"half-open circuit should have zero cooldown remaining",
|
||||||
"a whole cooldown",
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -46,7 +46,12 @@ func TestClientForRequest_TimeoutKeepsSSRFGuard(t *testing.T) {
|
|||||||
t.Run(target, func(t *testing.T) {
|
t.Run(target, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
client := engine.ExportClientForRequest(5, nil)
|
cfg := &delivery.HTTPTargetConfig{
|
||||||
|
URL: target,
|
||||||
|
Timeout: 5,
|
||||||
|
}
|
||||||
|
|
||||||
|
client := engine.ExportClientForRequest(cfg, nil)
|
||||||
|
|
||||||
require.NotSame(t, engine.ExportClient(), client,
|
require.NotSame(t, engine.ExportClient(), client,
|
||||||
"a per-target timeout must yield a "+
|
"a per-target timeout must yield a "+
|
||||||
@@ -96,7 +101,11 @@ func TestClientForRequest_NoTimeoutUnchanged(t *testing.T) {
|
|||||||
|
|
||||||
engine := newSSRFTestEngine()
|
engine := newSSRFTestEngine()
|
||||||
|
|
||||||
client := engine.ExportClientForRequest(0, nil)
|
cfg := &delivery.HTTPTargetConfig{
|
||||||
|
URL: "https://example.com/hook",
|
||||||
|
}
|
||||||
|
|
||||||
|
client := engine.ExportClientForRequest(cfg, nil)
|
||||||
|
|
||||||
assert.Same(t, engine.ExportClient(), client,
|
assert.Same(t, engine.ExportClient(), client,
|
||||||
"without a per-target timeout the shared client "+
|
"without a per-target timeout the shared client "+
|
||||||
@@ -114,8 +123,15 @@ func TestClientForRequest_HeadersKeepSSRFGuard(t *testing.T) {
|
|||||||
|
|
||||||
engine := newSSRFTestEngine()
|
engine := newSSRFTestEngine()
|
||||||
|
|
||||||
|
cfg := &delivery.HTTPTargetConfig{
|
||||||
|
URL: "https://example.com/with-headers",
|
||||||
|
Headers: map[string]string{
|
||||||
|
"X-Api-Key": "configured",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
client := engine.ExportClientForRequest(
|
client := engine.ExportClientForRequest(
|
||||||
0, []string{probeHeaderName},
|
cfg, []string{"X-Api-Key"},
|
||||||
)
|
)
|
||||||
|
|
||||||
require.NotNil(t, client.CheckRedirect,
|
require.NotNil(t, client.CheckRedirect,
|
||||||
|
|||||||
@@ -1,188 +0,0 @@
|
|||||||
package delivery_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/google/uuid"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"gorm.io/gorm"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// targetTotals reads one target's totals from a webhook database, all
|
|
||||||
// zero when it has no row.
|
|
||||||
func targetTotals(
|
|
||||||
t *testing.T, db *gorm.DB, targetID string,
|
|
||||||
) database.TargetTotals {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var rows []database.TargetTotals
|
|
||||||
|
|
||||||
require.NoError(t, db.Where("target_id = ?", targetID).
|
|
||||||
Find(&rows).Error)
|
|
||||||
|
|
||||||
if len(rows) == 0 {
|
|
||||||
return database.TargetTotals{TargetID: targetID}
|
|
||||||
}
|
|
||||||
|
|
||||||
return rows[0]
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestUpdateDeliveryStatus_FinishTimeAndTargetTotals pins what a status
|
|
||||||
// write records for the webhook page's statistics: the time a delivery
|
|
||||||
// finished, set only when it becomes delivered or failed and stored in
|
|
||||||
// the same form as created_at, and one more on its target's delivered
|
|
||||||
// or failed total.
|
|
||||||
func TestUpdateDeliveryStatus_FinishTimeAndTargetTotals(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
status database.DeliveryStatus
|
|
||||||
finished bool
|
|
||||||
delivered int64
|
|
||||||
failed int64
|
|
||||||
}{
|
|
||||||
{database.DeliveryStatusRetrying, false, 0, 0},
|
|
||||||
{database.DeliveryStatusDelivered, true, 1, 0},
|
|
||||||
{database.DeliveryStatusFailed, true, 0, 1},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(string(tt.status), func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := testWebhookDB(t)
|
|
||||||
e := testEngine(t, 1)
|
|
||||||
event := seedEvent(t, db, `{}`)
|
|
||||||
targetID := uuid.New().String()
|
|
||||||
d := seedDelivery(
|
|
||||||
t, db, event.ID, targetID,
|
|
||||||
database.DeliveryStatusPending,
|
|
||||||
)
|
|
||||||
|
|
||||||
before := time.Now()
|
|
||||||
|
|
||||||
require.NoError(t, e.ExportUpdateDeliveryStatus(
|
|
||||||
db, &d, tt.status,
|
|
||||||
))
|
|
||||||
|
|
||||||
var stored database.Delivery
|
|
||||||
|
|
||||||
require.NoError(t, db.First(&stored, "id = ?", d.ID).Error)
|
|
||||||
assert.Equal(t, tt.status, stored.Status)
|
|
||||||
|
|
||||||
if tt.finished {
|
|
||||||
require.NotNil(t, stored.FinishedAt)
|
|
||||||
assert.False(t, stored.FinishedAt.Before(before))
|
|
||||||
|
|
||||||
var createdAt, finishedAt string
|
|
||||||
|
|
||||||
require.NoError(t, db.Raw(
|
|
||||||
`SELECT CAST(created_at AS TEXT),
|
|
||||||
CAST(finished_at AS TEXT)
|
|
||||||
FROM deliveries WHERE id = ?`, d.ID,
|
|
||||||
).Row().Scan(&createdAt, &finishedAt))
|
|
||||||
assertStoredTime(t, createdAt)
|
|
||||||
assertStoredTime(t, finishedAt)
|
|
||||||
} else {
|
|
||||||
assert.Nil(t, stored.FinishedAt)
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equal(t, database.TargetTotals{
|
|
||||||
TargetID: targetID,
|
|
||||||
Delivered: tt.delivered,
|
|
||||||
Failed: tt.failed,
|
|
||||||
}, targetTotals(t, db, targetID))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestUpdateDeliveryStatus_DeletedDeliveryIsNotCounted covers a
|
|
||||||
// delivery retention deleted while the engine still held it. Failing
|
|
||||||
// it afterwards writes no row, so it adds no failure either: retention
|
|
||||||
// has already counted what it removed.
|
|
||||||
func TestUpdateDeliveryStatus_DeletedDeliveryIsNotCounted(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := testWebhookDB(t)
|
|
||||||
e := testEngine(t, 1)
|
|
||||||
event := seedEvent(t, db, `{}`)
|
|
||||||
targetID := uuid.New().String()
|
|
||||||
d := seedDelivery(
|
|
||||||
t, db, event.ID, targetID,
|
|
||||||
database.DeliveryStatusRetrying,
|
|
||||||
)
|
|
||||||
|
|
||||||
require.NoError(t, db.Unscoped().
|
|
||||||
Delete(&database.Delivery{}, "id = ?", d.ID).Error)
|
|
||||||
|
|
||||||
require.NoError(t, e.ExportUpdateDeliveryStatus(
|
|
||||||
db, &d, database.DeliveryStatusFailed,
|
|
||||||
))
|
|
||||||
|
|
||||||
assert.Equal(t, database.TargetTotals{TargetID: targetID},
|
|
||||||
targetTotals(t, db, targetID))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestUpdateDeliveryStatus_FinishedDeliveryIsNotSettledAgain covers a
|
|
||||||
// delivery settled a second time, as recovery can do when a worker has
|
|
||||||
// settled it since recovery read it. Neither status writes over the
|
|
||||||
// first, and the totals do not move.
|
|
||||||
func TestUpdateDeliveryStatus_FinishedDeliveryIsNotSettledAgain(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
finished := []database.DeliveryStatus{
|
|
||||||
database.DeliveryStatusDelivered,
|
|
||||||
database.DeliveryStatusFailed,
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, first := range finished {
|
|
||||||
t.Run(string(first), func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := testWebhookDB(t)
|
|
||||||
e := testEngine(t, 1)
|
|
||||||
event := seedEvent(t, db, `{}`)
|
|
||||||
targetID := uuid.New().String()
|
|
||||||
d := seedDelivery(
|
|
||||||
t, db, event.ID, targetID,
|
|
||||||
database.DeliveryStatusRetrying,
|
|
||||||
)
|
|
||||||
|
|
||||||
// The delivery as recovery read it, before the worker
|
|
||||||
// settled it.
|
|
||||||
readBefore := d
|
|
||||||
|
|
||||||
require.NoError(t, e.ExportUpdateDeliveryStatus(
|
|
||||||
db, &d, first,
|
|
||||||
))
|
|
||||||
|
|
||||||
var settled database.Delivery
|
|
||||||
|
|
||||||
require.NoError(t, db.First(&settled, "id = ?", d.ID).Error)
|
|
||||||
require.NotNil(t, settled.FinishedAt)
|
|
||||||
|
|
||||||
totals := targetTotals(t, db, targetID)
|
|
||||||
|
|
||||||
for _, again := range finished {
|
|
||||||
stale := readBefore
|
|
||||||
|
|
||||||
require.NoError(t, e.ExportUpdateDeliveryStatus(
|
|
||||||
db, &stale, again,
|
|
||||||
))
|
|
||||||
}
|
|
||||||
|
|
||||||
var stored database.Delivery
|
|
||||||
|
|
||||||
require.NoError(t, db.First(&stored, "id = ?", d.ID).Error)
|
|
||||||
assert.Equal(t, first, stored.Status)
|
|
||||||
require.NotNil(t, stored.FinishedAt)
|
|
||||||
assert.True(t, settled.FinishedAt.Equal(*stored.FinishedAt))
|
|
||||||
assert.Equal(t, totals, targetTotals(t, db, targetID))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+508
-472
File diff suppressed because it is too large
Load Diff
@@ -1,14 +1,15 @@
|
|||||||
package delivery_test
|
package delivery_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -17,10 +18,10 @@ import (
|
|||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
"gorm.io/driver/sqlite"
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
_ "modernc.org/sqlite"
|
_ "modernc.org/sqlite"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/database/databasetest"
|
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -42,12 +43,12 @@ func newISetup(t *testing.T) iSetup {
|
|||||||
wDB := iSeedWebhookDB(t, dbMgr, wID)
|
wDB := iSeedWebhookDB(t, dbMgr, wID)
|
||||||
|
|
||||||
return iSetup{
|
return iSetup{
|
||||||
MainDB: mainDB.DB(),
|
MainDB: mainDB,
|
||||||
DBMgr: dbMgr,
|
DBMgr: dbMgr,
|
||||||
WebhookID: wID,
|
WebhookID: wID,
|
||||||
WebhookDB: wDB,
|
WebhookDB: wDB,
|
||||||
Engine: delivery.NewTestEngineWithDB(
|
Engine: delivery.NewTestEngineWithDB(
|
||||||
mainDB,
|
database.NewTestDatabase(mainDB),
|
||||||
dbMgr,
|
dbMgr,
|
||||||
slog.New(slog.NewTextHandler(
|
slog.New(slog.NewTextHandler(
|
||||||
os.Stderr,
|
os.Stderr,
|
||||||
@@ -61,16 +62,34 @@ func newISetup(t *testing.T) iSetup {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// iMainDB opens a main database through database.Open, the way the
|
func iMainDB(t *testing.T) *gorm.DB {
|
||||||
// service opens it, so these tests cannot pass against journal and
|
|
||||||
// locking settings production does not use.
|
|
||||||
func iMainDB(t *testing.T) *database.Database {
|
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
db, err := database.Open(t.TempDir(), slog.New(slog.DiscardHandler))
|
dbPath := filepath.Join(
|
||||||
|
t.TempDir(), "main-test.db",
|
||||||
|
)
|
||||||
|
|
||||||
|
// Opened the way the service opens the main database, so these
|
||||||
|
// tests cannot pass against journal and locking settings
|
||||||
|
// production does not use.
|
||||||
|
sqlDB, err := database.OpenSQLite(
|
||||||
|
dbPath, database.SQLiteModeCreate,
|
||||||
|
)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
t.Cleanup(func() { _ = db.Close() })
|
t.Cleanup(func() { _ = sqlDB.Close() })
|
||||||
|
|
||||||
|
db, err := gorm.Open(
|
||||||
|
sqlite.Dialector{Conn: sqlDB}, &gorm.Config{},
|
||||||
|
)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
require.NoError(t, db.AutoMigrate(
|
||||||
|
&database.Webhook{},
|
||||||
|
&database.Target{},
|
||||||
|
&database.User{},
|
||||||
|
&database.Setting{},
|
||||||
|
))
|
||||||
|
|
||||||
return db
|
return db
|
||||||
}
|
}
|
||||||
@@ -80,7 +99,7 @@ func iDBManager(
|
|||||||
) *database.WebhookDBManager {
|
) *database.WebhookDBManager {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
return databasetest.NewWebhookDBManager(t, t.TempDir())
|
return database.NewTestWebhookDBManager(t.TempDir())
|
||||||
}
|
}
|
||||||
|
|
||||||
func iSeedWebhookDB(
|
func iSeedWebhookDB(
|
||||||
@@ -96,6 +115,17 @@ func iSeedWebhookDB(
|
|||||||
return db
|
return db
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func iHTTPConfig(url string) string {
|
||||||
|
cfg := delivery.HTTPTargetConfig{URL: url}
|
||||||
|
|
||||||
|
data, err := json.Marshal(cfg)
|
||||||
|
if err != nil {
|
||||||
|
panic("failed to marshal HTTPTargetConfig")
|
||||||
|
}
|
||||||
|
|
||||||
|
return string(data)
|
||||||
|
}
|
||||||
|
|
||||||
func iEngine(
|
func iEngine(
|
||||||
t *testing.T, workers int,
|
t *testing.T, workers int,
|
||||||
) *delivery.Engine {
|
) *delivery.Engine {
|
||||||
@@ -157,24 +187,24 @@ func iSeedDelivery(
|
|||||||
func iTask(
|
func iTask(
|
||||||
d database.Delivery,
|
d database.Delivery,
|
||||||
event database.Event,
|
event database.Event,
|
||||||
webhookID, targetID, name, targetURL string,
|
webhookID, targetID, name, config string,
|
||||||
maxAttempts, attemptNum int,
|
maxRetries, attemptNum int,
|
||||||
body *string,
|
body *string,
|
||||||
) delivery.Task {
|
) delivery.Task {
|
||||||
return delivery.Task{
|
return delivery.Task{
|
||||||
DeliveryID: d.ID,
|
DeliveryID: d.ID,
|
||||||
EventID: event.ID,
|
EventID: event.ID,
|
||||||
WebhookID: webhookID,
|
WebhookID: webhookID,
|
||||||
TargetID: targetID,
|
TargetID: targetID,
|
||||||
TargetName: name,
|
TargetName: name,
|
||||||
TargetType: database.TargetTypeHTTP,
|
TargetType: database.TargetTypeHTTP,
|
||||||
TargetURL: targetURL,
|
TargetConfig: config,
|
||||||
MaxAttempts: maxAttempts,
|
MaxRetries: maxRetries,
|
||||||
Method: event.Method,
|
Method: event.Method,
|
||||||
Headers: event.Headers,
|
Headers: event.Headers,
|
||||||
ContentType: event.ContentType,
|
ContentType: event.ContentType,
|
||||||
Body: body,
|
Body: body,
|
||||||
AttemptNum: attemptNum,
|
AttemptNum: attemptNum,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -227,9 +257,10 @@ func TestProcessNewTask_InlineBody(t *testing.T) {
|
|||||||
)
|
)
|
||||||
|
|
||||||
bodyStr := event.Body
|
bodyStr := event.Body
|
||||||
|
cfg := iHTTPConfig(ts.URL)
|
||||||
task := iTask(
|
task := iTask(
|
||||||
d, event, s.WebhookID, targetID,
|
d, event, s.WebhookID, targetID,
|
||||||
"test-target", ts.URL, 1, 1, &bodyStr,
|
"test-target", cfg, 0, 1, &bodyStr,
|
||||||
)
|
)
|
||||||
|
|
||||||
s.Engine.ExportProcessNewTask(
|
s.Engine.ExportProcessNewTask(
|
||||||
@@ -277,9 +308,10 @@ func TestProcessNewTask_LargeBody_FetchFromDB(
|
|||||||
database.DeliveryStatusPending,
|
database.DeliveryStatusPending,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
cfg := iHTTPConfig(ts.URL)
|
||||||
task := iTask(
|
task := iTask(
|
||||||
d, event, s.WebhookID, targetID,
|
d, event, s.WebhookID, targetID,
|
||||||
"test-large", ts.URL, 1, 1, nil,
|
"test-large", cfg, 0, 1, nil,
|
||||||
)
|
)
|
||||||
|
|
||||||
s.Engine.ExportProcessNewTask(
|
s.Engine.ExportProcessNewTask(
|
||||||
@@ -299,16 +331,16 @@ func TestProcessNewTask_InvalidWebhookID(t *testing.T) {
|
|||||||
s := newISetup(t)
|
s := newISetup(t)
|
||||||
|
|
||||||
task := delivery.Task{
|
task := delivery.Task{
|
||||||
DeliveryID: uuid.New().String(),
|
DeliveryID: uuid.New().String(),
|
||||||
EventID: uuid.New().String(),
|
EventID: uuid.New().String(),
|
||||||
WebhookID: uuid.New().String(),
|
WebhookID: uuid.New().String(),
|
||||||
TargetID: uuid.New().String(),
|
TargetID: uuid.New().String(),
|
||||||
TargetName: "test",
|
TargetName: "test",
|
||||||
TargetType: database.TargetTypeHTTP,
|
TargetType: database.TargetTypeHTTP,
|
||||||
TargetURL: "http://localhost:9999",
|
TargetConfig: iHTTPConfig("http://localhost:9999"),
|
||||||
MaxAttempts: 1,
|
MaxRetries: 0,
|
||||||
Body: nil,
|
Body: nil,
|
||||||
AttemptNum: 1,
|
AttemptNum: 1,
|
||||||
}
|
}
|
||||||
|
|
||||||
s.Engine.ExportProcessNewTask(
|
s.Engine.ExportProcessNewTask(
|
||||||
@@ -323,14 +355,9 @@ func TestProcessRetryTask_SuccessfulRetry(t *testing.T) {
|
|||||||
|
|
||||||
s := newISetup(t)
|
s := newISetup(t)
|
||||||
|
|
||||||
var receivedBody string
|
|
||||||
|
|
||||||
ts := httptest.NewServer(
|
ts := httptest.NewServer(
|
||||||
http.HandlerFunc(
|
http.HandlerFunc(
|
||||||
func(w http.ResponseWriter, r *http.Request) {
|
func(w http.ResponseWriter, _ *http.Request) {
|
||||||
body, _ := io.ReadAll(r.Body)
|
|
||||||
receivedBody = string(body)
|
|
||||||
|
|
||||||
w.WriteHeader(http.StatusOK)
|
w.WriteHeader(http.StatusOK)
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
@@ -349,6 +376,7 @@ func TestProcessRetryTask_SuccessfulRetry(t *testing.T) {
|
|||||||
)
|
)
|
||||||
|
|
||||||
bodyStr := event.Body
|
bodyStr := event.Body
|
||||||
|
cfg := iHTTPConfig(ts.URL)
|
||||||
|
|
||||||
// The target row exists because the engine confirms a scheduled
|
// The target row exists because the engine confirms a scheduled
|
||||||
// retry's target has not been deleted before it runs it. A retry
|
// retry's target has not been deleted before it runs it. A retry
|
||||||
@@ -357,20 +385,18 @@ func TestProcessRetryTask_SuccessfulRetry(t *testing.T) {
|
|||||||
// task. See https://git.eeqj.de/sneak/webhooker/issues/107.
|
// task. See https://git.eeqj.de/sneak/webhooker/issues/107.
|
||||||
iCreateTarget(
|
iCreateTarget(
|
||||||
t, s.MainDB, targetID, s.WebhookID, "retry-target",
|
t, s.MainDB, targetID, s.WebhookID, "retry-target",
|
||||||
database.TargetTypeHTTP, ts.URL, 5,
|
database.TargetTypeHTTP, cfg, 5,
|
||||||
)
|
)
|
||||||
|
|
||||||
task := iTask(
|
task := iTask(
|
||||||
d, event, s.WebhookID, targetID,
|
d, event, s.WebhookID, targetID,
|
||||||
"retry-target", ts.URL, 5, 2, &bodyStr,
|
"retry-target", cfg, 5, 2, &bodyStr,
|
||||||
)
|
)
|
||||||
|
|
||||||
s.Engine.ExportProcessRetryTask(
|
s.Engine.ExportProcessRetryTask(
|
||||||
context.TODO(), &task,
|
context.TODO(), &task,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert.Equal(t, event.Body, receivedBody)
|
|
||||||
|
|
||||||
iAssertStatus(t, s.WebhookDB, d.ID,
|
iAssertStatus(t, s.WebhookDB, d.ID,
|
||||||
database.DeliveryStatusDelivered,
|
database.DeliveryStatusDelivered,
|
||||||
)
|
)
|
||||||
@@ -395,9 +421,10 @@ func TestProcessRetryTask_SkipsNonRetryingDelivery(
|
|||||||
)
|
)
|
||||||
|
|
||||||
bodyStr := event.Body
|
bodyStr := event.Body
|
||||||
|
cfg := iHTTPConfig("http://localhost:1")
|
||||||
task := iTask(
|
task := iTask(
|
||||||
d, event, s.WebhookID, targetID,
|
d, event, s.WebhookID, targetID,
|
||||||
"skip-target", "http://localhost:1", 5, 2, &bodyStr,
|
"skip-target", cfg, 5, 2, &bodyStr,
|
||||||
)
|
)
|
||||||
|
|
||||||
s.Engine.ExportProcessRetryTask(
|
s.Engine.ExportProcessRetryTask(
|
||||||
@@ -416,14 +443,9 @@ func TestProcessRetryTask_LargeBody_FetchFromDB(
|
|||||||
|
|
||||||
s := newISetup(t)
|
s := newISetup(t)
|
||||||
|
|
||||||
var receivedBody string
|
|
||||||
|
|
||||||
ts := httptest.NewServer(
|
ts := httptest.NewServer(
|
||||||
http.HandlerFunc(
|
http.HandlerFunc(
|
||||||
func(w http.ResponseWriter, r *http.Request) {
|
func(w http.ResponseWriter, _ *http.Request) {
|
||||||
body, _ := io.ReadAll(r.Body)
|
|
||||||
receivedBody = string(body)
|
|
||||||
|
|
||||||
w.WriteHeader(http.StatusOK)
|
w.WriteHeader(http.StatusOK)
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
@@ -444,22 +466,22 @@ func TestProcessRetryTask_LargeBody_FetchFromDB(
|
|||||||
database.DeliveryStatusRetrying,
|
database.DeliveryStatusRetrying,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
cfg := iHTTPConfig(ts.URL)
|
||||||
|
|
||||||
iCreateTarget(
|
iCreateTarget(
|
||||||
t, s.MainDB, targetID, s.WebhookID, "retry-large",
|
t, s.MainDB, targetID, s.WebhookID, "retry-large",
|
||||||
database.TargetTypeHTTP, ts.URL, 5,
|
database.TargetTypeHTTP, cfg, 5,
|
||||||
)
|
)
|
||||||
|
|
||||||
task := iTask(
|
task := iTask(
|
||||||
d, event, s.WebhookID, targetID,
|
d, event, s.WebhookID, targetID,
|
||||||
"retry-large", ts.URL, 5, 2, nil,
|
"retry-large", cfg, 5, 2, nil,
|
||||||
)
|
)
|
||||||
|
|
||||||
s.Engine.ExportProcessRetryTask(
|
s.Engine.ExportProcessRetryTask(
|
||||||
context.TODO(), &task,
|
context.TODO(), &task,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert.Equal(t, largeBody, receivedBody)
|
|
||||||
|
|
||||||
iAssertStatus(t, s.WebhookDB, d.ID,
|
iAssertStatus(t, s.WebhookDB, d.ID,
|
||||||
database.DeliveryStatusDelivered,
|
database.DeliveryStatusDelivered,
|
||||||
)
|
)
|
||||||
@@ -488,11 +510,11 @@ func TestWorkerLifecycle_StartStop(t *testing.T) {
|
|||||||
task := iTask(
|
task := iTask(
|
||||||
d, event, s.WebhookID, targetID,
|
d, event, s.WebhookID, targetID,
|
||||||
"lifecycle-test", "",
|
"lifecycle-test", "",
|
||||||
1, 1, &bodyStr,
|
0, 1, &bodyStr,
|
||||||
)
|
)
|
||||||
task.TargetType = database.TargetTypeLog
|
task.TargetType = database.TargetTypeLog
|
||||||
|
|
||||||
s.Engine.ExportNotify([]delivery.Task{task})
|
s.Engine.Notify([]delivery.Task{task})
|
||||||
|
|
||||||
iWaitForDelivered(t, s.WebhookDB, d.ID)
|
iWaitForDelivered(t, s.WebhookDB, d.ID)
|
||||||
|
|
||||||
@@ -552,15 +574,16 @@ func TestWorkerLifecycle_ProcessesRetryChannel(
|
|||||||
s.Engine.ExportStart()
|
s.Engine.ExportStart()
|
||||||
|
|
||||||
bodyStr := event.Body
|
bodyStr := event.Body
|
||||||
|
cfg := iHTTPConfig(ts.URL)
|
||||||
|
|
||||||
iCreateTarget(
|
iCreateTarget(
|
||||||
t, s.MainDB, targetID, s.WebhookID, "retry-chan-test",
|
t, s.MainDB, targetID, s.WebhookID, "retry-chan-test",
|
||||||
database.TargetTypeHTTP, ts.URL, 5,
|
database.TargetTypeHTTP, cfg, 5,
|
||||||
)
|
)
|
||||||
|
|
||||||
task := iTask(
|
task := iTask(
|
||||||
d, event, s.WebhookID, targetID,
|
d, event, s.WebhookID, targetID,
|
||||||
"retry-chan-test", ts.URL, 5, 2, &bodyStr,
|
"retry-chan-test", cfg, 5, 2, &bodyStr,
|
||||||
)
|
)
|
||||||
|
|
||||||
s.Engine.ExportRetryCh() <- task
|
s.Engine.ExportRetryCh() <- task
|
||||||
@@ -626,21 +649,16 @@ func TestRecoverPendingDeliveries(t *testing.T) {
|
|||||||
|
|
||||||
iCreateTarget(t, s.MainDB, targetID,
|
iCreateTarget(t, s.MainDB, targetID,
|
||||||
s.WebhookID, "recovery-target",
|
s.WebhookID, "recovery-target",
|
||||||
database.TargetTypeLog, "", 1,
|
database.TargetTypeLog, "", 0,
|
||||||
)
|
)
|
||||||
|
|
||||||
iSeedPendingDeliveries(
|
iSeedPendingDeliveries(
|
||||||
t, s.WebhookDB, s.WebhookID, targetID, 3,
|
t, s.WebhookDB, s.WebhookID, targetID, 3,
|
||||||
)
|
)
|
||||||
|
|
||||||
// A recovered delivery still carries its event's query string.
|
|
||||||
require.NoError(t, s.WebhookDB.Model(&database.Event{}).
|
|
||||||
Where("webhook_id = ?", s.WebhookID).
|
|
||||||
Update("raw_query", eventQuery).Error)
|
|
||||||
|
|
||||||
s.Engine.ExportRecoverPendingDeliveries(
|
s.Engine.ExportRecoverPendingDeliveries(
|
||||||
context.Background(), s.WebhookDB,
|
context.Background(), s.WebhookDB,
|
||||||
s.WebhookID, atStart,
|
s.WebhookID,
|
||||||
)
|
)
|
||||||
|
|
||||||
for i := range 3 {
|
for i := range 3 {
|
||||||
@@ -652,8 +670,6 @@ func TestRecoverPendingDeliveries(t *testing.T) {
|
|||||||
database.TargetTypeLog,
|
database.TargetTypeLog,
|
||||||
task.TargetType,
|
task.TargetType,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert.Equal(t, eventQuery, task.RawQuery)
|
|
||||||
case <-time.After(2 * time.Second):
|
case <-time.After(2 * time.Second):
|
||||||
t.Fatalf("expected task %d", i)
|
t.Fatalf("expected task %d", i)
|
||||||
}
|
}
|
||||||
@@ -666,18 +682,18 @@ func iCreateTarget(
|
|||||||
mainDB *gorm.DB,
|
mainDB *gorm.DB,
|
||||||
targetID, webhookID, name string,
|
targetID, webhookID, name string,
|
||||||
targetType database.TargetType,
|
targetType database.TargetType,
|
||||||
targetURL string,
|
config string,
|
||||||
maxAttempts int,
|
maxRetries int,
|
||||||
) {
|
) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
target := database.Target{
|
target := database.Target{
|
||||||
WebhookID: webhookID,
|
WebhookID: webhookID,
|
||||||
Name: name,
|
Name: name,
|
||||||
Type: targetType,
|
Type: targetType,
|
||||||
Active: true,
|
Active: true,
|
||||||
URL: targetURL,
|
Config: config,
|
||||||
MaxAttempts: maxAttempts,
|
MaxRetries: maxRetries,
|
||||||
}
|
}
|
||||||
target.ID = targetID
|
target.ID = targetID
|
||||||
|
|
||||||
@@ -717,7 +733,7 @@ func TestRecoverWebhookDeliveries_RetryingDeliveries(
|
|||||||
iCreateTarget(t, s.MainDB, targetID,
|
iCreateTarget(t, s.MainDB, targetID,
|
||||||
s.WebhookID, "retry-recovery",
|
s.WebhookID, "retry-recovery",
|
||||||
database.TargetTypeHTTP,
|
database.TargetTypeHTTP,
|
||||||
"http://example.com/hook", 5,
|
iHTTPConfig("http://example.com/hook"), 5,
|
||||||
)
|
)
|
||||||
|
|
||||||
event := iSeedEvent(
|
event := iSeedEvent(
|
||||||
@@ -737,7 +753,7 @@ func TestRecoverWebhookDeliveries_RetryingDeliveries(
|
|||||||
)
|
)
|
||||||
|
|
||||||
s.Engine.ExportRecoverWebhookDeliveries(
|
s.Engine.ExportRecoverWebhookDeliveries(
|
||||||
context.Background(), s.WebhookID, atStart,
|
context.Background(), s.WebhookID,
|
||||||
)
|
)
|
||||||
|
|
||||||
select {
|
select {
|
||||||
@@ -777,7 +793,7 @@ func iSeedRetryingWithType(
|
|||||||
|
|
||||||
iCreateTarget(t, s.MainDB, targetID,
|
iCreateTarget(t, s.MainDB, targetID,
|
||||||
s.WebhookID, "mutated-target", targetType,
|
s.WebhookID, "mutated-target", targetType,
|
||||||
"http://example.com/hook", 5,
|
iHTTPConfig("http://example.com/hook"), 5,
|
||||||
)
|
)
|
||||||
|
|
||||||
event := iSeedEvent(
|
event := iSeedEvent(
|
||||||
@@ -861,7 +877,7 @@ func TestRecoverSingleRetry_TypeNoLongerRetries(
|
|||||||
)
|
)
|
||||||
|
|
||||||
s.Engine.ExportRecoverWebhookDeliveries(
|
s.Engine.ExportRecoverWebhookDeliveries(
|
||||||
context.Background(), s.WebhookID, atStart,
|
context.Background(), s.WebhookID,
|
||||||
)
|
)
|
||||||
|
|
||||||
iAssertTerminallyFailed(
|
iAssertTerminallyFailed(
|
||||||
@@ -869,7 +885,7 @@ func TestRecoverSingleRetry_TypeNoLongerRetries(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRecoverSingleRetry_OnTick_TypeNoLongerRetries(
|
func TestSweepSingleRetry_TypeNoLongerRetries(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
@@ -884,8 +900,8 @@ func TestRecoverSingleRetry_OnTick_TypeNoLongerRetries(
|
|||||||
t, s, database.TargetTypeDatabase,
|
t, s, database.TargetTypeDatabase,
|
||||||
)
|
)
|
||||||
|
|
||||||
s.Engine.ExportRecoverWebhookDeliveries(
|
s.Engine.ExportSweepWebhookRetries(
|
||||||
context.Background(), s.WebhookID, onTick,
|
context.Background(), s.WebhookID,
|
||||||
)
|
)
|
||||||
|
|
||||||
iAssertTerminallyFailed(
|
iAssertTerminallyFailed(
|
||||||
@@ -915,14 +931,14 @@ func TestFailUnretryableRetry_WritesNoTargetRow(
|
|||||||
|
|
||||||
targetID := uuid.New().String()
|
targetID := uuid.New().String()
|
||||||
|
|
||||||
// A Slack incoming-webhook URL: the target's URL IS the
|
// A Slack incoming-webhook URL: the target config IS the
|
||||||
// credential, which is what makes a leaked target row a
|
// credential, which is what makes a leaked target row a
|
||||||
// disclosure rather than a curiosity.
|
// disclosure rather than a curiosity.
|
||||||
hookURL := "https://hooks.slack.com/services/T00/B00/x"
|
hookURL := "https://hooks.slack.com/services/T00/B00/x"
|
||||||
|
|
||||||
iCreateTarget(t, s.MainDB, targetID,
|
iCreateTarget(t, s.MainDB, targetID,
|
||||||
s.WebhookID, "credential-bearing",
|
s.WebhookID, "credential-bearing",
|
||||||
database.TargetTypeLog, hookURL, 5,
|
database.TargetTypeLog, iHTTPConfig(hookURL), 5,
|
||||||
)
|
)
|
||||||
|
|
||||||
event := iSeedEvent(
|
event := iSeedEvent(
|
||||||
@@ -936,20 +952,37 @@ func TestFailUnretryableRetry_WritesNoTargetRow(
|
|||||||
|
|
||||||
iSeedFailedResult(t, s.WebhookDB, d.ID)
|
iSeedFailedResult(t, s.WebhookDB, d.ID)
|
||||||
|
|
||||||
s.Engine.ExportRecoverWebhookDeliveries(
|
s.Engine.ExportSweepWebhookRetries(
|
||||||
context.Background(), s.WebhookID, onTick,
|
context.Background(), s.WebhookID,
|
||||||
)
|
)
|
||||||
|
|
||||||
iAssertStatus(t, s.WebhookDB, d.ID,
|
iAssertStatus(t, s.WebhookDB, d.ID,
|
||||||
database.DeliveryStatusFailed,
|
database.DeliveryStatusFailed,
|
||||||
)
|
)
|
||||||
|
|
||||||
// The per-webhook database has no targets table, so writing a
|
// The table exists in the per-webhook database because GORM
|
||||||
// target row there would have failed the status write above.
|
// migrates the Delivery relation's model alongside it. It must
|
||||||
assert.False(t, s.WebhookDB.Migrator().HasTable("targets"),
|
// stay empty.
|
||||||
|
var targetRows int64
|
||||||
|
|
||||||
|
require.NoError(t, s.WebhookDB.
|
||||||
|
Table("targets").
|
||||||
|
Count(&targetRows).Error)
|
||||||
|
|
||||||
|
assert.Zero(t, targetRows,
|
||||||
"orphaned-retry terminal failure wrote a target row "+
|
"orphaned-retry terminal failure wrote a target row "+
|
||||||
"into the per-webhook event database",
|
"into the per-webhook event database",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var configs []string
|
||||||
|
|
||||||
|
require.NoError(t, s.WebhookDB.
|
||||||
|
Table("targets").
|
||||||
|
Pluck("config", &configs).Error)
|
||||||
|
|
||||||
|
assert.NotContains(
|
||||||
|
t, strings.Join(configs, " "), hookURL,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRecoverSingleRetry_UnknownTargetType(
|
func TestRecoverSingleRetry_UnknownTargetType(
|
||||||
@@ -968,13 +1001,13 @@ func TestRecoverSingleRetry_UnknownTargetType(
|
|||||||
deliveryID := iSeedRetryingWithType(t, s, unknown)
|
deliveryID := iSeedRetryingWithType(t, s, unknown)
|
||||||
|
|
||||||
s.Engine.ExportRecoverWebhookDeliveries(
|
s.Engine.ExportRecoverWebhookDeliveries(
|
||||||
context.Background(), s.WebhookID, atStart,
|
context.Background(), s.WebhookID,
|
||||||
)
|
)
|
||||||
|
|
||||||
iAssertTerminallyFailed(t, s, deliveryID, unknown)
|
iAssertTerminallyFailed(t, s, deliveryID, unknown)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRecoverSingleRetry_OnTick_UnknownTargetType(
|
func TestSweepSingleRetry_UnknownTargetType(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
@@ -989,8 +1022,8 @@ func TestRecoverSingleRetry_OnTick_UnknownTargetType(
|
|||||||
|
|
||||||
deliveryID := iSeedRetryingWithType(t, s, unknown)
|
deliveryID := iSeedRetryingWithType(t, s, unknown)
|
||||||
|
|
||||||
s.Engine.ExportRecoverWebhookDeliveries(
|
s.Engine.ExportSweepWebhookRetries(
|
||||||
context.Background(), s.WebhookID, onTick,
|
context.Background(), s.WebhookID,
|
||||||
)
|
)
|
||||||
|
|
||||||
iAssertTerminallyFailed(t, s, deliveryID, unknown)
|
iAssertTerminallyFailed(t, s, deliveryID, unknown)
|
||||||
@@ -1032,19 +1065,19 @@ func iCreateWebhook(
|
|||||||
require.NoError(t, mainDB.Create(&webhook).Error)
|
require.NoError(t, mainDB.Create(&webhook).Error)
|
||||||
}
|
}
|
||||||
|
|
||||||
// --- recoverDeliveries Tests ---
|
// --- recoverInFlight Tests ---
|
||||||
|
|
||||||
func TestRecoverDeliveries_NoWebhooks(t *testing.T) {
|
func TestRecoverInFlight_NoWebhooks(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
s := newISetup(t)
|
s := newISetup(t)
|
||||||
|
|
||||||
s.Engine.ExportRecoverDeliveries(
|
s.Engine.ExportRecoverInFlight(
|
||||||
context.Background(), atStart,
|
context.Background(),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRecoverDeliveries_WithPendingDeliveries(
|
func TestRecoverInFlight_WithPendingDeliveries(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
@@ -1058,7 +1091,7 @@ func TestRecoverDeliveries_WithPendingDeliveries(
|
|||||||
|
|
||||||
iCreateTarget(t, s.MainDB, targetID,
|
iCreateTarget(t, s.MainDB, targetID,
|
||||||
s.WebhookID, "recover-target",
|
s.WebhookID, "recover-target",
|
||||||
database.TargetTypeLog, "", 1,
|
database.TargetTypeLog, "", 0,
|
||||||
)
|
)
|
||||||
|
|
||||||
event := iSeedEvent(
|
event := iSeedEvent(
|
||||||
@@ -1071,8 +1104,8 @@ func TestRecoverDeliveries_WithPendingDeliveries(
|
|||||||
database.DeliveryStatusPending,
|
database.DeliveryStatusPending,
|
||||||
)
|
)
|
||||||
|
|
||||||
s.Engine.ExportRecoverDeliveries(
|
s.Engine.ExportRecoverInFlight(
|
||||||
context.Background(), atStart,
|
context.Background(),
|
||||||
)
|
)
|
||||||
|
|
||||||
select {
|
select {
|
||||||
@@ -1083,91 +1116,11 @@ func TestRecoverDeliveries_WithPendingDeliveries(
|
|||||||
database.TargetTypeLog, task.TargetType,
|
database.TargetTypeLog, task.TargetType,
|
||||||
)
|
)
|
||||||
case <-time.After(2 * time.Second):
|
case <-time.After(2 * time.Second):
|
||||||
t.Fatal("expected task from recovery at start")
|
t.Fatal("expected task from recoverInFlight")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestRecoverDeliveries_ReportsAMissingWebhookDatabase covers a webhook
|
// --- HTTP Config with custom headers ---
|
||||||
// whose database file is gone, after a partial restore say. Recovery at
|
|
||||||
// start opens every webhook's database, so the empty one made in its
|
|
||||||
// place is reported at start, naming the file
|
|
||||||
// (https://git.eeqj.de/sneak/webhooker/issues/290).
|
|
||||||
func TestRecoverDeliveries_ReportsAMissingWebhookDatabase(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
mainDB := iMainDB(t)
|
|
||||||
webhookID := uuid.New().String()
|
|
||||||
iCreateWebhook(t, mainDB.DB(), webhookID, "lost-database")
|
|
||||||
|
|
||||||
var logs bytes.Buffer
|
|
||||||
|
|
||||||
dbMgr := databasetest.NewWebhookDBManagerWithLogger(
|
|
||||||
t, t.TempDir(), slog.New(slog.NewTextHandler(&logs, nil)),
|
|
||||||
)
|
|
||||||
t.Cleanup(func() { _ = dbMgr.CloseAll() })
|
|
||||||
|
|
||||||
engine := delivery.NewTestEngineWithDB(
|
|
||||||
mainDB, dbMgr,
|
|
||||||
slog.New(slog.DiscardHandler),
|
|
||||||
&http.Client{Timeout: 5 * time.Second}, 1,
|
|
||||||
)
|
|
||||||
|
|
||||||
engine.ExportRecoverDeliveries(context.Background(), atStart)
|
|
||||||
|
|
||||||
assert.Contains(
|
|
||||||
t, logs.String(),
|
|
||||||
`level=WARN msg="created a new, empty database" webhook_id=`+
|
|
||||||
webhookID+" path="+dbMgr.DBPath(webhookID),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRecoverDeliveries_SkipsAWebhookDeletedAfterTheListIsRead covers a
|
|
||||||
// webhook deleted from the web interface while the recovery routine
|
|
||||||
// runs. Its database file is gone, and recovery must not create it
|
|
||||||
// again.
|
|
||||||
func TestRecoverDeliveries_SkipsAWebhookDeletedAfterTheListIsRead(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
mainDB := iMainDB(t)
|
|
||||||
webhookID := uuid.New().String()
|
|
||||||
iCreateWebhook(t, mainDB.DB(), webhookID, "deleted-during-recovery")
|
|
||||||
|
|
||||||
// The first query to return is recovery's read of the list of
|
|
||||||
// webhooks. Deleting the webhook right after it puts the delete
|
|
||||||
// between that read and the opening of the webhook's database.
|
|
||||||
deleted := false
|
|
||||||
|
|
||||||
require.NoError(t, mainDB.DB().Callback().Query().After("gorm:query").
|
|
||||||
Register("delete-after-list", func(*gorm.DB) {
|
|
||||||
if deleted {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
deleted = true
|
|
||||||
|
|
||||||
require.NoError(t, mainDB.DB().Delete(
|
|
||||||
&database.Webhook{}, "id = ?", webhookID,
|
|
||||||
).Error)
|
|
||||||
}))
|
|
||||||
|
|
||||||
dbMgr := databasetest.NewWebhookDBManager(t, t.TempDir())
|
|
||||||
t.Cleanup(func() { _ = dbMgr.CloseAll() })
|
|
||||||
|
|
||||||
engine := delivery.NewTestEngineWithDB(
|
|
||||||
mainDB, dbMgr,
|
|
||||||
slog.New(slog.DiscardHandler),
|
|
||||||
&http.Client{Timeout: 5 * time.Second}, 1,
|
|
||||||
)
|
|
||||||
|
|
||||||
engine.ExportRecoverDeliveries(context.Background(), atStart)
|
|
||||||
|
|
||||||
require.True(t, deleted)
|
|
||||||
assert.False(t, dbMgr.DBExists(webhookID))
|
|
||||||
}
|
|
||||||
|
|
||||||
// --- HTTP target with custom headers ---
|
|
||||||
|
|
||||||
func TestDeliverHTTP_CustomTargetHeaders(t *testing.T) {
|
func TestDeliverHTTP_CustomTargetHeaders(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
@@ -1189,6 +1142,16 @@ func TestDeliverHTTP_CustomTargetHeaders(t *testing.T) {
|
|||||||
)
|
)
|
||||||
defer ts.Close()
|
defer ts.Close()
|
||||||
|
|
||||||
|
cfg := delivery.HTTPTargetConfig{
|
||||||
|
URL: ts.URL,
|
||||||
|
Headers: map[string]string{
|
||||||
|
"Authorization": "Bearer secret-token",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
cfgJSON, err := json.Marshal(cfg)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
event := iSeedEvent(
|
event := iSeedEvent(
|
||||||
t, s.WebhookDB, s.WebhookID,
|
t, s.WebhookDB, s.WebhookID,
|
||||||
`{"auth":"test"}`,
|
`{"auth":"test"}`,
|
||||||
@@ -1203,12 +1166,9 @@ func TestDeliverHTTP_CustomTargetHeaders(t *testing.T) {
|
|||||||
bodyStr := event.Body
|
bodyStr := event.Body
|
||||||
task := iTask(
|
task := iTask(
|
||||||
d, event, s.WebhookID, targetID,
|
d, event, s.WebhookID, targetID,
|
||||||
"auth-target", ts.URL,
|
"auth-target", string(cfgJSON),
|
||||||
1, 1, &bodyStr,
|
0, 1, &bodyStr,
|
||||||
)
|
)
|
||||||
task.TargetHeaders = map[string]string{
|
|
||||||
headerAuthorization: "Bearer secret-token",
|
|
||||||
}
|
|
||||||
|
|
||||||
s.Engine.ExportProcessNewTask(
|
s.Engine.ExportProcessNewTask(
|
||||||
context.TODO(), &task,
|
context.TODO(), &task,
|
||||||
@@ -1221,14 +1181,11 @@ func TestDeliverHTTP_CustomTargetHeaders(t *testing.T) {
|
|||||||
|
|
||||||
// --- HTTP delivery with custom timeout ---
|
// --- HTTP delivery with custom timeout ---
|
||||||
|
|
||||||
// TestDeliverHTTP_TargetTimeout proves a task's TimeoutSeconds reaches
|
|
||||||
// the delivery through the engine. The destination answers after two
|
|
||||||
// seconds, inside the engine's own five-second client timeout, so only
|
|
||||||
// the task's one second can fail the delivery.
|
|
||||||
func TestDeliverHTTP_TargetTimeout(t *testing.T) {
|
func TestDeliverHTTP_TargetTimeout(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
s := newISetup(t)
|
db := testWebhookDB(t)
|
||||||
|
e := iEngine(t, 1)
|
||||||
|
|
||||||
ts := httptest.NewServer(
|
ts := httptest.NewServer(
|
||||||
http.HandlerFunc(
|
http.HandlerFunc(
|
||||||
@@ -1240,30 +1197,31 @@ func TestDeliverHTTP_TargetTimeout(t *testing.T) {
|
|||||||
)
|
)
|
||||||
defer ts.Close()
|
defer ts.Close()
|
||||||
|
|
||||||
event := iSeedEvent(
|
cfg := delivery.HTTPTargetConfig{
|
||||||
t, s.WebhookDB, s.WebhookID, `{"timeout":"test"}`,
|
URL: ts.URL,
|
||||||
)
|
Timeout: 1,
|
||||||
targetID := uuid.New().String()
|
}
|
||||||
|
|
||||||
d := iSeedDelivery(
|
cfgJSON, err := json.Marshal(cfg)
|
||||||
t, s.WebhookDB, event.ID, targetID,
|
require.NoError(t, err)
|
||||||
database.DeliveryStatusPending,
|
|
||||||
|
event, del := iSeedEventAndDelivery(
|
||||||
|
t, db, `{"timeout":"test"}`,
|
||||||
|
string(cfgJSON),
|
||||||
)
|
)
|
||||||
|
|
||||||
bodyStr := event.Body
|
task, d := iHTTPTaskAndDelivery(
|
||||||
task := iTask(
|
event, del, "timeout-target",
|
||||||
d, event, s.WebhookID, targetID,
|
string(cfgJSON), 0, 1,
|
||||||
"timeout-target", ts.URL, 1, 1, &bodyStr,
|
|
||||||
)
|
)
|
||||||
task.TimeoutSeconds = 1
|
|
||||||
|
|
||||||
s.Engine.ExportProcessNewTask(context.TODO(), &task)
|
e.ExportDeliverHTTP(context.TODO(), db, d, task)
|
||||||
|
|
||||||
iAssertStatus(t, s.WebhookDB, d.ID,
|
iAssertStatus(t, db, del.ID,
|
||||||
database.DeliveryStatusFailed,
|
database.DeliveryStatusFailed,
|
||||||
)
|
)
|
||||||
|
|
||||||
iAssertResultFailed(t, s.WebhookDB, d.ID)
|
iAssertResultFailed(t, db, del.ID)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestDeliverHTTP_CutsStoredResponseAtMaxBodyLog pins the size
|
// TestDeliverHTTP_CutsStoredResponseAtMaxBodyLog pins the size
|
||||||
@@ -1301,6 +1259,8 @@ func TestDeliverHTTP_CutsStoredResponseAtMaxBodyLog(
|
|||||||
))
|
))
|
||||||
defer ts.Close()
|
defer ts.Close()
|
||||||
|
|
||||||
|
cfgJSON := iHTTPConfig(ts.URL)
|
||||||
|
|
||||||
event := iSeedEvent(
|
event := iSeedEvent(
|
||||||
t, s.WebhookDB, s.WebhookID, `{"cut":"test"}`,
|
t, s.WebhookDB, s.WebhookID, `{"cut":"test"}`,
|
||||||
)
|
)
|
||||||
@@ -1314,7 +1274,7 @@ func TestDeliverHTTP_CutsStoredResponseAtMaxBodyLog(
|
|||||||
bodyStr := event.Body
|
bodyStr := event.Body
|
||||||
task := iTask(
|
task := iTask(
|
||||||
del, event, s.WebhookID, targetID,
|
del, event, s.WebhookID, targetID,
|
||||||
"cut-target", ts.URL, 1, 1, &bodyStr,
|
"cut-target", cfgJSON, 0, 1, &bodyStr,
|
||||||
)
|
)
|
||||||
|
|
||||||
s.Engine.ExportProcessNewTask(context.TODO(), &task)
|
s.Engine.ExportProcessNewTask(context.TODO(), &task)
|
||||||
@@ -1346,7 +1306,7 @@ func TestDeliverHTTP_CutsStoredResponseAtMaxBodyLog(
|
|||||||
func iSeedEventAndDelivery(
|
func iSeedEventAndDelivery(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
db *gorm.DB,
|
db *gorm.DB,
|
||||||
body string,
|
body, _ string,
|
||||||
) (database.Event, database.Delivery) {
|
) (database.Event, database.Delivery) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -1377,19 +1337,19 @@ func iSeedEventAndDelivery(
|
|||||||
func iHTTPTaskAndDelivery(
|
func iHTTPTaskAndDelivery(
|
||||||
event database.Event,
|
event database.Event,
|
||||||
del database.Delivery,
|
del database.Delivery,
|
||||||
name, targetURL string,
|
name, config string,
|
||||||
maxAttempts, attemptNum int,
|
maxRetries, attemptNum int,
|
||||||
) (*delivery.Task, *database.Delivery) {
|
) (*delivery.Task, *database.Delivery) {
|
||||||
task := &delivery.Task{
|
task := &delivery.Task{
|
||||||
DeliveryID: del.ID,
|
DeliveryID: del.ID,
|
||||||
EventID: event.ID,
|
EventID: event.ID,
|
||||||
WebhookID: event.WebhookID,
|
WebhookID: event.WebhookID,
|
||||||
TargetID: del.TargetID,
|
TargetID: del.TargetID,
|
||||||
TargetName: name,
|
TargetName: name,
|
||||||
TargetType: database.TargetTypeHTTP,
|
TargetType: database.TargetTypeHTTP,
|
||||||
TargetURL: targetURL,
|
TargetConfig: config,
|
||||||
MaxAttempts: maxAttempts,
|
MaxRetries: maxRetries,
|
||||||
AttemptNum: attemptNum,
|
AttemptNum: attemptNum,
|
||||||
}
|
}
|
||||||
|
|
||||||
d := &database.Delivery{
|
d := &database.Delivery{
|
||||||
@@ -1398,10 +1358,9 @@ func iHTTPTaskAndDelivery(
|
|||||||
Status: database.DeliveryStatusPending,
|
Status: database.DeliveryStatusPending,
|
||||||
Event: event,
|
Event: event,
|
||||||
Target: database.Target{
|
Target: database.Target{
|
||||||
Name: name,
|
Name: name,
|
||||||
Type: database.TargetTypeHTTP,
|
Type: database.TargetTypeHTTP,
|
||||||
URL: targetURL,
|
Config: config,
|
||||||
MaxAttempts: maxAttempts,
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
d.ID = del.ID
|
d.ID = del.ID
|
||||||
@@ -1429,30 +1388,30 @@ func iAssertResultFailed(
|
|||||||
assert.NotEmpty(t, result.Error)
|
assert.NotEmpty(t, result.Error)
|
||||||
}
|
}
|
||||||
|
|
||||||
// --- HTTP request with no URL ---
|
// --- HTTP request with invalid config ---
|
||||||
|
|
||||||
// TestDeliverHTTP_NoURL proves an http target without a URL, which
|
func TestDeliverHTTP_InvalidConfig(t *testing.T) {
|
||||||
// the forms never store, fails its delivery and records why.
|
|
||||||
func TestDeliverHTTP_NoURL(t *testing.T) {
|
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
db := testWebhookDB(t)
|
db := testWebhookDB(t)
|
||||||
e := iEngine(t, 1)
|
e := iEngine(t, 1)
|
||||||
|
|
||||||
event, del := iSeedEventAndDelivery(t, db, `{"url":"none"}`)
|
event, del := iSeedEventAndDelivery(
|
||||||
|
t, db, `{"config":"invalid"}`, "",
|
||||||
|
)
|
||||||
|
|
||||||
task, d := iHTTPTaskAndDelivery(event, del, "no-url", "", 1, 1)
|
task, d := iHTTPTaskAndDelivery(
|
||||||
|
event, del, "bad-config", `not-json`, 0, 1,
|
||||||
|
)
|
||||||
|
|
||||||
e.ExportDeliverHTTP(context.TODO(), db, d, task)
|
e.ExportDeliverHTTP(context.TODO(), db, d, task)
|
||||||
|
|
||||||
iAssertStatus(t, db, del.ID,
|
iAssertStatus(t, db, del.ID,
|
||||||
database.DeliveryStatusFailed,
|
database.DeliveryStatusFailed,
|
||||||
)
|
)
|
||||||
|
|
||||||
iAssertResultFailed(t, db, del.ID)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// --- notify batching ---
|
// --- Notify batching ---
|
||||||
|
|
||||||
func TestNotify_MultipleTasks(t *testing.T) {
|
func TestNotify_MultipleTasks(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
@@ -1467,7 +1426,7 @@ func TestNotify_MultipleTasks(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
e.ExportNotify(tasks)
|
e.Notify(tasks)
|
||||||
|
|
||||||
for i := range 5 {
|
for i := range 5 {
|
||||||
select {
|
select {
|
||||||
|
|||||||
@@ -2,19 +2,12 @@ package delivery_test
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"path/filepath"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"gorm.io/gorm/clause"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
)
|
)
|
||||||
@@ -29,7 +22,7 @@ const (
|
|||||||
// OnStart before the caller may enqueue work. A worker pool
|
// OnStart before the caller may enqueue work. A worker pool
|
||||||
// wrongly rooted in the already-done hook context has nothing
|
// wrongly rooted in the already-done hook context has nothing
|
||||||
// but ctx.Done() ready in its select, so it is deterministically
|
// but ctx.Done() ready in its select, so it is deterministically
|
||||||
// gone by the end of this window. Without the wait, notify would
|
// gone by the end of this window. Without the wait, Notify would
|
||||||
// race the pool's very first select, in which a ready ctx.Done()
|
// race the pool's very first select, in which a ready ctx.Done()
|
||||||
// and a ready deliveryCh are chosen between at random and a
|
// and a ready deliveryCh are chosen between at random and a
|
||||||
// doomed pool still delivers.
|
// doomed pool still delivers.
|
||||||
@@ -99,7 +92,7 @@ func requireStopHookExpires(
|
|||||||
// the caller can drive OnStop too.
|
// the caller can drive OnStop too.
|
||||||
//
|
//
|
||||||
// Callers must not seed pending or retrying deliveries before
|
// Callers must not seed pending or retrying deliveries before
|
||||||
// calling this: the recovery routine's run at start enqueues those,
|
// calling this: restart recovery enqueues those during startup,
|
||||||
// which would put work in the queue while the pool is still
|
// which would put work in the queue while the pool is still
|
||||||
// racing its first select.
|
// racing its first select.
|
||||||
func startEngineViaHook(
|
func startEngineViaHook(
|
||||||
@@ -149,7 +142,7 @@ func seedLogTask(
|
|||||||
bodyStr := event.Body
|
bodyStr := event.Body
|
||||||
task := iTask(
|
task := iTask(
|
||||||
d, event, s.WebhookID, targetID,
|
d, event, s.WebhookID, targetID,
|
||||||
"hook-context-test", "", 1, 1, &bodyStr,
|
"hook-context-test", "", 0, 1, &bodyStr,
|
||||||
)
|
)
|
||||||
task.TargetType = database.TargetTypeLog
|
task.TargetType = database.TargetTypeLog
|
||||||
|
|
||||||
@@ -178,11 +171,11 @@ func TestEngine_WorkersOutliveStartHookContext(t *testing.T) {
|
|||||||
_ = lc.hooks[0].OnStop(context.Background())
|
_ = lc.hooks[0].OnStop(context.Background())
|
||||||
})
|
})
|
||||||
|
|
||||||
// Seeded only after the pool has settled, so the recovery
|
// Seeded only after the pool has settled, so restart recovery
|
||||||
// routine's run at start cannot enqueue it.
|
// cannot enqueue it during startup.
|
||||||
deliveryID, task := seedLogTask(t, s)
|
deliveryID, task := seedLogTask(t, s)
|
||||||
|
|
||||||
s.Engine.ExportNotify([]delivery.Task{task})
|
s.Engine.Notify([]delivery.Task{task})
|
||||||
|
|
||||||
iWaitForDelivered(t, s.WebhookDB, deliveryID)
|
iWaitForDelivered(t, s.WebhookDB, deliveryID)
|
||||||
}
|
}
|
||||||
@@ -202,7 +195,7 @@ func TestEngine_StopHookStopsWorkers(t *testing.T) {
|
|||||||
// Let the pool prove it is running before stopping it, so a
|
// Let the pool prove it is running before stopping it, so a
|
||||||
// fast OnStop cannot pass by stopping something already dead.
|
// fast OnStop cannot pass by stopping something already dead.
|
||||||
firstID, firstTask := seedLogTask(t, s)
|
firstID, firstTask := seedLogTask(t, s)
|
||||||
s.Engine.ExportNotify([]delivery.Task{firstTask})
|
s.Engine.Notify([]delivery.Task{firstTask})
|
||||||
iWaitForDelivered(t, s.WebhookDB, firstID)
|
iWaitForDelivered(t, s.WebhookDB, firstID)
|
||||||
|
|
||||||
var stopErr error
|
var stopErr error
|
||||||
@@ -232,7 +225,7 @@ func TestEngine_StopHookStopsWorkers(t *testing.T) {
|
|||||||
// With every worker gone, a freshly notified task must sit
|
// With every worker gone, a freshly notified task must sit
|
||||||
// untouched in the queue rather than being delivered.
|
// untouched in the queue rather than being delivered.
|
||||||
secondID, secondTask := seedLogTask(t, s)
|
secondID, secondTask := seedLogTask(t, s)
|
||||||
s.Engine.ExportNotify([]delivery.Task{secondTask})
|
s.Engine.Notify([]delivery.Task{secondTask})
|
||||||
|
|
||||||
time.Sleep(200 * time.Millisecond)
|
time.Sleep(200 * time.Millisecond)
|
||||||
|
|
||||||
@@ -276,252 +269,3 @@ func TestEngine_StopHookHonoursStopTimeout(t *testing.T) {
|
|||||||
|
|
||||||
requireStopHookExpires(t, lc.hooks[0], "delivery engine")
|
requireStopHookExpires(t, lc.hooks[0], "delivery engine")
|
||||||
}
|
}
|
||||||
|
|
||||||
// seedHeldDelivery gives the setup's webhook an http target with
|
|
||||||
// maxAttempts, pointed at a receiver that keeps every request open until
|
|
||||||
// the client gives up on it, and seeds one delivery to that target at
|
|
||||||
// status. It returns the task that sends attempt attemptNum of the
|
|
||||||
// delivery, and a channel that is closed when a request reaches the
|
|
||||||
// receiver.
|
|
||||||
func seedHeldDelivery(
|
|
||||||
t *testing.T,
|
|
||||||
s iSetup,
|
|
||||||
maxAttempts int,
|
|
||||||
status database.DeliveryStatus,
|
|
||||||
attemptNum int,
|
|
||||||
) (delivery.Task, <-chan struct{}) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
arrived := make(chan struct{})
|
|
||||||
|
|
||||||
var once sync.Once
|
|
||||||
|
|
||||||
ts := httptest.NewServer(http.HandlerFunc(
|
|
||||||
func(_ http.ResponseWriter, r *http.Request) {
|
|
||||||
// The server notices the client hanging up only once the
|
|
||||||
// body has been read; until then r.Context() never ends.
|
|
||||||
_, _ = io.Copy(io.Discard, r.Body)
|
|
||||||
|
|
||||||
once.Do(func() { close(arrived) })
|
|
||||||
|
|
||||||
<-r.Context().Done()
|
|
||||||
},
|
|
||||||
))
|
|
||||||
t.Cleanup(ts.Close)
|
|
||||||
|
|
||||||
targetID := uuid.New().String()
|
|
||||||
|
|
||||||
iCreateWebhook(t, s.MainDB, s.WebhookID, "hook")
|
|
||||||
iCreateTarget(
|
|
||||||
t, s.MainDB, targetID, s.WebhookID, "held",
|
|
||||||
database.TargetTypeHTTP, ts.URL, maxAttempts,
|
|
||||||
)
|
|
||||||
|
|
||||||
event := iSeedEvent(t, s.WebhookDB, s.WebhookID, `{"held":true}`)
|
|
||||||
d := iSeedDelivery(t, s.WebhookDB, event.ID, targetID, status)
|
|
||||||
|
|
||||||
bodyStr := event.Body
|
|
||||||
|
|
||||||
return iTask(
|
|
||||||
d, event, s.WebhookID, targetID, "held", ts.URL,
|
|
||||||
maxAttempts, attemptNum, &bodyStr,
|
|
||||||
), arrived
|
|
||||||
}
|
|
||||||
|
|
||||||
// stopDuringAttempt waits until the receiver holds an attempt open, then
|
|
||||||
// stops the engine through its real stop hook.
|
|
||||||
func stopDuringAttempt(
|
|
||||||
t *testing.T, lc *recordingLifecycle, arrived <-chan struct{},
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-arrived:
|
|
||||||
case <-time.After(5 * time.Second):
|
|
||||||
t.Fatal("the delivery attempt never reached the receiver")
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NoError(t, lc.hooks[0].OnStop(context.Background()))
|
|
||||||
}
|
|
||||||
|
|
||||||
// requireRequeued runs the recovery routine as at start on the stopped
|
|
||||||
// engine and requires it to queue task's attempt again on ch.
|
|
||||||
func requireRequeued(
|
|
||||||
t *testing.T, s iSetup, ch <-chan delivery.Task, task delivery.Task,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
s.Engine.ExportRecoverWebhookDeliveries(
|
|
||||||
context.Background(), s.WebhookID, atStart,
|
|
||||||
)
|
|
||||||
|
|
||||||
select {
|
|
||||||
case got := <-ch:
|
|
||||||
assert.Equal(t, task.DeliveryID, got.DeliveryID)
|
|
||||||
assert.Equal(t, task.AttemptNum, got.AttemptNum)
|
|
||||||
case <-time.After(5 * time.Second):
|
|
||||||
t.Fatal("recovery at start did not queue the delivery again")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEngine_StopLeavesFireAndForgetAttemptPending is the regression
|
|
||||||
// test for a stop that turned an attempt in flight into a terminal
|
|
||||||
// failure. Stopping the engine cancels the request; for a target with
|
|
||||||
// no retries that was the delivery's only attempt, recorded as failed
|
|
||||||
// with "context canceled", and nothing ever sent it again. The
|
|
||||||
// delivery must instead stay pending, with no result and no count in
|
|
||||||
// the attempt metrics, for the recovery routine to send at the next
|
|
||||||
// start.
|
|
||||||
func TestEngine_StopLeavesFireAndForgetAttemptPending(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := newISetup(t)
|
|
||||||
reg := mIsolate(t, s)
|
|
||||||
lc := startEngineViaHook(t, s.Engine)
|
|
||||||
|
|
||||||
task, arrived := seedHeldDelivery(
|
|
||||||
t, s, 1, database.DeliveryStatusPending, 1,
|
|
||||||
)
|
|
||||||
|
|
||||||
s.Engine.ExportNotify([]delivery.Task{task})
|
|
||||||
stopDuringAttempt(t, lc, arrived)
|
|
||||||
|
|
||||||
iAssertStatus(
|
|
||||||
t, s.WebhookDB, task.DeliveryID, database.DeliveryStatusPending,
|
|
||||||
)
|
|
||||||
assert.Empty(t, iResults(t, s.WebhookDB, task.DeliveryID))
|
|
||||||
assert.Zero(t, mCounter(t, reg, mAttempts, mTypeHTTP))
|
|
||||||
assert.Zero(t, mHTTPDurations(t, reg))
|
|
||||||
|
|
||||||
requireRequeued(t, s, s.Engine.ExportDeliveryCh(), task)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEngine_StopLeavesLastRetryAttemptRetrying is the same regression
|
|
||||||
// for a target with retries, on its last attempt: the cancelled attempt
|
|
||||||
// used to exhaust the retries and end the delivery failed. It must stay
|
|
||||||
// retrying with only its earlier attempt recorded, and the recovery
|
|
||||||
// routine at the next start must send that last attempt again.
|
|
||||||
func TestEngine_StopLeavesLastRetryAttemptRetrying(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const maxAttempts = 2
|
|
||||||
|
|
||||||
s := newISetup(t)
|
|
||||||
lc := startEngineViaHook(t, s.Engine)
|
|
||||||
|
|
||||||
task, arrived := seedHeldDelivery(
|
|
||||||
t, s, maxAttempts, database.DeliveryStatusRetrying, maxAttempts,
|
|
||||||
)
|
|
||||||
iSeedFailedResult(t, s.WebhookDB, task.DeliveryID)
|
|
||||||
|
|
||||||
s.Engine.ExportScheduleRetry(task, 0)
|
|
||||||
stopDuringAttempt(t, lc, arrived)
|
|
||||||
|
|
||||||
iAssertStatus(
|
|
||||||
t, s.WebhookDB, task.DeliveryID, database.DeliveryStatusRetrying,
|
|
||||||
)
|
|
||||||
assert.Len(t, iResults(t, s.WebhookDB, task.DeliveryID), 1)
|
|
||||||
|
|
||||||
requireRequeued(t, s, s.Engine.ExportRetryCh(), task)
|
|
||||||
}
|
|
||||||
|
|
||||||
// deliverToArchive gives the setup's webhook a database target,
|
|
||||||
// runs one delivery to it through the running engine, and returns
|
|
||||||
// the target's archive file path. The archive writer holds the file
|
|
||||||
// open afterwards.
|
|
||||||
func deliverToArchive(t *testing.T, s iSetup) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
iCreateWebhook(t, s.MainDB, s.WebhookID, "hook")
|
|
||||||
|
|
||||||
tgt := &database.Target{
|
|
||||||
WebhookID: s.WebhookID,
|
|
||||||
Name: "archive",
|
|
||||||
Type: database.TargetTypeDatabase,
|
|
||||||
MaxAttempts: 1,
|
|
||||||
}
|
|
||||||
require.NoError(
|
|
||||||
t, s.MainDB.Omit(clause.Associations).Create(tgt).Error,
|
|
||||||
)
|
|
||||||
|
|
||||||
deliveryID, task := seedLogTask(t, s)
|
|
||||||
task.TargetID = tgt.ID
|
|
||||||
task.TargetType = database.TargetTypeDatabase
|
|
||||||
|
|
||||||
s.Engine.ExportNotify([]delivery.Task{task})
|
|
||||||
|
|
||||||
iWaitForDelivered(t, s.WebhookDB, deliveryID)
|
|
||||||
|
|
||||||
return filepath.Join(
|
|
||||||
filepath.Dir(s.DBMgr.DBPath(s.WebhookID)),
|
|
||||||
"archive-hook-archive-"+tgt.ID+".db",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEngine_StopHookClosesArchives is the regression test for an
|
|
||||||
// archive split across two files by a clean stop. The engine never
|
|
||||||
// closed its archive writers, so after a stop the archived rows
|
|
||||||
// could sit in archive-{id}.db-wal while archive-{id}.db held no
|
|
||||||
// table at all, and copying the .db on its own gave an empty
|
|
||||||
// database.
|
|
||||||
func TestEngine_StopHookClosesArchives(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := newISetup(t)
|
|
||||||
|
|
||||||
lc := startEngineViaHook(t, s.Engine)
|
|
||||||
|
|
||||||
path := deliverToArchive(t, s)
|
|
||||||
require.FileExists(
|
|
||||||
t, path+"-wal",
|
|
||||||
"an open archive should have a -wal for the stop to remove",
|
|
||||||
)
|
|
||||||
|
|
||||||
require.NoError(t, lc.hooks[0].OnStop(context.Background()))
|
|
||||||
|
|
||||||
wals, err := filepath.Glob(
|
|
||||||
filepath.Join(filepath.Dir(path), "archive-*.db-wal"),
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Empty(
|
|
||||||
t, wals, "a clean stop must leave no archive -wal behind",
|
|
||||||
)
|
|
||||||
|
|
||||||
// With no -wal beside it, the row can only be in the .db.
|
|
||||||
count, err := countArchivedRows(path)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Equal(t, int64(1), count)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEngine_StopHookTimeoutLeavesArchivesOpen covers a stop whose
|
|
||||||
// budget runs out while a worker is still running. The archive
|
|
||||||
// writers are left open, as a kill would leave them: closing them
|
|
||||||
// would wait for any write in progress, and that worker would then
|
|
||||||
// open new writers that nothing closes.
|
|
||||||
func TestEngine_StopHookTimeoutLeavesArchivesOpen(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := newISetup(t)
|
|
||||||
|
|
||||||
lc := startEngineViaHook(t, s.Engine)
|
|
||||||
|
|
||||||
path := deliverToArchive(t, s)
|
|
||||||
|
|
||||||
release := make(chan struct{})
|
|
||||||
|
|
||||||
t.Cleanup(func() {
|
|
||||||
close(release)
|
|
||||||
s.Engine.ExportArchives().EvictWebhook(s.WebhookID)
|
|
||||||
})
|
|
||||||
|
|
||||||
s.Engine.ExportWedgeWorker(release)
|
|
||||||
|
|
||||||
requireStopHookExpires(t, lc.hooks[0], "delivery engine")
|
|
||||||
|
|
||||||
// Closing the last handle on an archive removes its -wal, so a
|
|
||||||
// -wal still there is a writer still open.
|
|
||||||
require.FileExists(
|
|
||||||
t, path+"-wal",
|
|
||||||
"a stop that timed out must not close archive writers",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|||||||
+248
-475
File diff suppressed because it is too large
Load Diff
@@ -69,6 +69,7 @@ func TestEventDBHoldsNoTargetRows(t *testing.T) {
|
|||||||
))
|
))
|
||||||
defer ts.Close()
|
defer ts.Close()
|
||||||
|
|
||||||
|
cfg := iHTTPConfig(ts.URL)
|
||||||
targetID := uuid.New().String()
|
targetID := uuid.New().String()
|
||||||
dbPath := s.DBMgr.DBPath(s.WebhookID)
|
dbPath := s.DBMgr.DBPath(s.WebhookID)
|
||||||
|
|
||||||
@@ -84,7 +85,7 @@ func TestEventDBHoldsNoTargetRows(t *testing.T) {
|
|||||||
)
|
)
|
||||||
task := iTask(
|
task := iTask(
|
||||||
d, event, s.WebhookID, targetID,
|
d, event, s.WebhookID, targetID,
|
||||||
"leaky-target", ts.URL, 5, 1, &body,
|
"leaky-target", cfg, 5, 1, &body,
|
||||||
)
|
)
|
||||||
|
|
||||||
s.Engine.ExportProcessNewTask(context.TODO(), &task)
|
s.Engine.ExportProcessNewTask(context.TODO(), &task)
|
||||||
@@ -101,7 +102,7 @@ func TestEventDBHoldsNoTargetRows(t *testing.T) {
|
|||||||
// https://git.eeqj.de/sneak/webhooker/issues/107.
|
// https://git.eeqj.de/sneak/webhooker/issues/107.
|
||||||
iCreateTarget(
|
iCreateTarget(
|
||||||
t, s.MainDB, targetID, s.WebhookID, "leaky-target",
|
t, s.MainDB, targetID, s.WebhookID, "leaky-target",
|
||||||
database.TargetTypeHTTP, ts.URL, 5,
|
database.TargetTypeHTTP, cfg, 5,
|
||||||
)
|
)
|
||||||
|
|
||||||
rd := iSeedDelivery(
|
rd := iSeedDelivery(
|
||||||
@@ -110,7 +111,7 @@ func TestEventDBHoldsNoTargetRows(t *testing.T) {
|
|||||||
)
|
)
|
||||||
rTask := iTask(
|
rTask := iTask(
|
||||||
rd, event, s.WebhookID, targetID,
|
rd, event, s.WebhookID, targetID,
|
||||||
"leaky-target", ts.URL, 5, 2, &body,
|
"leaky-target", cfg, 5, 2, &body,
|
||||||
)
|
)
|
||||||
|
|
||||||
s.Engine.ExportProcessRetryTask(context.TODO(), &rTask)
|
s.Engine.ExportProcessRetryTask(context.TODO(), &rTask)
|
||||||
@@ -137,6 +138,7 @@ func TestEventDBHoldsNoTargetRowsOnFailedDelivery(t *testing.T) {
|
|||||||
))
|
))
|
||||||
defer ts.Close()
|
defer ts.Close()
|
||||||
|
|
||||||
|
cfg := iHTTPConfig(ts.URL)
|
||||||
targetID := uuid.New().String()
|
targetID := uuid.New().String()
|
||||||
|
|
||||||
event := iSeedEvent(
|
event := iSeedEvent(
|
||||||
@@ -150,7 +152,7 @@ func TestEventDBHoldsNoTargetRowsOnFailedDelivery(t *testing.T) {
|
|||||||
)
|
)
|
||||||
task := iTask(
|
task := iTask(
|
||||||
d, event, s.WebhookID, targetID,
|
d, event, s.WebhookID, targetID,
|
||||||
"leaky-target", ts.URL, 1, 1, &body,
|
"leaky-target", cfg, 0, 1, &body,
|
||||||
)
|
)
|
||||||
|
|
||||||
s.Engine.ExportProcessNewTask(context.TODO(), &task)
|
s.Engine.ExportProcessNewTask(context.TODO(), &task)
|
||||||
|
|||||||
@@ -93,6 +93,17 @@ func (s *tsSink) text(t *testing.T) string {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func tsSlackConfig(t *testing.T, url string) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
data, err := json.Marshal(
|
||||||
|
delivery.SlackTargetConfig{WebhookURL: url},
|
||||||
|
)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
return string(data)
|
||||||
|
}
|
||||||
|
|
||||||
// tsSeedEvent writes an event whose CreatedAt is tsEventCreatedAt
|
// tsSeedEvent writes an event whose CreatedAt is tsEventCreatedAt
|
||||||
// rather than the write time, so an assertion on the rendered
|
// rather than the write time, so an assertion on the rendered
|
||||||
// timestamp cannot pass by accident against "roughly now".
|
// timestamp cannot pass by accident against "roughly now".
|
||||||
@@ -131,17 +142,16 @@ func tsSeedEvent(
|
|||||||
// tsSeedTarget writes the slack target row into the main database.
|
// tsSeedTarget writes the slack target row into the main database.
|
||||||
// The retry path confirms the target still exists before sending.
|
// The retry path confirms the target still exists before sending.
|
||||||
func tsSeedTarget(
|
func tsSeedTarget(
|
||||||
t *testing.T, mainDB *gorm.DB, webhookID, webhookURL string,
|
t *testing.T, mainDB *gorm.DB, webhookID, config string,
|
||||||
) database.Target {
|
) database.Target {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
target := database.Target{
|
target := database.Target{
|
||||||
WebhookID: webhookID,
|
WebhookID: webhookID,
|
||||||
Name: "slack-sink",
|
Name: "slack-sink",
|
||||||
Type: database.TargetTypeSlack,
|
Type: database.TargetTypeSlack,
|
||||||
URL: webhookURL,
|
Config: config,
|
||||||
Active: true,
|
Active: true,
|
||||||
MaxAttempts: 1,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
require.NoError(t, mainDB.Create(&target).Error)
|
require.NoError(t, mainDB.Create(&target).Error)
|
||||||
@@ -165,8 +175,8 @@ func tsTask(
|
|||||||
TargetID: target.ID,
|
TargetID: target.ID,
|
||||||
TargetName: target.Name,
|
TargetName: target.Name,
|
||||||
TargetType: database.TargetTypeSlack,
|
TargetType: database.TargetTypeSlack,
|
||||||
TargetURL: target.URL,
|
TargetConfig: target.Config,
|
||||||
MaxAttempts: 1,
|
MaxRetries: 0,
|
||||||
Method: event.Method,
|
Method: event.Method,
|
||||||
Headers: event.Headers,
|
Headers: event.Headers,
|
||||||
ContentType: event.ContentType,
|
ContentType: event.ContentType,
|
||||||
@@ -215,7 +225,8 @@ func (c tsCase) run(t *testing.T) (iSetup, database.Delivery, string) {
|
|||||||
s := newISetup(t)
|
s := newISetup(t)
|
||||||
sink := newTSSink(t)
|
sink := newTSSink(t)
|
||||||
|
|
||||||
target := tsSeedTarget(t, s.MainDB, s.WebhookID, sink.URL)
|
cfg := tsSlackConfig(t, sink.URL)
|
||||||
|
target := tsSeedTarget(t, s.MainDB, s.WebhookID, cfg)
|
||||||
event := tsSeedEvent(t, s.WebhookDB, s.WebhookID)
|
event := tsSeedEvent(t, s.WebhookDB, s.WebhookID)
|
||||||
|
|
||||||
d := iSeedDelivery(
|
d := iSeedDelivery(
|
||||||
@@ -326,7 +337,8 @@ func TestFormatSlackMessageOverTaskReconstructedEvent(
|
|||||||
|
|
||||||
s := newISetup(t)
|
s := newISetup(t)
|
||||||
|
|
||||||
target := tsSeedTarget(t, s.MainDB, s.WebhookID, tsUndeliverableHook)
|
cfg := tsSlackConfig(t, tsUndeliverableHook)
|
||||||
|
target := tsSeedTarget(t, s.MainDB, s.WebhookID, cfg)
|
||||||
event := tsSeedEvent(t, s.WebhookDB, s.WebhookID)
|
event := tsSeedEvent(t, s.WebhookDB, s.WebhookID)
|
||||||
|
|
||||||
d := iSeedDelivery(
|
d := iSeedDelivery(
|
||||||
@@ -397,7 +409,8 @@ func TestEventReconstructionSurvivesAReapedRow(t *testing.T) {
|
|||||||
|
|
||||||
s := newISetup(t)
|
s := newISetup(t)
|
||||||
|
|
||||||
target := tsSeedTarget(t, s.MainDB, s.WebhookID, tsUndeliverableHook)
|
cfg := tsSlackConfig(t, tsUndeliverableHook)
|
||||||
|
target := tsSeedTarget(t, s.MainDB, s.WebhookID, cfg)
|
||||||
event := tsSeedEvent(t, s.WebhookDB, s.WebhookID)
|
event := tsSeedEvent(t, s.WebhookDB, s.WebhookID)
|
||||||
|
|
||||||
d := iSeedDelivery(
|
d := iSeedDelivery(
|
||||||
|
|||||||
+304
-116
@@ -9,14 +9,18 @@ import (
|
|||||||
"net/url"
|
"net/url"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
"sneak.berlin/go/webhooker/internal/archive"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/metrics"
|
"sneak.berlin/go/webhooker/internal/metrics"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// ErrExportArchiveWriterEvicted exposes the sentinel returned by
|
||||||
|
// an evicted archive writer. It carries the Err prefix rather
|
||||||
|
// than this file's usual Export one because it is a sentinel
|
||||||
|
// error.
|
||||||
|
var ErrExportArchiveWriterEvicted = errArchiveWriterEvicted
|
||||||
|
|
||||||
// Exported constants for test access.
|
// Exported constants for test access.
|
||||||
const (
|
const (
|
||||||
ExportDeliveryChannelSize = deliveryChannelSize
|
ExportDeliveryChannelSize = deliveryChannelSize
|
||||||
@@ -36,6 +40,11 @@ const (
|
|||||||
ExportPendingSweepMinAge = pendingSweepMinAge
|
ExportPendingSweepMinAge = pendingSweepMinAge
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// ExportIsBlockedIP exposes isBlockedIP for testing.
|
||||||
|
func ExportIsBlockedIP(ip net.IP) bool {
|
||||||
|
return isBlockedIP(ip)
|
||||||
|
}
|
||||||
|
|
||||||
// NewTestGuard builds an SSRF Guard from an explicit egress
|
// NewTestGuard builds an SSRF Guard from an explicit egress
|
||||||
// allowlist, without going through config. Passing no prefixes
|
// allowlist, without going through config. Passing no prefixes
|
||||||
// yields the default guard, which blocks every private/reserved
|
// yields the default guard, which blocks every private/reserved
|
||||||
@@ -61,11 +70,6 @@ func ExportBlockedNetworks() []*net.IPNet {
|
|||||||
return blockedNetworks
|
return blockedNetworks
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExportBlockedPublicNetworks exposes blockedPublicNetworks.
|
|
||||||
func ExportBlockedPublicNetworks() []*net.IPNet {
|
|
||||||
return blockedPublicNetworks
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExportIsForwardableHeader exposes isForwardableHeader.
|
// ExportIsForwardableHeader exposes isForwardableHeader.
|
||||||
func ExportIsForwardableHeader(name string) bool {
|
func ExportIsForwardableHeader(name string) bool {
|
||||||
return isForwardableHeader(name)
|
return isForwardableHeader(name)
|
||||||
@@ -77,10 +81,9 @@ func ExportIsForwardableHeader(name string) bool {
|
|||||||
func ExportApplyRequestHeaders(
|
func ExportApplyRequestHeaders(
|
||||||
req *http.Request,
|
req *http.Request,
|
||||||
event *database.Event,
|
event *database.Event,
|
||||||
configured map[string]string,
|
cfg *HTTPTargetConfig,
|
||||||
userAgent string,
|
|
||||||
) []string {
|
) []string {
|
||||||
return applyRequestHeaders(req, event, configured, userAgent)
|
return applyRequestHeaders(req, event, cfg)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExportTruncate exposes truncate for testing.
|
// ExportTruncate exposes truncate for testing.
|
||||||
@@ -98,19 +101,6 @@ func (e *Engine) ExportDeliverHTTP(
|
|||||||
e.httpTarget.Deliver(ctx, webhookDB, d, task, e)
|
e.httpTarget.Deliver(ctx, webhookDB, d, task, e)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExportDeliverHTTPWithScheduler delivers via the http target, handing
|
|
||||||
// any retry to sched instead of the engine, so a test can see the
|
|
||||||
// delay each retry is given.
|
|
||||||
func (e *Engine) ExportDeliverHTTPWithScheduler(
|
|
||||||
ctx context.Context,
|
|
||||||
webhookDB *gorm.DB,
|
|
||||||
d *database.Delivery,
|
|
||||||
task *Task,
|
|
||||||
sched Scheduler,
|
|
||||||
) {
|
|
||||||
e.httpTarget.Deliver(ctx, webhookDB, d, task, sched)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExportDeliverDatabase delivers via the database target.
|
// ExportDeliverDatabase delivers via the database target.
|
||||||
func (e *Engine) ExportDeliverDatabase(
|
func (e *Engine) ExportDeliverDatabase(
|
||||||
webhookDB *gorm.DB, d *database.Delivery,
|
webhookDB *gorm.DB, d *database.Delivery,
|
||||||
@@ -137,10 +127,9 @@ func (e *Engine) ExportDeliverSlack(
|
|||||||
d *database.Delivery,
|
d *database.Delivery,
|
||||||
) {
|
) {
|
||||||
task := &Task{
|
task := &Task{
|
||||||
DeliveryID: d.ID,
|
DeliveryID: d.ID,
|
||||||
TargetID: d.TargetID,
|
TargetID: d.TargetID,
|
||||||
MaxAttempts: d.Target.MaxAttempts,
|
AttemptNum: 1,
|
||||||
AttemptNum: 1,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
e.targets[database.TargetTypeSlack].Deliver(
|
e.targets[database.TargetTypeSlack].Deliver(
|
||||||
@@ -148,21 +137,6 @@ func (e *Engine) ExportDeliverSlack(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExportUpdateDeliveryStatus exposes updateDeliveryStatus. It passes no
|
|
||||||
// target type, so no metric moves.
|
|
||||||
func (e *Engine) ExportUpdateDeliveryStatus(
|
|
||||||
webhookDB *gorm.DB,
|
|
||||||
d *database.Delivery,
|
|
||||||
status database.DeliveryStatus,
|
|
||||||
) error {
|
|
||||||
return e.updateDeliveryStatus(webhookDB, d, "", status)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExportNotify exposes notify.
|
|
||||||
func (e *Engine) ExportNotify(tasks []Task) {
|
|
||||||
e.notify(tasks)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExportProcessNewTask exposes processNewTask.
|
// ExportProcessNewTask exposes processNewTask.
|
||||||
func (e *Engine) ExportProcessNewTask(
|
func (e *Engine) ExportProcessNewTask(
|
||||||
ctx context.Context, task *Task,
|
ctx context.Context, task *Task,
|
||||||
@@ -205,39 +179,37 @@ func (e *Engine) ExportGetCircuitBreaker(
|
|||||||
return e.httpTarget.getCircuitBreaker(targetID)
|
return e.httpTarget.getCircuitBreaker(targetID)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExportSetCircuitBreaker makes cb the http target's circuit breaker
|
// ExportParseHTTPConfig exposes parseHTTPConfig.
|
||||||
// for targetID, so a test can use one with a short cooldown.
|
func (e *Engine) ExportParseHTTPConfig(
|
||||||
func (e *Engine) ExportSetCircuitBreaker(
|
configJSON string,
|
||||||
targetID string, cb *CircuitBreaker,
|
) (*HTTPTargetConfig, error) {
|
||||||
) {
|
return parseHTTPConfig(configJSON)
|
||||||
e.httpTarget.circuitBreakers.Store(targetID, cb)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExportSetSlackCircuitBreaker is ExportSetCircuitBreaker for the
|
// ExportParseSlackConfig exposes parseSlackConfig.
|
||||||
// slack target.
|
func (e *Engine) ExportParseSlackConfig(
|
||||||
func (e *Engine) ExportSetSlackCircuitBreaker(
|
configJSON string,
|
||||||
targetID string, cb *CircuitBreaker,
|
) (*SlackTargetConfig, error) {
|
||||||
) {
|
return parseSlackConfig(configJSON)
|
||||||
e.slackTarget.circuitBreakers.Store(targetID, cb)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExportDoHTTPRequest exposes the http target's
|
// ExportDoHTTPRequest exposes the http target's
|
||||||
// doHTTPRequest.
|
// doHTTPRequest.
|
||||||
func (e *Engine) ExportDoHTTPRequest(
|
func (e *Engine) ExportDoHTTPRequest(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
target *database.Target,
|
cfg *HTTPTargetConfig,
|
||||||
event *database.Event,
|
event *database.Event,
|
||||||
) (int, string, int64, error) {
|
) (int, string, int64, error) {
|
||||||
return e.httpTarget.doHTTPRequest(ctx, target, event)
|
return e.httpTarget.doHTTPRequest(ctx, cfg, event)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExportClientForRequest exposes the http target's
|
// ExportClientForRequest exposes the http target's
|
||||||
// clientForRequest.
|
// clientForRequest.
|
||||||
func (e *Engine) ExportClientForRequest(
|
func (e *Engine) ExportClientForRequest(
|
||||||
timeoutSeconds int,
|
cfg *HTTPTargetConfig,
|
||||||
originScoped []string,
|
originScoped []string,
|
||||||
) *http.Client {
|
) *http.Client {
|
||||||
return e.httpTarget.clientForRequest(timeoutSeconds, originScoped)
|
return e.httpTarget.clientForRequest(cfg, originScoped)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ErrExportTooManyRedirects exposes the sentinel the redirect
|
// ErrExportTooManyRedirects exposes the sentinel the redirect
|
||||||
@@ -272,27 +244,32 @@ func (e *Engine) ExportRecoverPendingDeliveries(
|
|||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
webhookDB *gorm.DB,
|
webhookDB *gorm.DB,
|
||||||
webhookID string,
|
webhookID string,
|
||||||
atStart bool,
|
|
||||||
) {
|
) {
|
||||||
e.recoverPendingDeliveries(
|
e.recoverPendingDeliveries(
|
||||||
ctx, webhookDB, webhookID, atStart,
|
ctx, webhookDB, webhookID,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExportRecoverWebhookDeliveries exposes
|
// ExportRecoverWebhookDeliveries exposes
|
||||||
// recoverWebhookDeliveries.
|
// recoverWebhookDeliveries.
|
||||||
func (e *Engine) ExportRecoverWebhookDeliveries(
|
func (e *Engine) ExportRecoverWebhookDeliveries(
|
||||||
ctx context.Context, webhookID string, atStart bool,
|
ctx context.Context, webhookID string,
|
||||||
) {
|
) {
|
||||||
e.recoverWebhookDeliveries(ctx, webhookID, atStart)
|
e.recoverWebhookDeliveries(ctx, webhookID)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExportRecoverDeliveries exposes recoverDeliveries, the recovery
|
// ExportRecoverInFlight exposes recoverInFlight.
|
||||||
// routine.
|
func (e *Engine) ExportRecoverInFlight(
|
||||||
func (e *Engine) ExportRecoverDeliveries(
|
ctx context.Context,
|
||||||
ctx context.Context, atStart bool,
|
|
||||||
) {
|
) {
|
||||||
e.recoverDeliveries(ctx, atStart)
|
e.recoverInFlight(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExportSweepWebhookRetries exposes sweepWebhookRetries.
|
||||||
|
func (e *Engine) ExportSweepWebhookRetries(
|
||||||
|
ctx context.Context, webhookID string,
|
||||||
|
) {
|
||||||
|
e.sweepWebhookRetries(ctx, webhookID)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExportStart exposes start for testing.
|
// ExportStart exposes start for testing.
|
||||||
@@ -339,34 +316,9 @@ func (e *Engine) ExportRetainDelivery(deliveryID string) bool {
|
|||||||
|
|
||||||
// ExportRecoverRetryingDeliveries exposes recoverRetryingDeliveries.
|
// ExportRecoverRetryingDeliveries exposes recoverRetryingDeliveries.
|
||||||
func (e *Engine) ExportRecoverRetryingDeliveries(
|
func (e *Engine) ExportRecoverRetryingDeliveries(
|
||||||
ctx context.Context, webhookDB *gorm.DB, webhookID string,
|
webhookDB *gorm.DB, webhookID string,
|
||||||
) {
|
) {
|
||||||
e.recoverRetryingDeliveries(ctx, webhookDB, webhookID)
|
e.recoverRetryingDeliveries(webhookDB, webhookID)
|
||||||
}
|
|
||||||
|
|
||||||
// ExportFailMissingTarget exposes failMissingTarget, so a test can hand
|
|
||||||
// it a delivery as a batch read it earlier.
|
|
||||||
func (e *Engine) ExportFailMissingTarget(
|
|
||||||
webhookDB *gorm.DB,
|
|
||||||
webhookID string,
|
|
||||||
d *database.Delivery,
|
|
||||||
) {
|
|
||||||
e.failMissingTarget(webhookDB, webhookID, d)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExportSendRecoveredDeliveries exposes sendRecoveredDeliveries, so a
|
|
||||||
// test can hand it a target map that lacks a delivery's target.
|
|
||||||
func (e *Engine) ExportSendRecoveredDeliveries(
|
|
||||||
ctx context.Context,
|
|
||||||
webhookDB *gorm.DB,
|
|
||||||
deliveries []database.Delivery,
|
|
||||||
webhookID string,
|
|
||||||
targetMap map[string]database.Target,
|
|
||||||
settled map[string]struct{},
|
|
||||||
) {
|
|
||||||
e.sendRecoveredDeliveries(
|
|
||||||
ctx, webhookDB, deliveries, webhookID, targetMap, settled,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExportDeliveryCh returns the delivery channel.
|
// ExportDeliveryCh returns the delivery channel.
|
||||||
@@ -380,8 +332,7 @@ func (e *Engine) ExportRetryCh() chan Task {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// NewTestEngine creates an Engine for unit tests without
|
// NewTestEngine creates an Engine for unit tests without
|
||||||
// database dependencies. Its archive writers have no data
|
// database dependencies.
|
||||||
// directory, so a delivery to a database target fails.
|
|
||||||
func NewTestEngine(
|
func NewTestEngine(
|
||||||
log *slog.Logger,
|
log *slog.Logger,
|
||||||
client *http.Client,
|
client *http.Client,
|
||||||
@@ -392,8 +343,7 @@ func NewTestEngine(
|
|||||||
deliveryCh: make(chan Task, deliveryChannelSize),
|
deliveryCh: make(chan Task, deliveryChannelSize),
|
||||||
retryCh: make(chan Task, retryChannelSize),
|
retryCh: make(chan Task, retryChannelSize),
|
||||||
workers: workers,
|
workers: workers,
|
||||||
mtr: metrics.New(prometheus.NewRegistry()),
|
mtr: metrics.Default(),
|
||||||
archives: archive.NewWriters(archive.WritersParams{Logger: log}),
|
|
||||||
}
|
}
|
||||||
e.initTargets(client)
|
e.initTargets(client)
|
||||||
|
|
||||||
@@ -406,10 +356,9 @@ func NewTestEngineSmallRetry(
|
|||||||
log *slog.Logger,
|
log *slog.Logger,
|
||||||
) *Engine {
|
) *Engine {
|
||||||
e := &Engine{
|
e := &Engine{
|
||||||
log: log,
|
log: log,
|
||||||
retryCh: make(chan Task, 1),
|
retryCh: make(chan Task, 1),
|
||||||
mtr: metrics.New(prometheus.NewRegistry()),
|
mtr: metrics.Default(),
|
||||||
archives: archive.NewWriters(archive.WritersParams{Logger: log}),
|
|
||||||
}
|
}
|
||||||
e.initTargets(nil)
|
e.initTargets(nil)
|
||||||
|
|
||||||
@@ -417,8 +366,7 @@ func NewTestEngineSmallRetry(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// NewTestEngineWithDB creates an Engine with a real
|
// NewTestEngineWithDB creates an Engine with a real
|
||||||
// database and dbManager for integration tests, and archive
|
// database and dbManager for integration tests.
|
||||||
// writers of its own over the same two.
|
|
||||||
func NewTestEngineWithDB(
|
func NewTestEngineWithDB(
|
||||||
db *database.Database,
|
db *database.Database,
|
||||||
dbMgr *database.WebhookDBManager,
|
dbMgr *database.WebhookDBManager,
|
||||||
@@ -433,25 +381,16 @@ func NewTestEngineWithDB(
|
|||||||
deliveryCh: make(chan Task, deliveryChannelSize),
|
deliveryCh: make(chan Task, deliveryChannelSize),
|
||||||
retryCh: make(chan Task, retryChannelSize),
|
retryCh: make(chan Task, retryChannelSize),
|
||||||
workers: workers,
|
workers: workers,
|
||||||
mtr: metrics.New(prometheus.NewRegistry()),
|
mtr: metrics.Default(),
|
||||||
archives: archive.NewWriters(archive.WritersParams{
|
|
||||||
Database: db,
|
|
||||||
DBManager: dbMgr,
|
|
||||||
Logger: log,
|
|
||||||
}),
|
|
||||||
}
|
}
|
||||||
e.initTargets(client)
|
e.initTargets(client)
|
||||||
|
|
||||||
return e
|
return e
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExportArchives returns the engine's archive writers.
|
|
||||||
func (e *Engine) ExportArchives() *archive.Writers {
|
|
||||||
return e.archives
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExportSetMetrics substitutes the engine's metric set, so a test can
|
// ExportSetMetrics substitutes the engine's metric set, so a test can
|
||||||
// assert on collectors registered on a registry it holds.
|
// assert on collectors registered on a private registry instead of
|
||||||
|
// the process-wide ones every other test is also moving.
|
||||||
func (e *Engine) ExportSetMetrics(mtr *metrics.Set) {
|
func (e *Engine) ExportSetMetrics(mtr *metrics.Set) {
|
||||||
e.mtr = mtr
|
e.mtr = mtr
|
||||||
}
|
}
|
||||||
@@ -472,3 +411,252 @@ func NewTestCircuitBreaker(
|
|||||||
cooldown: cooldown,
|
cooldown: cooldown,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ExportArchivedEvent aliases the archive row type so black-box
|
||||||
|
// tests can construct and read archive rows.
|
||||||
|
type ExportArchivedEvent = archivedEvent
|
||||||
|
|
||||||
|
// ExportArchiveWriter wraps an archiveWriter so black-box tests
|
||||||
|
// can exercise the per-webhook archive file mechanics.
|
||||||
|
type ExportArchiveWriter struct {
|
||||||
|
w *archiveWriter
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewExportArchiveWriter builds an archive writer for tests,
|
||||||
|
// optionally overriding the reopen debounce (a non-positive
|
||||||
|
// debounce keeps the production default).
|
||||||
|
func NewExportArchiveWriter(
|
||||||
|
path string, log *slog.Logger, debounce time.Duration,
|
||||||
|
) *ExportArchiveWriter {
|
||||||
|
w := newArchiveWriter(path, log)
|
||||||
|
if debounce > 0 {
|
||||||
|
w.debounce = debounce
|
||||||
|
}
|
||||||
|
|
||||||
|
return &ExportArchiveWriter{w: w}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write archives a row through the writer.
|
||||||
|
func (e *ExportArchiveWriter) Write(
|
||||||
|
row ExportArchivedEvent, expiry time.Duration,
|
||||||
|
) error {
|
||||||
|
return e.w.write(row, expiry)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Open opens the archive file, pruning when expiry is positive.
|
||||||
|
func (e *ExportArchiveWriter) Open(expiry time.Duration) error {
|
||||||
|
return e.w.open(expiry)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reopen closes and reopens the archive file.
|
||||||
|
func (e *ExportArchiveWriter) Reopen(
|
||||||
|
expiry time.Duration,
|
||||||
|
) error {
|
||||||
|
return e.w.reopen(expiry)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reopens reports how many times the file has been opened.
|
||||||
|
func (e *ExportArchiveWriter) Reopens() int {
|
||||||
|
return e.w.reopens
|
||||||
|
}
|
||||||
|
|
||||||
|
// DB returns the writer's current open handle for row
|
||||||
|
// inspection in tests.
|
||||||
|
func (e *ExportArchiveWriter) DB() *gorm.DB {
|
||||||
|
return e.w.db
|
||||||
|
}
|
||||||
|
|
||||||
|
// Path returns the archive file the writer owns.
|
||||||
|
func (e *ExportArchiveWriter) Path() string {
|
||||||
|
return e.w.path
|
||||||
|
}
|
||||||
|
|
||||||
|
// OpenExisting opens the archive without permitting creation,
|
||||||
|
// the way the idle sweep does.
|
||||||
|
func (e *ExportArchiveWriter) OpenExisting(
|
||||||
|
expiry time.Duration,
|
||||||
|
) error {
|
||||||
|
return e.w.openMode(archiveModeExisting, expiry)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SweepExpired runs an idle sweep of the archive.
|
||||||
|
func (e *ExportArchiveWriter) SweepExpired(
|
||||||
|
expiry time.Duration,
|
||||||
|
) error {
|
||||||
|
return e.w.sweepExpired(expiry)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Evict marks the writer evicted and closes its handle, exactly
|
||||||
|
// as leaving the registry does.
|
||||||
|
func (e *ExportArchiveWriter) Evict() {
|
||||||
|
e.w.evict()
|
||||||
|
}
|
||||||
|
|
||||||
|
// HandleOpen reports whether the writer currently holds an open
|
||||||
|
// archive handle.
|
||||||
|
func (e *ExportArchiveWriter) HandleOpen() bool {
|
||||||
|
e.w.mu.Lock()
|
||||||
|
defer e.w.mu.Unlock()
|
||||||
|
|
||||||
|
return e.w.db != nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Same reports whether both wrappers refer to the very same
|
||||||
|
// underlying archive writer, so a test can prove a registry entry
|
||||||
|
// is the writer it was handed rather than a replacement.
|
||||||
|
func (e *ExportArchiveWriter) Same(
|
||||||
|
other *ExportArchiveWriter,
|
||||||
|
) bool {
|
||||||
|
return other != nil && e.w == other.w
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExportArchiveWriterFor returns the archive writer the registry
|
||||||
|
// currently caches for a webhook, or nil when none is cached. It
|
||||||
|
// never creates one, so a test can hold a reference to the very
|
||||||
|
// writer an eviction is about to detach.
|
||||||
|
func (e *Engine) ExportArchiveWriterFor(
|
||||||
|
webhookID string,
|
||||||
|
) *ExportArchiveWriter {
|
||||||
|
e.dbTarget.mu.Lock()
|
||||||
|
defer e.dbTarget.mu.Unlock()
|
||||||
|
|
||||||
|
w, ok := e.dbTarget.writers[webhookID]
|
||||||
|
if !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return &ExportArchiveWriter{w: w}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExportHasArchiveWriter reports whether the database target
|
||||||
|
// currently caches an archive writer for a webhook.
|
||||||
|
func (e *Engine) ExportHasArchiveWriter(
|
||||||
|
webhookID string,
|
||||||
|
) bool {
|
||||||
|
e.dbTarget.mu.Lock()
|
||||||
|
defer e.dbTarget.mu.Unlock()
|
||||||
|
|
||||||
|
_, ok := e.dbTarget.writers[webhookID]
|
||||||
|
|
||||||
|
return ok
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExportArchiveHandleOpen reports whether the cached archive
|
||||||
|
// writer for a webhook holds an open database handle. It
|
||||||
|
// returns false when no writer is cached.
|
||||||
|
func (e *Engine) ExportArchiveHandleOpen(
|
||||||
|
webhookID string,
|
||||||
|
) bool {
|
||||||
|
e.dbTarget.mu.Lock()
|
||||||
|
w, ok := e.dbTarget.writers[webhookID]
|
||||||
|
e.dbTarget.mu.Unlock()
|
||||||
|
|
||||||
|
if !ok {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
w.mu.Lock()
|
||||||
|
defer w.mu.Unlock()
|
||||||
|
|
||||||
|
return w.db != nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExportEnsureArchiveWriter creates (if needed) and returns the
|
||||||
|
// archive file path of the cached writer for a webhook, so a
|
||||||
|
// test can prime the registry the way a delivery would.
|
||||||
|
func (e *Engine) ExportEnsureArchiveWriter(
|
||||||
|
webhookID string,
|
||||||
|
) (string, error) {
|
||||||
|
w, err := e.dbTarget.writerFor(webhookID)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
return w.path, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExportSweepWriterFor takes a webhook's registry writer exactly
|
||||||
|
// as the idle sweep does, reporting whether the sweep had to
|
||||||
|
// create the entry. It lets a test drive the registry through the
|
||||||
|
// sweep's own entry point instead of choreographing goroutines.
|
||||||
|
func (e *Engine) ExportSweepWriterFor(
|
||||||
|
webhookID string,
|
||||||
|
) (*ExportArchiveWriter, bool, error) {
|
||||||
|
w, created, err := e.dbTarget.sweepWriterFor(webhookID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return &ExportArchiveWriter{w: w}, created, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExportReleaseSweepWriter releases a sweep-created registry entry
|
||||||
|
// exactly as a finished sweep does.
|
||||||
|
func (e *Engine) ExportReleaseSweepWriter(
|
||||||
|
webhookID string, w *ExportArchiveWriter,
|
||||||
|
) {
|
||||||
|
e.dbTarget.releaseSweepWriter(webhookID, w.w)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewTestArchiveSweeper builds an ArchiveSweeper backed by the
|
||||||
|
// given main database and engine, without the fx lifecycle.
|
||||||
|
// Intended for tests.
|
||||||
|
func NewTestArchiveSweeper(
|
||||||
|
db *database.Database,
|
||||||
|
eng *Engine,
|
||||||
|
log *slog.Logger,
|
||||||
|
) *ArchiveSweeper {
|
||||||
|
return &ArchiveSweeper{
|
||||||
|
db: db,
|
||||||
|
eng: eng,
|
||||||
|
log: log,
|
||||||
|
interval: time.Hour,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExportSweep runs a single archive sweep synchronously for
|
||||||
|
// tests.
|
||||||
|
func (s *ArchiveSweeper) ExportSweep(ctx context.Context) {
|
||||||
|
s.sweep(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExportStart starts the sweeper's background loop for tests.
|
||||||
|
func (s *ArchiveSweeper) ExportStart() {
|
||||||
|
s.start()
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExportRegisterHooks registers the sweeper's real fx lifecycle
|
||||||
|
// hooks on a lifecycle supplied by a test, so a test can drive
|
||||||
|
// the exact OnStart/OnStop functions the application runs and
|
||||||
|
// hand OnStart the kind of context fx actually supplies.
|
||||||
|
func (s *ArchiveSweeper) ExportRegisterHooks(lc fx.Lifecycle) {
|
||||||
|
s.registerHooks(lc)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExportStop stops the sweeper's background loop for tests.
|
||||||
|
func (s *ArchiveSweeper) ExportStop(ctx context.Context) error {
|
||||||
|
return s.stop(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExportWedgeLoop adds a goroutine to the sweeper's WaitGroup
|
||||||
|
// that never observes cancellation and returns only when release
|
||||||
|
// is closed. It stands in for a prune stuck on a locked archive.
|
||||||
|
func (s *ArchiveSweeper) ExportWedgeLoop(
|
||||||
|
release <-chan struct{},
|
||||||
|
) {
|
||||||
|
s.wg.Go(func() {
|
||||||
|
<-release
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExportSetInterval overrides the sweep interval for tests.
|
||||||
|
func (s *ArchiveSweeper) ExportSetInterval(d time.Duration) {
|
||||||
|
s.interval = d
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExportParseArchiveExpiry exposes parseArchiveExpiry.
|
||||||
|
func ExportParseArchiveExpiry(
|
||||||
|
configJSON string,
|
||||||
|
) (time.Duration, error) {
|
||||||
|
return parseArchiveExpiry(configJSON)
|
||||||
|
}
|
||||||
|
|||||||
@@ -6,10 +6,10 @@ import "sync"
|
|||||||
//
|
//
|
||||||
// A delivery is owned from the moment a task for it is handed to a
|
// A delivery is owned from the moment a task for it is handed to a
|
||||||
// channel or to a retry timer until the engine has no further plan for
|
// channel or to a retry timer until the engine has no further plan for
|
||||||
// it in memory. The recovery routine, at start and on every tick of the
|
// it in memory. Restart recovery and both arms of the periodic sweep
|
||||||
// sweep, re-dispatches only deliveries the set does not hold, which is
|
// re-dispatch only deliveries the set does not hold, which is what
|
||||||
// what makes it exact rather than a guess about how long a row has sat
|
// makes them exact rather than a guess about how long a row has sat at
|
||||||
// at pending.
|
// pending.
|
||||||
//
|
//
|
||||||
// This replaces reasoning from timestamps. A delivery's row says
|
// This replaces reasoning from timestamps. A delivery's row says
|
||||||
// pending from creation until its outcome is written, which covers
|
// pending from creation until its outcome is written, which covers
|
||||||
@@ -25,8 +25,8 @@ import "sync"
|
|||||||
// In-memory state is sufficient because a data directory admits one
|
// In-memory state is sufficient because a data directory admits one
|
||||||
// process: internal/datadir takes an flock on it at startup and a
|
// process: internal/datadir takes an flock on it at startup and a
|
||||||
// second instance refuses to run. Deliveries owned by a process that
|
// second instance refuses to run. Deliveries owned by a process that
|
||||||
// died are not in any successor's set, and the recovery routine's run
|
// died are not in any successor's set, and restart recovery is what
|
||||||
// at start is what picks those up.
|
// picks those up.
|
||||||
//
|
//
|
||||||
// References are counted rather than held as a plain set because
|
// References are counted rather than held as a plain set because
|
||||||
// ownership outlives the worker that took it. A target that schedules
|
// ownership outlives the worker that took it. A target that schedules
|
||||||
@@ -61,9 +61,9 @@ func (s *inflightSet) retain(deliveryID string) {
|
|||||||
|
|
||||||
// retainIdle takes the first reference on a delivery, and reports
|
// retainIdle takes the first reference on a delivery, and reports
|
||||||
// whether it got it. It fails when the engine already owns the
|
// whether it got it. It fails when the engine already owns the
|
||||||
// delivery, which is what makes two claimants — notify and the
|
// delivery, which is what makes two claimants — restart recovery and
|
||||||
// recovery routine, say — mutually exclusive rather than merely
|
// the sweep run concurrently, or two sweep arms — mutually exclusive
|
||||||
// atomic.
|
// rather than merely atomic.
|
||||||
func (s *inflightSet) retainIdle(deliveryID string) bool {
|
func (s *inflightSet) retainIdle(deliveryID string) bool {
|
||||||
s.mu.Lock()
|
s.mu.Lock()
|
||||||
defer s.mu.Unlock()
|
defer s.mu.Unlock()
|
||||||
|
|||||||
@@ -2,6 +2,8 @@ package delivery_test
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -30,7 +32,7 @@ func fSweepSetup(
|
|||||||
|
|
||||||
iCreateTarget(t, s.MainDB, targetID,
|
iCreateTarget(t, s.MainDB, targetID,
|
||||||
s.WebhookID, name,
|
s.WebhookID, name,
|
||||||
database.TargetTypeLog, "", 1,
|
database.TargetTypeLog, "", 0,
|
||||||
)
|
)
|
||||||
|
|
||||||
require.NoError(t, s.MainDB.Create(&database.Webhook{
|
require.NoError(t, s.MainDB.Create(&database.Webhook{
|
||||||
@@ -46,9 +48,8 @@ func fSweepSetup(
|
|||||||
//
|
//
|
||||||
// Every caller drives the dispatch paths synchronously and has already
|
// Every caller drives the dispatch paths synchronously and has already
|
||||||
// waited for them to return, so anything they queued is in the channel
|
// waited for them to return, so anything they queued is in the channel
|
||||||
// by now, and nothing is waited for. A timer here would race the queued
|
// by now. The short grace covers nothing but scheduler jitter, and is
|
||||||
// tasks: on a busy host it can be due by the time select looks, and
|
// kept small because one of these tests runs the drain forty times.
|
||||||
// select picks at random among the cases that are ready.
|
|
||||||
func fDrain(e *delivery.Engine) []delivery.Task {
|
func fDrain(e *delivery.Engine) []delivery.Task {
|
||||||
var out []delivery.Task
|
var out []delivery.Task
|
||||||
|
|
||||||
@@ -58,12 +59,39 @@ func fDrain(e *delivery.Engine) []delivery.Task {
|
|||||||
out = append(out, task)
|
out = append(out, task)
|
||||||
case task := <-e.ExportRetryCh():
|
case task := <-e.ExportRetryCh():
|
||||||
out = append(out, task)
|
out = append(out, task)
|
||||||
default:
|
case <-time.After(25 * time.Millisecond):
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestArchiveHandleIsWAL closes the last gap in the durability
|
||||||
|
// evidence: the main and per-webhook tiers each assert their journal
|
||||||
|
// mode on a live handle, and the archive tier gets its settings from
|
||||||
|
// the same code path but nothing checked the running file.
|
||||||
|
func TestArchiveHandleIsWAL(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
w := delivery.NewExportArchiveWriter(
|
||||||
|
filepath.Join(t.TempDir(), "archive-wal.db"),
|
||||||
|
archiveTestLogger(), 0,
|
||||||
|
)
|
||||||
|
|
||||||
|
require.NoError(t, w.Open(0))
|
||||||
|
|
||||||
|
var mode string
|
||||||
|
|
||||||
|
row := w.DB().Raw("pragma journal_mode").Row()
|
||||||
|
require.NoError(t, row.Scan(&mode))
|
||||||
|
assert.Equal(t, "wal", strings.ToLower(mode))
|
||||||
|
|
||||||
|
var busy string
|
||||||
|
|
||||||
|
row = w.DB().Raw("pragma busy_timeout").Row()
|
||||||
|
require.NoError(t, row.Scan(&busy))
|
||||||
|
assert.Equal(t, "10000", busy)
|
||||||
|
}
|
||||||
|
|
||||||
// TestSweepLeavesAQueuedDeliveryAlone is the case the age bound cannot
|
// TestSweepLeavesAQueuedDeliveryAlone is the case the age bound cannot
|
||||||
// see. The delivery is queued and untouched, so its row is arbitrarily
|
// see. The delivery is queued and untouched, so its row is arbitrarily
|
||||||
// old and still perfectly healthy; only ownership distinguishes it
|
// old and still perfectly healthy; only ownership distinguishes it
|
||||||
@@ -84,9 +112,9 @@ func TestSweepLeavesAQueuedDeliveryAlone(t *testing.T) {
|
|||||||
)
|
)
|
||||||
rAgePending(t, s.WebhookDB, d.ID)
|
rAgePending(t, s.WebhookDB, d.ID)
|
||||||
|
|
||||||
// Queued exactly as StoreEvent queues it, and never dequeued:
|
// Queued exactly as the receiver queues it, and never dequeued:
|
||||||
// no workers are running in this engine.
|
// no workers are running in this engine.
|
||||||
s.Engine.ExportNotify([]delivery.Task{{
|
s.Engine.Notify([]delivery.Task{{
|
||||||
DeliveryID: d.ID,
|
DeliveryID: d.ID,
|
||||||
EventID: event.ID,
|
EventID: event.ID,
|
||||||
WebhookID: s.WebhookID,
|
WebhookID: s.WebhookID,
|
||||||
@@ -95,8 +123,8 @@ func TestSweepLeavesAQueuedDeliveryAlone(t *testing.T) {
|
|||||||
|
|
||||||
require.Equal(t, 1, s.Engine.ExportInflightHeld())
|
require.Equal(t, 1, s.Engine.ExportInflightHeld())
|
||||||
|
|
||||||
s.Engine.ExportRecoverWebhookDeliveries(
|
s.Engine.ExportSweepWebhookRetries(
|
||||||
context.Background(), s.WebhookID, onTick,
|
context.Background(), s.WebhookID,
|
||||||
)
|
)
|
||||||
|
|
||||||
tasks := fDrain(s.Engine)
|
tasks := fDrain(s.Engine)
|
||||||
@@ -107,11 +135,9 @@ func TestSweepLeavesAQueuedDeliveryAlone(t *testing.T) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestRecoveryAndSweepDoNotDoubleDispatch drives the recovery routine's
|
// TestRecoveryAndSweepDoNotDoubleDispatch drives the two entry points
|
||||||
// run at start and a tick of the sweep at once against one aged pending
|
// the engine starts concurrently against one aged pending row. Before
|
||||||
// row. The engine runs them one after the other, but notify can race
|
// ownership they both dispatched it.
|
||||||
// either, and ownership is what keeps any two claimants from both
|
|
||||||
// sending. Before ownership the two runs both dispatched the row.
|
|
||||||
func TestRecoveryAndSweepDoNotDoubleDispatch(t *testing.T) {
|
func TestRecoveryAndSweepDoNotDoubleDispatch(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -135,12 +161,12 @@ func TestRecoveryAndSweepDoNotDoubleDispatch(t *testing.T) {
|
|||||||
|
|
||||||
wg.Go(func() {
|
wg.Go(func() {
|
||||||
s.Engine.ExportRecoverPendingDeliveries(
|
s.Engine.ExportRecoverPendingDeliveries(
|
||||||
ctx, s.WebhookDB, s.WebhookID, atStart,
|
ctx, s.WebhookDB, s.WebhookID,
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
wg.Go(func() {
|
wg.Go(func() {
|
||||||
s.Engine.ExportRecoverWebhookDeliveries(
|
s.Engine.ExportSweepWebhookRetries(
|
||||||
ctx, s.WebhookID, onTick,
|
ctx, s.WebhookID,
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
@@ -202,7 +228,7 @@ func TestOwnershipIsReleasedAfterDelivery(t *testing.T) {
|
|||||||
|
|
||||||
iCreateTarget(t, s.MainDB, targetID,
|
iCreateTarget(t, s.MainDB, targetID,
|
||||||
s.WebhookID, "released",
|
s.WebhookID, "released",
|
||||||
database.TargetTypeLog, "", 1,
|
database.TargetTypeLog, "", 0,
|
||||||
)
|
)
|
||||||
|
|
||||||
event := iSeedEvent(
|
event := iSeedEvent(
|
||||||
@@ -224,7 +250,7 @@ func TestOwnershipIsReleasedAfterDelivery(t *testing.T) {
|
|||||||
|
|
||||||
body := `{"released":true}`
|
body := `{"released":true}`
|
||||||
|
|
||||||
s.Engine.ExportNotify([]delivery.Task{{
|
s.Engine.Notify([]delivery.Task{{
|
||||||
DeliveryID: d.ID,
|
DeliveryID: d.ID,
|
||||||
EventID: event.ID,
|
EventID: event.ID,
|
||||||
WebhookID: s.WebhookID,
|
WebhookID: s.WebhookID,
|
||||||
@@ -247,75 +273,6 @@ func TestOwnershipIsReleasedAfterDelivery(t *testing.T) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestNotifyAfterRecoveryDoesNotSendAgain is the startup race of
|
|
||||||
// https://git.eeqj.de/sneak/webhooker/issues/299. StoreEvent has
|
|
||||||
// written a delivery, the recovery routine's run at start finds it
|
|
||||||
// pending, sends it and releases it, and only then does StoreEvent's
|
|
||||||
// notify for it arrive. Nothing owns the delivery by then, so notify
|
|
||||||
// takes it.
|
|
||||||
func TestNotifyAfterRecoveryDoesNotSendAgain(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
targetID := uuid.New().String()
|
|
||||||
s := fSweepSetup(t, targetID, "recovered")
|
|
||||||
|
|
||||||
event := iSeedEvent(
|
|
||||||
t, s.WebhookDB, s.WebhookID, `{"recovered":true}`,
|
|
||||||
)
|
|
||||||
|
|
||||||
d := iSeedDelivery(
|
|
||||||
t, s.WebhookDB, event.ID, targetID,
|
|
||||||
database.DeliveryStatusPending,
|
|
||||||
)
|
|
||||||
|
|
||||||
s.Engine.ExportStart()
|
|
||||||
|
|
||||||
defer func() {
|
|
||||||
require.NoError(
|
|
||||||
t, s.Engine.ExportStop(context.Background()),
|
|
||||||
)
|
|
||||||
}()
|
|
||||||
|
|
||||||
// The recovery routine's run at start sends the delivery and lets
|
|
||||||
// it go.
|
|
||||||
iWaitForDelivered(t, s.WebhookDB, d.ID)
|
|
||||||
require.Eventually(
|
|
||||||
t,
|
|
||||||
func() bool {
|
|
||||||
return s.Engine.ExportInflightHeld() == 0
|
|
||||||
},
|
|
||||||
5*time.Second, 20*time.Millisecond,
|
|
||||||
)
|
|
||||||
|
|
||||||
body := event.Body
|
|
||||||
|
|
||||||
s.Engine.ExportNotify([]delivery.Task{{
|
|
||||||
DeliveryID: d.ID,
|
|
||||||
EventID: event.ID,
|
|
||||||
WebhookID: s.WebhookID,
|
|
||||||
TargetID: targetID,
|
|
||||||
TargetName: "recovered",
|
|
||||||
TargetType: database.TargetTypeLog,
|
|
||||||
Body: &body,
|
|
||||||
EntrypointID: event.EntrypointID,
|
|
||||||
}})
|
|
||||||
|
|
||||||
// notify took the delivery, and a worker releases it once it has
|
|
||||||
// run the task.
|
|
||||||
require.Eventually(
|
|
||||||
t,
|
|
||||||
func() bool {
|
|
||||||
return s.Engine.ExportInflightHeld() == 0
|
|
||||||
},
|
|
||||||
5*time.Second, 20*time.Millisecond,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.Len(
|
|
||||||
t, iResults(t, s.WebhookDB, d.ID), 1,
|
|
||||||
"the delivery was sent a second time",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRetryingRecoverySkipsASuccessfulResult is the retrying-side twin
|
// TestRetryingRecoverySkipsASuccessfulResult is the retrying-side twin
|
||||||
// of the pending reconcile. A second attempt that reached the receiver
|
// of the pending reconcile. A second attempt that reached the receiver
|
||||||
// and whose status write then failed sits at retrying holding a
|
// and whose status write then failed sits at retrying holding a
|
||||||
@@ -338,7 +295,7 @@ func TestRetryingRecoverySkipsASuccessfulResult(t *testing.T) {
|
|||||||
rSeedResult(t, s.WebhookDB, d.ID, 2, true)
|
rSeedResult(t, s.WebhookDB, d.ID, 2, true)
|
||||||
|
|
||||||
s.Engine.ExportRecoverRetryingDeliveries(
|
s.Engine.ExportRecoverRetryingDeliveries(
|
||||||
context.Background(), s.WebhookDB, s.WebhookID,
|
s.WebhookDB, s.WebhookID,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert.Empty(
|
assert.Empty(
|
||||||
@@ -353,8 +310,8 @@ func TestRetryingRecoverySkipsASuccessfulResult(t *testing.T) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestRetryingSweepSkipsASuccessfulResult is the same rule on a tick of
|
// TestRetryingSweepSkipsASuccessfulResult is the same rule on the
|
||||||
// the sweep.
|
// periodic sweep's retrying arm.
|
||||||
func TestRetryingSweepSkipsASuccessfulResult(t *testing.T) {
|
func TestRetryingSweepSkipsASuccessfulResult(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -372,8 +329,8 @@ func TestRetryingSweepSkipsASuccessfulResult(t *testing.T) {
|
|||||||
rSeedResult(t, s.WebhookDB, d.ID, 1, false)
|
rSeedResult(t, s.WebhookDB, d.ID, 1, false)
|
||||||
rSeedResult(t, s.WebhookDB, d.ID, 2, true)
|
rSeedResult(t, s.WebhookDB, d.ID, 2, true)
|
||||||
|
|
||||||
s.Engine.ExportRecoverWebhookDeliveries(
|
s.Engine.ExportSweepWebhookRetries(
|
||||||
context.Background(), s.WebhookID, onTick,
|
context.Background(), s.WebhookID,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert.Empty(t, fDrain(s.Engine))
|
assert.Empty(t, fDrain(s.Engine))
|
||||||
@@ -424,8 +381,8 @@ func TestScheduledRetryIsNotSweptDuringBackoff(t *testing.T) {
|
|||||||
|
|
||||||
require.Equal(t, 1, s.Engine.ExportInflightHeld())
|
require.Equal(t, 1, s.Engine.ExportInflightHeld())
|
||||||
|
|
||||||
s.Engine.ExportRecoverWebhookDeliveries(
|
s.Engine.ExportSweepWebhookRetries(
|
||||||
context.Background(), s.WebhookID, onTick,
|
context.Background(), s.WebhookID,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert.Empty(
|
assert.Empty(
|
||||||
@@ -437,8 +394,7 @@ func TestScheduledRetryIsNotSweptDuringBackoff(t *testing.T) {
|
|||||||
|
|
||||||
// TestRedispatchStampsTheRow pins the cadence control: a stranded
|
// TestRedispatchStampsTheRow pins the cadence control: a stranded
|
||||||
// delivery that has just been handed out is not selected again by the
|
// delivery that has just been handed out is not selected again by the
|
||||||
// next tick a minute later. The stamp is stored in the same form as
|
// next tick a minute later.
|
||||||
// created_at.
|
|
||||||
func TestRedispatchStampsTheRow(t *testing.T) {
|
func TestRedispatchStampsTheRow(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -457,7 +413,7 @@ func TestRedispatchStampsTheRow(t *testing.T) {
|
|||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|
||||||
s.Engine.ExportRecoverWebhookDeliveries(ctx, s.WebhookID, onTick)
|
s.Engine.ExportSweepWebhookRetries(ctx, s.WebhookID)
|
||||||
require.Len(t, fDrain(s.Engine), 1)
|
require.Len(t, fDrain(s.Engine), 1)
|
||||||
|
|
||||||
var row database.Delivery
|
var row database.Delivery
|
||||||
@@ -469,13 +425,4 @@ func TestRedispatchStampsTheRow(t *testing.T) {
|
|||||||
"a re-dispatched delivery must be stamped so the "+
|
"a re-dispatched delivery must be stamped so the "+
|
||||||
"next tick does not select it again",
|
"next tick does not select it again",
|
||||||
)
|
)
|
||||||
|
|
||||||
var createdAt, updatedAt string
|
|
||||||
|
|
||||||
require.NoError(t, s.WebhookDB.Raw(
|
|
||||||
`SELECT CAST(created_at AS TEXT), CAST(updated_at AS TEXT)
|
|
||||||
FROM deliveries WHERE id = ?`, d.ID,
|
|
||||||
).Row().Scan(&createdAt, &updatedAt))
|
|
||||||
assertStoredTime(t, createdAt)
|
|
||||||
assertStoredTime(t, updatedAt)
|
|
||||||
}
|
}
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user