Compare commits
1
Commits
next
..
e67ac700c4
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e67ac700c4 |
+26
-83
@@ -1,84 +1,27 @@
|
|||||||
# .dockerignore does NOT use .gitignore semantics. Docker matches with
|
# .git is sent so the build can derive the version it stamps into the binary
|
||||||
# moby/patternmatcher: filepath.Match plus `**`, so `*` does not cross
|
# (script/version). Its config, which can hold a remote URL carrying a
|
||||||
# `/` and an unprefixed pattern is anchored at the context root. Every
|
# credential and which `git describe` does not need, is left out of a
|
||||||
# depth-independent pattern therefore needs `**/`, or `config/.env` and
|
# directory context. A context sent as a tar is not filtered by this file, so
|
||||||
# `certs/server.key` still ship while this file reads as solved. Only
|
# it carries .git/config unless its sender leaves it out.
|
||||||
# genuinely root-anchored entries go unprefixed. Never transplant these
|
.git/config
|
||||||
# into .gitignore, where `**/` is wrong.
|
|
||||||
|
# No tracked file may be listed here: git in the build would see it as
|
||||||
|
# deleted and mark the version -dirty.
|
||||||
#
|
#
|
||||||
# Matching is case-sensitive, so secrets use character ranges rather
|
# .ci-fingerprint is deliberately NOT excluded: it is the CI cache barrier
|
||||||
# than an ALL-CAPS twin, which would still miss `Server.Key`.
|
# that keeps the check stages from replaying a cached pass. See the lint
|
||||||
#
|
# stage of the Dockerfile.
|
||||||
# Extend with this repo's own host-built artifacts, written anchored:
|
bin/
|
||||||
# `/myapp`, never `**/myapp`, which also matches `cmd/myapp/` and
|
# Extracted from 3p/ by `make assets` inside the build; a host copy is not
|
||||||
# deletes the package directory from the context.
|
# needed. The tarball in 3p/ must stay in the context.
|
||||||
|
static/js/alpine.min.js
|
||||||
# .git is sent without its config. Without a VERSION build argument the
|
.env
|
||||||
# stage that compiles runs `git describe --tags --always` on .git, which
|
.env.*
|
||||||
# does not need .git/config; that file can hold a credential, such as a
|
*.db
|
||||||
# password in a remote URL or the token the CI checkout step stores there.
|
*.sqlite
|
||||||
# Each submodule keeps a config with the same exposure in its git directory
|
*.sqlite3
|
||||||
# under .git/modules/, nested again for a submodule's own submodules, or in
|
.DS_Store
|
||||||
# its own .git directory when it keeps one.
|
.idea/
|
||||||
# KNOWN GAP: a submodule whose name has a `config` segment (`config`,
|
.vscode/
|
||||||
# `deploy/config`, `config/lib`) loses its whole git directory, because
|
tmp/
|
||||||
# `**/.git/modules/**/config` also matches that segment's directory
|
temp/
|
||||||
# under .git/modules/. Go's version stamping then fails the build;
|
|
||||||
# nothing leaks. Name such a submodule without that segment:
|
|
||||||
# `git submodule add --name`.
|
|
||||||
**/.git/config
|
|
||||||
**/.git/modules/**/config
|
|
||||||
|
|
||||||
# Agent scratch: one full checkout of the repo per in-flight agent.
|
|
||||||
# Anchored because it occurs once where agents run at the repo root.
|
|
||||||
# KNOWN GAP: a repo running agents in subdirectories still ships
|
|
||||||
# `services/api/.claude/` and must add its own anchored entry.
|
|
||||||
.claude
|
|
||||||
|
|
||||||
# Environment files. `*.env` covers bare `.env` and the `prod.env`
|
|
||||||
# convention. Re-include a committed template with a negation if the
|
|
||||||
# build needs one: `!docs/example.env`.
|
|
||||||
**/*.[eE][nN][vV]
|
|
||||||
**/.[eE][nN][vV].*
|
|
||||||
**/.[eE][nN][vV][rR][cC]
|
|
||||||
|
|
||||||
# Private keys and the bundles carrying them. Public certificates
|
|
||||||
# (*.crt, *.cer) are deliberately absent: they are legitimate inputs.
|
|
||||||
**/*.[pP][eE][mM]
|
|
||||||
**/*.[kK][eE][yY]
|
|
||||||
**/*.[pP]12
|
|
||||||
**/*.[pP][fF][xX]
|
|
||||||
**/[iI][dD]_[rR][sS][aA]
|
|
||||||
**/[iI][dD]_[dD][sS][aA]
|
|
||||||
**/[iI][dD]_[eE][cC][dD][sS][aA]
|
|
||||||
**/[iI][dD]_[eE][cC][dD][sS][aA]_[sS][kK]
|
|
||||||
**/[iI][dD]_[eE][dD]25519
|
|
||||||
**/[iI][dD]_[eE][dD]25519_[sS][kK]
|
|
||||||
|
|
||||||
# Dependencies: restored inside the image, never copied in.
|
|
||||||
**/node_modules
|
|
||||||
|
|
||||||
# OS metadata.
|
|
||||||
**/.DS_Store
|
|
||||||
**/Thumbs.db
|
|
||||||
|
|
||||||
# Editor state: never a build input, and it churns COPY.
|
|
||||||
**/*.swp
|
|
||||||
**/*.swo
|
|
||||||
**/*~
|
|
||||||
**/*.bak
|
|
||||||
**/.idea
|
|
||||||
**/.vscode
|
|
||||||
**/*.sublime-*
|
|
||||||
|
|
||||||
# This repository's own host-built artifacts: the binary `make build`
|
|
||||||
# writes, and the Alpine.js file `make assets` extracts from 3p/ (the
|
|
||||||
# build extracts its own).
|
|
||||||
/bin
|
|
||||||
/static/js/alpine.min.js
|
|
||||||
|
|
||||||
# SQLite databases, which hold the session key and webhook payloads, at
|
|
||||||
# any depth.
|
|
||||||
**/*.db
|
|
||||||
**/*.sqlite
|
|
||||||
**/*.sqlite3
|
|
||||||
|
|||||||
@@ -10,6 +10,3 @@ insert_final_newline = true
|
|||||||
|
|
||||||
[Makefile]
|
[Makefile]
|
||||||
indent_style = tab
|
indent_style = tab
|
||||||
|
|
||||||
[*.go]
|
|
||||||
indent_style = tab
|
|
||||||
|
|||||||
@@ -1,9 +1,37 @@
|
|||||||
name: check
|
name: check
|
||||||
on: [push]
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- '**'
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
check:
|
check:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
# actions/checkout v4.2.2, 2026-02-22
|
- name: Checkout
|
||||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
|
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 2024-10-23
|
||||||
- run: script/cibuild
|
with:
|
||||||
|
# The superseded-status step needs history to walk ancestors (it
|
||||||
|
# aborts on a shallow clone).
|
||||||
|
fetch-depth: 0
|
||||||
|
|
||||||
|
- name: Mark superseded run statuses
|
||||||
|
# Gitea cancels the in-flight run when another commit is pushed to the
|
||||||
|
# same branch and records the cancellation as `failure`, so a commit
|
||||||
|
# that was never tested reads as a test result. The script rewrites
|
||||||
|
# those statuses to say what happened. See its header for why the
|
||||||
|
# state stays `failure` and not `skipped`.
|
||||||
|
env:
|
||||||
|
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||||
|
run: script/ci-mark-superseded
|
||||||
|
|
||||||
|
- name: Fingerprint the build context
|
||||||
|
# Writes the hash of the commit being checked into the context, which
|
||||||
|
# invalidates the `COPY . .` layer of every check stage: a commit
|
||||||
|
# that was never linted, format-checked, stylesheet-checked, tested
|
||||||
|
# and built cannot report success from cache.
|
||||||
|
run: git rev-parse HEAD > .ci-fingerprint
|
||||||
|
|
||||||
|
- name: Build Docker image (runs make fmt-check, golangci-lint, the stylesheet check, make test, make build)
|
||||||
|
run: script/cibuild
|
||||||
|
|||||||
+18
-50
@@ -1,53 +1,3 @@
|
|||||||
# OS
|
|
||||||
.DS_Store
|
|
||||||
Thumbs.db
|
|
||||||
|
|
||||||
# Editors
|
|
||||||
*.swp
|
|
||||||
*.swo
|
|
||||||
*~
|
|
||||||
*.bak
|
|
||||||
.idea/
|
|
||||||
.vscode/
|
|
||||||
*.sublime-*
|
|
||||||
|
|
||||||
# Agent scratch (worktrees of this repo, created and destroyed by
|
|
||||||
# in-flight tooling). Unanchored: .gitignore patterns already match at
|
|
||||||
# every depth, so no prefix is wanted here. This is not a .dockerignore
|
|
||||||
# entry and must not be given a `**/` prefix on the way into one.
|
|
||||||
.claude/
|
|
||||||
|
|
||||||
# Node
|
|
||||||
node_modules/
|
|
||||||
|
|
||||||
# Secrets. Unanchored like every entry above, so each matches at every
|
|
||||||
# depth. Matching is case-sensitive on Linux, so names use character
|
|
||||||
# ranges rather than a lowercase form that misses `Server.Key`.
|
|
||||||
|
|
||||||
# Environment files. `*.env` covers bare `.env` and the `prod.env`
|
|
||||||
# convention. Only the templates `example.env` and `sample.env` are
|
|
||||||
# re-included below. A repository that commits any other template adds
|
|
||||||
# its own negation after these lines, for example `!.env.example`.
|
|
||||||
*.[eE][nN][vV]
|
|
||||||
.[eE][nN][vV].*
|
|
||||||
.[eE][nN][vV][rR][cC]
|
|
||||||
!example.env
|
|
||||||
!sample.env
|
|
||||||
|
|
||||||
# Private keys and the bundles carrying them.
|
|
||||||
*.[pP][eE][mM]
|
|
||||||
*.[kK][eE][yY]
|
|
||||||
*.[pP]12
|
|
||||||
*.[pP][fF][xX]
|
|
||||||
[iI][dD]_[rR][sS][aA]
|
|
||||||
[iI][dD]_[dD][sS][aA]
|
|
||||||
[iI][dD]_[eE][cC][dD][sS][aA]
|
|
||||||
[iI][dD]_[eE][cC][dD][sS][aA]_[sS][kK]
|
|
||||||
[iI][dD]_[eE][dD]25519
|
|
||||||
[iI][dD]_[eE][dD]25519_[sS][kK]
|
|
||||||
|
|
||||||
# This repository's own entries, after the shared content above.
|
|
||||||
|
|
||||||
# Binaries
|
# Binaries
|
||||||
*.exe
|
*.exe
|
||||||
*.dll
|
*.dll
|
||||||
@@ -65,6 +15,21 @@ bin/
|
|||||||
# Go vendor directory
|
# Go vendor directory
|
||||||
vendor/
|
vendor/
|
||||||
|
|
||||||
|
# IDE specific files
|
||||||
|
.idea/
|
||||||
|
*.swp
|
||||||
|
*.swo
|
||||||
|
*~
|
||||||
|
.vscode/
|
||||||
|
|
||||||
|
# OS specific files
|
||||||
|
.DS_Store
|
||||||
|
Thumbs.db
|
||||||
|
|
||||||
|
# Environment and config files
|
||||||
|
.env
|
||||||
|
.env.local
|
||||||
|
|
||||||
# Data directory (SQLite databases)
|
# Data directory (SQLite databases)
|
||||||
data/
|
data/
|
||||||
*.db
|
*.db
|
||||||
@@ -78,6 +43,9 @@ data/
|
|||||||
tmp/
|
tmp/
|
||||||
temp/
|
temp/
|
||||||
|
|
||||||
|
# CI cache barrier, written into the build context by the check workflow
|
||||||
|
.ci-fingerprint
|
||||||
|
|
||||||
# Alpine.js, extracted by `make assets` from its tarball in 3p/, which is
|
# Alpine.js, extracted by `make assets` from its tarball in 3p/, which is
|
||||||
# what is committed.
|
# what is committed.
|
||||||
/static/js/alpine.min.js
|
/static/js/alpine.min.js
|
||||||
|
|||||||
+2
-73
@@ -10,21 +10,14 @@ run:
|
|||||||
|
|
||||||
linters:
|
linters:
|
||||||
default: all
|
default: all
|
||||||
enable:
|
|
||||||
# Successor to the deprecated gomodguard. Named explicitly, rather than
|
|
||||||
# left to `default: all`, because it carries the module policy below.
|
|
||||||
- gomodguard_v2
|
|
||||||
disable:
|
disable:
|
||||||
# Genuinely incompatible with project patterns
|
# Genuinely incompatible with project patterns
|
||||||
- exhaustruct # Requires all struct fields
|
- exhaustruct # Requires all struct fields
|
||||||
- exhaustruct_v5 # Requires all struct fields (successor to exhaustruct)
|
- depguard # Dependency allow/block lists
|
||||||
- godot # Requires comments to end with periods
|
- godot # Requires comments to end with periods
|
||||||
|
- wsl # Deprecated, replaced by wsl_v5
|
||||||
- wrapcheck # Too verbose for internal packages
|
- wrapcheck # Too verbose for internal packages
|
||||||
- varnamelen # Short names like db, id are idiomatic Go
|
- varnamelen # Short names like db, id are idiomatic Go
|
||||||
# Deprecated: the warning is attached to the old name, so it is
|
|
||||||
# silenced by disabling that name, not by enabling the successor.
|
|
||||||
- wsl # Deprecated, replaced by wsl_v5
|
|
||||||
- gomodguard # Deprecated, replaced by gomodguard_v2
|
|
||||||
settings:
|
settings:
|
||||||
lll:
|
lll:
|
||||||
line-length: 88
|
line-length: 88
|
||||||
@@ -35,70 +28,6 @@ linters:
|
|||||||
max-complexity: 15
|
max-complexity: 15
|
||||||
dupl:
|
dupl:
|
||||||
threshold: 100
|
threshold: 100
|
||||||
depguard:
|
|
||||||
# Test-support code must not be compiled into the shipped binary. A
|
|
||||||
# test-support package exists to hand a test privileges the program
|
|
||||||
# itself must never have, so a file that is not a test must not import
|
|
||||||
# one. Test files, and the files inside a package whose directory name
|
|
||||||
# ends in `test`, are where that code belongs, and are exempt.
|
|
||||||
#
|
|
||||||
# The deny list below is the one part of this file a repository is
|
|
||||||
# expected to extend, and the only part it may. depguard matches an
|
|
||||||
# import path against a list of prefixes, so it cannot be told "any path
|
|
||||||
# whose last segment ends in test"; a repository's own test-support
|
|
||||||
# packages have to be named here one at a time, by full import path,
|
|
||||||
# under a module path that differs from repository to repository. Add
|
|
||||||
# them; change nothing else.
|
|
||||||
rules:
|
|
||||||
test-support:
|
|
||||||
list-mode: lax
|
|
||||||
files:
|
|
||||||
- "$all"
|
|
||||||
- "!$test"
|
|
||||||
- "!**/*test/**"
|
|
||||||
deny:
|
|
||||||
- pkg: net/http/httptest
|
|
||||||
desc: >-
|
|
||||||
Test-support code belongs in test files and in packages whose
|
|
||||||
directory name ends in test, not in the shipped binary.
|
|
||||||
- pkg: sneak.berlin/go/webhooker/internal/config/configtest
|
|
||||||
desc: test support; a file that is not a test must not import it
|
|
||||||
- pkg: sneak.berlin/go/webhooker/internal/database/databasetest
|
|
||||||
desc: test support; a file that is not a test must not import it
|
|
||||||
- pkg: sneak.berlin/go/webhooker/internal/middleware/middlewaretest
|
|
||||||
desc: test support; a file that is not a test must not import it
|
|
||||||
# Only decisions already recorded in the Go package defaults are
|
|
||||||
# listed here. Every entry matches the module path exactly.
|
|
||||||
gomodguard_v2:
|
|
||||||
blocked:
|
|
||||||
- module: github.com/rs/zerolog
|
|
||||||
recommendations:
|
|
||||||
- log/slog
|
|
||||||
reason: "Structured logging is stdlib log/slog."
|
|
||||||
# One entry per pre-fork module path, because the later releases
|
|
||||||
# are separate paths. A prefix match would be shorter but would
|
|
||||||
# also reach github.com/go-redis/redismock, the test double for
|
|
||||||
# the successor these entries recommend.
|
|
||||||
- module: github.com/go-redis/redis
|
|
||||||
recommendations:
|
|
||||||
- github.com/redis/go-redis/v9
|
|
||||||
reason: "Pre-fork module; use the maintained go-redis v9."
|
|
||||||
- module: github.com/go-redis/redis/v7
|
|
||||||
recommendations:
|
|
||||||
- github.com/redis/go-redis/v9
|
|
||||||
reason: "Pre-fork module; use the maintained go-redis v9."
|
|
||||||
- module: github.com/go-redis/redis/v8
|
|
||||||
recommendations:
|
|
||||||
- github.com/redis/go-redis/v9
|
|
||||||
reason: "Pre-fork module; use the maintained go-redis v9."
|
|
||||||
- module: github.com/sergi/go-diff
|
|
||||||
recommendations:
|
|
||||||
- github.com/aymanbagabas/go-udiff
|
|
||||||
reason: "No unified diff output; use go-udiff."
|
|
||||||
- module: github.com/hexops/gotextdiff
|
|
||||||
recommendations:
|
|
||||||
- github.com/aymanbagabas/go-udiff
|
|
||||||
reason: "Unmaintained fork; use go-udiff."
|
|
||||||
|
|
||||||
issues:
|
issues:
|
||||||
max-issues-per-linter: 0
|
max-issues-per-linter: 0
|
||||||
|
|||||||
@@ -1,2 +0,0 @@
|
|||||||
node_modules/
|
|
||||||
yarn.lock
|
|
||||||
@@ -1,4 +0,0 @@
|
|||||||
{
|
|
||||||
"tabWidth": 4,
|
|
||||||
"proseWrap": "always"
|
|
||||||
}
|
|
||||||
@@ -1,3 +0,0 @@
|
|||||||
# Install into node_modules/: the Dockerfile's lint and Markdown stages run
|
|
||||||
# ESLint and prettier from node_modules/.bin.
|
|
||||||
nodeLinker: node-modules
|
|
||||||
+61
-164
@@ -1,3 +1,34 @@
|
|||||||
|
# Lint stage
|
||||||
|
# golangci/golangci-lint:v2.12.2 (Debian-based), 2026-08-07
|
||||||
|
# Using Debian-based image because mattn/go-sqlite3 (CGO) does not
|
||||||
|
# compile on Alpine musl (off64_t is a glibc type).
|
||||||
|
FROM golangci/golangci-lint:v2.12.2@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS lint
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends make && rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
WORKDIR /src
|
||||||
|
|
||||||
|
# Copy go mod files first for better layer caching
|
||||||
|
COPY go.mod go.sum ./
|
||||||
|
RUN go mod download
|
||||||
|
|
||||||
|
# Copy source code. In CI the context also carries .ci-fingerprint, which
|
||||||
|
# holds the hash of the commit being checked (see
|
||||||
|
# .gitea/workflows/check.yml). That invalidates this layer, so the checks
|
||||||
|
# below cannot report success by replaying a cached pass. Do not add it to
|
||||||
|
# .dockerignore.
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
# Run formatting check and linter. golangci-lint is invoked directly rather
|
||||||
|
# than through `make lint`: this stage is already the pinned linter image, and
|
||||||
|
# script/lint is a wrapper that builds Dockerfile.lint, so calling it here
|
||||||
|
# would need a docker daemon inside the build. Keep these steps in step with
|
||||||
|
# Dockerfile.lint, including --network=none (see its header for why).
|
||||||
|
RUN make fmt-check
|
||||||
|
RUN script/assets
|
||||||
|
RUN --network=none golangci-lint config verify --config .golangci.yml
|
||||||
|
RUN --network=none golangci-lint run --config .golangci.yml --build-tags browser ./...
|
||||||
|
|
||||||
# Stylesheet stages. static/css/tailwind.css is generated, by this pinned
|
# Stylesheet stages. static/css/tailwind.css is generated, by this pinned
|
||||||
# tailwindcss, from static/css/input.css and the files its @source lines
|
# tailwindcss, from static/css/input.css and the files its @source lines
|
||||||
# name. `make css` (script/css) writes it out from the css-output stage.
|
# name. `make css` (script/css) writes it out from the css-output stage.
|
||||||
@@ -34,151 +65,19 @@ RUN sed 's/}/}\n/g' static/css/tailwind.css > /tmp/committed.css \
|
|||||||
exit 1; \
|
exit 1; \
|
||||||
}
|
}
|
||||||
|
|
||||||
# JavaScript lint stages: ESLint, at the version package.json and yarn.lock
|
|
||||||
# pin, checks static/js/ against eslint.config.mjs. js-deps installs it, and
|
|
||||||
# prettier for the Markdown stages below. The lint phase below runs js-lint.
|
|
||||||
#
|
|
||||||
# The image's own corepack runs the yarn that package.json's packageManager
|
|
||||||
# field names, yarn 4.18.1 (released 2026-09-24), and checks it against the
|
|
||||||
# hash there. The image also ships yarn 1, which `corepack enable yarn`
|
|
||||||
# replaces.
|
|
||||||
# node:24.21.0-alpine (LTS), 2026-09-18
|
|
||||||
FROM node:24.21.0-alpine@sha256:ebfe2f90462722a7a4de65e91990e97fe0d401c70e0e762c5b53302f905ec1c1 AS js-deps
|
|
||||||
WORKDIR /src
|
|
||||||
COPY package.json yarn.lock .yarnrc.yml ./
|
|
||||||
RUN corepack enable yarn && yarn install --immutable --mode=skip-build
|
|
||||||
|
|
||||||
FROM js-deps AS js-lint
|
|
||||||
COPY . .
|
|
||||||
RUN --network=none node_modules/.bin/eslint static/js
|
|
||||||
|
|
||||||
# Markdown stages: prettier, at the version package.json and yarn.lock pin,
|
|
||||||
# formats every Markdown file in the tree with the settings in .prettierrc.
|
|
||||||
# `make fmt` (script/fmt) writes the formatted files out from markdown-output.
|
|
||||||
# markdown-check fails on any file prettier would change; `make fmt-check`
|
|
||||||
# runs it, and so does the build stage below.
|
|
||||||
FROM js-deps AS markdown
|
|
||||||
COPY . .
|
|
||||||
RUN --network=none node_modules/.bin/prettier --write '**/*.md' \
|
|
||||||
&& mkdir /out \
|
|
||||||
&& find . -name '*.md' ! -path './node_modules/*' -exec cp -p --parents {} /out \;
|
|
||||||
|
|
||||||
FROM scratch AS markdown-output
|
|
||||||
COPY --from=markdown /out /
|
|
||||||
|
|
||||||
FROM js-deps AS markdown-check
|
|
||||||
COPY . .
|
|
||||||
RUN --network=none node_modules/.bin/prettier --check '**/*.md'
|
|
||||||
|
|
||||||
# Lint phase: the Go formatting check and golangci-lint over the Go code,
|
|
||||||
# and ESLint over static/js/ through the copy from js-lint at the end.
|
|
||||||
# `make lint` (script/lint) builds this stage alone; the build stage below
|
|
||||||
# depends on it.
|
|
||||||
#
|
|
||||||
# golangci/golangci-lint:v2.14.0 (Debian-based), 2026-09-24
|
|
||||||
# Using Debian-based image because mattn/go-sqlite3 (CGO) does not
|
|
||||||
# compile on Alpine musl (off64_t is a glibc type).
|
|
||||||
FROM golangci/golangci-lint:v2.14.0@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint
|
|
||||||
|
|
||||||
WORKDIR /src
|
|
||||||
|
|
||||||
# Copy go mod files first for better layer caching
|
|
||||||
COPY go.mod go.sum ./
|
|
||||||
RUN go mod download
|
|
||||||
|
|
||||||
COPY . .
|
|
||||||
|
|
||||||
# gofmt and golangci-lint are invoked directly rather than through `make
|
|
||||||
# fmt-check` and `make lint`, which are themselves docker builds and would
|
|
||||||
# need a docker daemon inside this one. The Markdown half of `make
|
|
||||||
# fmt-check` is the markdown-check stage above.
|
|
||||||
RUN if [ -n "$(gofmt -s -l .)" ]; then echo "gofmt needed on:"; gofmt -s -l .; exit 1; fi
|
|
||||||
|
|
||||||
# static/static.go embeds the Alpine.js file this extracts from 3p/; without
|
|
||||||
# it the static package does not compile and cannot be linted.
|
|
||||||
RUN script/assets
|
|
||||||
|
|
||||||
# The golangci-lint steps run with --network=none. `golangci-lint config
|
|
||||||
# verify` is documented as fetching its JSON schema over HTTPS; this pinned
|
|
||||||
# image resolves the schema without any network, and --network=none enforces
|
|
||||||
# that. It also proves no linter reaches out at analysis time.
|
|
||||||
#
|
|
||||||
# `run` silently ignores config keys it does not recognize, so a typo would
|
|
||||||
# disable a setting without a word. `config verify` is what catches that.
|
|
||||||
RUN --network=none golangci-lint config verify --config .golangci.yml
|
|
||||||
# --build-tags browser also lints the browser test, which is built only with
|
|
||||||
# that tag (make test-browser).
|
|
||||||
RUN --network=none golangci-lint run --config .golangci.yml --build-tags browser ./...
|
|
||||||
|
|
||||||
# Nothing is wanted from js-lint; the copy is what makes this phase run it.
|
|
||||||
COPY --from=js-lint /src/yarn.lock /dev/null
|
|
||||||
|
|
||||||
# Test phase. -race needs cgo and so a C compiler, which the Debian Go image
|
|
||||||
# ships and the alpine one does not. `make test` (script/test) builds this
|
|
||||||
# stage alone; the build stage below depends on it.
|
|
||||||
#
|
|
||||||
# golang:1.26.1-bookworm (Debian-based), 2026-03-17
|
|
||||||
FROM golang:1.26.1-bookworm@sha256:4465644228bc2857a954b092167e12aa59c006a3492282a6c820bf4755fd64a4 AS test
|
|
||||||
|
|
||||||
WORKDIR /src
|
|
||||||
|
|
||||||
COPY go.mod go.sum ./
|
|
||||||
RUN go mod download
|
|
||||||
|
|
||||||
COPY . .
|
|
||||||
|
|
||||||
# static/static.go embeds the Alpine.js file this extracts from 3p/.
|
|
||||||
RUN script/assets
|
|
||||||
|
|
||||||
# -timeout applies to each package on its own, so 90s has only to clear the
|
|
||||||
# slowest one. -p 4 -parallel 8 keep the run under 2 GB of memory: at most
|
|
||||||
# four test binaries build or run at once, each with at most eight parallel
|
|
||||||
# tests. Under -race every test binary and every link costs a few hundred MB,
|
|
||||||
# so the defaults (one per core) add up to several GB on a many-core host.
|
|
||||||
#
|
|
||||||
# The first run has no -v: go test then prints one result line per package,
|
|
||||||
# with its coverage, and for a package that fails, everything its tests
|
|
||||||
# wrote. Verbose output from the whole suite passes the 2 MiB at which the
|
|
||||||
# Docker build cuts off a step's log, so on a failure only the tests that
|
|
||||||
# failed run again, with -v. go test reports a failed test as a line starting
|
|
||||||
# "--- FAIL: TestName" (a failed subtest's line is indented, and reruns with
|
|
||||||
# its parent) and a failed package as "FAIL<tab>package/path<tab>...". A
|
|
||||||
# failure that names no test, such as a build error or a timeout, is already
|
|
||||||
# shown in full, so there is nothing to rerun. The step fails after the rerun
|
|
||||||
# whatever its result: the first run already showed the suite is broken.
|
|
||||||
#
|
|
||||||
# TMPDIR, where the tests keep their SQLite databases, is a tmpfs: SQLite
|
|
||||||
# waits for the disk at every commit, and on a busy host that waiting was
|
|
||||||
# about 40% of the slowest package's run time. GOTMPDIR keeps go's own
|
|
||||||
# build files, the test binaries among them, on disk.
|
|
||||||
#
|
|
||||||
# bash with pipefail, so that the first run's status is go test's, not tee's.
|
|
||||||
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
|
|
||||||
RUN --mount=type=tmpfs,target=/tmp/tests,size=512m \
|
|
||||||
export TMPDIR=/tmp/tests GOTMPDIR=/tmp; \
|
|
||||||
go test -race -cover -p 4 -parallel 8 -timeout 90s ./... 2>&1 | tee /tmp/go-test.log && exit 0; \
|
|
||||||
tests="$(awk '/^--- FAIL: / { print $3 }' /tmp/go-test.log | paste -s -d '|' -)"; \
|
|
||||||
packages="$(awk '/^FAIL\t/ { print $2 }' /tmp/go-test.log)"; \
|
|
||||||
if [ -n "$tests" ]; then \
|
|
||||||
echo "--- Rerunning the failed tests with -v for details ---"; \
|
|
||||||
go test -race -v -p 4 -parallel 8 -timeout 90s -run "^($tests)\$" $packages; \
|
|
||||||
fi; \
|
|
||||||
exit 1
|
|
||||||
|
|
||||||
# Build stage
|
# Build stage
|
||||||
# golang:1.26.1-bookworm (Debian-based), 2026-03-17
|
# golang:1.26.1-bookworm (Debian-based), 2026-03-17
|
||||||
# Using Debian-based image because gorm.io/driver/sqlite pulls in
|
# Using Debian-based image because gorm.io/driver/sqlite pulls in
|
||||||
# mattn/go-sqlite3 (CGO), which does not compile on Alpine musl. The image
|
# mattn/go-sqlite3 (CGO), which does not compile on Alpine musl.
|
||||||
# ships git and make, which the version step below uses.
|
|
||||||
FROM golang:1.26.1-bookworm@sha256:4465644228bc2857a954b092167e12aa59c006a3492282a6c820bf4755fd64a4 AS builder
|
FROM golang:1.26.1-bookworm@sha256:4465644228bc2857a954b092167e12aa59c006a3492282a6c820bf4755fd64a4 AS builder
|
||||||
|
|
||||||
# Nothing is wanted from the lint and test phases or from the stylesheet and
|
# Depend on the lint and stylesheet check stages passing
|
||||||
# Markdown checks; the copies are what make BuildKit build them first, so
|
|
||||||
# this stage cannot run unless they all passed.
|
|
||||||
COPY --from=lint /src/go.sum /dev/null
|
COPY --from=lint /src/go.sum /dev/null
|
||||||
COPY --from=test /src/go.sum /dev/null
|
|
||||||
COPY --from=css-check /out/tailwind.css /dev/null
|
COPY --from=css-check /out/tailwind.css /dev/null
|
||||||
COPY --from=markdown-check /src/yarn.lock /dev/null
|
|
||||||
|
# jq is a runtime dependency of script/ci-mark-superseded, which the test
|
||||||
|
# suite executes. git is what script/version derives the version with.
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends make curl ca-certificates jq git && rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
# A build context sent as a tar archive keeps its files' owners, and git
|
# A build context sent as a tar archive keeps its files' owners, and git
|
||||||
# refuses to read a checkout owned by another user. Trust this one
|
# refuses to read a checkout owned by another user. Trust this one
|
||||||
@@ -191,45 +90,43 @@ WORKDIR /build
|
|||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
RUN go mod download
|
RUN go mod download
|
||||||
|
|
||||||
|
# Copy source code, including the .ci-fingerprint cache barrier described in
|
||||||
|
# the lint stage above.
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
|
# Run tests and build. Both first run script/assets, which extracts Alpine.js
|
||||||
|
# from its tarball in 3p/.
|
||||||
|
RUN make test
|
||||||
|
|
||||||
# Version stamped into the binary: the VERSION build arg when one is
|
# Version stamped into the binary: the VERSION build arg when one is
|
||||||
# given, otherwise what script/version derives from the .git the build
|
# given, otherwise what script/version derives from the .git the build
|
||||||
# context carries, so any `docker build .` of a clone stamps its commit.
|
# context carries, so any `docker build .` of a clone stamps its commit.
|
||||||
# With neither, as from a source tarball, it is "unknown".
|
# With neither, as from a source tarball, it is "unknown".
|
||||||
|
#
|
||||||
|
# Declared here, below the test step, so a changed version does not
|
||||||
|
# invalidate its cached layer.
|
||||||
ARG VERSION
|
ARG VERSION
|
||||||
|
|
||||||
# A context that carries .git must not stamp an empty version, "dev" or
|
# A context that carries .git must not stamp "unknown": that means git is
|
||||||
# "unknown": that means git is missing here or could not read the
|
# missing here or could not read the checkout, and the image could not be
|
||||||
# checkout, and the image could not be traced back to its commit.
|
# traced back to its commit.
|
||||||
RUN version="$(make version VERSION="$VERSION")"; \
|
RUN if [ -d .git ] && [ "$(make version VERSION="$VERSION")" = unknown ]; then \
|
||||||
if [ -e .git ]; then \
|
echo "version is unknown although the build context carries .git" >&2; \
|
||||||
case "$version" in ""|dev|unknown) \
|
exit 1; \
|
||||||
echo "version is '$version' although .git is present" >&2; \
|
|
||||||
exit 1 ;; \
|
|
||||||
esac; \
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Builds through the Makefile's build target, which runs script/assets
|
|
||||||
# (Alpine.js, extracted from its tarball in 3p/) first.
|
|
||||||
RUN make build VERSION="$VERSION"
|
RUN make build VERSION="$VERSION"
|
||||||
|
|
||||||
# Rebuild with static linking for the Alpine runtime. The cgo binary the
|
# Rebuild with static linking for Alpine runtime.
|
||||||
# step above builds is dynamically linked against glibc, which Alpine
|
# make build already verified compilation.
|
||||||
# (musl) does not have.
|
# The CGO binary from `make build` is dynamically linked against glibc,
|
||||||
|
# which doesn't exist on Alpine (musl). Rebuild with static linking so
|
||||||
|
# the binary runs on Alpine without glibc.
|
||||||
#
|
#
|
||||||
# A statically linked glibc still loads shared libraries at run time for
|
# The static flags go in through GO_LDFLAGS rather than a -ldflags of
|
||||||
# getaddrinfo and dlopen, and Alpine has none to load, so the build tags
|
# their own: the build target composes them with the -X that stamps the
|
||||||
# keep both out of the binary: netgo and osusergo select Go's own DNS
|
# version, so this relink cannot silently drop the stamp.
|
||||||
# resolver and user lookup instead of glibc's, and
|
RUN CGO_ENABLED=1 make build VERSION="$VERSION" GO_LDFLAGS='-extldflags "-static"'
|
||||||
# sqlite_omit_load_extension leaves out SQLite extension loading, which
|
|
||||||
# webhooker never uses.
|
|
||||||
#
|
|
||||||
# The static flags and tags go in through GO_LDFLAGS and GO_TAGS rather
|
|
||||||
# than a -ldflags of their own: the build target composes them with the
|
|
||||||
# -X that stamps the version, so this relink cannot silently drop the
|
|
||||||
# stamp.
|
|
||||||
RUN CGO_ENABLED=1 make build VERSION="$VERSION" GO_TAGS=netgo,osusergo,sqlite_omit_load_extension GO_LDFLAGS='-extldflags "-static"'
|
|
||||||
|
|
||||||
# Runtime stage
|
# Runtime stage
|
||||||
# alpine:3.21, 2026-03-17
|
# alpine:3.21, 2026-03-17
|
||||||
|
|||||||
+1
-1
@@ -16,7 +16,7 @@ COPY . .
|
|||||||
|
|
||||||
# The test binary embeds the templates and static files, so the browser
|
# The test binary embeds the templates and static files, so the browser
|
||||||
# stage needs nothing else. -p 4 keeps the compile's memory down, as in
|
# stage needs nothing else. -p 4 keeps the compile's memory down, as in
|
||||||
# the test phase of Dockerfile.
|
# script/test.
|
||||||
RUN make assets && go test -c -p 4 -tags browser -o /browser.test ./internal/server
|
RUN make assets && go test -c -p 4 -tags browser -o /browser.test ./internal/server
|
||||||
|
|
||||||
# chromedp/headless-shell:151.0.7922.109 (Debian trixie), 2026-08-11. The
|
# chromedp/headless-shell:151.0.7922.109 (Debian trixie), 2026-08-11. The
|
||||||
|
|||||||
@@ -0,0 +1,43 @@
|
|||||||
|
# Lint-only image, built by script/lint. golangci-lint is never installed on
|
||||||
|
# the host: the repo is COPYed into the pinned image and linted as a build
|
||||||
|
# step, so a successful build IS a clean lint. This works even when the docker
|
||||||
|
# daemon is remote and bind mounts are impossible.
|
||||||
|
#
|
||||||
|
# script/lint passes --no-cache-filter=lint. Without it an unchanged tree
|
||||||
|
# replays the lint stage from cache and the build succeeds in under a second
|
||||||
|
# having run no linter at all. Do not drop that flag.
|
||||||
|
#
|
||||||
|
# The lint steps run with --network=none. `golangci-lint config verify` is
|
||||||
|
# documented as fetching its JSON schema over HTTPS, which would make linting
|
||||||
|
# depend on an unpinned remote artifact; this pinned image resolves the schema
|
||||||
|
# without any network, and --network=none enforces that rather than trusting
|
||||||
|
# it. It also proves no linter reaches out at analysis time. If a future image
|
||||||
|
# bump makes either step need the network, this build fails loudly instead of
|
||||||
|
# quietly acquiring an unpinned dependency.
|
||||||
|
|
||||||
|
# golangci/golangci-lint:v2.12.2 (Debian-based), 2026-08-07
|
||||||
|
# Using Debian-based image because mattn/go-sqlite3 (CGO) does not
|
||||||
|
# compile on Alpine musl (off64_t is a glibc type).
|
||||||
|
FROM golangci/golangci-lint:v2.12.2@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS deps
|
||||||
|
|
||||||
|
WORKDIR /src
|
||||||
|
|
||||||
|
# Copy go mod files first for better layer caching. This stage is cacheable;
|
||||||
|
# only the lint stage below is forced to re-execute.
|
||||||
|
COPY go.mod go.sum ./
|
||||||
|
RUN go mod download
|
||||||
|
|
||||||
|
FROM deps AS lint
|
||||||
|
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
# static/static.go embeds the Alpine.js file this extracts from 3p/; without
|
||||||
|
# it the static package does not compile and cannot be linted.
|
||||||
|
RUN script/assets
|
||||||
|
|
||||||
|
# `run` silently ignores config keys it does not recognize, so a typo would
|
||||||
|
# disable a setting without a word. `config verify` is what catches that.
|
||||||
|
RUN --network=none golangci-lint config verify --config .golangci.yml
|
||||||
|
# --build-tags browser also lints the browser test, which is built only with
|
||||||
|
# that tag (make test-browser).
|
||||||
|
RUN --network=none golangci-lint run --config .golangci.yml --build-tags browser ./...
|
||||||
@@ -16,16 +16,10 @@ VERSION ?= $(shell script/version)
|
|||||||
# exists to correct.
|
# exists to correct.
|
||||||
override VERSION := $(or $(strip $(VERSION)),$(shell script/version))
|
override VERSION := $(or $(strip $(VERSION)),$(shell script/version))
|
||||||
|
|
||||||
# Extra linker flags and build tags for the build target. The static
|
# Extra linker flags for the build target. The static relink in the
|
||||||
# relink in the Dockerfile passes -extldflags and its build tags through
|
# Dockerfile adds -extldflags here rather than passing its own -ldflags,
|
||||||
# these rather than a -ldflags of its own, so composing flags cannot drop
|
# so composing flags cannot drop the version stamp.
|
||||||
# the version stamp.
|
|
||||||
#
|
|
||||||
# The build target itself always passes -trimpath and -s -w, as the Go
|
|
||||||
# Dockerfile in REPO_POLICIES.md does: no build paths, symbol table or
|
|
||||||
# debug information in the binary.
|
|
||||||
GO_LDFLAGS ?=
|
GO_LDFLAGS ?=
|
||||||
GO_TAGS ?=
|
|
||||||
|
|
||||||
bootstrap:
|
bootstrap:
|
||||||
@script/bootstrap
|
@script/bootstrap
|
||||||
@@ -55,7 +49,7 @@ check:
|
|||||||
@script/check
|
@script/check
|
||||||
|
|
||||||
build: assets
|
build: assets
|
||||||
go build -trimpath -tags '$(GO_TAGS)' -ldflags '$(strip -s -w -X main.version=$(VERSION) $(GO_LDFLAGS))' -o bin/webhooker ./cmd/webhooker
|
go build -ldflags '$(strip -X main.version=$(VERSION) $(GO_LDFLAGS))' -o bin/webhooker ./cmd/webhooker
|
||||||
|
|
||||||
run: build
|
run: build
|
||||||
./bin/webhooker
|
./bin/webhooker
|
||||||
|
|||||||
+86
-349
@@ -1,6 +1,6 @@
|
|||||||
---
|
---
|
||||||
title: Repository Policies
|
title: Repository Policies
|
||||||
last_modified: 2026-10-04
|
last_modified: 2026-08-07
|
||||||
---
|
---
|
||||||
|
|
||||||
This document covers repository structure, tooling, and workflow standards. Code
|
This document covers repository structure, tooling, and workflow standards. Code
|
||||||
@@ -60,28 +60,17 @@ style conventions are in separate documents:
|
|||||||
prerequisite since nvm requires bash. yarn is then pinned via
|
prerequisite since nvm requires bash. yarn is then pinned via
|
||||||
`corepack prepare yarn@<version> --activate`. Never install "latest" or "lts";
|
`corepack prepare yarn@<version> --activate`. Never install "latest" or "lts";
|
||||||
always exact versions. `script/cibuild` runs the CI build: it changes to the
|
always exact versions. `script/cibuild` runs the CI build: it changes to the
|
||||||
repo root, runs `script/bootstrap`, runs `script/check`, and builds the image
|
repo root and runs `docker build .`; the Gitea workflow calls it. Four further
|
||||||
with the version; the Gitea workflow calls it. **`script/cibuild` runs
|
scripts are our own extensions to the standard: `script/check` runs
|
||||||
`script/bootstrap` first**, because the workflow checks out the repo and runs
|
`script/test`, `script/lint`, and `script/fmt-check`; `script/precommit` is
|
||||||
nothing else, while `script/fmt-check` runs the formatter on the host: on a
|
what the git pre-commit hook runs, and it calls `script/check`;
|
||||||
pristine checkout with nothing installed the run dies there, after the
|
`script/install-precommit` installs the git pre-commit hook (the `make hooks`
|
||||||
containerised gates have passed. **The bootstrap alone is not enough**:
|
target shims to it); and `script/projectname` (literally that filename) simply
|
||||||
`script/bootstrap` installs node and yarn under nvm and leaves neither on the
|
outputs the project's name. Scripts that need the name call
|
||||||
`PATH` of the shell that called it, so a bare `yarn` still exits 127. The host
|
`script/projectname` — e.g. `script/docker` assembles its image tag from it —
|
||||||
entrypoints that need yarn — `script/fmt` and `script/fmt-check` — therefore
|
so those scripts stay byte-identical across all repos. Repo-type-specific
|
||||||
source nvm for the pinned node version before invoking it, exactly as
|
pre-commit extras (e.g. `go mod tidy` verification in Go repos) belong in
|
||||||
`script/bootstrap`'s own install step does. A runner carrying nothing but
|
`script/precommit`, not in the hook itself. Model scripts are at
|
||||||
docker and git then gets through `script/check`. Four further scripts are our
|
|
||||||
own extensions to the standard: `script/check` runs `script/test`,
|
|
||||||
`script/lint` and `script/fmt-check`; `script/precommit` is what the git
|
|
||||||
pre-commit hook runs, and it calls `script/check`; `script/install-precommit`
|
|
||||||
installs the git pre-commit hook (the `make hooks` target shims to it); and
|
|
||||||
`script/projectname` (literally that filename) simply outputs the project's
|
|
||||||
name. Scripts that need the name call `script/projectname` — e.g.
|
|
||||||
`script/docker` assembles its image tag from it — so those scripts stay
|
|
||||||
byte-identical across all repos. Repo-type-specific pre-commit extras (e.g.
|
|
||||||
`go mod tidy` verification in Go repos) belong in `script/precommit`, not in
|
|
||||||
the hook itself. Model scripts are at
|
|
||||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/script/<name>`. The README
|
`https://git.eeqj.de/sneak/prompts/raw/branch/main/script/<name>`. The README
|
||||||
must document the provided scripts in an **Entrypoints** section (see the
|
must document the provided scripts in an **Entrypoints** section (see the
|
||||||
README requirements below).
|
README requirements below).
|
||||||
@@ -100,198 +89,87 @@ style conventions are in separate documents:
|
|||||||
contributor should be able to understand the entire development workflow by
|
contributor should be able to understand the entire development workflow by
|
||||||
reading the Makefile.
|
reading the Makefile.
|
||||||
|
|
||||||
- Every repo should have a `Dockerfile`, and it carries the repo's gates: a
|
- Every repo should have a `Dockerfile`. All Dockerfiles must run `make check`
|
||||||
`lint` phase and a `test` phase, with the final stage depending on both so the
|
as a build step so the build fails if the branch is not green. For non-server
|
||||||
image cannot be built unless they pass. For non-server repos the final stage
|
repos, the Dockerfile should bring up a development environment and run
|
||||||
brings up a development environment; for server repos it is the runtime image.
|
`make check`. For server repos, `make check` should run as an early build
|
||||||
The gate phases and the build stage start from their pinned base images and
|
stage before the final image is assembled. Dockerfiles install development
|
||||||
install what those images lack either inline, as the canonical Go `Dockerfile`
|
prerequisites by running `script/bootstrap` rather than duplicating installs
|
||||||
below does for `git`, or by running `script/bootstrap`, as the `prompts`
|
inline; COPY `script/` and the dependency manifests (`package.json` +
|
||||||
repo's own `Dockerfile` does for its yarn packages. The development
|
`yarn.lock`, `go.mod` + `go.sum`, etc.) before running it so the bootstrap
|
||||||
environment stage installs development prerequisites by running
|
layer stays cached until dependencies change.
|
||||||
`script/bootstrap` rather than duplicating its installs inline. A stage that
|
|
||||||
runs `script/bootstrap` COPYs `script/` and the dependency manifests
|
|
||||||
(`package.json` + `yarn.lock`, `go.mod` + `go.sum`, etc.) before running it.
|
|
||||||
|
|
||||||
- **Linting and testing run in Docker, as phases of the `Dockerfile`.** There is
|
- **Dockerfiles must use a separate lint stage for fail-fast feedback.** Go
|
||||||
no separate lint file. `script/lint` and `script/test` each build one phase
|
repos use a multistage build where linting runs in an independent stage based
|
||||||
and nothing else:
|
on the `golangci/golangci-lint` image (pinned by hash). This stage runs
|
||||||
|
`make fmt-check` and `make lint` before the full build begins. The build stage
|
||||||
|
then declares an explicit dependency on the lint stage via
|
||||||
|
`COPY --from=lint /src/go.sum /dev/null`, which forces BuildKit to complete
|
||||||
|
linting before proceeding to compilation and tests. This ensures lint failures
|
||||||
|
surface in seconds rather than minutes, without blocking on dependency
|
||||||
|
download or compilation in the build stage.
|
||||||
|
|
||||||
```sh
|
The standard pattern for a Go repo Dockerfile is:
|
||||||
docker build --no-cache --target lint -t "$(script/projectname)-lint" .
|
|
||||||
docker build --no-cache --target test -t "$(script/projectname)-test" .
|
|
||||||
```
|
|
||||||
|
|
||||||
**A stage that is not the last one in the file is built only when the final
|
|
||||||
stage's chain depends on it, or when `--target` names it.** That is why the
|
|
||||||
two gates are always invoked by name here, and why the final stage carries a
|
|
||||||
`COPY --from=` of a harmless file from each of them: without that edge a
|
|
||||||
plain `docker build .` builds the last stage alone and exits 0 having linted
|
|
||||||
and tested nothing.
|
|
||||||
|
|
||||||
**Every `docker build` in `script/` is tagged**, here and in
|
|
||||||
`script/cibuild` and `script/docker`. An untagged build leaves a dangling
|
|
||||||
image behind on every invocation, on every developer host and every CI
|
|
||||||
runner; a tagged one replaces the previous image.
|
|
||||||
|
|
||||||
Inside a phase the tool is invoked directly — `golangci-lint`, `go test`,
|
|
||||||
`eslint`, `prettier` — never through `make lint` or `script/test`, which are
|
|
||||||
themselves a `docker build` and would recurse into a daemon that does not
|
|
||||||
exist in a build step. Formatting is the exception and stays on the host:
|
|
||||||
`script/fmt` writes the working tree, and `script/fmt-check` is its
|
|
||||||
read-only twin.
|
|
||||||
|
|
||||||
**No lint verdict may come from a host invocation of the linter.** On a
|
|
||||||
shared host golangci-lint reads a result cache keyed on file content rather
|
|
||||||
than location, so a second checkout of the same content is served the first
|
|
||||||
one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs
|
|
||||||
exit non-zero with `parallel golangci-lint is running` — a status a caller
|
|
||||||
cannot tell from real findings. Both have produced wrong verdicts in this
|
|
||||||
org, in both directions. A container has its own cache, its own `TMPDIR` and
|
|
||||||
a digest-pinned binary, so neither is reachable.
|
|
||||||
|
|
||||||
- **Any build that runs checks is built with `--no-cache`.** Docker invalidates
|
|
||||||
a `COPY` layer only when the copied content changes, so on an unchanged tree
|
|
||||||
the check `RUN` is served from cache, nothing executes, and the build still
|
|
||||||
exits 0. Every `docker build` in `script/` therefore passes `--no-cache`:
|
|
||||||
`script/lint`, `script/test`, `script/cibuild` and `script/docker` are the
|
|
||||||
four, and there is no fifth — `script/check` runs the two gate phases and
|
|
||||||
`script/fmt-check`, and builds no image of its own. A bare `docker build .` is
|
|
||||||
not evidence that anything ran: a sub-second build reporting success is a
|
|
||||||
cache hit, not a result. Never invalidate by pruning — `docker builder prune`
|
|
||||||
and friends destroy a build cache shared with every other build on the host.
|
|
||||||
When a check is added or changed, prove it works by planting a defect it must
|
|
||||||
catch and watching the run fail on it, then revert the defect. A green run
|
|
||||||
alone shows neither that the check ran nor that it covers what it should.
|
|
||||||
|
|
||||||
- **The gate phases are separate stages, and the build stage depends on both.**
|
|
||||||
The lint phase is based on the `golangci/golangci-lint` image (pinned by
|
|
||||||
hash), so lint failures surface in seconds rather than after a full compile,
|
|
||||||
and the test phase is based on the Debian Go image. The canonical Go repo
|
|
||||||
`Dockerfile`:
|
|
||||||
|
|
||||||
```dockerfile
|
```dockerfile
|
||||||
# Lint phase
|
# Lint stage — fast feedback on formatting and lint issues
|
||||||
# golangci/golangci-lint:v2.x.x, YYYY-MM-DD
|
# golangci/golangci-lint:v2.x.x, YYYY-MM-DD
|
||||||
FROM golangci/golangci-lint@sha256:... AS lint
|
FROM golangci/golangci-lint@sha256:... AS lint
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
RUN go mod download
|
RUN go mod download
|
||||||
COPY . .
|
COPY . .
|
||||||
RUN golangci-lint run --config .golangci.yml ./...
|
RUN make fmt-check
|
||||||
|
RUN make lint
|
||||||
|
|
||||||
# Test phase. -race needs cgo and so a C compiler, which the Debian Go
|
# Build stage
|
||||||
# image ships and the alpine one does not.
|
|
||||||
# golang:1.x, YYYY-MM-DD
|
|
||||||
FROM golang@sha256:... AS test
|
|
||||||
WORKDIR /src
|
|
||||||
COPY go.mod go.sum ./
|
|
||||||
RUN go mod download
|
|
||||||
COPY . .
|
|
||||||
RUN go test -timeout 90s -race -cover ./... || \
|
|
||||||
{ echo "--- Rerunning with -v for details ---"; \
|
|
||||||
go test -timeout 90s -race -v ./...; exit 1; }
|
|
||||||
|
|
||||||
# Build stage. Nothing is wanted from either phase above; the copies
|
|
||||||
# are what make BuildKit build them first, so this stage cannot run
|
|
||||||
# unless lint and test passed.
|
|
||||||
# golang:1.x-alpine, YYYY-MM-DD
|
# golang:1.x-alpine, YYYY-MM-DD
|
||||||
FROM golang@sha256:... AS builder
|
FROM golang@sha256:... AS builder
|
||||||
COPY --from=lint /src/go.sum /dev/null
|
|
||||||
COPY --from=test /src/go.sum /dev/null
|
|
||||||
RUN apk add --no-cache git
|
|
||||||
# A tar-stream context keeps the sender's file owners, which git refuses.
|
|
||||||
RUN git config --system --add safe.directory /src
|
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
|
|
||||||
|
# Force BuildKit to run the lint stage before proceeding
|
||||||
|
COPY --from=lint /src/go.sum /dev/null
|
||||||
|
|
||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
RUN go mod download
|
RUN go mod download
|
||||||
COPY . .
|
COPY . .
|
||||||
|
RUN make test
|
||||||
|
|
||||||
# The VERSION build arg when one is given, otherwise
|
ARG VERSION=dev
|
||||||
# `git describe --tags --always` on the .git in the build context. With
|
RUN CGO_ENABLED=0 go build -trimpath \
|
||||||
# .git present, a version that is still empty, dev or unknown fails the
|
|
||||||
# build: git is missing or could not read the checkout.
|
|
||||||
ARG VERSION
|
|
||||||
RUN VERSION="${VERSION:-$(git describe --tags --always)}"; \
|
|
||||||
if [ -e .git ]; then \
|
|
||||||
case "$VERSION" in ""|dev|unknown) \
|
|
||||||
echo "version is '$VERSION' although .git is present" >&2; \
|
|
||||||
exit 1 ;; \
|
|
||||||
esac; \
|
|
||||||
fi; \
|
|
||||||
CGO_ENABLED=0 go build -trimpath \
|
|
||||||
-ldflags="-s -w -X main.Version=${VERSION}" \
|
-ldflags="-s -w -X main.Version=${VERSION}" \
|
||||||
-o /app ./cmd/app/
|
-o /app ./cmd/app/
|
||||||
|
|
||||||
# Runtime stage, and the last one
|
# Runtime stage
|
||||||
FROM alpine@sha256:...
|
FROM alpine@sha256:...
|
||||||
COPY --from=builder /app /usr/local/bin/app
|
COPY --from=builder /app /usr/local/bin/app
|
||||||
ENTRYPOINT ["app"]
|
ENTRYPOINT ["app"]
|
||||||
```
|
```
|
||||||
|
|
||||||
Key points:
|
Key points:
|
||||||
- The lint phase uses the `golangci/golangci-lint` image directly (it has
|
- The lint stage uses the `golangci/golangci-lint` image directly (it
|
||||||
both Go and the linter), so nothing needs installing.
|
includes both Go and the linter), so there is no need to install the
|
||||||
- `COPY --from=<phase> /src/go.sum /dev/null` is a no-op copy whose only
|
linter separately.
|
||||||
purpose is the ordering edge. BuildKit runs stages in parallel by default,
|
- `COPY --from=lint /src/go.sum /dev/null` is a no-op file copy that creates
|
||||||
and a stage nothing depends on is not built at all, so without these two
|
a stage dependency. BuildKit runs stages in parallel by default; without
|
||||||
lines a red gate would not fail the build.
|
this line, the build stage would not wait for lint to finish and a lint
|
||||||
- Keep the runtime stage last, and if you add a stage after it, give it the
|
failure might not fail the overall build.
|
||||||
same two copies. A plain `docker build .` builds the last stage's chain
|
|
||||||
and nothing else.
|
|
||||||
- If the project uses `//go:embed` directives that reference build artifacts
|
- If the project uses `//go:embed` directives that reference build artifacts
|
||||||
(e.g. a web frontend compiled in a separate stage), the lint phase must
|
(e.g. a web frontend compiled in a separate stage), the lint stage must
|
||||||
create placeholder files so the embed directives resolve. Example:
|
create placeholder files so the embed directives resolve. Example:
|
||||||
`RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`.
|
`RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`.
|
||||||
- If the project requires CGO or system libraries for linting, install them
|
The lint stage should not depend on the actual build output — it exists to
|
||||||
in the lint phase. The `golangci/golangci-lint` image is Debian-based and
|
fail fast.
|
||||||
has no `apk`, so install with `apt-get` under the Debian package name
|
- If the project requires CGO or system libraries for linting (e.g.
|
||||||
(`libvips-dev`, where alpine says `vips-dev`), and delete the package
|
`vips-dev`), install them in the lint stage with `apk add`.
|
||||||
lists in the same `RUN`, so the layer does not keep them:
|
- The build stage runs `make test` after compilation setup. Tests run in the
|
||||||
|
build stage, not the lint stage, because they may require compiled
|
||||||
```dockerfile
|
artifacts or heavier dependencies.
|
||||||
RUN apt-get update \
|
|
||||||
&& apt-get install -y --no-install-recommends libvips-dev \
|
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
|
||||||
```
|
|
||||||
|
|
||||||
- `.dockerignore` lets `.git` into the build context. It keeps out every git
|
|
||||||
`config` at any depth (`**/.git/config`, `**/.git/modules/**/config`): the
|
|
||||||
repository's own, each submodule's under `.git/modules/`, and that of a
|
|
||||||
submodule keeping its own `.git` directory. `git describe` does not need
|
|
||||||
them, and each can hold a credential: a password in a remote URL, or the
|
|
||||||
token the CI checkout step stores there. A submodule whose name has a
|
|
||||||
`config` segment (`config`, `deploy/config`, `config/lib`) loses its whole
|
|
||||||
git directory to `**/.git/modules/**/config`, and Go's version stamping
|
|
||||||
then fails the build: give it a name without that segment
|
|
||||||
(`git submodule add --name`). The stage that compiles has `git` (the
|
|
||||||
Debian Go image has it; an alpine one needs `apk add --no-cache git`) and
|
|
||||||
takes the version from the `VERSION` build argument when one is given,
|
|
||||||
otherwise from `git describe --tags --always`. That gives the tag on a
|
|
||||||
tagged commit; on a later commit, the tag, the number of commits since it
|
|
||||||
and the short commit (`v1.2.3-4-gabc1234`); and the short commit when no
|
|
||||||
tag is reachable. The stage that compiles also marks its working directory
|
|
||||||
safe for git (`git config --system --add safe.directory /src`): a context
|
|
||||||
sent as a tar stream keeps the sender's file owners, and git refuses a
|
|
||||||
checkout owned by another user, so the version would come out empty.
|
|
||||||
`ARG VERSION` has no default, and the build fails if the context carries
|
|
||||||
`.git` and the version still comes out empty, `dev` or `unknown`. A plain
|
|
||||||
`docker build .` with no build arguments must succeed; a Dockerfile that
|
|
||||||
refuses an empty build argument drops that refusal and keeps the argument.
|
|
||||||
|
|
||||||
- Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that
|
- Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that
|
||||||
runs `script/cibuild` on push, and checks out the repo as its only other step.
|
runs `script/cibuild` (which runs `docker build .`) on push. Since the
|
||||||
That script bootstraps, runs the gate phases, and then builds the image, so a
|
Dockerfile already runs `make check`, a successful build implies all checks
|
||||||
successful run means every check passed; a bare `docker build .` does not
|
pass.
|
||||||
carry the same guarantee, because its gate phases may come from the cache. The
|
|
||||||
image build is uncached and so runs the gate phases a second time. That is the
|
|
||||||
price of the rule above, and it is worth paying: the image that ships is built
|
|
||||||
from a run of its own gates rather than from a cache entry. A separate
|
|
||||||
workflow limited to `main` by a `branches` list under `on: push` cannot be
|
|
||||||
checked by review: to try a change to it, add the feature branch to that list
|
|
||||||
and push, then remove the branch from the list again before merging. Keep any
|
|
||||||
job in it that publishes behind `if: github.ref_name == 'main'`, so the run
|
|
||||||
from the feature branch publishes nothing.
|
|
||||||
|
|
||||||
- Use platform-standard formatters: `black` for Python, `prettier` for
|
- Use platform-standard formatters: `black` for Python, `prettier` for
|
||||||
JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with
|
JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with
|
||||||
@@ -315,17 +193,15 @@ style conventions are in separate documents:
|
|||||||
suite that exceeds it fails. Under 20 seconds is the target. A suite between
|
suite that exceeds it fails. Under 20 seconds is the target. A suite between
|
||||||
20 and 60 seconds is still green, but the overage must be filed as an
|
20 and 60 seconds is still green, but the overage must be filed as an
|
||||||
improvement bug against that repo. Add a 90-second timeout to the test
|
improvement bug against that repo. Add a 90-second timeout to the test
|
||||||
invocation (`go test -timeout 90s`). The backstop deliberately sits above the
|
invocation in the Makefile (`go test -timeout 90s`). The backstop deliberately
|
||||||
hard cap so that it catches a genuinely hung test rather than a merely slow
|
sits above the hard cap so that it catches a genuinely hung test rather than a
|
||||||
one.
|
merely slow one.
|
||||||
|
|
||||||
- **The test command should use the conditional verbose rerun pattern.** Run
|
- **`make test` should use the conditional verbose rerun pattern.** Run tests
|
||||||
tests without `-v` (verbose) first. If tests fail, automatically rerun with
|
without `-v` (verbose) first. If tests fail, automatically rerun with `-v` to
|
||||||
`-v` to show full output. This keeps CI logs and `docker build` output clean
|
show full output. This keeps CI logs and `docker build` output clean on
|
||||||
on success (just package/suite summaries) while providing full diagnostic
|
success (just package/suite summaries) while providing full diagnostic detail
|
||||||
detail on failure (every test case, every assertion). The command lives in the
|
on failure (every test case, every assertion). The general shell pattern:
|
||||||
`test` phase of the `Dockerfile`, since `script/test` builds that phase; the
|
|
||||||
Makefile form below is the same pattern for any repo-local invocation:
|
|
||||||
|
|
||||||
```makefile
|
```makefile
|
||||||
test:
|
test:
|
||||||
@@ -338,26 +214,11 @@ style conventions are in separate documents:
|
|||||||
|
|
||||||
```makefile
|
```makefile
|
||||||
test:
|
test:
|
||||||
@go test -count=1 -timeout 90s -race -cover ./... || \
|
@go test -timeout 90s -race -cover ./... || \
|
||||||
{ echo "--- Rerunning with -v for details ---"; \
|
{ echo "--- Rerunning with -v for details ---"; \
|
||||||
go test -count=1 -timeout 90s -race -v ./...; exit 1; }
|
go test -timeout 90s -race -v ./...; exit 1; }
|
||||||
```
|
```
|
||||||
|
|
||||||
`-count=1` is required on both invocations: it defeats Go's test _result_
|
|
||||||
cache, so neither run can report a stored pass in place of running the
|
|
||||||
tests. It leaves the build cache alone, so it costs the runtime of the suite
|
|
||||||
and no recompilation.
|
|
||||||
|
|
||||||
That cache is Go's own, separate from Docker's layer cache. Go stores a
|
|
||||||
passing result in its cache directory (`GOCACHE`), and when the same tests
|
|
||||||
run again on unchanged code it prints that result, marked `(cached)`,
|
|
||||||
without running them. That matters on a developer's machine, where this
|
|
||||||
target runs and the directory lasts from one run to the next. The `test`
|
|
||||||
phase of the `Dockerfile` needs no `-count=1`: its base image holds no
|
|
||||||
result for this repo's tests and nothing before its `go test` step runs a
|
|
||||||
test, so there is nothing to replay. `--no-cache` (above) is what makes that
|
|
||||||
step run on an unchanged tree.
|
|
||||||
|
|
||||||
Python example:
|
Python example:
|
||||||
|
|
||||||
```makefile
|
```makefile
|
||||||
@@ -383,84 +244,10 @@ style conventions are in separate documents:
|
|||||||
must be in `.gitignore`. No exceptions.
|
must be in `.gitignore`. No exceptions.
|
||||||
|
|
||||||
- `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`),
|
- `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`),
|
||||||
editor files (`.swp`, `*~`), in-repo agent scratch directories (`.claude/`),
|
editor files (`.swp`, `*~`), language build artifacts, and `node_modules/`.
|
||||||
language build artifacts, and `node_modules/`. Fetch the standard `.gitignore`
|
Fetch the standard `.gitignore` from
|
||||||
from `https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when
|
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when setting up
|
||||||
setting up a new repo. These patterns are written to `.gitignore`'s own
|
a new repo.
|
||||||
semantics, in which an unanchored pattern already matches at every depth; they
|
|
||||||
are not a `.dockerignore` and must not be transplanted into one unmodified.
|
|
||||||
|
|
||||||
- **`.dockerignore` does not use `.gitignore` semantics, and copying patterns
|
|
||||||
across unmodified leaves secrets in the build context.** Docker matches with
|
|
||||||
`moby/patternmatcher`: `filepath.Match` semantics plus a `**` extension, so
|
|
||||||
`*` does not cross `/` and a pattern without a leading `**/` is anchored at
|
|
||||||
the build-context root. A `.dockerignore` listing `.env`, `*.pem` and `*.key`
|
|
||||||
therefore excludes only the copies at the repository root, while `config/.env`
|
|
||||||
and `certs/server.key` still reach the context and can land in an image layer
|
|
||||||
— which is more dangerous than a short file with no secret patterns at all,
|
|
||||||
because it reads as solved and stops anyone looking. Give every
|
|
||||||
depth-independent pattern the `**/` prefix and leave only genuinely
|
|
||||||
root-anchored entries unprefixed: `.claude`, and the repo's own host-built
|
|
||||||
binary, written `/myapp` and never `**/myapp`, which would also match
|
|
||||||
`cmd/myapp/` and delete the package directory from the context. Matching is
|
|
||||||
case-sensitive, and an ALL-CAPS twin per pattern still misses `Server.Key`, so
|
|
||||||
secret names use character ranges — `**/*.[kK][eE][yY]`, `**/*.[pP][eE][mM]`,
|
|
||||||
and likewise for `.envrc` and the extensionless SSH keys. Where such a pattern
|
|
||||||
also catches something the build needs, re-include it with a negation
|
|
||||||
(`!docs/example.env`); deleting the pattern reopens the exposure for every
|
|
||||||
other file it covers. Fetch the standard `.dockerignore` from
|
|
||||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.dockerignore` and extend
|
|
||||||
it with the repo's own artifacts.
|
|
||||||
|
|
||||||
- **In-repo agent scratch belongs in both files, written to each file's own
|
|
||||||
semantics.** `.claude/` holds one worktree per in-flight agent — an entire
|
|
||||||
additional checkout of the repo — so under `COPY . .` the build context
|
|
||||||
inflates by a multiple of the repo and another session's unreviewed work can
|
|
||||||
be copied into an image layer. In `.gitignore` the entry is `.claude/`,
|
|
||||||
unanchored. In `.dockerignore` it is `.claude`, anchored and with **no** `**/`
|
|
||||||
prefix, because the prefixed form would also delete any nested directory of
|
|
||||||
that name from the build. Anchoring carries a known gap that the canonical
|
|
||||||
`.dockerignore` states in its own comment, since consuming repos receive the
|
|
||||||
file and not the tracker: the directory is created in the agent's working
|
|
||||||
directory, so a repo running agents in subdirectories still ships
|
|
||||||
`services/api/.claude/` and must add its own anchored entry there.
|
|
||||||
|
|
||||||
- **A plain `docker build .` of a clone stamps the version that
|
|
||||||
`git describe --tags --always` gives**, derived from the `.git` in the build
|
|
||||||
context as the canonical `Dockerfile` above shows. Without its failure check,
|
|
||||||
a missing `git` or an unreadable checkout would leave `-X main.Version=` empty
|
|
||||||
and the build would still exit 0. `script/docker` and `script/cibuild` pass
|
|
||||||
the version they compute on the host; it takes precedence. They do this
|
|
||||||
byte-identically across repos:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
# Own line: a failing command substitution inside an argument does not
|
|
||||||
# trip `set -e`, so the inline form degrades to an empty constant.
|
|
||||||
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
|
|
||||||
[ -n "$version" ] || version="unknown"
|
|
||||||
docker build --no-cache \
|
|
||||||
--build-arg VERSION="$version" \
|
|
||||||
-t "$(script/projectname)" .
|
|
||||||
```
|
|
||||||
|
|
||||||
`--always` makes an untagged repo yield an abbreviated commit hash rather
|
|
||||||
than failing, and the `[ -n "$version" ]` line is the single place the
|
|
||||||
fallback is applied — a live check that fires on a build from an export with
|
|
||||||
no `.git` and on a repository with no commits yet. Do not fold it into the
|
|
||||||
substitution as `|| echo unknown`, which makes the guard unreachable. The
|
|
||||||
Dockerfile's side is `ARG VERSION` in the stage that compiles, declared
|
|
||||||
there because `ARG` is stage-scoped; passing `VERSION` to a repo whose
|
|
||||||
Dockerfile declares no such `ARG` is ignored and costs nothing, which is why
|
|
||||||
the scripts stay byte-identical. One consequence for CI: the standard
|
|
||||||
checkout action clones shallow and fetches no tags, so a repo that embeds a
|
|
||||||
tag-derived version must set `fetch-depth: 0` on its checkout step.
|
|
||||||
|
|
||||||
- **Verify `.dockerignore` by enumerating the image, not by reading the
|
|
||||||
patterns.** Plant files at the root _and_ at least two directories deep, build
|
|
||||||
a probe image that does `COPY . .`, and list what actually landed
|
|
||||||
(`docker run --rm --entrypoint find IMAGE /app`). The `transferring context`
|
|
||||||
size is not a substitute: a nested secret is a few bytes, and BuildKit
|
|
||||||
transfers only the delta from the previous build.
|
|
||||||
|
|
||||||
- **No build artifacts in version control.** Code-derived data (compiled
|
- **No build artifacts in version control.** Code-derived data (compiled
|
||||||
bundles, minified output, generated assets) must never be committed to the
|
bundles, minified output, generated assets) must never be committed to the
|
||||||
@@ -476,56 +263,12 @@ style conventions are in separate documents:
|
|||||||
- Make all changes on a feature branch. You can do whatever you want on a
|
- Make all changes on a feature branch. You can do whatever you want on a
|
||||||
feature branch.
|
feature branch.
|
||||||
|
|
||||||
- `.golangci.yml` is standardized. The vendored copy in a consuming repo must
|
- `.golangci.yml` is standardized and must _NEVER_ be modified by an agent, only
|
||||||
_NEVER_ be modified by an agent: fetch it from
|
manually by the user. Fetch from
|
||||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml` and keep it
|
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml`. The
|
||||||
byte-identical, so that no repo can quietly loosen its own linting. Linter
|
canonical golangci-lint version is v2.12.2 (released 2026-05-06), installed
|
||||||
configuration changes are made to the canonical copy in the `prompts` repo and
|
commit-pinned via
|
||||||
reach consuming repos by re-vendoring; an agent may open a PR against
|
`go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@c0d3ddc9cf3faa61a4e378e879ece580256d76e5`.
|
||||||
canonical, which only the user merges. One list is exempt from byte-identity,
|
|
||||||
because it cannot be written once for every repo: the `deny` list of the
|
|
||||||
`test-support` depguard rule, where a repo names its own test-support packages
|
|
||||||
by full import path. A repo adds entries there and changes nothing else, and a
|
|
||||||
re-vendor carries its entries forward. The canonical golangci-lint version is
|
|
||||||
v2.14.0 (released 2026-09-24), pinned as the digest of the lint phase's base
|
|
||||||
image
|
|
||||||
(`golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f`,
|
|
||||||
which reports `2.14.0 built with go1.27.0 from 114493f9`). A module's `go`
|
|
||||||
directive must not name a newer Go minor version than the one golangci-lint
|
|
||||||
was built with, or golangci-lint refuses to lint it: this release lints
|
|
||||||
`go 1.27.1` but not `go 1.28`. That digest is the only pin, since no repo
|
|
||||||
installs golangci-lint on the host. A repo sets the lint phase digest to the
|
|
||||||
one named here and re-vendors `.golangci.yml` in the same commit, whichever of
|
|
||||||
the two prompted the change: the canonical copy can name linters that an older
|
|
||||||
golangci-lint rejects, and a newer golangci-lint can add linters that
|
|
||||||
`default: all` switches on until the canonical copy disables them.
|
|
||||||
|
|
||||||
- **`script/bootstrap` installs a pinned tool by comparing versions, never by
|
|
||||||
testing presence.** An `if ! command -v <tool>; then install; fi` guard tests
|
|
||||||
`PATH` only, so on an already-provisioned machine the pin is inert and a
|
|
||||||
version bump is a silent no-op — while the Dockerfile, installing into a clean
|
|
||||||
image, gets the pinned version, so a local `make check` and `make docker` can
|
|
||||||
disagree about what the tool even is. The canonical form:
|
|
||||||
- compares the installed version against the pin over the **whole** version
|
|
||||||
token; a parser that stops at the first `-` reports `2.12.2` for a host
|
|
||||||
running `2.12.2-rc1` and skips the install;
|
|
||||||
- treats absent, non-zero, empty or unrecognised `--version` output as a
|
|
||||||
mismatch, so the failure direction is a redundant install and never a
|
|
||||||
skipped one;
|
|
||||||
- after installing, re-resolves the binary the way callers do — `hash -r`,
|
|
||||||
then through `PATH`, not through the directory the installer wrote to —
|
|
||||||
and fails naming the resolved path, since an install that a shadowing
|
|
||||||
binary hides succeeds while changing nothing any caller sees;
|
|
||||||
- is actually called, and prints the version on both success paths: a
|
|
||||||
function defined and never invoked has the same exit status and the same
|
|
||||||
empty output as one that worked.
|
|
||||||
|
|
||||||
Keep it POSIX sh: no arrays, no `[[`, no `grep -P`.
|
|
||||||
|
|
||||||
A Go tool a repo needs on the host is installed with `go install` pinned to
|
|
||||||
a commit hash (`go install <package>@<commit hash>`). It is never tracked as
|
|
||||||
a `go.mod` tool dependency or through a `tools.go` file, either of which
|
|
||||||
pulls the tool's own dependencies into the repo's `go.mod` and `go.sum`.
|
|
||||||
|
|
||||||
- When pinning images or packages by hash, add a comment above the reference
|
- When pinning images or packages by hash, add a comment above the reference
|
||||||
with the version and date (YYYY-MM-DD).
|
with the version and date (YYYY-MM-DD).
|
||||||
@@ -639,14 +382,12 @@ style conventions are in separate documents:
|
|||||||
settings.
|
settings.
|
||||||
|
|
||||||
- Avoid putting files in the repo root unless necessary. Root should contain
|
- Avoid putting files in the repo root unless necessary. Root should contain
|
||||||
only project-level config files (`README.md`, `AGENTS.md`, `Makefile`,
|
only project-level config files (`README.md`, `Makefile`, `Dockerfile`,
|
||||||
`Dockerfile`, `LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`,
|
`LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, and
|
||||||
and language-specific config). Everything else goes in a subdirectory.
|
language-specific config). Everything else goes in a subdirectory. Canonical
|
||||||
Canonical subdirectory names:
|
subdirectory names:
|
||||||
- `bin/` — executable scripts and tools
|
- `bin/` — executable scripts and tools
|
||||||
- `cmd/` — Go command entrypoints; thin only: one `main.go` per binary whose
|
- `cmd/` — Go command entrypoints
|
||||||
body is a single call into `internal/` or `pkg/`, no project logic in
|
|
||||||
`cmd/`
|
|
||||||
- `configs/` — configuration templates and examples
|
- `configs/` — configuration templates and examples
|
||||||
- `deploy/` — deployment manifests (k8s, compose, terraform)
|
- `deploy/` — deployment manifests (k8s, compose, terraform)
|
||||||
- `docs/` — documentation and markdown (README.md stays in root)
|
- `docs/` — documentation and markdown (README.md stays in root)
|
||||||
@@ -673,7 +414,3 @@ style conventions are in separate documents:
|
|||||||
- Go: `go.mod`, `go.sum`, `.golangci.yml`
|
- Go: `go.mod`, `go.sum`, `.golangci.yml`
|
||||||
- JS: `package.json`, `yarn.lock`, `.prettierrc`, `.prettierignore`
|
- JS: `package.json`, `yarn.lock`, `.prettierrc`, `.prettierignore`
|
||||||
- Python: `pyproject.toml`
|
- Python: `pyproject.toml`
|
||||||
|
|
||||||
- Guidance for coding agents lives in one `AGENTS.md` at the repository root. It
|
|
||||||
is never committed under a file or directory named after one agent tool, such
|
|
||||||
as `CLAUDE.md` or `.claude/`, and never split into separate memory files.
|
|
||||||
|
|||||||
@@ -2,71 +2,418 @@
|
|||||||
|
|
||||||
One issue per unit of work, one branch and one PR per issue:
|
One issue per unit of work, one branch and one PR per issue:
|
||||||
|
|
||||||
- ensure a tracked issue exists with a definition of done
|
* ensure a tracked issue exists with a definition of done
|
||||||
- branch from `next` (never from `main`)
|
* branch from `next` (never from `main`)
|
||||||
- do the work; open a PR based on `next` (never on `main`)
|
* do the work; open a PR based on `next` (never on `main`)
|
||||||
- pass an independent review, then the manager squash-merges into `next`
|
* pass an independent review, then the manager squash-merges into `next`
|
||||||
- push; nothing stays local-only
|
* push; nothing stays local-only
|
||||||
|
|
||||||
`next` is the branch for the next milestone and must stay green and mergeable to
|
`next` is the branch for the next milestone and must stay green and
|
||||||
`main` without notice. One `next` -> `main` PR accumulates the milestone;
|
mergeable to `main` without notice. One `next` -> `main` PR accumulates
|
||||||
releases are cut from `main` separately.
|
the milestone; releases are cut from `main` separately.
|
||||||
|
|
||||||
The tracker (https://git.eeqj.de/sneak/webhooker/issues) is the record of what
|
Issue branches do NOT touch this file — the manager maintains it on
|
||||||
is done and what is under way. This file keeps no list of either.
|
`next`. Every branch editing `TODO.md` conflicts with every other
|
||||||
|
(#112).
|
||||||
|
|
||||||
# Status
|
# Status
|
||||||
|
|
||||||
What is still open for 1.0.0 is in its milestone
|
The milestone (https://git.eeqj.de/sneak/webhooker/milestone/9) is the
|
||||||
(https://git.eeqj.de/sneak/webhooker/milestone/9). `v1.0.0` is not tagged, and
|
authoritative list, and the only place to read a count or a state of
|
||||||
only @sneak tags it.
|
play from. This file records where the project is, not what is in
|
||||||
|
flight: a sentence whose truth depends on a branch being unmerged is
|
||||||
|
wrong the moment it merges, and this file has been wrong that way
|
||||||
|
before.
|
||||||
|
|
||||||
Delivery is at-least-once by design, not by accident: a send whose result row
|
The durability defect that held the tag has landed
|
||||||
does not land is attempted again, so a receiver can see a duplicate. That is
|
(https://git.eeqj.de/sneak/webhooker/issues/256, commit `8d64259`).
|
||||||
deliberate — the alternative is a silent lost delivery — and the README says so
|
Every SQLite handle opens with WAL journaling and a busy timeout, a
|
||||||
under Rationale. It is not a defect to re-file.
|
bookkeeping write that fails leaves its delivery in a recoverable
|
||||||
|
state rather than a lying one, and recovery skips a delivery that
|
||||||
|
already has a successful result row. Final pre-tag verification
|
||||||
|
exercised it and confirmed it holds. Whatever the milestone still
|
||||||
|
shows open is what remains before `v1.0.0`.
|
||||||
|
|
||||||
|
Delivery is at-least-once by design, not by accident: a send whose
|
||||||
|
result row does not land is attempted again, so a receiver can see a
|
||||||
|
duplicate. That is deliberate — the alternative is a silent lost
|
||||||
|
delivery — and the README says so under Rationale. It is not a defect
|
||||||
|
to re-file.
|
||||||
|
|
||||||
# Next Step
|
# Next Step
|
||||||
|
|
||||||
@sneak runs webhooker in limited production, then tags `v1.0.0`. Merging `next`
|
Clear the rest of the open 1.0.0 milestone
|
||||||
into `main` is a separate act from tagging and waits on neither of those: `next`
|
(https://git.eeqj.de/sneak/webhooker/milestone/9) and tag `v1.0.0`.
|
||||||
is kept mergeable at all times, which is the point of the branch.
|
Merging `next` into `main` is a separate act from tagging and waits on
|
||||||
|
neither of those: `next` is kept mergeable at all times, which is the
|
||||||
|
point of the branch.
|
||||||
|
|
||||||
|
# Completed Steps
|
||||||
|
|
||||||
|
- 2026-08-24 Bind the plaintext HTTP listener deliberately, via
|
||||||
|
`BIND_ADDRESS` defaulting to `127.0.0.1`, and document the
|
||||||
|
reverse-proxy deployment. A hostname, an empty value or a value
|
||||||
|
carrying a port is a startup error, and the `Dockerfile` sets
|
||||||
|
`0.0.0.0` because a loopback bind inside a container is unreachable
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/268). The same commit
|
||||||
|
removed the shutdown race: `httpServer` is built in the constructor
|
||||||
|
rather than assigned from the serving goroutine, which orders the
|
||||||
|
write before every fx hook and rules out the nil dereference a
|
||||||
|
SIGTERM arriving first would have caused, and `sentryEnabled` is an
|
||||||
|
`atomic.Bool` (https://git.eeqj.de/sneak/webhooker/issues/226)
|
||||||
|
- 2026-08-24 Remove inbound request signature verification. The
|
||||||
|
entrypoint UUID is the authentication secret, so the per-entrypoint
|
||||||
|
shared secret, the `internal/signature` package, the receiver check,
|
||||||
|
the model fields and the forms are all gone. This reverses the
|
||||||
|
feature that landed earlier in the same milestone
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/67,
|
||||||
|
https://git.eeqj.de/sneak/webhooker/issues/279)
|
||||||
|
- 2026-08-24 Stamp the build version into the binary and render it in
|
||||||
|
the UI footer. `script/version` is the single source — `$VERSION`,
|
||||||
|
else `git describe --tags --always --dirty`, else `unknown` — so a
|
||||||
|
`make build` binary and a `make docker` image from one checkout
|
||||||
|
report the same thing, and nothing in it varies between two builds
|
||||||
|
of the same commit, which the release gate's byte-identical
|
||||||
|
assertion would catch
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/253)
|
||||||
|
- 2026-08-24 Derive cookie `Secure` and CSRF strictness from the
|
||||||
|
request transport rather than from `WEBHOOKER_ENVIRONMENT`. Behind a
|
||||||
|
real TLS proxy with the environment left at its `dev` default, the
|
||||||
|
session cookie silently lost `Secure` while the CSRF cookie on the
|
||||||
|
same response kept it. `X-Forwarded-Proto` is now matched
|
||||||
|
case-insensitively on its first comma-separated element, so `HTTPS`
|
||||||
|
and `https, http` no longer fall to the relaxed CSRF path
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/269)
|
||||||
|
- 2026-08-24 Roll back a failed webhook deletion instead of committing
|
||||||
|
it. A failing delete committed whatever had already succeeded,
|
||||||
|
hard-deleted the per-webhook event database anyway, and redirected as
|
||||||
|
though it had worked — orphaned config plus permanently destroyed
|
||||||
|
history, reported as success. All three delete positions now roll
|
||||||
|
back with the event database intact
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/262)
|
||||||
|
- 2026-08-24 Name a deleted target on its historical deliveries, marked
|
||||||
|
`(deleted)`, rather than leaving the event log unable to say where a
|
||||||
|
delivery went. A deleted target's credentials stay masked exactly as
|
||||||
|
a live one's, and it cannot become deliverable again through the
|
||||||
|
receiver, resubmit, replay, the edit form or the toggle
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/211)
|
||||||
|
- 2026-08-24 Bound both request-controlled `/metrics` label dimensions,
|
||||||
|
so the unauthenticated receiver is no longer a memory-exhaustion
|
||||||
|
vector: `handler` carries the chi route pattern, and `method` folds
|
||||||
|
anything chi cannot route onto a single `(unmatched)` sentinel. Both
|
||||||
|
were reproduced before the fix — 300 random method tokens took the
|
||||||
|
series count from 106 to 7,631, and path flooding reached 62,532 —
|
||||||
|
and a label audit across a live scrape found no third unbounded
|
||||||
|
dimension (https://git.eeqj.de/sneak/webhooker/issues/254,
|
||||||
|
https://git.eeqj.de/sneak/webhooker/issues/261)
|
||||||
|
- 2026-08-24 Validate `max_retries` on both target forms. `abc`, `2.7`
|
||||||
|
and `-5` silently became 0 — fire-and-forget — including on the edit
|
||||||
|
path, where it destroyed a working value, and `999999999` stored
|
||||||
|
verbatim. The ceiling of 20 is the `max` both templates already
|
||||||
|
declared (https://git.eeqj.de/sneak/webhooker/issues/221)
|
||||||
|
- 2026-08-24 Resubmit a stored event as a new undelivered event, so a
|
||||||
|
backend under development can be tested against real captured
|
||||||
|
traffic. Per-delivery replay cannot serve that: it re-sends one
|
||||||
|
finished delivery to its own original target, and a target created
|
||||||
|
for a dev backend has no prior delivery to replay. Resubmit
|
||||||
|
re-injects the stored event at the top of the receiver path and fans
|
||||||
|
it out to whatever targets are active now
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/250)
|
||||||
|
- 2026-08-20 Take an exclusive lock on `DATA_DIR` at startup, so two
|
||||||
|
instances on one directory cannot both deliver
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/201)
|
||||||
|
- 2026-08-20 Shut down the app when the HTTP listener fails. The
|
||||||
|
`OnStart` hook returned as soon as the serving goroutine was
|
||||||
|
spawned, so a failed listen left fx reporting RUNNING and a live
|
||||||
|
process with nothing bound — invisible to systemd and Docker restart
|
||||||
|
policies (https://git.eeqj.de/sneak/webhooker/issues/200)
|
||||||
|
- 2026-08-20 Stop target credentials leaking into the per-webhook event
|
||||||
|
databases (https://git.eeqj.de/sneak/webhooker/issues/206), log SQL
|
||||||
|
with placeholders rather than bound values
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/207), and fail loudly on
|
||||||
|
half-set metrics auth credentials
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/205)
|
||||||
|
- 2026-08-20 Read queue depths with `Find`, not `Scan`. `Scan` swaps
|
||||||
|
GORM's own trace recorder in for the logging adapter, and that
|
||||||
|
recorder does not implement `gorm.ParamsFilter`, so those statements
|
||||||
|
logged their bound values interpolated and bypassed the suppression
|
||||||
|
above. The two units gated green against a `next` that lacked the
|
||||||
|
other, and `next` went red when both landed
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/234)
|
||||||
|
- 2026-08-20 Render per-attempt delivery detail in the event log
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/202) and add replay of a
|
||||||
|
terminally failed delivery
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/203)
|
||||||
|
- 2026-08-20 Expose delivery metrics on `/metrics`
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/209) and document the
|
||||||
|
backup, restore and upgrade procedures
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/210)
|
||||||
|
- 2026-08-20 Add a `webhooker resetpw` subcommand and a bootstrap
|
||||||
|
banner. The admin bootstrap password was printed once among roughly
|
||||||
|
45 fx lines, and under `docker run -d` went to container logs subject
|
||||||
|
to rotation; there was no reset path at all, so recovery meant
|
||||||
|
hand-deleting the users row, documented nowhere. The password is read
|
||||||
|
from stdin or generated, never from argv where `/proc` would publish
|
||||||
|
it (https://git.eeqj.de/sneak/webhooker/issues/208)
|
||||||
|
- 2026-08-20 Add `ALLOWED_EGRESS_CIDRS`, an allowlist-only escape hatch
|
||||||
|
for the SSRF guard, so a self-hosted proxy can forward into the
|
||||||
|
operator's own network. The guard's always-blocked set cannot be
|
||||||
|
reopened by configuration
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/204)
|
||||||
|
- 2026-08-20 Harden operator-set target headers, which were carried
|
||||||
|
unsafely across a redirect
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/233)
|
||||||
|
- 2026-08-20 Add a target edit form with headers and timeout fields
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/127)
|
||||||
|
- 2026-08-18 Raise `script/test`'s per-package timeout from 30s to 90s,
|
||||||
|
matching the org-wide backstop. `go test` applies `-timeout` per
|
||||||
|
package, and `internal/handlers` had grown past the old budget: a
|
||||||
|
cache-defeated build failed outright at `GOMAXPROCS=4`, and every run
|
||||||
|
under deliberate host load breached 30s. The measurement table lives
|
||||||
|
in the script (#194)
|
||||||
|
- 2026-08-18 Re-sync `REPO_POLICIES.md` from `prompts`. The local copy
|
||||||
|
was stale and still mandated a 20s test target with a 30s timeout,
|
||||||
|
which the org replaced with a 60s cap and a 90s backstop. A synced
|
||||||
|
copy is not a source; reading it as one nearly produced a PR against
|
||||||
|
`prompts` proposing a change already merged there (#196)
|
||||||
|
- 2026-08-18 Report handler panics through the logger and answer 500.
|
||||||
|
chi v1.5.5's `Recoverer` scans for a `panic(0x` frame the runtime no
|
||||||
|
longer emits, then indexes `pkg[-1:]`, so it panicked inside its own
|
||||||
|
stack printer before writing a byte: the recovery never ran, the
|
||||||
|
client got a dropped connection instead of a 500, and the original
|
||||||
|
panic was lost. A local middleware replaces it, bounded by
|
||||||
|
`MaxPanicLogLineBytes` (#187)
|
||||||
|
- 2026-08-18 Route GORM's logger through `slog` and bound it. Every
|
||||||
|
`gorm.Open` left `logger.Default` in place at `Warn` with
|
||||||
|
`IgnoreRecordNotFoundError` false, so **every record-not-found
|
||||||
|
printed the fully interpolated SQL to stdout** — including the
|
||||||
|
client-chosen path on `/webhook/{uuid}` and the submitted username on
|
||||||
|
the login form, at no level the operator set and outside
|
||||||
|
`internal/logger` entirely. Three call sites, not the two the issue
|
||||||
|
named (#178)
|
||||||
|
- 2026-08-18 Bound every `slog` line against client-chosen text. Eight
|
||||||
|
sites reachable unauthenticated, found by reading every `slog` call in
|
||||||
|
the tree rather than only the one reported; the budget moved to a
|
||||||
|
shared `internal/logfield` so no second truncation exists. `DEBUG`
|
||||||
|
being off by default is not a bound and is not treated as one (#176)
|
||||||
|
- 2026-08-18 Stop a slow host turning a login-guard test into a
|
||||||
|
segfault. A non-fatal `assert` on an acquire result was dereferenced
|
||||||
|
on the next line, so one timing miss killed the whole
|
||||||
|
`internal/middleware` binary and reddened CI for unrelated PRs. The
|
||||||
|
fix also removed a real production race — `acquire` could shed a
|
||||||
|
request with a slot standing free, because Go picks uniformly among
|
||||||
|
ready `select` cases (#186)
|
||||||
|
- 2026-08-18 Send the chi route pattern to Sentry rather than the
|
||||||
|
concrete path. The receiver's path carries the entrypoint capability
|
||||||
|
token, so every Sentry event from `/webhook/{uuid}` shipped a live
|
||||||
|
credential to a third party. Request `Data`, `QueryString`, `Cookies`
|
||||||
|
and `Env` are dropped and headers reduced to an allowlist (#179)
|
||||||
|
- 2026-08-18 Read form fields from the POST body only. `r.FormValue`
|
||||||
|
merges the query string, so a login could be driven by URL parameters
|
||||||
|
— putting the password somewhere that lands in access logs, proxy
|
||||||
|
logs and browser history (#160)
|
||||||
|
- 2026-08-18 Verify login credentials before spending rate-limit
|
||||||
|
budget, so a flood of wrong passwords cannot lock out the account it
|
||||||
|
is guessing at. The manager took this decision rather than stall the
|
||||||
|
queue; it is flagged on the issue for reversal (#150)
|
||||||
|
- 2026-08-18 Run all linting in Docker via `Dockerfile.lint`. Host lint
|
||||||
|
was wrong in both directions from version skew and shared caches.
|
||||||
|
`script/lint` asserts the summary line, because `--no-cache-filter`
|
||||||
|
silently ignores a stage name it does not match — the flag that makes
|
||||||
|
the gate meaningful fails open (#109)
|
||||||
|
- 2026-08-18 Serve an event's full stored body over HTTP. The list
|
||||||
|
query truncates for rendering, and that truncated value was the only
|
||||||
|
way to read a body, so the full payload was unreachable (#157)
|
||||||
|
- 2026-08-18 Bound the access log line against client-chosen text.
|
||||||
|
`internal/logfield` budgets by *encoded* bytes, not runes, so a
|
||||||
|
handler's JSON escaping cannot multiply a field past its allowance
|
||||||
|
(#146)
|
||||||
|
- 2026-08-18 Mark superseded CI commits `failure` rather than
|
||||||
|
`skipped`. A skipped run rolls up green, so a commit that was never
|
||||||
|
tested reported success (#152)
|
||||||
|
- 2026-08-18 Set `fx.StopTimeout` inside the container stop grace, so
|
||||||
|
shutdown hooks are bounded by a deadline the orchestrator will
|
||||||
|
actually honour rather than being killed mid-flush (#134)
|
||||||
|
- 2026-08-17 Bucket IPv6 rate-limit keys by `/64`. A single allocation
|
||||||
|
hands out 2^64 addresses, so per-address keying let one client mint
|
||||||
|
unlimited buckets. Manager decision, recorded on the issue (#125)
|
||||||
|
- 2026-08-17 Correct release-blocking README and startup-warning
|
||||||
|
inaccuracies, including claims about behaviour the code does not have
|
||||||
|
(#151)
|
||||||
|
- 2026-08-17 Fetch and verify Alpine.js at build time against
|
||||||
|
`static/vendor.sha256` instead of committing the minified blob, so
|
||||||
|
the dependency is pinned by hash rather than by trust (#145)
|
||||||
|
- 2026-08-17 Bound the event log's rendered bodies in the query itself,
|
||||||
|
so a large stored payload cannot be read into memory just to be
|
||||||
|
truncated for display (#135)
|
||||||
|
- 2026-08-17 Mask the `http` target's destination URL in the UI: it can
|
||||||
|
carry a bearer credential in its path or query, and was rendered
|
||||||
|
verbatim. Manager decision to mask unconditionally (#115)
|
||||||
|
- 2026-08-14 Bound shutdown hooks by their stop context, so a hook that
|
||||||
|
hangs cannot hold the process past its grace period (#102)
|
||||||
|
- 2026-08-14 Render templates via a buffer rather than the
|
||||||
|
`ResponseWriter`, so a template error part-way through cannot commit
|
||||||
|
a 200 and then fail — the response is written only once it is whole
|
||||||
|
(#123)
|
||||||
|
- 2026-08-14 Align the session codec's max-age with the 7-day absolute
|
||||||
|
cap. The codec accepted cookies the session layer considered expired,
|
||||||
|
so the cap was enforced in one place and not the other (#108)
|
||||||
|
- 2026-08-12 Warn when `TRUSTED_PROXIES` is empty in production, where
|
||||||
|
the safe default silently discards forwarded headers and every client
|
||||||
|
rate-limits as the proxy's address (#149)
|
||||||
|
- 2026-08-12 Bound the receiver rate limit per client IP across the
|
||||||
|
whole `/webhook/*` route. The existing limiter keyed on the request
|
||||||
|
path and `/webhook/{uuid}` matches any single segment, so a client
|
||||||
|
that invented a fresh path per request minted a fresh bucket per
|
||||||
|
request: the limit on the only unauthenticated endpoint bounded
|
||||||
|
nothing in aggregate, and every request still cost an entrypoint
|
||||||
|
lookup before it 404ed. An outer limiter keyed on the client address
|
||||||
|
alone now bounds that, chained in front of the unchanged
|
||||||
|
per-entrypoint limiter (#139)
|
||||||
|
- 2026-08-12 Correct release-blocking documentation inaccuracies: the
|
||||||
|
README promised manual redelivery in the present tense in three
|
||||||
|
places when nothing implements it (the same false claim also sat in
|
||||||
|
the doc comment that was its source text), the env table omitted
|
||||||
|
`RETENTION_SWEEP_INTERVAL`, and `TODO.md` itself omitted five landed
|
||||||
|
units (#141)
|
||||||
|
- 2026-08-12 Make the CI gate execute the checks it reports on. The
|
||||||
|
workflow now writes a build-context fingerprint before calling
|
||||||
|
`script/cibuild`, so a code commit invalidates the `COPY` layer of
|
||||||
|
the lint and builder stages while a docs-only commit still replays
|
||||||
|
from cache; a superseding run also rewrites the `failure` status
|
||||||
|
Gitea leaves on commits it cancelled and never tested. Verified by
|
||||||
|
pushing a deliberately broken test and watching CI go red (#119)
|
||||||
|
- 2026-08-12 Require a positive `RETENTION_SWEEP_INTERVAL`: a
|
||||||
|
non-positive value reached `time.NewTicker` in both the retention
|
||||||
|
reaper and the archive sweeper, panicking two goroutines with no
|
||||||
|
recover after startup had already reported success (#140)
|
||||||
|
- 2026-08-12 Bound the `X-Forwarded-For` scan's allocation to the hop
|
||||||
|
cap: the reverse walk cuts entries with `strings.LastIndexByte`
|
||||||
|
instead of joining and splitting, so a 1 MB header allocates 16 bytes
|
||||||
|
rather than 1.6 MB per request on the unauthenticated receiver.
|
||||||
|
Semantics proven unchanged by differential testing against the
|
||||||
|
previous implementation (#133)
|
||||||
|
- 2026-08-12 Cap the `X-Forwarded-For` hop walk at 64 entries, so an
|
||||||
|
attacker-supplied chain cannot burn unbounded CPU in the rate-limit
|
||||||
|
key function; running off the end falls back to the peer address
|
||||||
|
(#124)
|
||||||
|
- 2026-08-12 Gate forwarded-header trust behind a `TRUSTED_PROXIES` CIDR
|
||||||
|
list: all three rate limiters key on the connection's own address
|
||||||
|
unless the direct peer is a configured proxy, in which case
|
||||||
|
`X-Forwarded-For` is walked right to left for the first non-proxy hop.
|
||||||
|
Default trusts nothing, and a set-but-unparseable value aborts
|
||||||
|
startup. Before this, any client could mint a fresh bucket or drain
|
||||||
|
another's by rotating a spoofed header (#88)
|
||||||
|
- 2026-08-11 Web UI cleanup: nav terminology unified on Webhooks, the
|
||||||
|
Profile settings placeholder removed, a progressive-enhancement copy
|
||||||
|
button for the entrypoint URL, and retention form copy that states the
|
||||||
|
actual policy (deletion by the reaper, 0 retains forever) (#57)
|
||||||
|
- 2026-08-11 Mask the webhook credential in delivery errors and logs:
|
||||||
|
Go embeds the request URL in `*url.Error`, so every transport failure
|
||||||
|
persisted the full Slack webhook URL into the per-webhook event
|
||||||
|
database via `DeliveryResult.Error`, a field a future REST API would
|
||||||
|
have served. `maskURLError` drops path, query and userinfo while
|
||||||
|
preserving the wrapped cause, so `errors.Is`/`As` and `Timeout()`
|
||||||
|
still work and DNS, TLS and timeout failures still read differently
|
||||||
|
(#118)
|
||||||
|
- 2026-08-11 Rate-limit the public webhook receiver endpoint
|
||||||
|
(`RECEIVER_RATE_LIMIT`, default 120/min), keyed on client IP plus
|
||||||
|
entrypoint path so one entrypoint cannot exhaust another's budget;
|
||||||
|
over-limit requests get 429 with `Retry-After`. It was the one
|
||||||
|
unauthenticated, internet-facing endpoint with no limit at all (#64)
|
||||||
|
- 2026-08-11 Enforce the body size limit before CSRF parses the form:
|
||||||
|
`MaxBodySize` is now first in all four form-parsing route groups, so
|
||||||
|
an oversized request is rejected with 413 instead of being read in
|
||||||
|
full by the CSRF middleware before any cap applied (#90)
|
||||||
|
- 2026-08-11 Mask target config on the source detail page, which
|
||||||
|
rendered the stored blob verbatim and so exposed the Slack
|
||||||
|
incoming-webhook URL — a bearer credential that cannot be revoked
|
||||||
|
per-holder. Config reaches the template only as a `TargetView` of
|
||||||
|
labelled fields, and header values are rendered as a count (#113)
|
||||||
|
- 2026-08-11 Allow `retention_days` of 0 to mean retain forever, via a
|
||||||
|
sentinel written in `BeforeSave` so the GORM column default cannot
|
||||||
|
win the race. Also bounds the reaper's cutoff arithmetic: day counts
|
||||||
|
above 106751 overflowed `time.Duration` and wrapped the cutoff into
|
||||||
|
the future, where every row matched and the sweep deleted everything
|
||||||
|
(#79)
|
||||||
|
- 2026-08-09 Inactivity-based session timeout: sliding idle expiry
|
||||||
|
(`SESSION_IDLE_TIMEOUT`, default `24h`) refreshed on authenticated
|
||||||
|
requests, with the 7-day absolute cap kept as an independent
|
||||||
|
backstop that activity never extends (#66)
|
||||||
|
- 2026-08-09 Restart recovery and the 60s retry sweep terminally fail an
|
||||||
|
orphaned `retrying` delivery whose target type no longer supports
|
||||||
|
retries, recording a `DeliveryResult` with the reason instead of
|
||||||
|
leaving the delivery stuck forever (#82)
|
||||||
|
- 2026-08-09 Root the delivery engine's worker pool and the retention
|
||||||
|
reaper's sweep loop at `context.Background()` rather than the fx
|
||||||
|
`OnStart` hook context (#97), which carries fx's 15s start timeout and
|
||||||
|
killed both roughly fifteen seconds after boot: the proxy silently
|
||||||
|
stopped delivering webhooks entirely, and the reaper never ran a
|
||||||
|
single sweep under its default one-hour interval
|
||||||
|
- 2026-08-09 Archive writer lifecycle (#89): deleting a webhook (or its
|
||||||
|
last `database` target) evicts the cached archive writer and closes
|
||||||
|
its handle while deliberately leaving `archive-{webhookID}.db` on
|
||||||
|
disk, and a new `ArchiveSweeper` prunes idle archives on the existing
|
||||||
|
`RETENTION_SWEEP_INTERVAL` without ever creating an archive file
|
||||||
|
- 2026-08-09 Configuration parsing fails loudly on set-but-unparseable
|
||||||
|
environment values: `envInt` removed in favour of `envPositiveInt`
|
||||||
|
plus a `PORT` range check, `envBool` now parses with
|
||||||
|
`strconv.ParseBool`, and defaults apply only to unset variables (#80)
|
||||||
|
- 2026-08-07 Automatic event retention cleanup based on
|
||||||
|
`retention_days`, deleting expired events, deliveries, and delivery
|
||||||
|
results from each per-webhook event database (#63)
|
||||||
|
- 2026-08-07 Update golangci-lint to v2.12.2 (Docker image digest in
|
||||||
|
`Dockerfile`, release-archive sha256 pins in `script/bootstrap`),
|
||||||
|
adopt the canonical `.golangci.yml` (v2 `linters.settings` layout so
|
||||||
|
`lll`/`funlen`/`cyclop`/`dupl` thresholds actually apply), and fix
|
||||||
|
all newly surfaced lint findings
|
||||||
|
- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints,
|
||||||
|
Makefile shims, README Entrypoints section
|
||||||
|
- 2026-03-25 pin golangci-lint Docker image for linting (#55)
|
||||||
|
- 2026-03-18 CSRF middleware detects TLS per-request, fixing login over
|
||||||
|
plain HTTP and behind reverse proxies (#54)
|
||||||
|
- 2026-03-17 root path redirects based on auth state (#52)
|
||||||
|
- 2026-03-17 CSRF protection, SSRF prevention for HTTP delivery targets
|
||||||
|
with DNS rebinding defense, and per-IP login rate limiting (#42)
|
||||||
|
- 2026-03-17 Slack target type for incoming webhook notifications (#47)
|
||||||
|
- 2026-03-17 Dockerfile absolute paths and static linking (#49);
|
||||||
|
absolute dev DATA_DIR default and clarified env docs (#46)
|
||||||
|
- 2026-03-05 security headers middleware, session regeneration on
|
||||||
|
login, request body size limits (#41)
|
||||||
|
- 2026-03-04 tests for delivery, middleware, and session packages
|
||||||
|
(#32); removed the build-architecture global (#31)
|
||||||
|
- 2026-03-04 1.0 MVP merge: Webhook/Entrypoint/Target rename, core
|
||||||
|
delivery engine with bounded worker pool and circuit breaker,
|
||||||
|
parallel fan-out, per-webhook event databases, management UI (#16)
|
||||||
|
- 2026-03-01 repo brought to REPO_POLICIES standards; TODO.md folded
|
||||||
|
into README (#6)
|
||||||
|
|
||||||
# Future Steps
|
# Future Steps
|
||||||
|
|
||||||
- Per-webhook rate limits, set in the web UI and enforced in the receiver
|
- Delivery status and retry management UI. Replay of a terminally
|
||||||
handler, on top of the receiver's existing limits, which `RECEIVER_RATE_LIMIT`
|
failed delivery and per-attempt detail already landed
|
||||||
sets for the whole server (see Rate Limiting in the README)
|
(https://git.eeqj.de/sneak/webhooker/issues/203,
|
||||||
- API key authentication for programmatic access (the `APIKey` model exists, but
|
https://git.eeqj.de/sneak/webhooker/issues/202)
|
||||||
nothing creates a key and no middleware checks one)
|
- Per-webhook rate limiting in the receiver handler (per-webhook config
|
||||||
|
plus handler enforcement; global limits must not apply to receiver
|
||||||
|
endpoints)
|
||||||
|
- API key authentication for programmatic access (APIKey model exists;
|
||||||
|
Bearer token middleware does not)
|
||||||
- REST API v1
|
- REST API v1
|
||||||
- CRUD for webhooks, entrypoints, targets
|
- CRUD for webhooks, entrypoints, targets
|
||||||
- event viewing and filtering endpoints
|
- event viewing and filtering endpoints
|
||||||
- event redelivery endpoint
|
- event redelivery endpoint
|
||||||
- OpenAPI specification
|
- OpenAPI specification
|
||||||
- Analytics in the web UI: response times summarised per webhook or per target,
|
- Analytics dashboard: success rates, response times, volume
|
||||||
and figures for all webhooks together. The webhook page already shows event,
|
- A remember-me option at login
|
||||||
delivery and failure counts and failure percentages, the target list counts
|
- Password reset flow for a forgotten password over the web. The
|
||||||
each target's deliveries, the event log and each event's page show every
|
authenticated password *change* flow already landed, and a lost
|
||||||
attempt's duration, and `/metrics` carries server-wide counts and delivery
|
password is recoverable from the console with `webhooker resetpw`
|
||||||
durations
|
|
||||||
- A remember-me option at login. Every session already survives a browser
|
|
||||||
restart: it lasts up to 7 days, and ends sooner after `SESSION_IDLE_TIMEOUT`
|
|
||||||
(default 24 hours) without activity
|
|
||||||
- Password reset flow for a forgotten password over the web. The authenticated
|
|
||||||
password _change_ flow already landed, and a lost password is recoverable from
|
|
||||||
the console with `webhooker resetpw`
|
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/208)
|
(https://git.eeqj.de/sneak/webhooker/issues/208)
|
||||||
- Later, nice to have
|
- Later, nice to have
|
||||||
- email delivery target type
|
- email delivery target type
|
||||||
- SNS and S3 delivery targets
|
- SNS and S3 delivery targets
|
||||||
- data transformations (e.g. a configurable Slack message format; the Slack
|
- data transformations (e.g. webhook to Slack message formatting)
|
||||||
target sends one fixed format today)
|
- JSONL file delivery with periodic S3 upload
|
||||||
- JSONL file delivery with periodic S3 upload (the `database` target already
|
- webhook event search and filtering
|
||||||
archives events to SQLite files, rotated hourly, daily or monthly if set,
|
- multi-user with role-based access control
|
||||||
and the webhook page offers them as one gzipped JSON download)
|
|
||||||
- webhook event search, and filtering beyond the event log's failed and
|
|
||||||
pending views
|
|
||||||
- multi-user with role-based access control (each webhook already belongs to
|
|
||||||
one account, but there is only the account made at first boot and no way
|
|
||||||
to add another)
|
|
||||||
|
|||||||
@@ -4,7 +4,6 @@ package main
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"log/slog"
|
|
||||||
"os"
|
"os"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -188,9 +187,6 @@ func newApp() *fx.App {
|
|||||||
fx.Provide(
|
fx.Provide(
|
||||||
globals.New,
|
globals.New,
|
||||||
logger.New,
|
logger.New,
|
||||||
// The plain logger the session, the middleware and the
|
|
||||||
// webhook database manager take.
|
|
||||||
func(l *logger.Logger) *slog.Logger { return l.Get() },
|
|
||||||
config.New,
|
config.New,
|
||||||
database.New,
|
database.New,
|
||||||
database.NewWebhookDBManager,
|
database.NewWebhookDBManager,
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ import (
|
|||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"sneak.berlin/go/webhooker/internal/config/configtest"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/datadir"
|
"sneak.berlin/go/webhooker/internal/datadir"
|
||||||
"sneak.berlin/go/webhooker/internal/resetpw"
|
"sneak.berlin/go/webhooker/internal/resetpw"
|
||||||
"sneak.berlin/go/webhooker/internal/server"
|
"sneak.berlin/go/webhooker/internal/server"
|
||||||
@@ -37,7 +37,7 @@ const dockerStopGrace = 10 * time.Second
|
|||||||
// fx.New applies options before it executes invokes, so the timeout
|
// fx.New applies options before it executes invokes, so the timeout
|
||||||
// is set whether or not the graph itself can be constructed here.
|
// is set whether or not the graph itself can be constructed here.
|
||||||
func TestNewApp_StopTimeout(t *testing.T) {
|
func TestNewApp_StopTimeout(t *testing.T) {
|
||||||
configtest.ClearEnv(t)
|
config.ClearEnvForTest(t)
|
||||||
t.Setenv("DATA_DIR", t.TempDir())
|
t.Setenv("DATA_DIR", t.TempDir())
|
||||||
|
|
||||||
got := newApp().StopTimeout()
|
got := newApp().StopTimeout()
|
||||||
@@ -75,7 +75,7 @@ func freePort(t *testing.T) int {
|
|||||||
// anything is built, and the run of logger.New, which happens before
|
// anything is built, and the run of logger.New, which happens before
|
||||||
// the configuration sets the level.
|
// the configuration sets the level.
|
||||||
func TestNewApp_SendsFxEventsToTheLogger(t *testing.T) {
|
func TestNewApp_SendsFxEventsToTheLogger(t *testing.T) {
|
||||||
configtest.ClearEnv(t)
|
config.ClearEnvForTest(t)
|
||||||
t.Setenv("DATA_DIR", t.TempDir())
|
t.Setenv("DATA_DIR", t.TempDir())
|
||||||
t.Setenv("PORT", strconv.Itoa(freePort(t)))
|
t.Setenv("PORT", strconv.Itoa(freePort(t)))
|
||||||
t.Setenv("DEBUG", "true")
|
t.Setenv("DEBUG", "true")
|
||||||
|
|||||||
@@ -1,18 +0,0 @@
|
|||||||
// ESLint configuration for static/js/. script/lint and the image build run
|
|
||||||
// ESLint in the Dockerfile's js-lint stage, never on the host.
|
|
||||||
//
|
|
||||||
// The rules are the ones the JavaScript styleguide linked from
|
|
||||||
// REPO_POLICIES.md states that a linter can check: const for everything,
|
|
||||||
// let only for a variable that is reassigned, never var.
|
|
||||||
export default [
|
|
||||||
// Alpine.js, extracted from 3p/ by make assets; not ours to lint.
|
|
||||||
{ ignores: ["static/js/alpine.min.js"] },
|
|
||||||
{
|
|
||||||
// The pages load static/js/app.js as a classic script, not a module.
|
|
||||||
languageOptions: { sourceType: "script" },
|
|
||||||
rules: {
|
|
||||||
"no-var": "error",
|
|
||||||
"prefer-const": "error",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
];
|
|
||||||
@@ -22,6 +22,7 @@ require (
|
|||||||
github.com/stretchr/testify v1.11.1
|
github.com/stretchr/testify v1.11.1
|
||||||
go.uber.org/fx v1.24.0
|
go.uber.org/fx v1.24.0
|
||||||
golang.org/x/crypto v0.38.0
|
golang.org/x/crypto v0.38.0
|
||||||
|
gopkg.in/yaml.v3 v3.0.1
|
||||||
gorm.io/driver/sqlite v1.5.4
|
gorm.io/driver/sqlite v1.5.4
|
||||||
gorm.io/gorm v1.25.5
|
gorm.io/gorm v1.25.5
|
||||||
modernc.org/sqlite v1.28.0
|
modernc.org/sqlite v1.28.0
|
||||||
@@ -58,7 +59,6 @@ require (
|
|||||||
golang.org/x/text v0.25.0 // indirect
|
golang.org/x/text v0.25.0 // indirect
|
||||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
|
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
|
||||||
google.golang.org/protobuf v1.31.0 // indirect
|
google.golang.org/protobuf v1.31.0 // indirect
|
||||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
|
||||||
lukechampine.com/uint128 v1.2.0 // indirect
|
lukechampine.com/uint128 v1.2.0 // indirect
|
||||||
modernc.org/cc/v3 v3.40.0 // indirect
|
modernc.org/cc/v3 v3.40.0 // indirect
|
||||||
modernc.org/ccgo/v3 v3.16.13 // indirect
|
modernc.org/ccgo/v3 v3.16.13 // indirect
|
||||||
|
|||||||
@@ -0,0 +1,387 @@
|
|||||||
|
package ciscript_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"maps"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"gopkg.in/yaml.v3"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// supersededDesc is the description script/ci-mark-superseded
|
||||||
|
// writes, and the one an earlier revision of it wrote alongside a
|
||||||
|
// `skipped` state.
|
||||||
|
supersededDesc = "Superseded by a newer commit; never tested"
|
||||||
|
|
||||||
|
// liveContext is the commit-status context Gitea uses for this
|
||||||
|
// repository's runs, as seen in its API. The script derives it from
|
||||||
|
// the workflow and job names rather than hardcoding it; the
|
||||||
|
// derivation is checked against this value below.
|
||||||
|
liveContext = "check / check (push)"
|
||||||
|
|
||||||
|
scriptPath = "../../script/ci-mark-superseded"
|
||||||
|
workflow = "../../.gitea/workflows/check.yml"
|
||||||
|
|
||||||
|
// failure is the only state that neither folds into a combined
|
||||||
|
// `success` (as `skipped` does) nor blocks the commit forever (as
|
||||||
|
// `pending` does).
|
||||||
|
failure = "failure"
|
||||||
|
)
|
||||||
|
|
||||||
|
// repo is a throwaway git history: parent is the commit a run would be
|
||||||
|
// cancelled on, head the commit that superseded it.
|
||||||
|
type repo struct {
|
||||||
|
dir string
|
||||||
|
head string
|
||||||
|
parent string
|
||||||
|
}
|
||||||
|
|
||||||
|
// scriptEnv is the run identity the Gitea runner exports and the script
|
||||||
|
// builds its context string from.
|
||||||
|
type scriptEnv struct {
|
||||||
|
workflow string
|
||||||
|
job string
|
||||||
|
event string
|
||||||
|
}
|
||||||
|
|
||||||
|
func defaultEnv() scriptEnv {
|
||||||
|
return scriptEnv{workflow: "check", job: "check", event: "push"}
|
||||||
|
}
|
||||||
|
|
||||||
|
func cancelled() commitStatus {
|
||||||
|
return commitStatus{
|
||||||
|
Context: liveContext,
|
||||||
|
Status: failure,
|
||||||
|
Description: "Has been cancelled",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func running() commitStatus {
|
||||||
|
return commitStatus{
|
||||||
|
Context: liveContext,
|
||||||
|
Status: "pending",
|
||||||
|
Description: "Has started running",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMarkSuperseded(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
cases := map[string]struct {
|
||||||
|
parent commitStatus
|
||||||
|
wantMark bool
|
||||||
|
}{
|
||||||
|
"a cancelled run is marked": {
|
||||||
|
parent: cancelled(),
|
||||||
|
wantMark: true,
|
||||||
|
},
|
||||||
|
"a laundered skipped status is marked": {
|
||||||
|
parent: commitStatus{
|
||||||
|
Context: liveContext,
|
||||||
|
Status: "skipped",
|
||||||
|
Description: supersededDesc,
|
||||||
|
},
|
||||||
|
wantMark: true,
|
||||||
|
},
|
||||||
|
"a genuine failure is left alone": {
|
||||||
|
parent: commitStatus{
|
||||||
|
Context: liveContext,
|
||||||
|
Status: failure,
|
||||||
|
Description: "Failing after 3m1s",
|
||||||
|
},
|
||||||
|
wantMark: false,
|
||||||
|
},
|
||||||
|
"a passing run is left alone": {
|
||||||
|
parent: commitStatus{
|
||||||
|
Context: liveContext,
|
||||||
|
Status: "success",
|
||||||
|
Description: "Successful in 2m52s",
|
||||||
|
},
|
||||||
|
wantMark: false,
|
||||||
|
},
|
||||||
|
"another context is left alone": {
|
||||||
|
parent: commitStatus{
|
||||||
|
Context: "other / other (push)",
|
||||||
|
Status: failure,
|
||||||
|
Description: "Has been cancelled",
|
||||||
|
},
|
||||||
|
wantMark: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, tc := range cases {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
requireTools(t)
|
||||||
|
|
||||||
|
history := newRepo(t)
|
||||||
|
fake, api := newFakeGitea(t)
|
||||||
|
fake.setStatus(history.head, running())
|
||||||
|
fake.setStatus(history.parent, tc.parent)
|
||||||
|
|
||||||
|
out, err := runScript(t, history, api, defaultEnv())
|
||||||
|
require.NoError(t, err, out)
|
||||||
|
|
||||||
|
posted := fake.postedFor(history.parent)
|
||||||
|
if !tc.wantMark {
|
||||||
|
require.Empty(t, posted)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
require.Equal(t, []postedStatus{{
|
||||||
|
Context: liveContext,
|
||||||
|
// Not `skipped`: Gitea's combined status folds
|
||||||
|
// that into `success`, which is what made a
|
||||||
|
// never-tested commit read green.
|
||||||
|
State: failure,
|
||||||
|
Description: supersededDesc,
|
||||||
|
}}, posted)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A second run must not rewrite what the first one wrote, or every
|
||||||
|
// later push would post a duplicate status.
|
||||||
|
func TestMarkSupersededIsIdempotent(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
requireTools(t)
|
||||||
|
|
||||||
|
history := newRepo(t)
|
||||||
|
fake, api := newFakeGitea(t)
|
||||||
|
fake.setStatus(history.head, running())
|
||||||
|
fake.setStatus(history.parent, cancelled())
|
||||||
|
|
||||||
|
for range 2 {
|
||||||
|
out, err := runScript(t, history, api, defaultEnv())
|
||||||
|
require.NoError(t, err, out)
|
||||||
|
}
|
||||||
|
|
||||||
|
require.Len(t, fake.postedFor(history.parent), 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Renaming the workflow or the job changes the context string Gitea
|
||||||
|
// uses. The script must say so instead of quietly matching nothing.
|
||||||
|
func TestMarkSupersededRejectsAnUnknownContext(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
requireTools(t)
|
||||||
|
|
||||||
|
history := newRepo(t)
|
||||||
|
fake, api := newFakeGitea(t)
|
||||||
|
fake.setStatus(history.head, running())
|
||||||
|
fake.setStatus(history.parent, cancelled())
|
||||||
|
|
||||||
|
env := defaultEnv()
|
||||||
|
env.job = "renamed"
|
||||||
|
|
||||||
|
out, err := runScript(t, history, api, env)
|
||||||
|
require.Error(t, err)
|
||||||
|
require.Contains(t, out, "renamed")
|
||||||
|
require.Contains(t, out, liveContext)
|
||||||
|
require.Empty(t, fake.postedFor(history.parent))
|
||||||
|
}
|
||||||
|
|
||||||
|
// ANCESTOR_LIMIT is a documented knob. A value that is set but unusable
|
||||||
|
// must abort: handing it to git and discarding the exit status left the
|
||||||
|
// walk empty and the step green, marking nothing.
|
||||||
|
func TestMarkSupersededRejectsAnUnparseableAncestorLimit(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
requireTools(t)
|
||||||
|
|
||||||
|
history := newRepo(t)
|
||||||
|
fake, api := newFakeGitea(t)
|
||||||
|
fake.setStatus(history.head, running())
|
||||||
|
fake.setStatus(history.parent, cancelled())
|
||||||
|
|
||||||
|
out, err := runScript(
|
||||||
|
t, history, api, defaultEnv(), "ANCESTOR_LIMIT=twenty",
|
||||||
|
)
|
||||||
|
require.Error(t, err)
|
||||||
|
require.Contains(t, out, "ANCESTOR_LIMIT")
|
||||||
|
require.Contains(t, out, "twenty")
|
||||||
|
require.Empty(t, fake.postedFor(history.parent))
|
||||||
|
}
|
||||||
|
|
||||||
|
// A status read that fails is not the same as a commit with nothing to
|
||||||
|
// do. Losing curl's exit status through a pipe made the two identical
|
||||||
|
// and left a laundered commit laundered with no signal.
|
||||||
|
func TestMarkSupersededFailsOnAnUnreadableAncestorStatus(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
requireTools(t)
|
||||||
|
|
||||||
|
history := newRepo(t)
|
||||||
|
fake, api := newFakeGitea(t)
|
||||||
|
fake.setStatus(history.head, running())
|
||||||
|
fake.setStatus(history.parent, cancelled())
|
||||||
|
fake.failStatusRead(history.parent)
|
||||||
|
|
||||||
|
out, err := runScript(t, history, api, defaultEnv())
|
||||||
|
require.Error(t, err)
|
||||||
|
require.Contains(t, out, history.parent)
|
||||||
|
require.Contains(t, out, "cannot read commit statuses")
|
||||||
|
require.Empty(t, fake.postedFor(history.parent))
|
||||||
|
}
|
||||||
|
|
||||||
|
// A shallow clone cannot resolve the parent, so it is indistinguishable
|
||||||
|
// from a root commit to rev-parse and the walk would exit 0 having
|
||||||
|
// marked nothing. It must abort instead: dropping `fetch-depth: 0` from
|
||||||
|
// the checkout step is one edit, and a silent no-op there restores the
|
||||||
|
// false-green bug this script exists to prevent.
|
||||||
|
func TestMarkSupersededRejectsAShallowRepository(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
requireTools(t)
|
||||||
|
|
||||||
|
history := shallowClone(t, newRepo(t))
|
||||||
|
fake, api := newFakeGitea(t)
|
||||||
|
fake.setStatus(history.head, running())
|
||||||
|
fake.setStatus(history.parent, cancelled())
|
||||||
|
|
||||||
|
out, err := runScript(t, history, api, defaultEnv())
|
||||||
|
require.Error(t, err)
|
||||||
|
require.Contains(t, out, "shallow repository")
|
||||||
|
require.Empty(t, fake.postedFor(history.parent))
|
||||||
|
require.Empty(t, fake.postedFor(history.head))
|
||||||
|
}
|
||||||
|
|
||||||
|
// shallowClone returns the same history as a depth-1 clone. The `file://`
|
||||||
|
// URL is required: git ignores --depth for a plain local path.
|
||||||
|
func shallowClone(t *testing.T, history repo) repo {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
|
||||||
|
//nolint:gosec // fixed argv, arguments are test-local paths
|
||||||
|
cmd := exec.CommandContext(t.Context(), "git", "clone", "-q",
|
||||||
|
"--depth=1", "file://"+history.dir, dir)
|
||||||
|
|
||||||
|
out, err := cmd.CombinedOutput()
|
||||||
|
require.NoError(t, err, string(out))
|
||||||
|
|
||||||
|
return repo{dir: dir, head: history.head, parent: history.parent}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The derived context must equal the one Gitea actually uses, which is
|
||||||
|
// built from the same workflow and job names.
|
||||||
|
func TestDerivedContextMatchesGitea(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
requireTools(t)
|
||||||
|
|
||||||
|
name, job := workflowIdentity(t)
|
||||||
|
|
||||||
|
history := newRepo(t)
|
||||||
|
fake, api := newFakeGitea(t)
|
||||||
|
fake.setStatus(history.head, running())
|
||||||
|
fake.setStatus(history.parent, cancelled())
|
||||||
|
|
||||||
|
out, err := runScript(t, history, api, scriptEnv{
|
||||||
|
workflow: name,
|
||||||
|
job: job,
|
||||||
|
event: "push",
|
||||||
|
})
|
||||||
|
require.NoError(t, err, out)
|
||||||
|
|
||||||
|
posted := fake.postedFor(history.parent)
|
||||||
|
require.Len(t, posted, 1)
|
||||||
|
require.Equal(t, liveContext, posted[0].Context)
|
||||||
|
}
|
||||||
|
|
||||||
|
// workflowIdentity reads the workflow name and its single job id out of
|
||||||
|
// the checked-in workflow file.
|
||||||
|
func workflowIdentity(t *testing.T) (string, string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
raw, err := os.ReadFile(workflow)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
var parsed struct {
|
||||||
|
Name string `yaml:"name"`
|
||||||
|
Jobs map[string]any `yaml:"jobs"`
|
||||||
|
}
|
||||||
|
|
||||||
|
require.NoError(t, yaml.Unmarshal(raw, &parsed))
|
||||||
|
|
||||||
|
jobs := slices.Collect(maps.Keys(parsed.Jobs))
|
||||||
|
require.Len(t, jobs, 1)
|
||||||
|
|
||||||
|
return parsed.Name, jobs[0]
|
||||||
|
}
|
||||||
|
|
||||||
|
func runScript(
|
||||||
|
t *testing.T, history repo, api string, env scriptEnv,
|
||||||
|
extra ...string,
|
||||||
|
) (string, error) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
script, err := filepath.Abs(scriptPath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
//nolint:gosec // fixed argv, repo-local script under test
|
||||||
|
cmd := exec.CommandContext(t.Context(), "sh", script)
|
||||||
|
cmd.Dir = history.dir
|
||||||
|
cmd.Env = append(os.Environ(),
|
||||||
|
"GITHUB_API_URL="+api,
|
||||||
|
"GITHUB_REPOSITORY=sneak/webhooker",
|
||||||
|
"GITHUB_SHA="+history.head,
|
||||||
|
"GITHUB_WORKFLOW="+env.workflow,
|
||||||
|
"GITHUB_JOB="+env.job,
|
||||||
|
"GITHUB_EVENT_NAME="+env.event,
|
||||||
|
"GITEA_TOKEN=test-token",
|
||||||
|
)
|
||||||
|
cmd.Env = append(cmd.Env, extra...)
|
||||||
|
|
||||||
|
out, err := cmd.CombinedOutput()
|
||||||
|
|
||||||
|
return string(out), err
|
||||||
|
}
|
||||||
|
|
||||||
|
func newRepo(t *testing.T) repo {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
|
||||||
|
git := func(args ...string) string {
|
||||||
|
//nolint:gosec // fixed argv, arguments are test constants
|
||||||
|
cmd := exec.CommandContext(t.Context(), "git", args...)
|
||||||
|
cmd.Dir = dir
|
||||||
|
|
||||||
|
out, err := cmd.CombinedOutput()
|
||||||
|
require.NoError(t, err, string(out))
|
||||||
|
|
||||||
|
return strings.TrimSpace(string(out))
|
||||||
|
}
|
||||||
|
|
||||||
|
commit := func(message string) string {
|
||||||
|
git(
|
||||||
|
"-c", "user.email=ci@example.invalid",
|
||||||
|
"-c", "user.name=ci",
|
||||||
|
"-c", "commit.gpgsign=false",
|
||||||
|
"commit", "-q", "--allow-empty", "-m", message,
|
||||||
|
)
|
||||||
|
|
||||||
|
return git("rev-parse", "HEAD")
|
||||||
|
}
|
||||||
|
|
||||||
|
git("init", "-q", "-b", "main")
|
||||||
|
|
||||||
|
parent := commit("parent")
|
||||||
|
head := commit("head")
|
||||||
|
|
||||||
|
return repo{dir: dir, head: head, parent: parent}
|
||||||
|
}
|
||||||
|
|
||||||
|
func requireTools(t *testing.T) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
for _, tool := range []string{"sh", "git", "curl", "jq"} {
|
||||||
|
_, err := exec.LookPath(tool)
|
||||||
|
if err != nil {
|
||||||
|
t.Skipf("%s is not installed: %v", tool, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
// Package ciscript holds the tests for the repository's CI shell
|
||||||
|
// scripts in script/. It carries no runtime code: the scripts run on
|
||||||
|
// the CI runner, not inside the binary, but their behaviour still has
|
||||||
|
// to be verified by the test suite.
|
||||||
|
//
|
||||||
|
// The scripts under test are outside the Go build graph, so `go test`'s
|
||||||
|
// result cache serves a stale PASS when only a script changed: run the
|
||||||
|
// container build, or GOFLAGS=-count=1, to trust a result here after
|
||||||
|
// editing script/.
|
||||||
|
package ciscript
|
||||||
@@ -0,0 +1,162 @@
|
|||||||
|
package ciscript_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// commitStatus is the part of an entry in Gitea's combined-status
|
||||||
|
// response that script/ci-mark-superseded reads.
|
||||||
|
type commitStatus struct {
|
||||||
|
Context string `json:"context"`
|
||||||
|
Status string `json:"status"`
|
||||||
|
Description string `json:"description"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// postedStatus is the part of a create-status request body the script
|
||||||
|
// writes.
|
||||||
|
type postedStatus struct {
|
||||||
|
Context string `json:"context"`
|
||||||
|
State string `json:"state"`
|
||||||
|
Description string `json:"description"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// fakeGitea serves the two endpoints the script talks to. Like Gitea,
|
||||||
|
// the newest status for a context replaces the previous one, so a
|
||||||
|
// second run of the script sees what the first one wrote.
|
||||||
|
type fakeGitea struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
statuses map[string][]commitStatus
|
||||||
|
posted map[string][]postedStatus
|
||||||
|
// failRead is a commit whose combined-status read answers HTTP
|
||||||
|
// 500, standing in for a status API that is down.
|
||||||
|
failRead string
|
||||||
|
}
|
||||||
|
|
||||||
|
// newFakeGitea returns the fake and the base URL to hand the script as
|
||||||
|
// GITHUB_API_URL.
|
||||||
|
func newFakeGitea(t *testing.T) (*fakeGitea, string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
fake := &fakeGitea{
|
||||||
|
mu: sync.Mutex{},
|
||||||
|
statuses: map[string][]commitStatus{},
|
||||||
|
posted: map[string][]postedStatus{},
|
||||||
|
failRead: "",
|
||||||
|
}
|
||||||
|
|
||||||
|
srv := httptest.NewServer(fake.routes())
|
||||||
|
t.Cleanup(srv.Close)
|
||||||
|
|
||||||
|
return fake, srv.URL
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeGitea) routes() http.Handler {
|
||||||
|
mux := http.NewServeMux()
|
||||||
|
|
||||||
|
mux.HandleFunc(
|
||||||
|
"GET /repos/{owner}/{repo}/commits/{sha}/status",
|
||||||
|
f.handleCombined,
|
||||||
|
)
|
||||||
|
mux.HandleFunc(
|
||||||
|
"POST /repos/{owner}/{repo}/statuses/{sha}",
|
||||||
|
f.handleCreate,
|
||||||
|
)
|
||||||
|
|
||||||
|
return mux
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeGitea) handleCombined(
|
||||||
|
w http.ResponseWriter, r *http.Request,
|
||||||
|
) {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
|
||||||
|
sha := r.PathValue("sha")
|
||||||
|
if f.failRead != "" && f.failRead == sha {
|
||||||
|
http.Error(w, "boom", http.StatusInternalServerError)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
body := struct {
|
||||||
|
Statuses []commitStatus `json:"statuses"`
|
||||||
|
}{Statuses: f.statuses[sha]}
|
||||||
|
|
||||||
|
payload, err := json.Marshal(body)
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
|
||||||
|
_, _ = w.Write(payload)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeGitea) handleCreate(w http.ResponseWriter, r *http.Request) {
|
||||||
|
var got postedStatus
|
||||||
|
|
||||||
|
err := json.NewDecoder(r.Body).Decode(&got)
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
sha := r.PathValue("sha")
|
||||||
|
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
|
||||||
|
f.posted[sha] = append(f.posted[sha], got)
|
||||||
|
f.replaceLocked(sha, commitStatus{
|
||||||
|
Context: got.Context,
|
||||||
|
Status: got.State,
|
||||||
|
Description: got.Description,
|
||||||
|
})
|
||||||
|
|
||||||
|
w.WriteHeader(http.StatusCreated)
|
||||||
|
}
|
||||||
|
|
||||||
|
// failStatusRead makes the combined-status read for one commit answer
|
||||||
|
// HTTP 500.
|
||||||
|
func (f *fakeGitea) failStatusRead(sha string) {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
|
||||||
|
f.failRead = sha
|
||||||
|
}
|
||||||
|
|
||||||
|
// setStatus gives a commit its latest status for a context.
|
||||||
|
func (f *fakeGitea) setStatus(sha string, status commitStatus) {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
|
||||||
|
f.replaceLocked(sha, status)
|
||||||
|
}
|
||||||
|
|
||||||
|
// postedFor returns the statuses the script created for a commit.
|
||||||
|
func (f *fakeGitea) postedFor(sha string) []postedStatus {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
|
||||||
|
return append([]postedStatus(nil), f.posted[sha]...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// replaceLocked requires f.mu.
|
||||||
|
func (f *fakeGitea) replaceLocked(sha string, status commitStatus) {
|
||||||
|
for i, existing := range f.statuses[sha] {
|
||||||
|
if existing.Context == status.Context {
|
||||||
|
f.statuses[sha][i] = status
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
f.statuses[sha] = append(f.statuses[sha], status)
|
||||||
|
}
|
||||||
@@ -68,10 +68,6 @@ const (
|
|||||||
// authenticated activity before it expires.
|
// authenticated activity before it expires.
|
||||||
defaultSessionIdleTimeout = 24 * time.Hour
|
defaultSessionIdleTimeout = 24 * time.Hour
|
||||||
|
|
||||||
// defaultHSTSMaxAge is the max-age of the Strict-Transport-Security
|
|
||||||
// header when HSTS_MAX_AGE is unset: ten years.
|
|
||||||
defaultHSTSMaxAge = 87600 * time.Hour
|
|
||||||
|
|
||||||
// defaultReceiverRateLimit is the default number of requests
|
// defaultReceiverRateLimit is the default number of requests
|
||||||
// per minute each client IP may send to a single webhook
|
// per minute each client IP may send to a single webhook
|
||||||
// receiver entrypoint. Generous for legitimate webhook
|
// receiver entrypoint. Generous for legitimate webhook
|
||||||
@@ -124,10 +120,6 @@ var ErrInvalidBindAddress = errors.New("invalid bind address")
|
|||||||
// the Sentry SDK cannot parse as a DSN.
|
// the Sentry SDK cannot parse as a DSN.
|
||||||
var ErrInvalidSentryDSN = errors.New("invalid Sentry DSN")
|
var ErrInvalidSentryDSN = errors.New("invalid Sentry DSN")
|
||||||
|
|
||||||
// ErrInvalidHSTSMaxAge is returned when HSTS_MAX_AGE is set to a
|
|
||||||
// negative duration or one that is not a whole number of seconds.
|
|
||||||
var ErrInvalidHSTSMaxAge = errors.New("invalid HSTS max-age")
|
|
||||||
|
|
||||||
// ErrDotEnvUnreadable is returned when the optional .env file exists
|
// ErrDotEnvUnreadable is returned when the optional .env file exists
|
||||||
// but cannot be read or parsed. A file that is not there is not an
|
// but cannot be read or parsed. A file that is not there is not an
|
||||||
// error; a file that is there and broken is.
|
// error; a file that is there and broken is.
|
||||||
@@ -177,10 +169,6 @@ type Config struct {
|
|||||||
// which a session expires. Non-positive disables idle expiry.
|
// which a session expires. Non-positive disables idle expiry.
|
||||||
SessionIdleTimeout time.Duration
|
SessionIdleTimeout time.Duration
|
||||||
|
|
||||||
// HSTSMaxAge is the max-age of the Strict-Transport-Security
|
|
||||||
// header. Zero means the header is not sent at all.
|
|
||||||
HSTSMaxAge time.Duration
|
|
||||||
|
|
||||||
// ReceiverRateLimit is the number of requests per minute each
|
// ReceiverRateLimit is the number of requests per minute each
|
||||||
// client IP may send to a single webhook receiver entrypoint.
|
// client IP may send to a single webhook receiver entrypoint.
|
||||||
ReceiverRateLimit int
|
ReceiverRateLimit int
|
||||||
@@ -636,27 +624,6 @@ func resolveEnvironment() (string, error) {
|
|||||||
return environment, nil
|
return environment, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// resolveHSTSMaxAge reads HSTS_MAX_AGE. Zero is allowed and turns the
|
|
||||||
// Strict-Transport-Security header off. A fraction of a second is an
|
|
||||||
// error rather than rounded, because the header carries whole seconds
|
|
||||||
// and would otherwise say something other than what was set.
|
|
||||||
func resolveHSTSMaxAge() (time.Duration, error) {
|
|
||||||
maxAge, err := envDuration("HSTS_MAX_AGE", defaultHSTSMaxAge)
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if maxAge < 0 || maxAge%time.Second != 0 {
|
|
||||||
return 0, fmt.Errorf(
|
|
||||||
"%w: HSTS_MAX_AGE must be zero or a positive whole "+
|
|
||||||
"number of seconds, got %s",
|
|
||||||
ErrInvalidHSTSMaxAge, maxAge,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return maxAge, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// resolveListener reads the two variables that describe the HTTP
|
// resolveListener reads the two variables that describe the HTTP
|
||||||
// listener: which port it claims and which address it claims it on.
|
// listener: which port it claims and which address it claims it on.
|
||||||
// They are read together because neither is meaningful alone, and
|
// They are read together because neither is meaningful alone, and
|
||||||
@@ -717,11 +684,6 @@ func loadFromEnv() (*Config, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
hstsMaxAge, err := resolveHSTSMaxAge()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
receiverRateLimit, err := envPositiveInt(
|
receiverRateLimit, err := envPositiveInt(
|
||||||
"RECEIVER_RATE_LIMIT",
|
"RECEIVER_RATE_LIMIT",
|
||||||
defaultReceiverRateLimit,
|
defaultReceiverRateLimit,
|
||||||
@@ -761,7 +723,6 @@ func loadFromEnv() (*Config, error) {
|
|||||||
SentryDSN: sentryDSN,
|
SentryDSN: sentryDSN,
|
||||||
RetentionSweepInterval: retentionSweepInterval,
|
RetentionSweepInterval: retentionSweepInterval,
|
||||||
SessionIdleTimeout: sessionIdleTimeout,
|
SessionIdleTimeout: sessionIdleTimeout,
|
||||||
HSTSMaxAge: hstsMaxAge,
|
|
||||||
ReceiverRateLimit: receiverRateLimit,
|
ReceiverRateLimit: receiverRateLimit,
|
||||||
TrustedProxies: trustedProxies,
|
TrustedProxies: trustedProxies,
|
||||||
AllowedEgressCIDRs: allowedEgressCIDRs,
|
AllowedEgressCIDRs: allowedEgressCIDRs,
|
||||||
@@ -846,7 +807,6 @@ func New(lc fx.Lifecycle, params ConfigParams) (*Config, error) {
|
|||||||
// disables idle expiry entirely, and that is worth showing
|
// disables idle expiry entirely, and that is worth showing
|
||||||
// back to the operator.
|
// back to the operator.
|
||||||
"sessionIdleTimeout", s.SessionIdleTimeout.String(),
|
"sessionIdleTimeout", s.SessionIdleTimeout.String(),
|
||||||
"hstsMaxAge", s.HSTSMaxAge.String(),
|
|
||||||
"receiverRateLimit", s.ReceiverRateLimit,
|
"receiverRateLimit", s.ReceiverRateLimit,
|
||||||
"trustedProxies", len(s.TrustedProxies),
|
"trustedProxies", len(s.TrustedProxies),
|
||||||
"allowedEgressCIDRs", len(s.AllowedEgressCIDRs),
|
"allowedEgressCIDRs", len(s.AllowedEgressCIDRs),
|
||||||
|
|||||||
@@ -11,7 +11,6 @@ import (
|
|||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"go.uber.org/fx/fxtest"
|
"go.uber.org/fx/fxtest"
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/config/configtest"
|
|
||||||
"sneak.berlin/go/webhooker/internal/globals"
|
"sneak.berlin/go/webhooker/internal/globals"
|
||||||
"sneak.berlin/go/webhooker/internal/logger"
|
"sneak.berlin/go/webhooker/internal/logger"
|
||||||
)
|
)
|
||||||
@@ -71,7 +70,7 @@ func TestEnvironmentConfig(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
configtest.ClearEnv(t)
|
config.ClearEnvForTest(t)
|
||||||
|
|
||||||
if tt.envValue != "" {
|
if tt.envValue != "" {
|
||||||
t.Setenv(
|
t.Setenv(
|
||||||
@@ -197,7 +196,7 @@ func TestRetentionSweepInterval(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
configtest.ClearEnv(t)
|
config.ClearEnvForTest(t)
|
||||||
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
||||||
|
|
||||||
if tt.set {
|
if tt.set {
|
||||||
@@ -336,7 +335,7 @@ func TestSessionIdleTimeout(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
configtest.ClearEnv(t)
|
config.ClearEnvForTest(t)
|
||||||
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
||||||
|
|
||||||
if tt.set {
|
if tt.set {
|
||||||
@@ -379,83 +378,6 @@ func testSessionIdleTimeoutSuccess(
|
|||||||
assert.Equal(t, expected, cfg.SessionIdleTimeout)
|
assert.Equal(t, expected, cfg.SessionIdleTimeout)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHSTSMaxAge(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
set bool
|
|
||||||
value string
|
|
||||||
expectError bool
|
|
||||||
// sentinel, when set, must be wrapped by the startup
|
|
||||||
// error; every error case must additionally name the
|
|
||||||
// variable in its message.
|
|
||||||
sentinel error
|
|
||||||
expected time.Duration
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: caseUnsetUsesDefault,
|
|
||||||
set: false,
|
|
||||||
expected: 87600 * time.Hour,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: caseValidValueParsed,
|
|
||||||
set: true,
|
|
||||||
value: "720h",
|
|
||||||
expected: 720 * time.Hour,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
// Zero turns the header off; it is not an error.
|
|
||||||
name: "zero is accepted",
|
|
||||||
set: true,
|
|
||||||
value: "0",
|
|
||||||
expected: 0,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: caseUnparseableFails,
|
|
||||||
set: true,
|
|
||||||
value: "10 years",
|
|
||||||
expectError: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
// A whole number of seconds, so only the sign is wrong.
|
|
||||||
name: "negative whole seconds fail startup",
|
|
||||||
set: true,
|
|
||||||
value: "-1s",
|
|
||||||
expectError: true,
|
|
||||||
sentinel: config.ErrInvalidHSTSMaxAge,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "fraction of a second fails startup",
|
|
||||||
set: true,
|
|
||||||
value: "1500ms",
|
|
||||||
expectError: true,
|
|
||||||
sentinel: config.ErrInvalidHSTSMaxAge,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
|
||||||
// is incompatible with parallel subtests.
|
|
||||||
configtest.ClearEnv(t)
|
|
||||||
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
|
||||||
|
|
||||||
if tt.set {
|
|
||||||
t.Setenv("HSTS_MAX_AGE", tt.value)
|
|
||||||
}
|
|
||||||
|
|
||||||
if tt.expectError {
|
|
||||||
expectStartupErrorFor(t, "HSTS_MAX_AGE", tt.sentinel)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg, err := buildConfig(t)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, tt.expected, cfg.HSTSMaxAge)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDefaultDataDir(t *testing.T) {
|
func TestDefaultDataDir(t *testing.T) {
|
||||||
for _, env := range []string{"", "dev", "prod"} {
|
for _, env := range []string{"", "dev", "prod"} {
|
||||||
name := env
|
name := env
|
||||||
@@ -466,7 +388,7 @@ func TestDefaultDataDir(t *testing.T) {
|
|||||||
t.Run("env="+name, func(t *testing.T) {
|
t.Run("env="+name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
configtest.ClearEnv(t)
|
config.ClearEnvForTest(t)
|
||||||
|
|
||||||
if env != "" {
|
if env != "" {
|
||||||
t.Setenv("WEBHOOKER_ENVIRONMENT", env)
|
t.Setenv("WEBHOOKER_ENVIRONMENT", env)
|
||||||
@@ -511,7 +433,7 @@ func TestDataDirHelper(t *testing.T) {
|
|||||||
t.Run(name, func(t *testing.T) {
|
t.Run(name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
configtest.ClearEnv(t)
|
config.ClearEnvForTest(t)
|
||||||
|
|
||||||
if set != "" {
|
if set != "" {
|
||||||
t.Setenv("DATA_DIR", set)
|
t.Setenv("DATA_DIR", set)
|
||||||
@@ -576,7 +498,7 @@ func TestReceiverRateLimit(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
configtest.ClearEnv(t)
|
config.ClearEnvForTest(t)
|
||||||
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
||||||
|
|
||||||
if tt.set {
|
if tt.set {
|
||||||
@@ -692,7 +614,7 @@ func TestTrustedProxies(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
configtest.ClearEnv(t)
|
config.ClearEnvForTest(t)
|
||||||
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
||||||
|
|
||||||
if tt.set {
|
if tt.set {
|
||||||
@@ -803,7 +725,7 @@ func TestAllowedEgressCIDRs(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
configtest.ClearEnv(t)
|
config.ClearEnvForTest(t)
|
||||||
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
||||||
|
|
||||||
if tt.set {
|
if tt.set {
|
||||||
@@ -875,7 +797,7 @@ func TestEgressAllowlistWarning(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
configtest.ClearEnv(t)
|
config.ClearEnvForTest(t)
|
||||||
t.Setenv("WEBHOOKER_ENVIRONMENT", config.EnvironmentDev)
|
t.Setenv("WEBHOOKER_ENVIRONMENT", config.EnvironmentDev)
|
||||||
|
|
||||||
if tt.allowed != "" {
|
if tt.allowed != "" {
|
||||||
@@ -1011,7 +933,7 @@ func TestMetricsAuthConfig(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
configtest.ClearEnv(t)
|
config.ClearEnvForTest(t)
|
||||||
|
|
||||||
if tt.username.set {
|
if tt.username.set {
|
||||||
t.Setenv("METRICS_USERNAME", tt.username.value)
|
t.Setenv("METRICS_USERNAME", tt.username.value)
|
||||||
|
|||||||
@@ -8,7 +8,6 @@ import (
|
|||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/config/configtest"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// dotEnvKey is a throwaway variable name the .env tests write and
|
// dotEnvKey is a throwaway variable name the .env tests write and
|
||||||
@@ -40,9 +39,9 @@ func writeDotEnv(t *testing.T, contents string) string {
|
|||||||
// normally rather than be refused for a file it was never meant to
|
// normally rather than be refused for a file it was never meant to
|
||||||
// have.
|
// have.
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // ClearEnv uses t.Setenv.
|
//nolint:paralleltest // ClearEnvForTest uses t.Setenv.
|
||||||
func TestLoadDotEnv_MissingFileIsFine(t *testing.T) {
|
func TestLoadDotEnv_MissingFileIsFine(t *testing.T) {
|
||||||
configtest.ClearEnv(t)
|
config.ClearEnvForTest(t)
|
||||||
|
|
||||||
absent := filepath.Join(t.TempDir(), config.DotEnvPath)
|
absent := filepath.Join(t.TempDir(), config.DotEnvPath)
|
||||||
require.NoError(t, config.LoadDotEnvFileForTest(absent))
|
require.NoError(t, config.LoadDotEnvFileForTest(absent))
|
||||||
@@ -55,9 +54,9 @@ func TestLoadDotEnv_MissingFileIsFine(t *testing.T) {
|
|||||||
// reaches the environment, which is the whole reason the file is read
|
// reaches the environment, which is the whole reason the file is read
|
||||||
// at all.
|
// at all.
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // ClearEnv uses t.Setenv.
|
//nolint:paralleltest // ClearEnvForTest uses t.Setenv.
|
||||||
func TestLoadDotEnv_AppliesValues(t *testing.T) {
|
func TestLoadDotEnv_AppliesValues(t *testing.T) {
|
||||||
configtest.ClearEnv(t)
|
config.ClearEnvForTest(t)
|
||||||
|
|
||||||
path := writeDotEnv(t, "# a comment\n"+dotEnvKey+"=from-dot-env\n")
|
path := writeDotEnv(t, "# a comment\n"+dotEnvKey+"=from-dot-env\n")
|
||||||
|
|
||||||
@@ -83,9 +82,9 @@ func TestLoadDotEnv_RealEnvironmentWins(t *testing.T) {
|
|||||||
// reverts to its default; the process used to start that way with no
|
// reverts to its default; the process used to start that way with no
|
||||||
// log line naming the file at all.
|
// log line naming the file at all.
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // ClearEnv uses t.Setenv.
|
//nolint:paralleltest // ClearEnvForTest uses t.Setenv.
|
||||||
func TestLoadDotEnv_MalformedFileAborts(t *testing.T) {
|
func TestLoadDotEnv_MalformedFileAborts(t *testing.T) {
|
||||||
configtest.ClearEnv(t)
|
config.ClearEnvForTest(t)
|
||||||
|
|
||||||
path := writeDotEnv(
|
path := writeDotEnv(
|
||||||
t, malformedDotEnv+dotEnvKey+"=from-dot-env\n",
|
t, malformedDotEnv+dotEnvKey+"=from-dot-env\n",
|
||||||
@@ -133,7 +132,7 @@ func TestLoadDotEnv_UnreadableFileAborts(t *testing.T) {
|
|||||||
//
|
//
|
||||||
//nolint:paralleltest // t.Chdir moves the whole process.
|
//nolint:paralleltest // t.Chdir moves the whole process.
|
||||||
func TestLoadDotEnv_ReadsTheWorkingDirectory(t *testing.T) {
|
func TestLoadDotEnv_ReadsTheWorkingDirectory(t *testing.T) {
|
||||||
configtest.ClearEnv(t)
|
config.ClearEnvForTest(t)
|
||||||
|
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
require.NoError(t, os.WriteFile(
|
require.NoError(t, os.WriteFile(
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ import (
|
|||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/config/configtest"
|
|
||||||
"sneak.berlin/go/webhooker/internal/globals"
|
"sneak.berlin/go/webhooker/internal/globals"
|
||||||
"sneak.berlin/go/webhooker/internal/logger"
|
"sneak.berlin/go/webhooker/internal/logger"
|
||||||
)
|
)
|
||||||
@@ -121,7 +120,7 @@ func TestEnvBool(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
configtest.ClearEnv(t)
|
config.ClearEnvForTest(t)
|
||||||
|
|
||||||
if tt.set {
|
if tt.set {
|
||||||
t.Setenv(testEnvKey, tt.value)
|
t.Setenv(testEnvKey, tt.value)
|
||||||
@@ -170,7 +169,7 @@ func runEnvIntCases(
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
configtest.ClearEnv(t)
|
config.ClearEnvForTest(t)
|
||||||
|
|
||||||
if tt.set {
|
if tt.set {
|
||||||
t.Setenv(testEnvKey, tt.value)
|
t.Setenv(testEnvKey, tt.value)
|
||||||
@@ -311,7 +310,7 @@ func TestEnvBindAddress(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
configtest.ClearEnv(t)
|
config.ClearEnvForTest(t)
|
||||||
|
|
||||||
if tt.set {
|
if tt.set {
|
||||||
t.Setenv(testEnvKey, tt.value)
|
t.Setenv(testEnvKey, tt.value)
|
||||||
@@ -477,7 +476,7 @@ func TestNewRejectsBadEnvValues(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
configtest.ClearEnv(t)
|
config.ClearEnvForTest(t)
|
||||||
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
||||||
t.Setenv(tt.key, tt.value)
|
t.Setenv(tt.key, tt.value)
|
||||||
|
|
||||||
@@ -639,7 +638,7 @@ func sentryEnvValueCases() []badEnvValueCase {
|
|||||||
// break the legitimate unset case: absent variables still get their
|
// break the legitimate unset case: absent variables still get their
|
||||||
// documented defaults.
|
// documented defaults.
|
||||||
func TestNewUsesDefaultsWhenUnset(t *testing.T) {
|
func TestNewUsesDefaultsWhenUnset(t *testing.T) {
|
||||||
configtest.ClearEnv(t)
|
config.ClearEnvForTest(t)
|
||||||
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
||||||
|
|
||||||
cfg, err := buildConfig(t)
|
cfg, err := buildConfig(t)
|
||||||
|
|||||||
@@ -6,7 +6,6 @@ import (
|
|||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/config/configtest"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// envKeySentryDSN is the variable envSentryDSN reads in production.
|
// envKeySentryDSN is the variable envSentryDSN reads in production.
|
||||||
@@ -101,7 +100,7 @@ func TestEnvSentryDSN(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
configtest.ClearEnv(t)
|
config.ClearEnvForTest(t)
|
||||||
|
|
||||||
if tt.set {
|
if tt.set {
|
||||||
t.Setenv(envKeySentryDSN, tt.value)
|
t.Setenv(envKeySentryDSN, tt.value)
|
||||||
|
|||||||
@@ -1,6 +1,4 @@
|
|||||||
// Package configtest holds test support for code that reads the
|
package config
|
||||||
// process environment.
|
|
||||||
package configtest
|
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"os"
|
"os"
|
||||||
@@ -8,12 +6,12 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ClearEnv unsets every variable in the process environment
|
// ClearEnvForTest unsets every variable in the process environment
|
||||||
// for the rest of the test, so a test sees only the variables it sets
|
// for the rest of the test, so a test sees only the variables it sets
|
||||||
// itself, not whatever the developer's shell exports. When the test
|
// itself, not whatever the developer's shell exports. When the test
|
||||||
// ends it leaves the environment exactly as it found it: each variable
|
// ends it leaves the environment exactly as it found it: each variable
|
||||||
// it unset is put back, and any variable added since is removed.
|
// it unset is put back, and any variable added since is removed.
|
||||||
func ClearEnv(t *testing.T) {
|
func ClearEnvForTest(t *testing.T) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
present := make(map[string]bool)
|
present := make(map[string]bool)
|
||||||
@@ -7,22 +7,21 @@ import (
|
|||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/config/configtest"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// TestClearEnv_RemovesAddedVariables pins that a variable set
|
// TestClearEnvForTest_RemovesAddedVariables pins that a variable set
|
||||||
// after the clear other than through t.Setenv, as a test's .env file
|
// after the clear other than through t.Setenv, as a test's .env file
|
||||||
// sets one, is gone once the test ends, so it cannot reach the tests
|
// sets one, is gone once the test ends, so it cannot reach the tests
|
||||||
// that run after it.
|
// that run after it.
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // ClearEnv uses t.Setenv.
|
//nolint:paralleltest // ClearEnvForTest uses t.Setenv.
|
||||||
func TestClearEnv_RemovesAddedVariables(t *testing.T) {
|
func TestClearEnvForTest_RemovesAddedVariables(t *testing.T) {
|
||||||
// The outer clear keeps a value of the key exported in the shell
|
// The outer clear keeps a value of the key exported in the shell
|
||||||
// from making it a variable the inner clear has to put back.
|
// from making it a variable the inner clear has to put back.
|
||||||
configtest.ClearEnv(t)
|
config.ClearEnvForTest(t)
|
||||||
|
|
||||||
t.Run("loads a .env file after the clear", func(t *testing.T) {
|
t.Run("loads a .env file after the clear", func(t *testing.T) {
|
||||||
configtest.ClearEnv(t)
|
config.ClearEnvForTest(t)
|
||||||
|
|
||||||
path := writeDotEnv(t, dotEnvKey+"=from-dot-env\n")
|
path := writeDotEnv(t, dotEnvKey+"=from-dot-env\n")
|
||||||
require.NoError(t, config.LoadDotEnvFileForTest(path))
|
require.NoError(t, config.LoadDotEnvFileForTest(path))
|
||||||
@@ -18,7 +18,6 @@ import (
|
|||||||
"sneak.berlin/go/webhooker/internal/banner"
|
"sneak.berlin/go/webhooker/internal/banner"
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/datadir"
|
"sneak.berlin/go/webhooker/internal/datadir"
|
||||||
"sneak.berlin/go/webhooker/internal/db/migrations"
|
|
||||||
"sneak.berlin/go/webhooker/internal/gormlog"
|
"sneak.berlin/go/webhooker/internal/gormlog"
|
||||||
"sneak.berlin/go/webhooker/internal/logger"
|
"sneak.berlin/go/webhooker/internal/logger"
|
||||||
)
|
)
|
||||||
@@ -250,21 +249,12 @@ func (d *Database) connectTo(dataDir string) error {
|
|||||||
return fmt.Errorf("migrating %s: %w", dbPath, err)
|
return fmt.Errorf("migrating %s: %w", dbPath, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = parseModels(
|
|
||||||
db, &Setting{}, &User{}, &APIKey{},
|
|
||||||
&Webhook{}, &Entrypoint{}, &Target{},
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("connecting to %s: %w", dbPath, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// migrate applies the main database's SQL files. See
|
|
||||||
// internal/db/migrations.
|
|
||||||
func (d *Database) migrate() error {
|
func (d *Database) migrate() error {
|
||||||
err := migrations.Apply(d.db, migrations.Main)
|
// Run GORM auto-migrations
|
||||||
|
err := d.Migrate()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
d.log.Error(
|
d.log.Error(
|
||||||
"failed to run database migrations",
|
"failed to run database migrations",
|
||||||
@@ -279,21 +269,6 @@ func (d *Database) migrate() error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// parseModels has GORM parse each model for db before the handle is
|
|
||||||
// shared. GORM parses a model the first time a handle uses it, and two
|
|
||||||
// goroutines doing that at once for models that refer to each other,
|
|
||||||
// as Webhook and Target do, race inside GORM.
|
|
||||||
func parseModels(db *gorm.DB, models ...any) error {
|
|
||||||
for _, model := range models {
|
|
||||||
err := (&gorm.Statement{DB: db}).Parse(model)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("parsing model %T: %w", model, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ensureAdminUser creates the bootstrap admin account when the
|
// ensureAdminUser creates the bootstrap admin account when the
|
||||||
// deployment has no users at all.
|
// deployment has no users at all.
|
||||||
func (d *Database) ensureAdminUser() error {
|
func (d *Database) ensureAdminUser() error {
|
||||||
|
|||||||
@@ -1,55 +0,0 @@
|
|||||||
// Package databasetest builds a WebhookDBManager for tests in other
|
|
||||||
// packages.
|
|
||||||
package databasetest
|
|
||||||
|
|
||||||
import (
|
|
||||||
"log/slog"
|
|
||||||
"os"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"go.uber.org/fx/fxtest"
|
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// NewWebhookDBManager creates a WebhookDBManager backed by the given
|
|
||||||
// data directory, logging at DEBUG to standard error.
|
|
||||||
func NewWebhookDBManager(
|
|
||||||
t *testing.T, dataDir string,
|
|
||||||
) *database.WebhookDBManager {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
return NewWebhookDBManagerWithLogger(
|
|
||||||
t,
|
|
||||||
dataDir,
|
|
||||||
slog.New(slog.NewTextHandler(
|
|
||||||
os.Stderr,
|
|
||||||
&slog.HandlerOptions{Level: slog.LevelDebug},
|
|
||||||
)),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewWebhookDBManagerWithLogger is NewWebhookDBManager with the
|
|
||||||
// logger supplied by the caller. The per-webhook databases this manager
|
|
||||||
// opens hand that logger to gormlog, so a test that needs to see the SQL
|
|
||||||
// the service emits can capture it.
|
|
||||||
//
|
|
||||||
// It is built through database.NewWebhookDBManager on a lifecycle that
|
|
||||||
// is never started, so nothing closes its databases but the caller.
|
|
||||||
func NewWebhookDBManagerWithLogger(
|
|
||||||
t *testing.T, dataDir string, log *slog.Logger,
|
|
||||||
) *database.WebhookDBManager {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
mgr, err := database.NewWebhookDBManager(
|
|
||||||
fxtest.NewLifecycle(t),
|
|
||||||
database.WebhookDBManagerParams{
|
|
||||||
Config: &config.Config{DataDir: dataDir},
|
|
||||||
Logger: log,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
return mgr
|
|
||||||
}
|
|
||||||
@@ -2,6 +2,7 @@ package database
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
"gorm.io/gorm/clause"
|
"gorm.io/gorm/clause"
|
||||||
@@ -17,14 +18,14 @@ const omitAssociationsCallback = "webhooker:omit_associations"
|
|||||||
//
|
//
|
||||||
// A per-webhook database holds the event tier only, but Delivery
|
// A per-webhook database holds the event tier only, but Delivery
|
||||||
// declares belongs-to Event and Target and the delivery engine fills
|
// declares belongs-to Event and Target and the delivery engine fills
|
||||||
// both in memory before writing. Without this guard GORM upserts those
|
// both in memory before writing. Without this guard GORM upserts
|
||||||
// parent rows here on the delivery and retry write paths, and the
|
// those parent rows here on the delivery and retry write paths,
|
||||||
// target, whose config holds destination URLs and bearer credentials,
|
// copying targets.config, which holds destination URLs and bearer
|
||||||
// goes to a targets table this file does not have, which fails the
|
// credentials, into the file most likely to be backed up or handed
|
||||||
// write. Registering the guard on the connection
|
// to someone else. Registering the guard on the connection covers
|
||||||
// covers every write path, including writes inside a transaction and
|
// every write path, including writes inside a transaction and write
|
||||||
// write paths added later. Every event-tier row this file holds is
|
// paths added later. Every event-tier row this file holds is written
|
||||||
// written explicitly, so nothing depends on the automatic save.
|
// explicitly, so nothing depends on the automatic save.
|
||||||
func omitAssociations(db *gorm.DB) error {
|
func omitAssociations(db *gorm.DB) error {
|
||||||
omit := func(tx *gorm.DB) {
|
omit := func(tx *gorm.DB) {
|
||||||
tx.Statement.Omits = append(
|
tx.Statement.Omits = append(
|
||||||
@@ -52,3 +53,107 @@ func omitAssociations(db *gorm.DB) error {
|
|||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// eventDBSweptVersion is the PRAGMA user_version purgeTargetRows
|
||||||
|
// stamps into a per-webhook database once it has removed any leaked
|
||||||
|
// target rows *and* the VACUUM that removes their bytes has returned.
|
||||||
|
// Nothing else in the tree uses user_version, so 0 means "not swept
|
||||||
|
// by this build".
|
||||||
|
//
|
||||||
|
// The stamp, not the DELETE, is what records that a file is done. A
|
||||||
|
// DELETE commits on its own, so a sweep that is interrupted or whose
|
||||||
|
// VACUUM fails leaves a file whose rows are gone but whose credential
|
||||||
|
// bytes are still in the free pages -- indistinguishable, by row
|
||||||
|
// count, from a file that never leaked. Both leave the stamp unset,
|
||||||
|
// so the next open sweeps again.
|
||||||
|
const eventDBSweptVersion = 1
|
||||||
|
|
||||||
|
// purgeTargetRows deletes target rows that an earlier build's
|
||||||
|
// association upsert wrote into a per-webhook database, and rewrites
|
||||||
|
// the file so their bytes are gone with them. AutoMigrate creates a
|
||||||
|
// targets table in every one of these files because Delivery declares
|
||||||
|
// a belongs-to Target, but nothing in the event tier may put rows in
|
||||||
|
// it. The rows it did put there are junk, not history: they carry an
|
||||||
|
// empty webhook_id, and delivery rows resolve their target against
|
||||||
|
// the main database, so nothing here refers to them.
|
||||||
|
//
|
||||||
|
// The DELETE only unlinks the rows: modernc.org/sqlite leaves
|
||||||
|
// secure_delete at SQLite's default of off, so the credential bytes
|
||||||
|
// stay readable in the file's free pages and a backup of a swept file
|
||||||
|
// would still hand them over. VACUUM rewrites the file without them.
|
||||||
|
//
|
||||||
|
// This runs before every migration and is gated on
|
||||||
|
// eventDBSweptVersion, so a file pays for the rewrite once, on the
|
||||||
|
// first open that finds it unstamped, and every open after that is a
|
||||||
|
// PRAGMA read. A file this build created is stamped before its
|
||||||
|
// targets table exists, so it never vacuums at all. A failure here
|
||||||
|
// fails the open with the stamp left unset, so the sweep is retried
|
||||||
|
// rather than skipped -- a webhook whose file cannot be swept stays
|
||||||
|
// unusable instead of quietly serving from a file that still holds
|
||||||
|
// recoverable credentials.
|
||||||
|
func purgeTargetRows(
|
||||||
|
db *gorm.DB, log *slog.Logger, webhookID string,
|
||||||
|
) error {
|
||||||
|
var version int
|
||||||
|
|
||||||
|
// Row().Scan, not (*gorm.DB).Scan: see internal/gormlog.
|
||||||
|
err := db.Raw("PRAGMA user_version").Row().Scan(&version)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"reading sweep marker of webhook database %s: %w",
|
||||||
|
webhookID, err,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
if version >= eventDBSweptVersion {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var purged int64
|
||||||
|
|
||||||
|
if db.Migrator().HasTable("targets") {
|
||||||
|
res := db.Exec("DELETE FROM targets")
|
||||||
|
if res.Error != nil {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"purging target rows from webhook database %s: %w",
|
||||||
|
webhookID, res.Error,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
purged = res.RowsAffected
|
||||||
|
|
||||||
|
// Unconditional: a zero row count here does not mean there is
|
||||||
|
// nothing to remove, only that no *live* row is left. See
|
||||||
|
// eventDBSweptVersion.
|
||||||
|
err = db.Exec("VACUUM").Error
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"purged %d leaked target rows from webhook database "+
|
||||||
|
"%s but vacuuming it failed, so the deleted "+
|
||||||
|
"target credentials are still recoverable from "+
|
||||||
|
"the file; it stays marked unswept and the next "+
|
||||||
|
"open retries: %w",
|
||||||
|
purged, webhookID, err,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
err = db.Exec(fmt.Sprintf(
|
||||||
|
"PRAGMA user_version = %d", eventDBSweptVersion,
|
||||||
|
)).Error
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"marking webhook database %s swept: %w", webhookID, err,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
if purged > 0 {
|
||||||
|
log.Warn(
|
||||||
|
"purged leaked target rows from per-webhook database",
|
||||||
|
"webhook_id", webhookID,
|
||||||
|
"rows", purged,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,15 +1,391 @@
|
|||||||
package database_test
|
package database_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
|
"database/sql"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
_ "modernc.org/sqlite"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/database/databasetest"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// testDataDirPerm is the mode the test data directory is created
|
||||||
|
// with.
|
||||||
|
const testDataDirPerm = 0o750
|
||||||
|
|
||||||
|
// eventDBDataDir returns a data directory that a WebhookDBManager
|
||||||
|
// can be pointed at.
|
||||||
|
func eventDBDataDir(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
dir := filepath.Join(t.TempDir(), "events")
|
||||||
|
require.NoError(t, os.MkdirAll(dir, testDataDirPerm))
|
||||||
|
|
||||||
|
return dir
|
||||||
|
}
|
||||||
|
|
||||||
|
// openRawEventDB opens the per-webhook database file directly,
|
||||||
|
// without the manager, so a test can put a file on disk in a state
|
||||||
|
// the manager has to cope with, or inspect one afterwards.
|
||||||
|
func openRawEventDB(
|
||||||
|
t *testing.T, dataDir, webhookID string,
|
||||||
|
) *sql.DB {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
path := filepath.Join(
|
||||||
|
dataDir, fmt.Sprintf("events-%s.db", webhookID),
|
||||||
|
)
|
||||||
|
|
||||||
|
sqlDB, err := sql.Open(
|
||||||
|
"sqlite",
|
||||||
|
fmt.Sprintf("file:%s?mode=rwc", path),
|
||||||
|
)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
t.Cleanup(func() { _ = sqlDB.Close() })
|
||||||
|
|
||||||
|
return sqlDB
|
||||||
|
}
|
||||||
|
|
||||||
|
// eventDBFileBytes reads a per-webhook database file off disk, so a
|
||||||
|
// test can assert on what the file itself still holds rather than on
|
||||||
|
// what a query returns.
|
||||||
|
func eventDBFileBytes(t *testing.T, dataDir, webhookID string) []byte {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
//nolint:gosec // reads a file the test just created under t.TempDir()
|
||||||
|
raw, err := os.ReadFile(filepath.Join(
|
||||||
|
dataDir, fmt.Sprintf("events-%s.db", webhookID),
|
||||||
|
))
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
|
||||||
|
// eventDBUserVersion returns the PRAGMA user_version of a per-webhook
|
||||||
|
// database file, which is the marker purgeTargetRows stamps once it
|
||||||
|
// has swept and vacuumed.
|
||||||
|
func eventDBUserVersion(t *testing.T, sqlDB *sql.DB) int {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
var version int
|
||||||
|
|
||||||
|
require.NoError(t, sqlDB.QueryRowContext(
|
||||||
|
t.Context(), "PRAGMA user_version",
|
||||||
|
).Scan(&version))
|
||||||
|
|
||||||
|
return version
|
||||||
|
}
|
||||||
|
|
||||||
|
// clearEventDBSweptMarker resets the sweep marker to 0, which is what
|
||||||
|
// a file written by a build without the sweep looks like. Tests that
|
||||||
|
// seed a leaked row have to create the file through the manager to
|
||||||
|
// get the real targets table shape, and that stamps it.
|
||||||
|
func clearEventDBSweptMarker(t *testing.T, sqlDB *sql.DB) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
_, err := sqlDB.ExecContext(t.Context(), "PRAGMA user_version = 0")
|
||||||
|
require.NoError(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// countTargetRows returns the number of rows in the targets table of
|
||||||
|
// a per-webhook database file, or -1 if the table does not exist.
|
||||||
|
func countTargetRows(t *testing.T, sqlDB *sql.DB) int {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
var tables int
|
||||||
|
|
||||||
|
require.NoError(t, sqlDB.QueryRowContext(
|
||||||
|
t.Context(),
|
||||||
|
"SELECT count(*) FROM sqlite_master "+
|
||||||
|
"WHERE type = 'table' AND name = 'targets'",
|
||||||
|
).Scan(&tables))
|
||||||
|
|
||||||
|
if tables == 0 {
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
|
||||||
|
var rows int
|
||||||
|
|
||||||
|
require.NoError(t, sqlDB.QueryRowContext(
|
||||||
|
t.Context(), "SELECT count(*) FROM targets",
|
||||||
|
).Scan(&rows))
|
||||||
|
|
||||||
|
return rows
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestOpenPurgesLeakedTargetRows covers the sweep for event
|
||||||
|
// databases written by a build that let GORM upsert target rows
|
||||||
|
// into them: opening the database clears them, and opening it again
|
||||||
|
// is a no-op.
|
||||||
|
func TestOpenPurgesLeakedTargetRows(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
dataDir := eventDBDataDir(t)
|
||||||
|
webhookID := uuid.New().String()
|
||||||
|
|
||||||
|
// Create the file the way the application does, so the targets
|
||||||
|
// table has exactly the shape AutoMigrate gives it, then write
|
||||||
|
// a leaked row into it the way the association upsert did.
|
||||||
|
initial := database.NewTestWebhookDBManager(dataDir)
|
||||||
|
|
||||||
|
_, err := initial.GetDB(webhookID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, initial.CloseAll())
|
||||||
|
|
||||||
|
seed := openRawEventDB(t, dataDir, webhookID)
|
||||||
|
|
||||||
|
_, err = seed.ExecContext(
|
||||||
|
t.Context(),
|
||||||
|
"INSERT INTO targets "+
|
||||||
|
"(id, webhook_id, name, type, config) "+
|
||||||
|
"VALUES (?, '', ?, ?, ?)",
|
||||||
|
uuid.New().String(),
|
||||||
|
"leaked-target",
|
||||||
|
"slack",
|
||||||
|
`{"webhookUrl":"https://hooks.example/T000/B000/secret"}`,
|
||||||
|
)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, 1, countTargetRows(t, seed))
|
||||||
|
clearEventDBSweptMarker(t, seed)
|
||||||
|
require.NoError(t, seed.Close())
|
||||||
|
|
||||||
|
mgr := database.NewTestWebhookDBManager(dataDir)
|
||||||
|
|
||||||
|
_, err = mgr.GetDB(webhookID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, mgr.CloseAll())
|
||||||
|
|
||||||
|
check := openRawEventDB(t, dataDir, webhookID)
|
||||||
|
assert.Zero(t, countTargetRows(t, check))
|
||||||
|
assert.Equal(
|
||||||
|
t, 1, eventDBUserVersion(t, check),
|
||||||
|
"a completed sweep must mark the file so later opens skip it",
|
||||||
|
)
|
||||||
|
require.NoError(t, check.Close())
|
||||||
|
|
||||||
|
// Idempotent: a second open leaves it at zero and does not
|
||||||
|
// error.
|
||||||
|
again := database.NewTestWebhookDBManager(dataDir)
|
||||||
|
|
||||||
|
_, err = again.GetDB(webhookID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, again.CloseAll())
|
||||||
|
|
||||||
|
recheck := openRawEventDB(t, dataDir, webhookID)
|
||||||
|
assert.Zero(t, countTargetRows(t, recheck))
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestOpenPurgeRemovesCredentialBytes covers the sweep at the level
|
||||||
|
// that matters for a backup handed to someone else: the leaked
|
||||||
|
// credential must be gone from the raw bytes of the file, not merely
|
||||||
|
// unreachable by query. A bare DELETE unlinks the row and leaves the
|
||||||
|
// bytes readable in the free pages, so this fails without the VACUUM
|
||||||
|
// in purgeTargetRows.
|
||||||
|
func TestOpenPurgeRemovesCredentialBytes(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
dataDir := eventDBDataDir(t)
|
||||||
|
webhookID := uuid.New().String()
|
||||||
|
credential := "T00000000/B00000000/" + uuid.New().String()
|
||||||
|
|
||||||
|
initial := database.NewTestWebhookDBManager(dataDir)
|
||||||
|
|
||||||
|
_, err := initial.GetDB(webhookID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, initial.CloseAll())
|
||||||
|
|
||||||
|
seed := openRawEventDB(t, dataDir, webhookID)
|
||||||
|
|
||||||
|
_, err = seed.ExecContext(
|
||||||
|
t.Context(),
|
||||||
|
"INSERT INTO targets "+
|
||||||
|
"(id, webhook_id, name, type, config) "+
|
||||||
|
"VALUES (?, '', ?, ?, ?)",
|
||||||
|
uuid.New().String(),
|
||||||
|
"leaked-target",
|
||||||
|
"slack",
|
||||||
|
fmt.Sprintf(
|
||||||
|
`{"webhookUrl":"https://hooks.example/%s"}`, credential,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
require.NoError(t, err)
|
||||||
|
clearEventDBSweptMarker(t, seed)
|
||||||
|
require.NoError(t, seed.Close())
|
||||||
|
|
||||||
|
// The seed has to be in the file for its absence later to mean
|
||||||
|
// anything.
|
||||||
|
require.True(
|
||||||
|
t,
|
||||||
|
bytes.Contains(
|
||||||
|
eventDBFileBytes(t, dataDir, webhookID),
|
||||||
|
[]byte(credential),
|
||||||
|
),
|
||||||
|
"seeded credential is not in the file, so this test proves nothing",
|
||||||
|
)
|
||||||
|
|
||||||
|
mgr := database.NewTestWebhookDBManager(dataDir)
|
||||||
|
|
||||||
|
_, err = mgr.GetDB(webhookID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, mgr.CloseAll())
|
||||||
|
|
||||||
|
assert.NotContains(
|
||||||
|
t,
|
||||||
|
string(eventDBFileBytes(t, dataDir, webhookID)),
|
||||||
|
credential,
|
||||||
|
"leaked credential is still recoverable from the raw file",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestOpenRevacuumsAfterIncompleteSweep covers the case a row count
|
||||||
|
// cannot see: the rows are already deleted but the file was never
|
||||||
|
// vacuumed, because an earlier sweep died between the two or its
|
||||||
|
// VACUUM failed. The credential bytes are still recoverable, and the
|
||||||
|
// unset marker is the only thing that says so, so the next open must
|
||||||
|
// vacuum rather than conclude from the empty table that there is
|
||||||
|
// nothing to do.
|
||||||
|
func TestOpenRevacuumsAfterIncompleteSweep(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
dataDir := eventDBDataDir(t)
|
||||||
|
webhookID := uuid.New().String()
|
||||||
|
credential := "T00000000/B00000000/" + uuid.New().String()
|
||||||
|
|
||||||
|
initial := database.NewTestWebhookDBManager(dataDir)
|
||||||
|
|
||||||
|
_, err := initial.GetDB(webhookID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, initial.CloseAll())
|
||||||
|
|
||||||
|
seed := openRawEventDB(t, dataDir, webhookID)
|
||||||
|
|
||||||
|
_, err = seed.ExecContext(
|
||||||
|
t.Context(),
|
||||||
|
"INSERT INTO targets "+
|
||||||
|
"(id, webhook_id, name, type, config) "+
|
||||||
|
"VALUES (?, '', ?, ?, ?)",
|
||||||
|
uuid.New().String(),
|
||||||
|
"leaked-target",
|
||||||
|
"slack",
|
||||||
|
fmt.Sprintf(
|
||||||
|
`{"webhookUrl":"https://hooks.example/%s"}`, credential,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// Exactly the state an interrupted sweep leaves: rows gone,
|
||||||
|
// marker unset, bytes still in the free pages.
|
||||||
|
_, err = seed.ExecContext(t.Context(), "DELETE FROM targets")
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Zero(t, countTargetRows(t, seed))
|
||||||
|
clearEventDBSweptMarker(t, seed)
|
||||||
|
require.NoError(t, seed.Close())
|
||||||
|
|
||||||
|
require.True(
|
||||||
|
t,
|
||||||
|
bytes.Contains(
|
||||||
|
eventDBFileBytes(t, dataDir, webhookID),
|
||||||
|
[]byte(credential),
|
||||||
|
),
|
||||||
|
"the deleted row's bytes must still be in the file, or this "+
|
||||||
|
"test proves nothing",
|
||||||
|
)
|
||||||
|
|
||||||
|
mgr := database.NewTestWebhookDBManager(dataDir)
|
||||||
|
|
||||||
|
_, err = mgr.GetDB(webhookID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, mgr.CloseAll())
|
||||||
|
|
||||||
|
assert.NotContains(
|
||||||
|
t,
|
||||||
|
string(eventDBFileBytes(t, dataDir, webhookID)),
|
||||||
|
credential,
|
||||||
|
"an interrupted sweep was not retried, so the credential is "+
|
||||||
|
"still recoverable from the raw file",
|
||||||
|
)
|
||||||
|
|
||||||
|
check := openRawEventDB(t, dataDir, webhookID)
|
||||||
|
assert.Equal(t, 1, eventDBUserVersion(t, check))
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestOpenSkipsSweptDatabase covers the other half of the marker: a
|
||||||
|
// file this build created is marked without ever being vacuumed, and
|
||||||
|
// a marked file is not swept again.
|
||||||
|
func TestOpenSkipsSweptDatabase(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
dataDir := eventDBDataDir(t)
|
||||||
|
webhookID := uuid.New().String()
|
||||||
|
|
||||||
|
mgr := database.NewTestWebhookDBManager(dataDir)
|
||||||
|
|
||||||
|
_, err := mgr.GetDB(webhookID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, mgr.CloseAll())
|
||||||
|
|
||||||
|
marked := openRawEventDB(t, dataDir, webhookID)
|
||||||
|
assert.Equal(t, 1, eventDBUserVersion(t, marked))
|
||||||
|
|
||||||
|
// A marked file is left alone, so a row written into it survives
|
||||||
|
// a reopen. Nothing writes target rows any more; this stands in
|
||||||
|
// for the sweep having run.
|
||||||
|
_, err = marked.ExecContext(
|
||||||
|
t.Context(),
|
||||||
|
"INSERT INTO targets "+
|
||||||
|
"(id, webhook_id, name, type, config) "+
|
||||||
|
"VALUES (?, '', ?, ?, ?)",
|
||||||
|
uuid.New().String(), "sentinel", "slack", `{}`,
|
||||||
|
)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, marked.Close())
|
||||||
|
|
||||||
|
again := database.NewTestWebhookDBManager(dataDir)
|
||||||
|
|
||||||
|
_, err = again.GetDB(webhookID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, again.CloseAll())
|
||||||
|
|
||||||
|
check := openRawEventDB(t, dataDir, webhookID)
|
||||||
|
assert.Equal(
|
||||||
|
t, 1, countTargetRows(t, check),
|
||||||
|
"a marked file must not be swept again",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestOpenSucceedsWithoutTargetsTable covers an existing event
|
||||||
|
// database that never grew a targets table. The sweep must not fail
|
||||||
|
// startup on it.
|
||||||
|
func TestOpenSucceedsWithoutTargetsTable(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
dataDir := eventDBDataDir(t)
|
||||||
|
webhookID := uuid.New().String()
|
||||||
|
|
||||||
|
seed := openRawEventDB(t, dataDir, webhookID)
|
||||||
|
|
||||||
|
_, err := seed.ExecContext(
|
||||||
|
t.Context(),
|
||||||
|
"CREATE TABLE events (id text PRIMARY KEY)",
|
||||||
|
)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, seed.Close())
|
||||||
|
|
||||||
|
mgr := database.NewTestWebhookDBManager(dataDir)
|
||||||
|
|
||||||
|
db, err := mgr.GetDB(webhookID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.NotNil(t, db)
|
||||||
|
require.NoError(t, mgr.CloseAll())
|
||||||
|
}
|
||||||
|
|
||||||
// TestEventDBCreateOmitsAssociations covers the connection-level
|
// TestEventDBCreateOmitsAssociations covers the connection-level
|
||||||
// guard directly: a Delivery carrying its Event and Target in
|
// guard directly: a Delivery carrying its Event and Target in
|
||||||
// memory, written through the manager's handle, must store only the
|
// memory, written through the manager's handle, must store only the
|
||||||
@@ -17,9 +393,10 @@ import (
|
|||||||
func TestEventDBCreateOmitsAssociations(t *testing.T) {
|
func TestEventDBCreateOmitsAssociations(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
dataDir := eventDBDataDir(t)
|
||||||
webhookID := uuid.New().String()
|
webhookID := uuid.New().String()
|
||||||
|
|
||||||
mgr := databasetest.NewWebhookDBManager(t, t.TempDir())
|
mgr := database.NewTestWebhookDBManager(dataDir)
|
||||||
|
|
||||||
db, err := mgr.GetDB(webhookID)
|
db, err := mgr.GetDB(webhookID)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -54,9 +431,8 @@ func TestEventDBCreateOmitsAssociations(t *testing.T) {
|
|||||||
require.NoError(t, db.Model(d).
|
require.NoError(t, db.Model(d).
|
||||||
Update("status", database.DeliveryStatusDelivered).
|
Update("status", database.DeliveryStatusDelivered).
|
||||||
Error)
|
Error)
|
||||||
|
|
||||||
// The file has no targets table, so the create and update above
|
|
||||||
// would have failed had GORM tried to save the target.
|
|
||||||
assert.False(t, db.Migrator().HasTable(&database.Target{}))
|
|
||||||
require.NoError(t, mgr.CloseAll())
|
require.NoError(t, mgr.CloseAll())
|
||||||
|
|
||||||
|
check := openRawEventDB(t, dataDir, webhookID)
|
||||||
|
assert.Zero(t, countTargetRows(t, check))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,8 +13,10 @@ import (
|
|||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TestWebhookDBManager_OpenAddsEventTierIndexes verifies that a new
|
// TestWebhookDBManager_OpenAddsEventTierIndexes verifies that opening a
|
||||||
// per-webhook database has these indexes.
|
// per-webhook database that predates these indexes creates them. It
|
||||||
|
// stands in for an older database file by dropping the indexes
|
||||||
|
// AutoMigrate just created, then reopening the same file.
|
||||||
func TestWebhookDBManager_OpenAddsEventTierIndexes(t *testing.T) {
|
func TestWebhookDBManager_OpenAddsEventTierIndexes(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -40,9 +42,27 @@ func TestWebhookDBManager_OpenAddsEventTierIndexes(t *testing.T) {
|
|||||||
db, err := mgr.GetDB(webhookID)
|
db, err := mgr.GetDB(webhookID)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// A fresh database has them.
|
||||||
|
for _, ix := range indexes {
|
||||||
|
require.True(t, db.Migrator().HasIndex(ix.model, ix.name))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stand in for a database file created before the indexes existed.
|
||||||
|
for _, ix := range indexes {
|
||||||
|
require.NoError(t, db.Migrator().DropIndex(ix.model, ix.name))
|
||||||
|
require.False(t, db.Migrator().HasIndex(ix.model, ix.name))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Drop the cached connection so the next open reopens the file and
|
||||||
|
// runs AutoMigrate against it, as a restart would.
|
||||||
|
require.NoError(t, mgr.CloseAll())
|
||||||
|
|
||||||
|
db, err = mgr.GetDB(webhookID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
for _, ix := range indexes {
|
for _, ix := range indexes {
|
||||||
assert.True(t, db.Migrator().HasIndex(ix.model, ix.name),
|
assert.True(t, db.Migrator().HasIndex(ix.model, ix.name),
|
||||||
"a new database should have %s", ix.name)
|
"opening the existing database should create %s", ix.name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -129,62 +149,6 @@ func TestEventTierQueriesUseTheirIndexes(t *testing.T) {
|
|||||||
Delete(&database.Event{}), "sqlite_autoindex_events_1 (id=?)")
|
Delete(&database.Event{}), "sqlite_autoindex_events_1 (id=?)")
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestEventLogFiltersUseTheStatusIndex does the same for the event log's
|
|
||||||
// Failed and Pending lists, of the newest events with a delivery in
|
|
||||||
// given statuses, and for their counts (eventsWithStatus and
|
|
||||||
// countEventsWithStatus in the handlers). The lists must also reach
|
|
||||||
// the events table only by ID: from the matching deliveries, then from
|
|
||||||
// the newest of those events.
|
|
||||||
func TestEventLogFiltersUseTheStatusIndex(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
mgr, lc := setupTestWebhookDBManager(t)
|
|
||||||
ctx := context.Background()
|
|
||||||
require.NoError(t, lc.Start(ctx))
|
|
||||||
|
|
||||||
defer func() { require.NoError(t, lc.Stop(ctx)) }()
|
|
||||||
|
|
||||||
webhookID := uuid.New().String()
|
|
||||||
|
|
||||||
db, err := mgr.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
dry := db.Session(&gorm.Session{DryRun: true})
|
|
||||||
byStatus := "idx_deliveries_status (status=? AND deleted_at=?)"
|
|
||||||
pending := []database.DeliveryStatus{
|
|
||||||
database.DeliveryStatusPending,
|
|
||||||
database.DeliveryStatusRetrying,
|
|
||||||
}
|
|
||||||
|
|
||||||
var (
|
|
||||||
rows []struct{ ID string }
|
|
||||||
count int64
|
|
||||||
)
|
|
||||||
|
|
||||||
matching := dry.Model(&database.Delivery{}).
|
|
||||||
Distinct("event_id").Where("status IN ?", pending)
|
|
||||||
newest := dry.Table("(?) AS matching", matching).
|
|
||||||
Joins("CROSS JOIN events ON events.id = matching.event_id").
|
|
||||||
Where(
|
|
||||||
"events.webhook_id = ? AND events.deleted_at IS NULL",
|
|
||||||
webhookID,
|
|
||||||
).
|
|
||||||
Order("events.created_at DESC").Limit(50).
|
|
||||||
Select("events.id AS event_id")
|
|
||||||
|
|
||||||
// Each step of the plan is printed in braces, so these name the
|
|
||||||
// lookup that follows each scan.
|
|
||||||
byID := "{SEARCH events USING INDEX sqlite_autoindex_events_1 (id=?)}"
|
|
||||||
|
|
||||||
assertPlanUses(t, db, dry.Table("(?) AS newest", newest).
|
|
||||||
Joins("CROSS JOIN events ON events.id = newest.event_id").
|
|
||||||
Select("id").Order("created_at DESC").Limit(50).Find(&rows),
|
|
||||||
byStatus, "{SCAN matching} "+byID, "{SCAN newest} "+byID)
|
|
||||||
assertPlanUses(t, db, dry.Model(&database.Delivery{}).
|
|
||||||
Distinct("event_id").Where("status IN ?", pending).Count(&count),
|
|
||||||
byStatus)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestStatisticsQueriesUseTheirIndexes does the same for the webhook
|
// TestStatisticsQueriesUseTheirIndexes does the same for the webhook
|
||||||
// page's statistics (readEventStats in the handlers): deliveries in
|
// page's statistics (readEventStats in the handlers): deliveries in
|
||||||
// progress, each target's deliveries finished since a time, which must
|
// progress, each target's deliveries finished since a time, which must
|
||||||
|
|||||||
@@ -56,10 +56,6 @@ type Delivery struct {
|
|||||||
// the index.
|
// the index.
|
||||||
FinishedAt *time.Time `gorm:"index:idx_deliveries_status,priority:3" json:"finishedAt,omitempty"`
|
FinishedAt *time.Time `gorm:"index:idx_deliveries_status,priority:3" json:"finishedAt,omitempty"`
|
||||||
|
|
||||||
// Replay is set on a delivery created by the event log's Replay
|
|
||||||
// action, so the pages can tell it from the delivery it repeats.
|
|
||||||
Replay bool `gorm:"not null;default:false" json:"replay"`
|
|
||||||
|
|
||||||
// Relations. No model marshals the record it belongs to:
|
// Relations. No model marshals the record it belongs to:
|
||||||
// Event.Deliveries and Target.Deliveries lead back here, and the
|
// Event.Deliveries and Target.Deliveries lead back here, and the
|
||||||
// JSON could loop.
|
// JSON could loop.
|
||||||
|
|||||||
@@ -30,10 +30,8 @@ type Event struct {
|
|||||||
WebhookID string `gorm:"type:uuid;not null" json:"webhookId"`
|
WebhookID string `gorm:"type:uuid;not null" json:"webhookId"`
|
||||||
EntrypointID string `gorm:"type:uuid;not null;index:idx_events_entrypoint_id,priority:1" json:"entrypointId"`
|
EntrypointID string `gorm:"type:uuid;not null;index:idx_events_entrypoint_id,priority:1" json:"entrypointId"`
|
||||||
|
|
||||||
// Request data. RawQuery is the receiving request's query string
|
// Request data
|
||||||
// as sent, without the leading "?".
|
|
||||||
Method string `gorm:"not null" json:"method"`
|
Method string `gorm:"not null" json:"method"`
|
||||||
RawQuery string `gorm:"type:text" json:"rawQuery"`
|
|
||||||
Headers string `gorm:"type:text" json:"headers"` // JSON
|
Headers string `gorm:"type:text" json:"headers"` // JSON
|
||||||
Body string `gorm:"type:text" json:"body"`
|
Body string `gorm:"type:text" json:"body"`
|
||||||
ContentType string `json:"contentType"`
|
ContentType string `json:"contentType"`
|
||||||
@@ -45,7 +43,8 @@ type Event struct {
|
|||||||
|
|
||||||
// ResubmittedFromID names the event this one was copied from by
|
// ResubmittedFromID names the event this one was copied from by
|
||||||
// an operator resubmit. It is nil for an event that arrived on
|
// an operator resubmit. It is nil for an event that arrived on
|
||||||
// the receiver. It is not a foreign key: the source event can be
|
// the receiver, which is every event created before the column
|
||||||
|
// existed. It is not a foreign key: the source event can be
|
||||||
// reaped by retention while its copies remain, and the id is
|
// reaped by retention while its copies remain, and the id is
|
||||||
// kept as the record of where the copy came from either way.
|
// kept as the record of where the copy came from either way.
|
||||||
ResubmittedFromID *string `gorm:"type:uuid;index:idx_events_resubmitted_from_id,priority:1;index:idx_events_entrypoint_id,priority:3" json:"resubmittedFromId,omitempty"`
|
ResubmittedFromID *string `gorm:"type:uuid;index:idx_events_resubmitted_from_id,priority:1;index:idx_events_entrypoint_id,priority:3" json:"resubmittedFromId,omitempty"`
|
||||||
|
|||||||
@@ -82,7 +82,7 @@ type Webhook struct {
|
|||||||
// value while building the insert statement, which runs after
|
// value while building the insert statement, which runs after
|
||||||
// BeforeSave; rewriting any later than this loses that race and the
|
// BeforeSave; rewriting any later than this loses that race and the
|
||||||
// row lands at 30 days. Living on the model also means a future call
|
// row lands at 30 days. Living on the model also means a future call
|
||||||
// site, such as a REST API or a fixture, cannot bypass it.
|
// site — a REST API, a fixture, a migration — cannot bypass it.
|
||||||
func (w *Webhook) BeforeSave(_ *gorm.DB) error {
|
func (w *Webhook) BeforeSave(_ *gorm.DB) error {
|
||||||
if w.RetentionDays <= 0 {
|
if w.RetentionDays <= 0 {
|
||||||
w.RetentionDays = RetentionForeverDays
|
w.RetentionDays = RetentionForeverDays
|
||||||
@@ -98,8 +98,8 @@ func (w *Webhook) BeforeSave(_ *gorm.DB) error {
|
|||||||
// are exempt from reaping.
|
// are exempt from reaping.
|
||||||
//
|
//
|
||||||
// It accepts the RetentionForeverDays sentinel written by BeforeSave
|
// It accepts the RetentionForeverDays sentinel written by BeforeSave
|
||||||
// and, defensively, any non-positive value, which BeforeSave treats
|
// and, defensively, the non-positive values that rows written before
|
||||||
// as a request to retain forever.
|
// the sentinel existed may still carry.
|
||||||
func retainsForever(retentionDays int) bool {
|
func retainsForever(retentionDays int) bool {
|
||||||
return retentionDays <= 0 ||
|
return retentionDays <= 0 ||
|
||||||
retentionDays >= RetentionForeverDays
|
retentionDays >= RetentionForeverDays
|
||||||
|
|||||||
@@ -0,0 +1,17 @@
|
|||||||
|
package database
|
||||||
|
|
||||||
|
// Migrate runs database migrations for the main application database.
|
||||||
|
// Only configuration-tier models are stored in the main database.
|
||||||
|
// Event-tier models (Event, Delivery, DeliveryResult, EventTotals,
|
||||||
|
// TargetTotals, EntrypointTotals) live in
|
||||||
|
// per-webhook dedicated databases managed by WebhookDBManager.
|
||||||
|
func (d *Database) Migrate() error {
|
||||||
|
return d.db.AutoMigrate(
|
||||||
|
&Setting{},
|
||||||
|
&User{},
|
||||||
|
&APIKey{},
|
||||||
|
&Webhook{},
|
||||||
|
&Entrypoint{},
|
||||||
|
&Target{},
|
||||||
|
)
|
||||||
|
}
|
||||||
@@ -253,7 +253,7 @@ func (r *RetentionReaper) reapWebhook(
|
|||||||
// row in the database: the sweep then deletes every event, delivery,
|
// row in the database: the sweep then deletes every event, delivery,
|
||||||
// and delivery result, including ones created seconds ago. Rejecting
|
// and delivery result, including ones created seconds ago. Rejecting
|
||||||
// out-of-range input at the form is the primary guard; saturating here
|
// out-of-range input at the form is the primary guard; saturating here
|
||||||
// means a value that reaches the row by any other path cannot turn a
|
// means an old row, a migration, or a future call site cannot turn a
|
||||||
// too-large retention into total data loss.
|
// too-large retention into total data loss.
|
||||||
func retentionCutoff(
|
func retentionCutoff(
|
||||||
now time.Time,
|
now time.Time,
|
||||||
|
|||||||
@@ -51,7 +51,7 @@ func setupRetentionTest(t *testing.T) *retentionTestEnv {
|
|||||||
|
|
||||||
mgr, err := database.NewWebhookDBManager(
|
mgr, err := database.NewWebhookDBManager(
|
||||||
lc,
|
lc,
|
||||||
database.WebhookDBManagerParams{Config: cfg, Logger: l.Get()},
|
database.WebhookDBManagerParams{Config: cfg, Logger: l},
|
||||||
)
|
)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
|||||||
@@ -1,168 +0,0 @@
|
|||||||
package database_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"log/slog"
|
|
||||||
"net/http"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/google/uuid"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"gorm.io/gorm"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database/databasetest"
|
|
||||||
)
|
|
||||||
|
|
||||||
// withFixedNow returns db with GORM's clock stopped at a time in UTC,
|
|
||||||
// so that the created and updated times it writes come back from
|
|
||||||
// SQLite equal to the row written, which holds them in UTC.
|
|
||||||
func withFixedNow(db *gorm.DB) (*gorm.DB, time.Time) {
|
|
||||||
at := time.Date(2026, 10, 13, 12, 30, 45, 123456789, time.UTC)
|
|
||||||
|
|
||||||
return db.Session(&gorm.Session{
|
|
||||||
NowFunc: func() time.Time { return at },
|
|
||||||
}), at
|
|
||||||
}
|
|
||||||
|
|
||||||
// newBaseModel returns a BaseModel with a new ID.
|
|
||||||
func newBaseModel() database.BaseModel {
|
|
||||||
return database.BaseModel{ID: uuid.New().String()}
|
|
||||||
}
|
|
||||||
|
|
||||||
// assertRoundTrip writes row, which must be the only row of its
|
|
||||||
// table, and reads back the same row.
|
|
||||||
func assertRoundTrip[T any](t *testing.T, db *gorm.DB, row *T) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
require.NoError(t, db.Create(row).Error)
|
|
||||||
|
|
||||||
var got T
|
|
||||||
|
|
||||||
require.NoError(t, db.First(&got).Error)
|
|
||||||
assert.Equal(t, *row, got)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestMainSchemaHoldsEveryModel writes a row of each main database
|
|
||||||
// model, every field set, into a database made from its SQL files.
|
|
||||||
func TestMainSchemaHoldsEveryModel(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
mainDB, err := database.Open(
|
|
||||||
t.TempDir(), slog.New(slog.DiscardHandler),
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
t.Cleanup(func() { _ = mainDB.Close() })
|
|
||||||
|
|
||||||
db, at := withFixedNow(mainDB.DB())
|
|
||||||
|
|
||||||
assertRoundTrip(t, db, &database.Setting{
|
|
||||||
Key: "session_key", Value: "value",
|
|
||||||
})
|
|
||||||
assertRoundTrip(t, db, &database.User{
|
|
||||||
BaseModel: newBaseModel(),
|
|
||||||
Username: "user",
|
|
||||||
Password: "password hash",
|
|
||||||
})
|
|
||||||
assertRoundTrip(t, db, &database.APIKey{
|
|
||||||
BaseModel: newBaseModel(),
|
|
||||||
UserID: uuid.New().String(),
|
|
||||||
Key: "key",
|
|
||||||
Description: "API key description",
|
|
||||||
LastUsedAt: &at,
|
|
||||||
})
|
|
||||||
assertRoundTrip(t, db, &database.Webhook{
|
|
||||||
BaseModel: newBaseModel(),
|
|
||||||
UserID: uuid.New().String(),
|
|
||||||
Name: "webhook",
|
|
||||||
Description: "webhook description",
|
|
||||||
RetentionDays: 7,
|
|
||||||
})
|
|
||||||
assertRoundTrip(t, db, &database.Entrypoint{
|
|
||||||
BaseModel: newBaseModel(),
|
|
||||||
WebhookID: uuid.New().String(),
|
|
||||||
Path: uuid.New().String(),
|
|
||||||
Description: "entrypoint description",
|
|
||||||
Active: true,
|
|
||||||
})
|
|
||||||
assertRoundTrip(t, db, &database.Target{
|
|
||||||
BaseModel: newBaseModel(),
|
|
||||||
WebhookID: uuid.New().String(),
|
|
||||||
Name: "target",
|
|
||||||
Type: database.TargetTypeHTTP,
|
|
||||||
Active: true,
|
|
||||||
Config: `{"url":"https://example.com/"}`,
|
|
||||||
MaxRetries: 3,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEventSchemaHoldsEveryModel does the same for a webhook's event
|
|
||||||
// database, which the manager makes from its SQL files.
|
|
||||||
func TestEventSchemaHoldsEveryModel(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
mgr := databasetest.NewWebhookDBManager(t, t.TempDir())
|
|
||||||
|
|
||||||
t.Cleanup(func() { _ = mgr.CloseAll() })
|
|
||||||
|
|
||||||
events, err := mgr.GetDB(uuid.New().String())
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
db, at := withFixedNow(events)
|
|
||||||
source := uuid.New().String()
|
|
||||||
|
|
||||||
assertRoundTrip(t, db, &database.Event{
|
|
||||||
BaseModel: newBaseModel(),
|
|
||||||
WebhookID: uuid.New().String(),
|
|
||||||
EntrypointID: uuid.New().String(),
|
|
||||||
Method: http.MethodPost,
|
|
||||||
RawQuery: "a=1",
|
|
||||||
Headers: `{"X":"Y"}`,
|
|
||||||
Body: "body",
|
|
||||||
ContentType: "text/plain",
|
|
||||||
BodyBytes: 4,
|
|
||||||
ResubmittedFromID: &source,
|
|
||||||
})
|
|
||||||
assertRoundTrip(t, db, &database.Delivery{
|
|
||||||
BaseModel: newBaseModel(),
|
|
||||||
EventID: uuid.New().String(),
|
|
||||||
TargetID: uuid.New().String(),
|
|
||||||
Status: database.DeliveryStatusDelivered,
|
|
||||||
FinishedAt: &at,
|
|
||||||
Replay: true,
|
|
||||||
})
|
|
||||||
assertRoundTrip(t, db, &database.DeliveryResult{
|
|
||||||
BaseModel: newBaseModel(),
|
|
||||||
DeliveryID: uuid.New().String(),
|
|
||||||
AttemptNum: 2,
|
|
||||||
Success: true,
|
|
||||||
StatusCode: 200,
|
|
||||||
ResponseBody: "ok",
|
|
||||||
Error: "error",
|
|
||||||
Duration: 15,
|
|
||||||
})
|
|
||||||
assertRoundTrip(t, db, &database.TargetTotals{
|
|
||||||
TargetID: uuid.New().String(),
|
|
||||||
Deliveries: 5,
|
|
||||||
Delivered: 3,
|
|
||||||
Failed: 1,
|
|
||||||
DeliveriesRemoved: 2,
|
|
||||||
FailedRemoved: 1,
|
|
||||||
})
|
|
||||||
assertRoundTrip(t, db, &database.EntrypointTotals{
|
|
||||||
EntrypointID: uuid.New().String(), LastEventAt: at,
|
|
||||||
})
|
|
||||||
|
|
||||||
// A new event database has its one row of event totals already.
|
|
||||||
require.NoError(t, database.AddEventTotals(db, database.EventTotals{
|
|
||||||
Events: 3, EventsRemoved: 1, LastEventAt: &at,
|
|
||||||
}))
|
|
||||||
|
|
||||||
var totals []database.EventTotals
|
|
||||||
|
|
||||||
require.NoError(t, db.Find(&totals).Error)
|
|
||||||
assert.Equal(t, []database.EventTotals{{
|
|
||||||
ID: 1, Events: 3, EventsRemoved: 1, LastEventAt: &at,
|
|
||||||
}}, totals)
|
|
||||||
}
|
|
||||||
@@ -118,8 +118,9 @@ const SQLiteFilePerm fs.FileMode = 0o600
|
|||||||
// SQLite still reports the absence rather than this function
|
// SQLite still reports the absence rather than this function
|
||||||
// materializing an empty database the caller asked not to create.
|
// materializing an empty database the caller asked not to create.
|
||||||
//
|
//
|
||||||
// The mode is set on every open, whoever created the file, so a
|
// Chmod of a file that already exists is what tightens a data
|
||||||
// database or sidecar found at a wider mode is tightened too.
|
// directory an earlier build left at 0644 — including a developer's
|
||||||
|
// own scratch directory — without any migration machinery.
|
||||||
func reserveSQLiteFile(path string, create bool) error {
|
func reserveSQLiteFile(path string, create bool) error {
|
||||||
if create {
|
if create {
|
||||||
// gosec G304: the path is the database file the caller asked
|
// gosec G304: the path is the database file the caller asked
|
||||||
@@ -241,19 +242,7 @@ func SQLiteDSN(path, mode string) string {
|
|||||||
)
|
)
|
||||||
q.Add("_pragma", "journal_mode(WAL)")
|
q.Add("_pragma", "journal_mode(WAL)")
|
||||||
|
|
||||||
// SQLite reads a file: URI's path only up to the first ? or #, and
|
return "file:" + path + "?" + q.Encode()
|
||||||
// decodes %XX escapes in it, so url.URL escapes the path. OmitHost
|
|
||||||
// keeps a relative path as "file:data/x.db"; written
|
|
||||||
// "file://data/x.db", SQLite would reject "data" as the URI's
|
|
||||||
// authority.
|
|
||||||
dsn := url.URL{
|
|
||||||
Scheme: "file",
|
|
||||||
Path: path,
|
|
||||||
OmitHost: true,
|
|
||||||
RawQuery: q.Encode(),
|
|
||||||
}
|
|
||||||
|
|
||||||
return dsn.String()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// OpenSQLite opens the SQLite file at path with the service's
|
// OpenSQLite opens the SQLite file at path with the service's
|
||||||
|
|||||||
@@ -2,8 +2,6 @@ package database_test
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"database/sql"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -114,79 +112,6 @@ func TestMainDBAppliesPragmasOnALiveHandle(t *testing.T) {
|
|||||||
assert.Equal(t, "10000", busy)
|
assert.Equal(t, "10000", busy)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestOpenSQLiteOpensThePathAskedFor uses a data directory whose name
|
|
||||||
// holds ?, # and %. SQLite reads a file: URI's path only up to a ? or
|
|
||||||
// #, and decodes %41 to A, so an unescaped path sent every database in
|
|
||||||
// that directory to one file outside it.
|
|
||||||
func TestOpenSQLiteOpensThePathAskedFor(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := filepath.Join(t.TempDir(), "hooks#1?x%41")
|
|
||||||
require.NoError(t, os.Mkdir(dir, 0o700))
|
|
||||||
|
|
||||||
for _, name := range []string{
|
|
||||||
database.MainDBFileName,
|
|
||||||
"events-" + uuid.New().String() + ".db",
|
|
||||||
} {
|
|
||||||
path := filepath.Join(dir, name)
|
|
||||||
|
|
||||||
sqlDB, err := database.OpenSQLite(
|
|
||||||
path, database.SQLiteModeCreate,
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
t.Cleanup(func() { assert.NoError(t, sqlDB.Close()) })
|
|
||||||
|
|
||||||
assertOpenedFile(t, sqlDB, path)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestOpenSQLiteOpensARelativePath opens a database at a path relative
|
|
||||||
// to the working directory, which is what a relative DATA_DIR such as
|
|
||||||
// the README's DATA_DIR=../webhooker-data gives OpenSQLite.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // t.Chdir moves the whole process.
|
|
||||||
func TestOpenSQLiteOpensARelativePath(t *testing.T) {
|
|
||||||
t.Chdir(t.TempDir())
|
|
||||||
require.NoError(t, os.Mkdir("data", 0o700))
|
|
||||||
|
|
||||||
path := filepath.Join("data", database.MainDBFileName)
|
|
||||||
|
|
||||||
sqlDB, err := database.OpenSQLite(path, database.SQLiteModeCreate)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
defer func() { require.NoError(t, sqlDB.Close()) }()
|
|
||||||
|
|
||||||
assertOpenedFile(t, sqlDB, path)
|
|
||||||
}
|
|
||||||
|
|
||||||
// assertOpenedFile checks that sqlDB is the SQLite file at path.
|
|
||||||
// database_list names the file SQLite actually opened, as an absolute
|
|
||||||
// path with symlinks resolved, so the files are compared, not the names.
|
|
||||||
func assertOpenedFile(t *testing.T, sqlDB *sql.DB, path string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var (
|
|
||||||
seq int
|
|
||||||
schema, file string
|
|
||||||
)
|
|
||||||
|
|
||||||
require.NoError(t, sqlDB.
|
|
||||||
QueryRowContext(t.Context(), "pragma database_list").
|
|
||||||
Scan(&seq, &schema, &file))
|
|
||||||
|
|
||||||
asked, err := os.Stat(path)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
opened, err := os.Stat(file)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
assert.True(
|
|
||||||
t, os.SameFile(asked, opened),
|
|
||||||
"asked for %s, SQLite opened %s", path, file,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestConcurrentReaderDoesNotBlockWrites is the unit-scale form of the
|
// TestConcurrentReaderDoesNotBlockWrites is the unit-scale form of the
|
||||||
// reproduction in
|
// reproduction in
|
||||||
// https://git.eeqj.de/sneak/webhooker/issues/256: an operator's
|
// https://git.eeqj.de/sneak/webhooker/issues/256: an operator's
|
||||||
|
|||||||
@@ -0,0 +1,47 @@
|
|||||||
|
package database
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"gorm.io/gorm"
|
||||||
|
)
|
||||||
|
|
||||||
|
// NewTestDatabase creates a Database wrapper around a pre-opened *gorm.DB.
|
||||||
|
// Intended for use in tests that need a *database.Database without the
|
||||||
|
// full fx lifecycle. The caller is responsible for closing the underlying
|
||||||
|
// sql.DB connection.
|
||||||
|
func NewTestDatabase(db *gorm.DB) *Database {
|
||||||
|
return &Database{
|
||||||
|
db: db,
|
||||||
|
log: slog.New(slog.NewTextHandler(
|
||||||
|
os.Stderr,
|
||||||
|
&slog.HandlerOptions{Level: slog.LevelDebug},
|
||||||
|
)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewTestWebhookDBManager creates a WebhookDBManager backed by the given
|
||||||
|
// data directory. Intended for use in tests without the fx lifecycle.
|
||||||
|
func NewTestWebhookDBManager(dataDir string) *WebhookDBManager {
|
||||||
|
return NewTestWebhookDBManagerWithLogger(
|
||||||
|
dataDir,
|
||||||
|
slog.New(slog.NewTextHandler(
|
||||||
|
os.Stderr,
|
||||||
|
&slog.HandlerOptions{Level: slog.LevelDebug},
|
||||||
|
)),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewTestWebhookDBManagerWithLogger is NewTestWebhookDBManager with the
|
||||||
|
// logger supplied by the caller. The per-webhook databases this manager
|
||||||
|
// opens hand that logger to gormlog, so a test that needs to see the SQL
|
||||||
|
// the service emits can capture it.
|
||||||
|
func NewTestWebhookDBManagerWithLogger(
|
||||||
|
dataDir string, log *slog.Logger,
|
||||||
|
) *WebhookDBManager {
|
||||||
|
return &WebhookDBManager{
|
||||||
|
dataDir: dataDir,
|
||||||
|
log: log,
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -14,8 +14,8 @@ import (
|
|||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/datadir"
|
"sneak.berlin/go/webhooker/internal/datadir"
|
||||||
"sneak.berlin/go/webhooker/internal/db/migrations"
|
|
||||||
"sneak.berlin/go/webhooker/internal/gormlog"
|
"sneak.berlin/go/webhooker/internal/gormlog"
|
||||||
|
"sneak.berlin/go/webhooker/internal/logger"
|
||||||
)
|
)
|
||||||
|
|
||||||
// WebhookDBManagerParams holds the fx dependencies for
|
// WebhookDBManagerParams holds the fx dependencies for
|
||||||
@@ -24,7 +24,7 @@ type WebhookDBManagerParams struct {
|
|||||||
fx.In
|
fx.In
|
||||||
|
|
||||||
Config *config.Config
|
Config *config.Config
|
||||||
Logger *slog.Logger
|
Logger *logger.Logger
|
||||||
}
|
}
|
||||||
|
|
||||||
// errInvalidCachedDBType indicates a type assertion failure
|
// errInvalidCachedDBType indicates a type assertion failure
|
||||||
@@ -70,7 +70,7 @@ func NewWebhookDBManager(
|
|||||||
) (*WebhookDBManager, error) {
|
) (*WebhookDBManager, error) {
|
||||||
m := &WebhookDBManager{
|
m := &WebhookDBManager{
|
||||||
dataDir: params.Config.DataDir,
|
dataDir: params.Config.DataDir,
|
||||||
log: params.Logger,
|
log: params.Logger.Get(),
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create data directory if it doesn't exist. datadir.DirPerm is the
|
// Create data directory if it doesn't exist. datadir.DirPerm is the
|
||||||
@@ -371,7 +371,18 @@ func (m *WebhookDBManager) openDB(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = migrations.Apply(db, migrations.Events)
|
err = purgeTargetRows(db, m.log, webhookID)
|
||||||
|
if err != nil {
|
||||||
|
_ = sqlDB.Close()
|
||||||
|
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run migrations for event-tier models only
|
||||||
|
err = db.AutoMigrate(
|
||||||
|
&Event{}, &Delivery{}, &DeliveryResult{},
|
||||||
|
&EventTotals{}, &TargetTotals{}, &EntrypointTotals{},
|
||||||
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = sqlDB.Close()
|
_ = sqlDB.Close()
|
||||||
|
|
||||||
@@ -381,15 +392,14 @@ func (m *WebhookDBManager) openDB(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = parseModels(
|
// A new database gets its row of event totals, all zero. Target
|
||||||
db, &Event{}, &Delivery{}, &DeliveryResult{},
|
// totals rows are created by the first delivery to each target.
|
||||||
&EventTotals{}, &TargetTotals{}, &EntrypointTotals{},
|
err = db.FirstOrCreate(&EventTotals{}).Error
|
||||||
)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = sqlDB.Close()
|
_ = sqlDB.Close()
|
||||||
|
|
||||||
return nil, fmt.Errorf(
|
return nil, fmt.Errorf(
|
||||||
"connecting to webhook database %s: %w",
|
"creating event totals for webhook database %s: %w",
|
||||||
webhookID, err,
|
webhookID, err,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,7 +18,6 @@ import (
|
|||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/database/databasetest"
|
|
||||||
"sneak.berlin/go/webhooker/internal/globals"
|
"sneak.berlin/go/webhooker/internal/globals"
|
||||||
"sneak.berlin/go/webhooker/internal/logger"
|
"sneak.berlin/go/webhooker/internal/logger"
|
||||||
)
|
)
|
||||||
@@ -51,7 +50,7 @@ func setupTestWebhookDBManager(
|
|||||||
lc,
|
lc,
|
||||||
database.WebhookDBManagerParams{
|
database.WebhookDBManagerParams{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
Logger: l.Get(),
|
Logger: l,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -118,8 +117,7 @@ func TestWebhookDBManager_ConcurrentFirstTouchOpensOnce(t *testing.T) {
|
|||||||
|
|
||||||
var logs bytes.Buffer
|
var logs bytes.Buffer
|
||||||
|
|
||||||
mgr := databasetest.NewWebhookDBManagerWithLogger(
|
mgr := database.NewTestWebhookDBManagerWithLogger(
|
||||||
t,
|
|
||||||
t.TempDir(),
|
t.TempDir(),
|
||||||
slog.New(slog.NewTextHandler(&logs, nil)),
|
slog.New(slog.NewTextHandler(&logs, nil)),
|
||||||
)
|
)
|
||||||
@@ -309,8 +307,7 @@ func TestWebhookDBManager_LostDatabaseIsLogged(t *testing.T) {
|
|||||||
|
|
||||||
var logs bytes.Buffer
|
var logs bytes.Buffer
|
||||||
|
|
||||||
mgr := databasetest.NewWebhookDBManagerWithLogger(
|
mgr := database.NewTestWebhookDBManagerWithLogger(
|
||||||
t,
|
|
||||||
t.TempDir(),
|
t.TempDir(),
|
||||||
slog.New(slog.NewTextHandler(&logs, nil)),
|
slog.New(slog.NewTextHandler(&logs, nil)),
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,9 +0,0 @@
|
|||||||
-- Migration 000: the table recording which migrations this file has
|
|
||||||
-- had. Applied before the others; it records itself.
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
|
||||||
version INTEGER PRIMARY KEY,
|
|
||||||
applied_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
|
||||||
);
|
|
||||||
|
|
||||||
INSERT OR IGNORE INTO schema_migrations (version) VALUES (0);
|
|
||||||
@@ -1,18 +0,0 @@
|
|||||||
-- Migration 001: the schema of a database target's archive file. The
|
|
||||||
-- archivedEvent model in internal/delivery must agree with it.
|
|
||||||
|
|
||||||
CREATE TABLE archived_events (
|
|
||||||
id integer PRIMARY KEY AUTOINCREMENT,
|
|
||||||
event_id text,
|
|
||||||
webhook_id text,
|
|
||||||
entrypoint_id text,
|
|
||||||
method text,
|
|
||||||
raw_query text,
|
|
||||||
headers text,
|
|
||||||
body text,
|
|
||||||
content_type text,
|
|
||||||
archived_at datetime
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX idx_archived_events_event_id ON archived_events (event_id);
|
|
||||||
CREATE INDEX idx_archived_events_archived_at ON archived_events (archived_at);
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
-- Migration 000: the table recording which migrations this file has
|
|
||||||
-- had. Applied before the others; it records itself.
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
|
||||||
version INTEGER PRIMARY KEY,
|
|
||||||
applied_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
|
||||||
);
|
|
||||||
|
|
||||||
INSERT OR IGNORE INTO schema_migrations (version) VALUES (0);
|
|
||||||
@@ -1,98 +0,0 @@
|
|||||||
-- Migration 001: the schema of a webhook's event database. The models
|
|
||||||
-- in internal/database must agree with it. The webhook, entrypoint and
|
|
||||||
-- target ids here name rows in the main database, which has those
|
|
||||||
-- tables; this file does not.
|
|
||||||
|
|
||||||
CREATE TABLE events (
|
|
||||||
id uuid,
|
|
||||||
created_at datetime,
|
|
||||||
updated_at datetime,
|
|
||||||
deleted_at datetime,
|
|
||||||
webhook_id uuid NOT NULL,
|
|
||||||
entrypoint_id uuid NOT NULL,
|
|
||||||
method text NOT NULL,
|
|
||||||
raw_query text,
|
|
||||||
headers text,
|
|
||||||
body text,
|
|
||||||
content_type text,
|
|
||||||
body_bytes integer NOT NULL,
|
|
||||||
resubmitted_from_id uuid,
|
|
||||||
PRIMARY KEY (id)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX idx_events_created_at ON events (created_at);
|
|
||||||
CREATE INDEX idx_events_deleted_at ON events (deleted_at);
|
|
||||||
CREATE INDEX idx_events_deleted_at_created_at
|
|
||||||
ON events (deleted_at, created_at);
|
|
||||||
CREATE INDEX idx_events_resubmitted_from_id
|
|
||||||
ON events (resubmitted_from_id, deleted_at);
|
|
||||||
CREATE INDEX idx_events_entrypoint_id
|
|
||||||
ON events (entrypoint_id, deleted_at, resubmitted_from_id, created_at);
|
|
||||||
|
|
||||||
CREATE TABLE deliveries (
|
|
||||||
id uuid,
|
|
||||||
created_at datetime,
|
|
||||||
updated_at datetime,
|
|
||||||
deleted_at datetime,
|
|
||||||
event_id uuid NOT NULL,
|
|
||||||
target_id uuid NOT NULL,
|
|
||||||
status text NOT NULL DEFAULT 'pending',
|
|
||||||
finished_at datetime,
|
|
||||||
replay numeric NOT NULL DEFAULT false,
|
|
||||||
PRIMARY KEY (id),
|
|
||||||
CONSTRAINT fk_events_deliveries
|
|
||||||
FOREIGN KEY (event_id) REFERENCES events (id)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX idx_deliveries_deleted_at ON deliveries (deleted_at);
|
|
||||||
CREATE INDEX idx_deliveries_event_id ON deliveries (event_id, deleted_at);
|
|
||||||
CREATE INDEX idx_deliveries_status
|
|
||||||
ON deliveries (status, deleted_at, finished_at, target_id);
|
|
||||||
|
|
||||||
CREATE TABLE delivery_results (
|
|
||||||
id uuid,
|
|
||||||
created_at datetime,
|
|
||||||
updated_at datetime,
|
|
||||||
deleted_at datetime,
|
|
||||||
delivery_id uuid NOT NULL,
|
|
||||||
attempt_num integer NOT NULL,
|
|
||||||
success numeric,
|
|
||||||
status_code integer,
|
|
||||||
response_body text,
|
|
||||||
error text,
|
|
||||||
duration integer,
|
|
||||||
PRIMARY KEY (id),
|
|
||||||
CONSTRAINT fk_deliveries_delivery_results
|
|
||||||
FOREIGN KEY (delivery_id) REFERENCES deliveries (id)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX idx_delivery_results_deleted_at
|
|
||||||
ON delivery_results (deleted_at);
|
|
||||||
CREATE INDEX idx_delivery_results_delivery_id
|
|
||||||
ON delivery_results (delivery_id, deleted_at);
|
|
||||||
|
|
||||||
CREATE TABLE event_totals (
|
|
||||||
id integer PRIMARY KEY AUTOINCREMENT,
|
|
||||||
events integer NOT NULL,
|
|
||||||
events_removed integer NOT NULL,
|
|
||||||
last_event_at datetime
|
|
||||||
);
|
|
||||||
|
|
||||||
-- The one row of event totals, all zero.
|
|
||||||
INSERT INTO event_totals (events, events_removed) VALUES (0, 0);
|
|
||||||
|
|
||||||
CREATE TABLE target_totals (
|
|
||||||
target_id uuid,
|
|
||||||
deliveries integer NOT NULL,
|
|
||||||
delivered integer NOT NULL,
|
|
||||||
failed integer NOT NULL,
|
|
||||||
deliveries_removed integer NOT NULL,
|
|
||||||
failed_removed integer NOT NULL,
|
|
||||||
PRIMARY KEY (target_id)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE entrypoint_totals (
|
|
||||||
entrypoint_id uuid,
|
|
||||||
last_event_at datetime NOT NULL,
|
|
||||||
PRIMARY KEY (entrypoint_id)
|
|
||||||
);
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
-- Migration 000: the table recording which migrations this file has
|
|
||||||
-- had. Applied before the others; it records itself.
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
|
||||||
version INTEGER PRIMARY KEY,
|
|
||||||
applied_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
|
||||||
);
|
|
||||||
|
|
||||||
INSERT OR IGNORE INTO schema_migrations (version) VALUES (0);
|
|
||||||
@@ -1,89 +0,0 @@
|
|||||||
-- Migration 001: the main database's schema. The models in
|
|
||||||
-- internal/database must agree with it.
|
|
||||||
|
|
||||||
CREATE TABLE settings (
|
|
||||||
"key" text,
|
|
||||||
value text NOT NULL,
|
|
||||||
PRIMARY KEY ("key")
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE users (
|
|
||||||
id uuid,
|
|
||||||
created_at datetime,
|
|
||||||
updated_at datetime,
|
|
||||||
deleted_at datetime,
|
|
||||||
username text NOT NULL,
|
|
||||||
password text NOT NULL,
|
|
||||||
PRIMARY KEY (id),
|
|
||||||
CONSTRAINT chk_users_username
|
|
||||||
CHECK (length(CAST(username AS BLOB)) <= 1024)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX idx_users_deleted_at ON users (deleted_at);
|
|
||||||
CREATE UNIQUE INDEX idx_users_username ON users (username);
|
|
||||||
|
|
||||||
CREATE TABLE api_keys (
|
|
||||||
id uuid,
|
|
||||||
created_at datetime,
|
|
||||||
updated_at datetime,
|
|
||||||
deleted_at datetime,
|
|
||||||
user_id uuid NOT NULL,
|
|
||||||
"key" text NOT NULL,
|
|
||||||
description text,
|
|
||||||
last_used_at datetime,
|
|
||||||
PRIMARY KEY (id),
|
|
||||||
CONSTRAINT fk_users_api_keys FOREIGN KEY (user_id) REFERENCES users (id)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX idx_api_keys_deleted_at ON api_keys (deleted_at);
|
|
||||||
CREATE UNIQUE INDEX idx_api_keys_key ON api_keys ("key");
|
|
||||||
|
|
||||||
CREATE TABLE webhooks (
|
|
||||||
id uuid,
|
|
||||||
created_at datetime,
|
|
||||||
updated_at datetime,
|
|
||||||
deleted_at datetime,
|
|
||||||
user_id uuid NOT NULL,
|
|
||||||
name text NOT NULL,
|
|
||||||
description text,
|
|
||||||
retention_days integer DEFAULT 30,
|
|
||||||
PRIMARY KEY (id),
|
|
||||||
CONSTRAINT fk_users_webhooks FOREIGN KEY (user_id) REFERENCES users (id)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX idx_webhooks_deleted_at ON webhooks (deleted_at);
|
|
||||||
|
|
||||||
CREATE TABLE entrypoints (
|
|
||||||
id uuid,
|
|
||||||
created_at datetime,
|
|
||||||
updated_at datetime,
|
|
||||||
deleted_at datetime,
|
|
||||||
webhook_id uuid NOT NULL,
|
|
||||||
path text NOT NULL,
|
|
||||||
description text,
|
|
||||||
active numeric DEFAULT true,
|
|
||||||
PRIMARY KEY (id),
|
|
||||||
CONSTRAINT fk_webhooks_entrypoints
|
|
||||||
FOREIGN KEY (webhook_id) REFERENCES webhooks (id)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX idx_entrypoints_deleted_at ON entrypoints (deleted_at);
|
|
||||||
CREATE UNIQUE INDEX idx_entrypoints_path ON entrypoints (path);
|
|
||||||
|
|
||||||
CREATE TABLE targets (
|
|
||||||
id uuid,
|
|
||||||
created_at datetime,
|
|
||||||
updated_at datetime,
|
|
||||||
deleted_at datetime,
|
|
||||||
webhook_id uuid NOT NULL,
|
|
||||||
name text NOT NULL,
|
|
||||||
type text NOT NULL,
|
|
||||||
active numeric DEFAULT true,
|
|
||||||
config text,
|
|
||||||
max_retries integer,
|
|
||||||
PRIMARY KEY (id),
|
|
||||||
CONSTRAINT fk_webhooks_targets
|
|
||||||
FOREIGN KEY (webhook_id) REFERENCES webhooks (id)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX idx_targets_deleted_at ON targets (deleted_at);
|
|
||||||
@@ -1,135 +0,0 @@
|
|||||||
// Package migrations holds the schema of each kind of SQLite file
|
|
||||||
// webhooker writes, as numbered SQL files embedded in the binary, and
|
|
||||||
// applies it when a file is opened.
|
|
||||||
//
|
|
||||||
// Each kind of file has its own directory. In each, 000_migration.sql
|
|
||||||
// creates only the schema_migrations table, and 001_schema.sql holds
|
|
||||||
// the whole schema. Before 1.0 a schema change is an edit to
|
|
||||||
// 001_schema.sql, never a new file.
|
|
||||||
package migrations
|
|
||||||
|
|
||||||
import (
|
|
||||||
"embed"
|
|
||||||
"fmt"
|
|
||||||
"io/fs"
|
|
||||||
"path"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"gorm.io/gorm"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The directories, one for each kind of database file.
|
|
||||||
const (
|
|
||||||
// Main is the main database, webhooker.db.
|
|
||||||
Main = "main"
|
|
||||||
|
|
||||||
// Events is a webhook's event database, events-{uuid}.db.
|
|
||||||
Events = "events"
|
|
||||||
|
|
||||||
// Archive is a database target's archive file.
|
|
||||||
Archive = "archive"
|
|
||||||
)
|
|
||||||
|
|
||||||
//go:embed main/*.sql events/*.sql archive/*.sql
|
|
||||||
var files embed.FS
|
|
||||||
|
|
||||||
// Apply runs each SQL file in dir, which is Main, Events or Archive,
|
|
||||||
// that the database has not had yet, in the order of the number its
|
|
||||||
// name starts with, and records that number in schema_migrations.
|
|
||||||
// 000_migration.sql creates that table and records itself, so it runs
|
|
||||||
// first, and only when the table is missing.
|
|
||||||
//
|
|
||||||
// It all runs in one transaction, so a failure leaves the file as it
|
|
||||||
// was.
|
|
||||||
func Apply(db *gorm.DB, dir string) error {
|
|
||||||
return db.Transaction(func(tx *gorm.DB) error {
|
|
||||||
var tables int
|
|
||||||
|
|
||||||
// Row().Scan, not (*gorm.DB).Scan: see internal/gormlog.
|
|
||||||
err := tx.Raw(
|
|
||||||
"SELECT count(*) FROM sqlite_master "+
|
|
||||||
"WHERE type = 'table' AND name = ?",
|
|
||||||
"schema_migrations",
|
|
||||||
).Row().Scan(&tables)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("looking for schema_migrations: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if tables == 0 {
|
|
||||||
err = run(tx, path.Join(dir, "000_migration.sql"))
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// fs.Glob lists the files in name order, which is number order.
|
|
||||||
names, err := fs.Glob(files, path.Join(dir, "*.sql"))
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("listing migrations: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, name := range names {
|
|
||||||
err = applyOnce(tx, name)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// applyOnce runs the migration file name and records its number,
|
|
||||||
// unless schema_migrations already has that number.
|
|
||||||
func applyOnce(tx *gorm.DB, name string) error {
|
|
||||||
number, _, _ := strings.Cut(path.Base(name), "_")
|
|
||||||
|
|
||||||
version, err := strconv.Atoi(number)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("numbering migration %s: %w", name, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var applied int
|
|
||||||
|
|
||||||
err = tx.Raw(
|
|
||||||
"SELECT count(*) FROM schema_migrations WHERE version = ?",
|
|
||||||
version,
|
|
||||||
).Row().Scan(&applied)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("checking migration %s: %w", name, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if applied > 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
err = run(tx, name)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = tx.Exec(
|
|
||||||
"INSERT INTO schema_migrations (version) VALUES (?)", version,
|
|
||||||
).Error
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("recording migration %s: %w", name, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// run executes every statement in the migration file name.
|
|
||||||
func run(tx *gorm.DB, name string) error {
|
|
||||||
script, err := files.ReadFile(name)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("reading migration %s: %w", name, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = tx.Exec(string(script)).Error
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("applying migration %s: %w", name, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,57 +0,0 @@
|
|||||||
package migrations_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"database/sql"
|
|
||||||
"log/slog"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"gorm.io/driver/sqlite"
|
|
||||||
"gorm.io/gorm"
|
|
||||||
_ "modernc.org/sqlite"
|
|
||||||
"sneak.berlin/go/webhooker/internal/db/migrations"
|
|
||||||
"sneak.berlin/go/webhooker/internal/gormlog"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestApplyRunsEachFileOnce applies each kind of database file's
|
|
||||||
// migrations to a new file twice. The first time records every file.
|
|
||||||
// The second finds them all recorded and runs none, which is the only
|
|
||||||
// way it can succeed: 001_schema.sql creates tables that then exist.
|
|
||||||
func TestApplyRunsEachFileOnce(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, dir := range []string{
|
|
||||||
migrations.Main, migrations.Events, migrations.Archive,
|
|
||||||
} {
|
|
||||||
t.Run(dir, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
sqlDB, err := sql.Open(
|
|
||||||
"sqlite", filepath.Join(t.TempDir(), "test.db"),
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
t.Cleanup(func() { _ = sqlDB.Close() })
|
|
||||||
|
|
||||||
db, err := gorm.Open(
|
|
||||||
sqlite.Dialector{Conn: sqlDB},
|
|
||||||
&gorm.Config{
|
|
||||||
Logger: gormlog.New(slog.New(slog.DiscardHandler)),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
require.NoError(t, migrations.Apply(db, dir))
|
|
||||||
require.NoError(t, migrations.Apply(db, dir))
|
|
||||||
|
|
||||||
var versions string
|
|
||||||
|
|
||||||
require.NoError(t, sqlDB.QueryRowContext(t.Context(),
|
|
||||||
"SELECT group_concat(version) FROM schema_migrations",
|
|
||||||
).Scan(&versions))
|
|
||||||
assert.Equal(t, "0,1", versions)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -201,7 +201,8 @@ func (s *ArchiveSweeper) sweep(ctx context.Context) {
|
|||||||
|
|
||||||
// sweepTarget prunes the archive of a single database target.
|
// sweepTarget prunes the archive of a single database target.
|
||||||
// A missing, empty, or "never" expiry parses as a zero duration
|
// A missing, empty, or "never" expiry parses as a zero duration
|
||||||
// and is skipped entirely, so those archives are never pruned.
|
// and is skipped entirely, so those archives keep exactly the
|
||||||
|
// behaviour they had before the sweep existed.
|
||||||
func (s *ArchiveSweeper) sweepTarget(target *database.Target) {
|
func (s *ArchiveSweeper) sweepTarget(target *database.Target) {
|
||||||
expiry, err := parseArchiveExpiry(target.Config)
|
expiry, err := parseArchiveExpiry(target.Config)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -21,8 +21,6 @@ import (
|
|||||||
"gorm.io/gorm/clause"
|
"gorm.io/gorm/clause"
|
||||||
_ "modernc.org/sqlite" // Pure Go SQLite driver.
|
_ "modernc.org/sqlite" // Pure Go SQLite driver.
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/database/databasetest"
|
|
||||||
"sneak.berlin/go/webhooker/internal/db/migrations"
|
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
"sneak.berlin/go/webhooker/internal/gormlog"
|
"sneak.berlin/go/webhooker/internal/gormlog"
|
||||||
)
|
)
|
||||||
@@ -61,14 +59,29 @@ func setupArchiveTest(t *testing.T) *archiveEnv {
|
|||||||
dataDir := t.TempDir()
|
dataDir := t.TempDir()
|
||||||
log := archiveTestLogger()
|
log := archiveTestLogger()
|
||||||
|
|
||||||
mainDB, err := database.Open(dataDir, slog.New(slog.DiscardHandler))
|
sqlDB, err := sql.Open(
|
||||||
|
"sqlite",
|
||||||
|
fmt.Sprintf(
|
||||||
|
"file:%s?mode=rwc",
|
||||||
|
filepath.Join(dataDir, "main.db"),
|
||||||
|
),
|
||||||
|
)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
t.Cleanup(func() { _ = mainDB.Close() })
|
t.Cleanup(func() { _ = sqlDB.Close() })
|
||||||
|
|
||||||
|
gdb, err := gorm.Open(
|
||||||
|
sqlite.Dialector{Conn: sqlDB},
|
||||||
|
&gorm.Config{Logger: gormlog.New(slog.New(slog.DiscardHandler))},
|
||||||
|
)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
mainDB := database.NewTestDatabase(gdb)
|
||||||
|
require.NoError(t, mainDB.Migrate())
|
||||||
|
|
||||||
eng := delivery.NewTestEngineWithDB(
|
eng := delivery.NewTestEngineWithDB(
|
||||||
mainDB,
|
mainDB,
|
||||||
databasetest.NewWebhookDBManager(t, dataDir),
|
database.NewTestWebhookDBManager(dataDir),
|
||||||
log,
|
log,
|
||||||
&http.Client{Timeout: 5 * time.Second},
|
&http.Client{Timeout: 5 * time.Second},
|
||||||
1,
|
1,
|
||||||
@@ -150,17 +163,6 @@ func (env *archiveEnv) seedArchiveRows(
|
|||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
path := env.archivePath(tgt)
|
path := env.archivePath(tgt)
|
||||||
seedArchiveFile(t, path, tgt.WebhookID, archivedAt...)
|
|
||||||
|
|
||||||
return path
|
|
||||||
}
|
|
||||||
|
|
||||||
// seedArchiveFile creates the archive file at path and inserts one row
|
|
||||||
// per supplied archived-at timestamp, as seedArchiveRows does.
|
|
||||||
func seedArchiveFile(
|
|
||||||
t *testing.T, path, webhookID string, archivedAt ...time.Time,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
sqlDB, err := sql.Open(
|
sqlDB, err := sql.Open(
|
||||||
"sqlite", fmt.Sprintf("file:%s?mode=rwc", path),
|
"sqlite", fmt.Sprintf("file:%s?mode=rwc", path),
|
||||||
@@ -173,12 +175,14 @@ func seedArchiveFile(
|
|||||||
)
|
)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
require.NoError(t, migrations.Apply(gdb, migrations.Archive))
|
require.NoError(
|
||||||
|
t, gdb.AutoMigrate(&delivery.ExportArchivedEvent{}),
|
||||||
|
)
|
||||||
|
|
||||||
for i, at := range archivedAt {
|
for i, at := range archivedAt {
|
||||||
row := delivery.ExportArchivedEvent{
|
row := delivery.ExportArchivedEvent{
|
||||||
EventID: fmt.Sprintf("ev-%d", i),
|
EventID: fmt.Sprintf("ev-%d", i),
|
||||||
WebhookID: webhookID,
|
WebhookID: tgt.WebhookID,
|
||||||
Method: http.MethodPost,
|
Method: http.MethodPost,
|
||||||
Body: `{"seeded":true}`,
|
Body: `{"seeded":true}`,
|
||||||
ArchivedAt: at,
|
ArchivedAt: at,
|
||||||
@@ -187,6 +191,8 @@ func seedArchiveFile(
|
|||||||
}
|
}
|
||||||
|
|
||||||
require.NoError(t, sqlDB.Close())
|
require.NoError(t, sqlDB.Close())
|
||||||
|
|
||||||
|
return path
|
||||||
}
|
}
|
||||||
|
|
||||||
// archivedEventIDs returns the event ids currently stored in an
|
// archivedEventIDs returns the event ids currently stored in an
|
||||||
@@ -780,9 +786,9 @@ func TestArchiveSweep_NeverExpiryUntouched(t *testing.T) {
|
|||||||
//
|
//
|
||||||
// The spec is stronger than that — a "never" archive is skipped
|
// The spec is stronger than that — a "never" archive is skipped
|
||||||
// before any file is touched — so the archive here exists but has
|
// before any file is touched — so the archive here exists but has
|
||||||
// never been migrated. Opening it at all would apply the archive
|
// never been migrated. Opening it at all would run AutoMigrate
|
||||||
// schema and create the archive table, which is exactly what must
|
// and create the archive table, which is exactly what must not
|
||||||
// not happen.
|
// happen.
|
||||||
func TestArchiveSweep_NeverExpirySkipsBeforeOpening(
|
func TestArchiveSweep_NeverExpirySkipsBeforeOpening(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
) {
|
) {
|
||||||
@@ -806,7 +812,7 @@ func TestArchiveSweep_NeverExpirySkipsBeforeOpening(
|
|||||||
|
|
||||||
// seedUnmigratedArchive creates an archive file that exists but
|
// seedUnmigratedArchive creates an archive file that exists but
|
||||||
// carries no archive schema, so any open of it is observable: the
|
// carries no archive schema, so any open of it is observable: the
|
||||||
// archive table appears only if something applied the schema.
|
// archive table appears only if something ran AutoMigrate.
|
||||||
func seedUnmigratedArchive(t *testing.T, path string) {
|
func seedUnmigratedArchive(t *testing.T, path string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
|
|||||||
@@ -110,7 +110,6 @@ type Task struct {
|
|||||||
MaxRetries int
|
MaxRetries int
|
||||||
|
|
||||||
Method string
|
Method string
|
||||||
RawQuery string
|
|
||||||
Headers string
|
Headers string
|
||||||
ContentType string
|
ContentType string
|
||||||
Body *string
|
Body *string
|
||||||
@@ -296,13 +295,12 @@ func (e *Engine) EvictTarget(targetID string) {
|
|||||||
e.dbTarget.evict(targetID)
|
e.dbTarget.evict(targetID)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Rename implements Archives. It renames every one of a database
|
// Rename implements Archives. It renames a database target's
|
||||||
// target's archive files to ArchiveFileName(webhookName, targetName,
|
// archive file to ArchiveFileName(webhookName, targetName,
|
||||||
// targetID), each keeping the period in its name, under the lock the
|
// targetID), under the lock the target's archive writes and the
|
||||||
// target's archive writes and the idle sweep take. It never replaces
|
// idle sweep take. It never replaces a file: if one already has the
|
||||||
// a file: if one already has a new name, the error is
|
// new name, the error is ErrArchiveNameTaken. The caller renames
|
||||||
// ErrArchiveNameTaken. The caller renames before it saves the new
|
// before it saves the new name: see databaseTarget.rename.
|
||||||
// name: see databaseTarget.rename.
|
|
||||||
func (e *Engine) Rename(
|
func (e *Engine) Rename(
|
||||||
targetID, webhookName, targetName string,
|
targetID, webhookName, targetName string,
|
||||||
) error {
|
) error {
|
||||||
@@ -1753,7 +1751,6 @@ func buildEventFromTask(task *Task) database.Event {
|
|||||||
event := database.Event{
|
event := database.Event{
|
||||||
EntrypointID: task.EntrypointID,
|
EntrypointID: task.EntrypointID,
|
||||||
Method: task.Method,
|
Method: task.Method,
|
||||||
RawQuery: task.RawQuery,
|
|
||||||
Headers: task.Headers,
|
Headers: task.Headers,
|
||||||
ContentType: task.ContentType,
|
ContentType: task.ContentType,
|
||||||
}
|
}
|
||||||
@@ -2104,7 +2101,6 @@ func buildRecoveryTask(
|
|||||||
TargetConfig: target.Config,
|
TargetConfig: target.Config,
|
||||||
MaxRetries: target.MaxRetries,
|
MaxRetries: target.MaxRetries,
|
||||||
Method: event.Method,
|
Method: event.Method,
|
||||||
RawQuery: event.RawQuery,
|
|
||||||
Headers: event.Headers,
|
Headers: event.Headers,
|
||||||
ContentType: event.ContentType,
|
ContentType: event.ContentType,
|
||||||
Body: bodyPtr,
|
Body: bodyPtr,
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -18,11 +19,12 @@ import (
|
|||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
"gorm.io/driver/sqlite"
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
_ "modernc.org/sqlite"
|
_ "modernc.org/sqlite"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/database/databasetest"
|
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
|
"sneak.berlin/go/webhooker/internal/gormlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// iSetup holds common integration test dependencies.
|
// iSetup holds common integration test dependencies.
|
||||||
@@ -43,12 +45,12 @@ func newISetup(t *testing.T) iSetup {
|
|||||||
wDB := iSeedWebhookDB(t, dbMgr, wID)
|
wDB := iSeedWebhookDB(t, dbMgr, wID)
|
||||||
|
|
||||||
return iSetup{
|
return iSetup{
|
||||||
MainDB: mainDB.DB(),
|
MainDB: mainDB,
|
||||||
DBMgr: dbMgr,
|
DBMgr: dbMgr,
|
||||||
WebhookID: wID,
|
WebhookID: wID,
|
||||||
WebhookDB: wDB,
|
WebhookDB: wDB,
|
||||||
Engine: delivery.NewTestEngineWithDB(
|
Engine: delivery.NewTestEngineWithDB(
|
||||||
mainDB,
|
database.NewTestDatabase(mainDB),
|
||||||
dbMgr,
|
dbMgr,
|
||||||
slog.New(slog.NewTextHandler(
|
slog.New(slog.NewTextHandler(
|
||||||
os.Stderr,
|
os.Stderr,
|
||||||
@@ -62,16 +64,35 @@ func newISetup(t *testing.T) iSetup {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// iMainDB opens a main database through database.Open, the way the
|
func iMainDB(t *testing.T) *gorm.DB {
|
||||||
// service opens it, so these tests cannot pass against journal and
|
|
||||||
// locking settings production does not use.
|
|
||||||
func iMainDB(t *testing.T) *database.Database {
|
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
db, err := database.Open(t.TempDir(), slog.New(slog.DiscardHandler))
|
dbPath := filepath.Join(
|
||||||
|
t.TempDir(), "main-test.db",
|
||||||
|
)
|
||||||
|
|
||||||
|
// Opened the way the service opens the main database, so these
|
||||||
|
// tests cannot pass against journal and locking settings
|
||||||
|
// production does not use.
|
||||||
|
sqlDB, err := database.OpenSQLite(
|
||||||
|
dbPath, database.SQLiteModeCreate,
|
||||||
|
)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
t.Cleanup(func() { _ = db.Close() })
|
t.Cleanup(func() { _ = sqlDB.Close() })
|
||||||
|
|
||||||
|
db, err := gorm.Open(
|
||||||
|
sqlite.Dialector{Conn: sqlDB},
|
||||||
|
&gorm.Config{Logger: gormlog.New(slog.New(slog.DiscardHandler))},
|
||||||
|
)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
require.NoError(t, db.AutoMigrate(
|
||||||
|
&database.Webhook{},
|
||||||
|
&database.Target{},
|
||||||
|
&database.User{},
|
||||||
|
&database.Setting{},
|
||||||
|
))
|
||||||
|
|
||||||
return db
|
return db
|
||||||
}
|
}
|
||||||
@@ -81,7 +102,7 @@ func iDBManager(
|
|||||||
) *database.WebhookDBManager {
|
) *database.WebhookDBManager {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
return databasetest.NewWebhookDBManager(t, t.TempDir())
|
return database.NewTestWebhookDBManager(t.TempDir())
|
||||||
}
|
}
|
||||||
|
|
||||||
func iSeedWebhookDB(
|
func iSeedWebhookDB(
|
||||||
@@ -652,11 +673,6 @@ func TestRecoverPendingDeliveries(t *testing.T) {
|
|||||||
t, s.WebhookDB, s.WebhookID, targetID, 3,
|
t, s.WebhookDB, s.WebhookID, targetID, 3,
|
||||||
)
|
)
|
||||||
|
|
||||||
// A recovered delivery still carries its event's query string.
|
|
||||||
require.NoError(t, s.WebhookDB.Model(&database.Event{}).
|
|
||||||
Where("webhook_id = ?", s.WebhookID).
|
|
||||||
Update("raw_query", eventQuery).Error)
|
|
||||||
|
|
||||||
s.Engine.ExportRecoverPendingDeliveries(
|
s.Engine.ExportRecoverPendingDeliveries(
|
||||||
context.Background(), s.WebhookDB,
|
context.Background(), s.WebhookDB,
|
||||||
s.WebhookID,
|
s.WebhookID,
|
||||||
@@ -671,8 +687,6 @@ func TestRecoverPendingDeliveries(t *testing.T) {
|
|||||||
database.TargetTypeLog,
|
database.TargetTypeLog,
|
||||||
task.TargetType,
|
task.TargetType,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert.Equal(t, eventQuery, task.RawQuery)
|
|
||||||
case <-time.After(2 * time.Second):
|
case <-time.After(2 * time.Second):
|
||||||
t.Fatalf("expected task %d", i)
|
t.Fatalf("expected task %d", i)
|
||||||
}
|
}
|
||||||
@@ -963,12 +977,29 @@ func TestFailUnretryableRetry_WritesNoTargetRow(
|
|||||||
database.DeliveryStatusFailed,
|
database.DeliveryStatusFailed,
|
||||||
)
|
)
|
||||||
|
|
||||||
// The per-webhook database has no targets table, so writing a
|
// The table exists in the per-webhook database because GORM
|
||||||
// target row there would have failed the status write above.
|
// migrates the Delivery relation's model alongside it. It must
|
||||||
assert.False(t, s.WebhookDB.Migrator().HasTable("targets"),
|
// stay empty.
|
||||||
|
var targetRows int64
|
||||||
|
|
||||||
|
require.NoError(t, s.WebhookDB.
|
||||||
|
Table("targets").
|
||||||
|
Count(&targetRows).Error)
|
||||||
|
|
||||||
|
assert.Zero(t, targetRows,
|
||||||
"orphaned-retry terminal failure wrote a target row "+
|
"orphaned-retry terminal failure wrote a target row "+
|
||||||
"into the per-webhook event database",
|
"into the per-webhook event database",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var configs []string
|
||||||
|
|
||||||
|
require.NoError(t, s.WebhookDB.
|
||||||
|
Table("targets").
|
||||||
|
Pluck("config", &configs).Error)
|
||||||
|
|
||||||
|
assert.NotContains(
|
||||||
|
t, strings.Join(configs, " "), hookURL,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRecoverSingleRetry_UnknownTargetType(
|
func TestRecoverSingleRetry_UnknownTargetType(
|
||||||
@@ -1116,17 +1147,17 @@ func TestRecoverInFlight_ReportsAMissingWebhookDatabase(t *testing.T) {
|
|||||||
|
|
||||||
mainDB := iMainDB(t)
|
mainDB := iMainDB(t)
|
||||||
webhookID := uuid.New().String()
|
webhookID := uuid.New().String()
|
||||||
iCreateWebhook(t, mainDB.DB(), webhookID, "lost-database")
|
iCreateWebhook(t, mainDB, webhookID, "lost-database")
|
||||||
|
|
||||||
var logs bytes.Buffer
|
var logs bytes.Buffer
|
||||||
|
|
||||||
dbMgr := databasetest.NewWebhookDBManagerWithLogger(
|
dbMgr := database.NewTestWebhookDBManagerWithLogger(
|
||||||
t, t.TempDir(), slog.New(slog.NewTextHandler(&logs, nil)),
|
t.TempDir(), slog.New(slog.NewTextHandler(&logs, nil)),
|
||||||
)
|
)
|
||||||
t.Cleanup(func() { _ = dbMgr.CloseAll() })
|
t.Cleanup(func() { _ = dbMgr.CloseAll() })
|
||||||
|
|
||||||
engine := delivery.NewTestEngineWithDB(
|
engine := delivery.NewTestEngineWithDB(
|
||||||
mainDB, dbMgr,
|
database.NewTestDatabase(mainDB), dbMgr,
|
||||||
slog.New(slog.DiscardHandler),
|
slog.New(slog.DiscardHandler),
|
||||||
&http.Client{Timeout: 5 * time.Second}, 1,
|
&http.Client{Timeout: 5 * time.Second}, 1,
|
||||||
)
|
)
|
||||||
@@ -1150,14 +1181,14 @@ func TestRecoverInFlight_SkipsAWebhookDeletedAfterTheListIsRead(
|
|||||||
|
|
||||||
mainDB := iMainDB(t)
|
mainDB := iMainDB(t)
|
||||||
webhookID := uuid.New().String()
|
webhookID := uuid.New().String()
|
||||||
iCreateWebhook(t, mainDB.DB(), webhookID, "deleted-during-recovery")
|
iCreateWebhook(t, mainDB, webhookID, "deleted-during-recovery")
|
||||||
|
|
||||||
// The first query to return is recovery's read of the list of
|
// The first query to return is recovery's read of the list of
|
||||||
// webhooks. Deleting the webhook right after it puts the delete
|
// webhooks. Deleting the webhook right after it puts the delete
|
||||||
// between that read and the opening of the webhook's database.
|
// between that read and the opening of the webhook's database.
|
||||||
deleted := false
|
deleted := false
|
||||||
|
|
||||||
require.NoError(t, mainDB.DB().Callback().Query().After("gorm:query").
|
require.NoError(t, mainDB.Callback().Query().After("gorm:query").
|
||||||
Register("delete-after-list", func(*gorm.DB) {
|
Register("delete-after-list", func(*gorm.DB) {
|
||||||
if deleted {
|
if deleted {
|
||||||
return
|
return
|
||||||
@@ -1165,16 +1196,16 @@ func TestRecoverInFlight_SkipsAWebhookDeletedAfterTheListIsRead(
|
|||||||
|
|
||||||
deleted = true
|
deleted = true
|
||||||
|
|
||||||
require.NoError(t, mainDB.DB().Delete(
|
require.NoError(t, mainDB.Delete(
|
||||||
&database.Webhook{}, "id = ?", webhookID,
|
&database.Webhook{}, "id = ?", webhookID,
|
||||||
).Error)
|
).Error)
|
||||||
}))
|
}))
|
||||||
|
|
||||||
dbMgr := databasetest.NewWebhookDBManager(t, t.TempDir())
|
dbMgr := database.NewTestWebhookDBManager(t.TempDir())
|
||||||
t.Cleanup(func() { _ = dbMgr.CloseAll() })
|
t.Cleanup(func() { _ = dbMgr.CloseAll() })
|
||||||
|
|
||||||
engine := delivery.NewTestEngineWithDB(
|
engine := delivery.NewTestEngineWithDB(
|
||||||
mainDB, dbMgr,
|
database.NewTestDatabase(mainDB), dbMgr,
|
||||||
slog.New(slog.DiscardHandler),
|
slog.New(slog.DiscardHandler),
|
||||||
&http.Client{Timeout: 5 * time.Second}, 1,
|
&http.Client{Timeout: 5 * time.Second}, 1,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"os"
|
"os"
|
||||||
"strconv"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
@@ -21,11 +21,12 @@ import (
|
|||||||
"github.com/prometheus/client_golang/prometheus"
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
"gorm.io/driver/sqlite"
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
_ "modernc.org/sqlite"
|
_ "modernc.org/sqlite"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/database/databasetest"
|
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
|
"sneak.berlin/go/webhooker/internal/gormlog"
|
||||||
"sneak.berlin/go/webhooker/internal/metrics"
|
"sneak.berlin/go/webhooker/internal/metrics"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -35,18 +36,35 @@ const testContentType = "application/json"
|
|||||||
func testWebhookDB(t *testing.T) *gorm.DB {
|
func testWebhookDB(t *testing.T) *gorm.DB {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
// Opened by the manager the service uses, so these tests cannot
|
dbPath := filepath.Join(
|
||||||
// pass against a schema, journal and locking settings or write
|
t.TempDir(), "events-test.db",
|
||||||
// guard production does not use.
|
|
||||||
mgr := databasetest.NewWebhookDBManagerWithLogger(
|
|
||||||
t, t.TempDir(), slog.New(slog.DiscardHandler),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
t.Cleanup(func() { _ = mgr.CloseAll() })
|
// Opened the way the service opens a per-webhook database, so
|
||||||
|
// these tests cannot pass against journal and locking settings
|
||||||
db, err := mgr.GetDB(uuid.New().String())
|
// production does not use.
|
||||||
|
sqlDB, err := database.OpenSQLite(
|
||||||
|
dbPath, database.SQLiteModeCreate,
|
||||||
|
)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
t.Cleanup(func() { _ = sqlDB.Close() })
|
||||||
|
|
||||||
|
db, err := gorm.Open(
|
||||||
|
sqlite.Dialector{Conn: sqlDB},
|
||||||
|
&gorm.Config{Logger: gormlog.New(slog.New(slog.DiscardHandler))},
|
||||||
|
)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
require.NoError(t, db.AutoMigrate(
|
||||||
|
&database.Event{},
|
||||||
|
&database.Delivery{},
|
||||||
|
&database.DeliveryResult{},
|
||||||
|
&database.EventTotals{},
|
||||||
|
&database.TargetTotals{},
|
||||||
|
))
|
||||||
|
require.NoError(t, db.Create(&database.EventTotals{}).Error)
|
||||||
|
|
||||||
return db
|
return db
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1972,10 +1990,6 @@ func assertLogLineComplete(
|
|||||||
"log line must contain the full request headers",
|
"log line must contain the full request headers",
|
||||||
)
|
)
|
||||||
|
|
||||||
assert.Contains(t, out, "raw_query="+strconv.Quote(event.RawQuery),
|
|
||||||
"log line must contain the query string",
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.Contains(t, out, event.EntrypointID,
|
assert.Contains(t, out, event.EntrypointID,
|
||||||
"log line must contain the entrypoint id",
|
"log line must contain the entrypoint id",
|
||||||
)
|
)
|
||||||
@@ -1998,7 +2012,6 @@ func TestDeliverLog_LogsFullContent(t *testing.T) {
|
|||||||
event := seedEvent(
|
event := seedEvent(
|
||||||
t, db, `{"log-body-marker":"abc123"}`,
|
t, db, `{"log-body-marker":"abc123"}`,
|
||||||
)
|
)
|
||||||
event.RawQuery = eventQuery
|
|
||||||
|
|
||||||
dlv := seedDelivery(
|
dlv := seedDelivery(
|
||||||
t, db, event.ID, uuid.New().String(),
|
t, db, event.ID, uuid.New().String(),
|
||||||
|
|||||||
@@ -501,32 +501,23 @@ func NewExportArchiveWriter(
|
|||||||
return &ExportArchiveWriter{w: w}
|
return &ExportArchiveWriter{w: w}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Write archives a row through the writer, into the file named
|
// Write archives a row through the writer.
|
||||||
// without a period.
|
|
||||||
func (e *ExportArchiveWriter) Write(
|
func (e *ExportArchiveWriter) Write(
|
||||||
row ExportArchivedEvent, expiry time.Duration,
|
row ExportArchivedEvent, expiry time.Duration,
|
||||||
) error {
|
) error {
|
||||||
return e.w.write(row, expiry, "")
|
return e.w.write(row, expiry)
|
||||||
}
|
|
||||||
|
|
||||||
// WritePeriod archives a row through the writer, into the file for
|
|
||||||
// period.
|
|
||||||
func (e *ExportArchiveWriter) WritePeriod(
|
|
||||||
row ExportArchivedEvent, expiry time.Duration, period string,
|
|
||||||
) error {
|
|
||||||
return e.w.write(row, expiry, period)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Open opens the archive file, pruning when expiry is positive.
|
// Open opens the archive file, pruning when expiry is positive.
|
||||||
func (e *ExportArchiveWriter) Open(expiry time.Duration) error {
|
func (e *ExportArchiveWriter) Open(expiry time.Duration) error {
|
||||||
return e.w.open(e.w.path, expiry)
|
return e.w.open(expiry)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Reopen closes and reopens the archive file.
|
// Reopen closes and reopens the archive file.
|
||||||
func (e *ExportArchiveWriter) Reopen(
|
func (e *ExportArchiveWriter) Reopen(
|
||||||
expiry time.Duration,
|
expiry time.Duration,
|
||||||
) error {
|
) error {
|
||||||
return e.w.reopen(e.w.path, expiry)
|
return e.w.reopen(expiry)
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetNow replaces the clock the writer measures its reopen
|
// SetNow replaces the clock the writer measures its reopen
|
||||||
@@ -556,7 +547,7 @@ func (e *ExportArchiveWriter) Path() string {
|
|||||||
func (e *ExportArchiveWriter) OpenExisting(
|
func (e *ExportArchiveWriter) OpenExisting(
|
||||||
expiry time.Duration,
|
expiry time.Duration,
|
||||||
) error {
|
) error {
|
||||||
return e.w.openMode(e.w.path, archiveModeExisting, expiry)
|
return e.w.openMode(archiveModeExisting, expiry)
|
||||||
}
|
}
|
||||||
|
|
||||||
// SweepExpired runs an idle sweep of the archive.
|
// SweepExpired runs an idle sweep of the archive.
|
||||||
|
|||||||
@@ -48,9 +48,8 @@ func fSweepSetup(
|
|||||||
//
|
//
|
||||||
// Every caller drives the dispatch paths synchronously and has already
|
// Every caller drives the dispatch paths synchronously and has already
|
||||||
// waited for them to return, so anything they queued is in the channel
|
// waited for them to return, so anything they queued is in the channel
|
||||||
// by now, and nothing is waited for. A timer here would race the queued
|
// by now. The short grace covers nothing but scheduler jitter, and is
|
||||||
// tasks: on a busy host it can be due by the time select looks, and
|
// kept small because one of these tests runs the drain forty times.
|
||||||
// select picks at random among the cases that are ready.
|
|
||||||
func fDrain(e *delivery.Engine) []delivery.Task {
|
func fDrain(e *delivery.Engine) []delivery.Task {
|
||||||
var out []delivery.Task
|
var out []delivery.Task
|
||||||
|
|
||||||
@@ -60,7 +59,7 @@ func fDrain(e *delivery.Engine) []delivery.Task {
|
|||||||
out = append(out, task)
|
out = append(out, task)
|
||||||
case task := <-e.ExportRetryCh():
|
case task := <-e.ExportRetryCh():
|
||||||
out = append(out, task)
|
out = append(out, task)
|
||||||
default:
|
case <-time.After(25 * time.Millisecond):
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -13,9 +14,11 @@ import (
|
|||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
"gorm.io/driver/sqlite"
|
||||||
|
"gorm.io/gorm"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/database/databasetest"
|
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
|
"sneak.berlin/go/webhooker/internal/gormlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// qdAggregateMarker identifies the queue-depth aggregate in the
|
// qdAggregateMarker identifies the queue-depth aggregate in the
|
||||||
@@ -46,13 +49,27 @@ func (q *qdSyncBuf) String() string {
|
|||||||
|
|
||||||
// qdMainDB opens a main database whose GORM logger is the service's
|
// qdMainDB opens a main database whose GORM logger is the service's
|
||||||
// adapter, writing through log.
|
// adapter, writing through log.
|
||||||
func qdMainDB(t *testing.T, log *slog.Logger) *database.Database {
|
func qdMainDB(t *testing.T, log *slog.Logger) *gorm.DB {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
db, err := database.Open(t.TempDir(), log)
|
sqlDB, err := database.OpenSQLite(
|
||||||
|
filepath.Join(t.TempDir(), "main-gormlog.db"),
|
||||||
|
database.SQLiteModeCreate,
|
||||||
|
)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
t.Cleanup(func() { _ = db.Close() })
|
t.Cleanup(func() { _ = sqlDB.Close() })
|
||||||
|
|
||||||
|
db, err := gorm.Open(
|
||||||
|
sqlite.Dialector{Conn: sqlDB},
|
||||||
|
&gorm.Config{Logger: gormlog.New(log)},
|
||||||
|
)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
require.NoError(t, db.AutoMigrate(
|
||||||
|
&database.Webhook{},
|
||||||
|
&database.Target{},
|
||||||
|
))
|
||||||
|
|
||||||
return db
|
return db
|
||||||
}
|
}
|
||||||
@@ -89,18 +106,18 @@ func TestQueueDepthSample_LogsNoBoundValue(t *testing.T) {
|
|||||||
))
|
))
|
||||||
|
|
||||||
mainDB := qdMainDB(t, log)
|
mainDB := qdMainDB(t, log)
|
||||||
dbMgr := databasetest.NewWebhookDBManagerWithLogger(
|
dbMgr := database.NewTestWebhookDBManagerWithLogger(
|
||||||
t, t.TempDir(), log,
|
t.TempDir(), log,
|
||||||
)
|
)
|
||||||
|
|
||||||
webhookID := uuid.New().String()
|
webhookID := uuid.New().String()
|
||||||
webhookDB := iSeedWebhookDB(t, dbMgr, webhookID)
|
webhookDB := iSeedWebhookDB(t, dbMgr, webhookID)
|
||||||
|
|
||||||
iCreateWebhook(t, mainDB.DB(), webhookID, "queue-depth-gormlog")
|
iCreateWebhook(t, mainDB, webhookID, "queue-depth-gormlog")
|
||||||
|
|
||||||
targetID := uuid.New().String()
|
targetID := uuid.New().String()
|
||||||
|
|
||||||
iCreateTarget(t, mainDB.DB(), targetID, webhookID,
|
iCreateTarget(t, mainDB, targetID, webhookID,
|
||||||
"queue-depth-gormlog-target", database.TargetTypeHTTP,
|
"queue-depth-gormlog-target", database.TargetTypeHTTP,
|
||||||
iHTTPConfig("https://example.com/hook"), 3,
|
iHTTPConfig("https://example.com/hook"), 3,
|
||||||
)
|
)
|
||||||
@@ -119,7 +136,7 @@ func TestQueueDepthSample_LogsNoBoundValue(t *testing.T) {
|
|||||||
)
|
)
|
||||||
|
|
||||||
engine := delivery.NewTestEngineWithDB(
|
engine := delivery.NewTestEngineWithDB(
|
||||||
mainDB,
|
database.NewTestDatabase(mainDB),
|
||||||
dbMgr,
|
dbMgr,
|
||||||
log,
|
log,
|
||||||
&http.Client{Timeout: 5 * time.Second},
|
&http.Client{Timeout: 5 * time.Second},
|
||||||
|
|||||||
@@ -39,13 +39,8 @@ type TargetConfigForm struct {
|
|||||||
// Timeout is the HTTP target's per-request timeout in seconds,
|
// Timeout is the HTTP target's per-request timeout in seconds,
|
||||||
// empty when unset.
|
// empty when unset.
|
||||||
Timeout string
|
Timeout string
|
||||||
// ForwardQuery is the HTTP target's setting that passes each
|
|
||||||
// event's query string on to it.
|
|
||||||
ForwardQuery bool
|
|
||||||
// Expiry is the database (archive) target's row expiry.
|
// Expiry is the database (archive) target's row expiry.
|
||||||
Expiry string
|
Expiry string
|
||||||
// Rotation is the database (archive) target's rotation.
|
|
||||||
Rotation string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewTargetConfigForm parses a target's stored configuration into
|
// NewTargetConfigForm parses a target's stored configuration into
|
||||||
@@ -70,7 +65,6 @@ func NewTargetConfigForm(
|
|||||||
URL: cfg.URL,
|
URL: cfg.URL,
|
||||||
Headers: FormatTargetHeaders(cfg.Headers),
|
Headers: FormatTargetHeaders(cfg.Headers),
|
||||||
Timeout: FormatTargetTimeout(cfg.Timeout),
|
Timeout: FormatTargetTimeout(cfg.Timeout),
|
||||||
ForwardQuery: cfg.ForwardQuery,
|
|
||||||
}, nil
|
}, nil
|
||||||
case database.TargetTypeSlack:
|
case database.TargetTypeSlack:
|
||||||
cfg, err := parseSlackConfig(t.Config)
|
cfg, err := parseSlackConfig(t.Config)
|
||||||
@@ -91,13 +85,11 @@ func NewTargetConfigForm(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// databaseConfigForm parses an archive target's optional expiry and
|
// databaseConfigForm parses an archive target's optional expiry.
|
||||||
// rotation. An absent, empty or never expiry yields an empty expiry,
|
// An absent, empty or never expiry yields an empty expiry, on which
|
||||||
// on which the edit form starts at never; saving it unchanged stores
|
// the edit form starts at never; saving it unchanged stores never,
|
||||||
// never, which means the same as an empty expiry. An absent rotation
|
// which means the same as an empty expiry. An expiry that is set
|
||||||
// is empty too, and the form starts at none. An expiry that is set
|
// but not a valid duration is an error, not a blank field.
|
||||||
// but not a valid duration, or a rotation that is not one of the
|
|
||||||
// four, is an error, not a blank field.
|
|
||||||
func databaseConfigForm(
|
func databaseConfigForm(
|
||||||
configJSON string,
|
configJSON string,
|
||||||
) (TargetConfigForm, error) {
|
) (TargetConfigForm, error) {
|
||||||
@@ -114,15 +106,8 @@ func databaseConfigForm(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = ValidateArchiveRotation(cfg.Rotation)
|
|
||||||
if err != nil {
|
|
||||||
return TargetConfigForm{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
form := TargetConfigForm{Rotation: cfg.Rotation}
|
|
||||||
|
|
||||||
if cfg.Expiry == "" || cfg.Expiry == archiveExpiryNever {
|
if cfg.Expiry == "" || cfg.Expiry == archiveExpiryNever {
|
||||||
return form, nil
|
return TargetConfigForm{}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
err = ValidateArchiveExpiry(cfg.Expiry)
|
err = ValidateArchiveExpiry(cfg.Expiry)
|
||||||
@@ -130,7 +115,5 @@ func databaseConfigForm(
|
|||||||
return TargetConfigForm{}, err
|
return TargetConfigForm{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
form.Expiry = cfg.Expiry
|
return TargetConfigForm{Expiry: cfg.Expiry}, nil
|
||||||
|
|
||||||
return form, nil
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -171,38 +171,29 @@ func httpConfigFields(t *database.Target) []ConfigField {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
if cfg.ForwardQuery {
|
|
||||||
fields = append(fields, ConfigField{
|
|
||||||
Label: "Query string",
|
|
||||||
Value: "passed on to this target",
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fields = append(fields, maxRetriesField(t))
|
fields = append(fields, maxRetriesField(t))
|
||||||
|
|
||||||
return fields
|
return fields
|
||||||
}
|
}
|
||||||
|
|
||||||
// maxRetriesField describes a target's retry count, which lives
|
// maxRetriesField describes a target's retry count, which lives
|
||||||
// on the target row rather than in its configuration blob. A
|
// on the target row rather than in its configuration blob.
|
||||||
// stored 0 makes a single attempt, so it is shown as 1.
|
|
||||||
func maxRetriesField(t *database.Target) ConfigField {
|
func maxRetriesField(t *database.Target) ConfigField {
|
||||||
attempts := strconv.Itoa(t.MaxRetries)
|
retries := strconv.Itoa(t.MaxRetries)
|
||||||
if t.MaxRetries == 0 {
|
if t.MaxRetries == 0 {
|
||||||
attempts = "1 (fire-and-forget: no retries, no circuit breaker)"
|
retries += " (fire-and-forget)"
|
||||||
}
|
}
|
||||||
|
|
||||||
return ConfigField{
|
return ConfigField{
|
||||||
Label: "Delivery attempts",
|
Label: "Max Retries",
|
||||||
Value: attempts,
|
Value: retries,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// databaseConfigFields describes an archive target by its
|
// databaseConfigFields describes an archive target by its
|
||||||
// expiry in plain units, such as "30 days", or "never" when
|
// expiry in plain units, such as "30 days", or "never" when
|
||||||
// the archive is kept forever, and by its rotation. An expiry
|
// the archive is kept forever. An expiry that is set but not
|
||||||
// that is set but not a valid duration, or a rotation that is
|
// a valid duration is reported as unavailable rather than
|
||||||
// not one of the four, is reported as unavailable rather than
|
|
||||||
// echoed back.
|
// echoed back.
|
||||||
func databaseConfigFields(configJSON string) []ConfigField {
|
func databaseConfigFields(configJSON string) []ConfigField {
|
||||||
expiry, err := parseArchiveExpiry(configJSON)
|
expiry, err := parseArchiveExpiry(configJSON)
|
||||||
@@ -210,22 +201,14 @@ func databaseConfigFields(configJSON string) []ConfigField {
|
|||||||
return unavailableConfigFields()
|
return unavailableConfigFields()
|
||||||
}
|
}
|
||||||
|
|
||||||
rotation, err := parseArchiveRotation(configJSON)
|
|
||||||
if err != nil {
|
|
||||||
return unavailableConfigFields()
|
|
||||||
}
|
|
||||||
|
|
||||||
value := archiveExpiryNever
|
value := archiveExpiryNever
|
||||||
if expiry > 0 {
|
if expiry > 0 {
|
||||||
value = plainDuration(expiry)
|
value = plainDuration(expiry)
|
||||||
}
|
}
|
||||||
|
|
||||||
return []ConfigField{{
|
return []ConfigField{{
|
||||||
Label: "Archive expiry",
|
Label: "Archive Expiry",
|
||||||
Value: value,
|
Value: value,
|
||||||
}, {
|
|
||||||
Label: "Archive rotation",
|
|
||||||
Value: rotation,
|
|
||||||
}}
|
}}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -32,7 +32,7 @@ const (
|
|||||||
viewMaskedOrigin = viewExampleOrigin + "/..."
|
viewMaskedOrigin = viewExampleOrigin + "/..."
|
||||||
viewUnavailable = "(unavailable)"
|
viewUnavailable = "(unavailable)"
|
||||||
viewExpiryNever = "never"
|
viewExpiryNever = "never"
|
||||||
viewMaxRetries = "Delivery attempts"
|
viewMaxRetries = "Max Retries"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestMaskedWebhookURL(t *testing.T) {
|
func TestMaskedWebhookURL(t *testing.T) {
|
||||||
@@ -190,7 +190,7 @@ func TestNewTargetViews_Slack(t *testing.T) {
|
|||||||
t,
|
t,
|
||||||
map[string]string{
|
map[string]string{
|
||||||
"Webhook URL": slackMaskedURL,
|
"Webhook URL": slackMaskedURL,
|
||||||
viewMaxRetries: "1 (fire-and-forget: no retries, no circuit breaker)",
|
viewMaxRetries: "0 (fire-and-forget)",
|
||||||
},
|
},
|
||||||
fieldMap(view.Config),
|
fieldMap(view.Config),
|
||||||
)
|
)
|
||||||
@@ -223,8 +223,7 @@ func TestNewTargetViews_HTTP(t *testing.T) {
|
|||||||
Type: database.TargetTypeHTTP,
|
Type: database.TargetTypeHTTP,
|
||||||
Config: `{"url":"` + viewExampleHook + `",` +
|
Config: `{"url":"` + viewExampleHook + `",` +
|
||||||
`"timeout":30,` +
|
`"timeout":30,` +
|
||||||
`"headers":{"Authorization":"Bearer sekrit"},` +
|
`"headers":{"Authorization":"Bearer sekrit"}}`,
|
||||||
`"forwardQuery":true}`,
|
|
||||||
MaxRetries: 5,
|
MaxRetries: 5,
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -236,7 +235,6 @@ func TestNewTargetViews_HTTP(t *testing.T) {
|
|||||||
"Destination URL": viewMaskedOrigin,
|
"Destination URL": viewMaskedOrigin,
|
||||||
"Timeout": "30s",
|
"Timeout": "30s",
|
||||||
"Headers": "1 configured",
|
"Headers": "1 configured",
|
||||||
"Query string": "passed on to this target",
|
|
||||||
viewMaxRetries: "5",
|
viewMaxRetries: "5",
|
||||||
},
|
},
|
||||||
fields,
|
fields,
|
||||||
@@ -260,7 +258,7 @@ func TestNewTargetViews_HTTPFireAndForget(t *testing.T) {
|
|||||||
t,
|
t,
|
||||||
map[string]string{
|
map[string]string{
|
||||||
"Destination URL": viewMaskedOrigin,
|
"Destination URL": viewMaskedOrigin,
|
||||||
viewMaxRetries: "1 (fire-and-forget: no retries, no circuit breaker)",
|
viewMaxRetries: "0 (fire-and-forget)",
|
||||||
},
|
},
|
||||||
fieldMap(view.Config),
|
fieldMap(view.Config),
|
||||||
)
|
)
|
||||||
@@ -330,49 +328,13 @@ func TestNewTargetViews_Database(t *testing.T) {
|
|||||||
|
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t,
|
t,
|
||||||
map[string]string{
|
map[string]string{"Archive Expiry": tc.want},
|
||||||
"Archive expiry": tc.want,
|
|
||||||
"Archive rotation": rotationNone,
|
|
||||||
},
|
|
||||||
fieldMap(view.Config),
|
fieldMap(view.Config),
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestNewTargetViews_DatabaseRotation proves the target list shows a
|
|
||||||
// database target's rotation, none when it has none stored.
|
|
||||||
func TestNewTargetViews_DatabaseRotation(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Each stored config, and the rotation the list shows for it.
|
|
||||||
tests := map[string]string{
|
|
||||||
"": rotationNone,
|
|
||||||
`{"rotation":""}`: rotationNone,
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, rotation := range []string{
|
|
||||||
rotationNone, rotationMonthly, rotationDaily, rotationHourly,
|
|
||||||
} {
|
|
||||||
tests[`{"rotation":"`+rotation+`"}`] = rotation
|
|
||||||
}
|
|
||||||
|
|
||||||
for config, want := range tests {
|
|
||||||
t.Run(config, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
view := viewFor(t, database.Target{
|
|
||||||
Type: database.TargetTypeDatabase,
|
|
||||||
Config: config,
|
|
||||||
})
|
|
||||||
|
|
||||||
assert.Equal(
|
|
||||||
t, want, fieldMap(view.Config)["Archive rotation"],
|
|
||||||
)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewTargetViews_Log(t *testing.T) {
|
func TestNewTargetViews_Log(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -420,10 +382,6 @@ func TestNewTargetViews_Unpresentable(t *testing.T) {
|
|||||||
Type: database.TargetTypeDatabase,
|
Type: database.TargetTypeDatabase,
|
||||||
Config: `{"expiry":"a fortnight"}`,
|
Config: `{"expiry":"a fortnight"}`,
|
||||||
},
|
},
|
||||||
"invalid archive rotation": {
|
|
||||||
Type: database.TargetTypeDatabase,
|
|
||||||
Config: weeklyConfig,
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for name, target := range tests {
|
for name, target := range tests {
|
||||||
|
|||||||
@@ -20,8 +20,7 @@ const archiveNameMaxLen = 40
|
|||||||
// from the per-webhook event database. The event is already
|
// from the per-webhook event database. The event is already
|
||||||
// persisted in the per-webhook event DB by the time delivery runs;
|
// persisted in the per-webhook event DB by the time delivery runs;
|
||||||
// the database target additionally writes a durable long-term copy
|
// the database target additionally writes a durable long-term copy
|
||||||
// into the file ArchiveFileName names, with a period added when the
|
// into the file ArchiveFileName names and then records a single
|
||||||
// target rotates (see archivePeriodPath), and then records a single
|
|
||||||
// attempt whose outcome reflects whether the archive write
|
// attempt whose outcome reflects whether the archive write
|
||||||
// succeeded. See archiveWriter for the close/reopen, auto-recreate,
|
// succeeded. See archiveWriter for the close/reopen, auto-recreate,
|
||||||
// and expiry semantics.
|
// and expiry semantics.
|
||||||
@@ -147,9 +146,8 @@ func (t *databaseTarget) Deliver(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// archive writes the full event as a row into the target's
|
// archive writes the full event as a row into the target's
|
||||||
// archive database, honouring the optional per-target expiry and
|
// archive database, honouring the optional per-target expiry
|
||||||
// rotation parsed from the target config JSON. With rotation, the
|
// parsed from the target config JSON.
|
||||||
// event goes to the file for the period of its receive time.
|
|
||||||
func (t *databaseTarget) archive(d *database.Delivery) error {
|
func (t *databaseTarget) archive(d *database.Delivery) error {
|
||||||
webhookID := d.Event.WebhookID
|
webhookID := d.Event.WebhookID
|
||||||
if webhookID == "" {
|
if webhookID == "" {
|
||||||
@@ -161,19 +159,6 @@ func (t *databaseTarget) archive(d *database.Delivery) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
rotation, err := parseArchiveRotation(d.Target.Config)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// An event whose stored row was gone before its delivery ran has
|
|
||||||
// no receive time (see Engine.hydrateEvent), and goes to the
|
|
||||||
// file for now.
|
|
||||||
receivedAt := d.Event.CreatedAt
|
|
||||||
if receivedAt.IsZero() {
|
|
||||||
receivedAt = time.Now()
|
|
||||||
}
|
|
||||||
|
|
||||||
w, err := t.writerFor(d.TargetID)
|
w, err := t.writerFor(d.TargetID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -184,13 +169,12 @@ func (t *databaseTarget) archive(d *database.Delivery) error {
|
|||||||
WebhookID: webhookID,
|
WebhookID: webhookID,
|
||||||
EntrypointID: d.Event.EntrypointID,
|
EntrypointID: d.Event.EntrypointID,
|
||||||
Method: d.Event.Method,
|
Method: d.Event.Method,
|
||||||
RawQuery: d.Event.RawQuery,
|
|
||||||
Headers: d.Event.Headers,
|
Headers: d.Event.Headers,
|
||||||
Body: d.Event.Body,
|
Body: d.Event.Body,
|
||||||
ContentType: d.Event.ContentType,
|
ContentType: d.Event.ContentType,
|
||||||
}
|
}
|
||||||
|
|
||||||
return w.write(row, expiry, archivePeriod(rotation, receivedAt))
|
return w.write(row, expiry)
|
||||||
}
|
}
|
||||||
|
|
||||||
// writerFor returns the archive writer for a database target,
|
// writerFor returns the archive writer for a database target,
|
||||||
@@ -291,10 +275,10 @@ func (t *databaseTarget) releaseSweepWriter(
|
|||||||
delete(t.writers, targetID)
|
delete(t.writers, targetID)
|
||||||
}
|
}
|
||||||
|
|
||||||
// newWriter builds the writer for a database target's archive. Its
|
// newWriter builds the writer for a database target's archive. The
|
||||||
// path is the one ArchivePath gives for the webhook and the target as
|
// file is the one ArchivePath gives for the webhook and the target as
|
||||||
// the main database names them now; from then on only rename changes
|
// the main database names them now; from then on only rename changes
|
||||||
// the name the writer uses. It does not touch the archive files.
|
// the name the writer uses. It does not touch the archive file.
|
||||||
func (t *databaseTarget) newWriter(
|
func (t *databaseTarget) newWriter(
|
||||||
targetID string,
|
targetID string,
|
||||||
) (*archiveWriter, error) {
|
) (*archiveWriter, error) {
|
||||||
@@ -322,10 +306,10 @@ func (t *databaseTarget) newWriter(
|
|||||||
return w, nil
|
return w, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// rename moves every one of a database target's archive files to the
|
// rename moves a database target's archive file to the name for
|
||||||
// name for webhookName and targetName. It goes through the target's
|
// webhookName and targetName. It goes through the target's writer,
|
||||||
// writer, so the move holds the lock that writes and the idle sweep
|
// so the move holds the lock that writes and the idle sweep take,
|
||||||
// take, and later writes use the new name.
|
// and later writes use the new name.
|
||||||
//
|
//
|
||||||
// The writer is created if there is none, and it stays cached. The
|
// The writer is created if there is none, and it stays cached. The
|
||||||
// handlers rename before they save the new name, so until the save
|
// handlers rename before they save the new name, so until the save
|
||||||
|
|||||||
@@ -14,7 +14,6 @@ import (
|
|||||||
"gorm.io/driver/sqlite"
|
"gorm.io/driver/sqlite"
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/db/migrations"
|
|
||||||
"sneak.berlin/go/webhooker/internal/gormlog"
|
"sneak.berlin/go/webhooker/internal/gormlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -86,10 +85,6 @@ type databaseTargetConfig struct {
|
|||||||
// archived rows are pruned, or "never" (the default) to
|
// archived rows are pruned, or "never" (the default) to
|
||||||
// keep them forever.
|
// keep them forever.
|
||||||
Expiry string `json:"expiry"`
|
Expiry string `json:"expiry"`
|
||||||
|
|
||||||
// Rotation is none (the default), monthly, daily or hourly: see
|
|
||||||
// archivePeriod.
|
|
||||||
Rotation string `json:"rotation"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// archivedEvent is one fully captured webhook event stored in a
|
// archivedEvent is one fully captured webhook event stored in a
|
||||||
@@ -102,7 +97,6 @@ type archivedEvent struct {
|
|||||||
WebhookID string
|
WebhookID string
|
||||||
EntrypointID string
|
EntrypointID string
|
||||||
Method string
|
Method string
|
||||||
RawQuery string
|
|
||||||
Headers string
|
Headers string
|
||||||
Body string
|
Body string
|
||||||
ContentType string
|
ContentType string
|
||||||
@@ -184,7 +178,7 @@ func ValidateArchiveExpiry(expiry string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// archiveWriter owns one database target's archive SQLite files.
|
// archiveWriter owns one database target's archive SQLite file.
|
||||||
// It serialises writes, and after each write closes and reopens
|
// It serialises writes, and after each write closes and reopens
|
||||||
// the file (debounced to at most once per debounce window) so
|
// the file (debounced to at most once per debounce window) so
|
||||||
// an operator can move the file away for offline archiving. The
|
// an operator can move the file away for offline archiving. The
|
||||||
@@ -193,15 +187,7 @@ func ValidateArchiveExpiry(expiry string) error {
|
|||||||
// on every open.
|
// on every open.
|
||||||
type archiveWriter struct {
|
type archiveWriter struct {
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
|
|
||||||
// path is the target's archive file as ArchivePath names it. A
|
|
||||||
// target that rotates writes to the files archivePeriodPath names
|
|
||||||
// for path and a period instead.
|
|
||||||
path string
|
path string
|
||||||
|
|
||||||
// current is the file db is open on.
|
|
||||||
current string
|
|
||||||
|
|
||||||
log *slog.Logger
|
log *slog.Logger
|
||||||
debounce time.Duration
|
debounce time.Duration
|
||||||
db *gorm.DB
|
db *gorm.DB
|
||||||
@@ -250,14 +236,12 @@ func newArchiveWriter(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// write appends the event as a row to the archive file for period
|
// write appends the event as a row, then applies the debounced
|
||||||
// (see archivePeriodPath), then applies the debounced close/reopen.
|
// close/reopen. It recreates the archive file if it was moved
|
||||||
// When period names a different file from the one open, the open one
|
// or removed since the last open. A positive expiry prunes rows
|
||||||
// is closed first. It recreates the archive file if it was moved or
|
// older than it on each (re)open.
|
||||||
// removed since the last open. A positive expiry prunes rows older
|
|
||||||
// than it on each (re)open.
|
|
||||||
func (w *archiveWriter) write(
|
func (w *archiveWriter) write(
|
||||||
row archivedEvent, expiry time.Duration, period string,
|
row archivedEvent, expiry time.Duration,
|
||||||
) error {
|
) error {
|
||||||
w.mu.Lock()
|
w.mu.Lock()
|
||||||
defer w.mu.Unlock()
|
defer w.mu.Unlock()
|
||||||
@@ -268,10 +252,8 @@ func (w *archiveWriter) write(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
file := archivePeriodPath(w.path, period)
|
if w.db == nil || !fileExists(w.path) {
|
||||||
|
err := w.reopen(expiry)
|
||||||
if w.db == nil || w.current != file || !fileExists(file) {
|
|
||||||
err := w.reopen(file, expiry)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -282,41 +264,41 @@ func (w *archiveWriter) write(
|
|||||||
err := w.db.Create(&row).Error
|
err := w.db.Create(&row).Error
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf(
|
return fmt.Errorf(
|
||||||
"archiving event to %s: %w", file, err,
|
"archiving event to %s: %w", w.path, err,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
if w.now().Sub(w.lastReopen) >= w.debounce {
|
if w.now().Sub(w.lastReopen) >= w.debounce {
|
||||||
return w.reopen(file, expiry)
|
return w.reopen(expiry)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// open opens (creating if missing) an archive file, migrates
|
// open opens (creating if missing) the archive file, migrates
|
||||||
// its schema, records the reopen time, and prunes expired rows
|
// its schema, records the reopen time, and prunes expired rows
|
||||||
// when expiry is positive.
|
// when expiry is positive.
|
||||||
func (w *archiveWriter) open(file string, expiry time.Duration) error {
|
func (w *archiveWriter) open(expiry time.Duration) error {
|
||||||
return w.openMode(file, archiveModeCreate, expiry)
|
return w.openMode(archiveModeCreate, expiry)
|
||||||
}
|
}
|
||||||
|
|
||||||
// openMode opens an archive file with the given SQLite URI
|
// openMode opens the archive file with the given SQLite URI
|
||||||
// mode, migrates its schema, records the reopen time, and
|
// mode, migrates its schema, records the reopen time, and
|
||||||
// prunes expired rows when expiry is positive. The write path
|
// prunes expired rows when expiry is positive. The write path
|
||||||
// passes archiveModeCreate so a missing file is recreated; the
|
// passes archiveModeCreate so a missing file is recreated; the
|
||||||
// idle sweep passes archiveModeExisting so a missing file is an
|
// idle sweep passes archiveModeExisting so a missing file is an
|
||||||
// error rather than a newly conjured empty archive.
|
// error rather than a newly conjured empty archive.
|
||||||
func (w *archiveWriter) openMode(
|
func (w *archiveWriter) openMode(
|
||||||
file, mode string, expiry time.Duration,
|
mode string, expiry time.Duration,
|
||||||
) error {
|
) error {
|
||||||
// Opened through database.OpenSQLite so an archive file carries
|
// Opened through database.OpenSQLite so an archive file carries
|
||||||
// the same WAL journaling, busy timeout, immediate-transaction
|
// the same WAL journaling, busy timeout, immediate-transaction
|
||||||
// locking, and pool bounds as every other database file. See
|
// locking, and pool bounds as every other database file. See
|
||||||
// internal/database/sqlite_open.go.
|
// internal/database/sqlite_open.go.
|
||||||
sqlDB, err := database.OpenSQLite(file, mode)
|
sqlDB, err := database.OpenSQLite(w.path, mode)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf(
|
return fmt.Errorf(
|
||||||
"opening archive database %s: %w", file, err,
|
"opening archive database %s: %w", w.path, err,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -332,21 +314,20 @@ func (w *archiveWriter) openMode(
|
|||||||
|
|
||||||
return fmt.Errorf(
|
return fmt.Errorf(
|
||||||
"connecting to archive database %s: %w",
|
"connecting to archive database %s: %w",
|
||||||
file, err,
|
w.path, err,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = migrations.Apply(gdb, migrations.Archive)
|
err = gdb.AutoMigrate(&archivedEvent{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = sqlDB.Close()
|
_ = sqlDB.Close()
|
||||||
|
|
||||||
return fmt.Errorf(
|
return fmt.Errorf(
|
||||||
"migrating archive database %s: %w", file, err,
|
"migrating archive database %s: %w", w.path, err,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
w.db = gdb
|
w.db = gdb
|
||||||
w.current = file
|
|
||||||
w.lastReopen = w.now()
|
w.lastReopen = w.now()
|
||||||
w.reopens++
|
w.reopens++
|
||||||
|
|
||||||
@@ -357,12 +338,12 @@ func (w *archiveWriter) openMode(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// reopen closes any open handle and opens file afresh. The
|
// reopen closes any open handle and opens the file afresh. The
|
||||||
// fresh open recreates the file if it was moved away.
|
// fresh open recreates the file if it was moved away.
|
||||||
func (w *archiveWriter) reopen(file string, expiry time.Duration) error {
|
func (w *archiveWriter) reopen(expiry time.Duration) error {
|
||||||
w.close()
|
w.close()
|
||||||
|
|
||||||
return w.open(file, expiry)
|
return w.open(expiry)
|
||||||
}
|
}
|
||||||
|
|
||||||
// close closes the underlying handle, if any.
|
// close closes the underlying handle, if any.
|
||||||
@@ -379,56 +360,22 @@ func (w *archiveWriter) close() {
|
|||||||
w.db = nil
|
w.db = nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// sweepExpired prunes the target's archive files, which may have
|
// sweepExpired prunes an archive that may have gone idle, with
|
||||||
// gone idle, with no write to trigger the usual on-reopen prune. It
|
// no write to trigger the usual on-reopen prune. It takes the
|
||||||
// lists the files under the writer's own mutex, then takes the mutex
|
// writer's own mutex for the whole operation, so a sweep is
|
||||||
// again for one file at a time, so a write waits for at most one
|
// ordered against concurrent writes rather than reaching around
|
||||||
// file's prune, and each prune is ordered against concurrent writes
|
// them to the file.
|
||||||
// rather than reaching around them to the file.
|
|
||||||
//
|
//
|
||||||
// It never creates an archive file: it prunes only the files
|
// It never creates the archive file: a missing file is skipped,
|
||||||
// archiveFiles lists, skips one that is gone by the time it is
|
// and the reopen uses archiveModeExisting so SQLite itself
|
||||||
// reached (moved away, or renamed since the listing), and opens each
|
// refuses to create one if the file disappears between the
|
||||||
// with archiveModeExisting so SQLite itself refuses to create one if
|
// check and the open.
|
||||||
// the file disappears between the check and the open. A file named
|
|
||||||
// for a period that the prune leaves empty is deleted.
|
|
||||||
//
|
//
|
||||||
// The archive is left CLOSED afterwards. An idle archive holding
|
// The archive is left CLOSED afterwards. An idle archive holding
|
||||||
// no handle is what keeps the operator's move-the-file-away
|
// no handle is what keeps the operator's move-the-file-away
|
||||||
// workflow working; the next write reopens (and recreates) the
|
// workflow working; the next write reopens (and recreates) the
|
||||||
// file as it always has.
|
// file as it always has.
|
||||||
func (w *archiveWriter) sweepExpired(expiry time.Duration) error {
|
func (w *archiveWriter) sweepExpired(expiry time.Duration) error {
|
||||||
w.mu.Lock()
|
|
||||||
files, err := archiveFiles(w.path)
|
|
||||||
w.mu.Unlock()
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
var errs []error
|
|
||||||
|
|
||||||
for _, file := range files {
|
|
||||||
err = w.sweepFile(file, expiry)
|
|
||||||
if errors.Is(err, errArchiveWriterEvicted) {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
errs = append(errs, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return errors.Join(errs...)
|
|
||||||
}
|
|
||||||
|
|
||||||
// sweepFile prunes one of the target's archive files for sweepExpired,
|
|
||||||
// holding w.mu while it does. It skips a file that is gone, and deletes
|
|
||||||
// the file, with its -wal and -shm, when it is named for a period and
|
|
||||||
// the prune leaves it empty.
|
|
||||||
func (w *archiveWriter) sweepFile(
|
|
||||||
file archiveFile, expiry time.Duration,
|
|
||||||
) error {
|
|
||||||
w.mu.Lock()
|
w.mu.Lock()
|
||||||
defer w.mu.Unlock()
|
defer w.mu.Unlock()
|
||||||
|
|
||||||
@@ -438,7 +385,7 @@ func (w *archiveWriter) sweepFile(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !fileExists(file.path) {
|
if !fileExists(w.path) {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -446,56 +393,26 @@ func (w *archiveWriter) sweepFile(
|
|||||||
// freshly opened file, matching the write path's semantics.
|
// freshly opened file, matching the write path's semantics.
|
||||||
w.close()
|
w.close()
|
||||||
|
|
||||||
err := w.openMode(file.path, archiveModeExisting, expiry)
|
err := w.openMode(archiveModeExisting, expiry)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if file.period == "" {
|
|
||||||
w.close()
|
w.close()
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var rows int64
|
|
||||||
|
|
||||||
err = w.db.Model(&archivedEvent{}).Count(&rows).Error
|
|
||||||
|
|
||||||
w.close()
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"counting rows in archive %s: %w", file.path, err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
if rows > 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, suffix := range []string{"", "-wal", "-shm"} {
|
|
||||||
err = os.Remove(file.path + suffix)
|
|
||||||
if err != nil && !errors.Is(err, fs.ErrNotExist) {
|
|
||||||
return fmt.Errorf("deleting empty archive file: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
w.log.Info("deleted empty archive file", "path", file.path)
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// rename gives every one of the target's archive files the new
|
// rename gives the archive file a new name in the same directory,
|
||||||
// name, keeping the period in the name of each (see
|
// and the writer uses the file under that name from now on. The
|
||||||
// archivePeriodPath), and the writer uses the files under that name
|
// handle is closed first, which folds the -wal into the .db; any
|
||||||
// from now on. The handle is closed first, which folds the -wal into
|
// -wal or -shm still beside the file (left by a crash) is moved with
|
||||||
// the .db; any -wal or -shm still beside a file (left by a crash) is
|
// it, because SQLite finds them by name. A missing file is not an
|
||||||
// moved with it, because SQLite finds them by name. A target with no
|
// error: the operator may have moved it away, and the next write
|
||||||
// files is not an error: the operator may have moved them away, and
|
// creates it under the new name.
|
||||||
// the next write creates its file under the new name.
|
|
||||||
//
|
//
|
||||||
// If a file already has one of the new names, nothing is moved and
|
// If a file already has the new name, nothing is moved and the
|
||||||
// the error is ErrArchiveNameTaken. If one file fails to move, those
|
// error is ErrArchiveNameTaken. If one file fails to move, those
|
||||||
// already moved are moved back before the error is returned, so the
|
// already moved are moved back before the error is returned, so the
|
||||||
// archive is never split across two names.
|
// archive is never split across two names.
|
||||||
func (w *archiveWriter) rename(name string) error {
|
func (w *archiveWriter) rename(name string) error {
|
||||||
@@ -513,53 +430,38 @@ func (w *archiveWriter) rename(name string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
files, err := archiveFiles(w.path)
|
suffixes := []string{"", "-wal", "-shm"}
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// from[i] moves to to[i].
|
for _, suffix := range suffixes {
|
||||||
var from, to []string
|
if fileExists(path + suffix) {
|
||||||
|
|
||||||
for _, file := range files {
|
|
||||||
renamed := archivePeriodPath(path, file.period)
|
|
||||||
|
|
||||||
for _, suffix := range []string{"", "-wal", "-shm"} {
|
|
||||||
from = append(from, file.path+suffix)
|
|
||||||
to = append(to, renamed+suffix)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, taken := range to {
|
|
||||||
if fileExists(taken) {
|
|
||||||
return fmt.Errorf(
|
return fmt.Errorf(
|
||||||
"%w: %s", ErrArchiveNameTaken, filepath.Base(taken),
|
"%w: %s", ErrArchiveNameTaken, name+suffix,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
w.close()
|
w.close()
|
||||||
|
|
||||||
for i := range from {
|
for i, suffix := range suffixes {
|
||||||
err = os.Rename(from[i], to[i])
|
err := os.Rename(w.path+suffix, path+suffix)
|
||||||
if err == nil || errors.Is(err, fs.ErrNotExist) {
|
if err == nil || errors.Is(err, fs.ErrNotExist) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
for j := range i {
|
for _, moved := range suffixes[:i] {
|
||||||
backErr := os.Rename(to[j], from[j])
|
backErr := os.Rename(path+moved, w.path+moved)
|
||||||
if backErr != nil && !errors.Is(backErr, fs.ErrNotExist) {
|
if backErr != nil && !errors.Is(backErr, fs.ErrNotExist) {
|
||||||
w.log.Error(
|
w.log.Error(
|
||||||
"failed to move archive file back",
|
"failed to move archive file back",
|
||||||
"from", to[j],
|
"from", path+moved,
|
||||||
"to", from[j],
|
"to", w.path+moved,
|
||||||
"error", backErr,
|
"error", backErr,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return fmt.Errorf(
|
return fmt.Errorf(
|
||||||
"renaming archive %s to %s: %w", from[i], to[i], err,
|
"renaming archive %s to %s: %w", w.path+suffix, path+suffix, err,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -597,7 +499,7 @@ func (w *archiveWriter) prune(expiry time.Duration) {
|
|||||||
if res.Error != nil {
|
if res.Error != nil {
|
||||||
w.log.Error(
|
w.log.Error(
|
||||||
"failed to prune expired archive rows",
|
"failed to prune expired archive rows",
|
||||||
"path", w.current,
|
"path", w.path,
|
||||||
"error", res.Error,
|
"error", res.Error,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -607,59 +509,38 @@ func (w *archiveWriter) prune(expiry time.Duration) {
|
|||||||
if res.RowsAffected > 0 {
|
if res.RowsAffected > 0 {
|
||||||
w.log.Info(
|
w.log.Info(
|
||||||
"pruned expired archive rows",
|
"pruned expired archive rows",
|
||||||
"path", w.current,
|
"path", w.path,
|
||||||
"rows_deleted", res.RowsAffected,
|
"rows_deleted", res.RowsAffected,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ArchiveFileInfo is what the metadata of a database target's archive
|
// ArchiveFileInfo is what the metadata of a database target's archive
|
||||||
// files says about them.
|
// file says about it.
|
||||||
type ArchiveFileInfo struct {
|
type ArchiveFileInfo struct {
|
||||||
// Files counts the files.
|
// Size is the bytes on disk of the file and its -wal together.
|
||||||
Files int
|
|
||||||
|
|
||||||
// Size is the bytes on disk of the files and their -wal together.
|
|
||||||
Size int64
|
Size int64
|
||||||
|
|
||||||
// Written is when a file or a -wal was last modified, whichever is
|
// Written is when the file or its -wal was last modified, whichever
|
||||||
// latest: a write lands in the -wal first.
|
// is later: a write lands in the -wal first.
|
||||||
Written time.Time
|
Written time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
// StatArchive reads the metadata of a database target's archive
|
// StatArchive reads the metadata of the archive file at path and of
|
||||||
// files, given the path ArchivePath gives it (see archiveFiles), and
|
// its -wal, without opening the archive. With no file at path, which is
|
||||||
// of their -wal, without opening them. With no files, which is so
|
// so before the first write and after the operator moved it away, the
|
||||||
// before the first write and after the operator moved them away, the
|
|
||||||
// error wraps fs.ErrNotExist.
|
// error wraps fs.ErrNotExist.
|
||||||
func StatArchive(path string) (ArchiveFileInfo, error) {
|
func StatArchive(path string) (ArchiveFileInfo, error) {
|
||||||
files, err := archiveFiles(path)
|
file, err := os.Stat(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ArchiveFileInfo{}, err
|
return ArchiveFileInfo{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
var info ArchiveFileInfo
|
info := ArchiveFileInfo{Size: file.Size(), Written: file.ModTime()}
|
||||||
|
|
||||||
for _, file := range files {
|
wal, err := os.Stat(path + "-wal")
|
||||||
db, err := os.Stat(file.path)
|
|
||||||
if errors.Is(err, fs.ErrNotExist) {
|
if errors.Is(err, fs.ErrNotExist) {
|
||||||
continue
|
return info, nil
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return ArchiveFileInfo{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
info.Files++
|
|
||||||
info.Size += db.Size()
|
|
||||||
|
|
||||||
if db.ModTime().After(info.Written) {
|
|
||||||
info.Written = db.ModTime()
|
|
||||||
}
|
|
||||||
|
|
||||||
wal, err := os.Stat(file.path + "-wal")
|
|
||||||
if errors.Is(err, fs.ErrNotExist) {
|
|
||||||
continue
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -671,13 +552,6 @@ func StatArchive(path string) (ArchiveFileInfo, error) {
|
|||||||
if wal.ModTime().After(info.Written) {
|
if wal.ModTime().After(info.Written) {
|
||||||
info.Written = wal.ModTime()
|
info.Written = wal.ModTime()
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if info.Files == 0 {
|
|
||||||
return ArchiveFileInfo{}, fmt.Errorf(
|
|
||||||
"no archive file for %s: %w", path, fs.ErrNotExist,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return info, nil
|
return info, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,14 +6,10 @@ import (
|
|||||||
"database/sql"
|
"database/sql"
|
||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"io/fs"
|
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"os"
|
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sync"
|
|
||||||
"time"
|
"time"
|
||||||
"unicode/utf8"
|
"unicode/utf8"
|
||||||
|
|
||||||
@@ -54,46 +50,21 @@ func ArchiveExportFileName(
|
|||||||
at.UTC().Format("20060102T150405Z") + ".json.gz"
|
at.UTC().Format("20060102T150405Z") + ".json.gz"
|
||||||
}
|
}
|
||||||
|
|
||||||
// ArchiveExport is a database target's archive listed for download. It
|
// ArchiveExport is a database target's archive opened for download.
|
||||||
// opens one of the target's files at a time, only when its rows are
|
// It reads the file on its own connection, inside one read-only
|
||||||
// about to be written out, and closes it before it opens the next, so
|
// transaction, so it writes out the archive as it stood when
|
||||||
// an export holds at most one file open however many the target has.
|
// OpenArchiveExport returned.
|
||||||
//
|
//
|
||||||
// Each file is read on its own connection inside one read-only
|
// Archives are in WAL mode, where a reader works from a snapshot and
|
||||||
// transaction, so its rows are written out as the file stood when it
|
// never blocks a writer: archive writes go on while an export is open,
|
||||||
// was opened. Archives are in WAL mode, where a reader works from a
|
// and the export does not see them. SQLite cannot checkpoint the -wal
|
||||||
// snapshot and never blocks a writer: archive writes go on while a file
|
// past an open snapshot, so the -wal grows until the export is closed.
|
||||||
// is open, and the export does not see them. SQLite cannot checkpoint
|
|
||||||
// a -wal past an open snapshot, so the open file's -wal grows until the
|
|
||||||
// export has written that file out.
|
|
||||||
type ArchiveExport struct {
|
type ArchiveExport struct {
|
||||||
// periods are the periods of the target's files when the export
|
|
||||||
// was listed, "" for the file without one, in the order
|
|
||||||
// archiveFiles lists them.
|
|
||||||
periods []string
|
|
||||||
|
|
||||||
// lock is held while currentPath is called and a file is opened,
|
|
||||||
// so that a rename, which holds it too, cannot move the file in
|
|
||||||
// between.
|
|
||||||
lock sync.Locker
|
|
||||||
|
|
||||||
// currentPath returns the path ArchivePath gives the target under
|
|
||||||
// the names stored for it now, which a rename may have changed since
|
|
||||||
// the export was listed.
|
|
||||||
currentPath func() (string, error)
|
|
||||||
|
|
||||||
log *slog.Logger
|
|
||||||
}
|
|
||||||
|
|
||||||
// exportFile is one archive file opened for an export.
|
|
||||||
type exportFile struct {
|
|
||||||
db *sql.DB
|
db *sql.DB
|
||||||
tx *gorm.DB
|
tx *gorm.DB
|
||||||
|
|
||||||
// period is the period in the file's name, "" for none.
|
// empty is true when there is nothing to read: no file, or a file
|
||||||
period string
|
// without the archive's table yet.
|
||||||
|
|
||||||
// empty is true for a file without the archive's table yet.
|
|
||||||
empty bool
|
empty bool
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -103,46 +74,26 @@ type exportedName struct {
|
|||||||
Name string `json:"name"`
|
Name string `json:"name"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewArchiveExport lists a database target's archive files for export,
|
// OpenArchiveExport opens the archive file at path for export and
|
||||||
// given the path ArchivePath gives it (see archiveFiles). It opens none
|
// takes the snapshot the export reads. It never creates the file: with
|
||||||
// of them. Its caller holds lock, which every rename of the target's
|
// no file at path, the export has no rows.
|
||||||
// files runs under, from reading the names path is made of until it
|
|
||||||
// returns, so the files it lists are the ones those names give.
|
|
||||||
//
|
//
|
||||||
// WriteGzipJSON, called without lock held, finds each file again by its
|
// Once it has returned, the file is open, so a rename or a move of it
|
||||||
// period under the path currentPath gives, holding lock while it does
|
// does not affect the export, which reads the same file under its new
|
||||||
// and while it opens the file, so a rename during the export loses no
|
// name.
|
||||||
// file. A file that is gone by then, emptied by the sweep or moved
|
//
|
||||||
// away, is skipped. The export never creates a file: with no files, it
|
// The transaction lasts as long as ctx does, so ctx must last for the
|
||||||
// has no rows.
|
// whole export.
|
||||||
func NewArchiveExport(
|
func OpenArchiveExport(
|
||||||
path string,
|
ctx context.Context, path string, log *slog.Logger,
|
||||||
lock sync.Locker,
|
|
||||||
currentPath func() (string, error),
|
|
||||||
log *slog.Logger,
|
|
||||||
) (*ArchiveExport, error) {
|
) (*ArchiveExport, error) {
|
||||||
files, err := archiveFiles(path)
|
if !fileExists(path) {
|
||||||
if err != nil {
|
return &ArchiveExport{empty: true}, nil
|
||||||
return nil, err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
x := &ArchiveExport{lock: lock, currentPath: currentPath, log: log}
|
db, err := database.OpenSQLite(path, archiveModeExisting)
|
||||||
|
|
||||||
for _, file := range files {
|
|
||||||
x.periods = append(x.periods, file.period)
|
|
||||||
}
|
|
||||||
|
|
||||||
return x, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// openExportFile opens one archive file for an export and takes its
|
|
||||||
// snapshot. The transaction lasts as long as ctx does.
|
|
||||||
func openExportFile(
|
|
||||||
ctx context.Context, file archiveFile, log *slog.Logger,
|
|
||||||
) (*exportFile, error) {
|
|
||||||
db, err := database.OpenSQLite(file.path, archiveModeExisting)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("opening archive %s: %w", file.path, err)
|
return nil, fmt.Errorf("opening archive %s: %w", path, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
gdb, err := gorm.Open(
|
gdb, err := gorm.Open(
|
||||||
@@ -155,7 +106,7 @@ func openExportFile(
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
_ = db.Close()
|
_ = db.Close()
|
||||||
|
|
||||||
return nil, fmt.Errorf("opening archive %s: %w", file.path, err)
|
return nil, fmt.Errorf("opening archive %s: %w", path, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ReadOnly makes the driver begin a deferred transaction in place
|
// ReadOnly makes the driver begin a deferred transaction in place
|
||||||
@@ -165,9 +116,7 @@ func openExportFile(
|
|||||||
if tx.Error != nil {
|
if tx.Error != nil {
|
||||||
_ = db.Close()
|
_ = db.Close()
|
||||||
|
|
||||||
return nil, fmt.Errorf(
|
return nil, fmt.Errorf("reading archive %s: %w", path, tx.Error)
|
||||||
"reading archive %s: %w", file.path, tx.Error,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// The transaction's first read is what takes the snapshot.
|
// The transaction's first read is what takes the snapshot.
|
||||||
@@ -178,27 +127,22 @@ func openExportFile(
|
|||||||
_ = tx.Rollback()
|
_ = tx.Rollback()
|
||||||
_ = db.Close()
|
_ = db.Close()
|
||||||
|
|
||||||
return nil, fmt.Errorf("reading archive %s: %w", file.path, err)
|
return nil, fmt.Errorf("reading archive %s: %w", path, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return &exportFile{
|
return &ArchiveExport{db: db, tx: tx, empty: tables == 0}, nil
|
||||||
db: db, tx: tx, period: file.period, empty: tables == 0,
|
|
||||||
}, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// WriteGzipJSON writes the export to w as one gzipped JSON object:
|
// WriteGzipJSON writes the export to w as one gzipped JSON object:
|
||||||
// webhook and target, each an id and a name; exported_at; and
|
// webhook and target, each an id and a name; exported_at; and
|
||||||
// archived_events, one object per archived row, keyed by column name,
|
// archived_events, one object per archived row, keyed by column name.
|
||||||
// the files in the order archiveFiles lists them. A row from a file
|
// A body that is not valid UTF-8 cannot be a JSON string, so it is
|
||||||
// named for a period has "period" beside its columns. A body that is
|
// written in base64, with "body_encoding": "base64" beside it.
|
||||||
// not valid UTF-8 cannot be a JSON string, so it is written in base64,
|
|
||||||
// with "body_encoding": "base64" beside it.
|
|
||||||
//
|
//
|
||||||
// Each row is written out before the next is read, so neither the
|
// Each row is written out before the next is read, so neither the
|
||||||
// archive nor its JSON is ever held in memory whole, and each file is
|
// archive nor its JSON is ever held in memory whole. After an error
|
||||||
// closed once its rows are written, before the next is opened. When it
|
// the gzip stream is left unfinished, so what was written does not
|
||||||
// returns, no file is open. After an error the gzip stream is left
|
// decompress as a whole file.
|
||||||
// unfinished, so what was written does not decompress as a whole file.
|
|
||||||
func (x *ArchiveExport) WriteGzipJSON(
|
func (x *ArchiveExport) WriteGzipJSON(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
w io.Writer,
|
w io.Writer,
|
||||||
@@ -225,35 +169,15 @@ func (x *ArchiveExport) WriteGzipJSON(
|
|||||||
return zw.Close()
|
return zw.Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
// openFile finds the target's archive file for period under the path
|
// Close ends the export's transaction and closes its connection.
|
||||||
// currentPath gives now, and opens it for the export, holding x.lock
|
func (x *ArchiveExport) Close() error {
|
||||||
// for both. For a file that is gone, the error wraps fs.ErrNotExist.
|
if x.db == nil {
|
||||||
func (x *ArchiveExport) openFile(
|
return nil
|
||||||
ctx context.Context, period string,
|
|
||||||
) (*exportFile, error) {
|
|
||||||
x.lock.Lock()
|
|
||||||
defer x.lock.Unlock()
|
|
||||||
|
|
||||||
path, err := x.currentPath()
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("finding archive file: %w", err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
file := archiveFile{path: archivePeriodPath(path, period), period: period}
|
_ = x.tx.Rollback()
|
||||||
|
|
||||||
_, err = os.Stat(file.path)
|
return x.db.Close()
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return openExportFile(ctx, file, x.log)
|
|
||||||
}
|
|
||||||
|
|
||||||
// close ends the file's transaction and closes its connection.
|
|
||||||
func (f *exportFile) close() error {
|
|
||||||
_ = f.tx.Rollback()
|
|
||||||
|
|
||||||
return f.db.Close()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// writeJSON writes head with archived_events added as its last key,
|
// writeJSON writes head with archived_events added as its last key,
|
||||||
@@ -283,84 +207,52 @@ func (x *ArchiveExport) writeJSON(
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// writeRows writes the archived rows of each file to w, one per line,
|
// writeRows writes each archived row to w, oldest first, one per line,
|
||||||
// separated by commas, opening each file in turn and closing it once
|
// separated by commas.
|
||||||
// its rows are written.
|
|
||||||
func (x *ArchiveExport) writeRows(ctx context.Context, w io.Writer) error {
|
func (x *ArchiveExport) writeRows(ctx context.Context, w io.Writer) error {
|
||||||
sep := "\n"
|
if x.empty {
|
||||||
|
|
||||||
for _, period := range x.periods {
|
|
||||||
f, err := x.openFile(ctx, period)
|
|
||||||
if errors.Is(err, fs.ErrNotExist) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
sep, err = f.writeRows(ctx, w, sep)
|
|
||||||
|
|
||||||
err = errors.Join(err, f.close())
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
|
||||||
|
|
||||||
// writeRows writes the file's archived rows to w, oldest first, the
|
|
||||||
// first after sep and each other after ",\n". It returns what goes
|
|
||||||
// before the next row: sep again when the file had no rows.
|
|
||||||
func (f *exportFile) writeRows(
|
|
||||||
ctx context.Context, w io.Writer, sep string,
|
|
||||||
) (string, error) {
|
|
||||||
if f.empty {
|
|
||||||
return sep, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
rows, err := f.tx.WithContext(ctx).
|
rows, err := x.tx.WithContext(ctx).
|
||||||
Model(&archivedEvent{}).Order("id").Rows()
|
Model(&archivedEvent{}).Order("id").Rows()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
defer func() { _ = rows.Close() }()
|
defer func() { _ = rows.Close() }()
|
||||||
|
|
||||||
for ; rows.Next(); sep = ",\n" {
|
for sep := "\n"; rows.Next(); sep = ",\n" {
|
||||||
var ev archivedEvent
|
var ev archivedEvent
|
||||||
|
|
||||||
err = f.tx.ScanRows(rows, &ev)
|
err = x.tx.ScanRows(rows, &ev)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = io.WriteString(w, sep)
|
_, err = io.WriteString(w, sep)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
err = writeRow(w, &ev, f.period)
|
err = writeRow(w, &ev)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return sep, rows.Err()
|
return rows.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
// writeRow writes an archived row to w as a JSON object keyed by
|
// writeRow writes an archived row to w as a JSON object keyed by
|
||||||
// column name, its body in base64 when it is not valid UTF-8, with
|
// column name, its body in base64 when it is not valid UTF-8.
|
||||||
// the period of its file beside them unless that is "".
|
func writeRow(w io.Writer, ev *archivedEvent) error {
|
||||||
func writeRow(w io.Writer, ev *archivedEvent, period string) error {
|
|
||||||
row := map[string]any{
|
row := map[string]any{
|
||||||
"id": ev.ID,
|
"id": ev.ID,
|
||||||
"event_id": ev.EventID,
|
"event_id": ev.EventID,
|
||||||
"webhook_id": ev.WebhookID,
|
"webhook_id": ev.WebhookID,
|
||||||
"entrypoint_id": ev.EntrypointID,
|
"entrypoint_id": ev.EntrypointID,
|
||||||
"method": ev.Method,
|
"method": ev.Method,
|
||||||
"raw_query": ev.RawQuery,
|
|
||||||
"headers": ev.Headers,
|
"headers": ev.Headers,
|
||||||
"body": ev.Body,
|
"body": ev.Body,
|
||||||
"content_type": ev.ContentType,
|
"content_type": ev.ContentType,
|
||||||
@@ -372,10 +264,6 @@ func writeRow(w io.Writer, ev *archivedEvent, period string) error {
|
|||||||
row["body_encoding"] = "base64"
|
row["body_encoding"] = "base64"
|
||||||
}
|
}
|
||||||
|
|
||||||
if period != "" {
|
|
||||||
row["period"] = period
|
|
||||||
}
|
|
||||||
|
|
||||||
line, err := json.Marshal(row)
|
line, err := json.Marshal(row)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -12,8 +12,6 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"runtime"
|
"runtime"
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -31,8 +29,14 @@ const (
|
|||||||
exportTargetName = "Long-term archive"
|
exportTargetName = "Long-term archive"
|
||||||
)
|
)
|
||||||
|
|
||||||
// binaryBody is a body that is not valid UTF-8.
|
const (
|
||||||
const binaryBody = "\xff\xfe\x00\x01binary\x80"
|
// binaryBody is a body that is not valid UTF-8.
|
||||||
|
binaryBody = "\xff\xfe\x00\x01binary\x80"
|
||||||
|
|
||||||
|
// openedEventID is the event the snapshot tests archive before
|
||||||
|
// they open the export.
|
||||||
|
openedEventID = "opened"
|
||||||
|
)
|
||||||
|
|
||||||
// writeExportTo writes export to w as the archive of the export tests'
|
// writeExportTo writes export to w as the archive of the export tests'
|
||||||
// webhook and target, exported at 2026-10-02T12:03:04Z.
|
// webhook and target, exported at 2026-10-02T12:03:04Z.
|
||||||
@@ -55,41 +59,19 @@ func writeExportTo(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// listExport lists the archive at path for export, as the archive of a
|
|
||||||
// target whose names do not change.
|
|
||||||
func listExport(t *testing.T, path string) *delivery.ArchiveExport {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
return newExport(t, path, &sync.Mutex{}, func() (string, error) {
|
|
||||||
return path, nil
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// newExport lists the archive at path for export, to find each file
|
|
||||||
// again under the path currentPath gives, holding lock while it does.
|
|
||||||
// Nothing else takes lock while it lists, so it does not hold lock.
|
|
||||||
func newExport(
|
|
||||||
t *testing.T,
|
|
||||||
path string,
|
|
||||||
lock sync.Locker,
|
|
||||||
currentPath func() (string, error),
|
|
||||||
) *delivery.ArchiveExport {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
export, err := delivery.NewArchiveExport(
|
|
||||||
path, lock, currentPath, archiveTestLogger(),
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
return export
|
|
||||||
}
|
|
||||||
|
|
||||||
// exportArchive runs a whole export of the archive at path and returns
|
// exportArchive runs a whole export of the archive at path and returns
|
||||||
// its JSON, decompressed and parsed.
|
// its JSON, decompressed and parsed.
|
||||||
func exportArchive(t *testing.T, path string) map[string]any {
|
func exportArchive(t *testing.T, path string) map[string]any {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
return writeExport(t, listExport(t, path))
|
export, err := delivery.OpenArchiveExport(
|
||||||
|
t.Context(), path, archiveTestLogger(),
|
||||||
|
)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
defer func() { require.NoError(t, export.Close()) }()
|
||||||
|
|
||||||
|
return writeExport(t, export)
|
||||||
}
|
}
|
||||||
|
|
||||||
// writeExport writes an opened export and returns its JSON,
|
// writeExport writes an opened export and returns its JSON,
|
||||||
@@ -166,7 +148,6 @@ func TestArchiveExport_MatchesStoredRows(t *testing.T) {
|
|||||||
WebhookID: exportWebhookID,
|
WebhookID: exportWebhookID,
|
||||||
EntrypointID: "ep-1",
|
EntrypointID: "ep-1",
|
||||||
Method: "POST",
|
Method: "POST",
|
||||||
RawQuery: eventQuery,
|
|
||||||
Headers: `{"X-Test":["yes"]}`,
|
Headers: `{"X-Test":["yes"]}`,
|
||||||
Body: body,
|
Body: body,
|
||||||
ContentType: testContentType,
|
ContentType: testContentType,
|
||||||
@@ -216,13 +197,12 @@ func assertExportedRow(
|
|||||||
assert.Equal(t, row.WebhookID, ev["webhook_id"])
|
assert.Equal(t, row.WebhookID, ev["webhook_id"])
|
||||||
assert.Equal(t, row.EntrypointID, ev["entrypoint_id"])
|
assert.Equal(t, row.EntrypointID, ev["entrypoint_id"])
|
||||||
assert.Equal(t, row.Method, ev["method"])
|
assert.Equal(t, row.Method, ev["method"])
|
||||||
assert.Equal(t, row.RawQuery, ev["raw_query"])
|
|
||||||
assert.Equal(t, row.Headers, ev["headers"])
|
assert.Equal(t, row.Headers, ev["headers"])
|
||||||
assert.Equal(t, row.ContentType, ev["content_type"])
|
assert.Equal(t, row.ContentType, ev["content_type"])
|
||||||
|
|
||||||
if row.Body != binaryBody {
|
if row.Body != binaryBody {
|
||||||
assert.Equal(t, row.Body, ev["body"])
|
assert.Equal(t, row.Body, ev["body"])
|
||||||
assert.Len(t, ev, 10, "the ten columns and nothing else: %v", ev)
|
assert.Len(t, ev, 9, "the nine columns and nothing else: %v", ev)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -231,7 +211,7 @@ func assertExportedRow(
|
|||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, binaryBody, string(body))
|
assert.Equal(t, binaryBody, string(body))
|
||||||
assert.Equal(t, "base64", ev["body_encoding"])
|
assert.Equal(t, "base64", ev["body_encoding"])
|
||||||
assert.Len(t, ev, 11, "the ten columns and body_encoding: %v", ev)
|
assert.Len(t, ev, 10, "the nine columns and body_encoding: %v", ev)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestArchiveExport_Empty proves an archive with nothing in it exports
|
// TestArchiveExport_Empty proves an archive with nothing in it exports
|
||||||
@@ -261,212 +241,69 @@ func TestArchiveExport_Empty(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// unlockHook is a sync.Locker that runs fn each time it is unlocked. An
|
// TestArchiveExport_ReadsOneSnapshot proves an export writes the
|
||||||
// export unlocks its lock right after it opens a file.
|
// archive as it was when it was opened, and holds up no archive
|
||||||
type unlockHook struct {
|
// write: a row written while the export is open is stored, and is not
|
||||||
sync.Mutex
|
// in the export. A write held up for the whole busy timeout would
|
||||||
|
// fail.
|
||||||
fn func()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u *unlockHook) Unlock() {
|
|
||||||
u.Mutex.Unlock()
|
|
||||||
u.fn()
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveExport_ReadsOneSnapshot proves an export writes a file
|
|
||||||
// out as it was when the export opened it, and holds up no archive
|
|
||||||
// write: a row written after the export was listed but before the file
|
|
||||||
// was opened is in the export, and one written while the file is open
|
|
||||||
// is stored, and is not. A write held up for the whole busy timeout
|
|
||||||
// would fail.
|
|
||||||
func TestArchiveExport_ReadsOneSnapshot(t *testing.T) {
|
func TestArchiveExport_ReadsOneSnapshot(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "archive.db")
|
path := filepath.Join(t.TempDir(), "archive.db")
|
||||||
w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
||||||
|
|
||||||
require.NoError(t, w.Write(delivery.ExportArchivedEvent{EventID: "listed"}, 0))
|
require.NoError(t, w.Write(delivery.ExportArchivedEvent{EventID: openedEventID}, 0))
|
||||||
|
|
||||||
|
export, err := delivery.OpenArchiveExport(
|
||||||
|
t.Context(), path, archiveTestLogger(),
|
||||||
|
)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
defer func() { require.NoError(t, export.Close()) }()
|
||||||
|
|
||||||
opened := &unlockHook{fn: func() {
|
|
||||||
require.NoError(t, w.Write(delivery.ExportArchivedEvent{EventID: "during"}, 0))
|
require.NoError(t, w.Write(delivery.ExportArchivedEvent{EventID: "during"}, 0))
|
||||||
}}
|
|
||||||
export := newExport(t, path, opened, func() (string, error) {
|
|
||||||
return path, nil
|
|
||||||
})
|
|
||||||
|
|
||||||
require.NoError(t, w.Write(delivery.ExportArchivedEvent{EventID: "before-open"}, 0))
|
|
||||||
|
|
||||||
assert.Equal(t,
|
assert.Equal(t,
|
||||||
[]string{"listed", "before-open"},
|
[]string{openedEventID}, exportedEventIDs(t, writeExport(t, export)),
|
||||||
exportedEventIDs(t, writeExport(t, export)),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var stored int64
|
var stored int64
|
||||||
|
|
||||||
require.NoError(t, openArchiveDBForRead(t, path).
|
require.NoError(t, openArchiveDBForRead(t, path).
|
||||||
Model(&delivery.ExportArchivedEvent{}).Count(&stored).Error)
|
Model(&delivery.ExportArchivedEvent{}).Count(&stored).Error)
|
||||||
assert.Equal(t, int64(3), stored)
|
assert.Equal(t, int64(2), stored)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestArchiveExport_FindsFilesAfterRename proves that renaming the
|
// TestArchiveExport_SurvivesRename proves that renaming the archive
|
||||||
// archive after an export has listed it, as renaming its webhook or
|
// while an export of it is open, as renaming its webhook or target
|
||||||
// target does, loses no file: the export finds each file again by its
|
// does, leaves the export reading the same file.
|
||||||
// period under the new name. A file moved away by then is skipped.
|
func TestArchiveExport_SurvivesRename(t *testing.T) {
|
||||||
func TestArchiveExport_FindsFilesAfterRename(t *testing.T) {
|
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
dir := t.TempDir()
|
path := filepath.Join(t.TempDir(), "archive-old.db")
|
||||||
path := filepath.Join(dir, "archive-old.db")
|
|
||||||
w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
||||||
|
|
||||||
for _, period := range []string{"", dayPeriod, hourPeriod} {
|
require.NoError(t, w.Write(delivery.ExportArchivedEvent{EventID: openedEventID}, 0))
|
||||||
require.NoError(t, w.WritePeriod(
|
|
||||||
delivery.ExportArchivedEvent{EventID: "in-" + period}, 0, period,
|
|
||||||
))
|
|
||||||
}
|
|
||||||
|
|
||||||
current := path
|
export, err := delivery.OpenArchiveExport(
|
||||||
export := newExport(t, path, &sync.Mutex{}, func() (string, error) {
|
t.Context(), path, archiveTestLogger(),
|
||||||
return current, nil
|
|
||||||
})
|
|
||||||
|
|
||||||
require.NoError(t, w.Rename("archive-new.db"))
|
|
||||||
|
|
||||||
current = filepath.Join(dir, "archive-new.db")
|
|
||||||
|
|
||||||
removeArchiveFiles(t, periodPath(current, dayPeriod))
|
|
||||||
|
|
||||||
assert.Equal(t,
|
|
||||||
[]string{"in-", "in-" + hourPeriod},
|
|
||||||
exportedEventIDs(t, writeExport(t, export)),
|
|
||||||
)
|
)
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveExport_EveryFileOldestFirst writes a row to a target's
|
|
||||||
// file without a period and to its files for a month, an hour and a
|
|
||||||
// day, and proves the export holds every row: the file without a
|
|
||||||
// period first, then the others oldest period first, each row from a
|
|
||||||
// file named for a period carrying that period. A file made after the
|
|
||||||
// export was listed is not in it.
|
|
||||||
func TestArchiveExport_EveryFileOldestFirst(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "archive-wh.db")
|
|
||||||
w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
|
||||||
|
|
||||||
// Written in an order that is not the export's.
|
|
||||||
for _, period := range []string{nextDayPeriod, "", hourPeriod, "2026-03"} {
|
|
||||||
require.NoError(t, w.WritePeriod(
|
|
||||||
delivery.ExportArchivedEvent{EventID: "in-" + period}, 0, period,
|
|
||||||
))
|
|
||||||
}
|
|
||||||
|
|
||||||
export := listExport(t, path)
|
|
||||||
|
|
||||||
require.NoError(t, w.WritePeriod(
|
|
||||||
delivery.ExportArchivedEvent{EventID: "later"}, 0, "2026-03-06",
|
|
||||||
))
|
|
||||||
|
|
||||||
events := exportedEvents(t, writeExport(t, export))
|
|
||||||
ids := make([]string, 0, len(events))
|
|
||||||
periods := make([]any, 0, len(events))
|
|
||||||
|
|
||||||
for _, ev := range events {
|
|
||||||
ids = append(ids, fmt.Sprint(ev["event_id"]))
|
|
||||||
periods = append(periods, ev["period"])
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equal(t,
|
|
||||||
[]string{"in-", "in-2026-03", "in-" + hourPeriod, "in-" + nextDayPeriod},
|
|
||||||
ids,
|
|
||||||
)
|
|
||||||
assert.Equal(t,
|
|
||||||
[]any{nil, "2026-03", hourPeriod, nextDayPeriod}, periods,
|
|
||||||
)
|
|
||||||
assert.NotContains(t, events[0], "period",
|
|
||||||
"a row from the file without a period has no period")
|
|
||||||
}
|
|
||||||
|
|
||||||
// openFilesPeak is an io.Writer that discards what it is given and
|
|
||||||
// records the most archive files in dir the process had open at any
|
|
||||||
// write, as /proc/self/fd lists the files a process has open.
|
|
||||||
type openFilesPeak struct {
|
|
||||||
dir string
|
|
||||||
max int
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *openFilesPeak) Write(b []byte) (int, error) {
|
|
||||||
fds, err := os.ReadDir("/proc/self/fd")
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
open := map[string]bool{}
|
|
||||||
|
|
||||||
for _, fd := range fds {
|
|
||||||
file, err := os.Readlink(filepath.Join("/proc/self/fd", fd.Name()))
|
|
||||||
if err == nil && filepath.Dir(file) == p.dir &&
|
|
||||||
strings.HasSuffix(file, ".db") {
|
|
||||||
open[file] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
p.max = max(p.max, len(open))
|
|
||||||
|
|
||||||
return len(b), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveExport_OneFileOpenAtATime exports a target with a file for
|
|
||||||
// each of 24 hours and proves the export never had more than one of
|
|
||||||
// them open, and had one open while it wrote. Each file holds a row of
|
|
||||||
// 48 KiB of random base64, which gzip shrinks little, so the export
|
|
||||||
// writes output while it reads each file.
|
|
||||||
func TestArchiveExport_OneFileOpenAtATime(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
if runtime.GOOS != "linux" {
|
|
||||||
t.Skip("only Linux lists a process's open files in /proc/self/fd")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Readlink gives each open file's path with no symbolic link in it.
|
|
||||||
dir, err := filepath.EvalSymlinks(t.TempDir())
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
path := filepath.Join(dir, "archive-wh.db")
|
defer func() { require.NoError(t, export.Close()) }()
|
||||||
w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
|
||||||
random := make([]byte, 36<<10)
|
|
||||||
|
|
||||||
for hour := range 24 {
|
require.NoError(t, w.Rename("archive-new.db"))
|
||||||
_, _ = rand.Read(random)
|
require.NoError(t, w.Write(delivery.ExportArchivedEvent{EventID: "after"}, 0))
|
||||||
|
require.NoFileExists(t, path)
|
||||||
|
|
||||||
require.NoError(t, w.WritePeriod(delivery.ExportArchivedEvent{
|
assert.Equal(t,
|
||||||
Body: base64.StdEncoding.EncodeToString(random),
|
[]string{openedEventID}, exportedEventIDs(t, writeExport(t, export)),
|
||||||
}, 0, fmt.Sprintf("2026-10-01-%02d", hour)))
|
)
|
||||||
}
|
|
||||||
|
|
||||||
// The writer's own handle on the last file is not the export's.
|
|
||||||
w.Evict()
|
|
||||||
|
|
||||||
// Through a buffer, the open files are listed once per 8 KiB of
|
|
||||||
// output, a few times for each file, rather than at each of gzip's
|
|
||||||
// small writes, which takes far longer.
|
|
||||||
peak := &openFilesPeak{dir: dir}
|
|
||||||
buffered := bufio.NewWriterSize(peak, 8<<10)
|
|
||||||
|
|
||||||
require.NoError(t, writeExportTo(t, listExport(t, path), buffered))
|
|
||||||
require.NoError(t, buffered.Flush())
|
|
||||||
assert.Equal(t, 1, peak.max)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// heapPeak is an io.Writer that discards what it is given and records
|
// heapPeak is an io.Writer that discards what it is given and records
|
||||||
// the largest heap it saw at a write. It collects garbage twice before
|
// the largest heap it saw at a write. It collects garbage before each
|
||||||
// each reading, so the heap it reads is what is still held. Once is not
|
// reading, so the heap it reads is what is still held.
|
||||||
// enough: the libraries the export calls (regexp, under GORM's table
|
|
||||||
// names, and encoding/json among them) cache buffers in a sync.Pool,
|
|
||||||
// which keeps them through one collection, so after one the reading
|
|
||||||
// counts however many happen to be cached. That varies from run to run
|
|
||||||
// by about as much as the limit in TestArchiveExport_Streams.
|
|
||||||
type heapPeak struct {
|
type heapPeak struct {
|
||||||
max uint64
|
max uint64
|
||||||
}
|
}
|
||||||
@@ -474,7 +311,6 @@ type heapPeak struct {
|
|||||||
func (p *heapPeak) Write(b []byte) (int, error) {
|
func (p *heapPeak) Write(b []byte) (int, error) {
|
||||||
var m runtime.MemStats
|
var m runtime.MemStats
|
||||||
|
|
||||||
runtime.GC()
|
|
||||||
runtime.GC()
|
runtime.GC()
|
||||||
runtime.ReadMemStats(&m)
|
runtime.ReadMemStats(&m)
|
||||||
p.max = max(p.max, m.HeapAlloc)
|
p.max = max(p.max, m.HeapAlloc)
|
||||||
@@ -502,10 +338,13 @@ func exportHeapGrowth(t *testing.T, rows, bodySize int) uint64 {
|
|||||||
}, 0))
|
}, 0))
|
||||||
}
|
}
|
||||||
|
|
||||||
export := listExport(t, path)
|
export, err := delivery.OpenArchiveExport(
|
||||||
|
t.Context(), path, archiveTestLogger(),
|
||||||
|
)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
defer func() { require.NoError(t, export.Close()) }()
|
||||||
|
|
||||||
// Twice, for the reason heapPeak gives.
|
|
||||||
runtime.GC()
|
|
||||||
runtime.GC()
|
runtime.GC()
|
||||||
|
|
||||||
var start runtime.MemStats
|
var start runtime.MemStats
|
||||||
|
|||||||
@@ -1,198 +0,0 @@
|
|||||||
package delivery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The archive rotations: how often a database target starts a new
|
|
||||||
// archive file. Every rotation but none puts the period of an event's
|
|
||||||
// receive time, in UTC, in the name of the file the event goes to,
|
|
||||||
// written in the layout beside it.
|
|
||||||
const (
|
|
||||||
archiveRotationNone = "none"
|
|
||||||
archiveRotationMonthly = "monthly"
|
|
||||||
archiveRotationDaily = "daily"
|
|
||||||
archiveRotationHourly = "hourly"
|
|
||||||
|
|
||||||
archiveMonthLayout = "2006-01"
|
|
||||||
archiveDayLayout = "2006-01-02"
|
|
||||||
archiveHourLayout = "2006-01-02-15"
|
|
||||||
)
|
|
||||||
|
|
||||||
// errArchiveRotationUnknown is returned for a rotation that is not one
|
|
||||||
// of the four.
|
|
||||||
var errArchiveRotationUnknown = errors.New(
|
|
||||||
"rotation must be none, monthly, daily or hourly",
|
|
||||||
)
|
|
||||||
|
|
||||||
// archiveFile is one of a database target's archive files, and the
|
|
||||||
// period in its name: "" for the file named without one.
|
|
||||||
type archiveFile struct {
|
|
||||||
path string
|
|
||||||
period string
|
|
||||||
}
|
|
||||||
|
|
||||||
// ValidateArchiveRotation checks a user-supplied archive rotation for
|
|
||||||
// a database target: empty or none (both meaning one file), monthly,
|
|
||||||
// daily or hourly.
|
|
||||||
func ValidateArchiveRotation(rotation string) error {
|
|
||||||
switch rotation {
|
|
||||||
case "", archiveRotationNone, archiveRotationMonthly,
|
|
||||||
archiveRotationDaily, archiveRotationHourly:
|
|
||||||
return nil
|
|
||||||
default:
|
|
||||||
return fmt.Errorf("%w: %q", errArchiveRotationUnknown, rotation)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseArchiveRotation reads the rotation from a database target's
|
|
||||||
// config JSON. An empty config or an empty rotation is none.
|
|
||||||
func parseArchiveRotation(configJSON string) (string, error) {
|
|
||||||
if configJSON == "" {
|
|
||||||
return archiveRotationNone, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var cfg databaseTargetConfig
|
|
||||||
|
|
||||||
err := json.Unmarshal([]byte(configJSON), &cfg)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("parsing database target config: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = ValidateArchiveRotation(cfg.Rotation)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
if cfg.Rotation == "" {
|
|
||||||
return archiveRotationNone, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return cfg.Rotation, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// archivePeriod returns the period, in UTC, that a rotation puts an
|
|
||||||
// event received at receivedAt in: "2026-10" for monthly,
|
|
||||||
// "2026-10-01" for daily, "2026-10-01-19" for hourly, and "" for none.
|
|
||||||
func archivePeriod(rotation string, receivedAt time.Time) string {
|
|
||||||
switch rotation {
|
|
||||||
case archiveRotationMonthly:
|
|
||||||
return receivedAt.UTC().Format(archiveMonthLayout)
|
|
||||||
case archiveRotationDaily:
|
|
||||||
return receivedAt.UTC().Format(archiveDayLayout)
|
|
||||||
case archiveRotationHourly:
|
|
||||||
return receivedAt.UTC().Format(archiveHourLayout)
|
|
||||||
default:
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// archivePeriodPath returns the path of a database target's archive
|
|
||||||
// file for a period: path, as ArchivePath gives it, with "-" and the
|
|
||||||
// period put before its ".db". The period "" gives path itself.
|
|
||||||
func archivePeriodPath(path, period string) string {
|
|
||||||
if period == "" {
|
|
||||||
return path
|
|
||||||
}
|
|
||||||
|
|
||||||
return strings.TrimSuffix(path, ".db") + "-" + period + ".db"
|
|
||||||
}
|
|
||||||
|
|
||||||
// ArchivePathAt returns the archive file a database target writes an
|
|
||||||
// event received at receivedAt to: ArchivePath's file, with the
|
|
||||||
// period in its name when the target rotates.
|
|
||||||
func ArchivePathAt(
|
|
||||||
dbMgr *database.WebhookDBManager,
|
|
||||||
webhook *database.Webhook,
|
|
||||||
target *database.Target,
|
|
||||||
receivedAt time.Time,
|
|
||||||
) (string, error) {
|
|
||||||
rotation, err := parseArchiveRotation(target.Config)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
return archivePeriodPath(
|
|
||||||
ArchivePath(dbMgr, webhook, target),
|
|
||||||
archivePeriod(rotation, receivedAt),
|
|
||||||
), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// archiveFiles lists the database target's archive files that exist,
|
|
||||||
// given the path ArchivePath gives it: the file at path, then each
|
|
||||||
// file archivePeriodPath names for path and a period, oldest period
|
|
||||||
// first. Which rotation wrote a file does not matter, so the files of
|
|
||||||
// an earlier rotation setting are listed too.
|
|
||||||
func archiveFiles(path string) ([]archiveFile, error) {
|
|
||||||
dir := filepath.Dir(path)
|
|
||||||
|
|
||||||
entries, err := os.ReadDir(dir)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("listing archive files: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
stem := strings.TrimSuffix(filepath.Base(path), ".db")
|
|
||||||
|
|
||||||
var files []archiveFile
|
|
||||||
|
|
||||||
for _, entry := range entries {
|
|
||||||
period, ok := archiveFilePeriod(stem, entry.Name())
|
|
||||||
if ok {
|
|
||||||
files = append(files, archiveFile{
|
|
||||||
path: filepath.Join(dir, entry.Name()),
|
|
||||||
period: period,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A month sorts before the days and hours in it.
|
|
||||||
slices.SortFunc(files, func(a, b archiveFile) int {
|
|
||||||
return strings.Compare(a.period, b.period)
|
|
||||||
})
|
|
||||||
|
|
||||||
return files, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// archiveFilePeriod reports whether name is the name of an archive
|
|
||||||
// file of the target whose file name without a period is stem+".db",
|
|
||||||
// and the period in it.
|
|
||||||
func archiveFilePeriod(stem, name string) (string, bool) {
|
|
||||||
rest, ok := strings.CutPrefix(name, stem)
|
|
||||||
if !ok {
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
|
|
||||||
rest, ok = strings.CutSuffix(rest, ".db")
|
|
||||||
if !ok {
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
|
|
||||||
if rest == "" {
|
|
||||||
return "", true
|
|
||||||
}
|
|
||||||
|
|
||||||
period, ok := strings.CutPrefix(rest, "-")
|
|
||||||
if !ok {
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, layout := range []string{
|
|
||||||
archiveMonthLayout, archiveDayLayout, archiveHourLayout,
|
|
||||||
} {
|
|
||||||
_, err := time.Parse(layout, period)
|
|
||||||
if err == nil {
|
|
||||||
return period, true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
@@ -1,390 +0,0 @@
|
|||||||
package delivery_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database/databasetest"
|
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The archive rotations, and the configs of a daily target and of one
|
|
||||||
// whose rotation is not one of the four.
|
|
||||||
const (
|
|
||||||
rotationNone = "none"
|
|
||||||
rotationMonthly = "monthly"
|
|
||||||
rotationDaily = "daily"
|
|
||||||
rotationHourly = "hourly"
|
|
||||||
|
|
||||||
dailyConfig = `{"rotation":"daily"}`
|
|
||||||
weeklyConfig = `{"rotation":"weekly"}`
|
|
||||||
)
|
|
||||||
|
|
||||||
// The periods the tests archive into most: two days, and an hour of
|
|
||||||
// the first.
|
|
||||||
const (
|
|
||||||
dayPeriod = "2026-03-04"
|
|
||||||
nextDayPeriod = "2026-03-05"
|
|
||||||
hourPeriod = "2026-03-04-05"
|
|
||||||
)
|
|
||||||
|
|
||||||
// periodPath returns the archive file for a period of the target whose
|
|
||||||
// file without a period is path.
|
|
||||||
func periodPath(path, period string) string {
|
|
||||||
return strings.TrimSuffix(path, ".db") + "-" + period + ".db"
|
|
||||||
}
|
|
||||||
|
|
||||||
// deliverReceivedAt delivers to a database target an event whose
|
|
||||||
// receive time is receivedAt, and returns the event's id. The receive
|
|
||||||
// time is what decides a rotated archive's file, so setting it is how
|
|
||||||
// these tests move the clock across a period boundary.
|
|
||||||
func (env *archiveEnv) deliverReceivedAt(
|
|
||||||
t *testing.T, tgt *database.Target, receivedAt time.Time,
|
|
||||||
) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
webhookDB := testWebhookDB(t)
|
|
||||||
event := seedEvent(t, webhookDB, `{"n":1}`)
|
|
||||||
event.CreatedAt = receivedAt
|
|
||||||
|
|
||||||
env.eng.ExportDeliverDatabase(
|
|
||||||
webhookDB, seedDatabaseTargetDelivery(t, webhookDB, event, tgt),
|
|
||||||
)
|
|
||||||
|
|
||||||
return event.ID
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestDeliverDatabase_RotatesAtEachPeriodBoundary delivers, for each
|
|
||||||
// rotation, an event received in the last second of a period and one
|
|
||||||
// received in the first second of the next, and checks each lands in
|
|
||||||
// the file named for its own period, in UTC. Rotation none keeps both
|
|
||||||
// in the one file.
|
|
||||||
func TestDeliverDatabase_RotatesAtEachPeriodBoundary(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
berlin := time.FixedZone("CEST", 2*60*60)
|
|
||||||
|
|
||||||
cases := []struct {
|
|
||||||
name string
|
|
||||||
rotation string
|
|
||||||
before, after time.Time
|
|
||||||
// periods are the periods of before and after.
|
|
||||||
periods [2]string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
rotationMonthly, rotationMonthly,
|
|
||||||
time.Date(2026, 1, 31, 23, 59, 59, 0, time.UTC),
|
|
||||||
time.Date(2026, 2, 1, 0, 0, 0, 0, time.UTC),
|
|
||||||
[2]string{"2026-01", "2026-02"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
rotationDaily, rotationDaily,
|
|
||||||
time.Date(2026, 3, 4, 23, 59, 59, 0, time.UTC),
|
|
||||||
time.Date(2026, 3, 5, 0, 0, 0, 0, time.UTC),
|
|
||||||
[2]string{dayPeriod, nextDayPeriod},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
// The same instants, received in a zone two hours ahead
|
|
||||||
// of UTC, where they fall on 5 March: the period is UTC's.
|
|
||||||
"daily in another zone", rotationDaily,
|
|
||||||
time.Date(2026, 3, 5, 1, 59, 59, 0, berlin),
|
|
||||||
time.Date(2026, 3, 5, 2, 0, 0, 0, berlin),
|
|
||||||
[2]string{dayPeriod, nextDayPeriod},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
rotationHourly, rotationHourly,
|
|
||||||
time.Date(2026, 3, 4, 5, 59, 59, 0, time.UTC),
|
|
||||||
time.Date(2026, 3, 4, 6, 0, 0, 0, time.UTC),
|
|
||||||
[2]string{hourPeriod, "2026-03-04-06"},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range cases {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
|
||||||
tgt := env.seedDatabaseTarget(t, `{"rotation":"`+tc.rotation+`"}`)
|
|
||||||
path := env.archivePath(tgt)
|
|
||||||
|
|
||||||
first := env.deliverReceivedAt(t, tgt, tc.before)
|
|
||||||
second := env.deliverReceivedAt(t, tgt, tc.after)
|
|
||||||
|
|
||||||
assert.Equal(t, []string{first},
|
|
||||||
archivedEventIDs(t, periodPath(path, tc.periods[0])))
|
|
||||||
assert.Equal(t, []string{second},
|
|
||||||
archivedEventIDs(t, periodPath(path, tc.periods[1])))
|
|
||||||
assert.NoFileExists(t, path,
|
|
||||||
"a rotated target never writes the file without a period")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Run(rotationNone, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
|
||||||
tgt := env.seedDatabaseTarget(t, `{"rotation":"`+rotationNone+`"}`)
|
|
||||||
|
|
||||||
first := env.deliverReceivedAt(t, tgt, cases[0].before)
|
|
||||||
second := env.deliverReceivedAt(t, tgt, cases[0].after)
|
|
||||||
|
|
||||||
assert.ElementsMatch(t, []string{first, second},
|
|
||||||
archivedEventIDs(t, env.archivePath(tgt)))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestDeliverDatabase_EventWithoutReceiveTime proves an event whose
|
|
||||||
// receive time is not known goes to the file for the time it is
|
|
||||||
// archived, rather than to one for the year 1.
|
|
||||||
func TestDeliverDatabase_EventWithoutReceiveTime(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
|
||||||
tgt := env.seedDatabaseTarget(t, `{"rotation":"`+rotationMonthly+`"}`)
|
|
||||||
path := env.archivePath(tgt)
|
|
||||||
|
|
||||||
before := time.Now().UTC().Format("2006-01")
|
|
||||||
id := env.deliverReceivedAt(t, tgt, time.Time{})
|
|
||||||
|
|
||||||
file := periodPath(path, time.Now().UTC().Format("2006-01"))
|
|
||||||
|
|
||||||
_, err := os.Stat(file)
|
|
||||||
if err != nil {
|
|
||||||
// The month turned during the delivery.
|
|
||||||
file = periodPath(path, before)
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equal(t, []string{id}, archivedEventIDs(t, file))
|
|
||||||
assert.NoFileExists(t, periodPath(path, "0001-01"))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestDeliverDatabase_RotationChangeKeepsOldFiles changes a target's
|
|
||||||
// rotation from none to daily between two events, and checks the
|
|
||||||
// second goes to the daily file while the first stays where it was.
|
|
||||||
func TestDeliverDatabase_RotationChangeKeepsOldFiles(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
|
||||||
tgt := env.seedDatabaseTarget(t, "")
|
|
||||||
path := env.archivePath(tgt)
|
|
||||||
at := time.Date(2026, 3, 4, 12, 0, 0, 0, time.UTC)
|
|
||||||
|
|
||||||
first := env.deliverReceivedAt(t, tgt, at)
|
|
||||||
|
|
||||||
tgt.Config = dailyConfig
|
|
||||||
second := env.deliverReceivedAt(t, tgt, at)
|
|
||||||
|
|
||||||
assert.Equal(t, []string{first}, archivedEventIDs(t, path))
|
|
||||||
assert.Equal(t, []string{second},
|
|
||||||
archivedEventIDs(t, periodPath(path, dayPeriod)))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveSweep_PrunesEveryFile gives a daily target three files:
|
|
||||||
// the file without a period, left from before it rotated, and two
|
|
||||||
// daily files. Each holds a row older than the expiry, and one daily
|
|
||||||
// file also a newer row. The sweep prunes the old row from every file,
|
|
||||||
// deletes the daily file it leaves empty, and keeps the file without a
|
|
||||||
// period although it is empty too.
|
|
||||||
func TestArchiveSweep_PrunesEveryFile(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
|
||||||
tgt := env.seedDatabaseTarget(
|
|
||||||
t, `{"expiry":"1h","rotation":"`+rotationDaily+`"}`,
|
|
||||||
)
|
|
||||||
path := env.archivePath(tgt)
|
|
||||||
emptied := periodPath(path, dayPeriod)
|
|
||||||
kept := periodPath(path, nextDayPeriod)
|
|
||||||
|
|
||||||
now := time.Now()
|
|
||||||
old := now.Add(-48 * time.Hour)
|
|
||||||
|
|
||||||
seedArchiveFile(t, path, tgt.WebhookID, old)
|
|
||||||
seedArchiveFile(t, emptied, tgt.WebhookID, old)
|
|
||||||
seedArchiveFile(t, kept, tgt.WebhookID, old, now.Add(-time.Minute))
|
|
||||||
|
|
||||||
env.sweeper.ExportSweep(t.Context())
|
|
||||||
|
|
||||||
assert.Empty(t, archivedEventIDs(t, path))
|
|
||||||
assert.Equal(t, []string{sweepRowNew}, archivedEventIDs(t, kept))
|
|
||||||
|
|
||||||
for _, suffix := range archiveFileSuffixes() {
|
|
||||||
assert.NoFileExists(t, emptied+suffix)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRename_MovesEveryFile renames a daily target that also has a
|
|
||||||
// file without a period, and checks every file moves to the new name
|
|
||||||
// with its period, rows and all, and that a later write uses the new
|
|
||||||
// name.
|
|
||||||
func TestRename_MovesEveryFile(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
|
||||||
tgt := env.seedDatabaseTarget(t, "")
|
|
||||||
oldPath := env.archivePath(tgt)
|
|
||||||
day := time.Date(2026, 3, 4, 12, 0, 0, 0, time.UTC)
|
|
||||||
|
|
||||||
unrotated := env.deliverReceivedAt(t, tgt, day)
|
|
||||||
|
|
||||||
tgt.Config = dailyConfig
|
|
||||||
first := env.deliverReceivedAt(t, tgt, day)
|
|
||||||
second := env.deliverReceivedAt(t, tgt, day.Add(24*time.Hour))
|
|
||||||
|
|
||||||
require.NoError(t, env.eng.Rename(tgt.ID, "Orders", "Long Term"))
|
|
||||||
|
|
||||||
newPath := filepath.Join(
|
|
||||||
env.dataDir, "archive-orders-long-term-"+tgt.ID+".db",
|
|
||||||
)
|
|
||||||
|
|
||||||
for _, old := range []string{
|
|
||||||
oldPath,
|
|
||||||
periodPath(oldPath, dayPeriod),
|
|
||||||
periodPath(oldPath, nextDayPeriod),
|
|
||||||
} {
|
|
||||||
assert.NoFileExists(t, old)
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equal(t, []string{unrotated}, archivedEventIDs(t, newPath))
|
|
||||||
assert.Equal(t, []string{first},
|
|
||||||
archivedEventIDs(t, periodPath(newPath, dayPeriod)))
|
|
||||||
assert.Equal(t, []string{second},
|
|
||||||
archivedEventIDs(t, periodPath(newPath, nextDayPeriod)))
|
|
||||||
|
|
||||||
third := env.deliverReceivedAt(t, tgt, day.Add(48*time.Hour))
|
|
||||||
assert.Equal(t, []string{third},
|
|
||||||
archivedEventIDs(t, periodPath(newPath, "2026-03-06")))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRename_NeverReplacesARotatedFile plants a file at the new name
|
|
||||||
// of a target's daily file, and proves the rename is refused and moves
|
|
||||||
// none of the target's files.
|
|
||||||
func TestRename_NeverReplacesARotatedFile(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
|
||||||
tgt := env.seedDatabaseTarget(t, dailyConfig)
|
|
||||||
oldPath := env.archivePath(tgt)
|
|
||||||
day := time.Date(2026, 3, 4, 12, 0, 0, 0, time.UTC)
|
|
||||||
|
|
||||||
env.deliverReceivedAt(t, tgt, day)
|
|
||||||
env.deliverReceivedAt(t, tgt, day.Add(24*time.Hour))
|
|
||||||
|
|
||||||
newPath := filepath.Join(
|
|
||||||
env.dataDir, "archive-orders-long-term-"+tgt.ID+".db",
|
|
||||||
)
|
|
||||||
planted := periodPath(newPath, nextDayPeriod)
|
|
||||||
require.NoError(t, os.WriteFile(planted, []byte("planted"), 0o600))
|
|
||||||
|
|
||||||
require.ErrorIs(
|
|
||||||
t, env.eng.Rename(tgt.ID, "Orders", "Long Term"),
|
|
||||||
delivery.ErrArchiveNameTaken,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.FileExists(t, periodPath(oldPath, dayPeriod))
|
|
||||||
assert.FileExists(t, periodPath(oldPath, nextDayPeriod))
|
|
||||||
assert.NoFileExists(t, periodPath(newPath, dayPeriod))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestStatArchive_EveryFile proves StatArchive counts and adds up every
|
|
||||||
// one of a target's files, takes the latest write of any of them, and
|
|
||||||
// leaves out files whose names only look like the target's.
|
|
||||||
func TestStatArchive_EveryFile(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
path := filepath.Join(dir, "archive-wh.db")
|
|
||||||
files := []string{
|
|
||||||
path, periodPath(path, "2026-03"), periodPath(path, hourPeriod),
|
|
||||||
}
|
|
||||||
written := time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC)
|
|
||||||
|
|
||||||
var size int64
|
|
||||||
|
|
||||||
for i, file := range files {
|
|
||||||
require.NoError(t, os.WriteFile(file, make([]byte, 100*(i+1)), 0o600))
|
|
||||||
|
|
||||||
size += int64(100 * (i + 1))
|
|
||||||
at := written.Add(-time.Duration(i) * time.Hour)
|
|
||||||
require.NoError(t, os.Chtimes(file, at, at))
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, other := range []string{
|
|
||||||
"archive-wh-2026-13.db", "archive-wh-2026-3.db",
|
|
||||||
"archive-wh-other.db", "archive-wh-2026-03.json",
|
|
||||||
"archive-whx.db",
|
|
||||||
} {
|
|
||||||
require.NoError(t,
|
|
||||||
os.WriteFile(filepath.Join(dir, other), []byte("x"), 0o600))
|
|
||||||
}
|
|
||||||
|
|
||||||
got, err := delivery.StatArchive(path)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, len(files), got.Files)
|
|
||||||
assert.Equal(t, size, got.Size)
|
|
||||||
assert.True(t, written.Equal(got.Written), got.Written)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchivePathAt names the file each rotation writes an event to.
|
|
||||||
func TestArchivePathAt(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dataDir := t.TempDir()
|
|
||||||
dbMgr := databasetest.NewWebhookDBManager(t, dataDir)
|
|
||||||
webhook := &database.Webhook{
|
|
||||||
BaseModel: database.BaseModel{ID: "wh-id"}, Name: "Orders",
|
|
||||||
}
|
|
||||||
at := time.Date(2026, 10, 1, 19, 30, 0, 0, time.UTC)
|
|
||||||
|
|
||||||
cases := map[string]string{
|
|
||||||
"": "",
|
|
||||||
`{"rotation":"` + rotationNone + `"}`: "",
|
|
||||||
`{"rotation":"` + rotationMonthly + `"}`: "-2026-10",
|
|
||||||
dailyConfig: "-2026-10-01",
|
|
||||||
`{"rotation":"` + rotationHourly + `"}`: "-2026-10-01-19",
|
|
||||||
}
|
|
||||||
|
|
||||||
for config, period := range cases {
|
|
||||||
target := &database.Target{
|
|
||||||
BaseModel: database.BaseModel{ID: "tgt-id"},
|
|
||||||
Name: "Archive",
|
|
||||||
Config: config,
|
|
||||||
}
|
|
||||||
|
|
||||||
got, err := delivery.ArchivePathAt(dbMgr, webhook, target, at)
|
|
||||||
require.NoError(t, err, config)
|
|
||||||
assert.Equal(t,
|
|
||||||
filepath.Join(
|
|
||||||
dataDir, "archive-orders-archive-tgt-id"+period+".db",
|
|
||||||
),
|
|
||||||
got, config,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err := delivery.ArchivePathAt(dbMgr, webhook, &database.Target{
|
|
||||||
Config: weeklyConfig,
|
|
||||||
}, at)
|
|
||||||
require.Error(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestValidateArchiveRotation accepts the four rotations, and empty,
|
|
||||||
// and refuses anything else.
|
|
||||||
func TestValidateArchiveRotation(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, ok := range []string{
|
|
||||||
"", rotationNone, rotationMonthly, rotationDaily, rotationHourly,
|
|
||||||
} {
|
|
||||||
require.NoError(t, delivery.ValidateArchiveRotation(ok), ok)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, bad := range []string{"weekly", "Daily", "hourly "} {
|
|
||||||
require.Error(t, delivery.ValidateArchiveRotation(bad), bad)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -85,7 +85,6 @@ func TestDeliverDatabase_ArchivesEvent(t *testing.T) {
|
|||||||
|
|
||||||
webhookDB := testWebhookDB(t)
|
webhookDB := testWebhookDB(t)
|
||||||
event := seedEvent(t, webhookDB, `{"archived":true}`)
|
event := seedEvent(t, webhookDB, `{"archived":true}`)
|
||||||
event.RawQuery = eventQuery
|
|
||||||
d := seedDatabaseTargetDelivery(t, webhookDB, event, tgt)
|
d := seedDatabaseTargetDelivery(t, webhookDB, event, tgt)
|
||||||
|
|
||||||
env.eng.ExportDeliverDatabase(webhookDB, d)
|
env.eng.ExportDeliverDatabase(webhookDB, d)
|
||||||
@@ -114,7 +113,6 @@ func TestDeliverDatabase_ArchivesEvent(t *testing.T) {
|
|||||||
assert.Equal(t, event.ID, rows[0].EventID)
|
assert.Equal(t, event.ID, rows[0].EventID)
|
||||||
assert.Equal(t, event.WebhookID, rows[0].WebhookID)
|
assert.Equal(t, event.WebhookID, rows[0].WebhookID)
|
||||||
assert.Equal(t, event.Method, rows[0].Method)
|
assert.Equal(t, event.Method, rows[0].Method)
|
||||||
assert.Equal(t, eventQuery, rows[0].RawQuery)
|
|
||||||
assert.JSONEq(t, `{"archived":true}`, rows[0].Body)
|
assert.JSONEq(t, `{"archived":true}`, rows[0].Body)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -220,7 +218,6 @@ func TestStatArchive(t *testing.T) {
|
|||||||
|
|
||||||
got, err := delivery.StatArchive(path)
|
got, err := delivery.StatArchive(path)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, 1, got.Files)
|
|
||||||
assert.Equal(t, file.Size()+wal.Size(), got.Size)
|
assert.Equal(t, file.Size()+wal.Size(), got.Size)
|
||||||
assert.True(t, written.Equal(got.Written), got.Written)
|
assert.True(t, written.Equal(got.Written), got.Written)
|
||||||
|
|
||||||
@@ -723,41 +720,3 @@ func TestArchiveWriter_RenameMovesBackOnFailure(t *testing.T) {
|
|||||||
assert.NoFileExists(t, filepath.Join(dir, newName))
|
assert.NoFileExists(t, filepath.Join(dir, newName))
|
||||||
assert.Equal(t, oldPath, w.Path())
|
assert.Equal(t, oldPath, w.Path())
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestArchiveWriter_RenameMovesBackEveryFile renames a target with a
|
|
||||||
// file without a period and a file for a month, and proves that when
|
|
||||||
// the month's file fails to move, the file already moved is moved back:
|
|
||||||
// both files are under the old name with their rows, and nothing is
|
|
||||||
// under the new name. The new name is 251 bytes, so the file without a
|
|
||||||
// period, with its -wal and -shm, can take it, but the month's file,
|
|
||||||
// eight bytes longer, cannot.
|
|
||||||
func TestArchiveWriter_RenameMovesBackEveryFile(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
oldPath := filepath.Join(dir, "archive-old.db")
|
|
||||||
monthPath := filepath.Join(dir, "archive-old-2026-03.db")
|
|
||||||
w := delivery.NewExportArchiveWriter(oldPath, archiveTestLogger(), 0)
|
|
||||||
|
|
||||||
require.NoError(t, w.WritePeriod(
|
|
||||||
delivery.ExportArchivedEvent{EventID: "in-none"}, 0, "",
|
|
||||||
))
|
|
||||||
require.NoError(t, w.WritePeriod(
|
|
||||||
delivery.ExportArchivedEvent{EventID: "in-month"}, 0, "2026-03",
|
|
||||||
))
|
|
||||||
|
|
||||||
require.Error(t, w.Rename(strings.Repeat("a", 248)+".db"))
|
|
||||||
|
|
||||||
assert.Equal(t, []string{"in-none"}, archivedEventIDs(t, oldPath))
|
|
||||||
assert.Equal(t, []string{"in-month"}, archivedEventIDs(t, monthPath))
|
|
||||||
|
|
||||||
entries, err := os.ReadDir(dir)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
for _, entry := range entries {
|
|
||||||
assert.True(t, strings.HasPrefix(entry.Name(), "archive-old"),
|
|
||||||
"%s is not under the old name", entry.Name())
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equal(t, oldPath, w.Path())
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -82,8 +82,8 @@ func isReservedTargetHeader(name string) bool {
|
|||||||
// one pair silently overwriting the other.
|
// one pair silently overwriting the other.
|
||||||
//
|
//
|
||||||
// An input with no pairs yields an empty map, which omitempty drops
|
// An input with no pairs yields an empty map, which omitempty drops
|
||||||
// from the stored config, so a target configured with no headers
|
// from the stored config: a target configured with no headers keeps
|
||||||
// stores no headers key at all.
|
// the same config JSON it had before this field existed.
|
||||||
func ParseTargetHeaders(raw string) (map[string]string, error) {
|
func ParseTargetHeaders(raw string) (map[string]string, error) {
|
||||||
headers := make(map[string]string)
|
headers := make(map[string]string)
|
||||||
|
|
||||||
|
|||||||
@@ -204,11 +204,10 @@ func TestNewTargetConfigForm(t *testing.T) {
|
|||||||
|
|
||||||
form, err = delivery.NewTargetConfigForm(&database.Target{
|
form, err = delivery.NewTargetConfigForm(&database.Target{
|
||||||
Type: database.TargetTypeDatabase,
|
Type: database.TargetTypeDatabase,
|
||||||
Config: `{"expiry":"720h","rotation":"daily"}`,
|
Config: `{"expiry":"720h"}`,
|
||||||
})
|
})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, "720h", form.Expiry)
|
assert.Equal(t, "720h", form.Expiry)
|
||||||
assert.Equal(t, rotationDaily, form.Rotation)
|
|
||||||
|
|
||||||
form, err = delivery.NewTargetConfigForm(&database.Target{
|
form, err = delivery.NewTargetConfigForm(&database.Target{
|
||||||
Type: database.TargetTypeLog,
|
Type: database.TargetTypeLog,
|
||||||
@@ -249,10 +248,6 @@ func TestNewTargetConfigForm_UnreadableConfigErrors(t *testing.T) {
|
|||||||
Type: database.TargetTypeDatabase,
|
Type: database.TargetTypeDatabase,
|
||||||
Config: `{"expiry":"soon"}`,
|
Config: `{"expiry":"soon"}`,
|
||||||
},
|
},
|
||||||
{
|
|
||||||
Type: database.TargetTypeDatabase,
|
|
||||||
Config: weeklyConfig,
|
|
||||||
},
|
|
||||||
{Type: database.TargetType("nope")},
|
{Type: database.TargetType("nope")},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -8,7 +8,6 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
|
||||||
"sort"
|
"sort"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
@@ -33,11 +32,6 @@ type HTTPTargetConfig struct {
|
|||||||
URL string `json:"url"`
|
URL string `json:"url"`
|
||||||
Headers map[string]string `json:"headers,omitempty"`
|
Headers map[string]string `json:"headers,omitempty"`
|
||||||
Timeout int `json:"timeout,omitempty"`
|
Timeout int `json:"timeout,omitempty"`
|
||||||
|
|
||||||
// ForwardQuery passes each event's query string on to the target,
|
|
||||||
// appended to URL. Off, the target URL is sent exactly as
|
|
||||||
// configured.
|
|
||||||
ForwardQuery bool `json:"forwardQuery,omitempty"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// httpCore holds the retry, backoff, and circuit-breaker
|
// httpCore holds the retry, backoff, and circuit-breaker
|
||||||
@@ -450,10 +444,6 @@ func (t *httpTarget) doHTTPRequest(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
if cfg.ForwardQuery {
|
|
||||||
appendQuery(req.URL, event.RawQuery)
|
|
||||||
}
|
|
||||||
|
|
||||||
originScoped := applyRequestHeaders(
|
originScoped := applyRequestHeaders(
|
||||||
req, event, cfg, t.eng.userAgent(),
|
req, event, cfg, t.eng.userAgent(),
|
||||||
)
|
)
|
||||||
@@ -484,19 +474,6 @@ func (t *httpTarget) doHTTPRequest(
|
|||||||
return resp.StatusCode, string(body), dur, nil
|
return resp.StatusCode, string(body), dur, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// appendQuery adds an event's query string to a delivery's URL, joined
|
|
||||||
// with "&" to any query string the target URL already has.
|
|
||||||
func appendQuery(u *url.URL, rawQuery string) {
|
|
||||||
switch {
|
|
||||||
case rawQuery == "":
|
|
||||||
return
|
|
||||||
case u.RawQuery == "":
|
|
||||||
u.RawQuery = rawQuery
|
|
||||||
default:
|
|
||||||
u.RawQuery += "&" + rawQuery
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// clientForRequest returns the client for one delivery attempt.
|
// clientForRequest returns the client for one delivery attempt.
|
||||||
// originScoped is the header set applyRequestHeaders built for that
|
// originScoped is the header set applyRequestHeaders built for that
|
||||||
// attempt; a request with neither a per-target timeout nor an
|
// attempt; a request with neither a per-target timeout nor an
|
||||||
|
|||||||
@@ -1,172 +0,0 @@
|
|||||||
package delivery_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/google/uuid"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
|
||||||
)
|
|
||||||
|
|
||||||
// eventQuery is the query string the events in these tests arrived
|
|
||||||
// with.
|
|
||||||
const eventQuery = "a=1&b=2"
|
|
||||||
|
|
||||||
// httpTargetConfig is the stored configuration of an HTTP target at
|
|
||||||
// targetURL.
|
|
||||||
func httpTargetConfig(
|
|
||||||
t *testing.T, targetURL string, forwardQuery bool,
|
|
||||||
) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
cfg, err := json.Marshal(delivery.HTTPTargetConfig{
|
|
||||||
URL: targetURL, ForwardQuery: forwardQuery,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
return string(cfg)
|
|
||||||
}
|
|
||||||
|
|
||||||
// deliverWithQuery sends one event that arrived with eventQuery to an
|
|
||||||
// HTTP target configured with cfg, through the path a received event's
|
|
||||||
// delivery takes, and returns the attempt it recorded.
|
|
||||||
func deliverWithQuery(t *testing.T, cfg string) database.DeliveryResult {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
s := newISetup(t)
|
|
||||||
|
|
||||||
event := iSeedEvent(t, s.WebhookDB, s.WebhookID, "{}")
|
|
||||||
d := iSeedDelivery(
|
|
||||||
t, s.WebhookDB, event.ID, uuid.NewString(),
|
|
||||||
database.DeliveryStatusPending,
|
|
||||||
)
|
|
||||||
task := iTask(
|
|
||||||
d, event, s.WebhookID, d.TargetID, "query", cfg, 0, 1, &event.Body,
|
|
||||||
)
|
|
||||||
task.RawQuery = eventQuery
|
|
||||||
|
|
||||||
s.Engine.ExportProcessNewTask(context.TODO(), &task)
|
|
||||||
|
|
||||||
var result database.DeliveryResult
|
|
||||||
|
|
||||||
require.NoError(t, s.WebhookDB.Where(
|
|
||||||
"delivery_id = ?", d.ID,
|
|
||||||
).First(&result).Error)
|
|
||||||
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestDeliverHTTP_ForwardQuery proves the URL a delivery is sent to:
|
|
||||||
// with the target's setting off, the target URL exactly as configured;
|
|
||||||
// with it on, the event's query string appended, joined with "&" to a
|
|
||||||
// query string the target URL already has.
|
|
||||||
func TestDeliverHTTP_ForwardQuery(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// The target URL's path, without and with a query string of its
|
|
||||||
// own.
|
|
||||||
const (
|
|
||||||
plain = "/in"
|
|
||||||
withQuery = "/in?key=k"
|
|
||||||
)
|
|
||||||
|
|
||||||
tests := map[string]struct {
|
|
||||||
path string
|
|
||||||
forwardQuery bool
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
"off": {
|
|
||||||
path: plain, want: plain,
|
|
||||||
},
|
|
||||||
"off, the target URL has a query string": {
|
|
||||||
path: withQuery, want: withQuery,
|
|
||||||
},
|
|
||||||
"on": {
|
|
||||||
path: plain, forwardQuery: true, want: plain + "?" + eventQuery,
|
|
||||||
},
|
|
||||||
"on, the target URL has a query string": {
|
|
||||||
path: withQuery, forwardQuery: true,
|
|
||||||
want: withQuery + "&" + eventQuery,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for name, tc := range tests {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
received := make(chan string, 1)
|
|
||||||
|
|
||||||
ts := httptest.NewServer(http.HandlerFunc(
|
|
||||||
func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
received <- r.RequestURI
|
|
||||||
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
},
|
|
||||||
))
|
|
||||||
t.Cleanup(ts.Close)
|
|
||||||
|
|
||||||
result := deliverWithQuery(t, httpTargetConfig(
|
|
||||||
t, ts.URL+tc.path, tc.forwardQuery,
|
|
||||||
))
|
|
||||||
|
|
||||||
assert.True(t, result.Success)
|
|
||||||
require.Len(t, received, 1)
|
|
||||||
assert.Equal(t, tc.want, <-received)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestDeliverHTTP_ForwardedQueryKeepsTheTargetURLMasked proves the
|
|
||||||
// credential in a target URL's own query string stays masked once the
|
|
||||||
// event's query string is appended to it: in a response or error that
|
|
||||||
// echoes the URL the target was sent, as the event log's Redactor shows
|
|
||||||
// it, and in the error a failed connection stores.
|
|
||||||
func TestDeliverHTTP_ForwardedQueryKeepsTheTargetURLMasked(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const secret = "s3cr3t"
|
|
||||||
|
|
||||||
received := make(chan string, 1)
|
|
||||||
|
|
||||||
ts := httptest.NewServer(http.HandlerFunc(
|
|
||||||
func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
received <- r.RequestURI
|
|
||||||
|
|
||||||
w.WriteHeader(http.StatusBadRequest)
|
|
||||||
},
|
|
||||||
))
|
|
||||||
t.Cleanup(ts.Close)
|
|
||||||
|
|
||||||
target := &database.Target{
|
|
||||||
Type: database.TargetTypeHTTP,
|
|
||||||
Config: httpTargetConfig(t, ts.URL+"/in?token="+secret, true),
|
|
||||||
}
|
|
||||||
|
|
||||||
deliverWithQuery(t, target.Config)
|
|
||||||
require.Len(t, received, 1)
|
|
||||||
|
|
||||||
sent := <-received
|
|
||||||
require.Equal(t, "/in?token="+secret+"&"+eventQuery, sent)
|
|
||||||
|
|
||||||
redactor := delivery.NewRedactor(target)
|
|
||||||
|
|
||||||
for _, echoed := range []string{sent, ts.URL + sent} {
|
|
||||||
shown := redactor.Redact("rejected " + echoed)
|
|
||||||
assert.NotContains(t, shown, secret, echoed)
|
|
||||||
assert.Contains(t, shown, delivery.RedactionMarker, echoed)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Nothing listens on port 1.
|
|
||||||
failed := deliverWithQuery(t, httpTargetConfig(
|
|
||||||
t, "http://127.0.0.1:1/in?token="+secret, true,
|
|
||||||
))
|
|
||||||
require.NotEmpty(t, failed.Error)
|
|
||||||
assert.NotContains(t, failed.Error, secret)
|
|
||||||
assert.NotContains(t, failed.Error, eventQuery)
|
|
||||||
}
|
|
||||||
@@ -9,9 +9,9 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// logTarget is a fire-and-forget target that logs the entire
|
// logTarget is a fire-and-forget target that logs the entire
|
||||||
// inbound webhook — the full request body, query string and
|
// inbound webhook — the full request body and headers, plus
|
||||||
// headers, plus the method, content type, and the webhook and
|
// the method, content type, and the webhook and entrypoint
|
||||||
// entrypoint ids — then records a single successful attempt.
|
// ids — then records a single successful attempt.
|
||||||
//
|
//
|
||||||
// This is the one log call in the service that deliberately writes
|
// This is the one log call in the service that deliberately writes
|
||||||
// unbounded client-chosen bytes, so it is the one exception to the
|
// unbounded client-chosen bytes, so it is the one exception to the
|
||||||
@@ -46,7 +46,6 @@ func (t *logTarget) Deliver(
|
|||||||
"webhook_id", d.Event.WebhookID,
|
"webhook_id", d.Event.WebhookID,
|
||||||
"entrypoint_id", d.Event.EntrypointID,
|
"entrypoint_id", d.Event.EntrypointID,
|
||||||
"method", d.Event.Method,
|
"method", d.Event.Method,
|
||||||
"raw_query", d.Event.RawQuery,
|
|
||||||
"content_type", d.Event.ContentType,
|
"content_type", d.Event.ContentType,
|
||||||
"headers", d.Event.Headers,
|
"headers", d.Event.Headers,
|
||||||
"body", d.Event.Body,
|
"body", d.Event.Body,
|
||||||
|
|||||||
@@ -13,9 +13,9 @@ import (
|
|||||||
const RedactionMarker = "(redacted)"
|
const RedactionMarker = "(redacted)"
|
||||||
|
|
||||||
// Redactor removes one target's own credential material from
|
// Redactor removes one target's own credential material from
|
||||||
// a delivery response body, whose text the target's remote
|
// text that target's remote peer chose: a delivery response
|
||||||
// peer chose, and from a stored delivery error as a second
|
// body, or a delivery error stored before the delivery path
|
||||||
// line behind the delivery path's own URL masking.
|
// learned to mask the URLs it embeds.
|
||||||
//
|
//
|
||||||
// It removes byte-identical echoes of strings taken from the
|
// It removes byte-identical echoes of strings taken from the
|
||||||
// target's stored configuration, and nothing else. Anything
|
// target's stored configuration, and nothing else. Anything
|
||||||
@@ -162,22 +162,18 @@ func targetSecrets(t *database.Target) []string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// urlSecrets returns the substrings of a destination URL that
|
// urlSecrets returns the substrings of a destination URL that
|
||||||
// must not survive into a rendered page: the whole URL; its
|
// must not survive into a rendered page: the whole URL, the
|
||||||
// path, unless that is empty or "/"; its query string, and the
|
// parts of it MaskURL elides, and any userinfo.
|
||||||
// request URI that carries it, which a remote echoing the
|
|
||||||
// request line shows even when the URL has no path; and its
|
|
||||||
// userinfo and password.
|
|
||||||
//
|
//
|
||||||
// No length floor is applied to the path, the query string or
|
// No length floor is applied to the path, and none to the
|
||||||
// the userinfo. A short path or a four-byte username is
|
// userinfo. A short path or a four-byte username is treated as
|
||||||
// treated as a credential exactly like a long one, because the
|
// a credential exactly like a long one, because the field takes
|
||||||
// field takes an arbitrary URL and no part of it can be
|
// an arbitrary URL and no part of it can be assumed non-secret —
|
||||||
// assumed non-secret — the same rule MaskURL applies.
|
// the same rule MaskURL applies. headerSecrets does carry a
|
||||||
// headerSecrets does carry a floor, and the difference is
|
// floor, and the difference is deliberate: a header is picked
|
||||||
// deliberate: a header is picked out by a name-shaped guess
|
// out by a name-shaped guess and its value may be ordinary
|
||||||
// and its value may be ordinary text, whereas a URL's path,
|
// text, whereas a URL's path and userinfo are credential
|
||||||
// query string and userinfo are credential material by
|
// material by position.
|
||||||
// position.
|
|
||||||
func urlSecrets(raw string) []string {
|
func urlSecrets(raw string) []string {
|
||||||
raw = strings.TrimSpace(raw)
|
raw = strings.TrimSpace(raw)
|
||||||
if raw == "" {
|
if raw == "" {
|
||||||
@@ -192,11 +188,12 @@ func urlSecrets(raw string) []string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if parsed.Path != "" && parsed.Path != "/" {
|
if parsed.Path != "" && parsed.Path != "/" {
|
||||||
secrets = append(secrets, parsed.EscapedPath())
|
requestURI := parsed.RequestURI()
|
||||||
}
|
secrets = append(secrets, requestURI)
|
||||||
|
|
||||||
if parsed.RawQuery != "" {
|
if escaped := parsed.EscapedPath(); escaped != requestURI {
|
||||||
secrets = append(secrets, parsed.RequestURI(), parsed.RawQuery)
|
secrets = append(secrets, escaped)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if parsed.User != nil {
|
if parsed.User != nil {
|
||||||
|
|||||||
@@ -202,48 +202,6 @@ func TestRedactor_RemovesHTTPURLQueryAndUserinfo(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestRedactor_RemovesEchoedQueryOfURLWithoutPath covers an
|
|
||||||
// HTTP target URL whose credential is all in its query string.
|
|
||||||
// Written with or without the "/", the request line sends it
|
|
||||||
// as "/?token=…", and a target passing the event's query string
|
|
||||||
// on sends that after an "&". The event's part stays visible:
|
|
||||||
// the event's page shows it anyway.
|
|
||||||
func TestRedactor_RemovesEchoedQueryOfURLWithoutPath(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const secret = "s3cr3t"
|
|
||||||
|
|
||||||
marker := delivery.RedactionMarker
|
|
||||||
|
|
||||||
// An echoed request line, and what the event log shows of it.
|
|
||||||
echoes := map[string]string{
|
|
||||||
"POST /?token=" + secret + " HTTP/1.1": "POST " + marker +
|
|
||||||
" HTTP/1.1",
|
|
||||||
"POST ?token=" + secret + " HTTP/1.1": "POST ?" + marker +
|
|
||||||
" HTTP/1.1",
|
|
||||||
"POST /?token=" + secret + "&a=1&b=2 HTTP/1.1": "POST " +
|
|
||||||
marker + "&a=1&b=2 HTTP/1.1",
|
|
||||||
"POST ?token=" + secret + "&a=1&b=2 HTTP/1.1": "POST ?" +
|
|
||||||
marker + "&a=1&b=2 HTTP/1.1",
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, dest := range []string{
|
|
||||||
"https://example.com/?token=" + secret,
|
|
||||||
"https://example.com?token=" + secret,
|
|
||||||
} {
|
|
||||||
r := delivery.NewRedactor(&database.Target{
|
|
||||||
Type: database.TargetTypeHTTP,
|
|
||||||
Config: `{"url":"` + dest + `"}`,
|
|
||||||
})
|
|
||||||
|
|
||||||
for echoed, want := range echoes {
|
|
||||||
assert.Equal(
|
|
||||||
t, want, r.Redact(echoed), "%s: %s", dest, echoed,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRedactor_LeavesUnrelatedTextAlone pins that the
|
// TestRedactor_LeavesUnrelatedTextAlone pins that the
|
||||||
// redactor matches literally: it does not guess at what a
|
// redactor matches literally: it does not guess at what a
|
||||||
// secret looks like, so ordinary response content survives.
|
// secret looks like, so ordinary response content survives.
|
||||||
|
|||||||
@@ -312,9 +312,13 @@ func TestFailMissingTarget_WritesNoTargetRow(
|
|||||||
database.DeliveryStatusFailed,
|
database.DeliveryStatusFailed,
|
||||||
)
|
)
|
||||||
|
|
||||||
// The per-webhook database has no targets table, so writing a
|
var configs []string
|
||||||
// target row there would have failed the status write above.
|
|
||||||
assert.False(t, s.WebhookDB.Migrator().HasTable("targets"),
|
require.NoError(t, s.WebhookDB.
|
||||||
|
Table("targets").
|
||||||
|
Pluck("config", &configs).Error)
|
||||||
|
|
||||||
|
assert.Empty(t, configs,
|
||||||
"the deleted-target terminal path wrote a target row "+
|
"the deleted-target terminal path wrote a target row "+
|
||||||
"into the per-webhook event database",
|
"into the per-webhook event database",
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -3,7 +3,6 @@ package gormlog_test
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"log/slog"
|
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -14,7 +13,6 @@ import (
|
|||||||
"go.uber.org/fx/fxtest"
|
"go.uber.org/fx/fxtest"
|
||||||
_ "modernc.org/sqlite" // Pure Go SQLite driver.
|
_ "modernc.org/sqlite" // Pure Go SQLite driver.
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/config/configtest"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/globals"
|
"sneak.berlin/go/webhooker/internal/globals"
|
||||||
"sneak.berlin/go/webhooker/internal/logger"
|
"sneak.berlin/go/webhooker/internal/logger"
|
||||||
@@ -130,7 +128,7 @@ func readFirstBootSecrets(
|
|||||||
func bootAtDebug(t *testing.T, dataDir string) string {
|
func bootAtDebug(t *testing.T, dataDir string) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
configtest.ClearEnv(t)
|
config.ClearEnvForTest(t)
|
||||||
t.Setenv("DEBUG", "true")
|
t.Setenv("DEBUG", "true")
|
||||||
t.Setenv("DATA_DIR", dataDir)
|
t.Setenv("DATA_DIR", dataDir)
|
||||||
|
|
||||||
@@ -147,7 +145,6 @@ func bootAtDebug(t *testing.T, dataDir string) string {
|
|||||||
fx.Provide(
|
fx.Provide(
|
||||||
globals.New,
|
globals.New,
|
||||||
logger.New,
|
logger.New,
|
||||||
func(l *logger.Logger) *slog.Logger { return l.Get() },
|
|
||||||
config.New,
|
config.New,
|
||||||
database.New,
|
database.New,
|
||||||
session.New,
|
session.New,
|
||||||
|
|||||||
@@ -135,11 +135,9 @@ func openDB(
|
|||||||
)
|
)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
require.NoError(t, gdb.Exec(
|
require.NoError(t, gdb.AutoMigrate(&thing{}))
|
||||||
"CREATE TABLE things (id text PRIMARY KEY, name text)",
|
|
||||||
).Error)
|
|
||||||
|
|
||||||
// Creating the table is not what any of these cases is about.
|
// Migration chatter is not what any of these cases is about.
|
||||||
buf.Reset()
|
buf.Reset()
|
||||||
|
|
||||||
return gdb
|
return gdb
|
||||||
|
|||||||
@@ -115,8 +115,8 @@ func skipDir(name string) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// walkNonTestGo parses every non-test .go file under root. It returns
|
// walkNonTestGo parses every non-test .go file under root. It returns
|
||||||
// the directories, relative to root, it parsed a file in or below,
|
// the directories, relative to root, it parsed a file in, along with
|
||||||
// along with every unguarded Scan it found.
|
// every unguarded Scan it found.
|
||||||
func walkNonTestGo(t *testing.T, root string) (map[string]bool, []string) {
|
func walkNonTestGo(t *testing.T, root string) (map[string]bool, []string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -155,12 +155,7 @@ func walkNonTestGo(t *testing.T, root string) (map[string]bool, []string) {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// The directories above count too, so that one holding
|
|
||||||
// only a nested package, as internal/db does, is not
|
|
||||||
// taken for one the walk skipped.
|
|
||||||
for ; dir != "."; dir = filepath.Dir(dir) {
|
|
||||||
walked[dir] = true
|
walked[dir] = true
|
||||||
}
|
|
||||||
|
|
||||||
for _, pos := range unguardedScans(fset, file) {
|
for _, pos := range unguardedScans(fset, file) {
|
||||||
hits = append(hits, relPosition(root, pos))
|
hits = append(hits, relPosition(root, pos))
|
||||||
@@ -210,9 +205,9 @@ func TestGormScanIsNeverCalledOutsideTests(t *testing.T) {
|
|||||||
walked, offenders := walkNonTestGo(t, root)
|
walked, offenders := walkNonTestGo(t, root)
|
||||||
|
|
||||||
// The module's packages are static, templates, and every directory
|
// The module's packages are static, templates, and every directory
|
||||||
// directly under cmd and internal. Each holds non-test code, itself
|
// directly under cmd and internal. Each holds non-test code, so one
|
||||||
// or in a package below it, so one the walk parsed nothing in or
|
// the walk parsed nothing in was skipped, and a Scan there would
|
||||||
// below was skipped, and a Scan there would pass unseen.
|
// pass unseen.
|
||||||
packages := []string{"static", "templates"}
|
packages := []string{"static", "templates"}
|
||||||
|
|
||||||
for _, parent := range []string{"cmd", "internal"} {
|
for _, parent := range []string{"cmd", "internal"} {
|
||||||
|
|||||||
@@ -10,10 +10,10 @@ const (
|
|||||||
tmplKeyArchiveExpiryChoices = "ArchiveExpiryChoices"
|
tmplKeyArchiveExpiryChoices = "ArchiveExpiryChoices"
|
||||||
)
|
)
|
||||||
|
|
||||||
// archiveChoice is one entry of a database target's archive expiry
|
// archiveExpiryChoice is one entry of a database target's archive
|
||||||
// or archive rotation select: the value stored, the label shown, and
|
// expiry select: the expiry stored, the label shown, and whether the
|
||||||
// whether the select starts on it.
|
// select starts on it.
|
||||||
type archiveChoice struct {
|
type archiveExpiryChoice struct {
|
||||||
Value string
|
Value string
|
||||||
Label string
|
Label string
|
||||||
Selected bool
|
Selected bool
|
||||||
@@ -21,8 +21,8 @@ type archiveChoice struct {
|
|||||||
|
|
||||||
// archiveExpiryChoices lists the archive expiries offered by the new
|
// archiveExpiryChoices lists the archive expiries offered by the new
|
||||||
// webhook page, the add target form and the target edit form.
|
// webhook page, the add target form and the target edit form.
|
||||||
func archiveExpiryChoices() []archiveChoice {
|
func archiveExpiryChoices() []archiveExpiryChoice {
|
||||||
return []archiveChoice{
|
return []archiveExpiryChoice{
|
||||||
{Value: archiveExpiryNever, Label: archiveExpiryNever},
|
{Value: archiveExpiryNever, Label: archiveExpiryNever},
|
||||||
{Value: "1h", Label: "1h"},
|
{Value: "1h", Label: "1h"},
|
||||||
{Value: "12h", Label: "12h"},
|
{Value: "12h", Label: "12h"},
|
||||||
@@ -37,7 +37,7 @@ func archiveExpiryChoices() []archiveChoice {
|
|||||||
// empty expiry selects never. An expiry that is not one of the
|
// empty expiry selects never. An expiry that is not one of the
|
||||||
// choices comes first as its own selected entry, so saving the form
|
// choices comes first as its own selected entry, so saving the form
|
||||||
// unchanged keeps it.
|
// unchanged keeps it.
|
||||||
func archiveExpiryOptions(expiry string) []archiveChoice {
|
func archiveExpiryOptions(expiry string) []archiveExpiryChoice {
|
||||||
if expiry == "" {
|
if expiry == "" {
|
||||||
expiry = archiveExpiryNever
|
expiry = archiveExpiryNever
|
||||||
}
|
}
|
||||||
@@ -52,7 +52,7 @@ func archiveExpiryOptions(expiry string) []archiveChoice {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
own := archiveChoice{Value: expiry, Label: expiry, Selected: true}
|
own := archiveExpiryChoice{Value: expiry, Label: expiry, Selected: true}
|
||||||
|
|
||||||
return append([]archiveChoice{own}, options...)
|
return append([]archiveExpiryChoice{own}, options...)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ import (
|
|||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"net/url"
|
"net/url"
|
||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
@@ -45,44 +44,26 @@ func expiryShown(
|
|||||||
require.Equal(t, http.StatusOK, w.Code)
|
require.Equal(t, http.StatusOK, w.Code)
|
||||||
|
|
||||||
return matched(
|
return matched(
|
||||||
`Archive expiry:</span>\s*<span>([^<]*)</span>`, w.Body.String(),
|
`Archive Expiry:</span>\s*<span>([^<]*)</span>`, w.Body.String(),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// expirySelected returns the target edit page and the expiries its
|
// expirySelected returns the target edit page and the expiries its
|
||||||
// expiry select starts on.
|
// select starts on.
|
||||||
func expirySelected(
|
func expirySelected(
|
||||||
t *testing.T, env *sourceTestEnv, webhookID, targetID string,
|
t *testing.T, env *sourceTestEnv, webhookID, targetID string,
|
||||||
) (string, []string) {
|
) (string, []string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
page := targetEditPage(t, env, webhookID, targetID)
|
|
||||||
|
|
||||||
return page, selectedIn(page, "expiry")
|
|
||||||
}
|
|
||||||
|
|
||||||
// targetEditPage returns a target's edit page.
|
|
||||||
func targetEditPage(
|
|
||||||
t *testing.T, env *sourceTestEnv, webhookID, targetID string,
|
|
||||||
) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
w := serveTarget(
|
w := serveTarget(
|
||||||
env, http.MethodGet,
|
env, http.MethodGet,
|
||||||
"/hook/"+webhookID+"/targets/"+targetID+"/edit", nil,
|
"/hook/"+webhookID+"/targets/"+targetID+"/edit", nil,
|
||||||
)
|
)
|
||||||
require.Equal(t, http.StatusOK, w.Code)
|
require.Equal(t, http.StatusOK, w.Code)
|
||||||
|
|
||||||
return w.Body.String()
|
page := w.Body.String()
|
||||||
}
|
|
||||||
|
|
||||||
// selectedIn returns the values the select named name on page starts
|
return page, matched(`<option value="([^"]*)" selected>`, page)
|
||||||
// on.
|
|
||||||
func selectedIn(page, name string) []string {
|
|
||||||
_, rest, _ := strings.Cut(page, `<select id="`+name+`" name="`+name+`"`)
|
|
||||||
options, _, _ := strings.Cut(rest, "</select>")
|
|
||||||
|
|
||||||
return matched(`<option value="([^"]*)" selected>`, options)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestArchiveExpiryChoices adds a database target with each archive
|
// TestArchiveExpiryChoices adds a database target with each archive
|
||||||
|
|||||||
@@ -1,42 +0,0 @@
|
|||||||
package handlers
|
|
||||||
|
|
||||||
const (
|
|
||||||
// archiveRotationNone is the archive rotation that keeps a
|
|
||||||
// database target's archive in one file. A stored empty rotation
|
|
||||||
// means the same.
|
|
||||||
archiveRotationNone = "none"
|
|
||||||
|
|
||||||
// tmplKeyArchiveRotationChoices is the template data key for the
|
|
||||||
// entries of a page's archive rotation select.
|
|
||||||
tmplKeyArchiveRotationChoices = "ArchiveRotationChoices"
|
|
||||||
)
|
|
||||||
|
|
||||||
// archiveRotationChoices lists the archive rotations offered by the
|
|
||||||
// new webhook page, the add target form and the target edit form.
|
|
||||||
func archiveRotationChoices() []archiveChoice {
|
|
||||||
return []archiveChoice{
|
|
||||||
{Value: archiveRotationNone, Label: archiveRotationNone},
|
|
||||||
{Value: "monthly", Label: "monthly"},
|
|
||||||
{Value: "daily", Label: "daily"},
|
|
||||||
{Value: "hourly", Label: "hourly"},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// archiveRotationOptions returns the choices with rotation selected;
|
|
||||||
// an empty rotation, or one that is not a choice, selects none. A
|
|
||||||
// stored rotation is always a choice: the forms refuse any other.
|
|
||||||
func archiveRotationOptions(rotation string) []archiveChoice {
|
|
||||||
options := archiveRotationChoices()
|
|
||||||
|
|
||||||
for i := range options {
|
|
||||||
if options[i].Value == rotation {
|
|
||||||
options[i].Selected = true
|
|
||||||
|
|
||||||
return options
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
options[0].Selected = true
|
|
||||||
|
|
||||||
return options
|
|
||||||
}
|
|
||||||
@@ -1,229 +0,0 @@
|
|||||||
package handlers_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"net/url"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
|
||||||
)
|
|
||||||
|
|
||||||
// rotationNone is the archive rotation that keeps one file.
|
|
||||||
const rotationNone = "none"
|
|
||||||
|
|
||||||
// rotationShown returns the archive rotations the webhook page's
|
|
||||||
// target list shows.
|
|
||||||
func rotationShown(
|
|
||||||
t *testing.T, env *sourceTestEnv, webhookID string,
|
|
||||||
) []string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
return matched(
|
|
||||||
`Archive rotation:</span>\s*<span>([^<]*)</span>`,
|
|
||||||
renderedPage(t, env, webhookID),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveRotationChoices adds a database target with each archive
|
|
||||||
// rotation the forms offer, and checks that it is stored as chosen,
|
|
||||||
// shown in the target list, and that the target edit form starts on
|
|
||||||
// it. It then edits the target to hourly, keeping its expiry.
|
|
||||||
func TestArchiveRotationChoices(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupSourceTest(t)
|
|
||||||
|
|
||||||
for _, rotation := range []string{rotationNone, "monthly", "daily", "hourly"} {
|
|
||||||
t.Run(rotation, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
webhook := seedWebhookWithRetention(t, env.db, 30)
|
|
||||||
|
|
||||||
form := url.Values{}
|
|
||||||
form.Set("name", "archive")
|
|
||||||
form.Set("type", string(database.TargetTypeDatabase))
|
|
||||||
form.Set("expiry", "720h")
|
|
||||||
form.Set("rotation", rotation)
|
|
||||||
|
|
||||||
w := serveTarget(
|
|
||||||
env, http.MethodPost, "/hook/"+webhook.ID+"/targets", form,
|
|
||||||
)
|
|
||||||
require.Equal(t, http.StatusSeeOther, w.Code, w.Body.String())
|
|
||||||
|
|
||||||
targets := targetsForWebhook(t, env.db, webhook.ID)
|
|
||||||
require.Len(t, targets, 1)
|
|
||||||
assert.JSONEq(t,
|
|
||||||
`{"expiry":"720h","rotation":"`+rotation+`"}`,
|
|
||||||
targets[0].Config,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.Equal(t,
|
|
||||||
[]string{rotation}, rotationShown(t, env, webhook.ID))
|
|
||||||
|
|
||||||
page := targetEditPage(t, env, webhook.ID, targets[0].ID)
|
|
||||||
assert.Equal(t, []string{rotation}, selectedIn(page, "rotation"))
|
|
||||||
|
|
||||||
form = url.Values{}
|
|
||||||
form.Set("name", "archive")
|
|
||||||
form.Set("expiry", "720h")
|
|
||||||
form.Set("rotation", "hourly")
|
|
||||||
|
|
||||||
w = submitTargetEdit(env, webhook.ID, targets[0].ID, form)
|
|
||||||
require.Equal(t, http.StatusSeeOther, w.Code, w.Body.String())
|
|
||||||
assert.JSONEq(t,
|
|
||||||
`{"expiry":"720h","rotation":"hourly"}`,
|
|
||||||
storedTarget(t, env, targets[0].ID).Config,
|
|
||||||
)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveRotationEditStartsOnNone checks the edit form of a
|
|
||||||
// database target with no rotation stored starts on none.
|
|
||||||
func TestArchiveRotationEditStartsOnNone(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupSourceTest(t)
|
|
||||||
webhook := seedWebhookWithRetention(t, env.db, 30)
|
|
||||||
target := seedConfiguredTarget(
|
|
||||||
t, env.db, webhook.ID, database.TargetTypeDatabase, "",
|
|
||||||
)
|
|
||||||
|
|
||||||
page := targetEditPage(t, env, webhook.ID, target.ID)
|
|
||||||
assert.Equal(t, []string{rotationNone}, selectedIn(page, "rotation"))
|
|
||||||
assert.Equal(t, []string{rotationNone}, rotationShown(t, env, webhook.ID))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveRotationRefused proves a rotation that is not one of the
|
|
||||||
// four is refused on the add target form and the target edit form, and
|
|
||||||
// that nothing is stored.
|
|
||||||
func TestArchiveRotationRefused(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupSourceTest(t)
|
|
||||||
webhook := seedWebhookWithRetention(t, env.db, 30)
|
|
||||||
|
|
||||||
form := url.Values{}
|
|
||||||
form.Set("name", "archive")
|
|
||||||
form.Set("type", string(database.TargetTypeDatabase))
|
|
||||||
form.Set("rotation", "weekly")
|
|
||||||
|
|
||||||
w := serveTarget(
|
|
||||||
env, http.MethodPost, "/hook/"+webhook.ID+"/targets", form,
|
|
||||||
)
|
|
||||||
assert.Equal(t, http.StatusBadRequest, w.Code)
|
|
||||||
assert.Contains(t, w.Body.String(), "Invalid archive rotation")
|
|
||||||
assert.Empty(t, targetsForWebhook(t, env.db, webhook.ID))
|
|
||||||
|
|
||||||
target := seedConfiguredTarget(
|
|
||||||
t, env.db, webhook.ID, database.TargetTypeDatabase,
|
|
||||||
`{"rotation":"daily"}`,
|
|
||||||
)
|
|
||||||
|
|
||||||
form.Del("type")
|
|
||||||
|
|
||||||
w = submitTargetEdit(env, webhook.ID, target.ID, form)
|
|
||||||
assert.Equal(t, http.StatusBadRequest, w.Code)
|
|
||||||
assert.Contains(t, w.Body.String(), "Invalid archive rotation")
|
|
||||||
assert.JSONEq(t,
|
|
||||||
`{"rotation":"daily"}`, storedTarget(t, env, target.ID).Config,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleSourceCreateSubmit_ArchiveRotation proves the new webhook
|
|
||||||
// page's archive rotation is stored on the archive target it creates.
|
|
||||||
func TestHandleSourceCreateSubmit_ArchiveRotation(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupSourceTest(t)
|
|
||||||
|
|
||||||
form := url.Values{}
|
|
||||||
form.Set("name", "rotated")
|
|
||||||
form.Set("archive", "on")
|
|
||||||
form.Set("archive_expiry", "720h")
|
|
||||||
form.Set("archive_rotation", "daily")
|
|
||||||
|
|
||||||
w := submitCreateForm(env, form)
|
|
||||||
require.Equal(t, http.StatusSeeOther, w.Code, w.Body.String())
|
|
||||||
|
|
||||||
var webhook database.Webhook
|
|
||||||
|
|
||||||
require.NoError(t, env.db.DB().
|
|
||||||
Where("name = ?", "rotated").First(&webhook).Error)
|
|
||||||
|
|
||||||
targets := targetsForWebhook(t, env.db, webhook.ID)
|
|
||||||
require.Len(t, targets, 1)
|
|
||||||
assert.JSONEq(t,
|
|
||||||
`{"expiry":"720h","rotation":"daily"}`, targets[0].Config,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveFileView_Rotated describes a daily target's archive files
|
|
||||||
// at two times. On a day that has a file, the view names that file;
|
|
||||||
// on the next, before any event, it names the file the next event
|
|
||||||
// will go to, not created yet. Both times the size is of every file
|
|
||||||
// together and the last write the latest of them.
|
|
||||||
func TestArchiveFileView_Rotated(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupSourceTest(t)
|
|
||||||
webhook := seedWebhookWithRetention(t, env.db, 30)
|
|
||||||
target := seedConfiguredTarget(
|
|
||||||
t, env.db, webhook.ID, database.TargetTypeDatabase,
|
|
||||||
`{"rotation":"daily"}`,
|
|
||||||
)
|
|
||||||
|
|
||||||
path := delivery.ArchivePath(env.dbMgr, &webhook, target)
|
|
||||||
stem := strings.TrimSuffix(path, ".db")
|
|
||||||
written := time.Date(2026, 10, 2, 9, 0, 0, 0, time.UTC)
|
|
||||||
|
|
||||||
for i, day := range []string{"2026-10-01", "2026-10-02"} {
|
|
||||||
file := stem + "-" + day + ".db"
|
|
||||||
require.NoError(t, os.WriteFile(file, make([]byte, 1000), 0o600))
|
|
||||||
|
|
||||||
at := written.Add(time.Duration(i-1) * 24 * time.Hour)
|
|
||||||
require.NoError(t, os.Chtimes(file, at, at))
|
|
||||||
}
|
|
||||||
|
|
||||||
view := env.handlers.ArchiveFileViewForTest(
|
|
||||||
&webhook, target, time.Date(2026, 10, 2, 23, 0, 0, 0, time.UTC),
|
|
||||||
)
|
|
||||||
assert.Equal(t, filepath.Base(stem)+"-2026-10-02.db", view.Name)
|
|
||||||
assert.Empty(t, view.Note)
|
|
||||||
assert.Equal(t, 2, view.Files)
|
|
||||||
assert.Equal(t, "2.0 kB", view.Size)
|
|
||||||
assert.Equal(t, "2026-10-02 09:00:00 UTC", view.WrittenUTC)
|
|
||||||
|
|
||||||
view = env.handlers.ArchiveFileViewForTest(
|
|
||||||
&webhook, target, time.Date(2026, 10, 3, 0, 0, 0, 0, time.UTC),
|
|
||||||
)
|
|
||||||
assert.Equal(t, filepath.Base(stem)+"-2026-10-03.db", view.Name)
|
|
||||||
assert.Equal(t, "not created yet", view.Note)
|
|
||||||
assert.Equal(t, 2, view.Files)
|
|
||||||
assert.Equal(t, "2.0 kB", view.Size)
|
|
||||||
|
|
||||||
page := targetList(t, renderedPage(t, env, webhook.ID))
|
|
||||||
assert.Contains(t, page, "Archive size: 2.0 kB in 2 files")
|
|
||||||
}
|
|
||||||
|
|
||||||
// renderedPage returns the webhook page.
|
|
||||||
func renderedPage(t *testing.T, env *sourceTestEnv, webhookID string) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
env.handlers.HandleSourceDetail().ServeHTTP(w, getRequest(
|
|
||||||
t, "/hook/"+webhookID, env.cookies,
|
|
||||||
map[string]string{sourceIDParam: webhookID},
|
|
||||||
))
|
|
||||||
require.Equal(t, http.StatusOK, w.Code)
|
|
||||||
|
|
||||||
return w.Body.String()
|
|
||||||
}
|
|
||||||
@@ -268,7 +268,7 @@ func (h *Handlers) rejectLogin(
|
|||||||
)))
|
)))
|
||||||
h.renderLoginError(
|
h.renderLoginError(
|
||||||
w, r,
|
w, r,
|
||||||
"Too many failed sign-in attempts. Please try again later.",
|
"Too many failed login attempts. Please try again later.",
|
||||||
http.StatusTooManyRequests,
|
http.StatusTooManyRequests,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -22,28 +22,23 @@ func TestDeliveryAttempts_ReadInTheTargetTypesOwnTerms(t *testing.T) {
|
|||||||
targetType database.TargetType
|
targetType database.TargetType
|
||||||
success bool
|
success bool
|
||||||
statusCode int
|
statusCode int
|
||||||
errText string
|
|
||||||
outcome string
|
outcome string
|
||||||
status string // "" when the attempt must show no status
|
status string // "" when the attempt must show no status
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
database.TargetTypeHTTP, false, 0, "",
|
database.TargetTypeHTTP, false, 0,
|
||||||
"failure", "Status: — (no response)",
|
"failure", "Status: — (no response)",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
database.TargetTypeSlack, true, http.StatusOK, "",
|
database.TargetTypeSlack, true, http.StatusOK,
|
||||||
"success", "Status: 200",
|
"success", "Status: 200",
|
||||||
},
|
},
|
||||||
{database.TargetTypeDatabase, true, 0, "", "archived", ""},
|
{database.TargetTypeDatabase, true, 0, "archived", ""},
|
||||||
{
|
{database.TargetTypeLog, true, 0, "written to the log", ""},
|
||||||
database.TargetTypeDatabase, false, 0,
|
|
||||||
"opening archive database: disk full", "failure", "",
|
|
||||||
},
|
|
||||||
{database.TargetTypeLog, true, 0, "", "written to the log", ""},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range cases {
|
for _, tc := range cases {
|
||||||
t.Run(string(tc.targetType)+" "+tc.outcome, func(t *testing.T) {
|
t.Run(string(tc.targetType), func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
f := newRecentEventsFixture(t)
|
f := newRecentEventsFixture(t)
|
||||||
@@ -59,7 +54,6 @@ func TestDeliveryAttempts_ReadInTheTargetTypesOwnTerms(t *testing.T) {
|
|||||||
AttemptNum: 1,
|
AttemptNum: 1,
|
||||||
Success: tc.success,
|
Success: tc.success,
|
||||||
StatusCode: tc.statusCode,
|
StatusCode: tc.statusCode,
|
||||||
Error: tc.errText,
|
|
||||||
},
|
},
|
||||||
).Error)
|
).Error)
|
||||||
|
|
||||||
@@ -74,10 +68,6 @@ func TestDeliveryAttempts_ReadInTheTargetTypesOwnTerms(t *testing.T) {
|
|||||||
for _, page := range pages {
|
for _, page := range pages {
|
||||||
assert.Contains(t, page, ">"+tc.outcome+"</span>")
|
assert.Contains(t, page, ">"+tc.outcome+"</span>")
|
||||||
|
|
||||||
if tc.errText != "" {
|
|
||||||
assert.Contains(t, page, "Error: "+tc.errText)
|
|
||||||
}
|
|
||||||
|
|
||||||
if tc.status == "" {
|
if tc.status == "" {
|
||||||
assert.NotContains(t, page, "Status:")
|
assert.NotContains(t, page, "Status:")
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package handlers
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"strconv"
|
||||||
|
|
||||||
"github.com/go-chi/chi"
|
"github.com/go-chi/chi"
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
@@ -282,7 +283,6 @@ func createReplayDelivery(
|
|||||||
EventID: event.ID,
|
EventID: event.ID,
|
||||||
TargetID: target.ID,
|
TargetID: target.ID,
|
||||||
Status: database.DeliveryStatusPending,
|
Status: database.DeliveryStatusPending,
|
||||||
Replay: true,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
err := webhookDB.Transaction(func(tx *gorm.DB) error {
|
err := webhookDB.Transaction(func(tx *gorm.DB) error {
|
||||||
@@ -310,7 +310,6 @@ func createReplayDelivery(
|
|||||||
TargetConfig: target.Config,
|
TargetConfig: target.Config,
|
||||||
MaxRetries: target.MaxRetries,
|
MaxRetries: target.MaxRetries,
|
||||||
Method: event.Method,
|
Method: event.Method,
|
||||||
RawQuery: event.RawQuery,
|
|
||||||
Headers: event.Headers,
|
Headers: event.Headers,
|
||||||
ContentType: event.ContentType,
|
ContentType: event.ContentType,
|
||||||
Body: replayBody(event.Body),
|
Body: replayBody(event.Body),
|
||||||
@@ -330,25 +329,24 @@ func replayBody(body string) *string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// redirectToEventLog redirects a replay or resubmit back to the event
|
// redirectToEventLog redirects a replay or resubmit back to the event
|
||||||
// log it was triggered from, carrying the outcome as its notice. A
|
// log it was triggered from, carrying the outcome as its notice and
|
||||||
// Replay form carries the list it was pressed in as show, so a replay
|
// the page number the form submitted.
|
||||||
// returns to the Failed or Pending list; a Resubmit form carries none,
|
|
||||||
// so a resubmit returns to the full log, where its new event is the
|
|
||||||
// newest.
|
|
||||||
func redirectToEventLog(
|
func redirectToEventLog(
|
||||||
w http.ResponseWriter,
|
w http.ResponseWriter,
|
||||||
r *http.Request,
|
r *http.Request,
|
||||||
webhook database.Webhook,
|
webhook database.Webhook,
|
||||||
code noticeCode,
|
code noticeCode,
|
||||||
) {
|
) {
|
||||||
location := withNotice("/hook/"+webhook.ID+"/events", code)
|
dest := withNotice("/hook/"+webhook.ID+"/events", code)
|
||||||
|
|
||||||
show := r.PostFormValue(showParam)
|
// The page is read from the form rather than the query string:
|
||||||
if eventLogStatuses(show) != nil {
|
// this is a POST, and its query string is what logs and Referer
|
||||||
location += "&" + showParam + "=" + show
|
// headers record.
|
||||||
|
if page := pageOrFirst(
|
||||||
|
r.PostFormValue("page"),
|
||||||
|
); page > 1 {
|
||||||
|
dest += "&page=" + strconv.Itoa(page)
|
||||||
}
|
}
|
||||||
|
|
||||||
http.Redirect( //nolint:gosec // show is checked by eventLogStatuses
|
http.Redirect(w, r, dest, http.StatusSeeOther)
|
||||||
w, r, location, http.StatusSeeOther,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,10 +1,8 @@
|
|||||||
package handlers_test
|
package handlers_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"io"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"strings"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
@@ -26,9 +24,6 @@ const paramDeliveryID = "deliveryID"
|
|||||||
// dispatches to it: the notifier is recorded, not run.
|
// dispatches to it: the notifier is recorded, not run.
|
||||||
const replayTargetURL = "http://93.184.216.34/hook"
|
const replayTargetURL = "http://93.184.216.34/hook"
|
||||||
|
|
||||||
// replayEventQuery is the query string a seeded event arrived with.
|
|
||||||
const replayEventQuery = "a=1&b=2"
|
|
||||||
|
|
||||||
// seedFailedDelivery records an event, a terminally failed delivery of
|
// seedFailedDelivery records an event, a terminally failed delivery of
|
||||||
// it to the given target, and the attempt that failed.
|
// it to the given target, and the attempt that failed.
|
||||||
func seedFailedDelivery(
|
func seedFailedDelivery(
|
||||||
@@ -45,7 +40,6 @@ func seedFailedDelivery(
|
|||||||
WebhookID: webhookID,
|
WebhookID: webhookID,
|
||||||
EntrypointID: "entrypoint-" + webhookID,
|
EntrypointID: "entrypoint-" + webhookID,
|
||||||
Method: http.MethodPost,
|
Method: http.MethodPost,
|
||||||
RawQuery: replayEventQuery,
|
|
||||||
Headers: `{"X-Test":["yes"]}`,
|
Headers: `{"X-Test":["yes"]}`,
|
||||||
Body: `{"replay":"me"}`,
|
Body: `{"replay":"me"}`,
|
||||||
ContentType: contentTypeJSON,
|
ContentType: contentTypeJSON,
|
||||||
@@ -300,10 +294,6 @@ func assertReplayTask(
|
|||||||
"replay must use the target's current configuration",
|
"replay must use the target's current configuration",
|
||||||
)
|
)
|
||||||
assert.Equal(t, event.Method, task.Method)
|
assert.Equal(t, event.Method, task.Method)
|
||||||
assert.Equal(
|
|
||||||
t, replayEventQuery, task.RawQuery,
|
|
||||||
"replay re-sends the stored query string",
|
|
||||||
)
|
|
||||||
assert.Equal(t, event.Headers, task.Headers)
|
assert.Equal(t, event.Headers, task.Headers)
|
||||||
assert.Equal(t, event.ContentType, task.ContentType)
|
assert.Equal(t, event.ContentType, task.ContentType)
|
||||||
assert.Equal(t, 1, task.AttemptNum)
|
assert.Equal(t, 1, task.AttemptNum)
|
||||||
@@ -314,15 +304,21 @@ func assertReplayTask(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// assertNoLeakedTarget proves the per-webhook database holds no
|
// assertNoLeakedTarget proves the per-webhook database holds no target
|
||||||
// target: an association write would have upserted the whole target,
|
// rows. AutoMigrate creates the table there because Delivery declares
|
||||||
// plaintext config and all, into the event database. See
|
// the relation, so it is a ROW that signals a leak: an association
|
||||||
|
// write would have upserted the whole target, plaintext config and
|
||||||
|
// all, into the event database. See
|
||||||
// https://git.eeqj.de/sneak/webhooker/issues/206.
|
// https://git.eeqj.de/sneak/webhooker/issues/206.
|
||||||
func assertNoLeakedTarget(t *testing.T, webhookDB *gorm.DB) {
|
func assertNoLeakedTarget(t *testing.T, webhookDB *gorm.DB) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
assert.False(
|
var leaked int64
|
||||||
t, webhookDB.Migrator().HasTable(&database.Target{}),
|
|
||||||
|
require.NoError(t, webhookDB.Unscoped().
|
||||||
|
Model(&database.Target{}).Count(&leaked).Error)
|
||||||
|
assert.Zero(
|
||||||
|
t, leaked,
|
||||||
"replay must not write the target into the event database",
|
"replay must not write the target into the event database",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -474,66 +470,6 @@ func TestHandleDeliveryReplay_RefusesWhileEarlierReplayInFlight(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestHandleDeliveryReplay_ReturnsToTheListItWasPressedIn proves a
|
|
||||||
// Replay pressed in the Failed list carries that list in its form and
|
|
||||||
// returns to it, and that a show value the event log does not know
|
|
||||||
// returns to the full log.
|
|
||||||
func TestHandleDeliveryReplay_ReturnsToTheListItWasPressedIn(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
dbMgr *database.WebhookDBManager
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
wh := seedWebhook(t, db)
|
|
||||||
tgt := seedConfiguredTarget(
|
|
||||||
t, db, wh.ID, database.TargetTypeHTTP,
|
|
||||||
`{"url":"`+replayTargetURL+`"}`,
|
|
||||||
)
|
|
||||||
|
|
||||||
_, original := seedFailedDelivery(t, dbMgr, wh.ID, tgt.ID)
|
|
||||||
|
|
||||||
assert.Contains(t, renderSourceLogsPageWithQuery(
|
|
||||||
t, h, sess, wh.ID, "?show=failed",
|
|
||||||
), `name="show" value="failed"`)
|
|
||||||
|
|
||||||
// The second replay is refused, as the first is still queued.
|
|
||||||
for _, tc := range []struct{ show, location string }{
|
|
||||||
{"failed", "/hook/" + wh.ID +
|
|
||||||
"/events?notice=replay-queued&show=failed"},
|
|
||||||
{"made-up", "/hook/" + wh.ID + "/events?notice=replay-in-flight"},
|
|
||||||
} {
|
|
||||||
req := postRequest(
|
|
||||||
"/hook/"+wh.ID+"/deliveries/"+original.ID+"/replay",
|
|
||||||
authenticatedCookies(
|
|
||||||
t, sess, deleteTestUserID, deleteTestUsername,
|
|
||||||
),
|
|
||||||
map[string]string{
|
|
||||||
paramSourceID: wh.ID,
|
|
||||||
paramDeliveryID: original.ID,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
req.Body = io.NopCloser(strings.NewReader("show=" + tc.show))
|
|
||||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
h.HandleDeliveryReplay().ServeHTTP(w, req)
|
|
||||||
|
|
||||||
require.Equal(t, http.StatusSeeOther, w.Code, tc.show)
|
|
||||||
assert.Equal(t, tc.location, w.Header().Get("Location"), tc.show)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleSourceLogs_RendersReplayControlAndBanner proves the action
|
// TestHandleSourceLogs_RendersReplayControlAndBanner proves the action
|
||||||
// reaches the page it belongs on: a finished delivery renders a POST
|
// reaches the page it belongs on: a finished delivery renders a POST
|
||||||
// form carrying a CSRF token, and the outcome code a refusal redirects
|
// form carrying a CSRF token, and the outcome code a refusal redirects
|
||||||
@@ -592,58 +528,3 @@ func TestHandleSourceLogs_RendersReplayControlAndBanner(t *testing.T) {
|
|||||||
assert.NotContains(t, unknown, "alert-success")
|
assert.NotContains(t, unknown, "alert-success")
|
||||||
assert.NotContains(t, unknown, "made-up")
|
assert.NotContains(t, unknown, "made-up")
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestHandleDeliveryReplay_LabelsTheReplay proves a delivery created
|
|
||||||
// by Replay is labelled as a replay in the event's summary line in the
|
|
||||||
// event log, and in the list of the event's deliveries there and on
|
|
||||||
// the event's page, while the delivery it repeats is not.
|
|
||||||
func TestHandleDeliveryReplay_LabelsTheReplay(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
dbMgr *database.WebhookDBManager
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
wh := seedWebhook(t, db)
|
|
||||||
tgt := seedConfiguredTarget(
|
|
||||||
t, db, wh.ID, database.TargetTypeHTTP,
|
|
||||||
`{"url":"`+replayTargetURL+`"}`,
|
|
||||||
)
|
|
||||||
|
|
||||||
event, original := seedFailedDelivery(t, dbMgr, wh.ID, tgt.ID)
|
|
||||||
|
|
||||||
w := postReplay(t, h, sess, wh.ID, original.ID)
|
|
||||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
|
||||||
|
|
||||||
eventLog := renderSourceLogsPage(t, h, sess, wh.ID)
|
|
||||||
|
|
||||||
assert.Contains(t, eventLog, tgt.Name+": failed")
|
|
||||||
assert.Contains(t, eventLog, tgt.Name+" (replay): pending")
|
|
||||||
|
|
||||||
w = serveEventPage(t, h, sess, wh.ID, event.ID)
|
|
||||||
require.Equal(t, http.StatusOK, w.Code)
|
|
||||||
|
|
||||||
// In each delivery list a row names the target, then the label if
|
|
||||||
// it is a replay, then its status: the replay is still pending, the
|
|
||||||
// original failed.
|
|
||||||
replayRow := tgt.Name + `</span> ` +
|
|
||||||
`<span class="text-xs text-gray-500">replay</span> ` +
|
|
||||||
`<span class="text-xs text-gray-400">pending</span>`
|
|
||||||
originalRow := tgt.Name + `</span> ` +
|
|
||||||
`<span class="text-xs text-red-600">failed</span>`
|
|
||||||
|
|
||||||
for _, page := range []string{eventLog, w.Body.String()} {
|
|
||||||
page = strings.Join(strings.Fields(page), " ")
|
|
||||||
|
|
||||||
assert.Contains(t, page, replayRow)
|
|
||||||
assert.Contains(t, page, originalRow)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -3,7 +3,6 @@ package handlers
|
|||||||
import (
|
import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/dustin/go-humanize"
|
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -12,12 +11,13 @@ import (
|
|||||||
//
|
//
|
||||||
// The bound is enforced here and in SQL, because this page's
|
// The bound is enforced here and in SQL, because this page's
|
||||||
// memory profile must not depend on a constant in another
|
// memory profile must not depend on a constant in another
|
||||||
// package staying where it is, or on every stored row having
|
// package staying where it is, and because rows predating the
|
||||||
// been written by the delivery engine under its own cap.
|
// delivery engine's own cap or restored from an archive are
|
||||||
|
// not covered by it at all.
|
||||||
//
|
//
|
||||||
// It happens to equal that engine cap, so a row written by the
|
// It happens to equal that engine cap, so a row written by the
|
||||||
// engine reaches this one exactly and is never cut twice.
|
// current engine reaches this one exactly and is never cut
|
||||||
// Nothing here may assume the two differ: see view.
|
// twice. Nothing here may assume the two differ: see view.
|
||||||
const maxRenderedResponseBytes = 4096
|
const maxRenderedResponseBytes = 4096
|
||||||
|
|
||||||
// deliveryResultColumns is the delivery attempt projection.
|
// deliveryResultColumns is the delivery attempt projection.
|
||||||
@@ -39,18 +39,14 @@ const deliveryResultColumns = "delivery_id, attempt_num, created_at, " +
|
|||||||
//
|
//
|
||||||
// Both Error and ResponseBody have been through the target's
|
// Both Error and ResponseBody have been through the target's
|
||||||
// Redactor. The engine already masks the URL out of the
|
// Redactor. The engine already masks the URL out of the
|
||||||
// errors it stores, so for errors this is a second line; for
|
// errors it stores, so for errors this is a second line
|
||||||
// response bodies it is the only line, and its reach is what
|
// covering rows written before it did; for response bodies it
|
||||||
|
// is the only line, and its reach is what
|
||||||
// delivery.Redactor documents.
|
// delivery.Redactor documents.
|
||||||
type DeliveryResultView struct {
|
type DeliveryResultView struct {
|
||||||
AttemptNum int
|
AttemptNum int
|
||||||
Success bool
|
Success bool
|
||||||
|
|
||||||
// Ran is how long ago the attempt was recorded, and RanUTC the
|
|
||||||
// full timestamp the page shows on hover.
|
|
||||||
Ran string
|
|
||||||
RanUTC string
|
|
||||||
|
|
||||||
// StatusCode is 0 when the attempt never got a response,
|
// StatusCode is 0 when the attempt never got a response,
|
||||||
// which is why the page asks HasStatusCode rather than
|
// which is why the page asks HasStatusCode rather than
|
||||||
// printing the number.
|
// printing the number.
|
||||||
@@ -164,8 +160,6 @@ func (r *deliveryResultRow) view(
|
|||||||
return DeliveryResultView{
|
return DeliveryResultView{
|
||||||
AttemptNum: r.AttemptNum,
|
AttemptNum: r.AttemptNum,
|
||||||
Success: r.Success,
|
Success: r.Success,
|
||||||
Ran: humanize.Time(r.CreatedAt),
|
|
||||||
RanUTC: r.CreatedAt.UTC().Format(time.DateTime) + " UTC",
|
|
||||||
StatusCode: r.StatusCode,
|
StatusCode: r.StatusCode,
|
||||||
Error: redactor.Redact(r.Error),
|
Error: redactor.Redact(r.Error),
|
||||||
DurationMS: r.Duration,
|
DurationMS: r.Duration,
|
||||||
|
|||||||
@@ -435,7 +435,9 @@ func TestHandleSourceLogs_BoundsRenderedAttempts(t *testing.T) {
|
|||||||
}).Error)
|
}).Error)
|
||||||
}
|
}
|
||||||
|
|
||||||
views := h.LoadEventLogViewsForTest(httptest.NewRecorder(), *wh)
|
views := h.LoadEventLogViewsForTest(
|
||||||
|
httptest.NewRecorder(), *wh, 1,
|
||||||
|
)
|
||||||
require.Len(t, views, 1)
|
require.Len(t, views, 1)
|
||||||
require.Len(t, views[0].Deliveries, 1)
|
require.Len(t, views[0].Deliveries, 1)
|
||||||
|
|
||||||
@@ -487,7 +489,9 @@ func TestHandleSourceLogs_BoundsOversizeResponse(t *testing.T) {
|
|||||||
stored := strings.Repeat("A", responseCap*4) + tail
|
stored := strings.Repeat("A", responseCap*4) + tail
|
||||||
seedFailedDeliveryWithResponse(t, dbMgr, wh.ID, tgt.ID, stored)
|
seedFailedDeliveryWithResponse(t, dbMgr, wh.ID, tgt.ID, stored)
|
||||||
|
|
||||||
views := h.LoadEventLogViewsForTest(httptest.NewRecorder(), *wh)
|
views := h.LoadEventLogViewsForTest(
|
||||||
|
httptest.NewRecorder(), *wh, 1,
|
||||||
|
)
|
||||||
require.Len(t, views, 1)
|
require.Len(t, views, 1)
|
||||||
require.Len(t, views[0].Deliveries, 1)
|
require.Len(t, views[0].Deliveries, 1)
|
||||||
require.Len(t, views[0].Deliveries[0].Results, 1)
|
require.Len(t, views[0].Deliveries[0].Results, 1)
|
||||||
|
|||||||
@@ -1,169 +0,0 @@
|
|||||||
package handlers
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
|
|
||||||
"github.com/go-chi/chi"
|
|
||||||
"github.com/google/uuid"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// HandleEntrypointCreate handles adding a new entrypoint.
|
|
||||||
func (h *Handlers) HandleEntrypointCreate() http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
userID, ok := h.getUserID(r)
|
|
||||||
if !ok {
|
|
||||||
http.Redirect(
|
|
||||||
w, r, "/pages/login", http.StatusSeeOther,
|
|
||||||
)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
sourceID := chi.URLParam(r, "sourceID")
|
|
||||||
|
|
||||||
var webhook database.Webhook
|
|
||||||
|
|
||||||
err := h.db.DB().Where(
|
|
||||||
"id = ? AND user_id = ?", sourceID, userID,
|
|
||||||
).First(&webhook).Error
|
|
||||||
if err != nil {
|
|
||||||
h.renderError(w, r, http.StatusNotFound)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// The body size cap is enforced by the MaxBodySize
|
|
||||||
// middleware, which runs before CSRF parses the form.
|
|
||||||
err = r.ParseForm()
|
|
||||||
if err != nil {
|
|
||||||
h.renderError(w, r, http.StatusBadRequest)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
description := r.PostFormValue("description")
|
|
||||||
|
|
||||||
entrypoint := &database.Entrypoint{
|
|
||||||
WebhookID: webhook.ID,
|
|
||||||
Path: uuid.New().String(),
|
|
||||||
Description: description,
|
|
||||||
Active: true,
|
|
||||||
}
|
|
||||||
|
|
||||||
err = h.db.DB().Create(entrypoint).Error
|
|
||||||
if err != nil {
|
|
||||||
h.serverError(w, r, "failed to create entrypoint", err)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
http.Redirect(
|
|
||||||
w, r, withNotice("/hook/"+webhook.ID, entrypointAdded),
|
|
||||||
http.StatusSeeOther,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// HandleEntrypointEdit handles changing an entrypoint's description.
|
|
||||||
// It writes only the description column, so the entrypoint keeps its
|
|
||||||
// URL, and an activate or deactivate saved since the page was shown
|
|
||||||
// is not undone.
|
|
||||||
func (h *Handlers) HandleEntrypointEdit() http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
userID, ok := h.getUserID(r)
|
|
||||||
if !ok {
|
|
||||||
http.Redirect(
|
|
||||||
w, r, "/pages/login", http.StatusSeeOther,
|
|
||||||
)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
sourceID := chi.URLParam(r, "sourceID")
|
|
||||||
entrypointID := chi.URLParam(r, "entrypointID")
|
|
||||||
|
|
||||||
var webhook database.Webhook
|
|
||||||
|
|
||||||
err := h.db.DB().Where(
|
|
||||||
"id = ? AND user_id = ?", sourceID, userID,
|
|
||||||
).First(&webhook).Error
|
|
||||||
if err != nil {
|
|
||||||
h.renderError(w, r, http.StatusNotFound)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// The body size cap is enforced by the MaxBodySize
|
|
||||||
// middleware, which runs before CSRF parses the form.
|
|
||||||
err = r.ParseForm()
|
|
||||||
if err != nil {
|
|
||||||
h.renderError(w, r, http.StatusBadRequest)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
result := h.db.DB().Model(&database.Entrypoint{}).Where(
|
|
||||||
"id = ? AND webhook_id = ?", entrypointID, webhook.ID,
|
|
||||||
).Update("description", r.PostFormValue("description"))
|
|
||||||
if result.Error != nil {
|
|
||||||
h.serverError(
|
|
||||||
w, r, "failed to edit entrypoint", result.Error,
|
|
||||||
)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// The id came from the URL and may name another webhook's
|
|
||||||
// entrypoint, which this webhook does not have.
|
|
||||||
if result.RowsAffected == 0 {
|
|
||||||
h.renderError(w, r, http.StatusNotFound)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
http.Redirect(
|
|
||||||
w, r, withNotice("/hook/"+webhook.ID, entrypointSaved),
|
|
||||||
http.StatusSeeOther,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// HandleEntrypointDelete handles deleting an entrypoint.
|
|
||||||
func (h *Handlers) HandleEntrypointDelete() http.HandlerFunc {
|
|
||||||
return h.deleteChildResource(
|
|
||||||
"entrypointID", &database.Entrypoint{},
|
|
||||||
"failed to delete entrypoint",
|
|
||||||
nil,
|
|
||||||
entrypointDeleted,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// HandleEntrypointToggle handles toggling an entrypoint's
|
|
||||||
// active state.
|
|
||||||
func (h *Handlers) HandleEntrypointToggle() http.HandlerFunc {
|
|
||||||
return h.toggleChildResource(
|
|
||||||
"entrypointID",
|
|
||||||
func(webhookID, childID string) (bool, error) {
|
|
||||||
var ep database.Entrypoint
|
|
||||||
|
|
||||||
err := h.db.DB().Where(
|
|
||||||
"id = ? AND webhook_id = ?",
|
|
||||||
childID, webhookID,
|
|
||||||
).First(&ep).Error
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Only the active column: saving the whole row would
|
|
||||||
// write back the description read above over an edit
|
|
||||||
// saved since.
|
|
||||||
active := !ep.Active
|
|
||||||
|
|
||||||
return active, h.db.DB().Model(&ep).
|
|
||||||
Update("active", active).Error
|
|
||||||
},
|
|
||||||
"failed to toggle entrypoint",
|
|
||||||
entrypointActivated, entrypointDeactivated,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
@@ -1,7 +1,6 @@
|
|||||||
package handlers
|
package handlers
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"math"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
"github.com/go-chi/chi"
|
"github.com/go-chi/chi"
|
||||||
@@ -60,9 +59,8 @@ func (h *Handlers) HandleEventDetail() http.HandlerFunc {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// The page shows every request header.
|
|
||||||
views, ok := h.eventLogViews(
|
views, ok := h.eventLogViews(
|
||||||
w, r, webhookDB, webhook.ID, rows, targets, math.MaxInt,
|
w, r, webhookDB, webhook.ID, rows, targets,
|
||||||
)
|
)
|
||||||
if !ok {
|
if !ok {
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -1,621 +0,0 @@
|
|||||||
package handlers
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"slices"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/dustin/go-humanize"
|
|
||||||
"gorm.io/gorm"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
|
||||||
)
|
|
||||||
|
|
||||||
// DeliveryView is the display-safe projection of a delivery
|
|
||||||
// for the event log page. Its target is a TargetView, so the
|
|
||||||
// stored configuration blob — which holds the target's
|
|
||||||
// credential — has no path to the template.
|
|
||||||
type DeliveryView struct {
|
|
||||||
ID string
|
|
||||||
Status database.DeliveryStatus
|
|
||||||
Target delivery.TargetView
|
|
||||||
|
|
||||||
// Replay is set on a delivery the Replay action created.
|
|
||||||
Replay bool
|
|
||||||
|
|
||||||
// Created is how long ago the delivery was created, and
|
|
||||||
// CreatedUTC the full timestamp the page shows on hover.
|
|
||||||
Created string
|
|
||||||
CreatedUTC string
|
|
||||||
|
|
||||||
// Results is this delivery's attempts in attempt order,
|
|
||||||
// bounded by maxRenderedAttempts. Without them a failure
|
|
||||||
// renders as the status word alone and says nothing about
|
|
||||||
// why.
|
|
||||||
Results []DeliveryResultView
|
|
||||||
|
|
||||||
// AttemptCount is how many attempts were recorded, which
|
|
||||||
// is more than len(Results) once the middle was dropped.
|
|
||||||
AttemptCount int
|
|
||||||
|
|
||||||
// AttemptsOmitted is how many attempts were dropped from
|
|
||||||
// the middle of Results. The page must show it, or the
|
|
||||||
// bound would hide history rather than fold it.
|
|
||||||
AttemptsOmitted int
|
|
||||||
|
|
||||||
// Paused is set while the delivery is retrying and its
|
|
||||||
// target's circuit breaker is open, and nil otherwise.
|
|
||||||
Paused *PausedView
|
|
||||||
}
|
|
||||||
|
|
||||||
// eventLogTarget is what the event log needs to know about
|
|
||||||
// one target: the display-safe view its template renders, and
|
|
||||||
// the redactor that keeps that target's own credential out of
|
|
||||||
// the text its remote peer chose. The two are kept together
|
|
||||||
// so a caller cannot pick up one without the other, and apart
|
|
||||||
// from TargetView so the secrets never reach a template.
|
|
||||||
type eventLogTarget struct {
|
|
||||||
View delivery.TargetView
|
|
||||||
Redactor delivery.Redactor
|
|
||||||
}
|
|
||||||
|
|
||||||
// The event log's show query parameter and its two values: the events
|
|
||||||
// with a failed delivery, and those with a delivery still pending or
|
|
||||||
// retrying.
|
|
||||||
const (
|
|
||||||
showParam = "show"
|
|
||||||
showFailed = "failed"
|
|
||||||
showPending = "pending"
|
|
||||||
)
|
|
||||||
|
|
||||||
// HandleSourceLogs shows the request/response logs for a
|
|
||||||
// webhook.
|
|
||||||
func (h *Handlers) HandleSourceLogs() http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
webhook, ok := h.ownedWebhook(w, r)
|
|
||||||
if !ok {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
targets, err := h.loadTargetMap(webhook.ID)
|
|
||||||
if err != nil {
|
|
||||||
// Without the map every delivery renders through a
|
|
||||||
// zero redactor, so failing the page is the only
|
|
||||||
// safe answer.
|
|
||||||
h.serverError(w, r, "failed to load targets", err)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Any other value of show lists every event, as no value
|
|
||||||
// does.
|
|
||||||
show := r.URL.Query().Get(showParam)
|
|
||||||
|
|
||||||
statuses := eventLogStatuses(show)
|
|
||||||
if statuses == nil {
|
|
||||||
show = ""
|
|
||||||
}
|
|
||||||
|
|
||||||
evts, total, ok := h.loadEventsWithDeliveries(
|
|
||||||
w, r, webhook, targets, statuses,
|
|
||||||
)
|
|
||||||
if !ok {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
failed, pending, err := h.countFailedAndPendingEvents(webhook.ID)
|
|
||||||
if err != nil {
|
|
||||||
h.serverError(w, r, "failed to count events", err)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
data := map[string]any{
|
|
||||||
tmplKeyWebhook: &webhook,
|
|
||||||
"Events": evts,
|
|
||||||
"TotalEvents": total,
|
|
||||||
"Show": show,
|
|
||||||
"FailedEvents": failed,
|
|
||||||
"PendingEvents": pending,
|
|
||||||
}
|
|
||||||
|
|
||||||
h.renderTemplate(w, r, "source_logs.html", data)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// loadTargetMap loads targets into a map of display-safe
|
|
||||||
// views keyed by target ID, each paired with its redactor.
|
|
||||||
// The projection happens here so that no caller can hand a
|
|
||||||
// raw target, configuration blob and all, to a template: the
|
|
||||||
// raw rows do not leave this function.
|
|
||||||
//
|
|
||||||
// The load is Unscoped because deleting a target only soft
|
|
||||||
// deletes the row while its deliveries survive in the
|
|
||||||
// per-webhook database. Both halves of the map need those rows:
|
|
||||||
// a scoped load leaves an old delivery with a zero redactor,
|
|
||||||
// which renders its response bodies unredacted, and with a zero
|
|
||||||
// view, which renders its target as a blank name.
|
|
||||||
//
|
|
||||||
// This map is historical display only. It is built for the event
|
|
||||||
// log and an event's own page, and reaches nothing but
|
|
||||||
// DeliveryView.Target: the target list on the source detail page,
|
|
||||||
// the edit form and the replay path each resolve targets
|
|
||||||
// themselves, and a deleted row is refused there as before.
|
|
||||||
func (h *Handlers) loadTargetMap(
|
|
||||||
webhookID string,
|
|
||||||
) (map[string]eventLogTarget, error) {
|
|
||||||
var targets []database.Target
|
|
||||||
|
|
||||||
err := h.db.DB().Unscoped().Where(
|
|
||||||
"webhook_id = ?", webhookID,
|
|
||||||
).Find(&targets).Error
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
targetMap := make(
|
|
||||||
map[string]eventLogTarget, len(targets),
|
|
||||||
)
|
|
||||||
|
|
||||||
for i := range targets {
|
|
||||||
targetMap[targets[i].ID] = eventLogTarget{
|
|
||||||
Redactor: delivery.NewRedactor(&targets[i]),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The views come from NewTargetViews rather than being
|
|
||||||
// rebuilt here, so the masking rules stay in one place and a
|
|
||||||
// deleted target's configuration is masked by the same code
|
|
||||||
// that masks a live one's.
|
|
||||||
for _, v := range delivery.NewTargetViews(targets) {
|
|
||||||
entry := targetMap[v.ID]
|
|
||||||
entry.View = v
|
|
||||||
targetMap[v.ID] = entry
|
|
||||||
}
|
|
||||||
|
|
||||||
return targetMap, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// loadEventsWithDeliveries loads the recentEventLimit newest events
|
|
||||||
// and their deliveries from the per-webhook database, and the total
|
|
||||||
// number of events stored. Given delivery statuses, both cover only
|
|
||||||
// the events with a delivery in one of them. Events come back as
|
|
||||||
// capped projections rather than database.Event rows: see
|
|
||||||
// eventLogColumns for why the cut happens in SQL.
|
|
||||||
//
|
|
||||||
// The bool reports whether the load succeeded. It is false
|
|
||||||
// once this has answered the request with an error, and the
|
|
||||||
// caller must then render nothing further.
|
|
||||||
func (h *Handlers) loadEventsWithDeliveries(
|
|
||||||
w http.ResponseWriter,
|
|
||||||
r *http.Request,
|
|
||||||
webhook database.Webhook,
|
|
||||||
targetMap map[string]eventLogTarget,
|
|
||||||
statuses []database.DeliveryStatus,
|
|
||||||
) ([]EventLogView, int64, bool) {
|
|
||||||
if !h.dbMgr.DBExists(webhook.ID) {
|
|
||||||
return nil, 0, true
|
|
||||||
}
|
|
||||||
|
|
||||||
webhookDB, err := h.dbMgr.GetDB(webhook.ID)
|
|
||||||
if err != nil {
|
|
||||||
h.serverError(
|
|
||||||
w, r, "failed to get webhook database", err,
|
|
||||||
)
|
|
||||||
|
|
||||||
return nil, 0, false
|
|
||||||
}
|
|
||||||
|
|
||||||
rows, totalEvents, err := loadEventLogRows(
|
|
||||||
webhookDB, webhook.ID, statuses,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
h.serverError(w, r, "failed to load events", err)
|
|
||||||
|
|
||||||
return nil, 0, false
|
|
||||||
}
|
|
||||||
|
|
||||||
result, ok := h.eventLogViews(
|
|
||||||
w, r, webhookDB, webhook.ID, rows, targetMap,
|
|
||||||
maxRenderedBodyBytes,
|
|
||||||
)
|
|
||||||
|
|
||||||
return result, totalEvents, ok
|
|
||||||
}
|
|
||||||
|
|
||||||
// eventLogViews projects loaded events for rendering, each with
|
|
||||||
// its deliveries, how many times it has been resubmitted and the
|
|
||||||
// entrypoint it arrived at (for a resubmitted copy, the one the
|
|
||||||
// request it copies arrived at), and with its request headers only
|
|
||||||
// when their text holds at most maxHeaderBytes. Like
|
|
||||||
// loadEventsWithDeliveries, it reports false once it has answered
|
|
||||||
// the request with an error.
|
|
||||||
func (h *Handlers) eventLogViews(
|
|
||||||
w http.ResponseWriter,
|
|
||||||
r *http.Request,
|
|
||||||
webhookDB *gorm.DB,
|
|
||||||
webhookID string,
|
|
||||||
rows []eventLogRow,
|
|
||||||
targetMap map[string]eventLogTarget,
|
|
||||||
maxHeaderBytes int,
|
|
||||||
) ([]EventLogView, bool) {
|
|
||||||
result := make([]EventLogView, len(rows))
|
|
||||||
eventDeliveries := make([][]database.Delivery, len(rows))
|
|
||||||
|
|
||||||
var deliveryIDs []string
|
|
||||||
|
|
||||||
eventIDs := make([]string, len(rows))
|
|
||||||
|
|
||||||
for i := range rows {
|
|
||||||
result[i] = rows[i].view(webhookID, maxHeaderBytes)
|
|
||||||
eventIDs[i] = rows[i].ID
|
|
||||||
|
|
||||||
webhookDB.Where(
|
|
||||||
"event_id = ?", rows[i].ID,
|
|
||||||
).Find(&eventDeliveries[i])
|
|
||||||
|
|
||||||
for j := range eventDeliveries[i] {
|
|
||||||
deliveryIDs = append(
|
|
||||||
deliveryIDs, eventDeliveries[i][j].ID,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
attempts, err := h.loadDeliveryResults(
|
|
||||||
webhookDB, deliveryIDs,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
h.serverError(
|
|
||||||
w, r, "failed to load delivery attempts", err,
|
|
||||||
)
|
|
||||||
|
|
||||||
return nil, false
|
|
||||||
}
|
|
||||||
|
|
||||||
resubmits, err := resubmitCounts(webhookDB, eventIDs)
|
|
||||||
if err != nil {
|
|
||||||
h.serverError(
|
|
||||||
w, r, "failed to count event resubmissions", err,
|
|
||||||
)
|
|
||||||
|
|
||||||
return nil, false
|
|
||||||
}
|
|
||||||
|
|
||||||
entrypoints, err := h.entrypointNames(webhookID)
|
|
||||||
if err != nil {
|
|
||||||
h.serverError(w, r, "failed to load entrypoints", err)
|
|
||||||
|
|
||||||
return nil, false
|
|
||||||
}
|
|
||||||
|
|
||||||
for i := range rows {
|
|
||||||
result[i].Deliveries = h.newDeliveryViews(
|
|
||||||
eventDeliveries[i], targetMap, attempts,
|
|
||||||
)
|
|
||||||
result[i].ResubmitCount = resubmits[rows[i].ID]
|
|
||||||
|
|
||||||
name, ok := entrypoints[rows[i].EntrypointID]
|
|
||||||
if !ok {
|
|
||||||
name = "deleted entrypoint"
|
|
||||||
}
|
|
||||||
|
|
||||||
result[i].Entrypoint = name
|
|
||||||
}
|
|
||||||
|
|
||||||
return result, true
|
|
||||||
}
|
|
||||||
|
|
||||||
// loadEventLogRows reads the event log projection of the
|
|
||||||
// recentEventLimit newest events, newest first, and the total number
|
|
||||||
// of events stored, both narrowed by statuses as eventsWithStatus
|
|
||||||
// narrows them.
|
|
||||||
func loadEventLogRows(
|
|
||||||
webhookDB *gorm.DB,
|
|
||||||
webhookID string,
|
|
||||||
statuses []database.DeliveryStatus,
|
|
||||||
) ([]eventLogRow, int64, error) {
|
|
||||||
totalEvents, err := countEventsWithStatus(
|
|
||||||
webhookDB, webhookID, statuses,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return nil, 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
var rows []eventLogRow
|
|
||||||
|
|
||||||
err = eventsWithStatus(webhookDB, webhookID, statuses).Select(
|
|
||||||
eventLogColumns,
|
|
||||||
maxRenderedBodyBytes, maxRenderedBodyBytes, maxRenderedBodyBytes,
|
|
||||||
).Order("created_at DESC").Limit(recentEventLimit).Find(&rows).Error
|
|
||||||
|
|
||||||
return rows, totalEvents, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// eventLogStatuses returns the delivery statuses the event log's show
|
|
||||||
// value lists events by, or nil for one that lists every event.
|
|
||||||
func eventLogStatuses(show string) []database.DeliveryStatus {
|
|
||||||
switch show {
|
|
||||||
case showFailed:
|
|
||||||
return []database.DeliveryStatus{database.DeliveryStatusFailed}
|
|
||||||
case showPending:
|
|
||||||
return []database.DeliveryStatus{
|
|
||||||
database.DeliveryStatusPending,
|
|
||||||
database.DeliveryStatusRetrying,
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// eventsWithStatus selects the webhook's events, or, given statuses,
|
|
||||||
// the recentEventLimit newest of those with at least one delivery in
|
|
||||||
// one of them.
|
|
||||||
//
|
|
||||||
// Given statuses, its cost follows the matching deliveries. SQLite
|
|
||||||
// never reorders a CROSS JOIN, so it reads each join's left side
|
|
||||||
// first: the distinct event IDs of the matching deliveries, through
|
|
||||||
// idx_deliveries_status; then each of those events by ID, sorted to
|
|
||||||
// keep the newest; then the rows of only the events kept, so no other
|
|
||||||
// event's body is read. With a plain "id IN (matching deliveries)"
|
|
||||||
// condition instead, SQLite, which keeps no statistics on these
|
|
||||||
// tables, walks every event newest first.
|
|
||||||
func eventsWithStatus(
|
|
||||||
webhookDB *gorm.DB,
|
|
||||||
webhookID string,
|
|
||||||
statuses []database.DeliveryStatus,
|
|
||||||
) *gorm.DB {
|
|
||||||
if statuses == nil {
|
|
||||||
return webhookDB.Model(&database.Event{}).Where(
|
|
||||||
"webhook_id = ?", webhookID,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
matching := webhookDB.Model(&database.Delivery{}).
|
|
||||||
Distinct("event_id").Where("status IN ?", statuses)
|
|
||||||
|
|
||||||
newest := webhookDB.Table("(?) AS matching", matching).
|
|
||||||
Joins("CROSS JOIN events ON events.id = matching.event_id").
|
|
||||||
Where(
|
|
||||||
"events.webhook_id = ? AND events.deleted_at IS NULL",
|
|
||||||
webhookID,
|
|
||||||
).
|
|
||||||
Order("events.created_at DESC").Limit(recentEventLimit).
|
|
||||||
Select("events.id AS event_id")
|
|
||||||
|
|
||||||
return webhookDB.Table("(?) AS newest", newest).
|
|
||||||
Joins("CROSS JOIN events ON events.id = newest.event_id")
|
|
||||||
}
|
|
||||||
|
|
||||||
// countEventsWithStatus counts the webhook's events with at least one
|
|
||||||
// delivery in one of the statuses, or every event when statuses is
|
|
||||||
// nil. Given statuses, it counts the distinct events of the matching
|
|
||||||
// deliveries and reads nothing but those deliveries, through
|
|
||||||
// idx_deliveries_status, where counting the events would read every
|
|
||||||
// event row. That is the same number, because retention deletes an
|
|
||||||
// event's deliveries with it.
|
|
||||||
func countEventsWithStatus(
|
|
||||||
webhookDB *gorm.DB,
|
|
||||||
webhookID string,
|
|
||||||
statuses []database.DeliveryStatus,
|
|
||||||
) (int64, error) {
|
|
||||||
var count int64
|
|
||||||
|
|
||||||
if statuses == nil {
|
|
||||||
err := webhookDB.Model(&database.Event{}).Where(
|
|
||||||
"webhook_id = ?", webhookID,
|
|
||||||
).Count(&count).Error
|
|
||||||
|
|
||||||
return count, err
|
|
||||||
}
|
|
||||||
|
|
||||||
err := webhookDB.Model(&database.Delivery{}).Distinct("event_id").
|
|
||||||
Where("status IN ?", statuses).Count(&count).Error
|
|
||||||
|
|
||||||
return count, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// countFailedAndPendingEvents returns how many of the webhook's events
|
|
||||||
// the event log lists when it shows only those with a failed delivery,
|
|
||||||
// and when it shows only those with a delivery pending or retrying.
|
|
||||||
func (h *Handlers) countFailedAndPendingEvents(
|
|
||||||
webhookID string,
|
|
||||||
) (int64, int64, error) {
|
|
||||||
if !h.dbMgr.DBExists(webhookID) {
|
|
||||||
return 0, 0, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
webhookDB, err := h.dbMgr.GetDB(webhookID)
|
|
||||||
if err != nil {
|
|
||||||
return 0, 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
failed, err := countEventsWithStatus(
|
|
||||||
webhookDB, webhookID, eventLogStatuses(showFailed),
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return 0, 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
pending, err := countEventsWithStatus(
|
|
||||||
webhookDB, webhookID, eventLogStatuses(showPending),
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return 0, 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return failed, pending, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// resubmitCounts reports, for each of the page's events, how many
|
|
||||||
// events have been resubmitted from it.
|
|
||||||
//
|
|
||||||
// One grouped query covers the page rather than one query per event.
|
|
||||||
// The page shows at most recentEventLimit events, far below SQLite's
|
|
||||||
// bound parameter ceiling, so it needs no chunking as the delivery
|
|
||||||
// result load does.
|
|
||||||
func resubmitCounts(
|
|
||||||
webhookDB *gorm.DB, eventIDs []string,
|
|
||||||
) (map[string]int, error) {
|
|
||||||
counts := make(map[string]int, len(eventIDs))
|
|
||||||
|
|
||||||
if len(eventIDs) == 0 {
|
|
||||||
return counts, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var rows []struct {
|
|
||||||
ResubmittedFromID string
|
|
||||||
Total int
|
|
||||||
}
|
|
||||||
|
|
||||||
err := webhookDB.Model(&database.Event{}).
|
|
||||||
Select("resubmitted_from_id, count(*) AS total").
|
|
||||||
Where("resubmitted_from_id IN ?", eventIDs).
|
|
||||||
Group("resubmitted_from_id").
|
|
||||||
Find(&rows).Error
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, row := range rows {
|
|
||||||
counts[row.ResubmittedFromID] = row.Total
|
|
||||||
}
|
|
||||||
|
|
||||||
return counts, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// deliveryIDChunkSize bounds how many delivery IDs go into one
|
|
||||||
// IN clause. SQLite refuses a statement carrying more than
|
|
||||||
// SQLITE_MAX_VARIABLE_NUMBER (32766) bound parameters, and a
|
|
||||||
// page holds one delivery per target per event, so a webhook
|
|
||||||
// with enough targets would turn the whole query into an error
|
|
||||||
// and the page into zero attempts.
|
|
||||||
const deliveryIDChunkSize = 500
|
|
||||||
|
|
||||||
// loadDeliveryResults loads the recorded attempts for the
|
|
||||||
// page's deliveries, keyed by delivery ID.
|
|
||||||
//
|
|
||||||
// Each response body is cut by SQLite rather than in Go, for
|
|
||||||
// the reason deliveryResultColumns gives. How many attempts a
|
|
||||||
// delivery has is the target's MaxRetries, which the
|
|
||||||
// authenticated operator sets; how many of them reach the page
|
|
||||||
// is bounded again by maxRenderedAttempts.
|
|
||||||
func (h *Handlers) loadDeliveryResults(
|
|
||||||
webhookDB *gorm.DB,
|
|
||||||
deliveryIDs []string,
|
|
||||||
) (map[string][]deliveryResultRow, error) {
|
|
||||||
byDelivery := make(map[string][]deliveryResultRow)
|
|
||||||
|
|
||||||
for chunk := range slices.Chunk(
|
|
||||||
deliveryIDs, deliveryIDChunkSize,
|
|
||||||
) {
|
|
||||||
var rows []deliveryResultRow
|
|
||||||
|
|
||||||
err := webhookDB.Model(
|
|
||||||
&database.DeliveryResult{},
|
|
||||||
).Select(
|
|
||||||
deliveryResultColumns, maxRenderedResponseBytes,
|
|
||||||
).Where(
|
|
||||||
"delivery_id IN ?", chunk,
|
|
||||||
).Order("attempt_num ASC").Find(&rows).Error
|
|
||||||
if err != nil {
|
|
||||||
// Returning what was loaded so far renders the
|
|
||||||
// deliveries in the failed chunk as never having run,
|
|
||||||
// which is indistinguishable from ones that really
|
|
||||||
// never ran. The page fails instead.
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
for i := range rows {
|
|
||||||
byDelivery[rows[i].DeliveryID] = append(
|
|
||||||
byDelivery[rows[i].DeliveryID], rows[i],
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return byDelivery, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// newDeliveryViews projects deliveries for rendering,
|
|
||||||
// resolving each one's target to its display-safe view and
|
|
||||||
// each one's attempts through that target's redactor. A
|
|
||||||
// retrying delivery also reads its target's circuit breaker.
|
|
||||||
func (h *Handlers) newDeliveryViews(
|
|
||||||
deliveries []database.Delivery,
|
|
||||||
targetMap map[string]eventLogTarget,
|
|
||||||
attempts map[string][]deliveryResultRow,
|
|
||||||
) []DeliveryView {
|
|
||||||
views := make([]DeliveryView, len(deliveries))
|
|
||||||
|
|
||||||
for i := range deliveries {
|
|
||||||
target := targetMap[deliveries[i].TargetID]
|
|
||||||
rows := attempts[deliveries[i].ID]
|
|
||||||
created := deliveries[i].CreatedAt
|
|
||||||
|
|
||||||
results, omitted := renderedAttempts(
|
|
||||||
rows, target.Redactor,
|
|
||||||
)
|
|
||||||
|
|
||||||
views[i] = DeliveryView{
|
|
||||||
ID: deliveries[i].ID,
|
|
||||||
Status: deliveries[i].Status,
|
|
||||||
Target: target.View,
|
|
||||||
Replay: deliveries[i].Replay,
|
|
||||||
Created: humanize.Time(created),
|
|
||||||
CreatedUTC: created.UTC().Format(time.DateTime) + " UTC",
|
|
||||||
Results: results,
|
|
||||||
AttemptCount: len(rows),
|
|
||||||
AttemptsOmitted: omitted,
|
|
||||||
}
|
|
||||||
|
|
||||||
if deliveries[i].Status == database.DeliveryStatusRetrying {
|
|
||||||
views[i].Paused = h.deliveryPausedView(
|
|
||||||
deliveries[i].TargetID, rows,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return views
|
|
||||||
}
|
|
||||||
|
|
||||||
// maxRenderedAttempts bounds how many of one delivery's
|
|
||||||
// attempts the page renders. Past it the middle is dropped and
|
|
||||||
// counted, keeping the first attempts and the last ones: how
|
|
||||||
// the delivery started failing and how it ended are what a
|
|
||||||
// reader needs, and the count says plainly that the rest was
|
|
||||||
// dropped rather than never recorded.
|
|
||||||
const (
|
|
||||||
renderedAttemptsHead = 10
|
|
||||||
renderedAttemptsTail = 10
|
|
||||||
maxRenderedAttempts = renderedAttemptsHead +
|
|
||||||
renderedAttemptsTail
|
|
||||||
)
|
|
||||||
|
|
||||||
// renderedAttempts projects a delivery's attempts through the
|
|
||||||
// target's redactor, at most maxRenderedAttempts of them, and
|
|
||||||
// reports how many it dropped.
|
|
||||||
func renderedAttempts(
|
|
||||||
rows []deliveryResultRow,
|
|
||||||
redactor delivery.Redactor,
|
|
||||||
) ([]DeliveryResultView, int) {
|
|
||||||
omitted := 0
|
|
||||||
|
|
||||||
if len(rows) > maxRenderedAttempts {
|
|
||||||
omitted = len(rows) - maxRenderedAttempts
|
|
||||||
|
|
||||||
kept := make(
|
|
||||||
[]deliveryResultRow, 0, maxRenderedAttempts,
|
|
||||||
)
|
|
||||||
kept = append(kept, rows[:renderedAttemptsHead]...)
|
|
||||||
kept = append(
|
|
||||||
kept, rows[len(rows)-renderedAttemptsTail:]...,
|
|
||||||
)
|
|
||||||
rows = kept
|
|
||||||
}
|
|
||||||
|
|
||||||
views := make([]DeliveryResultView, len(rows))
|
|
||||||
for i := range rows {
|
|
||||||
views[i] = rows[i].view(redactor)
|
|
||||||
}
|
|
||||||
|
|
||||||
return views, omitted
|
|
||||||
}
|
|
||||||
@@ -1,52 +0,0 @@
|
|||||||
package handlers_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestEventLog_TimesCarryTheirZone proves that the event log shows
|
|
||||||
// when an event arrived, and that it and the event's page show when
|
|
||||||
// each delivery was created and each attempt recorded: each as how
|
|
||||||
// long ago, with the full UTC time on hover.
|
|
||||||
func TestEventLog_TimesCarryTheirZone(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
f := newRecentEventsFixture(t)
|
|
||||||
target := seedTarget(t, f.db, f.webhook.ID, database.TargetTypeHTTP)
|
|
||||||
|
|
||||||
now := time.Now().UTC().Truncate(time.Second)
|
|
||||||
receivedAt := now.Add(-3 * time.Hour)
|
|
||||||
createdAt := now.Add(-90 * time.Minute)
|
|
||||||
ranAt := now.Add(-30 * time.Minute)
|
|
||||||
|
|
||||||
event := f.event(t, contentTypeJSON, "{}", receivedAt)
|
|
||||||
dlv := f.deliveryQueuedAt(
|
|
||||||
t, event, target.ID, database.DeliveryStatusDelivered, createdAt,
|
|
||||||
)
|
|
||||||
f.attempt(t, dlv, http.StatusOK, ranAt.Sub(createdAt))
|
|
||||||
|
|
||||||
w := serveEventPage(t, f.h, f.sess, f.webhook.ID, event.ID)
|
|
||||||
require.Equal(t, http.StatusOK, w.Code)
|
|
||||||
|
|
||||||
eventPage := w.Body.String()
|
|
||||||
eventLog := renderSourceLogsPage(t, f.h, f.sess, f.webhook.ID)
|
|
||||||
|
|
||||||
received := receivedAt.Format(time.DateTime)
|
|
||||||
assert.Contains(t, eventLog, `title="`+received+` UTC">3 hours ago</span>`)
|
|
||||||
assert.NotContains(t, eventLog, received+"</span>",
|
|
||||||
"an event's time must not be written without its zone")
|
|
||||||
assert.Contains(t, eventPage, received+" UTC")
|
|
||||||
|
|
||||||
for _, page := range []string{eventLog, eventPage} {
|
|
||||||
assert.Contains(t, page, `title="`+createdAt.Format(time.DateTime)+
|
|
||||||
` UTC">created 1 hour ago</span>`)
|
|
||||||
assert.Contains(t, page, `title="`+ranAt.Format(time.DateTime)+
|
|
||||||
` UTC">30 minutes ago</span>`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,15 +1,8 @@
|
|||||||
package handlers
|
package handlers
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
|
||||||
"net/http"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"time"
|
"time"
|
||||||
"unicode/utf8"
|
"unicode/utf8"
|
||||||
|
|
||||||
"github.com/dustin/go-humanize"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// eventLogColumns is the event log's projection. The casts to
|
// eventLogColumns is the event log's projection. The casts to
|
||||||
@@ -17,24 +10,16 @@ import (
|
|||||||
// bytes rather than characters, so the cap bounds the page in
|
// bytes rather than characters, so the cap bounds the page in
|
||||||
// bytes whatever the payload's encoding. Cutting in SQLite
|
// bytes whatever the payload's encoding. Cutting in SQLite
|
||||||
// rather than in Go is the point of the projection — an
|
// rather than in Go is the point of the projection — an
|
||||||
// oversized body, query string or set of request headers never
|
// oversized body never becomes a Go string at all.
|
||||||
// becomes a Go string at all.
|
|
||||||
const eventLogColumns = "id, created_at, method, content_type, " +
|
const eventLogColumns = "id, created_at, method, content_type, " +
|
||||||
"resubmitted_from_id, entrypoint_id, " +
|
"resubmitted_from_id, " +
|
||||||
"substr(cast(raw_query as blob), 1, ?) AS raw_query, " +
|
|
||||||
"length(cast(raw_query as blob)) AS raw_query_bytes, " +
|
|
||||||
"substr(cast(headers as blob), 1, ?) AS headers, " +
|
|
||||||
"length(cast(headers as blob)) AS headers_bytes, " +
|
|
||||||
"substr(cast(body as blob), 1, ?) AS body, " +
|
"substr(cast(body as blob), 1, ?) AS body, " +
|
||||||
"length(cast(body as blob)) AS body_bytes"
|
"length(cast(body as blob)) AS body_bytes"
|
||||||
|
|
||||||
// eventColumns is eventLogColumns for the event's own page, which
|
// eventColumns is eventLogColumns for the event's own page, which
|
||||||
// shows the whole body, the whole query string and every request
|
// shows the whole body.
|
||||||
// header.
|
|
||||||
const eventColumns = "id, created_at, method, content_type, " +
|
const eventColumns = "id, created_at, method, content_type, " +
|
||||||
"resubmitted_from_id, entrypoint_id, raw_query, " +
|
"resubmitted_from_id, " +
|
||||||
"length(cast(raw_query as blob)) AS raw_query_bytes, headers, " +
|
|
||||||
"length(cast(headers as blob)) AS headers_bytes, " +
|
|
||||||
"cast(body as blob) AS body, " +
|
"cast(body as blob) AS body, " +
|
||||||
"length(cast(body as blob)) AS body_bytes"
|
"length(cast(body as blob)) AS body_bytes"
|
||||||
|
|
||||||
@@ -43,39 +28,12 @@ const eventColumns = "id, created_at, method, content_type, " +
|
|||||||
// DeliveryView and TargetView.
|
// DeliveryView and TargetView.
|
||||||
type EventLogView struct {
|
type EventLogView struct {
|
||||||
ID string
|
ID string
|
||||||
|
CreatedAt time.Time
|
||||||
Method string
|
Method string
|
||||||
ContentType string
|
ContentType string
|
||||||
|
|
||||||
// Received is how long ago the event arrived, and ReceivedUTC
|
|
||||||
// the full timestamp.
|
|
||||||
Received string
|
|
||||||
ReceivedUTC string
|
|
||||||
|
|
||||||
Body BodyView
|
Body BodyView
|
||||||
|
|
||||||
// Entrypoint names the entrypoint the event arrived at. A
|
|
||||||
// resubmitted copy, even a copy of a copy, did not arrive; it
|
|
||||||
// names the one the request it copies arrived at. The name is
|
|
||||||
// the entrypoint's description, "Entrypoint" when it has none,
|
|
||||||
// or "deleted entrypoint", never its URL, which is the
|
|
||||||
// entrypoint's secret.
|
|
||||||
Entrypoint string
|
|
||||||
|
|
||||||
// RawQuery is the query string the event arrived with.
|
|
||||||
// RawQueryCut reports one left out, RawQuery then empty, because
|
|
||||||
// it holds more than maxRenderedBodyBytes; only the event log
|
|
||||||
// leaves it out.
|
|
||||||
RawQuery string
|
|
||||||
RawQueryCut bool
|
|
||||||
|
|
||||||
// Headers is the event's request headers as text, one
|
|
||||||
// "Name: value" line per value, sorted by name. HeadersCut
|
|
||||||
// reports headers left out because they hold more than
|
|
||||||
// maxRenderedBodyBytes, stored or as text; only the event log
|
|
||||||
// leaves them out.
|
|
||||||
Headers string
|
|
||||||
HeadersCut bool
|
|
||||||
|
|
||||||
// ResubmittedFromID names the event this one was copied
|
// ResubmittedFromID names the event this one was copied
|
||||||
// from, empty for an event that arrived on the receiver.
|
// from, empty for an event that arrived on the receiver.
|
||||||
ResubmittedFromID string
|
ResubmittedFromID string
|
||||||
@@ -96,132 +54,39 @@ func (v EventLogView) ResubmittedFrom() bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// eventLogRow is one row of the event log projection, or of
|
// eventLogRow is one row of the event log projection, or of
|
||||||
// eventColumns. In the event log its query string, headers and
|
// eventColumns. In the event log its body column arrives
|
||||||
// body columns arrive already cut to the cap by SQLite, each with
|
// already cut to the cap by SQLite, with the true size beside
|
||||||
// its true size beside it.
|
// it.
|
||||||
type eventLogRow struct {
|
type eventLogRow struct {
|
||||||
ID string
|
ID string
|
||||||
CreatedAt time.Time
|
CreatedAt time.Time
|
||||||
Method string
|
Method string
|
||||||
ContentType string
|
ContentType string
|
||||||
ResubmittedFromID *string
|
ResubmittedFromID *string
|
||||||
EntrypointID string
|
|
||||||
RawQuery string
|
|
||||||
RawQueryBytes int64
|
|
||||||
Headers string
|
|
||||||
HeadersBytes int64
|
|
||||||
Body []byte
|
Body []byte
|
||||||
BodyBytes int64
|
BodyBytes int64
|
||||||
}
|
}
|
||||||
|
|
||||||
// view projects a loaded row of the webhook's events for
|
// view projects a loaded row of the webhook's events for
|
||||||
// rendering. It shows the request headers when the row holds them
|
// rendering.
|
||||||
// whole and their text holds at most maxHeaderBytes.
|
func (r *eventLogRow) view(webhookID string) EventLogView {
|
||||||
func (r *eventLogRow) view(
|
|
||||||
webhookID string, maxHeaderBytes int,
|
|
||||||
) EventLogView {
|
|
||||||
var from string
|
var from string
|
||||||
if r.ResubmittedFromID != nil {
|
if r.ResubmittedFromID != nil {
|
||||||
from = *r.ResubmittedFromID
|
from = *r.ResubmittedFromID
|
||||||
}
|
}
|
||||||
|
|
||||||
headers, fit := requestHeaderLines(r.Headers, maxHeaderBytes)
|
|
||||||
|
|
||||||
rawQuery := r.RawQuery
|
|
||||||
rawQueryCut := r.RawQueryBytes > int64(len(rawQuery))
|
|
||||||
|
|
||||||
if rawQueryCut {
|
|
||||||
rawQuery = ""
|
|
||||||
}
|
|
||||||
|
|
||||||
return EventLogView{
|
return EventLogView{
|
||||||
ID: r.ID,
|
ID: r.ID,
|
||||||
|
CreatedAt: r.CreatedAt,
|
||||||
Method: r.Method,
|
Method: r.Method,
|
||||||
ContentType: r.ContentType,
|
ContentType: r.ContentType,
|
||||||
Received: humanize.Time(r.CreatedAt),
|
|
||||||
ReceivedUTC: r.CreatedAt.UTC().Format(time.DateTime) + " UTC",
|
|
||||||
Body: newBodyView(
|
Body: newBodyView(
|
||||||
"/hook/"+webhookID+"/events/"+r.ID, r.Body, r.BodyBytes,
|
"/hook/"+webhookID+"/events/"+r.ID, r.Body, r.BodyBytes,
|
||||||
),
|
),
|
||||||
RawQuery: rawQuery,
|
|
||||||
RawQueryCut: rawQueryCut,
|
|
||||||
Headers: strings.Join(headers, "\n"),
|
|
||||||
HeadersCut: !fit || r.HeadersBytes > int64(len(r.Headers)),
|
|
||||||
ResubmittedFromID: from,
|
ResubmittedFromID: from,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// requestHeaderLines turns an event's stored request headers, the
|
|
||||||
// JSON the receiver writes, into one "Name: value" line per value,
|
|
||||||
// sorted by name. Headers that do not parse, as when the event log
|
|
||||||
// has cut them, show as none. It reports false, with no lines, when
|
|
||||||
// the lines, each with the newline that follows it, would hold more
|
|
||||||
// than maxBytes: a header sent many times is stored with its name
|
|
||||||
// once but shown with it on every line.
|
|
||||||
func requestHeaderLines(headersJSON string, maxBytes int) ([]string, bool) {
|
|
||||||
var headers http.Header
|
|
||||||
|
|
||||||
if json.Unmarshal([]byte(headersJSON), &headers) != nil {
|
|
||||||
return nil, true
|
|
||||||
}
|
|
||||||
|
|
||||||
names := make([]string, 0, len(headers))
|
|
||||||
for name := range headers {
|
|
||||||
names = append(names, name)
|
|
||||||
}
|
|
||||||
|
|
||||||
slices.Sort(names)
|
|
||||||
|
|
||||||
var lines []string
|
|
||||||
|
|
||||||
size := 0
|
|
||||||
|
|
||||||
for _, name := range names {
|
|
||||||
for _, value := range headers[name] {
|
|
||||||
line := name + ": " + value
|
|
||||||
|
|
||||||
size += len(line) + len("\n")
|
|
||||||
if size > maxBytes {
|
|
||||||
return nil, false
|
|
||||||
}
|
|
||||||
|
|
||||||
lines = append(lines, line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return lines, true
|
|
||||||
}
|
|
||||||
|
|
||||||
// entrypointNames maps each of the webhook's entrypoints to the name
|
|
||||||
// an event that arrived at it shows: its description, or "Entrypoint"
|
|
||||||
// when it has none, as the webhook page names it. A deleted
|
|
||||||
// entrypoint is left out.
|
|
||||||
func (h *Handlers) entrypointNames(
|
|
||||||
webhookID string,
|
|
||||||
) (map[string]string, error) {
|
|
||||||
var entrypoints []database.Entrypoint
|
|
||||||
|
|
||||||
err := h.db.DB().Where(
|
|
||||||
"webhook_id = ?", webhookID,
|
|
||||||
).Find(&entrypoints).Error
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
names := make(map[string]string, len(entrypoints))
|
|
||||||
|
|
||||||
for i := range entrypoints {
|
|
||||||
name := entrypoints[i].Description
|
|
||||||
if name == "" {
|
|
||||||
name = "Entrypoint"
|
|
||||||
}
|
|
||||||
|
|
||||||
names[entrypoints[i].ID] = name
|
|
||||||
}
|
|
||||||
|
|
||||||
return names, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// trimPartialRune drops a trailing UTF-8 sequence that the
|
// trimPartialRune drops a trailing UTF-8 sequence that the
|
||||||
// byte-wise cut left incomplete, so a multi-byte rune severed
|
// byte-wise cut left incomplete, so a multi-byte rune severed
|
||||||
// at the cap does not surface as a mojibake tail.
|
// at the cap does not surface as a mojibake tail.
|
||||||
|
|||||||
@@ -75,7 +75,9 @@ func seedAndProject(
|
|||||||
wh := seedWebhook(t, db)
|
wh := seedWebhook(t, db)
|
||||||
seedEventWithBody(t, dbMgr, wh.ID, body)
|
seedEventWithBody(t, dbMgr, wh.ID, body)
|
||||||
|
|
||||||
views := h.LoadEventLogViewsForTest(httptest.NewRecorder(), *wh)
|
views := h.LoadEventLogViewsForTest(
|
||||||
|
httptest.NewRecorder(), *wh, 1,
|
||||||
|
)
|
||||||
require.Len(t, views, 1)
|
require.Len(t, views, 1)
|
||||||
|
|
||||||
return views[0]
|
return views[0]
|
||||||
|
|||||||
@@ -1,404 +0,0 @@
|
|||||||
package handlers_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/go-chi/chi"
|
|
||||||
"github.com/google/uuid"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"gorm.io/gorm/clause"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// arrivedAt is how a page names the entrypoint an event arrived at.
|
|
||||||
func arrivedAt(name string) string {
|
|
||||||
return `Arrived at <span class="text-gray-900 wrap-anywhere">` + name +
|
|
||||||
`</span>`
|
|
||||||
}
|
|
||||||
|
|
||||||
// copiedRequestArrivedAt is how a page names, for a resubmitted copy,
|
|
||||||
// the entrypoint the request it copies arrived at.
|
|
||||||
func copiedRequestArrivedAt(name string) string {
|
|
||||||
return `The request it copies arrived at ` +
|
|
||||||
`<span class="text-gray-900 wrap-anywhere">` + name + `</span>`
|
|
||||||
}
|
|
||||||
|
|
||||||
// headerBox is how a page shows an event's request header lines: as
|
|
||||||
// one block of text in a single box.
|
|
||||||
func headerBox(lines ...string) string {
|
|
||||||
return `<pre class="rounded-md border border-gray-200 bg-white p-2 ` +
|
|
||||||
`text-xs text-gray-700 overflow-x-auto whitespace-pre-wrap ` +
|
|
||||||
`break-all">` + strings.Join(lines, "\n") + `</pre>`
|
|
||||||
}
|
|
||||||
|
|
||||||
// showHeadersLink is the event log's link to an event's own page for
|
|
||||||
// request headers it leaves out.
|
|
||||||
func showHeadersLink(webhookID, eventID string) string {
|
|
||||||
return `<a href="/hook/` + webhookID + `/events/` + eventID +
|
|
||||||
`" class="btn-small">Show the request headers</a>`
|
|
||||||
}
|
|
||||||
|
|
||||||
// entrypoint records one of the fixture webhook's entrypoints.
|
|
||||||
func (f *recentEventsFixture) entrypoint(
|
|
||||||
t *testing.T, description string,
|
|
||||||
) *database.Entrypoint {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
ep := &database.Entrypoint{
|
|
||||||
WebhookID: f.webhook.ID,
|
|
||||||
Path: uuid.NewString(),
|
|
||||||
Description: description,
|
|
||||||
Active: true,
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NoError(t, f.db.DB().Omit(clause.Associations).Create(ep).Error)
|
|
||||||
|
|
||||||
return ep
|
|
||||||
}
|
|
||||||
|
|
||||||
// eventAt records an event that arrived at the entrypoint with the
|
|
||||||
// given request headers, stored as JSON as the receiver stores them.
|
|
||||||
func (f *recentEventsFixture) eventAt(
|
|
||||||
t *testing.T,
|
|
||||||
ep *database.Entrypoint,
|
|
||||||
headersJSON string,
|
|
||||||
receivedAt time.Time,
|
|
||||||
) *database.Event {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
event := &database.Event{
|
|
||||||
WebhookID: f.webhook.ID,
|
|
||||||
EntrypointID: ep.ID,
|
|
||||||
Method: http.MethodPost,
|
|
||||||
Headers: headersJSON,
|
|
||||||
Body: "{}",
|
|
||||||
BodyBytes: 2,
|
|
||||||
ContentType: contentTypeJSON,
|
|
||||||
}
|
|
||||||
event.CreatedAt = receivedAt
|
|
||||||
|
|
||||||
require.NoError(t, f.webhookDB.Omit(
|
|
||||||
clause.Associations,
|
|
||||||
).Create(event).Error)
|
|
||||||
|
|
||||||
return event
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEventRequest_EachEventShowsItsOwnEntrypointAndHeaders proves two
|
|
||||||
// events that arrived at two entrypoints each show their own
|
|
||||||
// entrypoint and request headers, in the event log and on their own
|
|
||||||
// pages, with the headers sorted by name, escaped and keeping their
|
|
||||||
// whitespace, and never the entrypoint's URL.
|
|
||||||
func TestEventRequest_EachEventShowsItsOwnEntrypointAndHeaders(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
f := newRecentEventsFixture(t)
|
|
||||||
billing := f.entrypoint(t, "Billing sender")
|
|
||||||
unnamed := f.entrypoint(t, "")
|
|
||||||
|
|
||||||
// Stored in reverse name order.
|
|
||||||
older := f.eventAt(t, billing,
|
|
||||||
`{"X-Shop-Event":["order.created"],`+
|
|
||||||
`"User-Agent":["shop/1 build\t7"],"Accept":["*/*"]}`,
|
|
||||||
time.Now().Add(-time.Minute))
|
|
||||||
newer := f.eventAt(t, unnamed,
|
|
||||||
`{"X-Shop-Event":["order.paid"],"X-Note":["<b>hi</b>"]}`,
|
|
||||||
time.Now())
|
|
||||||
|
|
||||||
olderShows := func(t *testing.T, page string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
assert.Contains(t, page, arrivedAt("Billing sender"))
|
|
||||||
assert.Contains(t, page, "No query string.")
|
|
||||||
assert.Contains(t, page, headerBox(
|
|
||||||
"Accept: */*",
|
|
||||||
"User-Agent: shop/1 build\t7",
|
|
||||||
"X-Shop-Event: order.created",
|
|
||||||
), "headers are sorted by name")
|
|
||||||
assert.NotContains(t, page, "order.paid")
|
|
||||||
assert.NotContains(t, page, billing.Path)
|
|
||||||
}
|
|
||||||
|
|
||||||
newerShows := func(t *testing.T, page string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
assert.Contains(t, page, arrivedAt("Entrypoint"))
|
|
||||||
assert.Contains(t, page, headerBox(
|
|
||||||
"X-Note: <b>hi</b>",
|
|
||||||
"X-Shop-Event: order.paid",
|
|
||||||
))
|
|
||||||
assert.NotContains(t, page, "<b>hi</b>")
|
|
||||||
assert.NotContains(t, page, "order.created")
|
|
||||||
assert.NotContains(t, page, unnamed.Path)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The log lists the newer event first, so everything between
|
|
||||||
// the two events' first mentions belongs to the newer one.
|
|
||||||
_, rest, found := strings.Cut(renderSourceLogsPage(
|
|
||||||
t, f.h, f.sess, f.webhook.ID,
|
|
||||||
), newer.ID)
|
|
||||||
require.True(t, found)
|
|
||||||
|
|
||||||
newerPart, olderPart, found := strings.Cut(rest, older.ID)
|
|
||||||
require.True(t, found)
|
|
||||||
|
|
||||||
newerShows(t, newerPart)
|
|
||||||
olderShows(t, olderPart)
|
|
||||||
|
|
||||||
w := serveEventPage(t, f.h, f.sess, f.webhook.ID, newer.ID)
|
|
||||||
require.Equal(t, http.StatusOK, w.Code)
|
|
||||||
newerShows(t, w.Body.String())
|
|
||||||
|
|
||||||
w = serveEventPage(t, f.h, f.sess, f.webhook.ID, older.ID)
|
|
||||||
require.Equal(t, http.StatusOK, w.Code)
|
|
||||||
olderShows(t, w.Body.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEventRequest_DeletedEntrypoint proves an event whose entrypoint
|
|
||||||
// has since been deleted says so in the event log and on its own page.
|
|
||||||
func TestEventRequest_DeletedEntrypoint(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
f := newRecentEventsFixture(t)
|
|
||||||
ep := f.entrypoint(t, "Retired sender")
|
|
||||||
event := f.eventAt(t, ep, `{}`, time.Now())
|
|
||||||
|
|
||||||
require.NoError(t, f.db.DB().Delete(ep).Error)
|
|
||||||
|
|
||||||
page := renderSourceLogsPage(t, f.h, f.sess, f.webhook.ID)
|
|
||||||
assert.Contains(t, page, arrivedAt("deleted entrypoint"))
|
|
||||||
assert.NotContains(t, page, "Retired sender")
|
|
||||||
|
|
||||||
w := serveEventPage(t, f.h, f.sess, f.webhook.ID, event.ID)
|
|
||||||
require.Equal(t, http.StatusOK, w.Code)
|
|
||||||
assert.Contains(t, w.Body.String(), arrivedAt("deleted entrypoint"))
|
|
||||||
assert.NotContains(t, w.Body.String(), "Retired sender")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEventRequest_ResubmittedCopy proves a resubmitted copy and a copy
|
|
||||||
// of that copy each say the request they copy arrived at the
|
|
||||||
// entrypoint, in the event log and on their own pages, and never that
|
|
||||||
// they did.
|
|
||||||
func TestEventRequest_ResubmittedCopy(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
f := newRecentEventsFixture(t)
|
|
||||||
ep := f.entrypoint(t, "Billing sender")
|
|
||||||
original := f.eventAt(t, ep, `{}`, time.Now().Add(-2*time.Minute))
|
|
||||||
copied := f.eventAt(t, ep, `{}`, time.Now().Add(-time.Minute))
|
|
||||||
copyOfCopy := f.eventAt(t, ep, `{}`, time.Now())
|
|
||||||
|
|
||||||
require.NoError(t, f.webhookDB.Model(copied).Update(
|
|
||||||
"resubmitted_from_id", original.ID,
|
|
||||||
).Error)
|
|
||||||
require.NoError(t, f.webhookDB.Model(copyOfCopy).Update(
|
|
||||||
"resubmitted_from_id", copied.ID,
|
|
||||||
).Error)
|
|
||||||
|
|
||||||
// The log lists the newest event first, and each event's Resubmit
|
|
||||||
// form comes before its entrypoint, so cutting the page at the
|
|
||||||
// copy's and the original's forms leaves each event's entrypoint
|
|
||||||
// in its own part.
|
|
||||||
copyOfCopyPart, rest, found := strings.Cut(
|
|
||||||
renderSourceLogsPage(t, f.h, f.sess, f.webhook.ID),
|
|
||||||
"/events/"+copied.ID+"/resubmit",
|
|
||||||
)
|
|
||||||
require.True(t, found)
|
|
||||||
|
|
||||||
copyPart, originalPart, found := strings.Cut(
|
|
||||||
rest, "/events/"+original.ID+"/resubmit",
|
|
||||||
)
|
|
||||||
require.True(t, found)
|
|
||||||
|
|
||||||
for _, part := range []string{copyOfCopyPart, copyPart} {
|
|
||||||
assert.Contains(t, part, copiedRequestArrivedAt("Billing sender"))
|
|
||||||
assert.NotContains(t, part, arrivedAt("Billing sender"))
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Contains(t, originalPart, arrivedAt("Billing sender"))
|
|
||||||
assert.NotContains(t, originalPart,
|
|
||||||
copiedRequestArrivedAt("Billing sender"))
|
|
||||||
|
|
||||||
for _, event := range []*database.Event{copied, copyOfCopy} {
|
|
||||||
w := serveEventPage(t, f.h, f.sess, f.webhook.ID, event.ID)
|
|
||||||
require.Equal(t, http.StatusOK, w.Code)
|
|
||||||
assert.Contains(t, w.Body.String(),
|
|
||||||
copiedRequestArrivedAt("Billing sender"))
|
|
||||||
assert.NotContains(t, w.Body.String(), arrivedAt("Billing sender"))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEventRequest_HeadersOverTheLimit proves the event log leaves out
|
|
||||||
// request headers that hold more than it shows of a body, whether
|
|
||||||
// stored or as lines, and links to the event's own page, which shows
|
|
||||||
// them all.
|
|
||||||
func TestEventRequest_HeadersOverTheLimit(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// The receiver stores each "<" as six bytes of JSON, so this
|
|
||||||
// header is over the limit stored but not as a line.
|
|
||||||
const lessThans = bodyCap/6 + 1
|
|
||||||
|
|
||||||
// A header sent many times is stored with its name once, and
|
|
||||||
// shown with it on every line.
|
|
||||||
repeatedName := "X-Repeated-" + strings.Repeat("r", 1000)
|
|
||||||
|
|
||||||
tests := map[string]struct {
|
|
||||||
headers http.Header
|
|
||||||
line string
|
|
||||||
}{
|
|
||||||
"stored": {
|
|
||||||
headers: http.Header{"X-Long": {strings.Repeat("<", lessThans)}},
|
|
||||||
line: "X-Long: " + strings.Repeat("<", lessThans),
|
|
||||||
},
|
|
||||||
"as lines": {
|
|
||||||
headers: http.Header{repeatedName: slices.Repeat([]string{""}, 41)},
|
|
||||||
line: repeatedName + ": ",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for name, tc := range tests {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
headersJSON, err := json.Marshal(tc.headers)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
f := newRecentEventsFixture(t)
|
|
||||||
ep := f.entrypoint(t, "Billing sender")
|
|
||||||
event := f.eventAt(t, ep, string(headersJSON), time.Now())
|
|
||||||
|
|
||||||
page := renderSourceLogsPage(t, f.h, f.sess, f.webhook.ID)
|
|
||||||
assert.Contains(t, page, showHeadersLink(f.webhook.ID, event.ID))
|
|
||||||
assert.NotContains(t, page, tc.line)
|
|
||||||
assert.Less(t, len(page), 4*bodyCap)
|
|
||||||
|
|
||||||
w := serveEventPage(t, f.h, f.sess, f.webhook.ID, event.ID)
|
|
||||||
require.Equal(t, http.StatusOK, w.Code)
|
|
||||||
assert.Contains(t, w.Body.String(), tc.line)
|
|
||||||
assert.NotContains(t, w.Body.String(), "Show the request headers")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEventRequest_ManyShortHeaderLines proves that for many short
|
|
||||||
// request header lines the event log writes no more than its limit,
|
|
||||||
// apart from escaping: lines that fill the limit show as one block of
|
|
||||||
// text, and one line more is left out with a link to the event's own
|
|
||||||
// page.
|
|
||||||
func TestEventRequest_ManyShortHeaderLines(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Each "A: " line and the newline after it hold four bytes, so
|
|
||||||
// this many lines fill the limit exactly. Each line in its own
|
|
||||||
// element would make the page many times the limit.
|
|
||||||
const fill = bodyCap / len("A: \n")
|
|
||||||
|
|
||||||
tests := map[string]struct {
|
|
||||||
lines int
|
|
||||||
shown bool
|
|
||||||
}{
|
|
||||||
"filling the limit": {lines: fill, shown: true},
|
|
||||||
"one over the limit": {lines: fill + 1, shown: false},
|
|
||||||
}
|
|
||||||
|
|
||||||
for name, tc := range tests {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
headersJSON, err := json.Marshal(http.Header{
|
|
||||||
"A": slices.Repeat([]string{""}, tc.lines),
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
f := newRecentEventsFixture(t)
|
|
||||||
ep := f.entrypoint(t, "Billing sender")
|
|
||||||
event := f.eventAt(t, ep, string(headersJSON), time.Now())
|
|
||||||
|
|
||||||
page := renderSourceLogsPage(t, f.h, f.sess, f.webhook.ID)
|
|
||||||
box := headerBox(slices.Repeat([]string{"A: "}, tc.lines)...)
|
|
||||||
link := showHeadersLink(f.webhook.ID, event.ID)
|
|
||||||
|
|
||||||
assert.Equal(t, tc.shown, strings.Contains(page, box))
|
|
||||||
assert.Equal(t, !tc.shown, strings.Contains(page, link))
|
|
||||||
assert.Less(t, len(page), 4*bodyCap)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleWebhook_StoresAndShowsTheQueryString posts to an
|
|
||||||
// entrypoint's URL with a query string and proves the event stores it
|
|
||||||
// as sent, and shows it escaped in the event log and on its own page,
|
|
||||||
// in a box like the one the request headers show in.
|
|
||||||
func TestHandleWebhook_StoresAndShowsTheQueryString(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
f := newRecentEventsFixture(t)
|
|
||||||
ep := seedEntrypoint(t, f.db, f.webhook.ID)
|
|
||||||
|
|
||||||
req := httptest.NewRequestWithContext(
|
|
||||||
context.Background(), http.MethodPost,
|
|
||||||
"/h/"+ep.Path+"?a=1&b=2", strings.NewReader("{}"),
|
|
||||||
)
|
|
||||||
|
|
||||||
rctx := chi.NewRouteContext()
|
|
||||||
rctx.URLParams.Add("uuid", ep.Path)
|
|
||||||
|
|
||||||
req = req.WithContext(context.WithValue(
|
|
||||||
req.Context(), chi.RouteCtxKey, rctx,
|
|
||||||
))
|
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
f.h.HandleWebhook().ServeHTTP(w, req)
|
|
||||||
|
|
||||||
require.Equal(t, http.StatusOK, w.Code)
|
|
||||||
|
|
||||||
var stored database.Event
|
|
||||||
|
|
||||||
require.NoError(t, f.webhookDB.First(&stored).Error)
|
|
||||||
assert.Equal(t, "a=1&b=2", stored.RawQuery)
|
|
||||||
|
|
||||||
page := renderSourceLogsPage(t, f.h, f.sess, f.webhook.ID)
|
|
||||||
assert.Contains(t, page, headerBox("a=1&b=2"))
|
|
||||||
|
|
||||||
w = serveEventPage(t, f.h, f.sess, f.webhook.ID, stored.ID)
|
|
||||||
require.Equal(t, http.StatusOK, w.Code)
|
|
||||||
assert.Contains(t, w.Body.String(), headerBox("a=1&b=2"))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEventRequest_QueryStringOverTheLimit proves the event log leaves
|
|
||||||
// out a query string that holds more than it shows of a body, and links
|
|
||||||
// to the event's own page, which shows it whole.
|
|
||||||
func TestEventRequest_QueryStringOverTheLimit(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
f := newRecentEventsFixture(t)
|
|
||||||
ep := f.entrypoint(t, "Billing sender")
|
|
||||||
event := f.eventAt(t, ep, `{}`, time.Now())
|
|
||||||
query := "q=" + strings.Repeat("x", bodyCap)
|
|
||||||
|
|
||||||
require.NoError(t, f.webhookDB.Model(event).Update(
|
|
||||||
"raw_query", query,
|
|
||||||
).Error)
|
|
||||||
|
|
||||||
page := renderSourceLogsPage(t, f.h, f.sess, f.webhook.ID)
|
|
||||||
assert.Contains(t, page, `<a href="/hook/`+f.webhook.ID+`/events/`+
|
|
||||||
event.ID+`" class="btn-small">Show the query string</a>`)
|
|
||||||
assert.NotContains(t, page, "q=x")
|
|
||||||
assert.Less(t, len(page), 4*bodyCap)
|
|
||||||
|
|
||||||
w := serveEventPage(t, f.h, f.sess, f.webhook.ID, event.ID)
|
|
||||||
require.Equal(t, http.StatusOK, w.Code)
|
|
||||||
assert.Contains(t, w.Body.String(), headerBox(query))
|
|
||||||
assert.NotContains(t, w.Body.String(), "Show the query string")
|
|
||||||
}
|
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user