Compare commits

..
Author SHA1 Message Date
sneak 0a112fad71 Keep make test under 2 GB of memory (closes #344)
check / check (push) Waiting to run
Every test that starts a database hashed the bootstrap admin password
with Argon2id at 64 MB, which under -race holds about 150 MB per hash,
across dozens of parallel tests. The test packages that hash now lower
the memory cost to 1 MB from TestMain. One test still hashes and
verifies at the shipped parameters, which are unchanged.

script/test also runs at most four packages and eight parallel tests
at once: unbounded, a many-core host linked and ran every test binary
together.

Model: opus-5-5
2026-10-01 19:49:01 +00:00
39 changed files with 476 additions and 264 deletions
+4 -2
View File
@@ -3,8 +3,10 @@
# stage of the Dockerfile. # stage of the Dockerfile.
.git/ .git/
bin/ bin/
# Extracted from 3p/ by `make assets` inside the build; a host copy is not # Third-party browser assets are fetched and hash-verified inside the build by
# needed. The tarball in 3p/ must stay in the context. # script/fetch-assets. Excluding any host copy keeps a developer's working tree
# from supplying the bytes that get shipped. The script and its
# static/vendor.sha256 manifest stay in the context.
static/js/alpine.min.js static/js/alpine.min.js
*.md *.md
LICENSE LICENSE
+4 -3
View File
@@ -46,6 +46,7 @@ temp/
# CI cache barrier, written into the build context by the check workflow # CI cache barrier, written into the build context by the check workflow
.ci-fingerprint .ci-fingerprint
# Alpine.js, extracted by `make assets` from its tarball in 3p/, which is # Third-party browser assets, fetched and hash-verified by
# what is committed. # script/fetch-assets against static/vendor.sha256. Not committed:
/static/js/alpine.min.js # REPO_POLICIES.md forbids minified bundles in version control.
/static/js/alpine.min.js
Binary file not shown.
+11 -4
View File
@@ -51,8 +51,15 @@ RUN go mod download
# the lint stage above. # the lint stage above.
COPY . . COPY . .
# Run tests and build. Both first run script/assets, which extracts Alpine.js # Fetch the third-party browser assets the UI serves. They are not committed
# from its tarball in 3p/. # (REPO_POLICIES.md forbids minified bundles in version control) and
# .dockerignore keeps any host copy out of the build context, so this step is
# the only way they enter the image. Each download is checked against a
# hardcoded sha256 and the build fails on mismatch; make test re-checks the
# hashes against the bytes go:embed actually put in the binary.
RUN script/fetch-assets
# Run tests and build
RUN make test RUN make test
# Version stamped into the binary. .dockerignore excludes .git/, so # Version stamped into the binary. .dockerignore excludes .git/, so
@@ -60,8 +67,8 @@ RUN make test
# host and passes it in. The default is what a bare `docker build .` # host and passes it in. The default is what a bare `docker build .`
# with no --build-arg gets, and it names no tag the tree may not be at. # with no --build-arg gets, and it names no tag the tree may not be at.
# #
# Declared here, below the test step, so a changed version does not # Declared here, below the test and asset steps, so a changed version
# invalidate its cached layer. # does not invalidate their cached layers.
ARG VERSION=unknown ARG VERSION=unknown
RUN make build VERSION="$VERSION" RUN make build VERSION="$VERSION"
+3 -3
View File
@@ -28,7 +28,7 @@ setup:
@script/setup @script/setup
assets: assets:
@script/assets @script/fetch-assets
test: test:
@script/test @script/test
@@ -45,13 +45,13 @@ fmt-check:
check: check:
@script/check @script/check
build: assets build:
go build -ldflags '$(strip -X main.version=$(VERSION) $(GO_LDFLAGS))' -o bin/webhooker ./cmd/webhooker go build -ldflags '$(strip -X main.version=$(VERSION) $(GO_LDFLAGS))' -o bin/webhooker ./cmd/webhooker
run: build run: build
./bin/webhooker ./bin/webhooker
dev: assets dev:
go run ./cmd/webhooker go run ./cmd/webhooker
deps: deps:
+42 -40
View File
@@ -21,6 +21,9 @@ before deploying one.
- Go 1.26.1+ (the version in `go.mod`) - Go 1.26.1+ (the version in `go.mod`)
- Docker (for linting, for the test stage of the CI gate, and for - Docker (for linting, for the test stage of the CI gate, and for
containerized deployment) containerized deployment)
- `curl`, used by `script/fetch-assets` to download the third-party
browser assets, which are not committed (`make bootstrap` installs
it if missing)
golangci-lint is not a prerequisite and must not be installed on the golangci-lint is not a prerequisite and must not be installed on the
host: `script/bootstrap` does not install it, and `make lint` runs the host: `script/bootstrap` does not install it, and `make lint` runs the
@@ -33,7 +36,9 @@ digest-pinned linter image via `Dockerfile.lint`.
git clone https://git.eeqj.de/sneak/webhooker.git git clone https://git.eeqj.de/sneak/webhooker.git
cd webhooker cd webhooker
# Install the Go toolchain if missing, and the Go dependencies # Install Go dependencies and the third-party browser assets.
# `make deps` alone is not enough: it only runs go mod download/tidy,
# and the checks below need the fetched assets.
make bootstrap make bootstrap
# Run all checks (test, lint, format check) # Run all checks (test, lint, format check)
@@ -53,7 +58,7 @@ make docker
```bash ```bash
make bootstrap # Install all dependencies (idempotent) make bootstrap # Install all dependencies (idempotent)
make setup # Bootstrap + install git pre-commit hook make setup # Bootstrap + install git pre-commit hook
make assets # Extract Alpine.js from 3p/ (test, check, build, dev run it) make assets # Fetch + verify third-party browser assets
make fmt # Format code (gofmt + goimports) make fmt # Format code (gofmt + goimports)
make fmt-check # Fail if gofmt would change anything (writes nothing) make fmt-check # Fail if gofmt would change anything (writes nothing)
make lint # Run golangci-lint in Docker (Dockerfile.lint) make lint # Run golangci-lint in Docker (Dockerfile.lint)
@@ -1216,15 +1221,14 @@ This repository adheres to the
standard: normalized scripts in `script/` are the entrypoints for the standard: normalized scripts in `script/` are the entrypoints for the
development workflow. Ten of the Makefile's seventeen targets are thin development workflow. Ten of the Makefile's seventeen targets are thin
shims that call them; `build`, `run`, `dev`, `deps`, `clean`, `css` and shims that call them; `build`, `run`, `dev`, `deps`, `clean`, `css` and
`version` are inline commands with no script behind them, though `build`, `version` are inline commands with no script behind them, though
`run` and `dev` first run `script/assets`, and `build` and `version` both `build` and `version` both take their value from `script/version`.
take their value from `script/version`.
`script/test`, `make build` and `make dev` each run `script/assets` `make check` needs the third-party browser assets in `static/`, which
first, which writes the ignored `static/js/alpine.min.js` (see are not committed, so run `make bootstrap` (or just `make assets`) once
[Third-party browser assets](#third-party-browser-assets)), so after cloning. Without them the tests fail with a message naming that
`make test`, `make check` and the pre-commit hook work on a fresh clone remedy. `make check` does not fetch them itself because it must not
without a separate step. change any files in the repo.
We provide: We provide:
@@ -1232,8 +1236,8 @@ We provide:
- `script/setup` — make a fresh clone ready for development - `script/setup` — make a fresh clone ready for development
(bootstrap, then install-precommit) (bootstrap, then install-precommit)
- `script/projectname` — output the project name ("webhooker") - `script/projectname` — output the project name ("webhooker")
- `script/assets` — extract Alpine.js from its tarball in `3p/` (see - `script/fetch-assets` — download the third-party browser assets into
[Third-party browser assets](#third-party-browser-assets)) `static/`, verifying each against its pinned sha256
- `script/test` — run the test suite - `script/test` — run the test suite
- `script/lint` — run golangci-lint in Docker (see Linting below) - `script/lint` — run golangci-lint in Docker (see Linting below)
- `script/fmt` — format all code (writes) - `script/fmt` — format all code (writes)
@@ -1255,25 +1259,24 @@ We provide:
## Third-party browser assets ## Third-party browser assets
The web UI serves one third-party script, Alpine.js. Its npm package tarball The web UI serves one third-party script, Alpine.js. It is **not** committed:
is committed as `3p/alpinejs-3.14.9.tgz`, byte for byte as the npm registry a minified bundle in the tree is unreviewable, and `REPO_POLICIES.md` bars
publishes it. It is a dependency, not this repo's build output, so both committed build artifacts and unpinned external references.
`REPO_POLICIES.md`'s rule against committed build artifacts does not apply.
The directory is `3p/` rather than `vendor/` because Go treats a root
`vendor/` directory as its module vendor directory.
`script/assets` (`make assets`) extracts the browser build, Instead `script/fetch-assets` downloads it from a pinned URL, checks the
`package/dist/cdn.min.js`, from the tarball to `static/js/alpine.min.js`, download against a hardcoded sha256, and installs it under `static/`. The
where `go:embed` picks it up. `script/test`, `make build` and `make dev` run sha256 of every installed asset is recorded in `static/vendor.sha256`, and
it first, and the Dockerfile builds through `make test` and `make build`, so `static/vendor_test.go` re-hashes the bytes `go:embed` put in the binary
nothing downloads Alpine.js. The extracted file is not committed, and against that manifest — so the pin is enforced on what actually ships, not
`.dockerignore` keeps any host copy out of the build context. merely written down. Any mismatch fails the build.
To move to a new version: download `make bootstrap` runs the fetch for local development, and the Dockerfile
`https://registry.npmjs.org/alpinejs/-/alpinejs-<version>.tgz`, check it runs it in the build stage; `.gitignore` and `.dockerignore` keep the
against the `dist.integrity` hash listed at artifact out of both the repo and the build context.
`https://registry.npmjs.org/alpinejs/<version>`, replace the tarball in `3p/`
with it, update its file name in `script/assets`, and run `make check`. To move to a new version: update the version, URL, and tarball sha256 in
`script/fetch-assets` and the asset sha256 in `static/vendor.sha256`, then
run `make assets && make check`.
## Rationale ## Rationale
@@ -2752,8 +2755,6 @@ imports. The entry point is `cmd/webhooker/main.go`.
``` ```
webhooker/ webhooker/
├── 3p/
│ └── alpinejs-3.14.9.tgz # Alpine.js npm package, extracted by make assets
├── cmd/webhooker/ ├── cmd/webhooker/
│ └── main.go # Entry point: subcommand dispatch; no args locks DATA_DIR and wires fx │ └── main.go # Entry point: subcommand dispatch; no args locks DATA_DIR and wires fx
├── internal/ ├── internal/
@@ -2847,7 +2848,8 @@ webhooker/
│ ├── css/tailwind.css # Generated stylesheet the pages load │ ├── css/tailwind.css # Generated stylesheet the pages load
│ ├── css/style.css # Older hand-written stylesheet, no longer loaded │ ├── css/style.css # Older hand-written stylesheet, no longer loaded
│ ├── js/app.js # Progressive-enhancement copy-to-clipboard │ ├── js/app.js # Progressive-enhancement copy-to-clipboard
│ └── js/alpine.min.js # Alpine.js, extracted from 3p/ by make assets, not committed │ ├── js/alpine.min.js # Alpine.js, fetched by script/fetch-assets, not committed
│ └── vendor.sha256 # Pinned hashes the fetched assets are verified against
├── templates/ # Go HTML templates (base, login, sources, etc.) ├── templates/ # Go HTML templates (base, login, sources, etc.)
├── script/ # Scripts to Rule Them All entrypoints ├── script/ # Scripts to Rule Them All entrypoints
├── Dockerfile # Three stages: lint, test+build, Alpine runtime ├── Dockerfile # Three stages: lint, test+build, Alpine runtime
@@ -3159,14 +3161,14 @@ version is fixed independently of the compiler's:
`make fmt-check`, then `golangci-lint config verify` and `make fmt-check`, then `golangci-lint config verify` and
`golangci-lint run`, both with `--network=none`. `golangci-lint run`, both with `--network=none`.
2. **Builder stage** (`golang:1.26.1-bookworm`) — depends on the lint 2. **Builder stage** (`golang:1.26.1-bookworm`) — depends on the lint
stage passing (it copies a file from it), runs `make test` and stage passing (it copies a file from it), runs `script/fetch-assets`
`make build` (both extract Alpine.js from `3p/` first), and finally to download and verify the third-party browser assets, then runs
rebuilds the binary with `CGO_ENABLED=1` and static linking so it `make test` and `make build`, and finally rebuilds the binary with
runs on musl. Both builds go through `make build`, the relink adding `CGO_ENABLED=1` and static linking so it runs on musl. Both builds
its `-extldflags` via `GO_LDFLAGS`, so neither can drop the `-X` that go through `make build`, the relink adding its `-extldflags` via
stamps the version. The version arrives as the `VERSION` build arg, `GO_LDFLAGS`, so neither can drop the `-X` that stamps the version.
since the context has no `.git` (see The version arrives as the `VERSION` build arg, since the context
[Version stamping](#version-stamping)). has no `.git` (see [Version stamping](#version-stamping)).
3. **Runtime stage** (`alpine:3.21`) — copies the static binary and 3. **Runtime stage** (`alpine:3.21`) — copies the static binary and
`deploy/docker-entrypoint.sh`, creates the `/var/lib/webhooker` `deploy/docker-entrypoint.sh`, creates the `/var/lib/webhooker`
directory for all SQLite databases, exposes port 8080, and includes directory for all SQLite databases, exposes port 8080, and includes
-5
View File
@@ -16,7 +16,6 @@ import (
"sneak.berlin/go/webhooker/internal/handlers" "sneak.berlin/go/webhooker/internal/handlers"
"sneak.berlin/go/webhooker/internal/healthcheck" "sneak.berlin/go/webhooker/internal/healthcheck"
"sneak.berlin/go/webhooker/internal/logger" "sneak.berlin/go/webhooker/internal/logger"
"sneak.berlin/go/webhooker/internal/metrics"
"sneak.berlin/go/webhooker/internal/middleware" "sneak.berlin/go/webhooker/internal/middleware"
"sneak.berlin/go/webhooker/internal/resetpw" "sneak.berlin/go/webhooker/internal/resetpw"
"sneak.berlin/go/webhooker/internal/server" "sneak.berlin/go/webhooker/internal/server"
@@ -178,10 +177,6 @@ func newApp() *fx.App {
healthcheck.New, healthcheck.New,
session.New, session.New,
handlers.New, handlers.New,
// The registry /metrics serves, and the delivery
// collectors registered on it.
metrics.NewRegistry,
metrics.New,
middleware.New, middleware.New,
// The one SSRF guard both target-creation validation // The one SSRF guard both target-creation validation
// and the delivery dialer consult, so they cannot // and the delivery dialer consult, so they cannot
+6
View File
@@ -79,3 +79,9 @@ func (d *Database) ExportSetBannerOut(w io.Writer) {
func DummyPasswordHashForTest() string { func DummyPasswordHashForTest() string {
return dummyPasswordHash() return dummyPasswordHash()
} }
// HashPasswordWithDefaultsForTest hashes with the shipped Argon2id
// parameters, which TestMain has lowered for HashPassword itself.
func HashPasswordWithDefaultsForTest(password string) (string, error) {
return hashPasswordWith(password, DefaultPasswordConfig())
}
+14
View File
@@ -0,0 +1,14 @@
package database_test
import (
"testing"
"sneak.berlin/go/webhooker/internal/database"
)
// TestMain lowers the password hashing cost before any test runs. See
// database.LowerPasswordHashCostForTest.
func TestMain(m *testing.M) {
database.LowerPasswordHashCostForTest()
m.Run()
}
+15 -1
View File
@@ -63,10 +63,24 @@ func DefaultPasswordConfig() *PasswordConfig {
} }
} }
// hashConfig is what HashPassword hashes with: the defaults above.
// Only a test binary changes it, through LowerPasswordHashCostForTest,
// before any test runs.
//
//nolint:gochecknoglobals // see above
var hashConfig = DefaultPasswordConfig()
// HashPassword generates an Argon2id hash of the password // HashPassword generates an Argon2id hash of the password
func HashPassword(password string) (string, error) { func HashPassword(password string) (string, error) {
config := DefaultPasswordConfig() return hashPasswordWith(password, hashConfig)
}
// hashPasswordWith generates an Argon2id hash of the password with
// the given parameters.
func hashPasswordWith(
password string,
config *PasswordConfig,
) (string, error) {
// Generate a salt // Generate a salt
salt := make([]byte, config.SaltLen) salt := make([]byte, config.SaltLen)
+30
View File
@@ -192,6 +192,36 @@ func TestHashPasswordUniqueness(t *testing.T) {
} }
} }
// TestHashPassword_ShippedParameters hashes and verifies at the
// shipped Argon2id parameters. Every other test hashes at the lowered
// memory cost TestMain sets, so this is the one that keeps production
// hashing covered. One hash and one verification: each costs 64 MB.
func TestHashPassword_ShippedParameters(t *testing.T) {
t.Parallel()
password := "correct horse battery staple"
hash, err := database.HashPasswordWithDefaultsForTest(password)
if err != nil {
t.Fatalf("hashing with the shipped parameters: %v", err)
}
const shipped = "$argon2id$v=19$m=65536,t=1,p=4$"
if !strings.HasPrefix(hash, shipped) {
t.Errorf("hash = %q, want prefix %q", hash, shipped)
}
valid, err := database.VerifyPassword(password, hash)
if err != nil {
t.Fatalf("VerifyPassword() error = %v", err)
}
if !valid {
t.Error("VerifyPassword() returned false for correct password")
}
}
// TestVerifyDummyPassword_DoesRealWork covers the anti-enumeration // TestVerifyDummyPassword_DoesRealWork covers the anti-enumeration
// path. Login charges an unknown username a verification against a // path. Login charges an unknown username a verification against a
// dummy hash so that a nonexistent account is not answered in // dummy hash so that a nonexistent account is not answered in
+17
View File
@@ -45,3 +45,20 @@ func NewTestWebhookDBManagerWithLogger(
log: log, log: log,
} }
} }
// testArgon2Memory is the Argon2id memory cost, in KiB, that test
// binaries hash with: 1 MB instead of the shipped 64 MB.
const testArgon2Memory = 1024
// LowerPasswordHashCostForTest makes HashPassword use a 1 MB Argon2id
// memory cost instead of the shipped 64 MB, for the rest of the
// process. Call it from TestMain, before any test runs.
//
// Every test that starts a database hashes the bootstrap admin
// password, and a package runs dozens of those tests in parallel.
// Under the race detector each 64 MB hash holds about 150 MB resident.
// VerifyPassword reads the cost from the hash it checks, so
// verification follows. Production code never calls this.
func LowerPasswordHashCostForTest() {
hashConfig.Memory = testArgon2Memory
}
+5 -6
View File
@@ -148,7 +148,6 @@ type EngineParams struct {
DBManager *database.WebhookDBManager DBManager *database.WebhookDBManager
Logger *logger.Logger Logger *logger.Logger
SSRFGuard *Guard SSRFGuard *Guard
Metrics *metrics.Set
} }
// Engine processes queued deliveries in the background // Engine processes queued deliveries in the background
@@ -168,10 +167,10 @@ type Engine struct {
retryCh chan Task retryCh chan Task
workers int workers int
// mtr is the delivery metric set. Production wires the one // mtr is the delivery metric set. Production wires the
// registered on the registry /metrics serves; a test can // process-wide one; a test can substitute a set registered on
// substitute a set registered on a registry it holds, so it can // a private registry so its assertions are not disturbed by
// gather what its own deliveries recorded. // deliveries other tests are making at the same time.
mtr *metrics.Set mtr *metrics.Set
// targets maps each target type to its implementation. // targets maps each target type to its implementation.
@@ -205,7 +204,7 @@ func New(
deliveryCh: make(chan Task, deliveryChannelSize), deliveryCh: make(chan Task, deliveryChannelSize),
retryCh: make(chan Task, retryChannelSize), retryCh: make(chan Task, retryChannelSize),
workers: defaultWorkers, workers: defaultWorkers,
mtr: params.Metrics, mtr: metrics.Default(),
} }
e.initTargets(&http.Client{ e.initTargets(&http.Client{
+5 -5
View File
@@ -9,7 +9,6 @@ import (
"net/url" "net/url"
"time" "time"
"github.com/prometheus/client_golang/prometheus"
"go.uber.org/fx" "go.uber.org/fx"
"gorm.io/gorm" "gorm.io/gorm"
"sneak.berlin/go/webhooker/internal/database" "sneak.berlin/go/webhooker/internal/database"
@@ -390,7 +389,7 @@ func NewTestEngine(
deliveryCh: make(chan Task, deliveryChannelSize), deliveryCh: make(chan Task, deliveryChannelSize),
retryCh: make(chan Task, retryChannelSize), retryCh: make(chan Task, retryChannelSize),
workers: workers, workers: workers,
mtr: metrics.New(prometheus.NewRegistry()), mtr: metrics.Default(),
} }
e.initTargets(client) e.initTargets(client)
@@ -405,7 +404,7 @@ func NewTestEngineSmallRetry(
e := &Engine{ e := &Engine{
log: log, log: log,
retryCh: make(chan Task, 1), retryCh: make(chan Task, 1),
mtr: metrics.New(prometheus.NewRegistry()), mtr: metrics.Default(),
} }
e.initTargets(nil) e.initTargets(nil)
@@ -428,7 +427,7 @@ func NewTestEngineWithDB(
deliveryCh: make(chan Task, deliveryChannelSize), deliveryCh: make(chan Task, deliveryChannelSize),
retryCh: make(chan Task, retryChannelSize), retryCh: make(chan Task, retryChannelSize),
workers: workers, workers: workers,
mtr: metrics.New(prometheus.NewRegistry()), mtr: metrics.Default(),
} }
e.initTargets(client) e.initTargets(client)
@@ -436,7 +435,8 @@ func NewTestEngineWithDB(
} }
// ExportSetMetrics substitutes the engine's metric set, so a test can // ExportSetMetrics substitutes the engine's metric set, so a test can
// assert on collectors registered on a registry it holds. // assert on collectors registered on a private registry instead of
// the process-wide ones every other test is also moving.
func (e *Engine) ExportSetMetrics(mtr *metrics.Set) { func (e *Engine) ExportSetMetrics(mtr *metrics.Set) {
e.mtr = mtr e.mtr = mtr
} }
+3 -2
View File
@@ -35,8 +35,9 @@ const (
) )
// mIsolate gives the setup's engine a metric set registered on a // mIsolate gives the setup's engine a metric set registered on a
// registry this test holds, so its exact assertions can gather from // private registry. The process-wide collectors are moved by every
// it. // other delivery test running in parallel, so exact assertions are
// only possible against a registry this test owns.
func mIsolate( func mIsolate(
t *testing.T, s iSetup, t *testing.T, s iSetup,
) *prometheus.Registry { ) *prometheus.Registry {
+14
View File
@@ -0,0 +1,14 @@
package gormlog_test
import (
"testing"
"sneak.berlin/go/webhooker/internal/database"
)
// TestMain lowers the password hashing cost before any test runs. See
// database.LowerPasswordHashCostForTest.
func TestMain(m *testing.M) {
database.LowerPasswordHashCostForTest()
m.Run()
}
+1 -4
View File
@@ -12,7 +12,6 @@ import (
"net/http" "net/http"
"sync/atomic" "sync/atomic"
"github.com/prometheus/client_golang/prometheus"
"go.uber.org/fx" "go.uber.org/fx"
"sneak.berlin/go/webhooker/internal/database" "sneak.berlin/go/webhooker/internal/database"
"sneak.berlin/go/webhooker/internal/delivery" "sneak.berlin/go/webhooker/internal/delivery"
@@ -62,8 +61,6 @@ type HandlersParams struct {
Notifier delivery.Notifier Notifier delivery.Notifier
Evictor delivery.WebhookEvictor Evictor delivery.WebhookEvictor
SSRFGuard *delivery.Guard SSRFGuard *delivery.Guard
Metrics *metrics.Set
Registry *prometheus.Registry
} }
// Handlers provides HTTP handler methods for all application // Handlers provides HTTP handler methods for all application
@@ -125,7 +122,7 @@ func New(
s.mw = params.Middleware s.mw = params.Middleware
s.notifier = params.Notifier s.notifier = params.Notifier
s.evictor = params.Evictor s.evictor = params.Evictor
s.mtr = params.Metrics s.mtr = metrics.Default()
s.ssrf = params.SSRFGuard s.ssrf = params.SSRFGuard
// Parse all page templates once at startup // Parse all page templates once at startup
-3
View File
@@ -20,7 +20,6 @@ import (
"sneak.berlin/go/webhooker/internal/handlers" "sneak.berlin/go/webhooker/internal/handlers"
"sneak.berlin/go/webhooker/internal/healthcheck" "sneak.berlin/go/webhooker/internal/healthcheck"
"sneak.berlin/go/webhooker/internal/logger" "sneak.berlin/go/webhooker/internal/logger"
"sneak.berlin/go/webhooker/internal/metrics"
"sneak.berlin/go/webhooker/internal/middleware" "sneak.berlin/go/webhooker/internal/middleware"
"sneak.berlin/go/webhooker/internal/session" "sneak.berlin/go/webhooker/internal/session"
) )
@@ -110,8 +109,6 @@ func newTestApp(
func(r *recordingEvictor) delivery.WebhookEvictor { func(r *recordingEvictor) delivery.WebhookEvictor {
return r return r
}, },
metrics.NewRegistry,
metrics.New,
middleware.New, middleware.New,
delivery.NewGuard, delivery.NewGuard,
handlers.New, handlers.New,
+14
View File
@@ -0,0 +1,14 @@
package handlers_test
import (
"testing"
"sneak.berlin/go/webhooker/internal/database"
)
// TestMain lowers the password hashing cost before any test runs. See
// database.LowerPasswordHashCostForTest.
func TestMain(m *testing.M) {
database.LowerPasswordHashCostForTest()
m.Run()
}
-21
View File
@@ -1,21 +0,0 @@
package handlers
import (
"net/http"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
// HandleMetrics returns the Prometheus scrape handler for the
// registry built by metrics.NewRegistry, which the HTTP, delivery, Go
// runtime and process collectors register on. It is what
// promhttp.Handler builds for the global default registry, including
// the promhttp_metric_handler_* series that count scrapes, pointed at
// that registry instead.
func (s *Handlers) HandleMetrics() http.HandlerFunc {
reg := s.params.Registry
return promhttp.InstrumentMetricHandler(
reg, promhttp.HandlerFor(reg, promhttp.HandlerOpts{}),
).ServeHTTP
}
+20 -27
View File
@@ -3,18 +3,17 @@
// deliveries are attempted, how they end, how long they take, how // deliveries are attempted, how they end, how long they take, how
// deep the queues are, and how many circuit breakers are open. // deep the queues are, and how many circuit breakers are open.
// //
// It also builds the registry the authenticated /metrics route // The inbound HTTP metrics come from the go-http-metrics recorder in
// serves. In production, these collectors, the inbound HTTP metrics // internal/middleware and land on prometheus.DefaultRegisterer. These
// recorded in internal/middleware, and the Go runtime and process // collectors register there too, so both surfaces are gathered by the
// collectors all register on that one registry, never on Prometheus's // one promhttp handler mounted on the authenticated /metrics route.
// global default.
package metrics package metrics
import ( import (
"sync"
"time" "time"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/collectors"
"github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promauto"
"sneak.berlin/go/webhooker/internal/database" "sneak.berlin/go/webhooker/internal/database"
) )
@@ -58,31 +57,25 @@ var knownTargetTypes = []database.TargetType{
database.TargetTypeSlack, database.TargetTypeSlack,
} }
// NewRegistry returns the registry /metrics serves, carrying the Go // defaultSet is the process-wide metric set, registered on the same
// runtime and process collectors that Prometheus's global default // registry the HTTP middleware and the /metrics handler already use.
// registry carries, so the go_* and process_* series stay in the // It is built on first use rather than in an init so that a test
// scrape. // binary that never touches metrics never registers them.
// //
// A registry of its own, rather than the global default, is what lets //nolint:gochecknoglobals // one process-wide registration, by design
// two dependency graphs in one process — two tests, say — each var defaultSet = sync.OnceValue(func() *Set {
// register their collectors without the second registration return New(prometheus.DefaultRegisterer)
// panicking. })
func NewRegistry() *prometheus.Registry {
reg := prometheus.NewRegistry()
reg.MustRegister(
collectors.NewGoCollector(),
collectors.NewProcessCollector(
collectors.ProcessCollectorOpts{},
),
)
return reg // Default returns the process-wide metric set.
func Default() *Set {
return defaultSet()
} }
// Set is one registered group of webhooker's delivery collectors. // Set is one registered group of webhooker's delivery collectors.
// Production builds one on the registry /metrics serves; tests build // Production uses the single Default set; tests build their own
// their own against a private registry so assertions are not // against a private registry so assertions are not disturbed by
// disturbed by deliveries other tests are making concurrently. // deliveries other tests are making concurrently.
type Set struct { type Set struct {
eventsReceived prometheus.Counter eventsReceived prometheus.Counter
deliveryAttempts *prometheus.CounterVec deliveryAttempts *prometheus.CounterVec
@@ -100,7 +93,7 @@ type Set struct {
// New registers a full set of delivery collectors on reg and returns // New registers a full set of delivery collectors on reg and returns
// it. It panics if reg already holds them, which is the intended // it. It panics if reg already holds them, which is the intended
// behaviour for a duplicate registration. // behaviour for a duplicate registration.
func New(reg *prometheus.Registry) *Set { func New(reg prometheus.Registerer) *Set {
factory := promauto.With(reg) factory := promauto.With(reg)
s := &Set{ s := &Set{
+2 -1
View File
@@ -10,7 +10,8 @@ import (
// MetricsMiddlewareForTest builds the metrics recording middleware // MetricsMiddlewareForTest builds the metrics recording middleware
// against a caller-supplied recorder, so a test can gather from its // against a caller-supplied recorder, so a test can gather from its
// own Prometheus registry without building a whole Middleware. // own Prometheus registry rather than the process-wide default one
// that Middleware.Metrics uses.
func MetricsMiddlewareForTest( func MetricsMiddlewareForTest(
rec httpmetrics.Recorder, rec httpmetrics.Recorder,
) func(http.Handler) http.Handler { ) func(http.Handler) http.Handler {
+8 -7
View File
@@ -7,6 +7,7 @@ import (
"github.com/go-chi/chi" "github.com/go-chi/chi"
httpmetrics "github.com/slok/go-http-metrics/metrics" httpmetrics "github.com/slok/go-http-metrics/metrics"
prommetrics "github.com/slok/go-http-metrics/metrics/prometheus"
ghmm "github.com/slok/go-http-metrics/middleware" ghmm "github.com/slok/go-http-metrics/middleware"
"github.com/slok/go-http-metrics/middleware/std" "github.com/slok/go-http-metrics/middleware/std"
) )
@@ -150,17 +151,17 @@ func (r boundedLabelRecorder) AddInflightRequests(
var _ httpmetrics.Recorder = boundedLabelRecorder{} var _ httpmetrics.Recorder = boundedLabelRecorder{}
// Metrics returns middleware that records Prometheus HTTP metrics // Metrics returns middleware that records Prometheus HTTP metrics on
// with the Middleware's one recorder, which New builds on the registry // the default registry, which is the one the /metrics route gathers.
// the /metrics route serves and NewForTest on a registry of its own.
// Every call reuses that recorder, so any number of routers can
// install it.
func (s *Middleware) Metrics() func(http.Handler) http.Handler { func (s *Middleware) Metrics() func(http.Handler) http.Handler {
return metricsMiddleware(s.metricsRecorder) return metricsMiddleware(
prommetrics.NewRecorder(prommetrics.Config{}),
)
} }
// metricsMiddleware builds the recording middleware against a given // metricsMiddleware builds the recording middleware against a given
// recorder, so tests can gather from a registry of their own. // recorder, so tests can gather from a registry of their own instead
// of the process-wide default.
func metricsMiddleware( func metricsMiddleware(
rec httpmetrics.Recorder, rec httpmetrics.Recorder,
) func(http.Handler) http.Handler { ) func(http.Handler) http.Handler {
+3 -28
View File
@@ -57,8 +57,9 @@ const (
// Server.setupWebhookRoutes inside it. That ordering is the whole // Server.setupWebhookRoutes inside it. That ordering is the whole
// defect, so a test that flattens it would prove nothing. // defect, so a test that flattens it would prove nothing.
// //
// The recorder writes to a registry of the test's own, so each test // The recorder writes to a registry of the test's own rather than the
// observes only its own traffic. // process-wide default one, so each test observes only its own
// traffic.
func metricsTestRouter( func metricsTestRouter(
t *testing.T, t *testing.T,
receiverLimit int, receiverLimit int,
@@ -454,29 +455,3 @@ func TestMetrics_StatusAndSizeStillRecorded(t *testing.T) {
"the interceptor must still count written bytes", "the interceptor must still count written bytes",
) )
} }
// TestMetrics_WorksOnNewForTestMiddleware pins that a Middleware built
// by NewForTest has a recorder of its own: its Metrics() serves a
// request instead of panicking, and a second one does not collide
// with the first.
func TestMetrics_WorksOnNewForTestMiddleware(t *testing.T) {
t.Parallel()
log := slog.New(slog.DiscardHandler)
cfg := &config.Config{Environment: "prod"}
ok := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte(okBody))
})
for range 2 {
h := middleware.NewForTest(log, cfg, nil).Metrics()(ok)
req := httptest.NewRequestWithContext(
t.Context(), http.MethodGet, okRoute, nil,
)
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
assert.Equal(t, http.StatusOK, w.Code)
}
}
+4 -19
View File
@@ -13,9 +13,6 @@ import (
"github.com/go-chi/chi" "github.com/go-chi/chi"
"github.com/go-chi/chi/middleware" "github.com/go-chi/chi/middleware"
"github.com/go-chi/cors" "github.com/go-chi/cors"
"github.com/prometheus/client_golang/prometheus"
httpmetrics "github.com/slok/go-http-metrics/metrics"
prommetrics "github.com/slok/go-http-metrics/metrics/prometheus"
"go.uber.org/fx" "go.uber.org/fx"
"sneak.berlin/go/webhooker/internal/config" "sneak.berlin/go/webhooker/internal/config"
"sneak.berlin/go/webhooker/internal/globals" "sneak.berlin/go/webhooker/internal/globals"
@@ -151,11 +148,10 @@ const (
type MiddlewareParams struct { type MiddlewareParams struct {
fx.In fx.In
Logger *logger.Logger Logger *logger.Logger
Globals *globals.Globals Globals *globals.Globals
Config *config.Config Config *config.Config
Session *session.Session Session *session.Session
Registry *prometheus.Registry
} }
// Middleware provides HTTP middleware for logging, CORS, auth, and // Middleware provides HTTP middleware for logging, CORS, auth, and
@@ -165,14 +161,6 @@ type Middleware struct {
params *MiddlewareParams params *MiddlewareParams
session *session.Session session *session.Session
// metricsRecorder records the inbound HTTP metrics. New builds
// it on the registry /metrics serves, NewForTest on a registry
// of its own. Either way it is built once per Middleware and
// Metrics reuses it, because building it registers its
// collectors, and a second registration on the same registry
// panics.
metricsRecorder httpmetrics.Recorder
// loginGuard counts failed credential verifications and bounds // loginGuard counts failed credential verifications and bounds
// concurrent password hashing. It is built on first use so that // concurrent password hashing. It is built on first use so that
// every construction path gets one; see guard(). // every construction path gets one; see guard().
@@ -191,9 +179,6 @@ func New(
s.params = &params s.params = &params
s.log = params.Logger.Get() s.log = params.Logger.Get()
s.session = params.Session s.session = params.Session
s.metricsRecorder = prommetrics.NewRecorder(
prommetrics.Config{Registry: params.Registry},
)
return s, nil return s, nil
} }
-8
View File
@@ -3,17 +3,12 @@ package middleware
import ( import (
"log/slog" "log/slog"
"github.com/prometheus/client_golang/prometheus"
prommetrics "github.com/slok/go-http-metrics/metrics/prometheus"
"sneak.berlin/go/webhooker/internal/config" "sneak.berlin/go/webhooker/internal/config"
"sneak.berlin/go/webhooker/internal/session" "sneak.berlin/go/webhooker/internal/session"
) )
// NewForTest creates a Middleware with the minimum dependencies // NewForTest creates a Middleware with the minimum dependencies
// needed for testing. This bypasses the fx lifecycle. // needed for testing. This bypasses the fx lifecycle.
//
// Its metrics recorder writes to a fresh registry of its own, so
// Metrics() works on it and two of them never collide.
func NewForTest( func NewForTest(
log *slog.Logger, log *slog.Logger,
cfg *config.Config, cfg *config.Config,
@@ -25,8 +20,5 @@ func NewForTest(
Config: cfg, Config: cfg,
}, },
session: sess, session: sess,
metricsRecorder: prommetrics.NewRecorder(
prommetrics.Config{Registry: prometheus.NewRegistry()},
),
} }
} }
+14
View File
@@ -0,0 +1,14 @@
package resetpw_test
import (
"testing"
"sneak.berlin/go/webhooker/internal/database"
)
// TestMain lowers the password hashing cost before any test runs. See
// database.LowerPasswordHashCostForTest.
func TestMain(m *testing.M) {
database.LowerPasswordHashCostForTest()
m.Run()
}
+1 -4
View File
@@ -24,7 +24,6 @@ import (
"sneak.berlin/go/webhooker/internal/handlers" "sneak.berlin/go/webhooker/internal/handlers"
"sneak.berlin/go/webhooker/internal/healthcheck" "sneak.berlin/go/webhooker/internal/healthcheck"
"sneak.berlin/go/webhooker/internal/logger" "sneak.berlin/go/webhooker/internal/logger"
"sneak.berlin/go/webhooker/internal/metrics"
"sneak.berlin/go/webhooker/internal/middleware" "sneak.berlin/go/webhooker/internal/middleware"
"sneak.berlin/go/webhooker/internal/resetpw" "sneak.berlin/go/webhooker/internal/resetpw"
"sneak.berlin/go/webhooker/internal/session" "sneak.berlin/go/webhooker/internal/session"
@@ -141,7 +140,7 @@ func (n *noopEvictor) EvictWebhook(string) {}
// and the database, exactly as internal/handlers builds them. // and the database, exactly as internal/handlers builds them.
// //
// One application per test function, not per case: every start that // One application per test function, not per case: every start that
// finds no account seeds one at 64 MB of Argon2id, and this package's // finds no account seeds one with an Argon2id hash, and this package's
// budget is not the place to spend that repeatedly. // budget is not the place to spend that repeatedly.
func newServerApp( func newServerApp(
t *testing.T, dir string, t *testing.T, dir string,
@@ -164,8 +163,6 @@ func newServerApp(
session.New, session.New,
func() delivery.Notifier { return &noopNotifier{} }, func() delivery.Notifier { return &noopNotifier{} },
func() delivery.WebhookEvictor { return &noopEvictor{} }, func() delivery.WebhookEvictor { return &noopEvictor{} },
metrics.NewRegistry,
metrics.New,
middleware.New, middleware.New,
delivery.NewGuard, delivery.NewGuard,
handlers.New, handlers.New,
+14
View File
@@ -0,0 +1,14 @@
package server_test
import (
"testing"
"sneak.berlin/go/webhooker/internal/database"
)
// TestMain lowers the password hashing cost before any test runs. See
// database.LowerPasswordHashCostForTest.
func TestMain(m *testing.M) {
database.LowerPasswordHashCostForTest()
m.Run()
}
+7 -1
View File
@@ -7,6 +7,7 @@ import (
sentryhttp "github.com/getsentry/sentry-go/http" sentryhttp "github.com/getsentry/sentry-go/http"
"github.com/go-chi/chi" "github.com/go-chi/chi"
"github.com/go-chi/chi/middleware" "github.com/go-chi/chi/middleware"
"github.com/prometheus/client_golang/prometheus/promhttp"
"sneak.berlin/go/webhooker/static" "sneak.berlin/go/webhooker/static"
) )
@@ -129,7 +130,12 @@ func (s *Server) setupRoutes() {
if s.params.Config.MetricsAuthEnabled() { if s.params.Config.MetricsAuthEnabled() {
s.router.Group(func(r chi.Router) { s.router.Group(func(r chi.Router) {
r.Use(s.mw.MetricsAuth()) r.Use(s.mw.MetricsAuth())
r.Get("/metrics", s.h.HandleMetrics()) r.Get(
"/metrics",
http.HandlerFunc(
promhttp.Handler().ServeHTTP,
),
)
}) })
} }
-46
View File
@@ -24,7 +24,6 @@ import (
"sneak.berlin/go/webhooker/internal/handlers" "sneak.berlin/go/webhooker/internal/handlers"
"sneak.berlin/go/webhooker/internal/healthcheck" "sneak.berlin/go/webhooker/internal/healthcheck"
"sneak.berlin/go/webhooker/internal/logger" "sneak.berlin/go/webhooker/internal/logger"
"sneak.berlin/go/webhooker/internal/metrics"
"sneak.berlin/go/webhooker/internal/middleware" "sneak.berlin/go/webhooker/internal/middleware"
"sneak.berlin/go/webhooker/internal/server" "sneak.berlin/go/webhooker/internal/server"
"sneak.berlin/go/webhooker/internal/session" "sneak.berlin/go/webhooker/internal/session"
@@ -114,8 +113,6 @@ func newTestEnvWithConfig(
session.New, session.New,
func() delivery.Notifier { return &noopNotifier{} }, func() delivery.Notifier { return &noopNotifier{} },
func() delivery.WebhookEvictor { return &noopEvictor{} }, func() delivery.WebhookEvictor { return &noopEvictor{} },
metrics.NewRegistry,
metrics.New,
middleware.New, middleware.New,
delivery.NewGuard, delivery.NewGuard,
handlers.New, handlers.New,
@@ -1030,46 +1027,3 @@ func TestMetricsRouteUnmountedOnHalfSetConfig(t *testing.T) {
}) })
} }
} }
// TestTwoMetricsRoutersInOneProcess pins
// https://git.eeqj.de/sneak/webhooker/issues/227: a second
// metrics-enabled router in one process used to panic, because the
// HTTP metrics registered on Prometheus's global default registry.
// Two routers are built over separate dependency graphs and a third
// over the first graph again, and each must still serve the HTTP,
// delivery, Go runtime and process series, and the series counting
// scrapes of /metrics itself.
func TestTwoMetricsRoutersInOneProcess(t *testing.T) {
t.Parallel()
first := newTestEnvWithConfig(
t, metricsConfig(t, metricsUser, metricsAuthValue),
)
second := newTestEnvWithConfig(
t, metricsConfig(t, metricsUser, metricsAuthValue),
)
third := &testEnv{
router: server.NewRouterForTest(
first.log.Get(), first.cfg, first.mw, first.hnd,
),
}
for _, env := range []*testEnv{first, second, third} {
env.get("/", nil)
scrape := env.metricsRequest(metricsUser, metricsAuthValue)
require.Equal(t, http.StatusOK, scrape.Code)
for _, series := range []string{
"http_request_duration_seconds",
"http_response_size_bytes",
"http_requests_inflight",
"webhooker_events_received_total",
"go_goroutines",
"process_start_time_seconds",
"promhttp_metric_handler_requests_total",
} {
assert.Contains(t, scrape.Body.String(), series)
}
}
}
+3 -3
View File
@@ -13,9 +13,9 @@ import (
// TestBaseTemplateScriptsAreServed walks every /s/ script the base // TestBaseTemplateScriptsAreServed walks every /s/ script the base
// template loads on each page and fetches it through the real router. // template loads on each page and fetches it through the real router.
// Alpine.js is extracted from its tarball in 3p/ at build time, so the // Alpine.js is fetched at build time rather than committed, so nothing
// file is not in the tree: this is the check that the page still gets // in the repo guarantees it is present: this is the check that the page
// the JavaScript it asks for. // still gets the JavaScript it asks for.
func TestBaseTemplateScriptsAreServed(t *testing.T) { func TestBaseTemplateScriptsAreServed(t *testing.T) {
t.Parallel() t.Parallel()
-16
View File
@@ -1,16 +0,0 @@
#!/bin/sh
# script/assets: extract Alpine.js from its npm package tarball, committed
# in 3p/, to static/js/alpine.min.js, where go:embed reads it. The
# extracted file is not committed. script/test, make build and make dev run
# this first.
set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
main() {
cd "$ROOT"
tar -xzOf 3p/alpinejs-3.14.9.tgz package/dist/cdn.min.js \
>static/js/alpine.min.js
}
main "$@"
+8 -1
View File
@@ -4,7 +4,9 @@
# installed tools are skipped. Base tooling comes from nix, apt, brew, # installed tools are skipped. Base tooling comes from nix, apt, brew,
# or apk (detected in that order); assumes NOTHING is present (not git, # or apk (detected in that order); assumes NOTHING is present (not git,
# make, or go). golangci-lint is deliberately not installed: linting runs # make, or go). golangci-lint is deliberately not installed: linting runs
# only in docker, via script/lint and Dockerfile.lint. # only in docker, via script/lint and Dockerfile.lint. Finishes by running
# script/fetch-assets, which installs the hash-pinned third-party browser
# assets the repo does not commit.
set -eu set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
@@ -67,6 +69,11 @@ main() {
go mod download go mod download
# Third-party browser assets are not committed; fetch and verify them
# so a fresh clone can build and test.
if missing curl; then pkg_install curl curl curl curl; fi
"$ROOT/script/fetch-assets"
echo "bootstrap complete" echo "bootstrap complete"
} }
+1 -2
View File
@@ -1,7 +1,6 @@
#!/bin/sh #!/bin/sh
# script/check: run all checks (test, lint, fmt-check). Our own # script/check: run all checks (test, lint, fmt-check). Our own
# extension to scripts-to-rule-them-all. # extension to scripts-to-rule-them-all. Must not modify any files.
# Writes only the ignored static/js/alpine.min.js, through script/test.
# Generic: usually needs no adaptation. # Generic: usually needs no adaptation.
set -eu set -eu
+104
View File
@@ -0,0 +1,104 @@
#!/bin/sh
# script/fetch-assets: download the third-party browser assets the web UI
# ships and install them under static/. Minified bundles are not committed
# (REPO_POLICIES.md: no build artifacts in version control), so the build
# fetches them here. Every download is verified against a hardcoded sha256
# before it is installed, and any mismatch aborts. Idempotent: an asset
# already present with its pinned hash is left alone.
set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
# The sha256 of each installed asset lives in static/vendor.sha256, in
# sha256sum(1) format, with paths relative to static/. That file is the
# single source of truth: this script verifies against it, and
# static/vendor_test.go asserts the bytes embedded into the binary match
# it, so the hash cannot rot into a value nothing checks.
MANIFEST="static/vendor.sha256"
# Alpine.js 3.14.9, 2026-08-17. Fetched from registry.npmjs.org, the
# publisher of record; the jsDelivr and unpkg copies are mirrors of this
# same tarball. dist/cdn.min.js is the browser build Alpine publishes for
# a <script> tag.
ALPINE_VERSION="3.14.9"
ALPINE_URL="https://registry.npmjs.org/alpinejs/-/alpinejs-${ALPINE_VERSION}.tgz"
# sha256 of alpinejs-3.14.9.tgz
ALPINE_TARBALL_SHA256="97dad7c0c81e659cfc8e7700055da9770f8186187cb9a8a76efb57e00d5ce52a"
ALPINE_MEMBER="package/dist/cdn.min.js"
ALPINE_DEST="js/alpine.min.js"
sha256_of() {
if command -v sha256sum >/dev/null 2>&1; then
sha256sum "$1" | cut -d' ' -f1
else
shasum -a 256 "$1" | cut -d' ' -f1
fi
}
# expected_sha256 <path-relative-to-static>
expected_sha256() {
awk -v want="$1" '$2 == want { print $1; found = 1 }
END { if (!found) exit 1 }' "$ROOT/$MANIFEST"
}
# verify <file> <expected-sha256> <what>
verify() {
actual="$(sha256_of "$1")"
if [ "$actual" != "$2" ]; then
echo "fetch-assets: sha256 mismatch for $3" >&2
echo " expected: $2" >&2
echo " actual: $actual" >&2
exit 1
fi
}
# up_to_date <path-relative-to-static> <expected-sha256>
up_to_date() {
[ -f "$ROOT/static/$1" ] || return 1
[ "$(sha256_of "$ROOT/static/$1")" = "$2" ]
}
fetch_alpine() {
want="$(expected_sha256 "$ALPINE_DEST")"
if up_to_date "$ALPINE_DEST" "$want"; then
echo "fetch-assets: static/$ALPINE_DEST already at $want"
return 0
fi
echo "fetch-assets: fetching Alpine.js $ALPINE_VERSION from $ALPINE_URL"
tmp="$(mktemp -d)"
trap 'rm -rf "$tmp"' EXIT INT TERM
curl -fsSL -o "$tmp/alpine.tgz" "$ALPINE_URL"
verify "$tmp/alpine.tgz" "$ALPINE_TARBALL_SHA256" "alpinejs-${ALPINE_VERSION}.tgz"
tar -xzOf "$tmp/alpine.tgz" "$ALPINE_MEMBER" >"$tmp/alpine.min.js"
verify "$tmp/alpine.min.js" "$want" "$ALPINE_MEMBER from alpinejs-${ALPINE_VERSION}.tgz"
mkdir -p "$(dirname "$ROOT/static/$ALPINE_DEST")"
cp "$tmp/alpine.min.js" "$ROOT/static/$ALPINE_DEST"
rm -rf "$tmp"
trap - EXIT INT TERM
echo "fetch-assets: installed static/$ALPINE_DEST ($want)"
}
# Re-check every manifest entry against what is now on disk, so an entry
# no script installs fails loudly instead of passing silently.
verify_manifest() {
while read -r want path; do
case "$want" in '' | '#'*) continue ;; esac
if [ ! -f "$ROOT/static/$path" ]; then
echo "fetch-assets: $MANIFEST lists static/$path, which is missing" >&2
exit 1
fi
verify "$ROOT/static/$path" "$want" "static/$path"
done <"$ROOT/$MANIFEST"
}
main() {
cd "$ROOT"
fetch_alpine
verify_manifest
echo "fetch-assets: all assets in $MANIFEST verified"
}
main "$@"
+6 -2
View File
@@ -22,14 +22,18 @@
# The one figure above 90s is GOMAXPROCS 1, a synthetic core floor rather than # The one figure above 90s is GOMAXPROCS 1, a synthetic core floor rather than
# a condition CI runs under. If a CPU-limited runner ever puts a real run near # a condition CI runs under. If a CPU-limited runner ever puts a real run near
# 67s, that is the datum to revisit the org figure with. # 67s, that is the datum to revisit the org figure with.
#
# -p 4 -parallel 8 keep the run under 2 GB of memory: at most four test
# binaries build or run at once, each with at most eight parallel tests. Under
# -race every test binary and every link costs a few hundred MB, so the
# defaults (one per core) add up to several GB on a many-core host.
set -eu set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
main() { main() {
cd "$ROOT" cd "$ROOT"
"$ROOT/script/assets" go test -v -race -p 4 -parallel 8 -timeout 90s ./...
go test -v -race -timeout 90s ./...
} }
main "$@" main "$@"
+1
View File
@@ -0,0 +1 @@
3ed1eed252488921df65e363d6715deb04d7f92aaedb9e52199fdf73cb1e0ad3 js/alpine.min.js
+92
View File
@@ -0,0 +1,92 @@
package static_test
import (
"bufio"
"crypto/sha256"
"encoding/hex"
"os"
"strings"
"testing"
"github.com/stretchr/testify/require"
"sneak.berlin/go/webhooker/static"
)
const manifestPath = "vendor.sha256"
// fetchHint is appended to every failure here: the assets the manifest
// covers are fetched by the build, not committed, so a fresh clone that
// has not run script/fetch-assets fails this test and should be told why.
const fetchHint = "run `script/fetch-assets` (or `make assets`) to install " +
"the pinned third-party assets"
// TestVendoredAssetsMatchManifest asserts that every asset listed in
// static/vendor.sha256 is embedded in the binary with exactly the pinned
// bytes. script/fetch-assets verifies the same hashes at download time;
// this test verifies them again on what actually ships, so a build that
// skipped, cached, or subverted the fetch cannot produce a binary serving
// unpinned third-party JavaScript.
func TestVendoredAssetsMatchManifest(t *testing.T) {
t.Parallel()
entries := readManifest(t)
require.NotEmpty(t, entries, "%s lists no assets", manifestPath)
for path, want := range entries {
t.Run(path, func(t *testing.T) {
t.Parallel()
data, err := static.Static.ReadFile(path)
require.NoErrorf(
t, err,
"%s is listed in %s but is not embedded; %s",
path, manifestPath, fetchHint,
)
sum := sha256.Sum256(data)
got := hex.EncodeToString(sum[:])
require.Equalf(
t, want, got,
"embedded %s does not match its pinned sha256 in %s; %s",
path, manifestPath, fetchHint,
)
})
}
}
// readManifest parses static/vendor.sha256, which is in sha256sum(1)
// format with paths relative to static/.
func readManifest(t *testing.T) map[string]string {
t.Helper()
f, err := os.Open(manifestPath)
require.NoError(t, err, "opening %s", manifestPath)
defer func() { require.NoError(t, f.Close()) }()
entries := make(map[string]string)
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" || strings.HasPrefix(line, "#") {
continue
}
fields := strings.Fields(line)
require.Lenf(
t, fields, 2,
"%s: malformed entry %q, want \"<sha256> <path>\"",
manifestPath, line,
)
sum, path := fields[0], fields[1]
require.Lenf(t, sum, 64, "%s: %q is not a sha256", manifestPath, sum)
entries[path] = sum
}
require.NoError(t, scanner.Err(), "reading %s", manifestPath)
return entries
}