From 81d4153e78238afe6092ea3010456d6d1a6e832d Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 04:58:47 +0200 Subject: [PATCH] Serve Prometheus metrics at /metrics behind basic auth (closes #94) With METRICS_USERNAME and METRICS_PASSWORD both set, the backend records request metrics through go-http-metrics in a registry of its own, with Go's runtime and process metrics, and serves them at GET /metrics behind basic auth; nginx passes /metrics to it. With neither set there is no such route; one alone, or a METRICS_USERNAME containing ":", stops the start with an error naming the setting. Only requests that reach the health check or POST /api/v1/reports are recorded, as the labels are path and method, which clients could otherwise make up without end; POST /api/v1/reports is registered by its full path for that. Deviation: go get and go mod tidy ran directly; no entrypoint added a Go dependency yet (issue #45). Model: opus-5-5 --- Dockerfile | 4 +- README.md | 12 ++- TODO.md | 13 ++++ backend/README.md | 42 ++++++++--- backend/go.mod | 16 +++- backend/go.sum | 46 +++++++++--- backend/internal/config/config.go | 21 ++++++ backend/internal/config/config_test.go | 26 +++++++ backend/internal/middleware/middleware.go | 27 +++++++ backend/internal/server/routes.go | 37 ++++++++-- backend/internal/server/routes_test.go | 89 +++++++++++++++++++++++ nginx.conf | 6 ++ 12 files changed, 306 insertions(+), 33 deletions(-) diff --git a/Dockerfile b/Dockerfile index 66a7d5c..19e1591 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # The one image netwatch ships: nginx serves the built frontend and -# passes /api/ and /.well-known/healthcheck to netwatch-server, the Go -# backend, which runs in the same container on loopback only. +# passes /api/, /.well-known/healthcheck and /metrics to netwatch-server, +# the Go backend, which runs in the same container on loopback only. # bin/entrypoint.sh starts and watches both. # Lint stage — fast feedback on formatting and lint issues. The diff --git a/README.md b/README.md index b9ab277..b4e4de7 100644 --- a/README.md +++ b/README.md @@ -201,9 +201,9 @@ static file host (S3, GCS, Cloudflare Pages, Vercel, Netlify, GitHub Pages) or use the Docker image behind a reverse proxy. The Docker image, built from `Dockerfile`, is the whole service in one -container: nginx serves the built frontend and passes `/api/` and -`/.well-known/healthcheck` to the Go backend, `netwatch-server`, which listens -only inside the container, on `127.0.0.1:8081`. The image: +container: nginx serves the built frontend and passes `/api/`, +`/.well-known/healthcheck` and `/metrics` to the Go backend, `netwatch-server`, +which listens only inside the container, on `127.0.0.1:8081`. The image: - Listens on port 8080 by default (override with `PORT` env var) - Takes the client address from `X-Forwarded-For` only on requests from the @@ -251,6 +251,12 @@ What the [upaas](https://git.eeqj.de/sneak/upaas) app for netwatch needs: connects from one can write its own `X-Forwarded-For`, and through a port Docker publishes, every client may connect from the Docker network's gateway, such as `172.17.0.1`. + - `METRICS_USERNAME` and `METRICS_PASSWORD`, default empty: with both set, + the backend records Prometheus metrics of its requests and serves them at + `/metrics` on the container port, to requests with this user name and + password as their basic auth credentials. With neither set, there are no + metrics and `/metrics` is not found. One set without the other, or a user + name containing `:`, stops the container - **Health check:** the image's `HEALTHCHECK` requests `/.well-known/healthcheck` through nginx every 30 seconds, so it fails unless both nginx and the backend answer. upaas reads the container's health 60 diff --git a/TODO.md b/TODO.md index e234b46..5f4e5c6 100644 --- a/TODO.md +++ b/TODO.md @@ -23,6 +23,19 @@ latest run passes. # Completed Steps +- 2026-10-04: the backend serves Prometheus metrics (issue #94). With + `METRICS_USERNAME` and `METRICS_PASSWORD` both set, it records request + duration and response size through `go-http-metrics` and serves them, with + Go's runtime and process metrics, at `GET /metrics` behind basic auth with + those credentials; nginx passes `/metrics` to it as it does `/api/`. With + neither set there are no metrics and `/metrics` is 404; one without the other + stops the start with an error naming both, and so does a `METRICS_USERNAME` + containing `:`, with an error naming it. Only requests that reach the health + check or `POST /api/v1/reports` are recorded, not `/metrics` itself and not + every request as `GO_HTTP_SERVER_CONVENTIONS.md` shows, because the labels are + the request's path and method, which clients can make up without end. For + that, `POST /api/v1/reports` is now registered by its full path instead of + inside a `/api/v1` route group; it answers as before - 2026-10-04: `script/` and `Makefile` follow the org models (issue #28): `make dev` shims to the new `script/dev`, the Vite dev server, and the new `make build` to `script/build`, the frontend production build. diff --git a/backend/README.md b/backend/README.md index a9498d2..80ca89f 100644 --- a/backend/README.md +++ b/backend/README.md @@ -88,6 +88,8 @@ project layout: | `TRUSTED_PROXIES` | loopback + RFC1918 | Comma-separated CIDRs whose `X-Forwarded-For` / `X-Real-IP` headers are trusted for client IP resolution | | `REPORTS_PER_MINUTE` | `60` | Reports each client address may send a minute; see [Report limits](#report-limits) | | `CORS_ALLOWED_ORIGINS` | empty | Comma-separated origins whose pages may call the API; see [CORS](#cors) | +| `METRICS_USERNAME` | empty | Basic auth user name for `/metrics`; see [Metrics](#metrics) | +| `METRICS_PASSWORD` | empty | Basic auth password for `/metrics`; see [Metrics](#metrics) | `TRUSTED_PROXIES` defaults to `127.0.0.1/32,::1/128,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16`. The loopback @@ -103,16 +105,16 @@ starting, with an error naming the variable. An empty variable counts as unset. ### Container image The root `Dockerfile` builds one image in which nginx listens on the public port -8080, serves the frontend, and proxies `/api/` and `/.well-known/healthcheck` to -this server. The image's entrypoint, `bin/entrypoint.sh`, starts the server as -user `netwatch` (uid 1000) with `BIND_ADDRESS=127.0.0.1` and `PORT=8081`, so -only nginx reaches it, and with `TRUSTED_PROXIES=127.0.0.1/32`, so it takes the -client address nginx passes on and no other. `DATA_DIR` is `/data/reports`, on -the `/data` volume; before starting the server, the entrypoint creates it and -gives it and `/data` to `netwatch` with `netwatch-server prepare-data-dir`, -which acts on nothing outside `/data`. nginx replaces the security headers this -server sets with those in the root `security-headers.conf`, so those are what -clients of the image see. +8080, serves the frontend, and proxies `/api/`, `/.well-known/healthcheck` and +`/metrics` to this server. The image's entrypoint, `bin/entrypoint.sh`, starts +the server as user `netwatch` (uid 1000) with `BIND_ADDRESS=127.0.0.1` and +`PORT=8081`, so only nginx reaches it, and with `TRUSTED_PROXIES=127.0.0.1/32`, +so it takes the client address nginx passes on and no other. `DATA_DIR` is +`/data/reports`, on the `/data` volume; before starting the server, the +entrypoint creates it and gives it and `/data` to `netwatch` with +`netwatch-server prepare-data-dir`, which acts on nothing outside `/data`. nginx +replaces the security headers this server sets with those in the root +`security-headers.conf`, so those are what clients of the image see. The container's own `TRUSTED_PROXIES` goes to nginx instead: IP addresses or CIDRs, separated by commas, of the reverse proxies in front of the container. @@ -176,6 +178,26 @@ origin, `scheme://host` with an optional `:port`, as browsers send it: no path, not even a trailing `/`, and no `*`. Any other entry stops the server from starting, with an error naming `CORS_ALLOWED_ORIGINS`. +### Metrics + +With both `METRICS_USERNAME` and `METRICS_PASSWORD` set, the server serves +Prometheus metrics at `GET /metrics` to requests with those as their basic auth +credentials, and answers any other with 401. For each request that reaches the +health check or `POST /api/v1/reports`, those the rate limit refuses included, +the metrics record its duration and response size, labelled with its path, +method and status; they also count those requests in progress, and include Go's +runtime and process metrics. No other request is recorded: not those to +`/metrics` itself, and not those answered before they reach either route, such +as a CORS preflight, or a request refused with 404 for a path no route has, 405 +for a method its route does not take, or 413 for declaring a body length over +the 1 MiB limit. A report whose body goes over the limit without declaring its +length reaches the route, is answered 413 there, and is recorded with that +status. Clients can make up any number of paths and methods, and each would add +labels to the metrics for as long as the server runs. With neither set, nothing +is recorded and `/metrics` answers 404. One without the other stops the server +from starting, with an error naming both; so does a `METRICS_USERNAME` +containing `:`, which basic auth cannot carry, with an error naming it. + ## TODO - Add integration test that POSTs a report and verifies the compressed output diff --git a/backend/go.mod b/backend/go.mod index a2462e8..881081a 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -3,20 +3,29 @@ module sneak.berlin/go/netwatch go 1.25.5 require ( + github.com/99designs/basicauth-go v0.0.0-20230316000542-bf6f9cbbf0f8 github.com/go-chi/chi/v5 v5.2.5 github.com/go-chi/cors v1.2.2 github.com/go-chi/httprate v0.16.0 github.com/joho/godotenv v1.5.1 - github.com/klauspost/compress v1.18.4 + github.com/klauspost/compress v1.19.1 + github.com/prometheus/client_golang v1.24.1 + github.com/slok/go-http-metrics v0.13.0 github.com/spf13/viper v1.21.0 go.uber.org/fx v1.24.0 ) require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/klauspost/cpuid/v2 v2.2.10 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.70.1 // indirect + github.com/prometheus/procfs v0.21.1 // indirect github.com/sagikazarmark/locafero v0.11.0 // indirect github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect github.com/spf13/afero v1.15.0 // indirect @@ -28,6 +37,7 @@ require ( go.uber.org/multierr v1.10.0 // indirect go.uber.org/zap v1.26.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/sys v0.30.0 // indirect - golang.org/x/text v0.28.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.40.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect ) diff --git a/backend/go.sum b/backend/go.sum index cac5e22..ea56244 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -1,3 +1,9 @@ +github.com/99designs/basicauth-go v0.0.0-20230316000542-bf6f9cbbf0f8 h1:nMpu1t4amK3vJWBibQ5X/Nv0aXL+b69TQf2uK5PH7Go= +github.com/99designs/basicauth-go v0.0.0-20230316000542-bf6f9cbbf0f8/go.mod h1:3cARGAK9CfW3HoxCy1a0G4TKrdiKke8ftOMEOHyySYs= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= @@ -12,26 +18,40 @@ github.com/go-chi/httprate v0.16.0 h1:8V5DH9j6pSK6UQoBsTpvMyFxycqaKEIToyPKzHJjUa github.com/go-chi/httprate v0.16.0/go.mod h1:A8lo+qRhk+s9LiuP5saS7XCGDXRXMcrueq0NfIuCa/I= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= -github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c= -github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= +github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU= +github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY= +github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc= +github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= +github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= +github.com/slok/go-http-metrics v0.13.0 h1:lQDyJJx9wKhmbliyUsZ2l6peGnXRHjsjoqPt5VYzcP8= +github.com/slok/go-http-metrics v0.13.0/go.mod h1:HIr7t/HbN2sJaunvnt9wKP9xoBBVZFo1/KiHU3b0w+4= github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= @@ -42,6 +62,8 @@ github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= @@ -54,18 +76,22 @@ go.uber.org/dig v1.19.0 h1:BACLhebsYdpQ7IROQ1AGPjrXcP5dF80U3gKoFzbaq/4= go.uber.org/dig v1.19.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE= go.uber.org/fx v1.24.0 h1:wE8mruvpg2kiiL1Vqd0CC+tr0/24XIB10Iwp2lLWzkg= go.uber.org/fx v1.24.0/go.mod h1:AmDeGyS+ZARGKM4tlH4FY2Jr63VjbEDJHtqXTGP5hbo= -go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= -go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= -golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= -golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go index 0abc49b..348380d 100644 --- a/backend/internal/config/config.go +++ b/backend/internal/config/config.go @@ -44,6 +44,15 @@ var ( errNotPort = errors.New("must be a port number, 1 to 65535") errNotBool = errors.New("must be true or false") errNotIP = errors.New("must be an IP address, or empty") + + errMetricsCredentials = errors.New( + "METRICS_USERNAME and METRICS_PASSWORD must be set together, " + + "or neither", + ) + errMetricsUsernameColon = errors.New( + "METRICS_USERNAME must not contain \":\", " + + "which basic auth cannot carry in a user name", + ) ) // Params defines the dependencies for Config. @@ -178,6 +187,18 @@ func (s *Config) check() error { } } + // The server records and serves metrics only with both set, so + // one alone is a mistake that would otherwise go unnoticed. + if (s.MetricsUsername == "") != (s.MetricsPassword == "") { + return errMetricsCredentials + } + + // Basic auth splits the credentials at the first ":", so with one + // in the user name every request to /metrics would get 401. + if strings.Contains(s.MetricsUsername, ":") { + return errMetricsUsernameColon + } + return checkOrigins(s.CORSAllowedOrigins) } diff --git a/backend/internal/config/config_test.go b/backend/internal/config/config_test.go index a75e31e..a6a42e5 100644 --- a/backend/internal/config/config_test.go +++ b/backend/internal/config/config_test.go @@ -103,6 +103,32 @@ func TestDataDirMaxBytesMustBeANumber(t *testing.T) { requireConfigError(t, "DATA_DIR_MAX_BYTES") } +// TestMetricsCredentialsGoTogether: with only one of the two set, the +// server would quietly serve no metrics, so the start fails, naming +// both. +func TestMetricsCredentialsGoTogether(t *testing.T) { + for _, set := range []string{"METRICS_USERNAME", "METRICS_PASSWORD"} { + t.Run(set, func(t *testing.T) { + t.Setenv("METRICS_USERNAME", "") + t.Setenv("METRICS_PASSWORD", "") + t.Setenv(set, "prometheus") + + requireConfigError(t, "METRICS_USERNAME") + requireConfigError(t, "METRICS_PASSWORD") + }) + } +} + +// TestMetricsUsernameMustNotContainColon: basic auth splits the +// credentials at the first ":", so such a user name would get 401 on +// every request to /metrics. +func TestMetricsUsernameMustNotContainColon(t *testing.T) { + t.Setenv("METRICS_USERNAME", "prom:etheus") + t.Setenv("METRICS_PASSWORD", "secret") + + requireConfigError(t, "METRICS_USERNAME") +} + // TestCORSAllowedOriginsMustBeOrigins: "*" would let every origin in, // and an entry that is not a plain origin would match no page. func TestCORSAllowedOriginsMustBeOrigins(t *testing.T) { diff --git a/backend/internal/middleware/middleware.go b/backend/internal/middleware/middleware.go index aff7864..6d97b6e 100644 --- a/backend/internal/middleware/middleware.go +++ b/backend/internal/middleware/middleware.go @@ -18,9 +18,14 @@ import ( "sneak.berlin/go/netwatch/internal/globals" "sneak.berlin/go/netwatch/internal/logger" + basicauth "github.com/99designs/basicauth-go" "github.com/go-chi/chi/v5/middleware" "github.com/go-chi/cors" "github.com/go-chi/httprate" + "github.com/prometheus/client_golang/prometheus" + metrics "github.com/slok/go-http-metrics/metrics/prometheus" + ghmm "github.com/slok/go-http-metrics/middleware" + "github.com/slok/go-http-metrics/middleware/std" "go.uber.org/fx" ) @@ -364,3 +369,25 @@ func (s *Middleware) RateLimit( ), ) } + +// Metrics returns middleware that records each request's duration and +// response size, and the requests in progress, in registry. They are +// labelled by the request path. +func (s *Middleware) Metrics( + registry prometheus.Registerer, +) func(http.Handler) http.Handler { + mdlw := ghmm.New(ghmm.Config{ + Recorder: metrics.NewRecorder(metrics.Config{Registry: registry}), + }) + + return std.HandlerProvider("", mdlw) +} + +// MetricsAuth returns middleware that lets a request through only with +// METRICS_USERNAME and METRICS_PASSWORD as its basic auth credentials, +// and answers any other with 401. +func (s *Middleware) MetricsAuth() func(http.Handler) http.Handler { + return basicauth.New("metrics", map[string][]string{ + s.params.Config.MetricsUsername: {s.params.Config.MetricsPassword}, + }) +} diff --git a/backend/internal/server/routes.go b/backend/internal/server/routes.go index 4c87f59..9f91301 100644 --- a/backend/internal/server/routes.go +++ b/backend/internal/server/routes.go @@ -5,6 +5,9 @@ import ( "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" ) const ( @@ -29,13 +32,37 @@ func (s *Server) SetupRoutes() { s.router.Use(s.mw.MaxBodyBytes(maxRequestBodyBytes)) s.router.Use(middleware.Timeout(requestTimeout)) - s.router.Get( - "/.well-known/healthcheck", - s.h.HandleHealthCheck(), + // The metrics go in a registry of this server's own, not in + // Prometheus' default one, which takes them only once per process. + registry := prometheus.NewRegistry() + registry.MustRegister( + collectors.NewGoCollector(), + collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), ) - s.router.Route("/api/v1", func(r chi.Router) { + // Requests are measured only once chi has matched them to one of + // these routes, by path and method. The metrics are labelled with + // both, which any client can make up, so measuring every request + // would let clients add labels without bound. A Route here would + // be matched by its path prefix alone, so each path is given in + // full. + s.router.Group(func(r chi.Router) { + // config.New refuses one of the two credentials without the + // other. + if s.params.Config.MetricsUsername != "" { + r.Use(s.mw.Metrics(registry)) + } + + r.Get("/.well-known/healthcheck", s.h.HandleHealthCheck()) + r.With(s.mw.RateLimit(s.params.Config.ReportsPerMinute)). - Post("/reports", s.h.HandleReport()) + Post("/api/v1/reports", s.h.HandleReport()) }) + + if s.params.Config.MetricsUsername != "" { + s.router.With(s.mw.MetricsAuth()). + Get("/metrics", promhttp.HandlerFor( + registry, promhttp.HandlerOpts{}, + ).ServeHTTP) + } } diff --git a/backend/internal/server/routes_test.go b/backend/internal/server/routes_test.go index 972debd..8418aa1 100644 --- a/backend/internal/server/routes_test.go +++ b/backend/internal/server/routes_test.go @@ -107,6 +107,95 @@ func TestCORSAllowedOriginsReachTheRouter(t *testing.T) { } } +// TestNoMetricsWithoutCredentials: with neither metrics setting set, +// there is no /metrics. +func TestNoMetricsWithoutCredentials(t *testing.T) { + t.Setenv("METRICS_USERNAME", "") + t.Setenv("METRICS_PASSWORD", "") + + srv := newServer(t) + srv.SetupRoutes() + + rec := httptest.NewRecorder() + req := httptest.NewRequestWithContext(t.Context(), + http.MethodGet, "/metrics", http.NoBody) + srv.ServeHTTP(rec, req) + + if rec.Code != http.StatusNotFound { + t.Fatalf("status = %d, want %d", rec.Code, http.StatusNotFound) + } +} + +// TestMetricsBehindBasicAuth: with both metrics settings set, /metrics +// answers only with them as basic auth credentials, and shows a +// request to a route but not one to a path no route has. +func TestMetricsBehindBasicAuth(t *testing.T) { + t.Setenv("METRICS_USERNAME", "prometheus") + t.Setenv("METRICS_PASSWORD", "right") + + srv := newServer(t) + srv.SetupRoutes() + + get := func(path, username, password string) *httptest.ResponseRecorder { + rec := httptest.NewRecorder() + req := httptest.NewRequestWithContext(t.Context(), + http.MethodGet, path, http.NoBody) + + if username != "" { + req.SetBasicAuth(username, password) + } + + srv.ServeHTTP(rec, req) + + return rec + } + + get("/.well-known/healthcheck", "", "") + get("/api/v1/no-such-route", "", "") + + for _, creds := range [][2]string{ + {"", ""}, + {"prometheus", "wrong"}, + {"someone", "right"}, + } { + rec := get("/metrics", creds[0], creds[1]) + if rec.Code != http.StatusUnauthorized { + t.Errorf("credentials %q: status = %d, want %d", + creds, rec.Code, http.StatusUnauthorized) + } + } + + rec := get("/metrics", "prometheus", "right") + if rec.Code != http.StatusOK { + t.Fatalf("right credentials: status = %d, want %d", + rec.Code, http.StatusOK) + } + + body := rec.Body.String() + if !strings.Contains(body, `handler="/.well-known/healthcheck"`) { + t.Errorf("metrics show no health check request:\n%s", body) + } + + if strings.Contains(body, "no-such-route") { + t.Errorf("metrics show a request to a path no route has:\n%s", body) + } + + if !strings.Contains(body, "go_goroutines") { + t.Errorf("metrics show no Go runtime metrics:\n%s", body) + } +} + +// TestMetricsInTwoServers: two servers in one process can both have +// metrics on. +func TestMetricsInTwoServers(t *testing.T) { + t.Setenv("METRICS_USERNAME", "prometheus") + t.Setenv("METRICS_PASSWORD", "right") + + for range 2 { + newServer(t).SetupRoutes() + } +} + // TestHealthCheckRejectsOversizeBody sends the health check, which // never reads its body, a body one byte over the limit. Only the // router-wide body limit can reject it. diff --git a/nginx.conf b/nginx.conf index 3ea963a..d8b0c67 100644 --- a/nginx.conf +++ b/nginx.conf @@ -68,4 +68,10 @@ server { location = /.well-known/healthcheck { proxy_pass http://127.0.0.1:8081; } + + # The backend's Prometheus metrics, behind its own basic auth. Unless + # METRICS_USERNAME and METRICS_PASSWORD are set, it answers 404. + location = /metrics { + proxy_pass http://127.0.0.1:8081; + } }