Author SHA1 Message Date
sneak 6b7d939234 Test that max_concurrent_processing defaults to at most 4
check / check (push) Waiting to run
The default is the number of CPUs Go uses, so a container with no CPU
limit on a host with many CPUs processes one image per CPU at once. These
tests fail until the default in the config and in imageprocessor.New is
held to 4 (#242).

Model: opus-5-5
2026-10-15 02:09:45 +00:00
6 changed files with 47 additions and 146 deletions
+1 -4
View File
@@ -621,10 +621,7 @@ See `configs/config.example.yml` for all options with defaults.
- **Metrics**: Prometheus, at `/metrics`: generic HTTP request metrics
(duration, response size, requests in flight) and the Go runtime and process
metrics; requests are measured and `/metrics` is served only when
`metrics.username` and `metrics.password` are set. A request's `handler` label
is the pattern of the route it matches, such as `/v1/image/*`, never its path.
Requests that match no route share the label `unmatched`: those the router
answers 404 or 405, and CORS preflights
`metrics.username` and `metrics.password` are set
- **Logging**: stdlib slog
## Entrypoints
-9
View File
@@ -30,15 +30,6 @@ P2: security: per-origin rate limiting
# Completed Steps
- 2026-10-15 the request metrics are labelled by route, not by path (closes
#254): their `handler` label was each request's path, so every new path, a 404
or an encrypted URL, added series to the Prometheus registry that were never
removed, and `/metrics` listed every encrypted URL's token. The label is now
the pattern of the chi route the request matches, such as `/v1/image/*`, or
`unmatched` for a request that matches none: one the router answers 404 or
405, or a CORS preflight. The metrics library takes the label before the
request is routed, so the metrics middleware asks the router which route
matches.
- 2026-10-13 `fit=outside` no longer makes an output above 8192 on a side
(closes #257): it scales the image to cover the requested size, and the side
scaled past that size was held to no maximum, so `1x8192?fit=outside` of a
@@ -32,6 +32,28 @@ func TestOmittedConcurrencyLimitsUseDefaults(t *testing.T) {
}
}
// TestOmittedMaxConcurrentProcessingIsAtMostFour checks that an omitted
// max_concurrent_processing is the number of CPUs Go uses, but never more
// than 4, so a container with no CPU limit on a host with many CPUs does not
// process one image per CPU at once.
func TestOmittedMaxConcurrentProcessingIsAtMostFour(t *testing.T) {
t.Parallel()
c, err := configFromYAML(t, signingKeyLine)
if err != nil {
t.Fatalf("minimal config should be valid, got error: %v", err)
}
want := min(runtime.GOMAXPROCS(0), 4)
t.Logf("GOMAXPROCS %d, MaxConcurrentProcessing %d",
runtime.GOMAXPROCS(0), c.MaxConcurrentProcessing)
if c.MaxConcurrentProcessing != want {
t.Errorf("MaxConcurrentProcessing = %d, want %d",
c.MaxConcurrentProcessing, want)
}
}
// TestExplicitConcurrencyLimitsAreUsed checks that valid values for the
// two limits are used as given.
func TestExplicitConcurrencyLimitsAreUsed(t *testing.T) {
@@ -132,6 +132,27 @@ func TestNewDefaultsMaxConcurrentProcessingToCPUs(t *testing.T) {
}
}
// TestNewDefaultMaxConcurrentProcessingIsAtMostFour checks that New, given
// no MaxConcurrentProcessing, makes one slot per CPU Go uses, but never more
// than 4.
func TestNewDefaultMaxConcurrentProcessingIsAtMostFour(t *testing.T) {
t.Parallel()
want := min(runtime.GOMAXPROCS(0), 4)
for _, limit := range []int{0, -1} {
proc := New(Params{MaxConcurrentProcessing: limit})
got := cap(proc.processingSemaphore)
t.Logf("MaxConcurrentProcessing %d, GOMAXPROCS %d: %d slots",
limit, runtime.GOMAXPROCS(0), got)
if got != want {
t.Errorf("MaxConcurrentProcessing %d: %d slots, want %d",
limit, got, want)
}
}
}
// TestProcessNeverExceedsMaxConcurrentProcessing starts more Process calls
// than MaxConcurrentProcessing allows and holds the first ones inside
// Process until the test lets them go. No more than the limit may be
@@ -1,91 +0,0 @@
package middleware
import (
"log/slog"
"maps"
"net/http"
"net/http/httptest"
"testing"
"github.com/go-chi/chi/v5"
"github.com/prometheus/client_golang/prometheus"
"sneak.berlin/go/pixa/internal/config"
)
// TestMetricsLabelsRequestsByRoute checks that the metrics middleware labels
// each request with the pattern of the route that answers it, never with its
// path: two paths one route answers share that route's pattern, and requests
// no route answers, a 404 or a 405, share "unmatched". With a label per path,
// any client could add series to the registry without limit. The middleware
// registers with a registry of its own here, as another test in this package
// registers with Prometheus' default registry, which can be done only once.
func TestMetricsLabelsRequestsByRoute(t *testing.T) {
t.Parallel()
registry := prometheus.NewRegistry()
mw := &Middleware{
log: slog.Default(),
config: &config.Config{},
metricsRegistry: registry,
}
// The image route is in a subrouter, as in pixa's router.
router := chi.NewRouter()
router.Use(mw.Metrics())
router.Route("/v1", func(r chi.Router) {
r.Get("/image/*", func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusOK)
})
})
for _, req := range []struct{ method, path string }{
{http.MethodGet, "/v1/image/a.example.com/one.jpg/1x1.png"},
{http.MethodGet, "/v1/image/b.example.com/two.jpg/1x1.png"},
{http.MethodGet, "/no-such-route"},
{http.MethodGet, "/v1/no-such-route"},
{http.MethodPost, "/v1/image/a.example.com/one.jpg/1x1.png"},
} {
router.ServeHTTP(httptest.NewRecorder(), httptest.NewRequestWithContext(
t.Context(), req.method, req.path, nil))
}
got := requestCountsByHandler(t, registry)
t.Logf("requests counted by handler label: %v", got)
want := map[string]uint64{"/v1/image/*": 2, "unmatched": 3}
if !maps.Equal(got, want) {
t.Errorf("requests counted by handler label = %v, want %v", got, want)
}
}
// requestCountsByHandler returns how many requests the request duration
// histogram in registry counted under each value of its handler label.
func requestCountsByHandler(
t *testing.T, registry *prometheus.Registry,
) map[string]uint64 {
t.Helper()
families, err := registry.Gather()
if err != nil {
t.Fatalf("registry.Gather() error = %v", err)
}
counts := map[string]uint64{}
for _, family := range families {
if family.GetName() != "http_request_duration_seconds" {
continue
}
for _, metric := range family.GetMetric() {
for _, label := range metric.GetLabel() {
if label.GetName() == "handler" {
counts[label.GetValue()] += metric.GetHistogram().GetSampleCount()
}
}
}
}
return counts
}
+3 -42
View File
@@ -11,11 +11,9 @@ import (
"time"
basicauth "github.com/99designs/basicauth-go"
"github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware"
"github.com/go-chi/cors"
"github.com/go-chi/httprate"
"github.com/prometheus/client_golang/prometheus"
metrics "github.com/slok/go-http-metrics/metrics/prometheus"
ghmm "github.com/slok/go-http-metrics/middleware"
"github.com/slok/go-http-metrics/middleware/std"
@@ -65,10 +63,6 @@ type Middleware struct {
log *slog.Logger
config *config.Config
clientIP *clientip.Resolver
// metricsRegistry is the registry Metrics registers its metrics with.
// nil is Prometheus' default registry, the one /metrics serves.
metricsRegistry prometheus.Registerer
}
// New creates a new Middleware instance.
@@ -215,50 +209,17 @@ func (s *Middleware) CORS() func(http.Handler) http.Handler {
})
}
// Metrics returns a Prometheus metrics middleware. Its handler label is the
// pattern of the chi route a request matches, such as /v1/image/*, or
// "unmatched" for a request that matches none, so the routes fix how many
// values the label has. It is never the path, with which any client could
// add series to the registry without limit.
// Metrics returns a Prometheus metrics middleware.
func (s *Middleware) Metrics() func(http.Handler) http.Handler {
mdlw := ghmm.New(ghmm.Config{
Recorder: metrics.NewRecorder(metrics.Config{
Registry: s.metricsRegistry,
}),
Recorder: metrics.NewRecorder(metrics.Config{}),
})
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
std.Handler(routePattern(r), mdlw, next).ServeHTTP(w, r)
})
return std.Handler("", mdlw, next)
}
}
// routePattern returns the pattern of the chi route r matches, or
// "unmatched" when it matches none or is not served by a chi router. The
// metrics library takes the label before the request is routed, so the
// router is asked which route matches, with the path it routes on:
// r.URL.RawPath when set, else r.URL.Path.
func routePattern(r *http.Request) string {
pattern := ""
rctx := chi.RouteContext(r.Context())
if rctx != nil {
path := r.URL.RawPath
if path == "" {
path = r.URL.Path
}
pattern = rctx.Routes.Find(chi.NewRouteContext(), r.Method, path)
}
if pattern == "" {
return "unmatched"
}
return pattern
}
// MetricsAuth returns a basic auth middleware for the metrics endpoint.
func (s *Middleware) MetricsAuth() func(http.Handler) http.Handler {
return basicauth.New(