Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c38f495260 | ||
|
|
bf4ae21974 | ||
|
|
358f01a2a4 | ||
|
|
6ca43e26d7 | ||
|
|
75ce7a9ea8 | ||
|
|
c23585dd6d |
@@ -418,7 +418,9 @@ HMAC-SHA256(secret, "host:path:query:width:height:format:expiration:quality:fit"
|
||||
|
||||
Where:
|
||||
|
||||
- `host` — source origin hostname (e.g. `cdn.example.com`)
|
||||
- `host` — source host as the URL writes it up to the first `/`, in lower case,
|
||||
keeping an IPv6 address's brackets and any `:` and port after it (e.g.
|
||||
`cdn.example.com`, `cdn.example.com:8443`, `[2001:db8::1]:8443`)
|
||||
- `path` — source path (e.g. `/photos/cat.jpg`), with each `%` written as `%25`
|
||||
and each `:` as `%3A`, so `/photos/a:b.jpg` is signed as `/photos/a%3Ab.jpg`
|
||||
- `query` — source query string, empty string if none, with `%` and `:` written
|
||||
@@ -473,6 +475,12 @@ entry that is neither a host name (letters, digits, hyphens, underscores and
|
||||
dots, with at most one leading dot) nor an IP address, such as one with a port
|
||||
or a `*.` wildcard, aborts startup.
|
||||
|
||||
pixa lower-cases the host of a `/v1/image/` URL before anything else uses it, so
|
||||
`CDN.Example.com` is matched, fetched, cached and counted against
|
||||
`upstream_connections_per_host` as `cdn.example.com`. Such a URL whose host has
|
||||
a port, as in `cdn.example.com:8443`, or a `:` with no port, needs a signature
|
||||
even when the host is allowlisted. A host holding `@` is refused with 400.
|
||||
|
||||
### Configuration
|
||||
|
||||
Every setting can be given as an environment variable, in a YAML config file
|
||||
@@ -631,7 +639,10 @@ See `configs/config.example.yml` for all options with defaults.
|
||||
- **Metrics**: Prometheus, at `/metrics`: generic HTTP request metrics
|
||||
(duration, response size, requests in flight) and the Go runtime and process
|
||||
metrics; requests are measured and `/metrics` is served only when
|
||||
`metrics.username` and `metrics.password` are set
|
||||
`metrics.username` and `metrics.password` are set. A request's `handler` label
|
||||
is the pattern of the route it matches, such as `/v1/image/*`, never its path.
|
||||
Requests that match no route share the label `unmatched`: those the router
|
||||
answers 404 or 405, and CORS preflights
|
||||
- **Logging**: stdlib slog
|
||||
|
||||
## Entrypoints
|
||||
|
||||
@@ -30,6 +30,26 @@ P2: security: per-origin rate limiting
|
||||
|
||||
# Completed Steps
|
||||
|
||||
- 2026-10-15 the request metrics are labelled by route, not by path (closes
|
||||
#254): their `handler` label was each request's path, so every new path, a 404
|
||||
or an encrypted URL, added series to the Prometheus registry that were never
|
||||
removed, and `/metrics` listed every encrypted URL's token. The label is now
|
||||
the pattern of the chi route the request matches, such as `/v1/image/*`, or
|
||||
`unmatched` for a request that matches none: one the router answers 404 or
|
||||
405, or a CORS preflight. The metrics library takes the label before the
|
||||
request is routed, so the metrics middleware asks the router which route
|
||||
matches.
|
||||
- 2026-10-15 the host of a `/v1/image/` URL is lower-cased, and one with a port
|
||||
needs a signature (closes #246): with `cdn.example.com` allowlisted,
|
||||
`CDN.Example.com` and `cdn.example.com:8443` passed the allowlist too, and
|
||||
each spelling was fetched and cached on its own and had connections of its own
|
||||
under `upstream_connections_per_host`. The image URL parser now lower-cases
|
||||
the host and refuses one holding `@`, which a URL reads as a user name and
|
||||
password, with 400. A host with a port, or with a `:` and no port, needs a
|
||||
signature even when it is allowlisted. That is checked in the image service,
|
||||
not in `allowlist.IsAllowed`, which also matches `referer_blocklist`, where a
|
||||
`Referer` is blocked whatever its port. A URL is signed over its host as it
|
||||
writes it, in lower case, with any IPv6 brackets, `:` and port kept.
|
||||
- 2026-10-15 a `:` in the source path or query no longer moves where the path
|
||||
ends and the query begins (closes #247): the signed data and the cache key
|
||||
joined host, path and query with `:` and no escaping, so a URL signed for
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"log/slog"
|
||||
"net/url"
|
||||
"runtime/debug"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
@@ -218,7 +219,16 @@ func (s *Service) ValidateRequest(req *ImageRequest) error {
|
||||
return fmt.Errorf("invalid source URL: %w", err)
|
||||
}
|
||||
|
||||
if s.allowlist.IsAllowed(parsedURL) {
|
||||
// An allowlist entry names no port, so a host followed by ":", with or
|
||||
// without a port, needs a signature. The port is read from the parsed
|
||||
// URL, as the fetcher reads it: in an IPv6 address written without
|
||||
// brackets, such as 2001:db8::1:8443, it is what follows the last ":".
|
||||
// This is checked here and not in allowlist.IsAllowed, which also matches
|
||||
// referer_blocklist, where a Referer must be blocked whatever its port.
|
||||
hostHasPort := parsedURL.Port() != "" ||
|
||||
strings.HasSuffix(parsedURL.Host, ":")
|
||||
|
||||
if !hostHasPort && s.allowlist.IsAllowed(parsedURL) {
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -250,6 +250,97 @@ func TestService_ValidateRequest_SignatureExactHostMatch(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestService_ValidateRequest_HostWithPortNeedsSignature verifies that a host
|
||||
// followed by a port, or by a colon with no port, matches no allowlist entry,
|
||||
// so that a request for it without a signature is refused, while the same
|
||||
// host with no port is allowlisted.
|
||||
func TestService_ValidateRequest_HostWithPortNeedsSignature(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
svc, _ := SetupTestService(t,
|
||||
WithSigningKey(testSigningKey),
|
||||
WithAllowlist(testHostCDN, "203.0.113.5", "2001:db8::1"),
|
||||
)
|
||||
|
||||
tests := []struct {
|
||||
host string
|
||||
allowlisted bool
|
||||
}{
|
||||
{testHostCDN, true},
|
||||
{"cdn.example.com:8443", false},
|
||||
{"cdn.example.com:443", false},
|
||||
{"cdn.example.com:", false},
|
||||
{"203.0.113.5", true},
|
||||
{"203.0.113.5:8080", false},
|
||||
{"[2001:db8::1]", true},
|
||||
{"[2001:db8::1]:8443", false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.host, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
req := &ImageRequest{
|
||||
SourceHost: tt.host,
|
||||
SourcePath: testPathCat,
|
||||
Size: Size{Width: 50, Height: 50},
|
||||
Format: FormatJPEG,
|
||||
Quality: 85,
|
||||
FitMode: FitCover,
|
||||
}
|
||||
|
||||
err := svc.ValidateRequest(req)
|
||||
t.Logf("ValidateRequest() for host %q with no signature: %v",
|
||||
tt.host, err)
|
||||
|
||||
if allowlisted := err == nil; allowlisted != tt.allowlisted {
|
||||
t.Errorf("host %q served without a signature = %v, want %v",
|
||||
tt.host, allowlisted, tt.allowlisted)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestService_ValidateRequest_UnbracketedIPv6PortNeedsSignature verifies that
|
||||
// an allowlisted IPv6 address written without brackets and followed by a
|
||||
// port, or by a colon with no port, needs a signature. Go's URL parser, which
|
||||
// the fetcher uses, reads 2001:db8::1:8443 as the host 2001:db8::1 with the
|
||||
// port 8443.
|
||||
func TestService_ValidateRequest_UnbracketedIPv6PortNeedsSignature(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
svc, _ := SetupTestService(t,
|
||||
WithSigningKey(testSigningKey),
|
||||
WithAllowlist("2001:db8::1"),
|
||||
)
|
||||
|
||||
for _, host := range []string{"2001:db8::1:8443", "2001:db8::1:"} {
|
||||
t.Run(host, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
req := &ImageRequest{
|
||||
SourceHost: host,
|
||||
SourcePath: testPathCat,
|
||||
Size: Size{Width: 50, Height: 50},
|
||||
Format: FormatJPEG,
|
||||
Quality: 85,
|
||||
FitMode: FitCover,
|
||||
}
|
||||
|
||||
err := svc.ValidateRequest(req)
|
||||
t.Logf("ValidateRequest() for host %q with no signature: %v",
|
||||
host, err)
|
||||
|
||||
if err == nil {
|
||||
t.Errorf("host %q served without a signature, want refused",
|
||||
host)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestService_Get_InvalidFile(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
var (
|
||||
ErrInvalidPath = errors.New("invalid image path")
|
||||
ErrMissingHost = errors.New("missing source host")
|
||||
ErrInvalidHost = errors.New("invalid source host")
|
||||
ErrMissingSize = errors.New("missing size specification")
|
||||
ErrInvalidSize = errors.New("invalid size format")
|
||||
ErrInvalidFormat = errors.New("invalid or unsupported format")
|
||||
@@ -143,6 +144,17 @@ func parseImageComponents(remainder string) (*ParsedURL, error) {
|
||||
return nil, ErrMissingHost
|
||||
}
|
||||
|
||||
// A URL reads what comes before "@" as a user name and password, and the
|
||||
// rest as the host
|
||||
if strings.Contains(host, "@") {
|
||||
return nil, fmt.Errorf("%w: %q holds @", ErrInvalidHost, host)
|
||||
}
|
||||
|
||||
// Host names do not depend on letter case, so the allowlist, the upstream
|
||||
// connection limits, the cache and the signature see one spelling of each
|
||||
// host
|
||||
host = strings.ToLower(host)
|
||||
|
||||
// Extract query string if present in path
|
||||
if qIndex := strings.Index(path, "?"); qIndex != -1 {
|
||||
query = path[qIndex+1:]
|
||||
|
||||
@@ -234,6 +234,49 @@ func TestParseImagePath(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseImagePath_LowerCasesHost verifies that the host is lower-cased, so
|
||||
// that every spelling of its letter case is the same host to the allowlist,
|
||||
// the upstream connection limits and the cache, and that the path keeps its
|
||||
// case.
|
||||
func TestParseImagePath_LowerCasesHost(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
got, err := ParseImagePath("CDN.Example.COM/Photos/Cat.jpg/800x600.webp")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseImagePath() error = %v", err)
|
||||
}
|
||||
|
||||
t.Logf("parsed host %q, path %q", got.Host, got.Path)
|
||||
|
||||
if got.Host != testHostCDN {
|
||||
t.Errorf("Host = %q, want %q", got.Host, testHostCDN)
|
||||
}
|
||||
|
||||
if got.Path != "/Photos/Cat.jpg" {
|
||||
t.Errorf("Path = %q, want %q", got.Path, "/Photos/Cat.jpg")
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseImagePath_RefusesUserinfo verifies that a host holding "@" is
|
||||
// refused: a URL reads what comes before the "@" as a user name and password
|
||||
// and the rest as the host.
|
||||
func TestParseImagePath_RefusesUserinfo(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, input := range []string{
|
||||
"user@cdn.example.com/photos/cat.jpg/800x600.webp",
|
||||
"user:password@cdn.example.com/photos/cat.jpg/800x600.webp",
|
||||
} {
|
||||
got, err := ParseImagePath(input)
|
||||
t.Logf("ParseImagePath(%q) = %+v, error = %v", input, got, err)
|
||||
|
||||
if !errors.Is(err, ErrInvalidHost) {
|
||||
t.Errorf("ParseImagePath(%q) error = %v, want %v",
|
||||
input, err, ErrInvalidHost)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParsedURL_ToImageRequest(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -8,12 +8,15 @@ import (
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
basicauth "github.com/99designs/basicauth-go"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/go-chi/chi/v5/middleware"
|
||||
"github.com/go-chi/cors"
|
||||
"github.com/go-chi/httprate"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
metrics "github.com/slok/go-http-metrics/metrics/prometheus"
|
||||
ghmm "github.com/slok/go-http-metrics/middleware"
|
||||
"github.com/slok/go-http-metrics/middleware/std"
|
||||
@@ -63,6 +66,10 @@ type Middleware struct {
|
||||
log *slog.Logger
|
||||
config *config.Config
|
||||
clientIP *clientip.Resolver
|
||||
|
||||
// metricsRegistry is the registry Metrics registers its metrics with.
|
||||
// nil is Prometheus' default registry, the one /metrics serves.
|
||||
metricsRegistry prometheus.Registerer
|
||||
}
|
||||
|
||||
// New creates a new Middleware instance.
|
||||
@@ -209,17 +216,61 @@ func (s *Middleware) CORS() func(http.Handler) http.Handler {
|
||||
})
|
||||
}
|
||||
|
||||
// Metrics returns a Prometheus metrics middleware.
|
||||
// Metrics returns a Prometheus metrics middleware. Its handler label is the
|
||||
// pattern of the chi route a request matches, such as /v1/image/*, or
|
||||
// "unmatched" for a request that matches none, so the routes fix how many
|
||||
// values the label has. It is never the path, with which any client could
|
||||
// add series to the registry without limit.
|
||||
func (s *Middleware) Metrics() func(http.Handler) http.Handler {
|
||||
mdlw := ghmm.New(ghmm.Config{
|
||||
Recorder: metrics.NewRecorder(metrics.Config{}),
|
||||
Recorder: metrics.NewRecorder(metrics.Config{
|
||||
Registry: s.metricsRegistry,
|
||||
}),
|
||||
})
|
||||
|
||||
return func(next http.Handler) http.Handler {
|
||||
return std.Handler("", mdlw, next)
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
std.Handler(routePattern(r), mdlw, next).ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// routePattern returns the pattern of the chi route r matches, or
|
||||
// "unmatched" when it matches none or is not served by a chi router. The
|
||||
// metrics library takes the label before the request is routed, so the
|
||||
// router is asked which route matches, with the path it routes on:
|
||||
// r.URL.RawPath when set, else r.URL.Path.
|
||||
func routePattern(r *http.Request) string {
|
||||
pattern := ""
|
||||
|
||||
rctx := chi.RouteContext(r.Context())
|
||||
if rctx != nil {
|
||||
path := r.URL.RawPath
|
||||
if path == "" {
|
||||
path = r.URL.Path
|
||||
}
|
||||
|
||||
pattern = rctx.Routes.Find(chi.NewRouteContext(), r.Method, path)
|
||||
|
||||
// For the path a subrouter is mounted at, such as /v1 or /v1/, Find
|
||||
// answers that path without asking the subrouter, which chi hands
|
||||
// the request as /.
|
||||
for _, route := range rctx.Routes.Routes() {
|
||||
if route.SubRoutes != nil &&
|
||||
route.Pattern == strings.TrimSuffix(pattern, "/")+"/*" &&
|
||||
!route.SubRoutes.Match(chi.NewRouteContext(), r.Method, "/") {
|
||||
pattern = ""
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if pattern == "" {
|
||||
return "unmatched"
|
||||
}
|
||||
|
||||
return pattern
|
||||
}
|
||||
|
||||
// MetricsAuth returns a basic auth middleware for the metrics endpoint.
|
||||
func (s *Middleware) MetricsAuth() func(http.Handler) http.Handler {
|
||||
return basicauth.New(
|
||||
|
||||
@@ -3,12 +3,15 @@ package middleware
|
||||
import (
|
||||
"bytes"
|
||||
"log/slog"
|
||||
"maps"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/config"
|
||||
@@ -198,7 +201,7 @@ func TestMetricsRecordsServedRequest(t *testing.T) {
|
||||
|
||||
// The line /metrics shows once one GET /test has been served.
|
||||
const want = `http_request_duration_seconds_count{` +
|
||||
`code="200",handler="/test",method="GET",service=""} 1`
|
||||
`code="200",handler="unmatched",method="GET",service=""} 1`
|
||||
|
||||
mw := &Middleware{log: slog.Default(), config: &config.Config{}}
|
||||
|
||||
@@ -220,6 +223,86 @@ func TestMetricsRecordsServedRequest(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestMetricsLabelsRequestsByRoute checks that the metrics middleware labels
|
||||
// each request with the pattern of the route that answers it, never with its
|
||||
// path: two paths one route answers share that route's pattern, and requests
|
||||
// no route answers, a 404 or a 405, share "unmatched", among them requests for
|
||||
// the path the subrouter is mounted at, /v1 and /v1/. With a label per path,
|
||||
// any client could add series to the registry without limit. The middleware
|
||||
// registers with a registry of its own here, as another test in this package
|
||||
// registers with Prometheus' default registry, which can be done only once.
|
||||
func TestMetricsLabelsRequestsByRoute(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
registry := prometheus.NewRegistry()
|
||||
mw := &Middleware{
|
||||
log: slog.Default(),
|
||||
config: &config.Config{},
|
||||
metricsRegistry: registry,
|
||||
}
|
||||
|
||||
// The image route is in a subrouter, as in pixa's router.
|
||||
router := chi.NewRouter()
|
||||
router.Use(mw.Metrics())
|
||||
router.Route("/v1", func(r chi.Router) {
|
||||
r.Get("/image/*", func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
})
|
||||
})
|
||||
|
||||
for _, req := range []struct{ method, path string }{
|
||||
{http.MethodGet, "/v1/image/a.example.com/one.jpg/1x1.png"},
|
||||
{http.MethodGet, "/v1/image/b.example.com/two.jpg/1x1.png"},
|
||||
{http.MethodGet, "/no-such-route"},
|
||||
{http.MethodGet, "/v1/no-such-route"},
|
||||
{http.MethodGet, "/v1"},
|
||||
{http.MethodGet, "/v1/"},
|
||||
{http.MethodPost, "/v1/image/a.example.com/one.jpg/1x1.png"},
|
||||
} {
|
||||
router.ServeHTTP(httptest.NewRecorder(), httptest.NewRequestWithContext(
|
||||
t.Context(), req.method, req.path, nil))
|
||||
}
|
||||
|
||||
got := requestCountsByHandler(t, registry)
|
||||
t.Logf("requests counted by handler label: %v", got)
|
||||
|
||||
want := map[string]uint64{"/v1/image/*": 2, "unmatched": 5}
|
||||
if !maps.Equal(got, want) {
|
||||
t.Errorf("requests counted by handler label = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// requestCountsByHandler returns how many requests the request duration
|
||||
// histogram in registry counted under each value of its handler label.
|
||||
func requestCountsByHandler(
|
||||
t *testing.T, registry *prometheus.Registry,
|
||||
) map[string]uint64 {
|
||||
t.Helper()
|
||||
|
||||
families, err := registry.Gather()
|
||||
if err != nil {
|
||||
t.Fatalf("registry.Gather() error = %v", err)
|
||||
}
|
||||
|
||||
counts := map[string]uint64{}
|
||||
|
||||
for _, family := range families {
|
||||
if family.GetName() != "http_request_duration_seconds" {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, metric := range family.GetMetric() {
|
||||
for _, label := range metric.GetLabel() {
|
||||
if label.GetName() == "handler" {
|
||||
counts[label.GetValue()] += metric.GetHistogram().GetSampleCount()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return counts
|
||||
}
|
||||
|
||||
// TestLoggingLeavesOutSubmittedSigningKey checks that a login, a POST /
|
||||
// whose form carries the signing key, leaves no trace of the key in the
|
||||
// request's log line.
|
||||
|
||||
Reference in New Issue
Block a user