Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6ebb6ed618 | ||
|
|
40387e90bb |
@@ -198,9 +198,11 @@ path under `/v1/` answers 200, in maintenance mode too.
|
|||||||
redirects to, is `localhost`, ends in `.localhost` or `.local`, or has an
|
redirects to, is `localhost`, ends in `.localhost` or `.local`, or has an
|
||||||
address in a blocked network (see `blocked_networks`); 429 over
|
address in a blocked network (see `blocked_networks`); 429 over
|
||||||
`image_requests_per_minute` (below); 415 when the upstream answers with a
|
`image_requests_per_minute` (below); 415 when the upstream answers with a
|
||||||
`Content-Type` that is not one of the source formats (below), or with an image
|
`Content-Type` that is not one of the source formats (below), with a body
|
||||||
whose first bytes libvips reads as another format; 502 when the upstream
|
whose first bytes are not those of the format its `Content-Type` names, or
|
||||||
answered with an error status, and for 5 minutes after that for the same
|
with an image whose first bytes libvips reads as another format; 502 when the
|
||||||
|
upstream answered with an error status, and for 5 minutes after that, or after
|
||||||
|
a body whose first bytes are not those of its `Content-Type`, for the same
|
||||||
source URL; 503 when pixa is busy or in maintenance mode; 500 for any other
|
source URL; 503 when pixa is busy or in maintenance mode; 500 for any other
|
||||||
failure.
|
failure.
|
||||||
- `GET` or `HEAD` `/v1/e/<token>/<name>` — an image through an encrypted URL
|
- `GET` or `HEAD` `/v1/e/<token>/<name>` — an image through an encrypted URL
|
||||||
@@ -279,8 +281,12 @@ above 8192.
|
|||||||
The source image may be JPEG, PNG, GIF, WebP, AVIF or JPEG XL. The upstream's
|
The source image may be JPEG, PNG, GIF, WebP, AVIF or JPEG XL. The upstream's
|
||||||
`Content-Type` must name its format, and the image's first bytes must match it.
|
`Content-Type` must name its format, and the image's first bytes must match it.
|
||||||
An upstream answering with any other `Content-Type`, SVG included, is refused
|
An upstream answering with any other `Content-Type`, SVG included, is refused
|
||||||
with 415, and so is an image whose first bytes libvips reads as another format,
|
with 415. So is a body whose first bytes are not those of the format its
|
||||||
such as an SVG whose first bytes pass the check for AVIF.
|
`Content-Type` names, such as a JPEG sent as `image/png`, text sent as an image,
|
||||||
|
or an empty body. pixa then does not fetch that source URL again for 5 minutes,
|
||||||
|
and answers 502 for it meanwhile, as after an upstream error status. An image
|
||||||
|
whose first bytes libvips reads as another format, such as an SVG whose first
|
||||||
|
bytes pass the check for AVIF, is refused with 415 too.
|
||||||
|
|
||||||
With the format `auto`, pixa chooses the format for each request from its
|
With the format `auto`, pixa chooses the format for each request from its
|
||||||
`Accept` header, in this order:
|
`Accept` header, in this order:
|
||||||
@@ -418,9 +424,7 @@ HMAC-SHA256(secret, "host:path:query:width:height:format:expiration:quality:fit"
|
|||||||
|
|
||||||
Where:
|
Where:
|
||||||
|
|
||||||
- `host` — source host as the URL writes it up to the first `/`, in lower case,
|
- `host` — source origin hostname (e.g. `cdn.example.com`)
|
||||||
keeping an IPv6 address's brackets and any `:` and port after it (e.g.
|
|
||||||
`cdn.example.com`, `cdn.example.com:8443`, `[2001:db8::1]:8443`)
|
|
||||||
- `path` — source path (e.g. `/photos/cat.jpg`), with each `%` written as `%25`
|
- `path` — source path (e.g. `/photos/cat.jpg`), with each `%` written as `%25`
|
||||||
and each `:` as `%3A`, so `/photos/a:b.jpg` is signed as `/photos/a%3Ab.jpg`
|
and each `:` as `%3A`, so `/photos/a:b.jpg` is signed as `/photos/a%3Ab.jpg`
|
||||||
- `query` — source query string, empty string if none, with `%` and `:` written
|
- `query` — source query string, empty string if none, with `%` and `:` written
|
||||||
@@ -475,12 +479,6 @@ entry that is neither a host name (letters, digits, hyphens, underscores and
|
|||||||
dots, with at most one leading dot) nor an IP address, such as one with a port
|
dots, with at most one leading dot) nor an IP address, such as one with a port
|
||||||
or a `*.` wildcard, aborts startup.
|
or a `*.` wildcard, aborts startup.
|
||||||
|
|
||||||
pixa lower-cases the host of a `/v1/image/` URL before anything else uses it, so
|
|
||||||
`CDN.Example.com` is matched, fetched, cached and counted against
|
|
||||||
`upstream_connections_per_host` as `cdn.example.com`. Such a URL whose host has
|
|
||||||
a port, as in `cdn.example.com:8443`, or a `:` with no port, needs a signature
|
|
||||||
even when the host is allowlisted. A host holding `@` is refused with 400.
|
|
||||||
|
|
||||||
### Configuration
|
### Configuration
|
||||||
|
|
||||||
Every setting can be given as an environment variable, in a YAML config file
|
Every setting can be given as an environment variable, in a YAML config file
|
||||||
@@ -639,10 +637,7 @@ See `configs/config.example.yml` for all options with defaults.
|
|||||||
- **Metrics**: Prometheus, at `/metrics`: generic HTTP request metrics
|
- **Metrics**: Prometheus, at `/metrics`: generic HTTP request metrics
|
||||||
(duration, response size, requests in flight) and the Go runtime and process
|
(duration, response size, requests in flight) and the Go runtime and process
|
||||||
metrics; requests are measured and `/metrics` is served only when
|
metrics; requests are measured and `/metrics` is served only when
|
||||||
`metrics.username` and `metrics.password` are set. A request's `handler` label
|
`metrics.username` and `metrics.password` are set
|
||||||
is the pattern of the route it matches, such as `/v1/image/*`, never its path.
|
|
||||||
Requests that match no route share the label `unmatched`: those the router
|
|
||||||
answers 404 or 405, and CORS preflights
|
|
||||||
- **Logging**: stdlib slog
|
- **Logging**: stdlib slog
|
||||||
|
|
||||||
## Entrypoints
|
## Entrypoints
|
||||||
|
|||||||
@@ -30,26 +30,15 @@ P2: security: per-origin rate limiting
|
|||||||
|
|
||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
- 2026-10-15 the request metrics are labelled by route, not by path (closes
|
- 2026-10-15 a source whose first bytes do not match its `Content-Type` answers
|
||||||
#254): their `handler` label was each request's path, so every new path, a 404
|
415 (closes #253): a body whose first bytes are not those of the format its
|
||||||
or an encrypted URL, added series to the Prometheus registry that were never
|
`Content-Type` names, such as a JPEG sent as `image/png`, text, or a body
|
||||||
removed, and `/metrics` listed every encrypted URL's token. The label is now
|
under 12 bytes, answered 500 on both image routes and was logged as an error
|
||||||
the pattern of the chi route the request matches, such as `/v1/image/*`, or
|
of pixa's, and each request for it fetched the whole body again. Both routes
|
||||||
`unmatched` for a request that matches none: one the router answers 404 or
|
now answer 415, and the image service logs it at warning level as a refused
|
||||||
405, or a CORS preflight. The metrics library takes the label before the
|
source. It also gets a negative cache entry, as an upstream error status does,
|
||||||
request is routed, so the metrics middleware asks the router which route
|
so the source URL is not fetched again for 5 minutes and answers 502
|
||||||
matches.
|
meanwhile.
|
||||||
- 2026-10-15 the host of a `/v1/image/` URL is lower-cased, and one with a port
|
|
||||||
needs a signature (closes #246): with `cdn.example.com` allowlisted,
|
|
||||||
`CDN.Example.com` and `cdn.example.com:8443` passed the allowlist too, and
|
|
||||||
each spelling was fetched and cached on its own and had connections of its own
|
|
||||||
under `upstream_connections_per_host`. The image URL parser now lower-cases
|
|
||||||
the host and refuses one holding `@`, which a URL reads as a user name and
|
|
||||||
password, with 400. A host with a port, or with a `:` and no port, needs a
|
|
||||||
signature even when it is allowlisted. That is checked in the image service,
|
|
||||||
not in `allowlist.IsAllowed`, which also matches `referer_blocklist`, where a
|
|
||||||
`Referer` is blocked whatever its port. A URL is signed over its host as it
|
|
||||||
writes it, in lower case, with any IPv6 brackets, `:` and port kept.
|
|
||||||
- 2026-10-15 a `:` in the source path or query no longer moves where the path
|
- 2026-10-15 a `:` in the source path or query no longer moves where the path
|
||||||
ends and the query begins (closes #247): the signed data and the cache key
|
ends and the query begins (closes #247): the signed data and the cache key
|
||||||
joined host, path and query with `:` and no escaping, so a URL signed for
|
joined host, path and query with `:` and no escaping, so a URL signed for
|
||||||
|
|||||||
@@ -0,0 +1,180 @@
|
|||||||
|
package handlers
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"image/color"
|
||||||
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
|
"slices"
|
||||||
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/go-chi/chi/v5"
|
||||||
|
"sneak.berlin/go/pixa/internal/encurl"
|
||||||
|
"sneak.berlin/go/pixa/internal/httpfetcher"
|
||||||
|
"sneak.berlin/go/pixa/internal/imgcache"
|
||||||
|
)
|
||||||
|
|
||||||
|
// mismatchedSourcePath is the path of the source that
|
||||||
|
// TestFirstBytesNotMatchingContentType serves on allowlistedHost. The mock
|
||||||
|
// fetcher names a file's Content-Type from its extension, so the source is
|
||||||
|
// sent as image/png.
|
||||||
|
const mismatchedSourcePath = "/images/source.png"
|
||||||
|
|
||||||
|
// newRoutesFetchingThrough returns both image routes, and the Handlers behind
|
||||||
|
// them, whose image service fetches through fetcher and has allowlistedHost on
|
||||||
|
// its allowlist. The handlers and the image service log to log.
|
||||||
|
func newRoutesFetchingThrough(
|
||||||
|
t *testing.T, fetcher httpfetcher.Fetcher, log *slog.Logger,
|
||||||
|
) (*Handlers, http.Handler) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
cache, err := imgcache.NewCache(setupTestDB(t), imgcache.CacheConfig{
|
||||||
|
StateDir: t.TempDir(),
|
||||||
|
NegativeTTL: 5 * time.Minute,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("imgcache.NewCache() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
svc, err := imgcache.NewService(&imgcache.ServiceConfig{
|
||||||
|
Cache: cache,
|
||||||
|
Fetcher: fetcher,
|
||||||
|
SigningKey: testSigningKey,
|
||||||
|
Allowlist: []string{allowlistedHost},
|
||||||
|
Logger: log,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("imgcache.NewService() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
encGen, err := encurl.NewGenerator(testSigningKey)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("encurl.NewGenerator() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
h := &Handlers{log: log, imgSvc: svc, encGen: encGen}
|
||||||
|
|
||||||
|
r := chi.NewRouter()
|
||||||
|
r.Get("/v1/image/*", h.HandleImage())
|
||||||
|
r.Get("/v1/e/{token}/*", h.HandleImageEnc())
|
||||||
|
|
||||||
|
return h, r
|
||||||
|
}
|
||||||
|
|
||||||
|
// mismatchedSourcePlainURL returns the /v1/image/ URL of mismatchedSourcePath
|
||||||
|
// on allowlistedHost at 50x50 as a JPEG.
|
||||||
|
func mismatchedSourcePlainURL(*testing.T, *Handlers) string {
|
||||||
|
return "/v1/image/" + allowlistedHost + mismatchedSourcePath + "/50x50.jpeg"
|
||||||
|
}
|
||||||
|
|
||||||
|
// mismatchedSourceEncURL returns an encrypted /v1/e/ URL, made with h's
|
||||||
|
// generator, of mismatchedSourcePath on allowlistedHost at 50x50 as a JPEG.
|
||||||
|
func mismatchedSourceEncURL(t *testing.T, h *Handlers) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
token, err := h.encGen.Generate(&encurl.Payload{
|
||||||
|
SourceHost: allowlistedHost,
|
||||||
|
SourcePath: mismatchedSourcePath,
|
||||||
|
Width: 50,
|
||||||
|
Height: 50,
|
||||||
|
Format: imgcache.FormatJPEG,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Generate() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return "/v1/e/" + token + "/img.jpg"
|
||||||
|
}
|
||||||
|
|
||||||
|
// loggedLevels reads the JSON log lines in logs and returns the messages
|
||||||
|
// logged at each level.
|
||||||
|
func loggedLevels(t *testing.T, logs *bytes.Buffer) map[string][]string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
t.Logf("logged:\n%s", logs.String())
|
||||||
|
|
||||||
|
messages := make(map[string][]string)
|
||||||
|
|
||||||
|
dec := json.NewDecoder(logs)
|
||||||
|
for dec.More() {
|
||||||
|
var line struct {
|
||||||
|
Level string `json:"level"`
|
||||||
|
Msg string `json:"msg"`
|
||||||
|
}
|
||||||
|
|
||||||
|
err := dec.Decode(&line)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("decoding log line: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
messages[line.Level] = append(messages[line.Level], line.Msg)
|
||||||
|
}
|
||||||
|
|
||||||
|
return messages
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestFirstBytesNotMatchingContentType checks that both image routes answer
|
||||||
|
// 415 for an upstream body whose first bytes are not those of the format its
|
||||||
|
// Content-Type names, and log it as a refused source, not as an error of
|
||||||
|
// pixa's. The next request for that source must answer 502 without fetching
|
||||||
|
// it again, as after an upstream error status.
|
||||||
|
func TestFirstBytesNotMatchingContentType(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
bodies := map[string][]byte{
|
||||||
|
"JPEG sent as image/png": generateTestJPEG(t, 10, 10, color.Black),
|
||||||
|
"text sent as image/png": []byte("This is just text, not a PNG"),
|
||||||
|
"body too short to name a format": []byte("short"),
|
||||||
|
}
|
||||||
|
|
||||||
|
// The source's URL on each image route.
|
||||||
|
sourceURLs := map[string]func(*testing.T, *Handlers) string{
|
||||||
|
"plain URL": mismatchedSourcePlainURL,
|
||||||
|
"encrypted URL": mismatchedSourceEncURL,
|
||||||
|
}
|
||||||
|
|
||||||
|
for bodyName, body := range bodies {
|
||||||
|
for urlName, sourceURL := range sourceURLs {
|
||||||
|
t.Run(urlName+", "+bodyName, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
fetcher := &countingFetcher{
|
||||||
|
Fetcher: httpfetcher.NewMock(fstest.MapFS{
|
||||||
|
allowlistedHost + mismatchedSourcePath: &fstest.MapFile{
|
||||||
|
Data: body,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
|
||||||
|
var logs bytes.Buffer
|
||||||
|
|
||||||
|
h, routes := newRoutesFetchingThrough(t, fetcher,
|
||||||
|
slog.New(slog.NewJSONHandler(&logs, nil)))
|
||||||
|
target := sourceURL(t, h)
|
||||||
|
|
||||||
|
checkErrorBody(t, sendGet(t, routes, target),
|
||||||
|
http.StatusUnsupportedMediaType, "unsupported source format")
|
||||||
|
|
||||||
|
levels := loggedLevels(t, &logs)
|
||||||
|
if len(levels[slog.LevelError.String()]) != 0 {
|
||||||
|
t.Errorf("logged at error level: %q",
|
||||||
|
levels[slog.LevelError.String()])
|
||||||
|
}
|
||||||
|
|
||||||
|
if !slices.Contains(levels[slog.LevelWarn.String()], "source refused") {
|
||||||
|
t.Error(`no "source refused" line logged at warning level`)
|
||||||
|
}
|
||||||
|
|
||||||
|
checkErrorBody(t, sendGet(t, routes, target),
|
||||||
|
http.StatusBadGateway, "upstream error")
|
||||||
|
|
||||||
|
if n := fetcher.fetches.Load(); n != 1 {
|
||||||
|
t.Errorf("upstream fetched %d times, want 1", n)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -15,6 +15,7 @@ import (
|
|||||||
"sneak.berlin/go/pixa/internal/httpfetcher"
|
"sneak.berlin/go/pixa/internal/httpfetcher"
|
||||||
"sneak.berlin/go/pixa/internal/imageprocessor"
|
"sneak.berlin/go/pixa/internal/imageprocessor"
|
||||||
"sneak.berlin/go/pixa/internal/imgcache"
|
"sneak.berlin/go/pixa/internal/imgcache"
|
||||||
|
"sneak.berlin/go/pixa/internal/magic"
|
||||||
)
|
)
|
||||||
|
|
||||||
// HandleImage handles the main image proxy route:
|
// HandleImage handles the main image proxy route:
|
||||||
@@ -220,6 +221,18 @@ func (s *Handlers) HandleTooManyRequests() http.HandlerFunc {
|
|||||||
func (s *Handlers) respondImageError(
|
func (s *Handlers) respondImageError(
|
||||||
w http.ResponseWriter, req *imgcache.ImageRequest, err error,
|
w http.ResponseWriter, req *imgcache.ImageRequest, err error,
|
||||||
) {
|
) {
|
||||||
|
// An upstream body whose first bytes are not those of the format its
|
||||||
|
// Content-Type names is the upstream's error, not pixa's: the image
|
||||||
|
// service has logged it as a refused source.
|
||||||
|
if errors.Is(err, magic.ErrMagicByteMismatch) ||
|
||||||
|
errors.Is(err, magic.ErrUnknownFormat) ||
|
||||||
|
errors.Is(err, magic.ErrNotEnoughData) {
|
||||||
|
s.respondError(w, "unsupported source format",
|
||||||
|
http.StatusUnsupportedMediaType)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
s.log.Error("failed to get image",
|
s.log.Error("failed to get image",
|
||||||
"host", req.SourceHost,
|
"host", req.SourceHost,
|
||||||
"path", req.SourcePath,
|
"path", req.SourcePath,
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ import (
|
|||||||
"sneak.berlin/go/pixa/internal/httpfetcher"
|
"sneak.berlin/go/pixa/internal/httpfetcher"
|
||||||
"sneak.berlin/go/pixa/internal/imageprocessor"
|
"sneak.berlin/go/pixa/internal/imageprocessor"
|
||||||
"sneak.berlin/go/pixa/internal/imgcache"
|
"sneak.berlin/go/pixa/internal/imgcache"
|
||||||
|
"sneak.berlin/go/pixa/internal/magic"
|
||||||
)
|
)
|
||||||
|
|
||||||
// HandleImageEnc handles requests to /v1/e/{token}/* for encrypted
|
// HandleImageEnc handles requests to /v1/e/{token}/* for encrypted
|
||||||
@@ -162,7 +163,10 @@ func (s *Handlers) handleImageError(w http.ResponseWriter, err error) {
|
|||||||
s.respondError(w, "server busy, try again later",
|
s.respondError(w, "server busy, try again later",
|
||||||
http.StatusServiceUnavailable)
|
http.StatusServiceUnavailable)
|
||||||
case errors.Is(err, httpfetcher.ErrInvalidContentType),
|
case errors.Is(err, httpfetcher.ErrInvalidContentType),
|
||||||
errors.Is(err, imageprocessor.ErrUnsupportedInputFormat):
|
errors.Is(err, imageprocessor.ErrUnsupportedInputFormat),
|
||||||
|
errors.Is(err, magic.ErrMagicByteMismatch),
|
||||||
|
errors.Is(err, magic.ErrUnknownFormat),
|
||||||
|
errors.Is(err, magic.ErrNotEnoughData):
|
||||||
s.respondError(w, "unsupported source format",
|
s.respondError(w, "unsupported source format",
|
||||||
http.StatusUnsupportedMediaType)
|
http.StatusUnsupportedMediaType)
|
||||||
case errors.Is(err, imageprocessor.ErrOutputTooLarge):
|
case errors.Is(err, imageprocessor.ErrOutputTooLarge):
|
||||||
|
|||||||
@@ -9,7 +9,6 @@ import (
|
|||||||
"log/slog"
|
"log/slog"
|
||||||
"net/url"
|
"net/url"
|
||||||
"runtime/debug"
|
"runtime/debug"
|
||||||
"strings"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/dustin/go-humanize"
|
"github.com/dustin/go-humanize"
|
||||||
@@ -219,16 +218,7 @@ func (s *Service) ValidateRequest(req *ImageRequest) error {
|
|||||||
return fmt.Errorf("invalid source URL: %w", err)
|
return fmt.Errorf("invalid source URL: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// An allowlist entry names no port, so a host followed by ":", with or
|
if s.allowlist.IsAllowed(parsedURL) {
|
||||||
// without a port, needs a signature. The port is read from the parsed
|
|
||||||
// URL, as the fetcher reads it: in an IPv6 address written without
|
|
||||||
// brackets, such as 2001:db8::1:8443, it is what follows the last ":".
|
|
||||||
// This is checked here and not in allowlist.IsAllowed, which also matches
|
|
||||||
// referer_blocklist, where a Referer must be blocked whatever its port.
|
|
||||||
hostHasPort := parsedURL.Port() != "" ||
|
|
||||||
strings.HasSuffix(parsedURL.Host, ":")
|
|
||||||
|
|
||||||
if !hostHasPort && s.allowlist.IsAllowed(parsedURL) {
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -448,13 +438,7 @@ func (s *Service) fetchAndProcess(
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
// Store negative cache for certain errors
|
// Store negative cache for certain errors
|
||||||
if isNegativeCacheable(err) {
|
if isNegativeCacheable(err) {
|
||||||
statusCode := extractStatusCode(err)
|
s.storeNegative(ctx, req, extractStatusCode(err), err)
|
||||||
|
|
||||||
storeErr := s.cache.StoreNegative(ctx, req, statusCode, err.Error())
|
|
||||||
if storeErr != nil {
|
|
||||||
s.log.Warn("failed to store negative cache entry",
|
|
||||||
"host", req.SourceHost, "path", req.SourcePath, "error", storeErr)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil, fmt.Errorf("upstream fetch failed: %w", err)
|
return nil, fmt.Errorf("upstream fetch failed: %w", err)
|
||||||
@@ -500,9 +484,16 @@ func (s *Service) fetchAndProcess(
|
|||||||
"cipher", fetchResult.TLSCipherSuite,
|
"cipher", fetchResult.TLSCipherSuite,
|
||||||
)
|
)
|
||||||
|
|
||||||
// Validate magic bytes match content type
|
// Validate magic bytes match content type. A body that does not is the
|
||||||
|
// upstream's error, so it gets a negative cache entry, as an upstream
|
||||||
|
// error status does.
|
||||||
err = magic.ValidateMagicBytes(sourceData, fetchResult.ContentType)
|
err = magic.ValidateMagicBytes(sourceData, fetchResult.ContentType)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
s.log.Warn("source refused", "request_id", middleware.GetReqID(ctx),
|
||||||
|
"host", req.SourceHost, "path", req.SourcePath,
|
||||||
|
"content_type", fetchResult.ContentType, "error", err)
|
||||||
|
s.storeNegative(ctx, req, httpStatusUnsupportedMediaType, err)
|
||||||
|
|
||||||
return nil, fmt.Errorf("content validation failed: %w", err)
|
return nil, fmt.Errorf("content validation failed: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -518,6 +509,19 @@ func (s *Service) fetchAndProcess(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// storeNegative stores a negative cache entry for req's source, whose fetch
|
||||||
|
// failed with err, so the source is not fetched again until the entry
|
||||||
|
// expires. A failure to store the entry is only logged.
|
||||||
|
func (s *Service) storeNegative(
|
||||||
|
ctx context.Context, req *ImageRequest, statusCode int, err error,
|
||||||
|
) {
|
||||||
|
storeErr := s.cache.StoreNegative(ctx, req, statusCode, err.Error())
|
||||||
|
if storeErr != nil {
|
||||||
|
s.log.Warn("failed to store negative cache entry",
|
||||||
|
"host", req.SourceHost, "path", req.SourcePath, "error", storeErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// processedVariant is a variant as processAndStore made it. Each request that
|
// processedVariant is a variant as processAndStore made it. Each request that
|
||||||
// shared its processing serves it through a reader of its own.
|
// shared its processing serves it through a reader of its own.
|
||||||
type processedVariant struct {
|
type processedVariant struct {
|
||||||
@@ -626,8 +630,9 @@ func signatureRequest(req *ImageRequest) *signature.Request {
|
|||||||
|
|
||||||
// HTTP status codes for error responses.
|
// HTTP status codes for error responses.
|
||||||
const (
|
const (
|
||||||
httpStatusBadGateway = 502
|
httpStatusBadGateway = 502
|
||||||
httpStatusInternalError = 500
|
httpStatusInternalError = 500
|
||||||
|
httpStatusUnsupportedMediaType = 415
|
||||||
)
|
)
|
||||||
|
|
||||||
// isNegativeCacheable returns true if the error should be cached.
|
// isNegativeCacheable returns true if the error should be cached.
|
||||||
|
|||||||
@@ -250,97 +250,6 @@ func TestService_ValidateRequest_SignatureExactHostMatch(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestService_ValidateRequest_HostWithPortNeedsSignature verifies that a host
|
|
||||||
// followed by a port, or by a colon with no port, matches no allowlist entry,
|
|
||||||
// so that a request for it without a signature is refused, while the same
|
|
||||||
// host with no port is allowlisted.
|
|
||||||
func TestService_ValidateRequest_HostWithPortNeedsSignature(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
svc, _ := SetupTestService(t,
|
|
||||||
WithSigningKey(testSigningKey),
|
|
||||||
WithAllowlist(testHostCDN, "203.0.113.5", "2001:db8::1"),
|
|
||||||
)
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
host string
|
|
||||||
allowlisted bool
|
|
||||||
}{
|
|
||||||
{testHostCDN, true},
|
|
||||||
{"cdn.example.com:8443", false},
|
|
||||||
{"cdn.example.com:443", false},
|
|
||||||
{"cdn.example.com:", false},
|
|
||||||
{"203.0.113.5", true},
|
|
||||||
{"203.0.113.5:8080", false},
|
|
||||||
{"[2001:db8::1]", true},
|
|
||||||
{"[2001:db8::1]:8443", false},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.host, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
req := &ImageRequest{
|
|
||||||
SourceHost: tt.host,
|
|
||||||
SourcePath: testPathCat,
|
|
||||||
Size: Size{Width: 50, Height: 50},
|
|
||||||
Format: FormatJPEG,
|
|
||||||
Quality: 85,
|
|
||||||
FitMode: FitCover,
|
|
||||||
}
|
|
||||||
|
|
||||||
err := svc.ValidateRequest(req)
|
|
||||||
t.Logf("ValidateRequest() for host %q with no signature: %v",
|
|
||||||
tt.host, err)
|
|
||||||
|
|
||||||
if allowlisted := err == nil; allowlisted != tt.allowlisted {
|
|
||||||
t.Errorf("host %q served without a signature = %v, want %v",
|
|
||||||
tt.host, allowlisted, tt.allowlisted)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestService_ValidateRequest_UnbracketedIPv6PortNeedsSignature verifies that
|
|
||||||
// an allowlisted IPv6 address written without brackets and followed by a
|
|
||||||
// port, or by a colon with no port, needs a signature. Go's URL parser, which
|
|
||||||
// the fetcher uses, reads 2001:db8::1:8443 as the host 2001:db8::1 with the
|
|
||||||
// port 8443.
|
|
||||||
func TestService_ValidateRequest_UnbracketedIPv6PortNeedsSignature(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
svc, _ := SetupTestService(t,
|
|
||||||
WithSigningKey(testSigningKey),
|
|
||||||
WithAllowlist("2001:db8::1"),
|
|
||||||
)
|
|
||||||
|
|
||||||
for _, host := range []string{"2001:db8::1:8443", "2001:db8::1:"} {
|
|
||||||
t.Run(host, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
req := &ImageRequest{
|
|
||||||
SourceHost: host,
|
|
||||||
SourcePath: testPathCat,
|
|
||||||
Size: Size{Width: 50, Height: 50},
|
|
||||||
Format: FormatJPEG,
|
|
||||||
Quality: 85,
|
|
||||||
FitMode: FitCover,
|
|
||||||
}
|
|
||||||
|
|
||||||
err := svc.ValidateRequest(req)
|
|
||||||
t.Logf("ValidateRequest() for host %q with no signature: %v",
|
|
||||||
host, err)
|
|
||||||
|
|
||||||
if err == nil {
|
|
||||||
t.Errorf("host %q served without a signature, want refused",
|
|
||||||
host)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestService_Get_InvalidFile(t *testing.T) {
|
func TestService_Get_InvalidFile(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -15,7 +15,6 @@ import (
|
|||||||
var (
|
var (
|
||||||
ErrInvalidPath = errors.New("invalid image path")
|
ErrInvalidPath = errors.New("invalid image path")
|
||||||
ErrMissingHost = errors.New("missing source host")
|
ErrMissingHost = errors.New("missing source host")
|
||||||
ErrInvalidHost = errors.New("invalid source host")
|
|
||||||
ErrMissingSize = errors.New("missing size specification")
|
ErrMissingSize = errors.New("missing size specification")
|
||||||
ErrInvalidSize = errors.New("invalid size format")
|
ErrInvalidSize = errors.New("invalid size format")
|
||||||
ErrInvalidFormat = errors.New("invalid or unsupported format")
|
ErrInvalidFormat = errors.New("invalid or unsupported format")
|
||||||
@@ -144,17 +143,6 @@ func parseImageComponents(remainder string) (*ParsedURL, error) {
|
|||||||
return nil, ErrMissingHost
|
return nil, ErrMissingHost
|
||||||
}
|
}
|
||||||
|
|
||||||
// A URL reads what comes before "@" as a user name and password, and the
|
|
||||||
// rest as the host
|
|
||||||
if strings.Contains(host, "@") {
|
|
||||||
return nil, fmt.Errorf("%w: %q holds @", ErrInvalidHost, host)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Host names do not depend on letter case, so the allowlist, the upstream
|
|
||||||
// connection limits, the cache and the signature see one spelling of each
|
|
||||||
// host
|
|
||||||
host = strings.ToLower(host)
|
|
||||||
|
|
||||||
// Extract query string if present in path
|
// Extract query string if present in path
|
||||||
if qIndex := strings.Index(path, "?"); qIndex != -1 {
|
if qIndex := strings.Index(path, "?"); qIndex != -1 {
|
||||||
query = path[qIndex+1:]
|
query = path[qIndex+1:]
|
||||||
|
|||||||
@@ -234,49 +234,6 @@ func TestParseImagePath(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestParseImagePath_LowerCasesHost verifies that the host is lower-cased, so
|
|
||||||
// that every spelling of its letter case is the same host to the allowlist,
|
|
||||||
// the upstream connection limits and the cache, and that the path keeps its
|
|
||||||
// case.
|
|
||||||
func TestParseImagePath_LowerCasesHost(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
got, err := ParseImagePath("CDN.Example.COM/Photos/Cat.jpg/800x600.webp")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ParseImagePath() error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Logf("parsed host %q, path %q", got.Host, got.Path)
|
|
||||||
|
|
||||||
if got.Host != testHostCDN {
|
|
||||||
t.Errorf("Host = %q, want %q", got.Host, testHostCDN)
|
|
||||||
}
|
|
||||||
|
|
||||||
if got.Path != "/Photos/Cat.jpg" {
|
|
||||||
t.Errorf("Path = %q, want %q", got.Path, "/Photos/Cat.jpg")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestParseImagePath_RefusesUserinfo verifies that a host holding "@" is
|
|
||||||
// refused: a URL reads what comes before the "@" as a user name and password
|
|
||||||
// and the rest as the host.
|
|
||||||
func TestParseImagePath_RefusesUserinfo(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, input := range []string{
|
|
||||||
"user@cdn.example.com/photos/cat.jpg/800x600.webp",
|
|
||||||
"user:password@cdn.example.com/photos/cat.jpg/800x600.webp",
|
|
||||||
} {
|
|
||||||
got, err := ParseImagePath(input)
|
|
||||||
t.Logf("ParseImagePath(%q) = %+v, error = %v", input, got, err)
|
|
||||||
|
|
||||||
if !errors.Is(err, ErrInvalidHost) {
|
|
||||||
t.Errorf("ParseImagePath(%q) error = %v, want %v",
|
|
||||||
input, err, ErrInvalidHost)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParsedURL_ToImageRequest(t *testing.T) {
|
func TestParsedURL_ToImageRequest(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -8,15 +8,12 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
basicauth "github.com/99designs/basicauth-go"
|
basicauth "github.com/99designs/basicauth-go"
|
||||||
"github.com/go-chi/chi/v5"
|
|
||||||
"github.com/go-chi/chi/v5/middleware"
|
"github.com/go-chi/chi/v5/middleware"
|
||||||
"github.com/go-chi/cors"
|
"github.com/go-chi/cors"
|
||||||
"github.com/go-chi/httprate"
|
"github.com/go-chi/httprate"
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
|
||||||
metrics "github.com/slok/go-http-metrics/metrics/prometheus"
|
metrics "github.com/slok/go-http-metrics/metrics/prometheus"
|
||||||
ghmm "github.com/slok/go-http-metrics/middleware"
|
ghmm "github.com/slok/go-http-metrics/middleware"
|
||||||
"github.com/slok/go-http-metrics/middleware/std"
|
"github.com/slok/go-http-metrics/middleware/std"
|
||||||
@@ -66,10 +63,6 @@ type Middleware struct {
|
|||||||
log *slog.Logger
|
log *slog.Logger
|
||||||
config *config.Config
|
config *config.Config
|
||||||
clientIP *clientip.Resolver
|
clientIP *clientip.Resolver
|
||||||
|
|
||||||
// metricsRegistry is the registry Metrics registers its metrics with.
|
|
||||||
// nil is Prometheus' default registry, the one /metrics serves.
|
|
||||||
metricsRegistry prometheus.Registerer
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// New creates a new Middleware instance.
|
// New creates a new Middleware instance.
|
||||||
@@ -216,61 +209,17 @@ func (s *Middleware) CORS() func(http.Handler) http.Handler {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// Metrics returns a Prometheus metrics middleware. Its handler label is the
|
// Metrics returns a Prometheus metrics middleware.
|
||||||
// pattern of the chi route a request matches, such as /v1/image/*, or
|
|
||||||
// "unmatched" for a request that matches none, so the routes fix how many
|
|
||||||
// values the label has. It is never the path, with which any client could
|
|
||||||
// add series to the registry without limit.
|
|
||||||
func (s *Middleware) Metrics() func(http.Handler) http.Handler {
|
func (s *Middleware) Metrics() func(http.Handler) http.Handler {
|
||||||
mdlw := ghmm.New(ghmm.Config{
|
mdlw := ghmm.New(ghmm.Config{
|
||||||
Recorder: metrics.NewRecorder(metrics.Config{
|
Recorder: metrics.NewRecorder(metrics.Config{}),
|
||||||
Registry: s.metricsRegistry,
|
|
||||||
}),
|
|
||||||
})
|
})
|
||||||
|
|
||||||
return func(next http.Handler) http.Handler {
|
return func(next http.Handler) http.Handler {
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
return std.Handler("", mdlw, next)
|
||||||
std.Handler(routePattern(r), mdlw, next).ServeHTTP(w, r)
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// routePattern returns the pattern of the chi route r matches, or
|
|
||||||
// "unmatched" when it matches none or is not served by a chi router. The
|
|
||||||
// metrics library takes the label before the request is routed, so the
|
|
||||||
// router is asked which route matches, with the path it routes on:
|
|
||||||
// r.URL.RawPath when set, else r.URL.Path.
|
|
||||||
func routePattern(r *http.Request) string {
|
|
||||||
pattern := ""
|
|
||||||
|
|
||||||
rctx := chi.RouteContext(r.Context())
|
|
||||||
if rctx != nil {
|
|
||||||
path := r.URL.RawPath
|
|
||||||
if path == "" {
|
|
||||||
path = r.URL.Path
|
|
||||||
}
|
|
||||||
|
|
||||||
pattern = rctx.Routes.Find(chi.NewRouteContext(), r.Method, path)
|
|
||||||
|
|
||||||
// For the path a subrouter is mounted at, such as /v1 or /v1/, Find
|
|
||||||
// answers that path without asking the subrouter, which chi hands
|
|
||||||
// the request as /.
|
|
||||||
for _, route := range rctx.Routes.Routes() {
|
|
||||||
if route.SubRoutes != nil &&
|
|
||||||
route.Pattern == strings.TrimSuffix(pattern, "/")+"/*" &&
|
|
||||||
!route.SubRoutes.Match(chi.NewRouteContext(), r.Method, "/") {
|
|
||||||
pattern = ""
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if pattern == "" {
|
|
||||||
return "unmatched"
|
|
||||||
}
|
|
||||||
|
|
||||||
return pattern
|
|
||||||
}
|
|
||||||
|
|
||||||
// MetricsAuth returns a basic auth middleware for the metrics endpoint.
|
// MetricsAuth returns a basic auth middleware for the metrics endpoint.
|
||||||
func (s *Middleware) MetricsAuth() func(http.Handler) http.Handler {
|
func (s *Middleware) MetricsAuth() func(http.Handler) http.Handler {
|
||||||
return basicauth.New(
|
return basicauth.New(
|
||||||
|
|||||||
@@ -3,15 +3,12 @@ package middleware
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"maps"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"net/url"
|
"net/url"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/go-chi/chi/v5"
|
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
|
||||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||||
|
|
||||||
"sneak.berlin/go/pixa/internal/config"
|
"sneak.berlin/go/pixa/internal/config"
|
||||||
@@ -201,7 +198,7 @@ func TestMetricsRecordsServedRequest(t *testing.T) {
|
|||||||
|
|
||||||
// The line /metrics shows once one GET /test has been served.
|
// The line /metrics shows once one GET /test has been served.
|
||||||
const want = `http_request_duration_seconds_count{` +
|
const want = `http_request_duration_seconds_count{` +
|
||||||
`code="200",handler="unmatched",method="GET",service=""} 1`
|
`code="200",handler="/test",method="GET",service=""} 1`
|
||||||
|
|
||||||
mw := &Middleware{log: slog.Default(), config: &config.Config{}}
|
mw := &Middleware{log: slog.Default(), config: &config.Config{}}
|
||||||
|
|
||||||
@@ -223,86 +220,6 @@ func TestMetricsRecordsServedRequest(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestMetricsLabelsRequestsByRoute checks that the metrics middleware labels
|
|
||||||
// each request with the pattern of the route that answers it, never with its
|
|
||||||
// path: two paths one route answers share that route's pattern, and requests
|
|
||||||
// no route answers, a 404 or a 405, share "unmatched", among them requests for
|
|
||||||
// the path the subrouter is mounted at, /v1 and /v1/. With a label per path,
|
|
||||||
// any client could add series to the registry without limit. The middleware
|
|
||||||
// registers with a registry of its own here, as another test in this package
|
|
||||||
// registers with Prometheus' default registry, which can be done only once.
|
|
||||||
func TestMetricsLabelsRequestsByRoute(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
registry := prometheus.NewRegistry()
|
|
||||||
mw := &Middleware{
|
|
||||||
log: slog.Default(),
|
|
||||||
config: &config.Config{},
|
|
||||||
metricsRegistry: registry,
|
|
||||||
}
|
|
||||||
|
|
||||||
// The image route is in a subrouter, as in pixa's router.
|
|
||||||
router := chi.NewRouter()
|
|
||||||
router.Use(mw.Metrics())
|
|
||||||
router.Route("/v1", func(r chi.Router) {
|
|
||||||
r.Get("/image/*", func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
for _, req := range []struct{ method, path string }{
|
|
||||||
{http.MethodGet, "/v1/image/a.example.com/one.jpg/1x1.png"},
|
|
||||||
{http.MethodGet, "/v1/image/b.example.com/two.jpg/1x1.png"},
|
|
||||||
{http.MethodGet, "/no-such-route"},
|
|
||||||
{http.MethodGet, "/v1/no-such-route"},
|
|
||||||
{http.MethodGet, "/v1"},
|
|
||||||
{http.MethodGet, "/v1/"},
|
|
||||||
{http.MethodPost, "/v1/image/a.example.com/one.jpg/1x1.png"},
|
|
||||||
} {
|
|
||||||
router.ServeHTTP(httptest.NewRecorder(), httptest.NewRequestWithContext(
|
|
||||||
t.Context(), req.method, req.path, nil))
|
|
||||||
}
|
|
||||||
|
|
||||||
got := requestCountsByHandler(t, registry)
|
|
||||||
t.Logf("requests counted by handler label: %v", got)
|
|
||||||
|
|
||||||
want := map[string]uint64{"/v1/image/*": 2, "unmatched": 5}
|
|
||||||
if !maps.Equal(got, want) {
|
|
||||||
t.Errorf("requests counted by handler label = %v, want %v", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// requestCountsByHandler returns how many requests the request duration
|
|
||||||
// histogram in registry counted under each value of its handler label.
|
|
||||||
func requestCountsByHandler(
|
|
||||||
t *testing.T, registry *prometheus.Registry,
|
|
||||||
) map[string]uint64 {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
families, err := registry.Gather()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("registry.Gather() error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
counts := map[string]uint64{}
|
|
||||||
|
|
||||||
for _, family := range families {
|
|
||||||
if family.GetName() != "http_request_duration_seconds" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, metric := range family.GetMetric() {
|
|
||||||
for _, label := range metric.GetLabel() {
|
|
||||||
if label.GetName() == "handler" {
|
|
||||||
counts[label.GetValue()] += metric.GetHistogram().GetSampleCount()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return counts
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLoggingLeavesOutSubmittedSigningKey checks that a login, a POST /
|
// TestLoggingLeavesOutSubmittedSigningKey checks that a login, a POST /
|
||||||
// whose form carries the signing key, leaves no trace of the key in the
|
// whose form carries the signing key, leaves no trace of the key in the
|
||||||
// request's log line.
|
// request's log line.
|
||||||
|
|||||||
Reference in New Issue
Block a user