Compare commits
8
Commits
v1.0.0-rc2
...
6572b3fe08
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6572b3fe08 | ||
|
|
1ba4c8d39f | ||
|
|
74ce122dc4 | ||
|
|
2790125cb2 | ||
|
|
c23585dd6d | ||
|
|
5d1e3ea871 | ||
|
|
761ade2e36 | ||
|
|
88e1b1cdfa |
@@ -26,6 +26,11 @@ make docker
|
|||||||
docker run -p 8080:8080 -e PIXA_SIGNING_KEY="$(openssl rand -base64 32)" pixa:latest
|
docker run -p 8080:8080 -e PIXA_SIGNING_KEY="$(openssl rand -base64 32)" pixa:latest
|
||||||
```
|
```
|
||||||
|
|
||||||
|
The example config allowlists no host (`allowlist_hosts`), so pixa serves an
|
||||||
|
image only for a signed or encrypted URL. Every image on an allowlisted host is
|
||||||
|
served to anyone without a signature, at any size and in any format, so
|
||||||
|
allowlist only hosts you control, where nobody else can put images.
|
||||||
|
|
||||||
A container takes its settings from environment variables (see Configuration
|
A container takes its settings from environment variables (see Configuration
|
||||||
below for the list). Only `PIXA_SIGNING_KEY` is required; if it is unset the
|
below for the list). Only `PIXA_SIGNING_KEY` is required; if it is unset the
|
||||||
container exits at startup naming the variable. Everything else has a built-in
|
container exits at startup naming the variable. Everything else has a built-in
|
||||||
@@ -115,7 +120,9 @@ What the [upaas](https://git.eeqj.de/sneak/upaas) app for pixa needs:
|
|||||||
- `PIXA_SIGNING_KEY` (required): secret for signed and encrypted URLs and
|
- `PIXA_SIGNING_KEY` (required): secret for signed and encrypted URLs and
|
||||||
login, 32+ characters, for example from `openssl rand -base64 32`
|
login, 32+ characters, for example from `openssl rand -base64 32`
|
||||||
- `PIXA_ALLOWLIST_HOSTS`: upstream hosts served without a signature,
|
- `PIXA_ALLOWLIST_HOSTS`: upstream hosts served without a signature,
|
||||||
comma-separated
|
comma-separated; default none. Every image on a listed host is served to
|
||||||
|
anyone without a signature, so list only hosts you control, where nobody
|
||||||
|
else can put images.
|
||||||
- `PIXA_CACHE_MAX_BYTES`: disk cache limit in bytes; `0` disables it;
|
- `PIXA_CACHE_MAX_BYTES`: disk cache limit in bytes; `0` disables it;
|
||||||
default 75% of (free space + what the cache holds)
|
default 75% of (free space + what the cache holds)
|
||||||
- the rest are in the table under Configuration below
|
- the rest are in the table under Configuration below
|
||||||
@@ -193,9 +200,10 @@ path under `/v1/` answers 200, in maintenance mode too.
|
|||||||
`image_requests_per_minute` (below); 415 when the upstream answers with a
|
`image_requests_per_minute` (below); 415 when the upstream answers with a
|
||||||
`Content-Type` that is not one of the source formats (below), or with an image
|
`Content-Type` that is not one of the source formats (below), or with an image
|
||||||
whose first bytes libvips reads as another format; 502 when the upstream
|
whose first bytes libvips reads as another format; 502 when the upstream
|
||||||
answered with an error status, and for 5 minutes after that for the same
|
answered with an error status, and for 5 minutes after that for an image of
|
||||||
source URL; 503 when pixa is busy or in maintenance mode; 500 for any other
|
the same source URL that pixa can neither serve from its cache nor make from
|
||||||
failure.
|
the source image still in its cache; 503 when pixa is busy or in maintenance
|
||||||
|
mode; 500 for any other failure.
|
||||||
- `GET` or `HEAD` `/v1/e/<token>/<name>` — an image through an encrypted URL
|
- `GET` or `HEAD` `/v1/e/<token>/<name>` — an image through an encrypted URL
|
||||||
(see Encrypted URLs). Needs: nothing but the URL. Answers: 200; 304 when
|
(see Encrypted URLs). Needs: nothing but the URL. Answers: 200; 304 when
|
||||||
`If-None-Match` matches the image's `ETag`; 400 for a token that does not
|
`If-None-Match` matches the image's `ETag`; 400 for a token that does not
|
||||||
@@ -306,9 +314,23 @@ cache at once, they share one upstream fetch (or one read of the cached source)
|
|||||||
and one transcode: the first request does the work, and the others wait for its
|
and one transcode: the first request does the work, and the others wait for its
|
||||||
image or its error, holding no upstream connection or processing slot of their
|
image or its error, holding no upstream connection or processing slot of their
|
||||||
own. A waiting request stops waiting when its own client goes away. The work
|
own. A waiting request stops waiting when its own client goes away. The work
|
||||||
goes on for the others even if the first request's client goes away, until that
|
goes on for the others even if the first request's client goes away. The shared
|
||||||
request's `downstream_timeout` ends. The shared fetch sends the first request's
|
fetch sends the first request's ID upstream, and the lines logged for the fetch
|
||||||
ID upstream, and the lines logged for the fetch and the transcode carry that ID.
|
and the transcode carry that ID.
|
||||||
|
|
||||||
|
A transcode whose request's `downstream_timeout` has ended (for a shared one,
|
||||||
|
the first request's) stops when it gets its processing slot or before it begins
|
||||||
|
encoding, and gives the slot back. libvips cannot stop an encode once it has
|
||||||
|
begun, so an encode keeps its slot until it finishes, which can be after
|
||||||
|
`downstream_timeout`. The limit of 8192 on each side of the source and of the
|
||||||
|
output bounds how long that takes. With one libvips thread, as pixad runs it,
|
||||||
|
encoding 8192x8192 random pixels takes up to about 1 second as JPEG, 15 as PNG,
|
||||||
|
16 as GIF, 63 as WebP, 110 as AVIF and 96 as JPEG XL, the slowest of 8 or 16
|
||||||
|
bits per sample, with or without an alpha channel. WebP, AVIF and JPEG XL are
|
||||||
|
that slow only with an alpha channel: 8-bit RGB takes about 44, 53 and 28
|
||||||
|
seconds. Decoding such a source adds up to about 11 seconds. A WebP, AVIF or
|
||||||
|
JPEG XL encode can therefore hold its slot past the default `downstream_timeout`
|
||||||
|
of 60 seconds.
|
||||||
|
|
||||||
The login form (`POST /`) is limited to 5 attempts per minute per client
|
The login form (`POST /`) is limited to 5 attempts per minute per client
|
||||||
address, counting an IPv6 client by its /64; an attempt over the limit is
|
address, counting an IPv6 client by its /64; an attempt over the limit is
|
||||||
@@ -411,9 +433,14 @@ HMAC-SHA256(secret, "host:path:query:width:height:format:expiration:quality:fit"
|
|||||||
|
|
||||||
Where:
|
Where:
|
||||||
|
|
||||||
- `host` — source origin hostname (e.g. `cdn.example.com`)
|
- `host` — source host as the URL writes it up to the first `/`, in lower case,
|
||||||
- `path` — source path (e.g. `/photos/cat.jpg`)
|
keeping an IPv6 address's brackets and any `:` and port after it (e.g.
|
||||||
- `query` — source query string, empty string if none
|
`cdn.example.com`, `cdn.example.com:8443`, `[2001:db8::1]:8443`)
|
||||||
|
- `path` — source path (e.g. `/photos/cat.jpg`), with each `%` written as `%25`
|
||||||
|
and each `:` as `%3A`, so `/photos/a:b.jpg` is signed as `/photos/a%3Ab.jpg`
|
||||||
|
- `query` — source query string, empty string if none, with `%` and `:` written
|
||||||
|
as in `path`; neither then holds a `:`, so path `/a:b` with no query and path
|
||||||
|
`/a` with query `b:` are signed differently
|
||||||
- `width` — requested width in pixels, `0` for original
|
- `width` — requested width in pixels, `0` for original
|
||||||
- `height` — requested height in pixels, `0` for original
|
- `height` — requested height in pixels, `0` for original
|
||||||
- `format` — output format, one of those listed under Routes, with `original`
|
- `format` — output format, one of those listed under Routes, with `original`
|
||||||
@@ -463,6 +490,12 @@ entry that is neither a host name (letters, digits, hyphens, underscores and
|
|||||||
dots, with at most one leading dot) nor an IP address, such as one with a port
|
dots, with at most one leading dot) nor an IP address, such as one with a port
|
||||||
or a `*.` wildcard, aborts startup.
|
or a `*.` wildcard, aborts startup.
|
||||||
|
|
||||||
|
pixa lower-cases the host of a `/v1/image/` URL before anything else uses it, so
|
||||||
|
`CDN.Example.com` is matched, fetched, cached and counted against
|
||||||
|
`upstream_connections_per_host` as `cdn.example.com`. Such a URL whose host has
|
||||||
|
a port, as in `cdn.example.com:8443`, or a `:` with no port, needs a signature
|
||||||
|
even when the host is allowlisted. A host holding `@` is refused with 400.
|
||||||
|
|
||||||
### Configuration
|
### Configuration
|
||||||
|
|
||||||
Every setting can be given as an environment variable, in a YAML config file
|
Every setting can be given as an environment variable, in a YAML config file
|
||||||
|
|||||||
@@ -30,6 +30,67 @@ P2: security: per-origin rate limiting
|
|||||||
|
|
||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
|
- 2026-10-15 a cached variant is served while its source URL is in the negative
|
||||||
|
cache (closes #245): `Get` checked the negative cache before the variant
|
||||||
|
cache, so once a fetch of the source failed, as one does when another variant
|
||||||
|
misses after the source was evicted and the upstream is down, every variant of
|
||||||
|
that source answered 502 for 5 minutes, also one on disk or one that could be
|
||||||
|
made from the cached source. The negative cache is now checked only just
|
||||||
|
before a fetch from the upstream, so a cached variant is served and a variant
|
||||||
|
not in the cache is made from the cached source when there is one. `README.md`
|
||||||
|
says so under `/v1/image/`. A refusal from the negative cache now counts as a
|
||||||
|
miss, as any other request that misses the variant cache does.
|
||||||
|
- 2026-10-15 the host of a `/v1/image/` URL is lower-cased, and one with a port
|
||||||
|
needs a signature (closes #246): with `cdn.example.com` allowlisted,
|
||||||
|
`CDN.Example.com` and `cdn.example.com:8443` passed the allowlist too, and
|
||||||
|
each spelling was fetched and cached on its own and had connections of its own
|
||||||
|
under `upstream_connections_per_host`. The image URL parser now lower-cases
|
||||||
|
the host and refuses one holding `@`, which a URL reads as a user name and
|
||||||
|
password, with 400. A host with a port, or with a `:` and no port, needs a
|
||||||
|
signature even when it is allowlisted. That is checked in the image service,
|
||||||
|
not in `allowlist.IsAllowed`, which also matches `referer_blocklist`, where a
|
||||||
|
`Referer` is blocked whatever its port. A URL is signed over its host as it
|
||||||
|
writes it, in lower case, with any IPv6 brackets, `:` and port kept.
|
||||||
|
- 2026-10-15 a `:` in the source path or query no longer moves where the path
|
||||||
|
ends and the query begins (closes #247): the signed data and the cache key
|
||||||
|
joined host, path and query with `:` and no escaping, so a URL signed for
|
||||||
|
`https://h/a:b` verified for `https://h/a?b:`; the two also shared a cache
|
||||||
|
key, and whichever was fetched first was served for both. In both, the path
|
||||||
|
and the query now have each `%` written as `%25` and each `:` as `%3A`; a path
|
||||||
|
and query holding neither are signed and keyed as before. Signature
|
||||||
|
Specification in `README.md` says so.
|
||||||
|
- 2026-10-15 a transcode stops once its request's `downstream_timeout` has
|
||||||
|
ended, unless it has begun encoding (closes #243): the image processor looked
|
||||||
|
at its context only while it waited for a processing slot, so an image was
|
||||||
|
decoded and encoded, holding the slot, however long after its request's
|
||||||
|
deadline that took. `Process` now returns the context's error, and gives the
|
||||||
|
slot back, when the context has ended by the time it has the slot or before it
|
||||||
|
encodes. libvips cannot stop an encode once it has begun, so that encode keeps
|
||||||
|
its slot until it finishes; the limit of 8192 on each side of the source and
|
||||||
|
of the output bounds how long that takes. With one libvips thread, 8192x8192
|
||||||
|
random pixels take up to about 1 second to encode as JPEG, 15 as PNG, 16 as
|
||||||
|
GIF, 63 as WebP, 110 as AVIF and 96 as JPEG XL, the last three with an alpha
|
||||||
|
channel, plus up to about 11 seconds to decode the source; `README.md` gives
|
||||||
|
these times and no longer says the work stops when `downstream_timeout` ends.
|
||||||
|
The settings of the WebP, AVIF and JPEG XL encoders, which can hold a slot
|
||||||
|
past the default `downstream_timeout` of 60 seconds, are unchanged.
|
||||||
|
- 2026-10-15 the example config allowlists no host (closes #249): it allowlisted
|
||||||
|
`github.com` and `user-images.githubusercontent.com`, where anyone can put an
|
||||||
|
image, so a deployment set up as Getting Started says served those images to
|
||||||
|
anyone without a signature, at any size and in any format.
|
||||||
|
`configs/config.example.yml` now leaves `allowlist_hosts` out, with one
|
||||||
|
commented entry under `example.com`. Getting Started and "Running under upaas"
|
||||||
|
in `README.md` say that every image on an allowlisted host is served without a
|
||||||
|
signature, so an allowlisted host must be one the operator controls.
|
||||||
|
- 2026-10-15 a chunked upstream body of exactly `upstream_max_response_size`
|
||||||
|
bytes is accepted (closes #248): once the limit was used up, the fetcher's
|
||||||
|
next read refused the body without looking for its end. Go's HTTP client
|
||||||
|
returns the end of a body sent with a `Content-Length` together with its last
|
||||||
|
bytes, but the end of a chunked body can come in a later read, so such a body
|
||||||
|
was refused after it had been downloaded whole, and the image routes
|
||||||
|
answered 500. Once the limit is used up, the fetcher now reads one more byte:
|
||||||
|
if the body has ended it is accepted, and if there is more it is refused with
|
||||||
|
`ErrResponseTooLarge`, as before.
|
||||||
- 2026-10-13 `fit=outside` no longer makes an output above 8192 on a side
|
- 2026-10-13 `fit=outside` no longer makes an output above 8192 on a side
|
||||||
(closes #257): it scales the image to cover the requested size, and the side
|
(closes #257): it scales the image to cover the requested size, and the side
|
||||||
scaled past that size was held to no maximum, so `1x8192?fit=outside` of a
|
scaled past that size was held to no maximum, so `1x8192?fit=outside` of a
|
||||||
@@ -476,11 +537,11 @@ P2: security: per-origin rate limiting
|
|||||||
cancellation but keeps its deadline, and the others wait for its image or
|
cancellation but keeps its deadline, and the others wait for its image or
|
||||||
error holding no upstream connection or processing slot, and stop waiting when
|
error holding no upstream connection or processing slot, and stop waiting when
|
||||||
their own context ends; the request doing the processing waits for it even
|
their own context ends; the request doing the processing waits for it even
|
||||||
then, up to its deadline; a request whose context has already ended starts
|
then, until it ends, which can be after its deadline; a request whose context
|
||||||
nothing; each request counts one miss, and the processing counts its fetch and
|
has already ended starts nothing; each request counts one miss, and the
|
||||||
transcode once; a panic while processing is reported to Sentry when
|
processing counts its fetch and transcode once; a panic while processing is
|
||||||
`sentry_dsn` is set and becomes an error for every waiting request instead of
|
reported to Sentry when `sentry_dsn` is set and becomes an error for every
|
||||||
stopping pixad; documented in `README.md`.
|
waiting request instead of stopping pixad; documented in `README.md`.
|
||||||
- 2026-09-29 only the image routes send CORS headers (closes #98): the CORS
|
- 2026-09-29 only the image routes send CORS headers (closes #98): the CORS
|
||||||
middleware, with the `access_control_allow_origin` origin, moved from the
|
middleware, with the `access_control_allow_origin` origin, moved from the
|
||||||
router root onto a `/v1` subrouter holding `/v1/image/` and `/v1/e/`, where it
|
router root onto a `/v1` subrouter holding `/v1/image/` and `/v1/e/`, where it
|
||||||
|
|||||||
@@ -45,15 +45,14 @@ state_dir: ./data
|
|||||||
signing_key: "CHANGE_ME_generate_with_openssl_rand_base64_32"
|
signing_key: "CHANGE_ME_generate_with_openssl_rand_base64_32"
|
||||||
|
|
||||||
# Hosts that don't require signatures (default: none)
|
# Hosts that don't require signatures (default: none)
|
||||||
|
# Every image on an allowlisted host is served to anyone without a
|
||||||
|
# signature, at any size and in any format, so list only hosts you
|
||||||
|
# control, where nobody else can put images.
|
||||||
# Use "." prefix for wildcard subdomain matching (e.g., ".example.com" matches "cdn.example.com")
|
# Use "." prefix for wildcard subdomain matching (e.g., ".example.com" matches "cdn.example.com")
|
||||||
# An entry that is neither a host name nor an IP address (IPv6 without
|
# An entry that is neither a host name nor an IP address (IPv6 without
|
||||||
# brackets), such as one with a port or a "*." wildcard, aborts startup.
|
# brackets), such as one with a port or a "*." wildcard, aborts startup.
|
||||||
allowlist_hosts:
|
# allowlist_hosts:
|
||||||
- s3.sneak.cloud
|
# - images.example.com
|
||||||
- static.sneak.cloud
|
|
||||||
- sneak.berlin
|
|
||||||
- github.com
|
|
||||||
- user-images.githubusercontent.com
|
|
||||||
|
|
||||||
# Hosts whose pages may not show pixa's images, written as for
|
# Hosts whose pages may not show pixa's images, written as for
|
||||||
# allowlist_hosts. A request to /v1/image/ or /v1/e/ whose Referer header
|
# allowlist_hosts. A request to /v1/image/ or /v1/e/ whose Referer header
|
||||||
|
|||||||
@@ -92,6 +92,32 @@ func TestGetStringSlice_Empty(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestExampleConfigAllowlistsNoHost loads configs/config.example.yml
|
||||||
|
// with a real signing key, as Getting Started in README.md has it copied
|
||||||
|
// and edited, and checks that it allowlists no host: every image on an
|
||||||
|
// allowlisted host is served to anyone without a signature.
|
||||||
|
func TestExampleConfigAllowlistsNoHost(t *testing.T) {
|
||||||
|
t.Setenv("PIXA_SIGNING_KEY", validTestSigningKey)
|
||||||
|
|
||||||
|
examplePath := filepath.Join("..", "..", "configs", "config.example.yml")
|
||||||
|
|
||||||
|
sc, err := loadTestConfig(examplePath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to load %s: %v", examplePath, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
c, err := newFromSmartConfig(sc)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s is not a valid config: %v", examplePath, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("allowlist_hosts in %s: %v", examplePath, c.AllowlistHosts)
|
||||||
|
|
||||||
|
if len(c.AllowlistHosts) != 0 {
|
||||||
|
t.Errorf("%s allowlists %v, want no host", examplePath, c.AllowlistHosts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// loadTestConfig is a helper to load a config file for testing.
|
// loadTestConfig is a helper to load a config file for testing.
|
||||||
func loadTestConfig(path string) (*smartconfig.Config, error) {
|
func loadTestConfig(path string) (*smartconfig.Config, error) {
|
||||||
return smartconfig.NewFromConfigPath(path)
|
return smartconfig.NewFromConfigPath(path)
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package handlers
|
package handlers
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/http"
|
"net/http"
|
||||||
@@ -47,3 +48,39 @@ func TestServerBusyAnswers503(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestProcessingTimedOutAnswers503 checks that both image routes answer 503
|
||||||
|
// with the server busy error when the image service reports that processing
|
||||||
|
// stopped at the deadline of the request doing it, wrapped as the service
|
||||||
|
// wraps it. A request that waited for that processing gets this error while
|
||||||
|
// its own deadline may have time left.
|
||||||
|
func TestProcessingTimedOutAnswers503(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
h := &Handlers{log: slog.New(slog.DiscardHandler)}
|
||||||
|
req := &imgcache.ImageRequest{SourceHost: "img.example.com", SourcePath: "/a.jpg"}
|
||||||
|
err := fmt.Errorf("%w: failed to encode: %w",
|
||||||
|
imgcache.ErrProcessingTimedOut, context.DeadlineExceeded)
|
||||||
|
|
||||||
|
plain := httptest.NewRecorder()
|
||||||
|
h.respondImageError(plain, req, err)
|
||||||
|
|
||||||
|
encrypted := httptest.NewRecorder()
|
||||||
|
h.handleImageError(encrypted, err)
|
||||||
|
|
||||||
|
for route, rec := range map[string]*httptest.ResponseRecorder{
|
||||||
|
"/v1/image/": plain, "/v1/e/": encrypted,
|
||||||
|
} {
|
||||||
|
t.Logf("%s: status = %d, body = %q", route, rec.Code, rec.Body.String())
|
||||||
|
|
||||||
|
if rec.Code != http.StatusServiceUnavailable {
|
||||||
|
t.Errorf("%s: status = %d, want %d",
|
||||||
|
route, rec.Code, http.StatusServiceUnavailable)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.Contains(rec.Body.String(), "server busy, try again later") {
|
||||||
|
t.Errorf("%s: body = %q, want the server busy error",
|
||||||
|
route, rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"slices"
|
"slices"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -297,6 +298,139 @@ func TestFetchMaxResponseSizeEnforced(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// fetchWithMaxResponseSize serves handler as the fake upstream and fetches
|
||||||
|
// its /image route with MaxResponseSize set to maxResponseSize.
|
||||||
|
func fetchWithMaxResponseSize(
|
||||||
|
t *testing.T,
|
||||||
|
maxResponseSize int64,
|
||||||
|
handler http.HandlerFunc,
|
||||||
|
) *FetchResult {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
srv := httptest.NewServer(handler)
|
||||||
|
t.Cleanup(srv.Close)
|
||||||
|
|
||||||
|
cfg := DefaultConfig()
|
||||||
|
cfg.MaxResponseSize = maxResponseSize
|
||||||
|
|
||||||
|
f, _ := newServerFetcher(t, srv, cfg)
|
||||||
|
|
||||||
|
res := fetchImage(t, f, "/image")
|
||||||
|
t.Cleanup(func() { _ = res.Content.Close() })
|
||||||
|
|
||||||
|
return res
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestFetchContentLengthBodyAtMaxResponseSize checks that a body sent with a
|
||||||
|
// Content-Length is read whole at exactly MaxResponseSize bytes and refused
|
||||||
|
// at one byte more.
|
||||||
|
func TestFetchContentLengthBodyAtMaxResponseSize(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const maxResponseSize = 8
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
size int
|
||||||
|
wantErr error
|
||||||
|
}{
|
||||||
|
{maxResponseSize, nil},
|
||||||
|
{maxResponseSize + 1, ErrResponseTooLarge},
|
||||||
|
} {
|
||||||
|
t.Run(strconv.Itoa(tc.size)+" bytes", func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
body := strings.Repeat("x", tc.size)
|
||||||
|
|
||||||
|
res := fetchWithMaxResponseSize(t, maxResponseSize,
|
||||||
|
func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
w.Header().Set("Content-Type", contentTypeJPEG)
|
||||||
|
w.Header().Set("Content-Length", strconv.Itoa(tc.size))
|
||||||
|
_, _ = io.WriteString(w, body)
|
||||||
|
})
|
||||||
|
|
||||||
|
if res.ContentLength != int64(tc.size) {
|
||||||
|
t.Fatalf("ContentLength = %d, want %d", res.ContentLength, tc.size)
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := io.ReadAll(res.Content)
|
||||||
|
t.Logf("body of %d bytes: read %d, error %v", tc.size, len(data), err)
|
||||||
|
|
||||||
|
if !errors.Is(err, tc.wantErr) {
|
||||||
|
t.Fatalf("read error = %v, want %v", err, tc.wantErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err == nil && string(data) != body {
|
||||||
|
t.Errorf("read %q, want %q", data, body)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestFetchChunkedBodyAtMaxResponseSize checks that a chunked body is read
|
||||||
|
// whole at exactly MaxResponseSize bytes and refused at one byte more. The
|
||||||
|
// upstream sends the end of the body only once the test has read
|
||||||
|
// MaxResponseSize bytes, so the read after them comes before the end of the
|
||||||
|
// body has arrived, as it can from a real upstream.
|
||||||
|
func TestFetchChunkedBodyAtMaxResponseSize(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const maxResponseSize = 8
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
size int
|
||||||
|
wantErr error
|
||||||
|
}{
|
||||||
|
{maxResponseSize, nil},
|
||||||
|
{maxResponseSize + 1, ErrResponseTooLarge},
|
||||||
|
} {
|
||||||
|
t.Run(strconv.Itoa(tc.size)+" bytes", func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
body := strings.Repeat("x", tc.size)
|
||||||
|
endBody := make(chan struct{})
|
||||||
|
|
||||||
|
res := fetchWithMaxResponseSize(t, maxResponseSize,
|
||||||
|
func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("Content-Type", contentTypeJPEG)
|
||||||
|
_, _ = io.WriteString(w, body)
|
||||||
|
// With no Content-Length, flushing sends the body
|
||||||
|
// chunked; its end is sent when the handler returns.
|
||||||
|
_ = http.NewResponseController(w).Flush()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-endBody:
|
||||||
|
case <-r.Context().Done():
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
if res.ContentLength != -1 {
|
||||||
|
t.Fatalf("ContentLength = %d, want -1 (chunked)", res.ContentLength)
|
||||||
|
}
|
||||||
|
|
||||||
|
head := make([]byte, maxResponseSize)
|
||||||
|
|
||||||
|
_, err := io.ReadFull(res.Content, head)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read the first %d bytes: %v", maxResponseSize, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
close(endBody)
|
||||||
|
|
||||||
|
rest, err := io.ReadAll(res.Content)
|
||||||
|
data := string(head) + string(rest)
|
||||||
|
t.Logf("body of %d bytes: read %d, error %v", tc.size, len(data), err)
|
||||||
|
|
||||||
|
if !errors.Is(err, tc.wantErr) {
|
||||||
|
t.Fatalf("read error = %v, want %v", err, tc.wantErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err == nil && data != body {
|
||||||
|
t.Errorf("read %q, want %q", data, body)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestFetchSemaphoreReleasedOnError(t *testing.T) {
|
func TestFetchSemaphoreReleasedOnError(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -660,14 +660,33 @@ func dialSSRFSafe(
|
|||||||
return nil, fmt.Errorf("%w to %s", errConnectFailed, host)
|
return nil, fmt.Errorf("%w to %s", errConnectFailed, host)
|
||||||
}
|
}
|
||||||
|
|
||||||
// limitedReader wraps a reader and limits the number of bytes read.
|
// limitedReader wraps a reader and refuses a body longer than remaining
|
||||||
|
// bytes with ErrResponseTooLarge. remaining goes below 0 once the body is
|
||||||
|
// found to be longer.
|
||||||
type limitedReader struct {
|
type limitedReader struct {
|
||||||
reader io.Reader
|
reader io.Reader
|
||||||
remaining int64
|
remaining int64
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Read reads at most the remaining bytes. Once they are read, it reads one
|
||||||
|
// more byte to tell the end of the body from more data: at the end it
|
||||||
|
// returns the wrapped reader's error, io.EOF; with more data it returns
|
||||||
|
// ErrResponseTooLarge, as does every later Read.
|
||||||
func (r *limitedReader) Read(p []byte) (int, error) {
|
func (r *limitedReader) Read(p []byte) (int, error) {
|
||||||
if r.remaining <= 0 {
|
if r.remaining < 0 {
|
||||||
|
return 0, ErrResponseTooLarge
|
||||||
|
}
|
||||||
|
|
||||||
|
if r.remaining == 0 {
|
||||||
|
var extra [1]byte
|
||||||
|
|
||||||
|
n, err := r.reader.Read(extra[:])
|
||||||
|
r.remaining -= int64(n)
|
||||||
|
|
||||||
|
if n == 0 {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
return 0, ErrResponseTooLarge
|
return 0, ErrResponseTooLarge
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,106 @@
|
|||||||
|
package imageprocessor
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// readerAfterDeadline is a Process input whose first Read waits until done
|
||||||
|
// is closed, so the deadline passes after Process has taken its slot and
|
||||||
|
// before it encodes.
|
||||||
|
type readerAfterDeadline struct {
|
||||||
|
data *bytes.Reader
|
||||||
|
done <-chan struct{}
|
||||||
|
waited bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *readerAfterDeadline) Read(p []byte) (int, error) {
|
||||||
|
if !r.waited {
|
||||||
|
r.waited = true
|
||||||
|
|
||||||
|
<-r.done
|
||||||
|
}
|
||||||
|
|
||||||
|
return r.data.Read(p)
|
||||||
|
}
|
||||||
|
|
||||||
|
// inputNotToBeRead is a Process input that fails the test when it is read.
|
||||||
|
type inputNotToBeRead struct {
|
||||||
|
t *testing.T
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r inputNotToBeRead) Read([]byte) (int, error) {
|
||||||
|
r.t.Error("Process read its input although its deadline had passed")
|
||||||
|
|
||||||
|
return 0, io.EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestProcessStopsWhenDeadlinePassed checks that Process returns
|
||||||
|
// context.DeadlineExceeded, and holds no slot afterwards, when its context's
|
||||||
|
// deadline has passed before it starts, then without reading its input, or
|
||||||
|
// while it reads its input. govips gives pixa no way to stop libvips once an
|
||||||
|
// encode has begun, so Process must not begin one for a request that has
|
||||||
|
// ended.
|
||||||
|
func TestProcessStopsWhenDeadlinePassed(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
input := createTestJPEG(t, 50, 50)
|
||||||
|
|
||||||
|
t.Run("before Process starts", func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
ctx, cancel := context.WithDeadline(t.Context(),
|
||||||
|
time.Now().Add(-time.Second))
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
// With a slot free, the select in acquireSlot takes the slot or sees
|
||||||
|
// the ended context at random, so Process is called many times
|
||||||
|
for range 100 {
|
||||||
|
checkProcessStopsAtDeadline(ctx, t, inputNotToBeRead{t: t})
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("while Process reads its input", func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
checkProcessStopsAtDeadline(ctx, t, &readerAfterDeadline{
|
||||||
|
data: bytes.NewReader(input), done: ctx.Done(),
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkProcessStopsAtDeadline runs Process on input with ctx, whose deadline
|
||||||
|
// has passed by the time Process could encode, and fails the test unless
|
||||||
|
// Process returns context.DeadlineExceeded and gives its slot back.
|
||||||
|
func checkProcessStopsAtDeadline(
|
||||||
|
ctx context.Context, t *testing.T, input io.Reader,
|
||||||
|
) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
proc := New(Params{MaxConcurrentProcessing: 1})
|
||||||
|
|
||||||
|
result, err := proc.Process(ctx, input, smallJPEGRequest())
|
||||||
|
if result != nil {
|
||||||
|
t.Logf("Process() made a %dx%d image after its deadline passed",
|
||||||
|
result.Width, result.Height)
|
||||||
|
|
||||||
|
_ = result.Content.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("Process() error = %v", err)
|
||||||
|
|
||||||
|
if !errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
t.Errorf("Process() error = %v, want context.DeadlineExceeded", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if held := len(proc.processingSemaphore); held != 0 {
|
||||||
|
t.Errorf("%d slots still held after Process() returned", held)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -208,7 +208,9 @@ func New(params Params) *ImageProcessor {
|
|||||||
// Process transforms an image according to the request. When
|
// Process transforms an image according to the request. When
|
||||||
// MaxConcurrentProcessing images are already being processed, it waits up
|
// MaxConcurrentProcessing images are already being processed, it waits up
|
||||||
// to ProcessingWaitTimeout for one to finish, then fails with
|
// to ProcessingWaitTimeout for one to finish, then fails with
|
||||||
// ErrTooManyImages.
|
// ErrTooManyImages. When ctx has ended once Process has a slot, or before it
|
||||||
|
// encodes, it returns ctx's error and gives the slot back. libvips cannot stop
|
||||||
|
// an encode once it has begun, so an encode keeps its slot until it finishes.
|
||||||
func (p *ImageProcessor) Process(
|
func (p *ImageProcessor) Process(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
input io.Reader,
|
input io.Reader,
|
||||||
@@ -290,7 +292,7 @@ func (p *ImageProcessor) Process(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Encode to target format
|
// Encode to target format
|
||||||
output, err := p.encode(img, outputFormat, req.Quality)
|
output, err := p.encode(ctx, img, outputFormat, req.Quality)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to encode: %w", err)
|
return nil, fmt.Errorf("failed to encode: %w", err)
|
||||||
}
|
}
|
||||||
@@ -417,25 +419,28 @@ func (p *ImageProcessor) WaitForProcessing(ctx context.Context) int {
|
|||||||
|
|
||||||
// acquireSlot takes a slot in processingSemaphore, waiting at most
|
// acquireSlot takes a slot in processingSemaphore, waiting at most
|
||||||
// processingWaitTimeout for one to free up, and returns the func that gives
|
// processingWaitTimeout for one to free up, and returns the func that gives
|
||||||
// it back. A free slot is taken even when ctx has ended; only the wait for
|
// it back. When ctx has ended by the time it has a slot, it gives the slot
|
||||||
// one stops when ctx ends, as the rest of Process does not check ctx.
|
// back and returns ctx's error.
|
||||||
func (p *ImageProcessor) acquireSlot(ctx context.Context) (func(), error) {
|
func (p *ImageProcessor) acquireSlot(ctx context.Context) (func(), error) {
|
||||||
release := func() { <-p.processingSemaphore }
|
|
||||||
|
|
||||||
select {
|
select {
|
||||||
case p.processingSemaphore <- struct{}{}:
|
case p.processingSemaphore <- struct{}{}:
|
||||||
return release, nil
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
select {
|
|
||||||
case p.processingSemaphore <- struct{}{}:
|
|
||||||
return release, nil
|
|
||||||
case <-time.After(p.processingWaitTimeout):
|
case <-time.After(p.processingWaitTimeout):
|
||||||
return nil, ErrTooManyImages
|
return nil, ErrTooManyImages
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return nil, ctx.Err()
|
return nil, ctx.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
release := func() { <-p.processingSemaphore }
|
||||||
|
|
||||||
|
// select takes the slot even when ctx has ended at the same moment
|
||||||
|
err := ctx.Err()
|
||||||
|
if err != nil {
|
||||||
|
release()
|
||||||
|
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return release, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// detectFormat returns the format string for a govips image type.
|
// detectFormat returns the format string for a govips image type.
|
||||||
@@ -527,10 +532,17 @@ func (p *ImageProcessor) resize(
|
|||||||
|
|
||||||
const defaultQuality = 85
|
const defaultQuality = 85
|
||||||
|
|
||||||
// encode encodes an image to the specified format.
|
// encode encodes an image to the specified format. libvips cannot stop an
|
||||||
|
// encode once it has begun, so when ctx has ended encode returns ctx's error
|
||||||
|
// without beginning one.
|
||||||
func (p *ImageProcessor) encode(
|
func (p *ImageProcessor) encode(
|
||||||
img *vips.ImageRef, format Format, quality int,
|
ctx context.Context, img *vips.ImageRef, format Format, quality int,
|
||||||
) ([]byte, error) {
|
) ([]byte, error) {
|
||||||
|
err := ctx.Err()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
if quality <= 0 {
|
if quality <= 0 {
|
||||||
quality = defaultQuality
|
quality = defaultQuality
|
||||||
}
|
}
|
||||||
@@ -540,7 +552,7 @@ func (p *ImageProcessor) encode(
|
|||||||
// libvips' built-in profile; govips' own sRGB path variable is set on
|
// libvips' built-in profile; govips' own sRGB path variable is set on
|
||||||
// first use but read without a lock, so concurrent requests race on it.
|
// first use but read without a lock, so concurrent requests race on it.
|
||||||
if img.HasICCProfile() {
|
if img.HasICCProfile() {
|
||||||
err := img.TransformICCProfileWithFallback("srgb", "srgb")
|
err = img.TransformICCProfileWithFallback("srgb", "srgb")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to convert to sRGB: %w", err)
|
return nil, fmt.Errorf("failed to convert to sRGB: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ import (
|
|||||||
|
|
||||||
"github.com/getsentry/sentry-go"
|
"github.com/getsentry/sentry-go"
|
||||||
"sneak.berlin/go/pixa/internal/httpfetcher"
|
"sneak.berlin/go/pixa/internal/httpfetcher"
|
||||||
|
"sneak.berlin/go/pixa/internal/imageprocessor"
|
||||||
"sneak.berlin/go/pixa/internal/magic"
|
"sneak.berlin/go/pixa/internal/magic"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -435,6 +436,47 @@ func TestService_Get_ReturnsByItsDeadline(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestService_Get_WaiterGetsProcessingTimedOut has a request with no deadline
|
||||||
|
// join the held fetch of a first request with a short one. With the only
|
||||||
|
// processing slot held, the first request's transcode waits for it until the
|
||||||
|
// first request's deadline ends, and stops then. The waiting request, whose
|
||||||
|
// own context has not ended, must get ErrProcessingTimedOut, which the image
|
||||||
|
// routes answer 503 as when pixa is busy.
|
||||||
|
func TestService_Get_WaiterGetsProcessingTimedOut(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
svc, fixtures, fetcher := setupHeldFetchService(t)
|
||||||
|
svc.processor = imageprocessor.New(
|
||||||
|
imageprocessor.Params{MaxConcurrentProcessing: 1},
|
||||||
|
)
|
||||||
|
|
||||||
|
holdProcessingSlot(t, svc.processor)
|
||||||
|
|
||||||
|
const firstTimeout = 500 * time.Millisecond
|
||||||
|
|
||||||
|
firstCtx, cancelFirst := context.WithTimeout(t.Context(), firstTimeout)
|
||||||
|
defer cancelFirst()
|
||||||
|
|
||||||
|
first := startGet(firstCtx, svc, photoVariant(fixtures, 85, FitCover))
|
||||||
|
|
||||||
|
<-fetcher.started
|
||||||
|
|
||||||
|
waiter := startGet(t.Context(), svc, photoVariant(fixtures, 85, FitCover))
|
||||||
|
|
||||||
|
time.Sleep(arrivalWait)
|
||||||
|
fetcher.releaseFetches()
|
||||||
|
|
||||||
|
t.Logf("first request: Get() error = %v", (<-first).err)
|
||||||
|
|
||||||
|
got := <-waiter
|
||||||
|
t.Logf("waiting request: Get() error = %v", got.err)
|
||||||
|
|
||||||
|
if !errors.Is(got.err, ErrProcessingTimedOut) {
|
||||||
|
t.Errorf("waiting request: Get() error = %v, want %v",
|
||||||
|
got.err, ErrProcessingTimedOut)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// panickingFetcher panics on every fetch.
|
// panickingFetcher panics on every fetch.
|
||||||
type panickingFetcher struct{}
|
type panickingFetcher struct{}
|
||||||
|
|
||||||
|
|||||||
@@ -94,6 +94,71 @@ func TestNegativeCache_Expired(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestService_Get_ServesCachedVariantOfNegativeCachedURL caches a variant of
|
||||||
|
// the test photo, then keeps a failed fetch of the photo's URL in the negative
|
||||||
|
// cache, as a failed fetch for another variant does once the source has been
|
||||||
|
// evicted. The cached variant is still served, and a variant not in the cache
|
||||||
|
// is made from the cached source. Once the source is evicted, a variant not in
|
||||||
|
// the cache needs a fetch and is refused.
|
||||||
|
func TestService_Get_ServesCachedVariantOfNegativeCachedURL(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
svc, fixtures := SetupTestService(t)
|
||||||
|
|
||||||
|
resp, err := svc.Get(t.Context(), photoVariant(fixtures, 85, FitCover))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Get() first request error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_ = resp.Content.Close()
|
||||||
|
|
||||||
|
err = svc.cache.StoreNegative(t.Context(), photoVariant(fixtures, 85, FitCover),
|
||||||
|
httpStatusBadGateway, "upstream answered 503")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("StoreNegative() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err = svc.Get(t.Context(), photoVariant(fixtures, 85, FitCover))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Get() of the cached variant error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_ = resp.Content.Close()
|
||||||
|
|
||||||
|
if resp.CacheStatus != CacheHit {
|
||||||
|
t.Errorf("cached variant CacheStatus = %q, want %q",
|
||||||
|
resp.CacheStatus, CacheHit)
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err = svc.Get(t.Context(), photoVariant(fixtures, 40, FitCover))
|
||||||
|
t.Logf("Get() of a variant made from the cached source: error = %v", err)
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Get() of a variant made from the cached source error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_ = resp.Content.Close()
|
||||||
|
|
||||||
|
contentHash, _, err := svc.cache.LookupSource(t.Context(),
|
||||||
|
photoVariant(fixtures, 85, FitCover))
|
||||||
|
if err != nil || contentHash == "" {
|
||||||
|
t.Fatalf("LookupSource() = %q, %v, want the cached source", contentHash, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = svc.cache.evictSourceBlob(t.Context(), contentHash)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("evictSourceBlob() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = svc.Get(t.Context(), photoVariant(fixtures, 60, FitCover))
|
||||||
|
t.Logf("Get() of a variant whose source was evicted: error = %v", err)
|
||||||
|
|
||||||
|
if !errors.Is(err, ErrNegativeCached) {
|
||||||
|
t.Errorf("Get() of a variant whose source was evicted error = %v, want %v",
|
||||||
|
err, ErrNegativeCached)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestService_Get_ReturnsErrorForNegativeCachedURL(t *testing.T) {
|
func TestService_Get_ReturnsErrorForNegativeCachedURL(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
"log/slog"
|
"log/slog"
|
||||||
"net/url"
|
"net/url"
|
||||||
"runtime/debug"
|
"runtime/debug"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/dustin/go-humanize"
|
"github.com/dustin/go-humanize"
|
||||||
@@ -128,26 +129,20 @@ func NewService(cfg *ServiceConfig) (*Service, error) {
|
|||||||
// ErrNegativeCached is returned when a URL is in the negative cache (recently failed).
|
// ErrNegativeCached is returned when a URL is in the negative cache (recently failed).
|
||||||
var ErrNegativeCached = errors.New("request is in negative cache (recently failed)")
|
var ErrNegativeCached = errors.New("request is in negative cache (recently failed)")
|
||||||
|
|
||||||
|
// ErrProcessingTimedOut is returned when processing an image stopped because
|
||||||
|
// the downstream_timeout of the request doing it ended, to that request and to
|
||||||
|
// every request that waited for its image.
|
||||||
|
var ErrProcessingTimedOut = errors.New(
|
||||||
|
"image processing stopped at the deadline of the request doing it")
|
||||||
|
|
||||||
// Get retrieves a processed image, fetching and processing if necessary.
|
// Get retrieves a processed image, fetching and processing if necessary.
|
||||||
|
// While its source URL is in the negative cache, a cached variant is still
|
||||||
|
// served and a variant not in the cache is still made from a cached source;
|
||||||
|
// only a request that needs the source fetched is refused.
|
||||||
func (s *Service) Get(ctx context.Context, req *ImageRequest) (*ImageResponse, error) {
|
func (s *Service) Get(ctx context.Context, req *ImageRequest) (*ImageResponse, error) {
|
||||||
// Propagate AllowHTTP setting to the request
|
// Propagate AllowHTTP setting to the request
|
||||||
req.AllowHTTP = s.allowHTTP
|
req.AllowHTTP = s.allowHTTP
|
||||||
|
|
||||||
// Check negative cache first - skip fetching for recently-failed URLs
|
|
||||||
negHit, err := s.cache.checkNegativeCache(ctx, req)
|
|
||||||
if err != nil {
|
|
||||||
s.log.Warn("negative cache check failed", "error", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if negHit {
|
|
||||||
s.log.Debug("negative cache hit",
|
|
||||||
"host", req.SourceHost,
|
|
||||||
"path", req.SourcePath,
|
|
||||||
)
|
|
||||||
|
|
||||||
return nil, fmt.Errorf("%w: %w", httpfetcher.ErrUpstreamError, ErrNegativeCached)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check variant cache first (disk only, no DB)
|
// Check variant cache first (disk only, no DB)
|
||||||
result, err := s.cache.Lookup(ctx, req)
|
result, err := s.cache.Lookup(ctx, req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -218,7 +213,16 @@ func (s *Service) ValidateRequest(req *ImageRequest) error {
|
|||||||
return fmt.Errorf("invalid source URL: %w", err)
|
return fmt.Errorf("invalid source URL: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if s.allowlist.IsAllowed(parsedURL) {
|
// An allowlist entry names no port, so a host followed by ":", with or
|
||||||
|
// without a port, needs a signature. The port is read from the parsed
|
||||||
|
// URL, as the fetcher reads it: in an IPv6 address written without
|
||||||
|
// brackets, such as 2001:db8::1:8443, it is what follows the last ":".
|
||||||
|
// This is checked here and not in allowlist.IsAllowed, which also matches
|
||||||
|
// referer_blocklist, where a Referer must be blocked whatever its port.
|
||||||
|
hostHasPort := parsedURL.Port() != "" ||
|
||||||
|
strings.HasSuffix(parsedURL.Host, ":")
|
||||||
|
|
||||||
|
if !hostHasPort && s.allowlist.IsAllowed(parsedURL) {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -423,12 +427,28 @@ func (s *Service) processFromSourceOrFetch(
|
|||||||
|
|
||||||
// fetchAndProcess fetches from upstream, processes, and caches the result.
|
// fetchAndProcess fetches from upstream, processes, and caches the result.
|
||||||
// It counts the fetch with the bytes read from upstream, including when
|
// It counts the fetch with the bytes read from upstream, including when
|
||||||
// reading the response or a later step fails.
|
// reading the response or a later step fails. It fetches nothing while the
|
||||||
|
// source URL is in the negative cache.
|
||||||
func (s *Service) fetchAndProcess(
|
func (s *Service) fetchAndProcess(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
req *ImageRequest,
|
req *ImageRequest,
|
||||||
cacheKey VariantKey,
|
cacheKey VariantKey,
|
||||||
) (*processedVariant, error) {
|
) (*processedVariant, error) {
|
||||||
|
// Check negative cache - skip fetching for recently-failed URLs
|
||||||
|
negHit, err := s.cache.checkNegativeCache(ctx, req)
|
||||||
|
if err != nil {
|
||||||
|
s.log.Warn("negative cache check failed", "error", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if negHit {
|
||||||
|
s.log.Debug("negative cache hit",
|
||||||
|
"host", req.SourceHost,
|
||||||
|
"path", req.SourcePath,
|
||||||
|
)
|
||||||
|
|
||||||
|
return nil, fmt.Errorf("%w: %w", httpfetcher.ErrUpstreamError, ErrNegativeCached)
|
||||||
|
}
|
||||||
|
|
||||||
// Fetch from upstream
|
// Fetch from upstream
|
||||||
sourceURL := req.SourceURL()
|
sourceURL := req.SourceURL()
|
||||||
|
|
||||||
|
|||||||
@@ -250,6 +250,97 @@ func TestService_ValidateRequest_SignatureExactHostMatch(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestService_ValidateRequest_HostWithPortNeedsSignature verifies that a host
|
||||||
|
// followed by a port, or by a colon with no port, matches no allowlist entry,
|
||||||
|
// so that a request for it without a signature is refused, while the same
|
||||||
|
// host with no port is allowlisted.
|
||||||
|
func TestService_ValidateRequest_HostWithPortNeedsSignature(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
svc, _ := SetupTestService(t,
|
||||||
|
WithSigningKey(testSigningKey),
|
||||||
|
WithAllowlist(testHostCDN, "203.0.113.5", "2001:db8::1"),
|
||||||
|
)
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
host string
|
||||||
|
allowlisted bool
|
||||||
|
}{
|
||||||
|
{testHostCDN, true},
|
||||||
|
{"cdn.example.com:8443", false},
|
||||||
|
{"cdn.example.com:443", false},
|
||||||
|
{"cdn.example.com:", false},
|
||||||
|
{"203.0.113.5", true},
|
||||||
|
{"203.0.113.5:8080", false},
|
||||||
|
{"[2001:db8::1]", true},
|
||||||
|
{"[2001:db8::1]:8443", false},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.host, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
req := &ImageRequest{
|
||||||
|
SourceHost: tt.host,
|
||||||
|
SourcePath: testPathCat,
|
||||||
|
Size: Size{Width: 50, Height: 50},
|
||||||
|
Format: FormatJPEG,
|
||||||
|
Quality: 85,
|
||||||
|
FitMode: FitCover,
|
||||||
|
}
|
||||||
|
|
||||||
|
err := svc.ValidateRequest(req)
|
||||||
|
t.Logf("ValidateRequest() for host %q with no signature: %v",
|
||||||
|
tt.host, err)
|
||||||
|
|
||||||
|
if allowlisted := err == nil; allowlisted != tt.allowlisted {
|
||||||
|
t.Errorf("host %q served without a signature = %v, want %v",
|
||||||
|
tt.host, allowlisted, tt.allowlisted)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestService_ValidateRequest_UnbracketedIPv6PortNeedsSignature verifies that
|
||||||
|
// an allowlisted IPv6 address written without brackets and followed by a
|
||||||
|
// port, or by a colon with no port, needs a signature. Go's URL parser, which
|
||||||
|
// the fetcher uses, reads 2001:db8::1:8443 as the host 2001:db8::1 with the
|
||||||
|
// port 8443.
|
||||||
|
func TestService_ValidateRequest_UnbracketedIPv6PortNeedsSignature(
|
||||||
|
t *testing.T,
|
||||||
|
) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
svc, _ := SetupTestService(t,
|
||||||
|
WithSigningKey(testSigningKey),
|
||||||
|
WithAllowlist("2001:db8::1"),
|
||||||
|
)
|
||||||
|
|
||||||
|
for _, host := range []string{"2001:db8::1:8443", "2001:db8::1:"} {
|
||||||
|
t.Run(host, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
req := &ImageRequest{
|
||||||
|
SourceHost: host,
|
||||||
|
SourcePath: testPathCat,
|
||||||
|
Size: Size{Width: 50, Height: 50},
|
||||||
|
Format: FormatJPEG,
|
||||||
|
Quality: 85,
|
||||||
|
FitMode: FitCover,
|
||||||
|
}
|
||||||
|
|
||||||
|
err := svc.ValidateRequest(req)
|
||||||
|
t.Logf("ValidateRequest() for host %q with no signature: %v",
|
||||||
|
host, err)
|
||||||
|
|
||||||
|
if err == nil {
|
||||||
|
t.Errorf("host %q served without a signature, want refused",
|
||||||
|
host)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestService_Get_InvalidFile(t *testing.T) {
|
func TestService_Get_InvalidFile(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import (
|
|||||||
"log/slog"
|
"log/slog"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -371,11 +372,13 @@ func HashPath(path string) PathHash {
|
|||||||
|
|
||||||
// CacheKey generates a unique key for a request variant.
|
// CacheKey generates a unique key for a request variant.
|
||||||
// Format: sha256(host:path:query:width:height:format:quality:fit_mode)
|
// Format: sha256(host:path:query:width:height:format:quality:fit_mode)
|
||||||
|
// The path and query go through escapePercentAndColon, so that a ":" in
|
||||||
|
// either cannot be taken for the one between them.
|
||||||
func CacheKey(req *ImageRequest) VariantKey {
|
func CacheKey(req *ImageRequest) VariantKey {
|
||||||
data := fmt.Sprintf("%s:%s:%s:%d:%d:%s:%d:%s",
|
data := fmt.Sprintf("%s:%s:%s:%d:%d:%s:%d:%s",
|
||||||
req.SourceHost,
|
req.SourceHost,
|
||||||
req.SourcePath,
|
escapePercentAndColon(req.SourcePath),
|
||||||
req.SourceQuery,
|
escapePercentAndColon(req.SourceQuery),
|
||||||
req.Size.Width,
|
req.Size.Width,
|
||||||
req.Size.Height,
|
req.Size.Height,
|
||||||
req.Format,
|
req.Format,
|
||||||
@@ -387,6 +390,14 @@ func CacheKey(req *ImageRequest) VariantKey {
|
|||||||
return VariantKey(hex.EncodeToString(h[:]))
|
return VariantKey(hex.EncodeToString(h[:]))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// escapePercentAndColon returns s with each "%" written as "%25" and each
|
||||||
|
// ":" as "%3A", so that the result holds no ":".
|
||||||
|
func escapePercentAndColon(s string) string {
|
||||||
|
s = strings.ReplaceAll(s, "%", "%25")
|
||||||
|
|
||||||
|
return strings.ReplaceAll(s, ":", "%3A")
|
||||||
|
}
|
||||||
|
|
||||||
// VariantStorage handles key-based file storage for processed image variants.
|
// VariantStorage handles key-based file storage for processed image variants.
|
||||||
// Files are stored at: <basedir>/<ab>/<cd>/<cache_key>
|
// Files are stored at: <basedir>/<ab>/<cd>/<cache_key>
|
||||||
// Metadata is stored at: <basedir>/<ab>/<cd>/<cache_key>.meta
|
// Metadata is stored at: <basedir>/<ab>/<cd>/<cache_key>.meta
|
||||||
|
|||||||
@@ -407,6 +407,62 @@ func TestCacheKey(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestCacheKey_ColonOrPercentInPathOrQuery checks that two different source
|
||||||
|
// URLs whose path or query holds ":" or "%" get different keys.
|
||||||
|
func TestCacheKey_ColonOrPercentInPathOrQuery(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
path1, query1 string
|
||||||
|
path2, query2 string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "colon moved from path to query",
|
||||||
|
path1: "/a:b", query1: "",
|
||||||
|
path2: "/a", query2: "b:",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "colon in path and its percent-encoding",
|
||||||
|
path1: "/a:b", query1: "",
|
||||||
|
path2: "/a%3Ab", query2: "",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range cases {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
key1 := CacheKey(&ImageRequest{
|
||||||
|
SourceHost: testHostCDN,
|
||||||
|
SourcePath: tt.path1,
|
||||||
|
SourceQuery: tt.query1,
|
||||||
|
Size: Size{Width: 800, Height: 600},
|
||||||
|
Format: FormatWebP,
|
||||||
|
Quality: 85,
|
||||||
|
FitMode: FitCover,
|
||||||
|
})
|
||||||
|
key2 := CacheKey(&ImageRequest{
|
||||||
|
SourceHost: testHostCDN,
|
||||||
|
SourcePath: tt.path2,
|
||||||
|
SourceQuery: tt.query2,
|
||||||
|
Size: Size{Width: 800, Height: 600},
|
||||||
|
Format: FormatWebP,
|
||||||
|
Quality: 85,
|
||||||
|
FitMode: FitCover,
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Logf("CacheKey() of path %q query %q = %s", tt.path1, tt.query1, key1)
|
||||||
|
t.Logf("CacheKey() of path %q query %q = %s", tt.path2, tt.query2, key2)
|
||||||
|
|
||||||
|
if key1 == key2 {
|
||||||
|
t.Errorf("CacheKey() gave path %q query %q and path %q query %q "+
|
||||||
|
"the same key", tt.path1, tt.query1, tt.path2, tt.query2)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestVariantStorage_StoreLogsFailedMetaWrite verifies that a .meta write
|
// TestVariantStorage_StoreLogsFailedMetaWrite verifies that a .meta write
|
||||||
// that fails is logged at warn and the store still succeeds.
|
// that fails is logged at warn and the store still succeeds.
|
||||||
func TestVariantStorage_StoreLogsFailedMetaWrite(t *testing.T) {
|
func TestVariantStorage_StoreLogsFailedMetaWrite(t *testing.T) {
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ import (
|
|||||||
var (
|
var (
|
||||||
ErrInvalidPath = errors.New("invalid image path")
|
ErrInvalidPath = errors.New("invalid image path")
|
||||||
ErrMissingHost = errors.New("missing source host")
|
ErrMissingHost = errors.New("missing source host")
|
||||||
|
ErrInvalidHost = errors.New("invalid source host")
|
||||||
ErrMissingSize = errors.New("missing size specification")
|
ErrMissingSize = errors.New("missing size specification")
|
||||||
ErrInvalidSize = errors.New("invalid size format")
|
ErrInvalidSize = errors.New("invalid size format")
|
||||||
ErrInvalidFormat = errors.New("invalid or unsupported format")
|
ErrInvalidFormat = errors.New("invalid or unsupported format")
|
||||||
@@ -143,6 +144,17 @@ func parseImageComponents(remainder string) (*ParsedURL, error) {
|
|||||||
return nil, ErrMissingHost
|
return nil, ErrMissingHost
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A URL reads what comes before "@" as a user name and password, and the
|
||||||
|
// rest as the host
|
||||||
|
if strings.Contains(host, "@") {
|
||||||
|
return nil, fmt.Errorf("%w: %q holds @", ErrInvalidHost, host)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Host names do not depend on letter case, so the allowlist, the upstream
|
||||||
|
// connection limits, the cache and the signature see one spelling of each
|
||||||
|
// host
|
||||||
|
host = strings.ToLower(host)
|
||||||
|
|
||||||
// Extract query string if present in path
|
// Extract query string if present in path
|
||||||
if qIndex := strings.Index(path, "?"); qIndex != -1 {
|
if qIndex := strings.Index(path, "?"); qIndex != -1 {
|
||||||
query = path[qIndex+1:]
|
query = path[qIndex+1:]
|
||||||
|
|||||||
@@ -234,6 +234,49 @@ func TestParseImagePath(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestParseImagePath_LowerCasesHost verifies that the host is lower-cased, so
|
||||||
|
// that every spelling of its letter case is the same host to the allowlist,
|
||||||
|
// the upstream connection limits and the cache, and that the path keeps its
|
||||||
|
// case.
|
||||||
|
func TestParseImagePath_LowerCasesHost(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
got, err := ParseImagePath("CDN.Example.COM/Photos/Cat.jpg/800x600.webp")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ParseImagePath() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("parsed host %q, path %q", got.Host, got.Path)
|
||||||
|
|
||||||
|
if got.Host != testHostCDN {
|
||||||
|
t.Errorf("Host = %q, want %q", got.Host, testHostCDN)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got.Path != "/Photos/Cat.jpg" {
|
||||||
|
t.Errorf("Path = %q, want %q", got.Path, "/Photos/Cat.jpg")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestParseImagePath_RefusesUserinfo verifies that a host holding "@" is
|
||||||
|
// refused: a URL reads what comes before the "@" as a user name and password
|
||||||
|
// and the rest as the host.
|
||||||
|
func TestParseImagePath_RefusesUserinfo(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
for _, input := range []string{
|
||||||
|
"user@cdn.example.com/photos/cat.jpg/800x600.webp",
|
||||||
|
"user:password@cdn.example.com/photos/cat.jpg/800x600.webp",
|
||||||
|
} {
|
||||||
|
got, err := ParseImagePath(input)
|
||||||
|
t.Logf("ParseImagePath(%q) = %+v, error = %v", input, got, err)
|
||||||
|
|
||||||
|
if !errors.Is(err, ErrInvalidHost) {
|
||||||
|
t.Errorf("ParseImagePath(%q) error = %v, want %v",
|
||||||
|
input, err, ErrInvalidHost)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestParsedURL_ToImageRequest(t *testing.T) {
|
func TestParsedURL_ToImageRequest(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,113 @@
|
|||||||
|
package signature_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"sneak.berlin/go/pixa/internal/signature"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestSigner_Verify_ColonCannotMovePathQueryBoundary checks that a URL
|
||||||
|
// signed for path "/a:b" with no query does not verify for path "/a" with
|
||||||
|
// query "b:", a different resource on the same host.
|
||||||
|
func TestSigner_Verify_ColonCannotMovePathQueryBoundary(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
signer := signature.New(goldenSigningKey)
|
||||||
|
|
||||||
|
signed := &signature.Request{
|
||||||
|
SourceHost: testHost,
|
||||||
|
SourcePath: "/a:b",
|
||||||
|
Width: 800,
|
||||||
|
Height: 600,
|
||||||
|
Format: testFormatWebP,
|
||||||
|
Quality: 85,
|
||||||
|
FitMode: testFitCover,
|
||||||
|
Expires: time.Now().Add(1 * time.Hour),
|
||||||
|
}
|
||||||
|
signed.Signature = signer.Sign(signed)
|
||||||
|
|
||||||
|
replayed := *signed
|
||||||
|
replayed.SourcePath = "/a"
|
||||||
|
replayed.SourceQuery = "b:"
|
||||||
|
|
||||||
|
t.Logf("signature for path %q query %q: %s",
|
||||||
|
signed.SourcePath, signed.SourceQuery, signed.Signature)
|
||||||
|
t.Logf("signature for path %q query %q: %s",
|
||||||
|
replayed.SourcePath, replayed.SourceQuery, signer.Sign(&replayed))
|
||||||
|
|
||||||
|
err := signer.Verify(&replayed)
|
||||||
|
if !errors.Is(err, signature.ErrInvalid) {
|
||||||
|
t.Errorf("Verify() of path %q query %q with the signature for "+
|
||||||
|
"path %q = %v, want %v",
|
||||||
|
replayed.SourcePath, replayed.SourceQuery, signed.SourcePath,
|
||||||
|
err, signature.ErrInvalid)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSigner_EscapedGoldenVectors pins the signature of requests whose path
|
||||||
|
// or query holds ":" or "%", which are signed with "%" written as "%25" and
|
||||||
|
// ":" as "%3A". The expected values were computed once, outside pixa, from
|
||||||
|
// the signed data in each comment.
|
||||||
|
func TestSigner_EscapedGoldenVectors(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
signer := signature.New(goldenSigningKey)
|
||||||
|
|
||||||
|
vectors := []struct {
|
||||||
|
name string
|
||||||
|
path string
|
||||||
|
query string
|
||||||
|
wantSignature string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "colon in path",
|
||||||
|
path: "/photos/a:b.jpg",
|
||||||
|
// Signed data:
|
||||||
|
// "cdn.example.com:/photos/a%3Ab.jpg::800:600:webp:1704067200:85:cover"
|
||||||
|
wantSignature: "WnEpZ7bQp1Zvw8WDatlj2vu-4wYKWE3BFBizV8m__6k=",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "colon in query",
|
||||||
|
path: testPath,
|
||||||
|
query: "t=1:2",
|
||||||
|
// Signed data:
|
||||||
|
// "cdn.example.com:/photos/cat.jpg:t=1%3A2:800:600:webp:1704067200:85:cover"
|
||||||
|
wantSignature: "85kRoN8hhr4cCUkEUjNvUsvVYKjYYfWdeffx5TjJjqA=",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "percent in path and query",
|
||||||
|
path: "/photos/a%20b.jpg",
|
||||||
|
query: "q=50%",
|
||||||
|
// Signed data:
|
||||||
|
// "cdn.example.com:/photos/a%2520b.jpg:q=50%25:800:600:webp:1704067200:85:cover"
|
||||||
|
wantSignature: "MjDiiRdgXMrsjqPMsENtD4aG1hp1e-a6zy67vKTX4CQ=",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range vectors {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
req := &signature.Request{
|
||||||
|
SourceHost: testHost,
|
||||||
|
SourcePath: tt.path,
|
||||||
|
SourceQuery: tt.query,
|
||||||
|
Width: 800,
|
||||||
|
Height: 600,
|
||||||
|
Format: testFormatWebP,
|
||||||
|
Quality: 85,
|
||||||
|
FitMode: testFitCover,
|
||||||
|
Expires: time.Unix(goldenExpiresUnix, 0),
|
||||||
|
}
|
||||||
|
|
||||||
|
gotSignature := signer.Sign(req)
|
||||||
|
t.Logf("Sign() of path %q query %q = %s", tt.path, tt.query, gotSignature)
|
||||||
|
|
||||||
|
if gotSignature != tt.wantSignature {
|
||||||
|
t.Errorf("Sign() = %q, want %q", gotSignature, tt.wantSignature)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -10,6 +10,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"net/url"
|
"net/url"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -154,15 +155,17 @@ func (s *Signer) GenerateSignedURL(
|
|||||||
|
|
||||||
// buildSignatureData creates the string to be signed.
|
// buildSignatureData creates the string to be signed.
|
||||||
// Format: "host:path:query:width:height:format:expiration:quality:fit"
|
// Format: "host:path:query:width:height:format:expiration:quality:fit"
|
||||||
// All components are used verbatim (exact match). No normalization,
|
// The path and query go through escapePercentAndColon, so that a ":" in
|
||||||
// suffix matching, or wildcard expansion is performed. Quality and fit
|
// either cannot be taken for the one between them; every other component
|
||||||
// are the effective transform values, so replaying a signed URL with a
|
// is used verbatim. No normalization, suffix matching, or wildcard
|
||||||
// different quality or fit mode fails verification.
|
// expansion is performed. Quality and fit are the effective transform
|
||||||
|
// values, so replaying a signed URL with a different quality or fit mode
|
||||||
|
// fails verification.
|
||||||
func (s *Signer) buildSignatureData(req *Request) string {
|
func (s *Signer) buildSignatureData(req *Request) string {
|
||||||
return fmt.Sprintf("%s:%s:%s:%d:%d:%s:%d:%d:%s",
|
return fmt.Sprintf("%s:%s:%s:%d:%d:%s:%d:%d:%s",
|
||||||
req.SourceHost,
|
req.SourceHost,
|
||||||
req.SourcePath,
|
escapePercentAndColon(req.SourcePath),
|
||||||
req.SourceQuery,
|
escapePercentAndColon(req.SourceQuery),
|
||||||
req.Width,
|
req.Width,
|
||||||
req.Height,
|
req.Height,
|
||||||
req.Format,
|
req.Format,
|
||||||
@@ -172,6 +175,14 @@ func (s *Signer) buildSignatureData(req *Request) string {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// escapePercentAndColon returns s with each "%" written as "%25" and each
|
||||||
|
// ":" as "%3A", so that the result holds no ":".
|
||||||
|
func escapePercentAndColon(s string) string {
|
||||||
|
s = strings.ReplaceAll(s, "%", "%25")
|
||||||
|
|
||||||
|
return strings.ReplaceAll(s, ":", "%3A")
|
||||||
|
}
|
||||||
|
|
||||||
// ParseParams extracts signature and expiration from query parameters.
|
// ParseParams extracts signature and expiration from query parameters.
|
||||||
func ParseParams(sig, expStr string) (string, time.Time, error) {
|
func ParseParams(sig, expStr string) (string, time.Time, error) {
|
||||||
if expStr == "" {
|
if expStr == "" {
|
||||||
|
|||||||
Reference in New Issue
Block a user