Compare commits
10
Commits
7014d88956
...
next
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c2e32d987b | ||
|
|
20874ba2af | ||
|
|
b32077c380 | ||
|
|
9bccaaf85a | ||
|
|
851a374e53 | ||
|
|
0de6972c2d | ||
|
|
092176d917 | ||
|
|
7f87aa17ca | ||
|
|
fa2ec3da6a | ||
|
|
a041e73550 |
No files matched your search
@@ -70,4 +70,3 @@ vendor/
|
|||||||
# Local dev configs
|
# Local dev configs
|
||||||
config.yml
|
config.yml
|
||||||
config.yaml
|
config.yaml
|
||||||
config.dev.yml
|
|
||||||
@@ -79,12 +79,15 @@ docker-versioned:
|
|||||||
docker-test:
|
docker-test:
|
||||||
@script/test
|
@script/test
|
||||||
|
|
||||||
# Run local dev server in Docker
|
# Run local dev server in Docker, with debug on and a new random signing
|
||||||
|
# key each run. Log in to the page with the key it prints.
|
||||||
devserver: docker-versioned devserver-stop
|
devserver: docker-versioned devserver-stop
|
||||||
|
@key="$$(head -c 32 /dev/urandom | base64)" && \
|
||||||
docker run -d --name pixad-dev -p 8080:8080 \
|
docker run -d --name pixad-dev -p 8080:8080 \
|
||||||
-v $(CURDIR)/config.dev.yml:/etc/pixa/config.yml:ro \
|
-e PIXA_SIGNING_KEY="$$key" -e PIXA_DEBUG=true \
|
||||||
pixa:latest
|
pixa:latest >/dev/null && \
|
||||||
@echo "pixad running at http://localhost:8080"
|
echo "pixad running at http://localhost:8080" && \
|
||||||
|
echo "signing key: $$key"
|
||||||
|
|
||||||
# Stop dev server
|
# Stop dev server
|
||||||
devserver-stop:
|
devserver-stop:
|
||||||
|
|||||||
@@ -15,9 +15,10 @@ In this document, the upstream host is the server that holds an original image.
|
|||||||
## Getting Started
|
## Getting Started
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# clone and build
|
# clone, install what compiling needs, and build
|
||||||
git clone https://git.eeqj.de/sneak/pixa.git
|
git clone https://git.eeqj.de/sneak/pixa.git
|
||||||
cd pixa
|
cd pixa
|
||||||
|
script/bootstrap --cgo
|
||||||
make build
|
make build
|
||||||
|
|
||||||
# make a config file from the example and put a real signing key in it
|
# make a config file from the example and put a real signing key in it
|
||||||
@@ -26,18 +27,30 @@ openssl rand -base64 32 # prints a new signing key
|
|||||||
$EDITOR config.yml # replace the signing_key placeholder with it
|
$EDITOR config.yml # replace the signing_key placeholder with it
|
||||||
./bin/pixad --config config.yml
|
./bin/pixad --config config.yml
|
||||||
|
|
||||||
# or build and run the Docker image
|
# or build and run the Docker image with that signing key
|
||||||
make docker
|
make docker
|
||||||
docker run -p 8080:8080 -e PIXA_SIGNING_KEY="$(openssl rand -base64 32)" pixa:latest
|
docker run -p 8080:8080 -e PIXA_SIGNING_KEY='the new signing key' pixa:latest
|
||||||
```
|
```
|
||||||
|
|
||||||
`make build` compiles pixa on your machine into `bin/pixad`. `pixad` is the
|
`script/bootstrap --cgo` installs Go, a C compiler and the image libraries pixa
|
||||||
server program. Compiling needs the libraries `script/bootstrap --cgo` installs
|
needs (see Deployment). `make build` then compiles pixa on your machine into
|
||||||
(see Deployment).
|
`bin/pixad`. `pixad` is the server program.
|
||||||
|
|
||||||
The signing key is pixa's one required secret. It must be 32 characters or
|
The signing key is pixa's one required secret. It must be 32 characters or
|
||||||
longer. The placeholder in the example file is refused at startup.
|
longer. The placeholder in the example file is refused at startup.
|
||||||
|
|
||||||
|
A container takes its settings from environment variables (see Configuration).
|
||||||
|
Only `PIXA_SIGNING_KEY` is required. Without it, the container exits at startup
|
||||||
|
and names the variable. Every other setting has a built-in default. A config
|
||||||
|
file mounted at `/etc/pixa/config.yml` is read when present. An environment
|
||||||
|
variable wins over the same setting in the file.
|
||||||
|
|
||||||
|
pixa's login page is at `http://localhost:8080/`. Logging in there works only
|
||||||
|
with `debug: true` in `config.yml`, or `-e PIXA_DEBUG=true` for the container.
|
||||||
|
Otherwise pixa refuses the login with 403, because the page was opened over
|
||||||
|
plain HTTP. Use `debug` only on your own machine. A pixa that others use serves
|
||||||
|
its pages over HTTPS through a reverse proxy (see Deployment).
|
||||||
|
|
||||||
Once pixa runs, it serves images through URLs of this form:
|
Once pixa runs, it serves images through URLs of this form:
|
||||||
|
|
||||||
```
|
```
|
||||||
@@ -60,11 +73,25 @@ The example config allowlists no host. Every image on an allowlisted host is
|
|||||||
served to anyone, without a signature, at any size and in any format. So
|
served to anyone, without a signature, at any size and in any format. So
|
||||||
allowlist only hosts you control, where nobody else can put images.
|
allowlist only hosts you control, where nobody else can put images.
|
||||||
|
|
||||||
A container takes its settings from environment variables (see Configuration).
|
To see pixa serve an image, sign a URL for an image on any public HTTPS server.
|
||||||
Only `PIXA_SIGNING_KEY` is required. Without it, the container exits at startup
|
Run this in a second terminal while pixa runs, with your own signing key and
|
||||||
and names the variable. Every other setting has a built-in default. A config
|
image:
|
||||||
file mounted at `/etc/pixa/config.yml` is read when present. An environment
|
|
||||||
variable wins over the same setting in the file.
|
```bash
|
||||||
|
key='the new signing key'
|
||||||
|
image_host=cdn.example.com # a public HTTPS server
|
||||||
|
image_path=/photos/cat.jpg # an image on it
|
||||||
|
exp=$(($(date +%s) + 3600)) # the URL expires in one hour
|
||||||
|
sig=$(printf '%s' "$image_host:$image_path::400:300:webp:$exp:85:cover" |
|
||||||
|
openssl dgst -sha256 -hmac "$key" -binary | base64 | tr '+/' '-_')
|
||||||
|
curl --fail-with-body -o cat.webp \
|
||||||
|
"http://localhost:8080/v1/image/$image_host$image_path/400x300.webp?sig=$sig&exp=$exp"
|
||||||
|
```
|
||||||
|
|
||||||
|
`cat.webp` is then the image at 400x300. If pixa refuses the request, curl
|
||||||
|
reports an error and `cat.webp` holds pixa's error message instead. The image
|
||||||
|
path must not hold a `%`, a `:` or a query. Signed URLs explains how to sign
|
||||||
|
those.
|
||||||
|
|
||||||
## Deployment
|
## Deployment
|
||||||
|
|
||||||
@@ -182,7 +209,8 @@ run under. Its app for pixa needs:
|
|||||||
is half the CPUs pixa sees times 4.2 GiB, plus 64 times 50 MiB. On a host
|
is half the CPUs pixa sees times 4.2 GiB, plus 64 times 50 MiB. On a host
|
||||||
where pixa sees 48 CPUs, that is 24 times 4.2 GiB plus 3.1 GiB, about 104 GiB.
|
where pixa sees 48 CPUs, that is 24 times 4.2 GiB plus 3.1 GiB, about 104 GiB.
|
||||||
Set `PIXA_MAX_CONCURRENT_PROCESSING` and `PIXA_UPSTREAM_CONNECTIONS` so that
|
Set `PIXA_MAX_CONCURRENT_PROCESSING` and `PIXA_UPSTREAM_CONNECTIONS` so that
|
||||||
this fits the memory the container may use.
|
this fits the memory the container may use. An SVG source is rendered in a
|
||||||
|
process of its own, which can use up to 1 GiB more.
|
||||||
- **Temporary files:** while pixa processes a large source, libvips writes the
|
- **Temporary files:** while pixa processes a large source, libvips writes the
|
||||||
decoded pixels to a file in `/tmp`. The file is removed from the directory
|
decoded pixels to a file in `/tmp`. The file is removed from the directory
|
||||||
once it is written, so a listing of `/tmp` does not show it. It keeps its
|
once it is written, so a listing of `/tmp` does not show it. It keeps its
|
||||||
@@ -285,6 +313,7 @@ to any method the list does not give for it.
|
|||||||
- 304 when `If-None-Match` matches the image's `ETag`.
|
- 304 when `If-None-Match` matches the image's `ETag`.
|
||||||
- 400 for a URL or parameter that is not valid.
|
- 400 for a URL or parameter that is not valid.
|
||||||
- 400 when the format is `auto` and the `Accept` header is not valid.
|
- 400 when the format is `auto` and the `Accept` header is not valid.
|
||||||
|
- 400 for `orig` of an SVG source at a size other than `orig`.
|
||||||
- 406 when the format is `auto` and `Accept` allows none of the formats it
|
- 406 when the format is `auto` and `Accept` allows none of the formats it
|
||||||
chooses from.
|
chooses from.
|
||||||
- 401 for a missing or wrong signature.
|
- 401 for a missing or wrong signature.
|
||||||
@@ -305,13 +334,28 @@ to any method the list does not give for it.
|
|||||||
format its `Content-Type` names.
|
format its `Content-Type` names.
|
||||||
- 415 when the upstream sends an image whose first bytes libvips reads as
|
- 415 when the upstream sends an image whose first bytes libvips reads as
|
||||||
another format.
|
another format.
|
||||||
|
- 415 for an SVG source with one side more than 8192 times the other, in any
|
||||||
|
format but `orig`.
|
||||||
|
- 415 for an SVG source that is not valid, or that takes more than 20
|
||||||
|
seconds or 1 GiB of memory to render.
|
||||||
- 502 when the upstream answers with an error status.
|
- 502 when the upstream answers with an error status.
|
||||||
- 502 for 5 minutes after that for the same source URL, when pixa has
|
- 502 for 5 minutes after that for the same source URL, when pixa has
|
||||||
neither the image nor its source in its cache.
|
neither the image nor its source in its cache.
|
||||||
- 502 likewise for 5 minutes after a body whose first bytes are not those of
|
- 502 likewise for 5 minutes after a body whose first bytes are not those of
|
||||||
its `Content-Type`.
|
its `Content-Type`.
|
||||||
|
- 502 when the upstream host, or a host it redirects to, has no address.
|
||||||
|
- 502 when the upstream redirects to a URL that is not `https`, or redirects
|
||||||
|
10 times.
|
||||||
|
- 502 when the fetch fails before the upstream answers, such as on a refused
|
||||||
|
connection or a failed TLS handshake.
|
||||||
- 503 when pixa is busy (see Limits).
|
- 503 when pixa is busy (see Limits).
|
||||||
- 503 in maintenance mode.
|
- 503 in maintenance mode.
|
||||||
|
- 504 when the address lookup of the upstream host, or of a host it
|
||||||
|
redirects to, times out.
|
||||||
|
- 504 when the connection to the upstream host, or to a host it redirects
|
||||||
|
to, times out.
|
||||||
|
- 504 when the upstream has not finished the TLS handshake within 10
|
||||||
|
seconds.
|
||||||
- 504 when the upstream has not sent its response headers within
|
- 504 when the upstream has not sent its response headers within
|
||||||
`upstream_fetch_timeout`.
|
`upstream_fetch_timeout`.
|
||||||
- 500 when that time runs out while the image itself is still arriving.
|
- 500 when that time runs out while the image itself is still arriving.
|
||||||
@@ -322,6 +366,8 @@ to any method the list does not give for it.
|
|||||||
- 304 when `If-None-Match` matches the image's `ETag`.
|
- 304 when `If-None-Match` matches the image's `ETag`.
|
||||||
- 400 for a token that does not decrypt.
|
- 400 for a token that does not decrypt.
|
||||||
- 400 for a token that asks for a size or fit that is not valid.
|
- 400 for a token that asks for a size or fit that is not valid.
|
||||||
|
- 400 for a token that asks for `orig` of an SVG source at a size other than
|
||||||
|
`orig`.
|
||||||
- 410 once the token has expired.
|
- 410 once the token has expired.
|
||||||
- 400 when the token's format is `auto` and the `Accept` header is not
|
- 400 when the token's format is `auto` and the `Accept` header is not
|
||||||
valid.
|
valid.
|
||||||
@@ -376,6 +422,10 @@ instead. Plain HTTP is for development on the browser's own machine. The login
|
|||||||
session cookie is always marked `Secure`. Over plain HTTP a browser keeps such a
|
session cookie is always marked `Secure`. Over plain HTTP a browser keeps such a
|
||||||
cookie only for its own machine (`localhost`), if at all.
|
cookie only for its own machine (`localhost`), if at all.
|
||||||
|
|
||||||
|
A form refused with 403 gets a page that names the cause and what to do about
|
||||||
|
it. pixa logs a `form refused` warning with the request ID and the form-token
|
||||||
|
library's reason, such as `origin invalid`.
|
||||||
|
|
||||||
The image routes answer the errors listed for them with a JSON body holding
|
The image routes answer the errors listed for them with a JSON body holding
|
||||||
`error`, `status` and `timestamp`.
|
`error`, `status` and `timestamp`.
|
||||||
|
|
||||||
@@ -460,10 +510,11 @@ outside ASCII. pixa signs and caches the query in that form. So
|
|||||||
`img%3Fname=my%20cat` and `img%3Fname=my%2520cat` in the image URL are the same
|
`img%3Fname=my%20cat` and `img%3Fname=my%2520cat` in the image URL are the same
|
||||||
source query, `name=my%20cat`.
|
source query, `name=my%20cat`.
|
||||||
|
|
||||||
**Sizes.** The largest width or height is 8192, for the source and for the
|
**Sizes.** The largest width or height is 8192, for the output and for a source
|
||||||
output. A size with one side `0` has that side worked out from the other side
|
in any format but SVG. A size with one side `0` has that side worked out from
|
||||||
and the source's proportions. The side is rounded down, but is at least 1. When
|
the other side and the source's proportions. The side is rounded down, but is at
|
||||||
it would come out above 8192, the request is refused with 400. The fit modes:
|
least 1. When it would come out above 8192, the request is refused with 400. The
|
||||||
|
fit modes:
|
||||||
|
|
||||||
- `cover`: scale the image to cover the size, keeping its proportions, and cut
|
- `cover`: scale the image to cover the size, keeping its proportions, and cut
|
||||||
out the centre. The output is exactly the size.
|
out the centre. The output is exactly the size.
|
||||||
@@ -483,9 +534,9 @@ applies to the format `orig` when the source is AVIF, and to `auto` when it
|
|||||||
chooses AVIF. So a client whose `Accept` names `image/avif` but not `image/jxl`
|
chooses AVIF. So a client whose `Accept` names `image/avif` but not `image/jxl`
|
||||||
is refused such a size.
|
is refused such a size.
|
||||||
|
|
||||||
**Source formats.** The source may be JPEG, PNG, GIF, WebP, AVIF or JPEG XL. The
|
**Source formats.** The source may be JPEG, PNG, GIF, WebP, AVIF, JPEG XL or
|
||||||
upstream's `Content-Type` must name its format, and the image's first bytes must
|
SVG. The upstream's `Content-Type` must name its format, and the image's first
|
||||||
match it. An upstream answering with any other `Content-Type`, SVG included, is
|
bytes must match it. An upstream answering with any other `Content-Type` is
|
||||||
refused with 415. So is a body whose first bytes are not those of the format its
|
refused with 415. So is a body whose first bytes are not those of the format its
|
||||||
`Content-Type` names: a JPEG sent as `image/png`, text sent as an image, or an
|
`Content-Type` names: a JPEG sent as `image/png`, text sent as an image, or an
|
||||||
empty body. pixa then does not fetch that source URL again for 5 minutes, as
|
empty body. pixa then does not fetch that source URL again for 5 minutes, as
|
||||||
@@ -494,6 +545,20 @@ need that fetch. An image whose first bytes libvips reads as another format is
|
|||||||
refused with 415 too. An SVG whose first bytes pass the check for AVIF is one
|
refused with 415 too. An SVG whose first bytes pass the check for AVIF is one
|
||||||
example.
|
example.
|
||||||
|
|
||||||
|
**SVG sources.** `orig` of an SVG source is the SVG itself, unchanged, and is
|
||||||
|
cached like any other image. It is given only at the `orig` size, since pixa
|
||||||
|
does not resize an SVG as SVG. `orig` of an SVG at any other size is refused
|
||||||
|
with 400. A browser that opens an SVG from pixa applies its styles and draws its
|
||||||
|
embedded `data:` images, but runs none of its scripts and loads nothing else.
|
||||||
|
|
||||||
|
In any format but `orig`, pixa renders an SVG source at the requested size, so
|
||||||
|
an enlarged SVG stays sharp. Rendering fetches no URL and reads no file that the
|
||||||
|
SVG refers to. In any format but `orig`, an SVG at the `orig` size is rendered
|
||||||
|
at its own size. In any format but `orig`, an SVG above 8192 on a side is
|
||||||
|
refused with 400 at the `orig` size. In any format but `orig`, an SVG with one
|
||||||
|
side more than 8192 times the other is refused with 415. Rendered with no side
|
||||||
|
above 8192, its shorter side would be under 1 pixel.
|
||||||
|
|
||||||
**The format `auto`.** With `auto`, pixa chooses the format for each request
|
**The format `auto`.** With `auto`, pixa chooses the format for each request
|
||||||
from its `Accept` header, in this order:
|
from its `Accept` header, in this order:
|
||||||
|
|
||||||
@@ -704,8 +769,9 @@ written percent-encoded.
|
|||||||
|
|
||||||
### Image Metadata
|
### Image Metadata
|
||||||
|
|
||||||
pixa decodes and re-encodes every image it serves, and removes all metadata from
|
pixa decodes and re-encodes every image it serves, except `orig` of an SVG
|
||||||
the output. This cannot be turned off. The metadata removed is:
|
source, which is the SVG unchanged. It removes all metadata from every image it
|
||||||
|
re-encodes. This cannot be turned off. The metadata removed is:
|
||||||
|
|
||||||
- EXIF, which can hold the GPS position, the camera make, model and serial
|
- EXIF, which can hold the GPS position, the camera make, model and serial
|
||||||
number, the capture time and an embedded thumbnail;
|
number, the capture time and an embedded thumbnail;
|
||||||
@@ -714,14 +780,16 @@ the output. This cannot be turned off. The metadata removed is:
|
|||||||
|
|
||||||
The rest of the rules:
|
The rest of the rules:
|
||||||
|
|
||||||
- The `orig` format means the source's own format, not the source's bytes. An
|
- For a source in any format but SVG, the `orig` format means the source's own
|
||||||
`orig` image is re-encoded and stripped like any other.
|
format, not the source's bytes. An `orig` image is re-encoded and stripped
|
||||||
|
like any other. For an SVG source, `orig` is the SVG's own bytes, its metadata
|
||||||
|
included.
|
||||||
- An image with an EXIF orientation is turned upright first, so it displays the
|
- An image with an EXIF orientation is turned upright first, so it displays the
|
||||||
same without the tag. A requested size applies to the upright image.
|
same without the tag. A requested size applies to the upright image.
|
||||||
- An image with an ICC profile is first converted to sRGB, the standard colour
|
- An image with an ICC profile is first converted to sRGB, the standard colour
|
||||||
space, since clients show an image with no profile as sRGB. Colours outside
|
space, since clients show an image with no profile as sRGB. Colours outside
|
||||||
sRGB are clipped: the most saturated ones in a Display P3 photo, for example.
|
sRGB are clipped: the most saturated ones in a Display P3 photo, for example.
|
||||||
- JPEG XL output is the one exception, with libvips 8.16 and later. libvips then
|
- JPEG XL output is an exception, with libvips 8.16 and later. libvips then
|
||||||
writes an EXIF block of its own, which pixa cannot stop. The block holds the
|
writes an EXIF block of its own, which pixa cannot stop. The block holds the
|
||||||
image's size and fixed values: an orientation of 1 and a resolution of 72 dpi.
|
image's size and fixed values: an orientation of 1 and a resolution of 72 dpi.
|
||||||
Nothing in it comes from the source. The Docker image has libvips 8.16, so its
|
Nothing in it comes from the source. The Docker image has libvips 8.16, so its
|
||||||
@@ -756,11 +824,11 @@ through, gives no way to stop an encode once it has begun. So an encode keeps
|
|||||||
its slot until it finishes, which can be after `downstream_timeout`.
|
its slot until it finishes, which can be after `downstream_timeout`.
|
||||||
|
|
||||||
**Encode times.** The limits on image size bound how long a conversion can take.
|
**Encode times.** The limits on image size bound how long a conversion can take.
|
||||||
They are 8192 on each side of the source and of the output, and 16,777,216
|
They are 8192 on each side of the output and of a source in any format but SVG,
|
||||||
pixels for an AVIF output. Decoding an 8192x8192 source takes up to about 11
|
and 16,777,216 pixels for an AVIF output. Decoding an 8192x8192 source takes up
|
||||||
seconds. The WebP and JPEG XL encoders run below libvips' default effort. AVIF
|
to about 11 seconds. The WebP and JPEG XL encoders run below libvips' default
|
||||||
runs at the lowest effort `govips` can set. With one libvips thread, as pixa
|
effort. AVIF runs at the lowest effort `govips` can set. With one libvips
|
||||||
runs it, encoding 8192x8192 random pixels takes up to about:
|
thread, as pixa runs it, encoding 8192x8192 random pixels takes up to about:
|
||||||
|
|
||||||
- 1 second as JPEG;
|
- 1 second as JPEG;
|
||||||
- 15 seconds as PNG;
|
- 15 seconds as PNG;
|
||||||
@@ -773,7 +841,8 @@ about 34 seconds with an alpha channel, and 22 without. At 8192x8192 and quality
|
|||||||
85 it would take about 96 seconds with an alpha channel, and 52 without. The 34
|
85 it would take about 96 seconds with an alpha channel, and 52 without. The 34
|
||||||
seconds hold at any quality and at 8 or 16 bits per sample. No encode within the
|
seconds hold at any quality and at 8 or 16 bits per sample. No encode within the
|
||||||
limits takes more than about 40 seconds. The default `downstream_timeout` is 60
|
limits takes more than about 40 seconds. The default `downstream_timeout` is 60
|
||||||
seconds.
|
seconds. An SVG source is rendered in a process of its own, which pixa stops
|
||||||
|
after 20 seconds and limits to 1 GiB of memory.
|
||||||
|
|
||||||
**Processing and connections.** `max_concurrent_processing` images are decoded
|
**Processing and connections.** `max_concurrent_processing` images are decoded
|
||||||
and encoded at once. A request that finds every slot in use waits up to 10
|
and encoded at once. A request that finds every slot in use waits up to 10
|
||||||
@@ -1167,10 +1236,33 @@ Each item below is one setting, named by its config key:
|
|||||||
- the Go runtime and process metrics
|
- the Go runtime and process metrics
|
||||||
- `pixa_cache_hits_total`: image requests served from the cache
|
- `pixa_cache_hits_total`: image requests served from the cache
|
||||||
- `pixa_cache_misses_total`: image requests not served from the cache
|
- `pixa_cache_misses_total`: image requests not served from the cache
|
||||||
|
- `pixa_source_cache_hits_total`: variants made or checked whose source was
|
||||||
|
in the cache
|
||||||
|
- `pixa_source_cache_misses_total`: variants made or checked whose source
|
||||||
|
was not in the cache
|
||||||
|
- `pixa_cache_usage_bytes`: the bytes of sources and variants in the cache,
|
||||||
|
as eviction last read them; eviction runs after each store and every five
|
||||||
|
minutes
|
||||||
|
- `pixa_cache_max_bytes`: the disk cache limit, `cache_max_bytes`
|
||||||
|
- `pixa_cache_evictions_total`: sources and variants removed by eviction
|
||||||
|
- `pixa_cache_evicted_bytes_total`: the bytes those evictions freed
|
||||||
- `pixa_upstream_fetches_total`: fetches of a source image that read any
|
- `pixa_upstream_fetches_total`: fetches of a source image that read any
|
||||||
bytes
|
bytes
|
||||||
- `pixa_upstream_fetch_bytes_total`: the bytes those fetches read
|
- `pixa_upstream_fetch_bytes_total`: the bytes those fetches read
|
||||||
|
- `pixa_upstream_fetch_duration_seconds`: a histogram of how long those
|
||||||
|
fetches took, up to the last byte of the response
|
||||||
|
- `pixa_upstream_fetch_errors_total`: fetches of a source image that failed,
|
||||||
|
before or while reading the response
|
||||||
- `pixa_transforms_total`: images converted
|
- `pixa_transforms_total`: images converted
|
||||||
|
- `pixa_transform_duration_seconds`: a histogram of how long those
|
||||||
|
conversions took, including the wait for a processing slot
|
||||||
|
- `pixa_transforms_in_progress`: images being converted or waiting for a
|
||||||
|
processing slot
|
||||||
|
|
||||||
|
pixa measures the `pixa_` metrics whether or not `/metrics` is served. They
|
||||||
|
have no labels, so their number never grows with the hosts and paths pixa
|
||||||
|
serves. Requests that share work (see Limits) count one source cache hit or
|
||||||
|
miss between them.
|
||||||
|
|
||||||
A request's `handler` label is the pattern of the route it matches, such as
|
A request's `handler` label is the pattern of the route it matches, such as
|
||||||
`/v1/image/*`, never its path. Requests that match no route share the label
|
`/v1/image/*`, never its path. Requests that match no route share the label
|
||||||
|
|||||||
@@ -27,12 +27,5 @@ P2: security: per-origin rate limiting
|
|||||||
- P2: HTTP response handling
|
- P2: HTTP response handling
|
||||||
- Last-Modified headers
|
- Last-Modified headers
|
||||||
- P2: operational
|
- P2: operational
|
||||||
- Prometheus metrics still missing:
|
|
||||||
- cache hits and misses, split into source images and variants
|
|
||||||
- cache size on disk and its limit
|
|
||||||
- evictions and the bytes they free
|
|
||||||
- upstream fetch duration and errors
|
|
||||||
- transcode duration
|
|
||||||
- images in processing
|
|
||||||
- measure the 1k to 5k req/s target with `script/loadtest` on a machine not
|
- measure the 1k to 5k req/s target with `script/loadtest` on a machine not
|
||||||
shared with other work
|
shared with other work
|
||||||
+13
-2
@@ -1,10 +1,21 @@
|
|||||||
// Package main is the entry point for the pixad image proxy server.
|
// Package main is the entry point for the pixad image proxy server. pixad
|
||||||
|
// also runs as a child process of itself, to read the size of an SVG source
|
||||||
|
// or to render it (see imageprocessor.RunAsSVGChildProcess).
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import "sneak.berlin/go/pixa/internal/app"
|
import (
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"sneak.berlin/go/pixa/internal/app"
|
||||||
|
"sneak.berlin/go/pixa/internal/imageprocessor"
|
||||||
|
)
|
||||||
|
|
||||||
var Version string //nolint:gochecknoglobals // set by ldflags
|
var Version string //nolint:gochecknoglobals // set by ldflags
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
|
if imageprocessor.IsSVGChildProcess(os.Args[1:]) {
|
||||||
|
os.Exit(imageprocessor.RunAsSVGChildProcess(os.Args[1:]))
|
||||||
|
}
|
||||||
|
|
||||||
app.Run(Version)
|
app.Run(Version)
|
||||||
}
|
}
|
||||||
@@ -158,8 +158,9 @@ access_control_allow_origin: "*"
|
|||||||
# Sentry DSN for error reporting (default: empty, which turns it off)
|
# Sentry DSN for error reporting (default: empty, which turns it off)
|
||||||
sentry_dsn: ""
|
sentry_dsn: ""
|
||||||
|
|
||||||
# Username and password for /metrics, set together (default: unset). Metrics
|
# Username and password for /metrics, set together (default: unset). pixa
|
||||||
# are measured and /metrics is served only when both are set.
|
# serves /metrics, and measures HTTP requests, only when both are set. It
|
||||||
|
# measures the pixa_ metrics either way.
|
||||||
# metrics:
|
# metrics:
|
||||||
# username: "admin"
|
# username: "admin"
|
||||||
# password: "secret"
|
# password: "secret"
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ require (
|
|||||||
github.com/prometheus/client_golang v1.23.2
|
github.com/prometheus/client_golang v1.23.2
|
||||||
github.com/slok/go-http-metrics v0.13.0
|
github.com/slok/go-http-metrics v0.13.0
|
||||||
github.com/spf13/cobra v1.10.2
|
github.com/spf13/cobra v1.10.2
|
||||||
|
go.uber.org/dig v1.19.0
|
||||||
go.uber.org/fx v1.24.0
|
go.uber.org/fx v1.24.0
|
||||||
golang.org/x/crypto v0.41.0
|
golang.org/x/crypto v0.41.0
|
||||||
golang.org/x/sync v0.19.0
|
golang.org/x/sync v0.19.0
|
||||||
@@ -126,7 +127,6 @@ require (
|
|||||||
go.opentelemetry.io/otel v1.36.0 // indirect
|
go.opentelemetry.io/otel v1.36.0 // indirect
|
||||||
go.opentelemetry.io/otel/metric v1.36.0 // indirect
|
go.opentelemetry.io/otel/metric v1.36.0 // indirect
|
||||||
go.opentelemetry.io/otel/trace v1.36.0 // indirect
|
go.opentelemetry.io/otel/trace v1.36.0 // indirect
|
||||||
go.uber.org/dig v1.19.0 // indirect
|
|
||||||
go.uber.org/multierr v1.11.0 // indirect
|
go.uber.org/multierr v1.11.0 // indirect
|
||||||
go.uber.org/zap v1.27.0 // indirect
|
go.uber.org/zap v1.27.0 // indirect
|
||||||
go.yaml.in/yaml/v2 v2.4.2 // indirect
|
go.yaml.in/yaml/v2 v2.4.2 // indirect
|
||||||
|
|||||||
+43
-2
@@ -2,13 +2,16 @@
|
|||||||
package app
|
package app
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
"syscall"
|
"syscall"
|
||||||
|
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
|
"go.uber.org/dig"
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"sneak.berlin/go/pixa/internal/config"
|
"sneak.berlin/go/pixa/internal/config"
|
||||||
"sneak.berlin/go/pixa/internal/database"
|
"sneak.berlin/go/pixa/internal/database"
|
||||||
@@ -53,7 +56,10 @@ func run(_ *cobra.Command, _ []string) {
|
|||||||
// A write to a closed stdout or stderr must not end the process.
|
// A write to a closed stdout or stderr must not end the process.
|
||||||
signal.Ignore(syscall.SIGPIPE)
|
signal.Ignore(syscall.SIGPIPE)
|
||||||
|
|
||||||
fx.New(
|
app := fx.New(
|
||||||
|
// pixad logs with slog only; runApp prints a startup or shutdown
|
||||||
|
// error itself.
|
||||||
|
fx.NopLogger,
|
||||||
fx.Provide(
|
fx.Provide(
|
||||||
config.New,
|
config.New,
|
||||||
database.New,
|
database.New,
|
||||||
@@ -71,5 +77,40 @@ func run(_ *cobra.Command, _ []string) {
|
|||||||
func(log *logger.Logger) { log.Identify() },
|
func(log *logger.Logger) { log.Identify() },
|
||||||
func(*server.Server) {},
|
func(*server.Server) {},
|
||||||
),
|
),
|
||||||
).Run()
|
)
|
||||||
|
|
||||||
|
exitCode := runApp(app, os.Stderr)
|
||||||
|
if exitCode != 0 {
|
||||||
|
os.Exit(exitCode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// runApp starts app, waits for a signal or a shutdown request, stops app,
|
||||||
|
// and returns the status pixad exits with, as fx.App.Run does. It prints
|
||||||
|
// a start or stop error to stderr, once, as the error that caused it:
|
||||||
|
// dig.RootCause drops the frames fx and dig wrap around it.
|
||||||
|
func runApp(app *fx.App, stderr io.Writer) int {
|
||||||
|
startCtx, cancelStart := context.WithTimeout(context.Background(), app.StartTimeout())
|
||||||
|
defer cancelStart()
|
||||||
|
|
||||||
|
err := app.Start(startCtx)
|
||||||
|
if err != nil {
|
||||||
|
_, _ = fmt.Fprintln(stderr, dig.RootCause(err))
|
||||||
|
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
shutdown := <-app.Wait()
|
||||||
|
|
||||||
|
stopCtx, cancelStop := context.WithTimeout(context.Background(), app.StopTimeout())
|
||||||
|
defer cancelStop()
|
||||||
|
|
||||||
|
err = app.Stop(stopCtx)
|
||||||
|
if err != nil {
|
||||||
|
_, _ = fmt.Fprintln(stderr, dig.RootCause(err))
|
||||||
|
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
return shutdown.ExitCode
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,65 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.uber.org/fx"
|
||||||
|
)
|
||||||
|
|
||||||
|
var errTestStopFailed = errors.New("test stop hook failed")
|
||||||
|
|
||||||
|
// TestRunAppPrintsStopErrorOnce checks that when a stop hook fails, runApp
|
||||||
|
// prints the hook's error once and returns exit status 1.
|
||||||
|
func TestRunAppPrintsStopErrorOnce(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
app := fx.New(
|
||||||
|
fx.NopLogger,
|
||||||
|
fx.Invoke(func(lifecycle fx.Lifecycle, shutdowner fx.Shutdowner) {
|
||||||
|
lifecycle.Append(fx.Hook{
|
||||||
|
// Ask to shut down as soon as the app has started, so
|
||||||
|
// that runApp goes on to stop it.
|
||||||
|
OnStart: func(context.Context) error { return shutdowner.Shutdown() },
|
||||||
|
OnStop: func(context.Context) error { return errTestStopFailed },
|
||||||
|
})
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
|
exitCode := runApp(app, &stderr)
|
||||||
|
if exitCode != 1 {
|
||||||
|
t.Errorf("runApp() = %d, want 1", exitCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
want := errTestStopFailed.Error() + "\n"
|
||||||
|
if stderr.String() != want {
|
||||||
|
t.Errorf("stderr = %q, want %q", stderr.String(), want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRunAppReturnsShutdownExitCode checks that runApp returns the exit
|
||||||
|
// code a shutdown request names.
|
||||||
|
func TestRunAppReturnsShutdownExitCode(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
app := fx.New(
|
||||||
|
fx.NopLogger,
|
||||||
|
fx.Invoke(func(lifecycle fx.Lifecycle, shutdowner fx.Shutdowner) {
|
||||||
|
lifecycle.Append(fx.Hook{
|
||||||
|
OnStart: func(context.Context) error {
|
||||||
|
return shutdowner.Shutdown(fx.ExitCode(3))
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
|
||||||
|
exitCode := runApp(app, io.Discard)
|
||||||
|
if exitCode != 3 {
|
||||||
|
t.Errorf("runApp() = %d, want 3", exitCode)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,177 @@
|
|||||||
|
package app_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"strconv"
|
||||||
|
"syscall"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"sneak.berlin/go/pixa/internal/app"
|
||||||
|
)
|
||||||
|
|
||||||
|
// These tests run pixad in a child process: the test binary itself,
|
||||||
|
// started again with runPixadInChildEnvVar set, which makes TestMain run
|
||||||
|
// pixad instead of the tests.
|
||||||
|
const runPixadInChildEnvVar = "TEST_RUN_PIXAD_IN_CHILD"
|
||||||
|
|
||||||
|
const validTestSigningKey = "test-signing-key-for-app-tests-0123456789"
|
||||||
|
|
||||||
|
func TestMain(m *testing.M) {
|
||||||
|
if os.Getenv(runPixadInChildEnvVar) == "1" {
|
||||||
|
os.Args = []string{"pixad"}
|
||||||
|
|
||||||
|
app.Run("test")
|
||||||
|
os.Exit(0)
|
||||||
|
}
|
||||||
|
|
||||||
|
os.Exit(m.Run())
|
||||||
|
}
|
||||||
|
|
||||||
|
// pixadCommand returns a command that runs pixad with only the given
|
||||||
|
// environment, in an empty directory with an empty HOME, so that no
|
||||||
|
// config file is found.
|
||||||
|
func pixadCommand(t *testing.T, env ...string) (*exec.Cmd, *bytes.Buffer) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
testBinary := os.Args[0]
|
||||||
|
cmd := exec.CommandContext(t.Context(), testBinary) //nolint:gosec // not tainted
|
||||||
|
cmd.Dir = t.TempDir()
|
||||||
|
// A test binary built with -cover warns on stderr at exit when
|
||||||
|
// GOCOVERDIR is not set.
|
||||||
|
cmd.Env = append([]string{
|
||||||
|
runPixadInChildEnvVar + "=1",
|
||||||
|
"HOME=" + t.TempDir(),
|
||||||
|
"GOCOVERDIR=" + t.TempDir(),
|
||||||
|
}, env...)
|
||||||
|
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
|
cmd.Stderr = &stderr
|
||||||
|
|
||||||
|
return cmd, &stderr
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRunPrintsStartupErrorOnce checks that an invalid setting is
|
||||||
|
// printed once, as the line that names it, and nothing else goes to
|
||||||
|
// stderr.
|
||||||
|
func TestRunPrintsStartupErrorOnce(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
cmd, stderr := pixadCommand(t, "PIXA_SIGNING_KEY=short")
|
||||||
|
|
||||||
|
err := cmd.Run()
|
||||||
|
|
||||||
|
var exitErr *exec.ExitError
|
||||||
|
if !errors.As(err, &exitErr) || exitErr.ExitCode() != 1 {
|
||||||
|
t.Fatalf("pixad: got %v, want exit status 1; stderr:\n%s", err, stderr)
|
||||||
|
}
|
||||||
|
|
||||||
|
want := `config key "signing_key" (environment variable PIXA_SIGNING_KEY): ` +
|
||||||
|
"value too short: must be at least 32 characters, got 5\n"
|
||||||
|
if stderr.String() != want {
|
||||||
|
t.Errorf("stderr:\n%s\nwant:\n%s", stderr, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRunWritesNothingToStderrOnStartAndStop starts pixad, waits until
|
||||||
|
// it answers its health check, stops it with SIGTERM, and checks that it
|
||||||
|
// wrote nothing to stderr.
|
||||||
|
func TestRunWritesNothingToStderrOnStartAndStop(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
port := freePort(t)
|
||||||
|
cmd, stderr := pixadCommand(t,
|
||||||
|
"PORT="+strconv.Itoa(port),
|
||||||
|
"PIXA_STATE_DIR="+t.TempDir(),
|
||||||
|
"PIXA_SIGNING_KEY="+validTestSigningKey,
|
||||||
|
)
|
||||||
|
|
||||||
|
err := cmd.Start()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("starting pixad: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
waitForHealthCheck(t, cmd, port, stderr)
|
||||||
|
|
||||||
|
err = cmd.Process.Signal(syscall.SIGTERM)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("signalling pixad: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = cmd.Wait()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("pixad: got %v, want exit status 0; stderr:\n%s", err, stderr)
|
||||||
|
}
|
||||||
|
|
||||||
|
if stderr.Len() != 0 {
|
||||||
|
t.Errorf("stderr is not empty:\n%s", stderr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// freePort returns a TCP port on the loopback address that nothing
|
||||||
|
// listens on.
|
||||||
|
func freePort(t *testing.T) int {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
var listenConfig net.ListenConfig
|
||||||
|
|
||||||
|
listener, err := listenConfig.Listen(t.Context(), "tcp", "127.0.0.1:0")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("finding a free port: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
addr, ok := listener.Addr().(*net.TCPAddr)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("listener address %v is not a TCP address", listener.Addr())
|
||||||
|
}
|
||||||
|
|
||||||
|
err = listener.Close()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("closing listener: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return addr.Port
|
||||||
|
}
|
||||||
|
|
||||||
|
// waitForHealthCheck waits up to 30 seconds for pixad, started as cmd, to
|
||||||
|
// answer its health check on port. If it does not, it stops pixad and
|
||||||
|
// fails the test with what pixad wrote to stderr.
|
||||||
|
func waitForHealthCheck(t *testing.T, cmd *exec.Cmd, port int, stderr *bytes.Buffer) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
url := fmt.Sprintf("http://127.0.0.1:%d/.well-known/healthcheck.json", port)
|
||||||
|
deadline := time.Now().Add(30 * time.Second)
|
||||||
|
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, url, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("health check request: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := http.DefaultClient.Do(req)
|
||||||
|
if err == nil {
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode == http.StatusOK {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
time.Sleep(100 * time.Millisecond)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop pixad first: stderr is complete, and safe to read, only once
|
||||||
|
// cmd.Wait has returned.
|
||||||
|
_ = cmd.Process.Kill()
|
||||||
|
_ = cmd.Wait()
|
||||||
|
|
||||||
|
t.Fatalf("pixad did not answer its health check on port %d; stderr:\n%s",
|
||||||
|
port, stderr)
|
||||||
|
}
|
||||||
@@ -55,7 +55,7 @@ func newCSRFTestRouter(t *testing.T) (*Handlers, http.Handler) {
|
|||||||
config: cfg,
|
config: cfg,
|
||||||
sessMgr: session.NewManager(keys.SessionHashKey, keys.SessionBlockKey),
|
sessMgr: session.NewManager(keys.SessionHashKey, keys.SessionBlockKey),
|
||||||
encGen: encurl.NewGenerator(keys.EncryptedURLKey),
|
encGen: encurl.NewGenerator(keys.EncryptedURLKey),
|
||||||
csrfProtect: newCSRFProtect(keys.CSRFKey, cfg.Debug),
|
csrfProtect: newCSRFProtect(keys.CSRFKey, cfg.Debug, slog.New(slog.DiscardHandler)),
|
||||||
}
|
}
|
||||||
|
|
||||||
r := chi.NewRouter()
|
r := chi.NewRouter()
|
||||||
|
|||||||
@@ -43,7 +43,7 @@ func newBodyLimitTestRouter(
|
|||||||
config: cfg,
|
config: cfg,
|
||||||
sessMgr: session.NewManager(keys.SessionHashKey, keys.SessionBlockKey),
|
sessMgr: session.NewManager(keys.SessionHashKey, keys.SessionBlockKey),
|
||||||
encGen: encurl.NewGenerator(keys.EncryptedURLKey),
|
encGen: encurl.NewGenerator(keys.EncryptedURLKey),
|
||||||
csrfProtect: newCSRFProtect(keys.CSRFKey, cfg.Debug),
|
csrfProtect: newCSRFProtect(keys.CSRFKey, cfg.Debug, slog.New(slog.DiscardHandler)),
|
||||||
}
|
}
|
||||||
|
|
||||||
r := chi.NewRouter()
|
r := chi.NewRouter()
|
||||||
|
|||||||
+106
-2
@@ -1,18 +1,43 @@
|
|||||||
package handlers
|
package handlers
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
|
"errors"
|
||||||
"html/template"
|
"html/template"
|
||||||
|
"log/slog"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
|
||||||
|
"github.com/go-chi/chi/v5/middleware"
|
||||||
"github.com/gorilla/csrf"
|
"github.com/gorilla/csrf"
|
||||||
|
|
||||||
"sneak.berlin/go/pixa/internal/seal"
|
"sneak.berlin/go/pixa/internal/seal"
|
||||||
|
"sneak.berlin/go/pixa/internal/templates"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The causes the form-refused page names, each with what to do about it.
|
||||||
|
const (
|
||||||
|
causePlainHTTP = "This page was opened over plain HTTP while debug is " +
|
||||||
|
"off. Open it over HTTPS, or turn debug on to use plain HTTP on " +
|
||||||
|
"your own machine."
|
||||||
|
causeHTTPSWithDebug = "This page was opened over HTTPS while debug is " +
|
||||||
|
"on. Turn debug off, or open the page over plain HTTP."
|
||||||
|
causeOtherHost = "The form was sent from another host than the one in " +
|
||||||
|
"the request's Host header. A reverse proxy in front of pixa must " +
|
||||||
|
"pass the Host header on unchanged."
|
||||||
|
causeNoReferer = "The browser sent neither an Origin nor a Referer " +
|
||||||
|
"header, and over HTTPS pixa needs one to check where the form " +
|
||||||
|
"came from. Let the browser send the Referer header to this site."
|
||||||
|
causeStaleToken = "The form token is missing or out of date. Reload " +
|
||||||
|
"the page and submit the form again."
|
||||||
)
|
)
|
||||||
|
|
||||||
// newCSRFProtect builds the CSRF-protection middleware for the
|
// newCSRFProtect builds the CSRF-protection middleware for the
|
||||||
// state-mutating HTML form routes. The token lives in its own cookie,
|
// state-mutating HTML form routes. The token lives in its own cookie,
|
||||||
// independent of the session cookie, so it also protects the login POST
|
// independent of the session cookie, so it also protects the login POST
|
||||||
// where no session exists yet (login CSRF). key signs that cookie.
|
// where no session exists yet (login CSRF). key signs that cookie. A
|
||||||
|
// refused form is answered by the handler that handleFormRefused returns.
|
||||||
|
// log gets a warning with the library's reason, such as "origin invalid".
|
||||||
//
|
//
|
||||||
// When plaintext is true (local HTTP development), requests are marked
|
// When plaintext is true (local HTTP development), requests are marked
|
||||||
// plaintext so the library neither demands an https Referer nor sets a
|
// plaintext so the library neither demands an https Referer nor sets a
|
||||||
@@ -20,13 +45,14 @@ import (
|
|||||||
// service runs behind a TLS-terminating proxy, so plaintext is false and
|
// service runs behind a TLS-terminating proxy, so plaintext is false and
|
||||||
// the library enforces its https Referer origin check.
|
// the library enforces its https Referer origin check.
|
||||||
func newCSRFProtect(
|
func newCSRFProtect(
|
||||||
key [seal.KeySize]byte, plaintext bool,
|
key [seal.KeySize]byte, plaintext bool, log *slog.Logger,
|
||||||
) func(http.Handler) http.Handler {
|
) func(http.Handler) http.Handler {
|
||||||
protect := csrf.Protect(
|
protect := csrf.Protect(
|
||||||
key[:],
|
key[:],
|
||||||
csrf.Path("/"),
|
csrf.Path("/"),
|
||||||
csrf.Secure(!plaintext),
|
csrf.Secure(!plaintext),
|
||||||
csrf.SameSite(csrf.SameSiteStrictMode),
|
csrf.SameSite(csrf.SameSiteStrictMode),
|
||||||
|
csrf.ErrorHandler(handleFormRefused(plaintext, log)),
|
||||||
)
|
)
|
||||||
|
|
||||||
if !plaintext {
|
if !plaintext {
|
||||||
@@ -42,6 +68,84 @@ func newCSRFProtect(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// handleFormRefused returns the handler the CSRF middleware calls when it
|
||||||
|
// refuses a request. It logs the library's reason with the request ID and
|
||||||
|
// answers 403 with a page naming the cause and what to do. A reason that is
|
||||||
|
// not a refusal, such as a failure to make a token, answers 500.
|
||||||
|
func handleFormRefused(plaintext bool, log *slog.Logger) http.Handler {
|
||||||
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
reason := csrf.FailureReason(r)
|
||||||
|
requestID := middleware.GetReqID(r.Context())
|
||||||
|
|
||||||
|
cause := formRefusedCause(r, reason, plaintext)
|
||||||
|
if cause == "" {
|
||||||
|
log.Error("CSRF middleware failed",
|
||||||
|
"request_id", requestID, "error", reason)
|
||||||
|
http.Error(w, "Internal server error",
|
||||||
|
http.StatusInternalServerError)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Warn("form refused", "request_id", requestID, "reason", reason)
|
||||||
|
|
||||||
|
var page bytes.Buffer
|
||||||
|
|
||||||
|
err := templates.Render(&page, "form_refused.html",
|
||||||
|
struct{ Cause string }{Cause: cause})
|
||||||
|
if err != nil {
|
||||||
|
log.Error("failed to render form refused template", "error", err)
|
||||||
|
http.Error(w, "Internal server error",
|
||||||
|
http.StatusInternalServerError)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||||
|
w.WriteHeader(http.StatusForbidden)
|
||||||
|
|
||||||
|
_, err = page.WriteTo(w)
|
||||||
|
if err != nil {
|
||||||
|
log.Error("failed to write form refused page", "error", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// formRefusedCause returns the cause the form-refused page names for the
|
||||||
|
// CSRF middleware's reason, or "" when the reason is not a refusal.
|
||||||
|
func formRefusedCause(r *http.Request, reason error, plaintext bool) string {
|
||||||
|
switch {
|
||||||
|
case errors.Is(reason, csrf.ErrNoToken), errors.Is(reason, csrf.ErrBadToken):
|
||||||
|
return causeStaleToken
|
||||||
|
case errors.Is(reason, csrf.ErrNoReferer):
|
||||||
|
return causeNoReferer
|
||||||
|
case errors.Is(reason, csrf.ErrBadOrigin):
|
||||||
|
return pageAddressCause(r.Header.Get("Origin"), r.Host, plaintext)
|
||||||
|
case errors.Is(reason, csrf.ErrBadReferer):
|
||||||
|
return pageAddressCause(r.Referer(), r.Host, plaintext)
|
||||||
|
default:
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// pageAddressCause returns the cause for a form whose page address, the
|
||||||
|
// Origin or Referer header, the CSRF middleware refused. The middleware
|
||||||
|
// expects the request's own host, with https when plaintext is false and
|
||||||
|
// http when it is true. So on the request's own host, only the scheme was
|
||||||
|
// wrong.
|
||||||
|
func pageAddressCause(pageAddress, host string, plaintext bool) string {
|
||||||
|
page, err := url.Parse(pageAddress)
|
||||||
|
if err != nil || page.Host != host {
|
||||||
|
return causeOtherHost
|
||||||
|
}
|
||||||
|
|
||||||
|
if plaintext {
|
||||||
|
return causeHTTPSWithDebug
|
||||||
|
}
|
||||||
|
|
||||||
|
return causePlainHTTP
|
||||||
|
}
|
||||||
|
|
||||||
// CSRF returns the CSRF-protection middleware for the login and generator
|
// CSRF returns the CSRF-protection middleware for the login and generator
|
||||||
// form routes.
|
// form routes.
|
||||||
func (s *Handlers) CSRF() func(http.Handler) http.Handler {
|
func (s *Handlers) CSRF() func(http.Handler) http.Handler {
|
||||||
|
|||||||
@@ -0,0 +1,205 @@
|
|||||||
|
package handlers
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"html/template"
|
||||||
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/go-chi/chi/v5/middleware"
|
||||||
|
"github.com/gorilla/csrf"
|
||||||
|
|
||||||
|
"sneak.berlin/go/pixa/internal/seal"
|
||||||
|
)
|
||||||
|
|
||||||
|
// refusedFormRequestID is the request ID of the requests postRefusedForm
|
||||||
|
// sends, which the warning for a refused form must carry.
|
||||||
|
const refusedFormRequestID = "test-request-id"
|
||||||
|
|
||||||
|
// TestFormRefusedNamesCause verifies that a form the CSRF middleware refuses
|
||||||
|
// answers 403 with a page naming the cause, and logs a warning with the
|
||||||
|
// request ID and the middleware's reason.
|
||||||
|
func TestFormRefusedNamesCause(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
debug bool
|
||||||
|
origin string
|
||||||
|
referer string
|
||||||
|
wantCause string
|
||||||
|
wantReason error
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "plain HTTP origin with debug off",
|
||||||
|
origin: "http://example.com",
|
||||||
|
wantCause: causePlainHTTP,
|
||||||
|
wantReason: csrf.ErrBadOrigin,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "plain HTTP referer with debug off",
|
||||||
|
referer: "http://example.com/",
|
||||||
|
wantCause: causePlainHTTP,
|
||||||
|
wantReason: csrf.ErrBadReferer,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "origin on another host",
|
||||||
|
origin: "https://proxy.example.net",
|
||||||
|
wantCause: causeOtherHost,
|
||||||
|
wantReason: csrf.ErrBadOrigin,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "referer on another host",
|
||||||
|
referer: "https://proxy.example.net/",
|
||||||
|
wantCause: causeOtherHost,
|
||||||
|
wantReason: csrf.ErrBadReferer,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "HTTPS origin with debug on",
|
||||||
|
debug: true,
|
||||||
|
origin: "https://example.com",
|
||||||
|
wantCause: causeHTTPSWithDebug,
|
||||||
|
wantReason: csrf.ErrBadOrigin,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "no origin or referer with debug off",
|
||||||
|
wantCause: causeNoReferer,
|
||||||
|
wantReason: csrf.ErrNoReferer,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "missing token",
|
||||||
|
debug: true,
|
||||||
|
origin: "http://example.com",
|
||||||
|
wantCause: causeStaleToken,
|
||||||
|
wantReason: csrf.ErrNoToken,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
rec, logLine := postRefusedForm(t, tt.debug, tt.origin, tt.referer,
|
||||||
|
nil, "")
|
||||||
|
checkFormRefused(t, rec, logLine, tt.wantCause, tt.wantReason)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestFormRefusedNamesCauseForForeignToken verifies that a form whose token
|
||||||
|
// does not match its CSRF cookie answers 403 with a page naming
|
||||||
|
// causeStaleToken, and logs a warning with the request ID and the
|
||||||
|
// middleware's reason.
|
||||||
|
func TestFormRefusedNamesCauseForForeignToken(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
// The router and postRefusedForm derive their CSRF key from
|
||||||
|
// testSigningKey, so cookiesA is valid there and only tokenB is wrong.
|
||||||
|
_, router := newCSRFTestRouter(t)
|
||||||
|
cookiesA, _ := csrfCredentials(t, router, nil)
|
||||||
|
_, tokenB := csrfCredentials(t, router, nil)
|
||||||
|
|
||||||
|
rec, logLine := postRefusedForm(t, true, "http://example.com", "",
|
||||||
|
cookiesA, tokenB)
|
||||||
|
checkFormRefused(t, rec, logLine, causeStaleToken, csrf.ErrBadToken)
|
||||||
|
}
|
||||||
|
|
||||||
|
// postRefusedForm sends a POST with the given cookies to the CSRF
|
||||||
|
// middleware built for debug. It adds the form token and the Origin and
|
||||||
|
// Referer headers when they are not empty. It returns the response and the
|
||||||
|
// one JSON log line the middleware wrote.
|
||||||
|
func postRefusedForm(
|
||||||
|
t *testing.T, debug bool, origin, referer string,
|
||||||
|
cookies []*http.Cookie, token string,
|
||||||
|
) (*httptest.ResponseRecorder, map[string]any) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
keys, err := seal.DeriveKeys(testSigningKey)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("seal.DeriveKeys() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var logs bytes.Buffer
|
||||||
|
|
||||||
|
log := slog.New(slog.NewJSONHandler(&logs, nil))
|
||||||
|
reached := http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {
|
||||||
|
t.Error("the refused form reached the handler behind the middleware")
|
||||||
|
})
|
||||||
|
srv := newCSRFProtect(keys.CSRFKey, debug, log)(reached)
|
||||||
|
|
||||||
|
form := url.Values{loginKeyField: {"x"}}
|
||||||
|
if token != "" {
|
||||||
|
form.Set(csrfTokenField, token)
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx := context.WithValue(context.Background(),
|
||||||
|
middleware.RequestIDKey, refusedFormRequestID)
|
||||||
|
req := httptest.NewRequestWithContext(ctx, http.MethodPost,
|
||||||
|
"http://example.com/", strings.NewReader(form.Encode()))
|
||||||
|
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||||
|
|
||||||
|
for _, c := range cookies {
|
||||||
|
req.AddCookie(c)
|
||||||
|
}
|
||||||
|
|
||||||
|
if origin != "" {
|
||||||
|
req.Header.Set("Origin", origin)
|
||||||
|
}
|
||||||
|
|
||||||
|
if referer != "" {
|
||||||
|
req.Header.Set("Referer", referer)
|
||||||
|
}
|
||||||
|
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
srv.ServeHTTP(rec, req)
|
||||||
|
|
||||||
|
var logLine map[string]any
|
||||||
|
|
||||||
|
err = json.Unmarshal(logs.Bytes(), &logLine)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("decoding log line %q: %v", logs.String(), err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return rec, logLine
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkFormRefused checks that rec is a 403 page naming wantCause and that
|
||||||
|
// logLine is a warning carrying the request ID and wantReason.
|
||||||
|
func checkFormRefused(
|
||||||
|
t *testing.T, rec *httptest.ResponseRecorder, logLine map[string]any,
|
||||||
|
wantCause string, wantReason error,
|
||||||
|
) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
if rec.Code != http.StatusForbidden {
|
||||||
|
t.Errorf("status = %d, want %d", rec.Code, http.StatusForbidden)
|
||||||
|
}
|
||||||
|
|
||||||
|
body := rec.Body.String()
|
||||||
|
t.Logf("body: %s", body)
|
||||||
|
|
||||||
|
if !strings.Contains(body, template.HTMLEscapeString(wantCause)) {
|
||||||
|
t.Errorf("page does not name the cause %q", wantCause)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("log line: %v", logLine)
|
||||||
|
|
||||||
|
if logLine["level"] != "WARN" || logLine["msg"] != "form refused" {
|
||||||
|
t.Errorf("log line = %v, want a WARN \"form refused\"", logLine)
|
||||||
|
}
|
||||||
|
|
||||||
|
if logLine["request_id"] != refusedFormRequestID {
|
||||||
|
t.Errorf("logged request_id = %v, want %q",
|
||||||
|
logLine["request_id"], refusedFormRequestID)
|
||||||
|
}
|
||||||
|
|
||||||
|
if logLine["reason"] != wantReason.Error() {
|
||||||
|
t.Errorf("logged reason = %v, want %q", logLine["reason"], wantReason)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -56,14 +56,16 @@ type Handlers struct {
|
|||||||
|
|
||||||
// New creates a new Handlers instance.
|
// New creates a new Handlers instance.
|
||||||
func New(_ fx.Lifecycle, params Params) *Handlers {
|
func New(_ fx.Lifecycle, params Params) *Handlers {
|
||||||
|
log := params.Logger.Get()
|
||||||
|
|
||||||
return &Handlers{
|
return &Handlers{
|
||||||
log: params.Logger.Get(),
|
log: log,
|
||||||
hc: params.Healthcheck,
|
hc: params.Healthcheck,
|
||||||
config: params.Config,
|
config: params.Config,
|
||||||
imgSvc: params.ImageService,
|
imgSvc: params.ImageService,
|
||||||
sessMgr: params.SessionManager,
|
sessMgr: params.SessionManager,
|
||||||
encGen: params.EncryptedURLGenerator,
|
encGen: params.EncryptedURLGenerator,
|
||||||
csrfProtect: newCSRFProtect(params.Keys.CSRFKey, params.Config.Debug),
|
csrfProtect: newCSRFProtect(params.Keys.CSRFKey, params.Config.Debug, log),
|
||||||
signer: signature.New(params.Config.SigningKey),
|
signer: signature.New(params.Config.SigningKey),
|
||||||
allowlist: allowlist.New(params.Config.AllowlistHosts),
|
allowlist: allowlist.New(params.Config.AllowlistHosts),
|
||||||
refererBlocklist: allowlist.New(params.Config.RefererBlocklist),
|
refererBlocklist: allowlist.New(params.Config.RefererBlocklist),
|
||||||
|
|||||||
@@ -234,12 +234,8 @@ func (s *Handlers) HandleTooManyRequests() http.HandlerFunc {
|
|||||||
func (s *Handlers) respondImageError(
|
func (s *Handlers) respondImageError(
|
||||||
w http.ResponseWriter, req *imagerequest.Request, err error,
|
w http.ResponseWriter, req *imagerequest.Request, err error,
|
||||||
) {
|
) {
|
||||||
// An upstream body whose first bytes are not those of the format its
|
// The image service has logged this as a refused source
|
||||||
// Content-Type names is the upstream's error, not pixa's: the image
|
if isFirstBytesMismatch(err) {
|
||||||
// service has logged it as a refused source.
|
|
||||||
if errors.Is(err, magic.ErrMagicByteMismatch) ||
|
|
||||||
errors.Is(err, magic.ErrUnknownFormat) ||
|
|
||||||
errors.Is(err, magic.ErrNotEnoughData) {
|
|
||||||
s.respondError(w, "unsupported source format",
|
s.respondError(w, "unsupported source format",
|
||||||
http.StatusUnsupportedMediaType)
|
http.StatusUnsupportedMediaType)
|
||||||
|
|
||||||
@@ -254,7 +250,9 @@ func (s *Handlers) respondImageError(
|
|||||||
switch {
|
switch {
|
||||||
case errors.Is(err, httpfetcher.ErrSSRFBlocked):
|
case errors.Is(err, httpfetcher.ErrSSRFBlocked):
|
||||||
status, message = http.StatusForbidden, "forbidden"
|
status, message = http.StatusForbidden, "forbidden"
|
||||||
case errors.Is(err, httpfetcher.ErrUpstreamError):
|
case errors.Is(err, httpfetcher.ErrUpstreamError),
|
||||||
|
errors.Is(err, httpfetcher.ErrInvalidHost),
|
||||||
|
errors.Is(err, httpfetcher.ErrUpstreamRequestFailed):
|
||||||
status, message = http.StatusBadGateway, "upstream error"
|
status, message = http.StatusBadGateway, "upstream error"
|
||||||
case errors.Is(err, httpfetcher.ErrUpstreamTimeout):
|
case errors.Is(err, httpfetcher.ErrUpstreamTimeout):
|
||||||
status, message = http.StatusGatewayTimeout, "upstream timeout"
|
status, message = http.StatusGatewayTimeout, "upstream timeout"
|
||||||
@@ -267,11 +265,29 @@ func (s *Handlers) respondImageError(
|
|||||||
errors.Is(err, imageprocessor.ErrUnsupportedInputFormat):
|
errors.Is(err, imageprocessor.ErrUnsupportedInputFormat):
|
||||||
status = http.StatusUnsupportedMediaType
|
status = http.StatusUnsupportedMediaType
|
||||||
message = "unsupported source format"
|
message = "unsupported source format"
|
||||||
|
case errors.Is(err, imageprocessor.ErrSVGTooThin):
|
||||||
|
status = http.StatusUnsupportedMediaType
|
||||||
|
message = fmt.Sprintf("SVG source too thin to render: one side is "+
|
||||||
|
"more than %d times the other", imageprocessor.MaxInputDimension)
|
||||||
|
case errors.Is(err, imageprocessor.ErrSVGRenderFailed):
|
||||||
|
status = http.StatusUnsupportedMediaType
|
||||||
|
message = fmt.Sprintf("SVG source could not be rendered: it is not "+
|
||||||
|
"valid, or it takes more than %d seconds or %d GiB of memory to "+
|
||||||
|
"render", int(imageprocessor.SVGRenderTimeout.Seconds()),
|
||||||
|
imageprocessor.MaxSVGRenderMemory/bytesPerGiB)
|
||||||
case errors.Is(err, imageprocessor.ErrOutputTooLarge):
|
case errors.Is(err, imageprocessor.ErrOutputTooLarge):
|
||||||
status = http.StatusBadRequest
|
status = http.StatusBadRequest
|
||||||
message = fmt.Sprintf("size too large for the source: the side "+
|
message = fmt.Sprintf("size too large for the source: the side "+
|
||||||
"computed from its proportions is above %d",
|
"computed from its proportions is above %d",
|
||||||
imageprocessor.MaxOutputDimension)
|
imageprocessor.MaxOutputDimension)
|
||||||
|
case errors.Is(err, imageprocessor.ErrSVGNotOriginalSize):
|
||||||
|
status = http.StatusBadRequest
|
||||||
|
message = "an SVG source is returned as SVG only at the orig size; " +
|
||||||
|
"ask for another format to resize it"
|
||||||
|
case errors.Is(err, imageprocessor.ErrSVGTooLargeForOrigSize):
|
||||||
|
status = http.StatusBadRequest
|
||||||
|
message = fmt.Sprintf("SVG source above %d on a side: give a size "+
|
||||||
|
"to render it at", imageprocessor.MaxOutputDimension)
|
||||||
case errors.Is(err, imageprocessor.ErrAVIFOutputTooLarge):
|
case errors.Is(err, imageprocessor.ErrAVIFOutputTooLarge):
|
||||||
status = http.StatusBadRequest
|
status = http.StatusBadRequest
|
||||||
message = fmt.Sprintf("size too large for AVIF: the output "+
|
message = fmt.Sprintf("size too large for AVIF: the output "+
|
||||||
@@ -295,6 +311,18 @@ func (s *Handlers) respondImageError(
|
|||||||
s.respondError(w, message, status)
|
s.respondError(w, message, status)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// bytesPerGiB is the number of bytes in a GiB.
|
||||||
|
const bytesPerGiB = 1 << 30
|
||||||
|
|
||||||
|
// isFirstBytesMismatch reports whether err refuses an upstream body whose
|
||||||
|
// first bytes are not those of the format its Content-Type names. That is the
|
||||||
|
// upstream's error, not pixa's.
|
||||||
|
func isFirstBytesMismatch(err error) bool {
|
||||||
|
return errors.Is(err, magic.ErrMagicByteMismatch) ||
|
||||||
|
errors.Is(err, magic.ErrUnknownFormat) ||
|
||||||
|
errors.Is(err, magic.ErrNotEnoughData)
|
||||||
|
}
|
||||||
|
|
||||||
// cacheControl returns the Cache-Control header for an image served through a
|
// cacheControl returns the Cache-Control header for an image served through a
|
||||||
// URL that expires at expires, or never when expires is the zero time. A cache
|
// URL that expires at expires, or never when expires is the zero time. A cache
|
||||||
// may keep the image for a year, but not past the URL's expiry, after which
|
// may keep the image for a year, but not past the URL's expiry, after which
|
||||||
@@ -313,6 +341,26 @@ func cacheControl(expires time.Time) string {
|
|||||||
return fmt.Sprintf("public, max-age=%d, immutable", int64(maxAge/time.Second))
|
return fmt.Sprintf("public, max-age=%d, immutable", int64(maxAge/time.Second))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// svgContentSecurityPolicyValue is the Content-Security-Policy header value
|
||||||
|
// of an SVG response, in place of the middleware's. A browser that opens the
|
||||||
|
// SVG applies its styles and draws its embedded data: images, but runs none
|
||||||
|
// of its scripts and loads nothing else. The styles need unsafe-inline, as an
|
||||||
|
// SVG keeps them inline, in style elements and attributes.
|
||||||
|
const svgContentSecurityPolicyValue = "default-src 'none'; " +
|
||||||
|
"style-src 'unsafe-inline'; " +
|
||||||
|
"img-src data:; " +
|
||||||
|
"frame-ancestors 'none'; " +
|
||||||
|
"sandbox"
|
||||||
|
|
||||||
|
// setSVGContentSecurityPolicy sets the Content-Security-Policy in header to
|
||||||
|
// svgContentSecurityPolicyValue when contentType is that of SVG. Any other
|
||||||
|
// response keeps the middleware's policy.
|
||||||
|
func setSVGContentSecurityPolicy(header http.Header, contentType string) {
|
||||||
|
if contentType == imagerequest.FormatSVG.MIMEType() {
|
||||||
|
header.Set("Content-Security-Policy", svgContentSecurityPolicyValue)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// refuseBlockedReferer answers 403 with a JSON error when the request's Referer
|
// refuseBlockedReferer answers 403 with a JSON error when the request's Referer
|
||||||
// names a host on referer_blocklist, and reports whether it answered. A request
|
// names a host on referer_blocklist, and reports whether it answered. A request
|
||||||
// with no Referer, or one that does not parse as a URL with a host, is not
|
// with no Referer, or one that does not parse as a URL with a host, is not
|
||||||
@@ -395,6 +443,7 @@ func (s *Handlers) serveImage(
|
|||||||
|
|
||||||
// Set response headers
|
// Set response headers
|
||||||
w.Header().Set("Content-Type", resp.ContentType)
|
w.Header().Set("Content-Type", resp.ContentType)
|
||||||
|
setSVGContentSecurityPolicy(w.Header(), resp.ContentType)
|
||||||
|
|
||||||
if resp.ContentLength > 0 {
|
if resp.ContentLength > 0 {
|
||||||
w.Header().Set("Content-Length", strconv.FormatInt(resp.ContentLength, 10))
|
w.Header().Set("Content-Length", strconv.FormatInt(resp.ContentLength, 10))
|
||||||
|
|||||||
@@ -185,13 +185,13 @@ func TestHandleImageEnc_MaxAge(t *testing.T) {
|
|||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
expires time.Time
|
ttl time.Duration
|
||||||
wantAtLeast int
|
wantAtLeast int
|
||||||
wantAtMost int
|
wantAtMost int
|
||||||
}{
|
}{
|
||||||
{"60 second TTL", time.Now().Add(time.Minute), 50, 60},
|
{"60 second TTL", time.Minute, 50, 60},
|
||||||
{"two-year TTL", time.Now().Add(2 * 365 * 24 * time.Hour), 31536000, 31536000},
|
{"two-year TTL", 2 * 365 * 24 * time.Hour, 31536000, 31536000},
|
||||||
{"no TTL", time.Time{}, 31536000, 31536000},
|
{"no TTL", 0, 31536000, 31536000},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
@@ -200,6 +200,14 @@ func TestHandleImageEnc_MaxAge(t *testing.T) {
|
|||||||
|
|
||||||
h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
||||||
|
|
||||||
|
// The expiry is set here, not in the table: a parallel subtest
|
||||||
|
// can wait behind other tests for more than the 10 seconds the
|
||||||
|
// 60 second case allows.
|
||||||
|
var expires time.Time
|
||||||
|
if tt.ttl != 0 {
|
||||||
|
expires = time.Now().Add(tt.ttl)
|
||||||
|
}
|
||||||
|
|
||||||
token, err := h.encGen.Generate(&imagerequest.Request{
|
token, err := h.encGen.Generate(&imagerequest.Request{
|
||||||
Source: imagerequest.Source{
|
Source: imagerequest.Source{
|
||||||
Host: signedHost,
|
Host: signedHost,
|
||||||
@@ -209,7 +217,7 @@ func TestHandleImageEnc_MaxAge(t *testing.T) {
|
|||||||
Size: imagerequest.Size{Width: 50, Height: 50},
|
Size: imagerequest.Size{Width: 50, Height: 50},
|
||||||
Format: imagerequest.FormatJPEG,
|
Format: imagerequest.FormatJPEG,
|
||||||
},
|
},
|
||||||
}, tt.expires)
|
}, expires)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Generate() error = %v", err)
|
t.Fatalf("Generate() error = %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import (
|
|||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
|
"syscall"
|
||||||
"testing"
|
"testing"
|
||||||
"testing/fstest"
|
"testing/fstest"
|
||||||
"time"
|
"time"
|
||||||
@@ -461,13 +462,20 @@ func TestImageRoutes_SameErrorAnswers(t *testing.T) {
|
|||||||
{"upstream error status",
|
{"upstream error status",
|
||||||
fmt.Errorf("%w: status 404", httpfetcher.ErrUpstreamError),
|
fmt.Errorf("%w: status 404", httpfetcher.ErrUpstreamError),
|
||||||
http.StatusBadGateway, slog.LevelWarn},
|
http.StatusBadGateway, slog.LevelWarn},
|
||||||
|
{"upstream host with no address",
|
||||||
|
fmt.Errorf("%w: cdn.example.com", httpfetcher.ErrInvalidHost),
|
||||||
|
http.StatusBadGateway, slog.LevelWarn},
|
||||||
|
{"connection refused",
|
||||||
|
fmt.Errorf("%w: %w", httpfetcher.ErrUpstreamRequestFailed,
|
||||||
|
syscall.ECONNREFUSED),
|
||||||
|
http.StatusBadGateway, slog.LevelWarn},
|
||||||
{"no response headers within the fetch timeout",
|
{"no response headers within the fetch timeout",
|
||||||
httpfetcher.ErrUpstreamTimeout, http.StatusGatewayTimeout,
|
httpfetcher.ErrUpstreamTimeout, http.StatusGatewayTimeout,
|
||||||
slog.LevelWarn},
|
slog.LevelWarn},
|
||||||
{"no free upstream connection", httpfetcher.ErrTooManyConnections,
|
{"no free upstream connection", httpfetcher.ErrTooManyConnections,
|
||||||
http.StatusServiceUnavailable, slog.LevelWarn},
|
http.StatusServiceUnavailable, slog.LevelWarn},
|
||||||
{"Content-Type not a source format",
|
{"Content-Type not a source format",
|
||||||
fmt.Errorf("%w: image/svg+xml", httpfetcher.ErrInvalidContentType),
|
fmt.Errorf("%w: image/tiff", httpfetcher.ErrInvalidContentType),
|
||||||
http.StatusUnsupportedMediaType, slog.LevelWarn},
|
http.StatusUnsupportedMediaType, slog.LevelWarn},
|
||||||
{"any other failure", io.ErrUnexpectedEOF,
|
{"any other failure", io.ErrUnexpectedEOF,
|
||||||
http.StatusInternalServerError, slog.LevelError},
|
http.StatusInternalServerError, slog.LevelError},
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ import (
|
|||||||
|
|
||||||
// TestUnsupportedSourceFormatAnswers415 checks that respondImageError, which
|
// TestUnsupportedSourceFormatAnswers415 checks that respondImageError, which
|
||||||
// answers the errors of both image routes, answers 415 with a clear error when
|
// answers the errors of both image routes, answers 415 with a clear error when
|
||||||
// the upstream answers with a content type pixa does not accept, such as SVG,
|
// the upstream answers with a content type pixa does not accept, such as TIFF,
|
||||||
// and when libvips does not read a source as the format its first bytes
|
// and when libvips does not read a source as the format its first bytes
|
||||||
// show, wrapped as the service wraps them.
|
// show, wrapped as the service wraps them.
|
||||||
func TestUnsupportedSourceFormatAnswers415(t *testing.T) {
|
func TestUnsupportedSourceFormatAnswers415(t *testing.T) {
|
||||||
@@ -23,11 +23,11 @@ func TestUnsupportedSourceFormatAnswers415(t *testing.T) {
|
|||||||
|
|
||||||
h := &Handlers{log: slog.New(slog.DiscardHandler)}
|
h := &Handlers{log: slog.New(slog.DiscardHandler)}
|
||||||
req := &imagerequest.Request{
|
req := &imagerequest.Request{
|
||||||
Source: imagerequest.Source{Host: "img.example.com", Path: "/a.svg"},
|
Source: imagerequest.Source{Host: "img.example.com", Path: "/a.tiff"},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, err := range []error{
|
for _, err := range []error{
|
||||||
fmt.Errorf("upstream fetch failed: %w: image/svg+xml",
|
fmt.Errorf("upstream fetch failed: %w: image/tiff",
|
||||||
httpfetcher.ErrInvalidContentType),
|
httpfetcher.ErrInvalidContentType),
|
||||||
fmt.Errorf("image processing failed: %w: libvips does not read the input as avif",
|
fmt.Errorf("image processing failed: %w: libvips does not read the input as avif",
|
||||||
imageprocessor.ErrUnsupportedInputFormat),
|
imageprocessor.ErrUnsupportedInputFormat),
|
||||||
@@ -47,3 +47,67 @@ func TestUnsupportedSourceFormatAnswers415(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestSVGTooThinAnswers415 checks that respondImageError answers 415 with a
|
||||||
|
// clear error for an SVG source too thin to render, wrapped as the service
|
||||||
|
// wraps it.
|
||||||
|
func TestSVGTooThinAnswers415(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
h := &Handlers{log: slog.New(slog.DiscardHandler)}
|
||||||
|
req := &imagerequest.Request{
|
||||||
|
Source: imagerequest.Source{Host: "svg.example.com", Path: "/a.svg"},
|
||||||
|
}
|
||||||
|
err := fmt.Errorf("image processing failed: %w: 81930x10",
|
||||||
|
imageprocessor.ErrSVGTooThin)
|
||||||
|
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
h.respondImageError(rec, req, err)
|
||||||
|
t.Logf("for %v: %d %s", err, rec.Code, rec.Body.String())
|
||||||
|
|
||||||
|
if rec.Code != http.StatusUnsupportedMediaType {
|
||||||
|
t.Errorf("status = %d, want %d", rec.Code,
|
||||||
|
http.StatusUnsupportedMediaType)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.Contains(rec.Body.String(), "SVG source too thin to render") {
|
||||||
|
t.Errorf("body = %q, want the SVG too thin error", rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSVGRenderFailedAnswers415 checks that respondImageError answers 415,
|
||||||
|
// naming the limits, for an SVG source that a child process could not read
|
||||||
|
// or render, wrapped as the service wraps it.
|
||||||
|
func TestSVGRenderFailedAnswers415(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
h := &Handlers{log: slog.New(slog.DiscardHandler)}
|
||||||
|
err := fmt.Errorf("image processing failed: %w: it took more than 20s",
|
||||||
|
imageprocessor.ErrSVGRenderFailed)
|
||||||
|
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
h.respondImageError(rec, &imagerequest.Request{}, err)
|
||||||
|
t.Logf("for %v: %d %s", err, rec.Code, rec.Body.String())
|
||||||
|
|
||||||
|
checkErrorBody(t, rec, http.StatusUnsupportedMediaType,
|
||||||
|
"SVG source could not be rendered: it is not valid, or it takes more "+
|
||||||
|
"than 20 seconds or 1 GiB of memory to render")
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSVGTooLargeForOrigSizeAnswers400 checks that respondImageError answers
|
||||||
|
// 400, asking for a size, for an SVG source above 8192 on a side asked for at
|
||||||
|
// the orig size in a raster format, wrapped as the service wraps it.
|
||||||
|
func TestSVGTooLargeForOrigSizeAnswers400(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
h := &Handlers{log: slog.New(slog.DiscardHandler)}
|
||||||
|
err := fmt.Errorf("image processing failed: %w: 10000x10 is above 8192",
|
||||||
|
imageprocessor.ErrSVGTooLargeForOrigSize)
|
||||||
|
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
h.respondImageError(rec, &imagerequest.Request{}, err)
|
||||||
|
t.Logf("for %v: %d %s", err, rec.Code, rec.Body.String())
|
||||||
|
|
||||||
|
checkErrorBody(t, rec, http.StatusBadRequest,
|
||||||
|
"SVG source above 8192 on a side: give a size to render it at")
|
||||||
|
}
|
||||||
@@ -69,7 +69,11 @@ var (
|
|||||||
ErrInvalidContentType = errors.New("invalid or unsupported content type")
|
ErrInvalidContentType = errors.New("invalid or unsupported content type")
|
||||||
ErrUpstreamError = errors.New("upstream server error")
|
ErrUpstreamError = errors.New("upstream server error")
|
||||||
ErrUpstreamTimeout = errors.New("upstream request timeout")
|
ErrUpstreamTimeout = errors.New("upstream request timeout")
|
||||||
ErrTooManyConnections = errors.New("too many concurrent upstream connections")
|
// ErrUpstreamRequestFailed is a fetch that failed before the upstream
|
||||||
|
// answered, other than by timeout or cancellation: a failed connection
|
||||||
|
// or TLS handshake, or a redirect the fetcher refuses.
|
||||||
|
ErrUpstreamRequestFailed = errors.New("upstream request failed")
|
||||||
|
ErrTooManyConnections = errors.New("too many concurrent upstream connections")
|
||||||
)
|
)
|
||||||
|
|
||||||
// Internal fetcher errors.
|
// Internal fetcher errors.
|
||||||
@@ -124,6 +128,7 @@ func allowedContentTypes() []string {
|
|||||||
imagerequest.FormatWebP.MIMEType(),
|
imagerequest.FormatWebP.MIMEType(),
|
||||||
imagerequest.FormatAVIF.MIMEType(),
|
imagerequest.FormatAVIF.MIMEType(),
|
||||||
imagerequest.FormatJXL.MIMEType(),
|
imagerequest.FormatJXL.MIMEType(),
|
||||||
|
imagerequest.FormatSVG.MIMEType(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -283,11 +288,7 @@ func (f *HTTPFetcher) Fetch(
|
|||||||
fetchDuration := time.Since(startTime)
|
fetchDuration := time.Since(startTime)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, context.DeadlineExceeded) {
|
return nil, upstreamRequestError(err)
|
||||||
return nil, ErrUpstreamTimeout
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil, fmt.Errorf("upstream request failed: %w", err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// The deferred release gives the connection back, as no body follows
|
// The deferred release gives the connection back, as no body follows
|
||||||
@@ -316,6 +317,33 @@ func (f *HTTPFetcher) Fetch(
|
|||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// upstreamRequestError returns the error Fetch reports when its upstream
|
||||||
|
// request fails with err:
|
||||||
|
// - err wrapped in ErrUpstreamTimeout for a timeout, so the log names what
|
||||||
|
// timed out.
|
||||||
|
// - err, still matching context.Canceled, when the caller cancelled the
|
||||||
|
// fetch.
|
||||||
|
// - err wrapped in ErrUpstreamRequestFailed otherwise.
|
||||||
|
func upstreamRequestError(err error) error {
|
||||||
|
var netErr net.Error
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, context.DeadlineExceeded),
|
||||||
|
errors.Is(err, ErrUpstreamTimeout),
|
||||||
|
errors.As(err, &netErr) && netErr.Timeout():
|
||||||
|
// A timeout other than the fetch timeout, such as the TLS
|
||||||
|
// handshake timeout, does not match context.DeadlineExceeded. An
|
||||||
|
// address lookup or a connection that timed out is
|
||||||
|
// ErrUpstreamTimeout already, so its log line names that error twice.
|
||||||
|
return fmt.Errorf("%w: %w", ErrUpstreamTimeout, err)
|
||||||
|
case errors.Is(err, context.Canceled):
|
||||||
|
// The caller cancelled the fetch; the upstream did not fail
|
||||||
|
return fmt.Errorf("upstream request cancelled: %w", err)
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("%w: %w", ErrUpstreamRequestFailed, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// requestHeaders returns the headers of a fetch: User-Agent, Accept, the
|
// requestHeaders returns the headers of a fetch: User-Agent, Accept, the
|
||||||
// ID of the request the fetch serves, so the fetch can be found in the
|
// ID of the request the fetch serves, so the fetch can be found in the
|
||||||
// upstream host's logs, and a non-empty ifNoneMatch or ifModifiedSince.
|
// upstream host's logs, and a non-empty ifNoneMatch or ifModifiedSince.
|
||||||
@@ -505,7 +533,7 @@ func validateURL(ctx context.Context, fetchURL *neturl.URL, allowHTTP bool) erro
|
|||||||
// Resolve the host to check IP addresses
|
// Resolve the host to check IP addresses
|
||||||
addrs, err := net.DefaultResolver.LookupIPAddr(ctx, host)
|
addrs, err := net.DefaultResolver.LookupIPAddr(ctx, host)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("%w: %s", ErrInvalidHost, host)
|
return addressLookupError(host, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
private := slices.ContainsFunc(addrs, func(addr net.IPAddr) bool {
|
private := slices.ContainsFunc(addrs, func(addr net.IPAddr) bool {
|
||||||
@@ -518,6 +546,23 @@ func validateURL(ctx context.Context, fetchURL *neturl.URL, allowHTTP bool) erro
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// addressLookupError returns the error for a failed address lookup of host:
|
||||||
|
// - ErrUpstreamTimeout when the lookup timed out.
|
||||||
|
// - err, still matching context.Canceled, when the lookup was cancelled.
|
||||||
|
// - ErrInvalidHost otherwise.
|
||||||
|
func addressLookupError(host string, err error) error {
|
||||||
|
var netErr net.Error
|
||||||
|
if errors.As(err, &netErr) && netErr.Timeout() {
|
||||||
|
return fmt.Errorf("%w: address lookup of %s", ErrUpstreamTimeout, host)
|
||||||
|
}
|
||||||
|
|
||||||
|
if errors.Is(err, context.Canceled) {
|
||||||
|
return fmt.Errorf("address lookup cancelled: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Errorf("%w: %s", ErrInvalidHost, host)
|
||||||
|
}
|
||||||
|
|
||||||
// isLocalhost checks if the host is localhost.
|
// isLocalhost checks if the host is localhost.
|
||||||
func isLocalhost(host string) bool {
|
func isLocalhost(host string) bool {
|
||||||
host = strings.ToLower(host)
|
host = strings.ToLower(host)
|
||||||
@@ -646,7 +691,7 @@ func dialSSRFSafe(
|
|||||||
// Resolve the address
|
// Resolve the address
|
||||||
ips, err := net.DefaultResolver.LookupIP(ctx, "ip", host)
|
ips, err := net.DefaultResolver.LookupIP(ctx, "ip", host)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %s", ErrInvalidHost, host)
|
return nil, addressLookupError(host, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check all resolved IPs
|
// Check all resolved IPs
|
||||||
@@ -657,7 +702,10 @@ func dialSSRFSafe(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Connect using the first valid IP
|
// Connect using the first valid IP
|
||||||
var dialer net.Dialer
|
var (
|
||||||
|
dialer net.Dialer
|
||||||
|
lastDialErr error
|
||||||
|
)
|
||||||
|
|
||||||
for _, ip := range ips {
|
for _, ip := range ips {
|
||||||
addr := net.JoinHostPort(ip.String(), port)
|
addr := net.JoinHostPort(ip.String(), port)
|
||||||
@@ -666,9 +714,18 @@ func dialSSRFSafe(
|
|||||||
if err == nil {
|
if err == nil {
|
||||||
return conn, nil
|
return conn, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
lastDialErr = err
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil, fmt.Errorf("%w to %s", errConnectFailed, host)
|
// Only the last address tried decides whether the connection timed out
|
||||||
|
var netErr net.Error
|
||||||
|
if errors.As(lastDialErr, &netErr) && netErr.Timeout() {
|
||||||
|
return nil, fmt.Errorf("%w: connect to %s: %w",
|
||||||
|
ErrUpstreamTimeout, host, lastDialErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil, fmt.Errorf("%w to %s: %w", errConnectFailed, host, lastDialErr)
|
||||||
}
|
}
|
||||||
|
|
||||||
// limitedReader wraps a reader and refuses a body longer than remaining
|
// limitedReader wraps a reader and refuses a body longer than remaining
|
||||||
|
|||||||
@@ -279,7 +279,7 @@ func TestDetectContentTypeFromPath(t *testing.T) {
|
|||||||
{"foo/bar.gif", imagerequest.FormatGIF.MIMEType()},
|
{"foo/bar.gif", imagerequest.FormatGIF.MIMEType()},
|
||||||
{"foo/bar.webp", imagerequest.FormatWebP.MIMEType()},
|
{"foo/bar.webp", imagerequest.FormatWebP.MIMEType()},
|
||||||
{"foo/bar.avif", imagerequest.FormatAVIF.MIMEType()},
|
{"foo/bar.avif", imagerequest.FormatAVIF.MIMEType()},
|
||||||
{"foo/bar.svg", contentTypeSVG},
|
{"foo/bar.svg", imagerequest.FormatSVG.MIMEType()},
|
||||||
{"foo/bar.bin", contentTypeOctetStream},
|
{"foo/bar.bin", contentTypeOctetStream},
|
||||||
{"foo/bar", contentTypeOctetStream},
|
{"foo/bar", contentTypeOctetStream},
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -67,11 +67,9 @@ func (m *MockFetcher) Fetch(
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// The content types the mock fetcher gives files in no format pixa reads.
|
// contentTypeOctetStream is the content type the mock fetcher gives files in
|
||||||
const (
|
// no format pixa reads.
|
||||||
contentTypeSVG = "image/svg+xml"
|
const contentTypeOctetStream = "application/octet-stream"
|
||||||
contentTypeOctetStream = "application/octet-stream"
|
|
||||||
)
|
|
||||||
|
|
||||||
// detectContentTypeFromPath returns the MIME type based on file extension.
|
// detectContentTypeFromPath returns the MIME type based on file extension.
|
||||||
func detectContentTypeFromPath(path string) string {
|
func detectContentTypeFromPath(path string) string {
|
||||||
@@ -91,7 +89,7 @@ func detectContentTypeFromPath(path string) string {
|
|||||||
case strings.HasSuffix(path, ".jxl"):
|
case strings.HasSuffix(path, ".jxl"):
|
||||||
return imagerequest.FormatJXL.MIMEType()
|
return imagerequest.FormatJXL.MIMEType()
|
||||||
case strings.HasSuffix(path, ".svg"):
|
case strings.HasSuffix(path, ".svg"):
|
||||||
return contentTypeSVG
|
return imagerequest.FormatSVG.MIMEType()
|
||||||
default:
|
default:
|
||||||
return contentTypeOctetStream
|
return contentTypeOctetStream
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,213 @@
|
|||||||
|
package httpfetcher
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
|
"syscall"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestFetchFailedBeforeAnswer checks the error of a fetch whose connection
|
||||||
|
// fails:
|
||||||
|
// - refused: ErrUpstreamRequestFailed.
|
||||||
|
// - timed out: ErrUpstreamTimeout, still matching the dial's error, even for
|
||||||
|
// a timeout error that does not match context.DeadlineExceeded, such as
|
||||||
|
// the TLS handshake timeout.
|
||||||
|
// - cancelled by its caller: context.Canceled, not ErrUpstreamRequestFailed.
|
||||||
|
func TestFetchFailedBeforeAnswer(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
dial func(cancel context.CancelFunc) error
|
||||||
|
wantErrs []error
|
||||||
|
wantNotErr error
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "connection refused",
|
||||||
|
dial: func(context.CancelFunc) error {
|
||||||
|
return syscall.ECONNREFUSED
|
||||||
|
},
|
||||||
|
wantErrs: []error{ErrUpstreamRequestFailed},
|
||||||
|
wantNotErr: context.Canceled,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "timed out while connecting",
|
||||||
|
dial: func(context.CancelFunc) error {
|
||||||
|
return os.ErrDeadlineExceeded
|
||||||
|
},
|
||||||
|
wantErrs: []error{ErrUpstreamTimeout, os.ErrDeadlineExceeded},
|
||||||
|
wantNotErr: ErrUpstreamRequestFailed,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "cancelled while connecting",
|
||||||
|
dial: func(cancel context.CancelFunc) error {
|
||||||
|
cancel()
|
||||||
|
|
||||||
|
return context.Canceled
|
||||||
|
},
|
||||||
|
wantErrs: []error{context.Canceled},
|
||||||
|
wantNotErr: ErrUpstreamRequestFailed,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(testContext(t))
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
f := New(&Config{
|
||||||
|
Timeout: 30 * time.Second,
|
||||||
|
MaxResponseSize: 50 << 20,
|
||||||
|
AllowHTTP: true,
|
||||||
|
MaxConnectionsPerHost: 20,
|
||||||
|
MaxConnections: 64,
|
||||||
|
DialContext: func(context.Context, string, string) (net.Conn, error) {
|
||||||
|
return nil, tt.dial(cancel)
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
_, err := f.Fetch(ctx, upstreamSource("/image"), "", "")
|
||||||
|
t.Logf("Fetch() error: %v", err)
|
||||||
|
|
||||||
|
for _, wantErr := range tt.wantErrs {
|
||||||
|
if !errors.Is(err, wantErr) {
|
||||||
|
t.Errorf("Fetch() error = %v, want %v", err, wantErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if errors.Is(err, tt.wantNotErr) {
|
||||||
|
t.Errorf("Fetch() error = %v, must not be %v", err, tt.wantNotErr)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestFetchAddressLookupTimedOut checks that a fetch whose address lookup
|
||||||
|
// times out fails with ErrUpstreamTimeout, both for the lookup of the source
|
||||||
|
// host and for the lookup while connecting. The error must match neither
|
||||||
|
// ErrInvalidHost nor ErrUpstreamRequestFailed, which the image routes answer
|
||||||
|
// 502. A context whose deadline has passed makes the lookup time out.
|
||||||
|
func TestFetchAddressLookupTimedOut(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
checkTimedOut := func(t *testing.T, err error) {
|
||||||
|
t.Helper()
|
||||||
|
t.Logf("Fetch() error: %v", err)
|
||||||
|
|
||||||
|
if !errors.Is(err, ErrUpstreamTimeout) {
|
||||||
|
t.Errorf("Fetch() error = %v, want ErrUpstreamTimeout", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if errors.Is(err, ErrInvalidHost) || errors.Is(err, ErrUpstreamRequestFailed) {
|
||||||
|
t.Errorf("Fetch() error = %v, must match neither ErrInvalidHost "+
|
||||||
|
"nor ErrUpstreamRequestFailed", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("lookup of the source host", func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(testContext(t), 0)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
f := New(&Config{
|
||||||
|
Timeout: 30 * time.Second,
|
||||||
|
MaxResponseSize: 50 << 20,
|
||||||
|
MaxConnectionsPerHost: 20,
|
||||||
|
MaxConnections: 64,
|
||||||
|
})
|
||||||
|
|
||||||
|
_, err := f.Fetch(ctx, imageSourceOn("cdn.example.com"), "", "")
|
||||||
|
checkTimedOut(t, err)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("lookup while connecting", func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
// The source host is an address, which validateURL does not look up;
|
||||||
|
// the dialer looks up a name
|
||||||
|
f := New(&Config{
|
||||||
|
Timeout: 30 * time.Second,
|
||||||
|
MaxResponseSize: 50 << 20,
|
||||||
|
MaxConnectionsPerHost: 20,
|
||||||
|
MaxConnections: 64,
|
||||||
|
DialContext: func(ctx context.Context, network, _ string) (net.Conn, error) {
|
||||||
|
ctx, cancel := context.WithTimeout(ctx, 0)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
return dialSSRFSafe(ctx, network, "img.example.com:443", nil)
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
_, err := f.Fetch(testContext(t), upstreamSource("/image"), "", "")
|
||||||
|
checkTimedOut(t, err)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestFetchConnectionTimedOut checks that a fetch whose connection in
|
||||||
|
// dialSSRFSafe times out fails with ErrUpstreamTimeout, which the image
|
||||||
|
// routes answer 504. The error must match neither ErrUpstreamRequestFailed
|
||||||
|
// nor ErrInvalidHost, which they answer 502. A context whose deadline has
|
||||||
|
// passed makes the connection time out without touching the network.
|
||||||
|
func TestFetchConnectionTimedOut(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
f := New(&Config{
|
||||||
|
Timeout: 30 * time.Second,
|
||||||
|
MaxResponseSize: 50 << 20,
|
||||||
|
MaxConnectionsPerHost: 20,
|
||||||
|
MaxConnections: 64,
|
||||||
|
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||||
|
ctx, cancel := context.WithTimeout(ctx, 0)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
return dialSSRFSafe(ctx, network, addr, nil)
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
_, err := f.Fetch(testContext(t), upstreamSource("/image"), "", "")
|
||||||
|
t.Logf("Fetch() error: %v", err)
|
||||||
|
|
||||||
|
if !errors.Is(err, ErrUpstreamTimeout) {
|
||||||
|
t.Errorf("Fetch() error = %v, want ErrUpstreamTimeout", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if errors.Is(err, ErrUpstreamRequestFailed) || errors.Is(err, ErrInvalidHost) {
|
||||||
|
t.Errorf("Fetch() error = %v, must match neither "+
|
||||||
|
"ErrUpstreamRequestFailed nor ErrInvalidHost", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestFetchAddressLookupCancelled checks that a fetch whose address lookup
|
||||||
|
// its caller cancelled fails with an error matching context.Canceled, not
|
||||||
|
// ErrInvalidHost, which the image routes answer 502.
|
||||||
|
func TestFetchAddressLookupCancelled(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(testContext(t))
|
||||||
|
cancel()
|
||||||
|
|
||||||
|
f := New(&Config{
|
||||||
|
Timeout: 30 * time.Second,
|
||||||
|
MaxResponseSize: 50 << 20,
|
||||||
|
MaxConnectionsPerHost: 20,
|
||||||
|
MaxConnections: 64,
|
||||||
|
})
|
||||||
|
|
||||||
|
_, err := f.Fetch(ctx, imageSourceOn("cdn.example.com"), "", "")
|
||||||
|
t.Logf("Fetch() error: %v", err)
|
||||||
|
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Errorf("Fetch() error = %v, want context.Canceled", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if errors.Is(err, ErrInvalidHost) {
|
||||||
|
t.Errorf("Fetch() error = %v, must not be ErrInvalidHost", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,4 +1,7 @@
|
|||||||
// Package imageprocessor provides image format conversion and resizing using libvips.
|
// Package imageprocessor provides image format conversion and resizing using libvips.
|
||||||
|
// It reads the size of an SVG source and renders it in child processes, each
|
||||||
|
// the running program started again (see RunAsSVGChildProcess). pixa kills
|
||||||
|
// them after SVGRenderTimeout, and each may use at most MaxSVGRenderMemory.
|
||||||
package imageprocessor
|
package imageprocessor
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -7,6 +10,8 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"math"
|
||||||
|
"strconv"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -113,10 +118,43 @@ var ErrUnsupportedOutputFormat = errors.New("unsupported output format")
|
|||||||
// ErrUnsupportedInputFormat is returned, before the input is decoded, when
|
// ErrUnsupportedInputFormat is returned, before the input is decoded, when
|
||||||
// libvips reads the input's first bytes as another format than its source
|
// libvips reads the input's first bytes as another format than its source
|
||||||
// format, or as none. An SVG whose first bytes pass the first-bytes check for
|
// format, or as none. An SVG whose first bytes pass the first-bytes check for
|
||||||
// AVIF is one example. A source format other than the six pixa reads is also
|
// AVIF is one example. A source format other than the seven pixa reads is
|
||||||
// refused with it.
|
// also refused with it.
|
||||||
var ErrUnsupportedInputFormat = errors.New("unsupported input format")
|
var ErrUnsupportedInputFormat = errors.New("unsupported input format")
|
||||||
|
|
||||||
|
// ErrSVGNotOriginalSize is returned when orig is asked of an SVG source at a
|
||||||
|
// size other than orig. pixa returns an SVG only as it is; a raster format
|
||||||
|
// gives it a size.
|
||||||
|
var ErrSVGNotOriginalSize = errors.New(
|
||||||
|
"an SVG is returned as SVG only at the orig size")
|
||||||
|
|
||||||
|
// ErrSVGTooThin is returned, before rendering, when an SVG source has one
|
||||||
|
// side more than MaxInputDimension times the other. Rendered with its longer
|
||||||
|
// side at most MaxInputDimension, its shorter side would be under 1 pixel.
|
||||||
|
var ErrSVGTooThin = errors.New("SVG source too thin to render")
|
||||||
|
|
||||||
|
// ErrSVGTooLargeForOrigSize is returned, before rendering, when a format other
|
||||||
|
// than orig asks for an SVG source at the orig size and the SVG has a side
|
||||||
|
// above MaxOutputDimension.
|
||||||
|
var ErrSVGTooLargeForOrigSize = errors.New(
|
||||||
|
"SVG source too large to render at its own size")
|
||||||
|
|
||||||
|
// SVGRenderTimeout is how long the child processes that read the size of one
|
||||||
|
// SVG source and render it may run, counted from the start of the first,
|
||||||
|
// before pixa kills them.
|
||||||
|
const SVGRenderTimeout = 20 * time.Second
|
||||||
|
|
||||||
|
// MaxSVGRenderMemory is the most memory, in bytes, that each child process
|
||||||
|
// reading the size of an SVG source or rendering it may use. It is the
|
||||||
|
// child's RLIMIT_DATA, which limits its heap and its other private writable
|
||||||
|
// memory.
|
||||||
|
const MaxSVGRenderMemory = 1 << 30 // 1 GiB
|
||||||
|
|
||||||
|
// ErrSVGRenderFailed is returned when a child process fails to read the size
|
||||||
|
// of an SVG source or to render it: librsvg cannot read the SVG, or the
|
||||||
|
// child takes more than SVGRenderTimeout or MaxSVGRenderMemory.
|
||||||
|
var ErrSVGRenderFailed = errors.New("SVG source could not be rendered")
|
||||||
|
|
||||||
// ErrTooManyImages is returned when MaxConcurrentProcessing images are being
|
// ErrTooManyImages is returned when MaxConcurrentProcessing images are being
|
||||||
// processed and none finishes within ProcessingWaitTimeout.
|
// processed and none finishes within ProcessingWaitTimeout.
|
||||||
var ErrTooManyImages = errors.New("too many images being processed at once")
|
var ErrTooManyImages = errors.New("too many images being processed at once")
|
||||||
@@ -134,6 +172,10 @@ type ImageProcessor struct {
|
|||||||
processingSemaphore chan struct{}
|
processingSemaphore chan struct{}
|
||||||
// processingWaitTimeout is ProcessingWaitTimeout; tests shorten it.
|
// processingWaitTimeout is ProcessingWaitTimeout; tests shorten it.
|
||||||
processingWaitTimeout time.Duration
|
processingWaitTimeout time.Duration
|
||||||
|
// svgRenderTimeout is SVGRenderTimeout; tests shorten it.
|
||||||
|
svgRenderTimeout time.Duration
|
||||||
|
// maxSVGRenderMemory is MaxSVGRenderMemory; tests lower it.
|
||||||
|
maxSVGRenderMemory int
|
||||||
}
|
}
|
||||||
|
|
||||||
// Params holds configuration for creating an ImageProcessor. Both fields
|
// Params holds configuration for creating an ImageProcessor. Both fields
|
||||||
@@ -153,6 +195,8 @@ func New(params Params) *ImageProcessor {
|
|||||||
maxInputBytes: params.MaxInputBytes,
|
maxInputBytes: params.MaxInputBytes,
|
||||||
processingSemaphore: make(chan struct{}, params.MaxConcurrentProcessing),
|
processingSemaphore: make(chan struct{}, params.MaxConcurrentProcessing),
|
||||||
processingWaitTimeout: ProcessingWaitTimeout,
|
processingWaitTimeout: ProcessingWaitTimeout,
|
||||||
|
svgRenderTimeout: SVGRenderTimeout,
|
||||||
|
maxSVGRenderMemory: MaxSVGRenderMemory,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -164,7 +208,10 @@ func New(params Params) *ImageProcessor {
|
|||||||
// encodes, it returns ctx's error and gives the slot back. govips gives pixa
|
// encodes, it returns ctx's error and gives the slot back. govips gives pixa
|
||||||
// no way to stop libvips once an encode has begun, so an encode keeps its
|
// no way to stop libvips once an encode has begun, so an encode keeps its
|
||||||
// slot until it finishes. An AVIF output of more than MaxAVIFOutputPixels is
|
// slot until it finishes. An AVIF output of more than MaxAVIFOutputPixels is
|
||||||
// refused with ErrAVIFOutputTooLarge before anything is decoded.
|
// refused with ErrAVIFOutputTooLarge before anything is decoded. orig of an
|
||||||
|
// SVG source is the SVG as it is, since libvips cannot save SVG; any other
|
||||||
|
// format renders it at the output size. Child processes read an SVG's size
|
||||||
|
// and render it, as processSVG describes.
|
||||||
func (p *ImageProcessor) Process(
|
func (p *ImageProcessor) Process(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
input io.Reader,
|
input io.Reader,
|
||||||
@@ -183,35 +230,18 @@ func (p *ImageProcessor) Process(
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Input libvips would not load as sourceFormat is refused before it is
|
if sourceFormat == imagerequest.FormatSVG {
|
||||||
// decoded
|
return p.processSVG(ctx, data, transform)
|
||||||
err = checkInputFormat(data, sourceFormat)
|
}
|
||||||
|
|
||||||
|
img, err := loadImage(data, sourceFormat)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Decode image
|
|
||||||
img, err := vips.NewImageFromBuffer(data)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to decode image: %w", err)
|
|
||||||
}
|
|
||||||
defer img.Close()
|
defer img.Close()
|
||||||
|
|
||||||
// Turn the image upright now: encode strips the EXIF orientation tag,
|
origWidth, origHeight := img.Width(), img.Height()
|
||||||
// and sizes below must be worked out on the upright image.
|
outputFormat := resolveOutputFormat(transform.Format, sourceFormat)
|
||||||
err = img.AutoRotate()
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to auto-rotate: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get original dimensions
|
|
||||||
origWidth := img.Width()
|
|
||||||
origHeight := img.Height()
|
|
||||||
|
|
||||||
// Validate input dimensions to prevent DoS via decompression bombs
|
|
||||||
if origWidth > MaxInputDimension || origHeight > MaxInputDimension {
|
|
||||||
return nil, ErrInputTooLarge
|
|
||||||
}
|
|
||||||
|
|
||||||
// Determine target dimensions; too large ones are refused before resizing
|
// Determine target dimensions; too large ones are refused before resizing
|
||||||
targetWidth, targetHeight, err := targetDimensions(transform.Size,
|
targetWidth, targetHeight, err := targetDimensions(transform.Size,
|
||||||
@@ -220,12 +250,6 @@ func (p *ImageProcessor) Process(
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// orig is the source's own format; encode refuses an empty format
|
|
||||||
outputFormat := transform.Format
|
|
||||||
if outputFormat == imagerequest.FormatOriginal {
|
|
||||||
outputFormat = sourceFormat
|
|
||||||
}
|
|
||||||
|
|
||||||
// Determine output dimensions; too large ones are refused before
|
// Determine output dimensions; too large ones are refused before
|
||||||
// resizing, which with FitCover already decodes the input
|
// resizing, which with FitCover already decodes the input
|
||||||
outputWidth, outputHeight, err := outputDimensions(origWidth, origHeight,
|
outputWidth, outputHeight, err := outputDimensions(origWidth, origHeight,
|
||||||
@@ -242,8 +266,104 @@ func (p *ImageProcessor) Process(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Encode to target format
|
return p.encodeResult(ctx, img, outputFormat, transform.Quality,
|
||||||
output, err := p.encode(ctx, img, outputFormat, transform.Quality)
|
origWidth, origHeight)
|
||||||
|
}
|
||||||
|
|
||||||
|
// WaitForProcessing waits until no image is being processed, or until ctx
|
||||||
|
// ends, and returns how many images were still being processed then. It
|
||||||
|
// waits by taking each slot in processingSemaphore as it frees up until it
|
||||||
|
// holds them all, or until ctx ends, then gives back the slots it took.
|
||||||
|
func (p *ImageProcessor) WaitForProcessing(ctx context.Context) int {
|
||||||
|
taken := 0
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
for range taken {
|
||||||
|
<-p.processingSemaphore
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
for taken < cap(p.processingSemaphore) {
|
||||||
|
select {
|
||||||
|
case p.processingSemaphore <- struct{}{}:
|
||||||
|
taken++
|
||||||
|
case <-ctx.Done():
|
||||||
|
return len(p.processingSemaphore) - taken
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// processSVG is Process for an SVG source, data. Apart from the first-bytes
|
||||||
|
// check, pixad hands the SVG to libvips only in child processes (see
|
||||||
|
// runSVGChildProcess). One reads the SVG's size. For a raster format a second
|
||||||
|
// renders it as PNG, which is then resized and encoded as any other image.
|
||||||
|
// pixa kills them once svgRenderTimeout has passed since the first one
|
||||||
|
// started.
|
||||||
|
func (p *ImageProcessor) processSVG(
|
||||||
|
ctx context.Context, data []byte, transform *imagerequest.Transform,
|
||||||
|
) (*Result, error) {
|
||||||
|
err := checkInputFormat(data, imagerequest.FormatSVG)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
deadline := time.Now().Add(p.svgRenderTimeout)
|
||||||
|
|
||||||
|
origWidth, origHeight, err := p.readSVGSize(ctx, deadline, data)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
outputFormat := resolveOutputFormat(transform.Format, imagerequest.FormatSVG)
|
||||||
|
if outputFormat == imagerequest.FormatSVG {
|
||||||
|
return svgAsItIs(data, transform.Size, origWidth, origHeight)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = checkSVGOrigSize(transform.Size, origWidth, origHeight)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
targetWidth, targetHeight, err := targetDimensions(transform.Size,
|
||||||
|
origWidth, origHeight)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
outputWidth, outputHeight, err := outputDimensions(origWidth, origHeight,
|
||||||
|
targetWidth, targetHeight, transform.FitMode, outputFormat)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// The SVG is rendered at about the output size, to stay sharp
|
||||||
|
img, err := p.renderSVG(ctx, deadline, data, origWidth, origHeight,
|
||||||
|
outputWidth, outputHeight)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer img.Close()
|
||||||
|
|
||||||
|
if targetWidth != origWidth || targetHeight != origHeight {
|
||||||
|
err := p.resize(img, outputWidth, outputHeight, transform.FitMode)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to resize: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return p.encodeResult(ctx, img, outputFormat, transform.Quality,
|
||||||
|
origWidth, origHeight)
|
||||||
|
}
|
||||||
|
|
||||||
|
// encodeResult encodes img as format at quality and returns it as the result
|
||||||
|
// for a source of origWidth x origHeight.
|
||||||
|
func (p *ImageProcessor) encodeResult(
|
||||||
|
ctx context.Context, img *vips.ImageRef, format imagerequest.Format,
|
||||||
|
quality, origWidth, origHeight int,
|
||||||
|
) (*Result, error) {
|
||||||
|
output, err := p.encode(ctx, img, format, quality)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to encode: %w", err)
|
return nil, fmt.Errorf("failed to encode: %w", err)
|
||||||
}
|
}
|
||||||
@@ -251,7 +371,7 @@ func (p *ImageProcessor) Process(
|
|||||||
return &Result{
|
return &Result{
|
||||||
Content: io.NopCloser(bytes.NewReader(output)),
|
Content: io.NopCloser(bytes.NewReader(output)),
|
||||||
ContentLength: int64(len(output)),
|
ContentLength: int64(len(output)),
|
||||||
ContentType: outputFormat.MIMEType(),
|
ContentType: format.MIMEType(),
|
||||||
Width: img.Width(),
|
Width: img.Width(),
|
||||||
Height: img.Height(),
|
Height: img.Height(),
|
||||||
InputWidth: origWidth,
|
InputWidth: origWidth,
|
||||||
@@ -290,29 +410,125 @@ func targetDimensions(
|
|||||||
return width, height, nil
|
return width, height, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// WaitForProcessing waits until no image is being processed, or until ctx
|
// resolveOutputFormat returns the format of the output for the requested
|
||||||
// ends, and returns how many images were still being processed then. It
|
// format: sourceFormat for orig, and format itself for any other. encode
|
||||||
// waits by taking each slot in processingSemaphore as it frees up until it
|
// refuses an empty format.
|
||||||
// holds them all, or until ctx ends, then gives back the slots it took.
|
func resolveOutputFormat(
|
||||||
func (p *ImageProcessor) WaitForProcessing(ctx context.Context) int {
|
format, sourceFormat imagerequest.Format,
|
||||||
taken := 0
|
) imagerequest.Format {
|
||||||
|
if format == imagerequest.FormatOriginal {
|
||||||
defer func() {
|
return sourceFormat
|
||||||
for range taken {
|
|
||||||
<-p.processingSemaphore
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
for taken < cap(p.processingSemaphore) {
|
|
||||||
select {
|
|
||||||
case p.processingSemaphore <- struct{}{}:
|
|
||||||
taken++
|
|
||||||
case <-ctx.Done():
|
|
||||||
return len(p.processingSemaphore) - taken
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return 0
|
return format
|
||||||
|
}
|
||||||
|
|
||||||
|
// loadImage refuses data that libvips would not load as sourceFormat, then
|
||||||
|
// decodes it and turns the image upright: encode strips the EXIF orientation
|
||||||
|
// tag, and sizes must be worked out on the upright image. An image with a
|
||||||
|
// side above MaxInputDimension is ErrInputTooLarge, which guards against
|
||||||
|
// decompression bombs.
|
||||||
|
func loadImage(
|
||||||
|
data []byte, sourceFormat imagerequest.Format,
|
||||||
|
) (*vips.ImageRef, error) {
|
||||||
|
err := checkInputFormat(data, sourceFormat)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
img, err := vips.NewImageFromBuffer(data)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to decode image: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = img.AutoRotate()
|
||||||
|
if err != nil {
|
||||||
|
img.Close()
|
||||||
|
|
||||||
|
return nil, fmt.Errorf("failed to auto-rotate: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if img.Width() > MaxInputDimension || img.Height() > MaxInputDimension {
|
||||||
|
img.Close()
|
||||||
|
|
||||||
|
return nil, ErrInputTooLarge
|
||||||
|
}
|
||||||
|
|
||||||
|
return img, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// svgAsItIs returns the SVG data unchanged, which is orig of an SVG source
|
||||||
|
// at the orig size. width x height is the SVG's own size. Any other size is
|
||||||
|
// ErrSVGNotOriginalSize.
|
||||||
|
func svgAsItIs(
|
||||||
|
data []byte, size imagerequest.Size, width, height int,
|
||||||
|
) (*Result, error) {
|
||||||
|
if !size.OriginalSize() {
|
||||||
|
return nil, fmt.Errorf("%w: asked for %dx%d", ErrSVGNotOriginalSize,
|
||||||
|
size.Width, size.Height)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &Result{
|
||||||
|
Content: io.NopCloser(bytes.NewReader(data)),
|
||||||
|
ContentLength: int64(len(data)),
|
||||||
|
ContentType: imagerequest.FormatSVG.MIMEType(),
|
||||||
|
Width: width,
|
||||||
|
Height: height,
|
||||||
|
InputWidth: width,
|
||||||
|
InputHeight: height,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkSVGOrigSize returns ErrSVGTooLargeForOrigSize when size is the orig
|
||||||
|
// size and the SVG source, of width x height, has a side above
|
||||||
|
// MaxOutputDimension. processSVG calls it after returning orig of the SVG, so
|
||||||
|
// the SVG would be rendered at its own size.
|
||||||
|
func checkSVGOrigSize(size imagerequest.Size, width, height int) error {
|
||||||
|
if size.OriginalSize() && max(width, height) > MaxOutputDimension {
|
||||||
|
return fmt.Errorf("%w: %dx%d is above %d", ErrSVGTooLargeForOrigSize,
|
||||||
|
width, height, MaxOutputDimension)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// svgRenderSize returns the least size in the proportions of width x height,
|
||||||
|
// an SVG's own size, that covers outputWidth x outputHeight, so that resize
|
||||||
|
// hardly changes it. Neither side is above MaxInputDimension.
|
||||||
|
func svgRenderSize(width, height, outputWidth, outputHeight int) (int, int) {
|
||||||
|
scale := min(
|
||||||
|
max(float64(outputWidth)/float64(width),
|
||||||
|
float64(outputHeight)/float64(height)),
|
||||||
|
float64(MaxInputDimension)/float64(max(width, height)))
|
||||||
|
|
||||||
|
return int(math.Round(float64(width) * scale)),
|
||||||
|
int(math.Round(float64(height) * scale))
|
||||||
|
}
|
||||||
|
|
||||||
|
// renderSVG renders the SVG data, whose own size is width x height, at
|
||||||
|
// svgRenderSize for outputWidth x outputHeight, in a child process that pixa
|
||||||
|
// kills at deadline (see runSVGChildProcess). It returns the rendered image,
|
||||||
|
// read from the child's PNG. An SVG with one side more than
|
||||||
|
// MaxInputDimension times the other is ErrSVGTooThin.
|
||||||
|
func (p *ImageProcessor) renderSVG(
|
||||||
|
ctx context.Context, deadline time.Time, data []byte,
|
||||||
|
width, height, outputWidth, outputHeight int,
|
||||||
|
) (*vips.ImageRef, error) {
|
||||||
|
if max(width, height) > MaxInputDimension*min(width, height) {
|
||||||
|
return nil, fmt.Errorf("%w: %dx%d", ErrSVGTooThin, width, height)
|
||||||
|
}
|
||||||
|
|
||||||
|
renderWidth, renderHeight := svgRenderSize(width, height,
|
||||||
|
outputWidth, outputHeight)
|
||||||
|
|
||||||
|
png, err := p.runSVGChildProcess(ctx, deadline, data,
|
||||||
|
renderSVGAsPNGArgument, strconv.Itoa(renderWidth),
|
||||||
|
strconv.Itoa(renderHeight))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return loadImage(png, imagerequest.FormatPNG)
|
||||||
}
|
}
|
||||||
|
|
||||||
// acquireSlot takes a slot in processingSemaphore, waiting at most
|
// acquireSlot takes a slot in processingSemaphore, waiting at most
|
||||||
@@ -397,6 +613,8 @@ func vipsImageType(format imagerequest.Format) vips.ImageType {
|
|||||||
return vips.ImageTypeAVIF
|
return vips.ImageTypeAVIF
|
||||||
case imagerequest.FormatJXL:
|
case imagerequest.FormatJXL:
|
||||||
return vips.ImageTypeJXL
|
return vips.ImageTypeJXL
|
||||||
|
case imagerequest.FormatSVG:
|
||||||
|
return vips.ImageTypeSVG
|
||||||
case imagerequest.FormatOriginal, imagerequest.FormatAuto:
|
case imagerequest.FormatOriginal, imagerequest.FormatAuto:
|
||||||
return vips.ImageTypeUnknown
|
return vips.ImageTypeUnknown
|
||||||
default:
|
default:
|
||||||
@@ -575,7 +793,9 @@ func (p *ImageProcessor) encode(
|
|||||||
case imagerequest.FormatJXL:
|
case imagerequest.FormatJXL:
|
||||||
return exportJXL(img, quality)
|
return exportJXL(img, quality)
|
||||||
|
|
||||||
case imagerequest.FormatOriginal, imagerequest.FormatAuto:
|
// libvips cannot save SVG; Process returns an SVG source as it is
|
||||||
|
case imagerequest.FormatSVG, imagerequest.FormatOriginal,
|
||||||
|
imagerequest.FormatAuto:
|
||||||
return nil, fmt.Errorf("%w: %s", ErrUnsupportedOutputFormat, format)
|
return nil, fmt.Errorf("%w: %s", ErrUnsupportedOutputFormat, format)
|
||||||
|
|
||||||
default:
|
default:
|
||||||
|
|||||||
@@ -19,6 +19,11 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func TestMain(m *testing.M) {
|
func TestMain(m *testing.M) {
|
||||||
|
// The SVG tests start this test binary again as SVG child processes
|
||||||
|
if IsSVGChildProcess(os.Args[1:]) {
|
||||||
|
os.Exit(RunAsSVGChildProcess(os.Args[1:]))
|
||||||
|
}
|
||||||
|
|
||||||
initVips()
|
initVips()
|
||||||
|
|
||||||
code := m.Run()
|
code := m.Run()
|
||||||
|
|||||||
@@ -51,7 +51,7 @@ func TestImageProcessor_LoadsEachSourceFormat(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TestImageProcessor_NotASourceFormatRefused verifies that a source format
|
// TestImageProcessor_NotASourceFormatRefused verifies that a source format
|
||||||
// other than the six, such as orig, auto or none, is refused with
|
// other than the seven, such as orig, auto or none, is refused with
|
||||||
// ErrUnsupportedInputFormat. The input is text, which libvips reads as no
|
// ErrUnsupportedInputFormat. The input is text, which libvips reads as no
|
||||||
// format.
|
// format.
|
||||||
func TestImageProcessor_NotASourceFormatRefused(t *testing.T) {
|
func TestImageProcessor_NotASourceFormatRefused(t *testing.T) {
|
||||||
|
|||||||
@@ -0,0 +1,255 @@
|
|||||||
|
package imageprocessor
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"strconv"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/davidbyttow/govips/v2/vips"
|
||||||
|
)
|
||||||
|
|
||||||
|
// An SVG child process is the running program started again with one of
|
||||||
|
// these arguments first, naming what it does. Its command line, after the
|
||||||
|
// program's name, is one of:
|
||||||
|
//
|
||||||
|
// read-svg-size MEMORY_LIMIT
|
||||||
|
// render-svg-as-png MEMORY_LIMIT WIDTH HEIGHT
|
||||||
|
//
|
||||||
|
// MEMORY_LIMIT is in bytes. The child reads the SVG on standard input. It
|
||||||
|
// writes the SVG's size as WIDTH HEIGHT, or the SVG rendered at WIDTH x
|
||||||
|
// HEIGHT as PNG, to standard output.
|
||||||
|
const (
|
||||||
|
readSVGSizeArgument = "read-svg-size"
|
||||||
|
renderSVGAsPNGArgument = "render-svg-as-png"
|
||||||
|
)
|
||||||
|
|
||||||
|
// errSVGChildProcessArguments is returned for a command line that is not
|
||||||
|
// that of an SVG child process.
|
||||||
|
var errSVGChildProcessArguments = errors.New(
|
||||||
|
"not the command line of an SVG child process")
|
||||||
|
|
||||||
|
// IsSVGChildProcess reports whether args, a process's command line after the
|
||||||
|
// program's name, are those of an SVG child process.
|
||||||
|
func IsSVGChildProcess(args []string) bool {
|
||||||
|
return len(args) > 0 &&
|
||||||
|
(args[0] == readSVGSizeArgument || args[0] == renderSVGAsPNGArgument)
|
||||||
|
}
|
||||||
|
|
||||||
|
// RunAsSVGChildProcess runs this process as the SVG child process that args,
|
||||||
|
// its command line after the program's name, ask for. It returns the status
|
||||||
|
// to exit with: 0 once it has written its result, and 1 after writing an
|
||||||
|
// error to standard error, which the parent discards.
|
||||||
|
func RunAsSVGChildProcess(args []string) int {
|
||||||
|
err := runAsSVGChildProcess(args, os.Stdin, os.Stdout)
|
||||||
|
if err != nil {
|
||||||
|
_, _ = fmt.Fprintln(os.Stderr, err)
|
||||||
|
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// runAsSVGChildProcess does the work of RunAsSVGChildProcess, reading the SVG
|
||||||
|
// from stdin and writing the result to stdout. It sets the process's limits
|
||||||
|
// before it reads the SVG.
|
||||||
|
func runAsSVGChildProcess(
|
||||||
|
args []string, stdin io.Reader, stdout io.Writer,
|
||||||
|
) error {
|
||||||
|
numbers, err := parseSVGChildProcessArguments(args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = setSVGChildProcessLimits(numbers[0])
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
initVips()
|
||||||
|
|
||||||
|
svg, err := io.ReadAll(stdin)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to read the SVG: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if args[0] == readSVGSizeArgument {
|
||||||
|
return writeSVGSize(svg, stdout)
|
||||||
|
}
|
||||||
|
|
||||||
|
return writeSVGAsPNG(svg, numbers[1], numbers[2], stdout)
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseSVGChildProcessArguments returns the numbers on the command line of an
|
||||||
|
// SVG child process, args: the memory limit, then the width and height for
|
||||||
|
// render-svg-as-png.
|
||||||
|
func parseSVGChildProcessArguments(args []string) ([]int, error) {
|
||||||
|
wantArgs := 2
|
||||||
|
if len(args) > 0 && args[0] == renderSVGAsPNGArgument {
|
||||||
|
wantArgs = 4
|
||||||
|
}
|
||||||
|
|
||||||
|
if !IsSVGChildProcess(args) || len(args) != wantArgs {
|
||||||
|
return nil, fmt.Errorf("%w: %q", errSVGChildProcessArguments, args)
|
||||||
|
}
|
||||||
|
|
||||||
|
numbers := make([]int, 0, len(args)-1)
|
||||||
|
|
||||||
|
for _, arg := range args[1:] {
|
||||||
|
number, err := strconv.Atoi(arg)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %q: %w", errSVGChildProcessArguments,
|
||||||
|
args, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
numbers = append(numbers, number)
|
||||||
|
}
|
||||||
|
|
||||||
|
return numbers, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// setSVGChildProcessLimits sets this process's RLIMIT_DATA to memoryLimit
|
||||||
|
// bytes, so that allocating heap or other private writable memory past it
|
||||||
|
// fails. RLIMIT_AS would also count the address space the Go runtime reserves
|
||||||
|
// but does not use. It also sets RLIMIT_CORE to 0: librsvg crashes when an
|
||||||
|
// allocation fails, and the crash would write a core file of up to
|
||||||
|
// memoryLimit bytes into pixad's working directory.
|
||||||
|
func setSVGChildProcessLimits(memoryLimit int) error {
|
||||||
|
if memoryLimit < 1 {
|
||||||
|
return fmt.Errorf("%w: memory limit %d", errSVGChildProcessArguments,
|
||||||
|
memoryLimit)
|
||||||
|
}
|
||||||
|
|
||||||
|
err := syscall.Setrlimit(syscall.RLIMIT_DATA, &syscall.Rlimit{
|
||||||
|
Cur: uint64(memoryLimit),
|
||||||
|
Max: uint64(memoryLimit),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to limit memory to %d bytes: %w",
|
||||||
|
memoryLimit, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = syscall.Setrlimit(syscall.RLIMIT_CORE, &syscall.Rlimit{})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to turn off core files: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeSVGSize writes the size of svg to stdout as WIDTH HEIGHT. librsvg
|
||||||
|
// reads the whole SVG for it.
|
||||||
|
func writeSVGSize(svg []byte, stdout io.Writer) error {
|
||||||
|
img, err := vips.NewImageFromBuffer(svg)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to read the SVG: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
defer img.Close()
|
||||||
|
|
||||||
|
_, err = fmt.Fprintf(stdout, "%d %d\n", img.Width(), img.Height())
|
||||||
|
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeSVGAsPNG renders svg at width x height and writes it to stdout as PNG.
|
||||||
|
// libvips' thumbnail has librsvg render the SVG at that size directly, at
|
||||||
|
// any scale: govips takes a density only in whole dots per inch, which cannot
|
||||||
|
// render an SVG below 1/72 of its own size. Given the SVG as bytes with no
|
||||||
|
// file name, librsvg loads no file or URL the SVG refers to.
|
||||||
|
func writeSVGAsPNG(svg []byte, width, height int, stdout io.Writer) error {
|
||||||
|
img, err := vips.NewThumbnailWithSizeFromBuffer(svg, width, height,
|
||||||
|
vips.InterestingNone, vips.SizeBoth)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to render the SVG: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
defer img.Close()
|
||||||
|
|
||||||
|
png, err := exportPNG(img)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to save the SVG as PNG: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = stdout.Write(png)
|
||||||
|
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// readSVGSize reads the size of the SVG data in a child process that pixa
|
||||||
|
// kills at deadline (see runSVGChildProcess).
|
||||||
|
func (p *ImageProcessor) readSVGSize(
|
||||||
|
ctx context.Context, deadline time.Time, data []byte,
|
||||||
|
) (int, int, error) {
|
||||||
|
output, err := p.runSVGChildProcess(ctx, deadline, data,
|
||||||
|
readSVGSizeArgument)
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
var width, height int
|
||||||
|
|
||||||
|
_, err = fmt.Sscan(string(output), &width, &height)
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, fmt.Errorf("failed to read the size %q an SVG child "+
|
||||||
|
"process wrote: %w", output, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return width, height, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// runSVGChildProcess starts the running program again as an SVG child
|
||||||
|
// process, with argument, p.maxSVGRenderMemory and extraArgs as its command
|
||||||
|
// line, and waits for it. It gives the child data on standard input,
|
||||||
|
// discards its standard error, and returns its standard output. pixa kills
|
||||||
|
// the child at deadline. A child that fails or is killed is
|
||||||
|
// ErrSVGRenderFailed, unless ctx has ended: runSVGChildProcess then returns
|
||||||
|
// ctx's error. Failing to start the child is another error.
|
||||||
|
func (p *ImageProcessor) runSVGChildProcess(
|
||||||
|
ctx context.Context, deadline time.Time, data []byte,
|
||||||
|
argument string, extraArgs ...string,
|
||||||
|
) ([]byte, error) {
|
||||||
|
executable, err := os.Executable()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to find the running program: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
childCtx, cancel := context.WithDeadline(ctx, deadline)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
args := append([]string{argument, strconv.Itoa(p.maxSVGRenderMemory)},
|
||||||
|
extraArgs...)
|
||||||
|
|
||||||
|
//nolint:gosec // the running program, with arguments pixa makes
|
||||||
|
cmd := exec.CommandContext(childCtx, executable, args...)
|
||||||
|
cmd.Stdin = bytes.NewReader(data)
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
|
||||||
|
cmd.Stdout = &stdout
|
||||||
|
|
||||||
|
err = cmd.Run()
|
||||||
|
|
||||||
|
var exitError *exec.ExitError
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case err == nil:
|
||||||
|
return stdout.Bytes(), nil
|
||||||
|
case ctx.Err() != nil:
|
||||||
|
return nil, ctx.Err()
|
||||||
|
case childCtx.Err() != nil:
|
||||||
|
return nil, fmt.Errorf("%w: it took more than %v", ErrSVGRenderFailed,
|
||||||
|
p.svgRenderTimeout)
|
||||||
|
case errors.As(err, &exitError):
|
||||||
|
return nil, fmt.Errorf("%w: %w", ErrSVGRenderFailed, err)
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("failed to run an SVG child process: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,503 @@
|
|||||||
|
package imageprocessor
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/davidbyttow/govips/v2/vips"
|
||||||
|
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||||
|
)
|
||||||
|
|
||||||
|
// testSVG is a 16x12 SVG, black on its left half and white on its right.
|
||||||
|
const testSVG = `<svg xmlns="http://www.w3.org/2000/svg" width="16" height="12">` +
|
||||||
|
`<rect width="16" height="12" fill="white"/>` +
|
||||||
|
`<rect width="8" height="12" fill="black"/></svg>`
|
||||||
|
|
||||||
|
// newSVGTestProcessor returns a processor for the SVG tests.
|
||||||
|
func newSVGTestProcessor() *ImageProcessor {
|
||||||
|
return New(Params{MaxInputBytes: 1 << 20, MaxConcurrentProcessing: 1})
|
||||||
|
}
|
||||||
|
|
||||||
|
// processSVG processes svg, as an SVG source, with transform.
|
||||||
|
func processSVG(
|
||||||
|
t *testing.T, svg string, transform imagerequest.Transform,
|
||||||
|
) (*Result, []byte, error) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
result, err := newSVGTestProcessor().Process(t.Context(),
|
||||||
|
bytes.NewReader([]byte(svg)), imagerequest.FormatSVG, &transform)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
output, err := io.ReadAll(result.Content)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("reading the output: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return result, output, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestImageProcessor_SVGOrigIsTheSVG checks that orig of an SVG at the orig
|
||||||
|
// size is the SVG itself, never another format, and that orig at another
|
||||||
|
// size is ErrSVGNotOriginalSize.
|
||||||
|
func TestImageProcessor_SVGOrigIsTheSVG(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
result, output, err := processSVG(t, testSVG,
|
||||||
|
imagerequest.Transform{Format: imagerequest.FormatOriginal})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("orig of the SVG: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("orig of the SVG: %s, %dx%d", result.ContentType, result.Width,
|
||||||
|
result.Height)
|
||||||
|
|
||||||
|
if result.ContentType != "image/svg+xml" || string(output) != testSVG {
|
||||||
|
t.Errorf("orig of the SVG = %s %q, want image/svg+xml %q",
|
||||||
|
result.ContentType, output, testSVG)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, _, err = processSVG(t, testSVG, imagerequest.Transform{
|
||||||
|
Size: imagerequest.Size{Width: 32, Height: 24},
|
||||||
|
Format: imagerequest.FormatOriginal,
|
||||||
|
})
|
||||||
|
if !errors.Is(err, ErrSVGNotOriginalSize) {
|
||||||
|
t.Errorf("orig of the SVG at 32x24: error = %v, want %v", err,
|
||||||
|
ErrSVGNotOriginalSize)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestImageProcessor_RendersSVGToEachRasterFormat checks that an SVG is
|
||||||
|
// rendered to each raster format at the requested size.
|
||||||
|
func TestImageProcessor_RendersSVGToEachRasterFormat(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
for _, format := range []imagerequest.Format{
|
||||||
|
imagerequest.FormatJPEG, imagerequest.FormatPNG, imagerequest.FormatGIF,
|
||||||
|
imagerequest.FormatWebP, imagerequest.FormatAVIF, imagerequest.FormatJXL,
|
||||||
|
} {
|
||||||
|
result, output, err := processSVG(t, testSVG, imagerequest.Transform{
|
||||||
|
Size: imagerequest.Size{Width: 40, Height: 30},
|
||||||
|
Format: format,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("SVG as %s: %v", format, err)
|
||||||
|
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("SVG as %s: %s, %dx%d, %d bytes", format, result.ContentType,
|
||||||
|
result.Width, result.Height, len(output))
|
||||||
|
|
||||||
|
if result.ContentType != format.MIMEType() ||
|
||||||
|
vips.DetermineImageType(output) != vipsImageType(format) ||
|
||||||
|
result.Width != 40 || result.Height != 30 {
|
||||||
|
t.Errorf("SVG as %s = %s, %dx%d; want %s, 40x30", format,
|
||||||
|
result.ContentType, result.Width, result.Height, format.MIMEType())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestImageProcessor_RendersSVGSharpWhenEnlarged checks that a 16x12 SVG
|
||||||
|
// asked for at 1600x1200 is rendered at that size, not rendered at 16x12 and
|
||||||
|
// enlarged: 10 pixels either side of the edge between its black and white
|
||||||
|
// halves are black and white, where an enlarged image is grey there.
|
||||||
|
func TestImageProcessor_RendersSVGSharpWhenEnlarged(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
_, output, err := processSVG(t, testSVG, imagerequest.Transform{
|
||||||
|
Size: imagerequest.Size{Width: 1600, Height: 1200},
|
||||||
|
Format: imagerequest.FormatPNG,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("SVG as PNG at 1600x1200: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
img, err := vips.NewImageFromBuffer(output)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("decoding the PNG: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Cleanup(img.Close)
|
||||||
|
|
||||||
|
for x, want := range map[int]float64{790: 0, 810: 255} {
|
||||||
|
point, err := img.GetPoint(x, 600)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetPoint(%d, 600): %v", x, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("pixel at %d,600: %v", x, point)
|
||||||
|
|
||||||
|
if point[0] != want {
|
||||||
|
t.Errorf("pixel at %d,600 = %v, want %v", x, point[0], want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestImageProcessor_SVGFetchesNothing checks that rendering an SVG draws no
|
||||||
|
// image it refers to: one at a URL on a local server, by href or xlink:href,
|
||||||
|
// or a local file that exists. Each SVG renders without error, white where
|
||||||
|
// the image would be. An SVG that uses an external entity at that server is
|
||||||
|
// a separate case. No SVG makes a request to the server.
|
||||||
|
func TestImageProcessor_SVGFetchesNothing(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var requests atomic.Int64
|
||||||
|
|
||||||
|
// Blue is 128 in every pixel of this PNG.
|
||||||
|
png := createTestPNG(t, 4, 4)
|
||||||
|
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(
|
||||||
|
func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
requests.Add(1)
|
||||||
|
t.Logf("request for %s", r.URL)
|
||||||
|
w.Header().Set("Content-Type", "image/png")
|
||||||
|
_, _ = w.Write(png)
|
||||||
|
}))
|
||||||
|
t.Cleanup(server.Close)
|
||||||
|
|
||||||
|
pngPath := filepath.Join(t.TempDir(), "image.png")
|
||||||
|
|
||||||
|
err := os.WriteFile(pngPath, png, 0o600)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("writing the PNG file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, reference := range []string{
|
||||||
|
`href="` + server.URL + `/image.png"`,
|
||||||
|
`xlink:href="` + server.URL + `/xlink.png"`,
|
||||||
|
`href="file://` + pngPath + `"`,
|
||||||
|
} {
|
||||||
|
svg := `<svg xmlns="http://www.w3.org/2000/svg" ` +
|
||||||
|
`xmlns:xlink="http://www.w3.org/1999/xlink" width="16" height="12">` +
|
||||||
|
`<rect width="16" height="12" fill="white"/>` +
|
||||||
|
`<image ` + reference + ` width="16" height="12"/></svg>`
|
||||||
|
|
||||||
|
_, output, err := processSVG(t, svg, imagerequest.Transform{
|
||||||
|
Format: imagerequest.FormatPNG,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("SVG with image %s as PNG: %v", reference, err)
|
||||||
|
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
img, err := vips.NewImageFromBuffer(output)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("decoding the PNG: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
point, err := img.GetPoint(8, 6)
|
||||||
|
img.Close()
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetPoint(8, 6): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("SVG with image %s: pixel at 8,6 = %v", reference, point)
|
||||||
|
|
||||||
|
if point[2] != 255 {
|
||||||
|
t.Errorf("SVG with image %s: blue at 8,6 = %v, want 255: the "+
|
||||||
|
"image is drawn", reference, point[2])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
svg := `<?xml version="1.0"?>` +
|
||||||
|
`<!DOCTYPE svg [<!ENTITY outside SYSTEM "` + server.URL + `/entity">]>` +
|
||||||
|
`<svg xmlns="http://www.w3.org/2000/svg" width="16" height="12">` +
|
||||||
|
`<text y="10">&outside;</text></svg>`
|
||||||
|
|
||||||
|
// librsvg loads no external entity, so it refuses this SVG: outside is
|
||||||
|
// not defined.
|
||||||
|
_, _, err = processSVG(t, svg, imagerequest.Transform{
|
||||||
|
Format: imagerequest.FormatPNG,
|
||||||
|
})
|
||||||
|
t.Logf("SVG with an external entity as PNG: error = %v", err)
|
||||||
|
|
||||||
|
if requests.Load() != 0 {
|
||||||
|
t.Errorf("rendering the SVGs made %d requests, want none",
|
||||||
|
requests.Load())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestImageProcessor_RendersSVGLargerThanMaxInputDimension checks that an
|
||||||
|
// SVG whose own size is above MaxInputDimension is limited only by the
|
||||||
|
// output checks: it is rendered at a raster size, orig of it is the SVG
|
||||||
|
// itself, and a raster format at its own size is ErrSVGTooLargeForOrigSize.
|
||||||
|
// The 1000000x1000000 one is rendered at a ten-thousandth of its own size.
|
||||||
|
func TestImageProcessor_RendersSVGLargerThanMaxInputDimension(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
for _, svg := range []string{
|
||||||
|
`<svg xmlns="http://www.w3.org/2000/svg" width="10000" height="10">` +
|
||||||
|
`<rect width="10000" height="10"/></svg>`,
|
||||||
|
`<svg xmlns="http://www.w3.org/2000/svg" width="1000000" ` +
|
||||||
|
`height="1000000"><rect width="1000000" height="1000000"/></svg>`,
|
||||||
|
} {
|
||||||
|
result, _, err := processSVG(t, svg, imagerequest.Transform{
|
||||||
|
Size: imagerequest.Size{Width: 100, Height: 100},
|
||||||
|
Format: imagerequest.FormatPNG,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("%s as PNG at 100x100: %v", svg, err)
|
||||||
|
} else {
|
||||||
|
t.Logf("%s as PNG at 100x100: %dx%d", svg, result.Width, result.Height)
|
||||||
|
|
||||||
|
if result.Width != 100 || result.Height != 100 {
|
||||||
|
t.Errorf("%s as PNG at 100x100 = %dx%d, want 100x100", svg,
|
||||||
|
result.Width, result.Height)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
result, output, err := processSVG(t, svg,
|
||||||
|
imagerequest.Transform{Format: imagerequest.FormatOriginal})
|
||||||
|
if err != nil || result.ContentType != imagerequest.FormatSVG.MIMEType() ||
|
||||||
|
string(output) != svg {
|
||||||
|
t.Errorf("orig of %s: error = %v, want the SVG itself", svg, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, _, err = processSVG(t, svg,
|
||||||
|
imagerequest.Transform{Format: imagerequest.FormatPNG})
|
||||||
|
if !errors.Is(err, ErrSVGTooLargeForOrigSize) {
|
||||||
|
t.Errorf("%s as PNG at its own size: error = %v, want %v", svg,
|
||||||
|
err, ErrSVGTooLargeForOrigSize)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestImageProcessor_RefusesSVGTooThin checks that an SVG with one side more
|
||||||
|
// than MaxInputDimension times the other is ErrSVGTooThin at a raster size,
|
||||||
|
// that one at exactly that proportion is rendered, and that orig of either
|
||||||
|
// is the SVG itself.
|
||||||
|
func TestImageProcessor_RefusesSVGTooThin(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
tooThin := `<svg xmlns="http://www.w3.org/2000/svg" width="81930" ` +
|
||||||
|
`height="10"><rect width="81930" height="10"/></svg>`
|
||||||
|
atTheLimit := `<svg xmlns="http://www.w3.org/2000/svg" width="81920" ` +
|
||||||
|
`height="10"><rect width="81920" height="10"/></svg>`
|
||||||
|
size := imagerequest.Size{Width: 100, Height: 100}
|
||||||
|
|
||||||
|
_, _, err := processSVG(t, tooThin,
|
||||||
|
imagerequest.Transform{Size: size, Format: imagerequest.FormatPNG})
|
||||||
|
if !errors.Is(err, ErrSVGTooThin) {
|
||||||
|
t.Errorf("81930x10 SVG as PNG at 100x100: error = %v, want %v", err,
|
||||||
|
ErrSVGTooThin)
|
||||||
|
}
|
||||||
|
|
||||||
|
result, _, err := processSVG(t, atTheLimit,
|
||||||
|
imagerequest.Transform{Size: size, Format: imagerequest.FormatPNG})
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("81920x10 SVG as PNG at 100x100: %v", err)
|
||||||
|
} else if result.Width != 100 || result.Height != 100 {
|
||||||
|
t.Errorf("81920x10 SVG as PNG at 100x100 = %dx%d, want 100x100",
|
||||||
|
result.Width, result.Height)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, svg := range []string{tooThin, atTheLimit} {
|
||||||
|
result, output, err := processSVG(t, svg,
|
||||||
|
imagerequest.Transform{Format: imagerequest.FormatOriginal})
|
||||||
|
if err != nil || result.ContentType != imagerequest.FormatSVG.MIMEType() ||
|
||||||
|
string(output) != svg {
|
||||||
|
t.Errorf("orig of %s: error = %v, want the SVG itself", svg, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// testSVGRenderTimeout is the time limit of the SVG time limit tests, short
|
||||||
|
// so that they finish in a few seconds. Under the race detector a child
|
||||||
|
// process takes about a second to start and read an SVG's size.
|
||||||
|
const testSVGRenderTimeout = 3 * time.Second
|
||||||
|
|
||||||
|
// processSVGTimed has processor process svg, as an SVG source, with transform,
|
||||||
|
// and returns the error and how long it took.
|
||||||
|
func processSVGTimed(
|
||||||
|
t *testing.T, processor *ImageProcessor, svg string,
|
||||||
|
transform imagerequest.Transform,
|
||||||
|
) (time.Duration, error) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
start := time.Now()
|
||||||
|
_, err := processor.Process(t.Context(), strings.NewReader(svg),
|
||||||
|
imagerequest.FormatSVG, &transform)
|
||||||
|
|
||||||
|
return time.Since(start), err
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkProcessesAnotherImage checks that processor, after refusing an SVG,
|
||||||
|
// processes a PNG.
|
||||||
|
func checkProcessesAnotherImage(t *testing.T, processor *ImageProcessor) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
_, err := processor.Process(t.Context(),
|
||||||
|
bytes.NewReader(createTestPNG(t, 64, 48)), imagerequest.FormatPNG,
|
||||||
|
&imagerequest.Transform{
|
||||||
|
Size: imagerequest.Size{Width: 32, Height: 24},
|
||||||
|
Format: imagerequest.FormatJPEG,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("a PNG after the SVG: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestImageProcessor_StopsSlowSVGRender checks that an SVG that takes about
|
||||||
|
// 12 seconds to render, 600 shapes each drawn with an feTurbulence filter
|
||||||
|
// effect, is ErrSVGRenderFailed soon after a time limit of
|
||||||
|
// testSVGRenderTimeout, not when the render would end. The processor then
|
||||||
|
// processes another image.
|
||||||
|
func TestImageProcessor_StopsSlowSVGRender(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
svg := `<svg xmlns="http://www.w3.org/2000/svg" width="256" height="256">` +
|
||||||
|
`<filter id="noise">` +
|
||||||
|
`<feTurbulence baseFrequency="0.05" numOctaves="4"/></filter>` +
|
||||||
|
strings.Repeat(`<rect width="256" height="256" filter="url(#noise)"/>`,
|
||||||
|
600) +
|
||||||
|
`</svg>`
|
||||||
|
|
||||||
|
processor := newSVGTestProcessor()
|
||||||
|
processor.svgRenderTimeout = testSVGRenderTimeout
|
||||||
|
|
||||||
|
elapsed, err := processSVGTimed(t, processor, svg, imagerequest.Transform{
|
||||||
|
Size: imagerequest.Size{Width: 256, Height: 256},
|
||||||
|
Format: imagerequest.FormatPNG,
|
||||||
|
})
|
||||||
|
t.Logf("slow SVG as PNG: error %v after %v", err, elapsed)
|
||||||
|
|
||||||
|
if !errors.Is(err, ErrSVGRenderFailed) {
|
||||||
|
t.Errorf("slow SVG as PNG: error = %v, want %v", err,
|
||||||
|
ErrSVGRenderFailed)
|
||||||
|
}
|
||||||
|
|
||||||
|
if elapsed > testSVGRenderTimeout+2*time.Second {
|
||||||
|
t.Errorf("slow SVG as PNG: refused after %v, want soon after %v",
|
||||||
|
elapsed, testSVGRenderTimeout)
|
||||||
|
}
|
||||||
|
|
||||||
|
checkProcessesAnotherImage(t, processor)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestImageProcessor_StopsSlowSVGSizeRead checks that orig of an SVG whose
|
||||||
|
// size takes about 10 seconds to read is ErrSVGRenderFailed soon after a time
|
||||||
|
// limit of testSVGRenderTimeout. To read the size, librsvg matches each of the
|
||||||
|
// SVG's 2500 CSS rules against each of its 2500 shapes, walking up the 75
|
||||||
|
// groups around it. The processor then processes another image.
|
||||||
|
func TestImageProcessor_StopsSlowSVGSizeRead(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var rules strings.Builder
|
||||||
|
|
||||||
|
for i := range 2500 {
|
||||||
|
fmt.Fprintf(&rules, ".group%d rect { fill: red }", i)
|
||||||
|
}
|
||||||
|
|
||||||
|
svg := `<svg xmlns="http://www.w3.org/2000/svg" width="16" height="12">` +
|
||||||
|
`<style>` + rules.String() + `</style>` + strings.Repeat(`<g>`, 75) +
|
||||||
|
strings.Repeat(`<rect width="1" height="1"/>`, 2500) +
|
||||||
|
strings.Repeat(`</g>`, 75) + `</svg>`
|
||||||
|
|
||||||
|
processor := newSVGTestProcessor()
|
||||||
|
processor.svgRenderTimeout = testSVGRenderTimeout
|
||||||
|
|
||||||
|
elapsed, err := processSVGTimed(t, processor, svg,
|
||||||
|
imagerequest.Transform{Format: imagerequest.FormatOriginal})
|
||||||
|
t.Logf("orig of the SVG: error %v after %v", err, elapsed)
|
||||||
|
|
||||||
|
if !errors.Is(err, ErrSVGRenderFailed) {
|
||||||
|
t.Errorf("orig of the SVG: error = %v, want %v", err,
|
||||||
|
ErrSVGRenderFailed)
|
||||||
|
}
|
||||||
|
|
||||||
|
if elapsed > testSVGRenderTimeout+2*time.Second {
|
||||||
|
t.Errorf("orig of the SVG: refused after %v, want soon after %v",
|
||||||
|
elapsed, testSVGRenderTimeout)
|
||||||
|
}
|
||||||
|
|
||||||
|
checkProcessesAnotherImage(t, processor)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestImageProcessor_StopsSVGRenderOverMemoryLimit checks that an SVG that
|
||||||
|
// needs about 500 MiB more than a child process's own memory to render, a
|
||||||
|
// chain of 2000 blurs that each keep their result, is ErrSVGRenderFailed with
|
||||||
|
// a memory limit of 512 MiB, and that a plain SVG renders under the same
|
||||||
|
// limit. A child process needs about 130 MiB of its own, and about 350 MiB
|
||||||
|
// under the race detector, so the chain also renders under MaxSVGRenderMemory.
|
||||||
|
func TestImageProcessor_StopsSVGRenderOverMemoryLimit(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var blurs strings.Builder
|
||||||
|
|
||||||
|
blurs.WriteString(`<feGaussianBlur stdDeviation="3" result="blur0"/>`)
|
||||||
|
|
||||||
|
for i := 1; i < 2000; i++ {
|
||||||
|
fmt.Fprintf(&blurs,
|
||||||
|
`<feGaussianBlur in="blur%d" stdDeviation="3" result="blur%d"/>`,
|
||||||
|
i-1, i)
|
||||||
|
}
|
||||||
|
|
||||||
|
svg := `<svg xmlns="http://www.w3.org/2000/svg" width="256" height="256">` +
|
||||||
|
`<filter id="blurs" color-interpolation-filters="sRGB">` +
|
||||||
|
blurs.String() + `</filter>` +
|
||||||
|
`<rect width="256" height="256" fill="red" filter="url(#blurs)"/></svg>`
|
||||||
|
size := imagerequest.Size{Width: 256, Height: 256}
|
||||||
|
|
||||||
|
processor := newSVGTestProcessor()
|
||||||
|
processor.maxSVGRenderMemory = 512 << 20
|
||||||
|
|
||||||
|
elapsed, err := processSVGTimed(t, processor, svg,
|
||||||
|
imagerequest.Transform{Size: size, Format: imagerequest.FormatPNG})
|
||||||
|
t.Logf("SVG of 2000 blurs as PNG: error %v after %v", err, elapsed)
|
||||||
|
|
||||||
|
if !errors.Is(err, ErrSVGRenderFailed) {
|
||||||
|
t.Errorf("SVG of 2000 blurs as PNG: error = %v, want %v", err,
|
||||||
|
ErrSVGRenderFailed)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = processSVGTimed(t, processor, testSVG,
|
||||||
|
imagerequest.Transform{Size: size, Format: imagerequest.FormatPNG})
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("plain SVG as PNG after it: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSVGRenderSize checks the size at which an SVG is rendered to cover an
|
||||||
|
// output size, and that it is held to MaxInputDimension.
|
||||||
|
func TestSVGRenderSize(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
width, height int
|
||||||
|
outputWidth, outputHeight int
|
||||||
|
wantWidth, wantHeight int
|
||||||
|
}{
|
||||||
|
{"own size", 16, 12, 16, 12, 16, 12},
|
||||||
|
{"100 times larger", 16, 12, 1600, 1200, 1600, 1200},
|
||||||
|
{"wider than the SVG's proportions", 16, 12, 160, 12, 160, 120},
|
||||||
|
{"10 times smaller", 100, 100, 10, 10, 10, 10},
|
||||||
|
{"held to MaxInputDimension", 8192, 1, 8192, 8192, 8192, 1},
|
||||||
|
{"held to MaxInputDimension when enlarged", 1024, 1, 8192, 8192, 8192, 8},
|
||||||
|
{"far above MaxInputDimension", 1000000, 1000000, 100, 100, 100, 100},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
gotWidth, gotHeight := svgRenderSize(tt.width, tt.height,
|
||||||
|
tt.outputWidth, tt.outputHeight)
|
||||||
|
if gotWidth != tt.wantWidth || gotHeight != tt.wantHeight {
|
||||||
|
t.Errorf("%s: svgRenderSize(%d, %d, %d, %d) = %dx%d, want %dx%d",
|
||||||
|
tt.name, tt.width, tt.height, tt.outputWidth, tt.outputHeight,
|
||||||
|
gotWidth, gotHeight, tt.wantWidth, tt.wantHeight)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -37,11 +37,15 @@ type Proxy struct {
|
|||||||
// same time share one fetch and one transcode.
|
// same time share one fetch and one transcode.
|
||||||
variantsInProgress singleflight.Group
|
variantsInProgress singleflight.Group
|
||||||
|
|
||||||
// The counts of upstream fetches and transforms, made and registered by
|
// The metrics of upstream fetches and transforms, made and registered by
|
||||||
// New. They are kept in memory, so adding to one waits for nothing.
|
// New. They are kept in memory, so updating one waits for nothing.
|
||||||
upstreamFetches prometheus.Counter
|
upstreamFetches prometheus.Counter
|
||||||
upstreamFetchBytes prometheus.Counter
|
upstreamFetchBytes prometheus.Counter
|
||||||
transforms prometheus.Counter
|
upstreamFetchDurations prometheus.Histogram
|
||||||
|
upstreamFetchErrors prometheus.Counter
|
||||||
|
transforms prometheus.Counter
|
||||||
|
transformDurations prometheus.Histogram
|
||||||
|
transformsInProgress prometheus.Gauge
|
||||||
}
|
}
|
||||||
|
|
||||||
// Config holds the proxy's configuration.
|
// Config holds the proxy's configuration.
|
||||||
@@ -62,9 +66,9 @@ type Config struct {
|
|||||||
// errCacheRequired is returned when the proxy is built without a cache.
|
// errCacheRequired is returned when the proxy is built without a cache.
|
||||||
var errCacheRequired = errors.New("cache is required")
|
var errCacheRequired = errors.New("cache is required")
|
||||||
|
|
||||||
// New creates a new image proxy and registers its counters on
|
// New creates a new image proxy and registers its metrics on
|
||||||
// metricsRegistry. pixad passes Prometheus' default registry, the one
|
// metricsRegistry. pixad passes Prometheus' default registry, the one
|
||||||
// /metrics serves. It fails when metricsRegistry already holds counters of
|
// /metrics serves. It fails when metricsRegistry already holds metrics of
|
||||||
// the same names.
|
// the same names.
|
||||||
func New(
|
func New(
|
||||||
cfg *Config, metricsRegistry prometheus.Registerer,
|
cfg *Config, metricsRegistry prometheus.Registerer,
|
||||||
@@ -99,18 +103,39 @@ func New(
|
|||||||
Name: "pixa_upstream_fetch_bytes_total",
|
Name: "pixa_upstream_fetch_bytes_total",
|
||||||
Help: "Bytes read from upstream by fetches of source images.",
|
Help: "Bytes read from upstream by fetches of source images.",
|
||||||
}),
|
}),
|
||||||
|
upstreamFetchDurations: prometheus.NewHistogram(prometheus.HistogramOpts{
|
||||||
|
Name: "pixa_upstream_fetch_duration_seconds",
|
||||||
|
Help: "Duration of the fetches pixa_upstream_fetches_total counts, " +
|
||||||
|
"until the response's last byte.",
|
||||||
|
}),
|
||||||
|
upstreamFetchErrors: prometheus.NewCounter(prometheus.CounterOpts{
|
||||||
|
Name: "pixa_upstream_fetch_errors_total",
|
||||||
|
Help: "Fetches of a source image from upstream that failed, " +
|
||||||
|
"before or while reading the response.",
|
||||||
|
}),
|
||||||
transforms: prometheus.NewCounter(prometheus.CounterOpts{
|
transforms: prometheus.NewCounter(prometheus.CounterOpts{
|
||||||
Name: "pixa_transforms_total",
|
Name: "pixa_transforms_total",
|
||||||
Help: "Images the image processor transformed.",
|
Help: "Images the image processor transformed.",
|
||||||
}),
|
}),
|
||||||
|
transformDurations: prometheus.NewHistogram(prometheus.HistogramOpts{
|
||||||
|
Name: "pixa_transform_duration_seconds",
|
||||||
|
Help: "Duration of the transforms pixa_transforms_total counts, " +
|
||||||
|
"waiting for a processing slot included.",
|
||||||
|
}),
|
||||||
|
transformsInProgress: prometheus.NewGauge(prometheus.GaugeOpts{
|
||||||
|
Name: "pixa_transforms_in_progress",
|
||||||
|
Help: "Images being transformed or waiting for a processing slot.",
|
||||||
|
}),
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, counter := range []prometheus.Counter{
|
for _, metric := range []prometheus.Collector{
|
||||||
p.upstreamFetches, p.upstreamFetchBytes, p.transforms,
|
p.upstreamFetches, p.upstreamFetchBytes, p.upstreamFetchDurations,
|
||||||
|
p.upstreamFetchErrors, p.transforms, p.transformDurations,
|
||||||
|
p.transformsInProgress,
|
||||||
} {
|
} {
|
||||||
err = metricsRegistry.Register(counter)
|
err = metricsRegistry.Register(metric)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to register the proxy's counters: %w", err)
|
return nil, fmt.Errorf("failed to register the proxy's metrics: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -341,7 +366,10 @@ func (p *Proxy) loadCachedSource(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// processFromSourceOrFetch makes the variant req asks for, from the cached
|
// processFromSourceOrFetch makes the variant req asks for, from the cached
|
||||||
// source when there is one and from a fetched source otherwise.
|
// source when there is one and from a fetched source otherwise. It counts a
|
||||||
|
// source cache miss when it fetches the source because the source has no row
|
||||||
|
// in the cache or its cached file cannot be used, and a source cache hit
|
||||||
|
// otherwise.
|
||||||
//
|
//
|
||||||
// A cached source last checked with upstream CacheTTL or more ago is checked
|
// A cached source last checked with upstream CacheTTL or more ago is checked
|
||||||
// first: upstream is asked to send it only if it changed since. When it
|
// first: upstream is asked to send it only if it changed since. When it
|
||||||
@@ -362,6 +390,8 @@ func (p *Proxy) processFromSourceOrFetch(
|
|||||||
p.log.Warn("failed to look up cached source, fetching", "error", err)
|
p.log.Warn("failed to look up cached source, fetching", "error", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
p.cache.IncrementSourceStats(false)
|
||||||
|
|
||||||
return p.fetchAndProcess(ctx, req, cacheKey)
|
return p.fetchAndProcess(ctx, req, cacheKey)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -375,16 +405,21 @@ func (p *Proxy) processFromSourceOrFetch(
|
|||||||
case err != nil:
|
case err != nil:
|
||||||
p.recordFailedCheck(ctx, req, err)
|
p.recordFailedCheck(ctx, req, err)
|
||||||
default:
|
default:
|
||||||
|
p.cache.IncrementSourceStats(true)
|
||||||
|
|
||||||
return p.processFetched(ctx, req, cacheKey, fetched)
|
return p.processFetched(ctx, req, cacheKey, fetched)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
variant := p.renewVariant(ctx, cacheKey, cached)
|
variant := p.renewVariant(ctx, cacheKey, cached)
|
||||||
if variant != nil {
|
if variant != nil {
|
||||||
|
p.cache.IncrementSourceStats(true)
|
||||||
|
|
||||||
return variant, nil
|
return variant, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
source, sourceSize, sourceFormat := p.loadCachedSource(ctx, req.Source)
|
source, sourceSize, sourceFormat := p.loadCachedSource(ctx, req.Source)
|
||||||
|
p.cache.IncrementSourceStats(source != nil)
|
||||||
|
|
||||||
// Fetch from upstream if we don't have source data or it's empty
|
// Fetch from upstream if we don't have source data or it's empty
|
||||||
if source == nil {
|
if source == nil {
|
||||||
@@ -581,36 +616,32 @@ type fetchedSource struct {
|
|||||||
// fetchSource fetches req's source from upstream and stores it in the cache.
|
// fetchSource fetches req's source from upstream and stores it in the cache.
|
||||||
// A non-empty ifNoneMatch or ifModifiedSince asks upstream to send the source
|
// A non-empty ifNoneMatch or ifModifiedSince asks upstream to send the source
|
||||||
// only if it changed since; errSourceNotModified says it did not. It counts
|
// only if it changed since; errSourceNotModified says it did not. It counts
|
||||||
// the fetch with the bytes read from upstream, including when reading the
|
// the fetch with the bytes read from upstream and its duration, including
|
||||||
// response or a later step fails. It fetches nothing while the source URL is
|
// when reading the response or a later step fails, and counts a fetch that
|
||||||
// in the negative cache. The caller closes the returned response.
|
// fails or whose response cannot be read as an upstream fetch error. It
|
||||||
|
// fetches nothing while the source URL is in the negative cache. The caller
|
||||||
|
// closes the returned response.
|
||||||
func (p *Proxy) fetchSource(
|
func (p *Proxy) fetchSource(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
req *imagerequest.Request,
|
req *imagerequest.Request,
|
||||||
ifNoneMatch, ifModifiedSince string,
|
ifNoneMatch, ifModifiedSince string,
|
||||||
) (*fetchedSource, error) {
|
) (*fetchedSource, error) {
|
||||||
// Check negative cache - skip fetching for recently-failed URLs
|
err := p.refuseNegativeCached(ctx, req)
|
||||||
negHit, err := p.cache.Failure(ctx, req.Source)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
p.log.Warn("negative cache check failed", "error", err)
|
return nil, err
|
||||||
}
|
|
||||||
|
|
||||||
if negHit {
|
|
||||||
p.log.Debug("negative cache hit",
|
|
||||||
"host", req.Source.Host,
|
|
||||||
"path", req.Source.Path,
|
|
||||||
)
|
|
||||||
|
|
||||||
return nil, fmt.Errorf("%w: %w", httpfetcher.ErrUpstreamError, ErrNegativeCached)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fetch from upstream
|
// Fetch from upstream
|
||||||
p.log.Debug("fetching from upstream", "host", req.Source.Host,
|
p.log.Debug("fetching from upstream", "host", req.Source.Host,
|
||||||
"path", req.Source.Path, "query", req.Source.Query)
|
"path", req.Source.Path, "query", req.Source.Query)
|
||||||
|
|
||||||
|
fetchStart := time.Now()
|
||||||
|
|
||||||
fetchResult, err := p.fetcher.Fetch(ctx, req.Source,
|
fetchResult, err := p.fetcher.Fetch(ctx, req.Source,
|
||||||
ifNoneMatch, ifModifiedSince)
|
ifNoneMatch, ifModifiedSince)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
p.upstreamFetchErrors.Inc()
|
||||||
|
|
||||||
// Store negative cache for certain errors
|
// Store negative cache for certain errors
|
||||||
if isNegativeCacheable(err) {
|
if isNegativeCacheable(err) {
|
||||||
p.storeNegative(ctx, req)
|
p.storeNegative(ctx, req)
|
||||||
@@ -634,9 +665,11 @@ func (p *Proxy) fetchSource(
|
|||||||
fetchBytes := int64(len(sourceData))
|
fetchBytes := int64(len(sourceData))
|
||||||
|
|
||||||
// Counted also when the request context has ended meanwhile
|
// Counted also when the request context has ended meanwhile
|
||||||
p.incrementUpstreamFetch(fetchBytes)
|
p.incrementUpstreamFetch(fetchBytes, time.Since(fetchStart))
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
p.upstreamFetchErrors.Inc()
|
||||||
|
|
||||||
_ = fetchResult.Content.Close()
|
_ = fetchResult.Content.Close()
|
||||||
|
|
||||||
return nil, fmt.Errorf("failed to read upstream response: %w", err)
|
return nil, fmt.Errorf("failed to read upstream response: %w", err)
|
||||||
@@ -677,6 +710,29 @@ func (p *Proxy) fetchSource(
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// refuseNegativeCached returns an error wrapping ErrNegativeCached while
|
||||||
|
// req's source URL is in the negative cache, and nil otherwise. A failure to
|
||||||
|
// read the negative cache is only logged, and returns nil.
|
||||||
|
func (p *Proxy) refuseNegativeCached(
|
||||||
|
ctx context.Context, req *imagerequest.Request,
|
||||||
|
) error {
|
||||||
|
negHit, err := p.cache.Failure(ctx, req.Source)
|
||||||
|
if err != nil {
|
||||||
|
p.log.Warn("negative cache check failed", "error", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !negHit {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
p.log.Debug("negative cache hit",
|
||||||
|
"host", req.Source.Host,
|
||||||
|
"path", req.Source.Path,
|
||||||
|
)
|
||||||
|
|
||||||
|
return fmt.Errorf("%w: %w", httpfetcher.ErrUpstreamError, ErrNegativeCached)
|
||||||
|
}
|
||||||
|
|
||||||
// fetchedSourceInfo returns what the cache keeps about a source upstream
|
// fetchedSourceInfo returns what the cache keeps about a source upstream
|
||||||
// just sent: fetched and checked now, with the ETag and Last-Modified
|
// just sent: fetched and checked now, with the ETag and Last-Modified
|
||||||
// headers of fetchResult.
|
// headers of fetchResult.
|
||||||
@@ -722,15 +778,16 @@ func (p *Proxy) logUpstreamFetch(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// incrementUpstreamFetch counts one upstream fetch that read fetchBytes bytes.
|
// incrementUpstreamFetch counts one upstream fetch that read fetchBytes bytes
|
||||||
// A fetch that read no bytes is not counted.
|
// and took duration. A fetch that read no bytes is not counted.
|
||||||
func (p *Proxy) incrementUpstreamFetch(fetchBytes int64) {
|
func (p *Proxy) incrementUpstreamFetch(fetchBytes int64, duration time.Duration) {
|
||||||
if fetchBytes <= 0 {
|
if fetchBytes <= 0 {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
p.upstreamFetches.Inc()
|
p.upstreamFetches.Inc()
|
||||||
p.upstreamFetchBytes.Add(float64(fetchBytes))
|
p.upstreamFetchBytes.Add(float64(fetchBytes))
|
||||||
|
p.upstreamFetchDurations.Observe(duration.Seconds())
|
||||||
}
|
}
|
||||||
|
|
||||||
// storeNegative stores a negative cache entry for req's source, whose fetch
|
// storeNegative stores a negative cache entry for req's source, whose fetch
|
||||||
@@ -772,8 +829,7 @@ func (p *Proxy) processAndStore(
|
|||||||
|
|
||||||
// ctx has the deadline of the request doing the processing, which can
|
// ctx has the deadline of the request doing the processing, which can
|
||||||
// end while a request waiting for its image has time left
|
// end while a request waiting for its image has time left
|
||||||
processResult, err := p.processor.Process(ctx, source, sourceFormat,
|
processResult, err := p.transform(ctx, source, sourceFormat, &req.Transform)
|
||||||
&req.Transform)
|
|
||||||
if errors.Is(err, context.DeadlineExceeded) {
|
if errors.Is(err, context.DeadlineExceeded) {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrProcessingTimedOut, err)
|
return nil, fmt.Errorf("%w: %w", ErrProcessingTimedOut, err)
|
||||||
}
|
}
|
||||||
@@ -786,6 +842,7 @@ func (p *Proxy) processAndStore(
|
|||||||
|
|
||||||
// Counted also when the request context has ended meanwhile
|
// Counted also when the request context has ended meanwhile
|
||||||
p.transforms.Inc()
|
p.transforms.Inc()
|
||||||
|
p.transformDurations.Observe(processDuration.Seconds())
|
||||||
|
|
||||||
// Read processed content
|
// Read processed content
|
||||||
processedData, err := io.ReadAll(processResult.Content)
|
processedData, err := io.ReadAll(processResult.Content)
|
||||||
@@ -838,6 +895,21 @@ func (p *Proxy) processAndStore(
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// transform has the image processor transform the image read from source
|
||||||
|
// as requested says, and counts it in pixa_transforms_in_progress until the
|
||||||
|
// processor returns or panics.
|
||||||
|
func (p *Proxy) transform(
|
||||||
|
ctx context.Context,
|
||||||
|
source io.Reader,
|
||||||
|
sourceFormat imagerequest.Format,
|
||||||
|
requested *imagerequest.Transform,
|
||||||
|
) (*imageprocessor.Result, error) {
|
||||||
|
p.transformsInProgress.Inc()
|
||||||
|
defer p.transformsInProgress.Dec()
|
||||||
|
|
||||||
|
return p.processor.Process(ctx, source, sourceFormat, requested)
|
||||||
|
}
|
||||||
|
|
||||||
// isNegativeCacheable returns true if the error should be cached.
|
// isNegativeCacheable returns true if the error should be cached.
|
||||||
func isNegativeCacheable(err error) bool {
|
func isNegativeCacheable(err error) bool {
|
||||||
return errors.Is(err, httpfetcher.ErrUpstreamError)
|
return errors.Is(err, httpfetcher.ErrUpstreamError)
|
||||||
|
|||||||
@@ -0,0 +1,213 @@
|
|||||||
|
package imageproxy
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"sneak.berlin/go/pixa/internal/imageprocessor"
|
||||||
|
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||||
|
)
|
||||||
|
|
||||||
|
// metricValues holds the source cache, upstream fetch and transform metrics
|
||||||
|
// of a proxy and its cache. A histogram is given by its count of
|
||||||
|
// observations.
|
||||||
|
type metricValues struct {
|
||||||
|
sourceCacheHits uint64
|
||||||
|
sourceCacheMisses uint64
|
||||||
|
fetchDurations uint64
|
||||||
|
fetchErrors uint64
|
||||||
|
transformDuration uint64
|
||||||
|
inProgress float64
|
||||||
|
}
|
||||||
|
|
||||||
|
// readMetricValues reads the metrics of svc and its cache that
|
||||||
|
// metricValues holds from their metrics registry.
|
||||||
|
func readMetricValues(t *testing.T, svc *testProxy) metricValues {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
families, err := svc.registry.Gather()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to gather the metrics: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var values metricValues
|
||||||
|
|
||||||
|
for _, family := range families {
|
||||||
|
metric := family.GetMetric()[0]
|
||||||
|
|
||||||
|
switch family.GetName() {
|
||||||
|
case "pixa_source_cache_hits_total":
|
||||||
|
values.sourceCacheHits = uint64(metric.GetCounter().GetValue())
|
||||||
|
case "pixa_source_cache_misses_total":
|
||||||
|
values.sourceCacheMisses = uint64(metric.GetCounter().GetValue())
|
||||||
|
case "pixa_upstream_fetch_duration_seconds":
|
||||||
|
values.fetchDurations = metric.GetHistogram().GetSampleCount()
|
||||||
|
case "pixa_upstream_fetch_errors_total":
|
||||||
|
values.fetchErrors = uint64(metric.GetCounter().GetValue())
|
||||||
|
case "pixa_transform_duration_seconds":
|
||||||
|
values.transformDuration = metric.GetHistogram().GetSampleCount()
|
||||||
|
case "pixa_transforms_in_progress":
|
||||||
|
values.inProgress = metric.GetGauge().GetValue()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return values
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestProxy_Get_CountsSourceCacheFetchesAndTransforms walks Get through a
|
||||||
|
// miss that fetches the source, a hit, a miss that uses the cached source
|
||||||
|
// and a miss whose source is not found upstream, checking the source cache,
|
||||||
|
// upstream fetch and transform metrics after each.
|
||||||
|
func TestProxy_Get_CountsSourceCacheFetchesAndTransforms(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
svc, fixtures := setupTestProxy(t)
|
||||||
|
|
||||||
|
// want is source cache hits, source cache misses, fetch durations
|
||||||
|
// observed, fetch errors, transform durations observed, transforms in
|
||||||
|
// progress.
|
||||||
|
steps := []struct {
|
||||||
|
name string
|
||||||
|
path string
|
||||||
|
size int
|
||||||
|
wantErr bool
|
||||||
|
want metricValues
|
||||||
|
}{
|
||||||
|
{"miss that fetches the source", testPathPhoto, 50, false,
|
||||||
|
metricValues{0, 1, 1, 0, 1, 0}},
|
||||||
|
{"hit", testPathPhoto, 50, false,
|
||||||
|
metricValues{0, 1, 1, 0, 1, 0}},
|
||||||
|
{"miss that uses the cached source", testPathPhoto, 25, false,
|
||||||
|
metricValues{1, 1, 1, 0, 2, 0}},
|
||||||
|
{"miss whose source is not found", "/images/absent.jpg", 50, true,
|
||||||
|
metricValues{1, 2, 1, 1, 2, 0}},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, step := range steps {
|
||||||
|
resp, err := svc.Get(t.Context(), &imagerequest.Request{
|
||||||
|
Source: imagerequest.Source{
|
||||||
|
Host: fixtures.GoodHost,
|
||||||
|
Path: step.path,
|
||||||
|
},
|
||||||
|
Transform: imagerequest.Transform{
|
||||||
|
Size: imagerequest.Size{Width: step.size, Height: step.size},
|
||||||
|
Format: imagerequest.FormatJPEG,
|
||||||
|
Quality: 85,
|
||||||
|
FitMode: imagerequest.FitCover,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if (err != nil) != step.wantErr {
|
||||||
|
t.Fatalf("%s: Get() error = %v, want error %t", step.name, err, step.wantErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err == nil {
|
||||||
|
_ = resp.Content.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
got := readMetricValues(t, svc)
|
||||||
|
t.Logf("after the %s: metrics = %+v", step.name, got)
|
||||||
|
|
||||||
|
if got != step.want {
|
||||||
|
t.Fatalf("after the %s: metrics = %+v, want %+v", step.name, got, step.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestProxy_Get_CountsMissingSourceFileAsSourceCacheMiss deletes the file of
|
||||||
|
// a cached source but keeps its row. A request for a second variant of that
|
||||||
|
// source must fetch the source again, and count a source cache miss, not a
|
||||||
|
// hit.
|
||||||
|
func TestProxy_Get_CountsMissingSourceFileAsSourceCacheMiss(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
svc, fixtures := setupTestProxy(t)
|
||||||
|
|
||||||
|
// A first request caches the photo as a source.
|
||||||
|
resp, err := svc.Get(t.Context(), widthOnlyRequest(fixtures, 50))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("first Get() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_ = resp.Content.Close()
|
||||||
|
|
||||||
|
source := widthOnlyRequest(fixtures, 50).Source
|
||||||
|
|
||||||
|
err = os.Remove(sourceFilePath(svc, source.Key()))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to delete the cached source file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
before := readMetricValues(t, svc)
|
||||||
|
|
||||||
|
resp, err = svc.Get(t.Context(), widthOnlyRequest(fixtures, 25))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Get() of a second variant error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_ = resp.Content.Close()
|
||||||
|
|
||||||
|
after := readMetricValues(t, svc)
|
||||||
|
t.Logf("before the second variant: metrics = %+v; after: %+v", before, after)
|
||||||
|
|
||||||
|
if after.sourceCacheHits != before.sourceCacheHits ||
|
||||||
|
after.sourceCacheMisses != before.sourceCacheMisses+1 {
|
||||||
|
t.Errorf("source cache hits went from %d to %d and misses from %d to %d, "+
|
||||||
|
"want no new hit and one more miss", before.sourceCacheHits,
|
||||||
|
after.sourceCacheHits, before.sourceCacheMisses, after.sourceCacheMisses)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestProxy_Get_CountsTransformWaitingForSlotAsInProgress holds the only
|
||||||
|
// processing slot and starts a request that must transform the photo. While
|
||||||
|
// the request waits for the slot, pixa_transforms_in_progress must be 1; once
|
||||||
|
// the request returns, 0.
|
||||||
|
func TestProxy_Get_CountsTransformWaitingForSlotAsInProgress(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
svc, fixtures := setupTestProxy(t)
|
||||||
|
svc.processor = imageprocessor.New(
|
||||||
|
imageprocessor.Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1},
|
||||||
|
)
|
||||||
|
|
||||||
|
release := holdProcessingSlot(t, svc.processor)
|
||||||
|
|
||||||
|
result := startGet(t.Context(), svc, widthOnlyRequest(fixtures, 50))
|
||||||
|
|
||||||
|
// The request fetches the photo first, then waits for the slot.
|
||||||
|
deadline := time.Now().Add(5 * time.Second)
|
||||||
|
for readMetricValues(t, svc).inProgress != 1 && time.Now().Before(deadline) {
|
||||||
|
time.Sleep(10 * time.Millisecond)
|
||||||
|
}
|
||||||
|
|
||||||
|
waiting := readMetricValues(t, svc).inProgress
|
||||||
|
t.Logf("while the request waits for the slot: transforms in progress = %v",
|
||||||
|
waiting)
|
||||||
|
|
||||||
|
select {
|
||||||
|
case got := <-result:
|
||||||
|
t.Fatalf("Get() returned while the only processing slot was held, "+
|
||||||
|
"error = %v", got.err)
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
if waiting != 1 {
|
||||||
|
t.Fatalf("transforms in progress = %v while the request waits for the "+
|
||||||
|
"slot, want 1", waiting)
|
||||||
|
}
|
||||||
|
|
||||||
|
release()
|
||||||
|
|
||||||
|
got := <-result
|
||||||
|
if got.err != nil {
|
||||||
|
t.Fatalf("Get() error = %v", got.err)
|
||||||
|
}
|
||||||
|
|
||||||
|
returned := readMetricValues(t, svc).inProgress
|
||||||
|
t.Logf("once the request returned: transforms in progress = %v", returned)
|
||||||
|
|
||||||
|
if returned != 0 {
|
||||||
|
t.Errorf("transforms in progress = %v once the request returned, want 0",
|
||||||
|
returned)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -82,15 +82,3 @@ func TestProxy_Get_ServesCachedVariantOfNegativeCachedURL(t *testing.T) {
|
|||||||
err, ErrNegativeCached)
|
err, ErrNegativeCached)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestProxy_Get_ReturnsErrorForNegativeCachedURL(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// This test verifies that Proxy.Get() checks the negative cache
|
|
||||||
// We can't easily test the full pipeline without vips, but we can
|
|
||||||
// verify the error type
|
|
||||||
err := ErrNegativeCached
|
|
||||||
if !errors.Is(err, ErrNegativeCached) {
|
|
||||||
t.Error("ErrNegativeCached should be identifiable with errors.Is")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -14,7 +14,8 @@ import (
|
|||||||
var ErrInvalidFitMode = errors.New("invalid fit mode")
|
var ErrInvalidFitMode = errors.New("invalid fit mode")
|
||||||
|
|
||||||
// Format is an image format. pixa reads and writes the formats from JPEG to
|
// Format is an image format. pixa reads and writes the formats from JPEG to
|
||||||
// GIF below. In a request, orig and auto stand for one of them.
|
// GIF below. It reads SVG and writes it only as orig of an SVG source. In a
|
||||||
|
// request, orig and auto stand for another format.
|
||||||
type Format string
|
type Format string
|
||||||
|
|
||||||
// Supported image formats.
|
// Supported image formats.
|
||||||
@@ -26,6 +27,7 @@ const (
|
|||||||
FormatAVIF Format = "avif"
|
FormatAVIF Format = "avif"
|
||||||
FormatJXL Format = "jxl"
|
FormatJXL Format = "jxl"
|
||||||
FormatGIF Format = "gif"
|
FormatGIF Format = "gif"
|
||||||
|
FormatSVG Format = "svg"
|
||||||
|
|
||||||
// FormatAuto stands for JPEG XL, AVIF, WebP or JPEG, chosen for each
|
// FormatAuto stands for JPEG XL, AVIF, WebP or JPEG, chosen for each
|
||||||
// request from its Accept header once the URL's signature or token has
|
// request from its Accept header once the URL's signature or token has
|
||||||
@@ -50,6 +52,8 @@ func (f Format) MIMEType() string {
|
|||||||
return "image/jxl"
|
return "image/jxl"
|
||||||
case FormatGIF:
|
case FormatGIF:
|
||||||
return "image/gif"
|
return "image/gif"
|
||||||
|
case FormatSVG:
|
||||||
|
return "image/svg+xml"
|
||||||
case FormatOriginal, FormatAuto:
|
case FormatOriginal, FormatAuto:
|
||||||
return ""
|
return ""
|
||||||
default:
|
default:
|
||||||
|
|||||||
+21
-24
@@ -111,10 +111,16 @@ type Cache struct {
|
|||||||
pendingAccessTimesDone chan struct{}
|
pendingAccessTimesDone chan struct{}
|
||||||
pendingAccessTimesCancel context.CancelFunc
|
pendingAccessTimesCancel context.CancelFunc
|
||||||
|
|
||||||
// The counts of cache hits and misses, made and registered by
|
// The cache's metrics, made and registered by registerMetrics (see
|
||||||
// registerCounters (see counters.go)
|
// metrics.go)
|
||||||
cacheHits prometheus.Counter
|
cacheHits prometheus.Counter
|
||||||
cacheMisses prometheus.Counter
|
cacheMisses prometheus.Counter
|
||||||
|
sourceCacheHits prometheus.Counter
|
||||||
|
sourceCacheMisses prometheus.Counter
|
||||||
|
usageBytes prometheus.Gauge
|
||||||
|
maxBytes prometheus.Gauge
|
||||||
|
evictions prometheus.Counter
|
||||||
|
evictedBytes prometheus.Counter
|
||||||
|
|
||||||
// reconciliationPageSize is the most rows one read of a content table
|
// reconciliationPageSize is the most rows one read of a content table
|
||||||
// returns in the reconciliation pass. NewCache sets it to
|
// returns in the reconciliation pass. NewCache sets it to
|
||||||
@@ -128,7 +134,7 @@ type Cache struct {
|
|||||||
reconciliationReadTestHook func(rows int)
|
reconciliationReadTestHook func(rows int)
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewCache creates a new cache instance and registers its counters on
|
// NewCache creates a new cache instance and registers its metrics on
|
||||||
// metricsRegistry. pixad passes Prometheus' default registry, the one
|
// metricsRegistry. pixad passes Prometheus' default registry, the one
|
||||||
// /metrics serves. It makes no database call.
|
// /metrics serves. It makes no database call.
|
||||||
func NewCache(
|
func NewCache(
|
||||||
@@ -157,7 +163,7 @@ func NewCache(
|
|||||||
reconciliationPageSize: defaultReconciliationPageSize,
|
reconciliationPageSize: defaultReconciliationPageSize,
|
||||||
}
|
}
|
||||||
|
|
||||||
err := c.registerCounters(metricsRegistry)
|
err := c.registerMetrics(metricsRegistry)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -656,17 +662,6 @@ func (c *Cache) CleanExpired(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// IncrementStats counts a cache hit or miss. It only adds to the cache's
|
|
||||||
// Prometheus counters in memory (see counters.go), so it never waits for the
|
|
||||||
// database or fails.
|
|
||||||
func (c *Cache) IncrementStats(hit bool) {
|
|
||||||
if hit {
|
|
||||||
c.cacheHits.Inc()
|
|
||||||
} else {
|
|
||||||
c.cacheMisses.Inc()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// touchVariant hands the access time of a variant served from the cache to
|
// touchVariant hands the access time of a variant served from the cache to
|
||||||
// the goroutine StartPendingAccessTimeWrites starts, which writes it in its
|
// the goroutine StartPendingAccessTimeWrites starts, which writes it in its
|
||||||
// next batch. It never waits: while maxPendingAccessTimes access times of
|
// next batch. It never waits: while maxPendingAccessTimes access times of
|
||||||
@@ -740,16 +735,18 @@ func (c *Cache) finishOpeningVariant(
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// readFormat reads the first bytes of a source or variant file, without
|
// readFormat reads up to magic.MaxMagicBytes first bytes of a source or
|
||||||
// moving its offset, and returns the format they show. Sources and variants
|
// variant file, without moving its offset, and returns the format they show.
|
||||||
// are in the same six formats.
|
// Sources and variants are in the same seven formats; a variant is SVG only
|
||||||
|
// as orig of an SVG source. A file shorter than magic.MinMagicBytes is
|
||||||
|
// io.EOF.
|
||||||
func readFormat(content io.ReaderAt) (imagerequest.Format, error) {
|
func readFormat(content io.ReaderAt) (imagerequest.Format, error) {
|
||||||
firstBytes := make([]byte, magic.MinMagicBytes)
|
firstBytes := make([]byte, magic.MaxMagicBytes)
|
||||||
|
|
||||||
_, err := content.ReadAt(firstBytes, 0)
|
n, err := content.ReadAt(firstBytes, 0)
|
||||||
if err != nil {
|
if err != nil && (!errors.Is(err, io.EOF) || n < magic.MinMagicBytes) {
|
||||||
return "", fmt.Errorf("failed to read the first bytes: %w", err)
|
return "", fmt.Errorf("failed to read the first bytes: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return magic.DetectFormat(firstBytes)
|
return magic.DetectFormat(firstBytes[:n])
|
||||||
}
|
}
|
||||||
@@ -98,8 +98,6 @@ func TestVariant_RefusesFileOfNoOutputFormat(t *testing.T) {
|
|||||||
wantErr error
|
wantErr error
|
||||||
}{
|
}{
|
||||||
{"text", []byte("this is text, not an image"), magic.ErrUnknownFormat},
|
{"text", []byte("this is text, not an image"), magic.ErrUnknownFormat},
|
||||||
{"SVG", []byte(`<svg xmlns="http://www.w3.org/2000/svg"></svg>`),
|
|
||||||
magic.ErrUnknownFormat},
|
|
||||||
{"JPEG cut to 10 bytes", jpegImage[:10], io.EOF},
|
{"JPEG cut to 10 bytes", jpegImage[:10], io.EOF},
|
||||||
{"empty", nil, io.EOF},
|
{"empty", nil, io.EOF},
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,31 +0,0 @@
|
|||||||
package imgcache
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
|
||||||
)
|
|
||||||
|
|
||||||
// registerCounters makes the cache's counters and registers them on
|
|
||||||
// metricsRegistry. The counters are kept in memory, so adding to one waits
|
|
||||||
// for nothing and makes no database call. It fails when metricsRegistry
|
|
||||||
// already holds counters of the same names.
|
|
||||||
func (c *Cache) registerCounters(metricsRegistry prometheus.Registerer) error {
|
|
||||||
c.cacheHits = prometheus.NewCounter(prometheus.CounterOpts{
|
|
||||||
Name: "pixa_cache_hits_total",
|
|
||||||
Help: "Image requests served from the cache.",
|
|
||||||
})
|
|
||||||
c.cacheMisses = prometheus.NewCounter(prometheus.CounterOpts{
|
|
||||||
Name: "pixa_cache_misses_total",
|
|
||||||
Help: "Image requests not served from the cache.",
|
|
||||||
})
|
|
||||||
|
|
||||||
for _, counter := range []prometheus.Counter{c.cacheHits, c.cacheMisses} {
|
|
||||||
err := metricsRegistry.Register(counter)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to register the cache's counters: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -92,8 +92,9 @@ type evictionCandidate struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// EvictToLimit evicts least-recently-used cache entries until total
|
// EvictToLimit evicts least-recently-used cache entries until total
|
||||||
// tracked usage is at or below the configured MaxBytes limit. It is a
|
// tracked usage is at or below the configured MaxBytes limit. It sets the
|
||||||
// no-op when the cache is disabled.
|
// pixa_cache_usage_bytes gauge to each usage it reads. It is a no-op when
|
||||||
|
// the cache is disabled.
|
||||||
func (c *Cache) EvictToLimit(ctx context.Context) error {
|
func (c *Cache) EvictToLimit(ctx context.Context) error {
|
||||||
if c.config.MaxBytes == 0 {
|
if c.config.MaxBytes == 0 {
|
||||||
return nil
|
return nil
|
||||||
@@ -105,6 +106,8 @@ func (c *Cache) EvictToLimit(ctx context.Context) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
c.usageBytes.Set(float64(usage))
|
||||||
|
|
||||||
if usage <= c.config.MaxBytes {
|
if usage <= c.config.MaxBytes {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -345,37 +348,95 @@ func (c *Cache) sourceCandidates(ctx context.Context) ([]evictionCandidate, erro
|
|||||||
return candidates, nil
|
return candidates, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// evictVariant removes one variant: accounting row first, then the
|
// evictVariant removes one variant (see removeVariant) and counts the
|
||||||
// file, so the database never references a deleted file.
|
// eviction with the size its row recorded.
|
||||||
func (c *Cache) evictVariant(
|
func (c *Cache) evictVariant(
|
||||||
ctx context.Context, cacheKey imagerequest.VariantKey,
|
ctx context.Context, cacheKey imagerequest.VariantKey,
|
||||||
) error {
|
) error {
|
||||||
_, err := c.db.ExecContext(ctx,
|
sizeBytes, err := c.removeVariant(ctx, cacheKey)
|
||||||
`DELETE FROM variant_content WHERE cache_key = ?`, string(cacheKey))
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to delete variant accounting row: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = c.variants.Delete(string(cacheKey))
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
c.countEviction(sizeBytes)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// evictSourceBlob removes one source: its row first, then its file, as
|
// removeVariant removes one variant: accounting row first, then the
|
||||||
// evictVariant does for a variant. It holds no lock: a store of the same
|
// file, so the database never references a deleted file. It returns the
|
||||||
// source in between can leave the source's row with no file, which
|
// size the row recorded, or 0 when the variant had no row.
|
||||||
// Source treats as a miss.
|
func (c *Cache) removeVariant(
|
||||||
func (c *Cache) evictSourceBlob(ctx context.Context, key imagerequest.SourceKey) error {
|
ctx context.Context, cacheKey imagerequest.VariantKey,
|
||||||
_, err := c.db.ExecContext(ctx,
|
) (int64, error) {
|
||||||
`DELETE FROM source_content WHERE source_key = ?`, string(key))
|
sizeBytes, err := c.deleteRowReturningSize(ctx,
|
||||||
|
`DELETE FROM variant_content WHERE cache_key = ? RETURNING size_bytes`,
|
||||||
|
string(cacheKey))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to delete source content row: %w", err)
|
return 0, fmt.Errorf("failed to delete variant accounting row: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return c.srcContent.Delete(string(key))
|
err = c.variants.Delete(string(cacheKey))
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return sizeBytes, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// evictSourceBlob removes one source (see removeSourceBlob) and counts the
|
||||||
|
// eviction with the size its row recorded.
|
||||||
|
func (c *Cache) evictSourceBlob(ctx context.Context, key imagerequest.SourceKey) error {
|
||||||
|
sizeBytes, err := c.removeSourceBlob(ctx, key)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
c.countEviction(sizeBytes)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// removeSourceBlob removes one source: its row first, then its file, as
|
||||||
|
// removeVariant does for a variant, and returns the size the row recorded,
|
||||||
|
// or 0 when the source had no row. It holds no lock: a store of the same
|
||||||
|
// source in between can leave the source's row with no file, which Source
|
||||||
|
// treats as a miss.
|
||||||
|
func (c *Cache) removeSourceBlob(
|
||||||
|
ctx context.Context, key imagerequest.SourceKey,
|
||||||
|
) (int64, error) {
|
||||||
|
sizeBytes, err := c.deleteRowReturningSize(ctx,
|
||||||
|
`DELETE FROM source_content WHERE source_key = ? RETURNING size_bytes`,
|
||||||
|
string(key))
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to delete source content row: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = c.srcContent.Delete(string(key))
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return sizeBytes, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// deleteRowReturningSize runs deleteQuery, which deletes the row of key and
|
||||||
|
// returns its size_bytes, and returns that size, or 0 when there was no row.
|
||||||
|
func (c *Cache) deleteRowReturningSize(
|
||||||
|
ctx context.Context, deleteQuery string, key string,
|
||||||
|
) (int64, error) {
|
||||||
|
var sizeBytes int64
|
||||||
|
|
||||||
|
err := c.db.QueryRowContext(ctx, deleteQuery, key).Scan(&sizeBytes)
|
||||||
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return sizeBytes, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// notifyWritePressure wakes the background evictor after a store, so
|
// notifyWritePressure wakes the background evictor after a store, so
|
||||||
@@ -673,7 +734,9 @@ func (c *Cache) reconcileVariantFile(
|
|||||||
c.log.Warn("removing damaged cached variant file", "path", path,
|
c.log.Warn("removing damaged cached variant file", "path", path,
|
||||||
"file_bytes", info.Size(), "recorded_bytes", recordedSize)
|
"file_bytes", info.Size(), "recorded_bytes", recordedSize)
|
||||||
|
|
||||||
return c.evictVariant(ctx, cacheKey)
|
_, err = c.removeVariant(ctx, cacheKey)
|
||||||
|
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// adoptVariantFile inserts an accounting row for a variant file that
|
// adoptVariantFile inserts an accounting row for a variant file that
|
||||||
@@ -835,7 +898,9 @@ func (c *Cache) reconcileSourceFile(
|
|||||||
c.log.Warn("removing damaged cached source file", "path", path,
|
c.log.Warn("removing damaged cached source file", "path", path,
|
||||||
"file_bytes", info.Size(), "recorded_bytes", recordedSize)
|
"file_bytes", info.Size(), "recorded_bytes", recordedSize)
|
||||||
|
|
||||||
return c.evictSourceBlob(ctx, key)
|
_, err = c.removeSourceBlob(ctx, key)
|
||||||
|
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// removeUntrackedSourceFile deletes a source blob file that has no
|
// removeUntrackedSourceFile deletes a source blob file that has no
|
||||||
|
|||||||
@@ -0,0 +1,88 @@
|
|||||||
|
package imgcache
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
|
)
|
||||||
|
|
||||||
|
// registerMetrics makes the cache's metrics and registers them on
|
||||||
|
// metricsRegistry. The metrics are kept in memory, so updating one waits
|
||||||
|
// for nothing and makes no database call. It fails when metricsRegistry
|
||||||
|
// already holds metrics of the same names.
|
||||||
|
func (c *Cache) registerMetrics(metricsRegistry prometheus.Registerer) error {
|
||||||
|
c.cacheHits = prometheus.NewCounter(prometheus.CounterOpts{
|
||||||
|
Name: "pixa_cache_hits_total",
|
||||||
|
Help: "Image requests served from the cache.",
|
||||||
|
})
|
||||||
|
c.cacheMisses = prometheus.NewCounter(prometheus.CounterOpts{
|
||||||
|
Name: "pixa_cache_misses_total",
|
||||||
|
Help: "Image requests not served from the cache.",
|
||||||
|
})
|
||||||
|
c.sourceCacheHits = prometheus.NewCounter(prometheus.CounterOpts{
|
||||||
|
Name: "pixa_source_cache_hits_total",
|
||||||
|
Help: "Variants made or checked whose source was in the cache.",
|
||||||
|
})
|
||||||
|
c.sourceCacheMisses = prometheus.NewCounter(prometheus.CounterOpts{
|
||||||
|
Name: "pixa_source_cache_misses_total",
|
||||||
|
Help: "Variants made or checked whose source was not in the cache.",
|
||||||
|
})
|
||||||
|
c.usageBytes = prometheus.NewGauge(prometheus.GaugeOpts{
|
||||||
|
Name: "pixa_cache_usage_bytes",
|
||||||
|
Help: "Bytes of sources and variants in the cache, as eviction last read them.",
|
||||||
|
})
|
||||||
|
c.maxBytes = prometheus.NewGauge(prometheus.GaugeOpts{
|
||||||
|
Name: "pixa_cache_max_bytes",
|
||||||
|
Help: "The disk cache limit in bytes, cache_max_bytes.",
|
||||||
|
})
|
||||||
|
c.evictions = prometheus.NewCounter(prometheus.CounterOpts{
|
||||||
|
Name: "pixa_cache_evictions_total",
|
||||||
|
Help: "Sources and variants removed by eviction.",
|
||||||
|
})
|
||||||
|
c.evictedBytes = prometheus.NewCounter(prometheus.CounterOpts{
|
||||||
|
Name: "pixa_cache_evicted_bytes_total",
|
||||||
|
Help: "Bytes of the sources and variants removed by eviction.",
|
||||||
|
})
|
||||||
|
|
||||||
|
c.maxBytes.Set(float64(c.config.MaxBytes))
|
||||||
|
|
||||||
|
for _, metric := range []prometheus.Collector{
|
||||||
|
c.cacheHits, c.cacheMisses, c.sourceCacheHits, c.sourceCacheMisses,
|
||||||
|
c.usageBytes, c.maxBytes, c.evictions, c.evictedBytes,
|
||||||
|
} {
|
||||||
|
err := metricsRegistry.Register(metric)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to register the cache's metrics: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// IncrementStats counts a cache hit or miss of an image request. It only
|
||||||
|
// adds to the cache's counters in memory, so it never waits for the
|
||||||
|
// database or fails.
|
||||||
|
func (c *Cache) IncrementStats(hit bool) {
|
||||||
|
if hit {
|
||||||
|
c.cacheHits.Inc()
|
||||||
|
} else {
|
||||||
|
c.cacheMisses.Inc()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// IncrementSourceStats counts whether a variant being made or checked found
|
||||||
|
// its source in the cache. Like IncrementStats, it never waits or fails.
|
||||||
|
func (c *Cache) IncrementSourceStats(hit bool) {
|
||||||
|
if hit {
|
||||||
|
c.sourceCacheHits.Inc()
|
||||||
|
} else {
|
||||||
|
c.sourceCacheMisses.Inc()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// countEviction counts one source or variant of sizeBytes bytes removed by
|
||||||
|
// eviction.
|
||||||
|
func (c *Cache) countEviction(sizeBytes int64) {
|
||||||
|
c.evictions.Inc()
|
||||||
|
c.evictedBytes.Add(float64(sizeBytes))
|
||||||
|
}
|
||||||
@@ -0,0 +1,137 @@
|
|||||||
|
package imgcache
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
|
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||||
|
)
|
||||||
|
|
||||||
|
// gatheredValue returns the value of the metric name on registry: a
|
||||||
|
// counter's or a gauge's value.
|
||||||
|
func gatheredValue(t *testing.T, registry *prometheus.Registry, name string) float64 {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
families, err := registry.Gather()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to gather the metrics: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, family := range families {
|
||||||
|
if family.GetName() != name {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
metric := family.GetMetric()[0]
|
||||||
|
if metric.GetGauge() != nil {
|
||||||
|
return metric.GetGauge().GetValue()
|
||||||
|
}
|
||||||
|
|
||||||
|
return metric.GetCounter().GetValue()
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Fatalf("the metric %s is not on the registry", name)
|
||||||
|
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestEvictToLimit_CountsEvictionsAndUsage stores four variants of 1000
|
||||||
|
// bytes in a cache limited to 3000 bytes. EvictToLimit must count the one
|
||||||
|
// eviction and its bytes, and set the usage and limit gauges.
|
||||||
|
func TestEvictToLimit_CountsEvictionsAndUsage(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const limit = 3000
|
||||||
|
|
||||||
|
registry := prometheus.NewRegistry()
|
||||||
|
|
||||||
|
cache, err := NewCache(evictionTestDB(t),
|
||||||
|
Config{StateDir: t.TempDir(), MaxBytes: limit}, registry)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewCache() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
keys := []imagerequest.VariantKey{
|
||||||
|
testVariantKeyOne, testVariantKeyTwo,
|
||||||
|
testVariantKeyThree, testVariantKeyFour,
|
||||||
|
}
|
||||||
|
|
||||||
|
for i, key := range keys {
|
||||||
|
storeEvictionTestVariant(t, cache, key, bytes.Repeat([]byte{0x01}, 1000))
|
||||||
|
setVariantLastAccessed(t, cache, key,
|
||||||
|
time.Now().Add(-time.Duration(len(keys)-i)*time.Hour))
|
||||||
|
}
|
||||||
|
|
||||||
|
err = cache.EvictToLimit(t.Context())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("EvictToLimit() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
want := map[string]float64{
|
||||||
|
"pixa_cache_evictions_total": 1,
|
||||||
|
"pixa_cache_evicted_bytes_total": 1000,
|
||||||
|
"pixa_cache_usage_bytes": 3000,
|
||||||
|
"pixa_cache_max_bytes": limit,
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, value := range want {
|
||||||
|
got := gatheredValue(t, registry, name)
|
||||||
|
t.Logf("%s = %v", name, got)
|
||||||
|
|
||||||
|
if got != value {
|
||||||
|
t.Errorf("%s = %v, want %v", name, got, value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestEvictNotCheckedRecently_CountsEvictions stores a source of 6 bytes and
|
||||||
|
// a variant of 7 bytes last checked with upstream 31 days ago.
|
||||||
|
// EvictNotCheckedRecently must count both evictions and their 13 bytes.
|
||||||
|
func TestEvictNotCheckedRecently_CountsEvictions(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
registry := prometheus.NewRegistry()
|
||||||
|
|
||||||
|
cache, err := NewCache(evictionTestDB(t),
|
||||||
|
Config{StateDir: t.TempDir(), MaxBytes: 1 << 30}, registry)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewCache() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
old := time.Now().Add(-31 * 24 * time.Hour)
|
||||||
|
|
||||||
|
_, err = cache.PutSource(t.Context(),
|
||||||
|
imagerequest.Source{Host: testHostCDN, Path: "/old.jpg"},
|
||||||
|
bytes.NewReader([]byte("source")),
|
||||||
|
SourceInfo{FetchedAt: old, CheckedAt: old})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("PutSource() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = cache.PutVariant(t.Context(), testVariantKeyOne,
|
||||||
|
SourceInfo{CheckedAt: old}, bytes.NewReader([]byte("variant")))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("PutVariant() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = cache.EvictNotCheckedRecently(t.Context())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("EvictNotCheckedRecently() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
want := map[string]float64{
|
||||||
|
"pixa_cache_evictions_total": 2,
|
||||||
|
"pixa_cache_evicted_bytes_total": 13,
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, value := range want {
|
||||||
|
got := gatheredValue(t, registry, name)
|
||||||
|
t.Logf("%s = %v", name, got)
|
||||||
|
|
||||||
|
if got != value {
|
||||||
|
t.Errorf("%s = %v, want %v", name, got, value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -216,8 +216,6 @@ func TestSource_RefusesFileOfNoSourceFormat(t *testing.T) {
|
|||||||
wantErr error
|
wantErr error
|
||||||
}{
|
}{
|
||||||
{"text", []byte("this is text, not an image"), magic.ErrUnknownFormat},
|
{"text", []byte("this is text, not an image"), magic.ErrUnknownFormat},
|
||||||
{"SVG", []byte(`<svg xmlns="http://www.w3.org/2000/svg"></svg>`),
|
|
||||||
magic.ErrUnknownFormat},
|
|
||||||
{"JPEG cut to 10 bytes", jpegImage[:10], io.EOF},
|
{"JPEG cut to 10 bytes", jpegImage[:10], io.EOF},
|
||||||
{"empty file", nil, io.EOF},
|
{"empty file", nil, io.EOF},
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,54 @@
|
|||||||
|
package imgcache
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||||
|
)
|
||||||
|
|
||||||
|
// testSVG is an SVG with a comment before its root element, so that its
|
||||||
|
// first magic.MinMagicBytes bytes do not show its format.
|
||||||
|
const testSVG = `<!-- a drawing --><svg xmlns="http://www.w3.org/2000/svg" ` +
|
||||||
|
`width="4" height="4"/>`
|
||||||
|
|
||||||
|
// TestSourceAndVariant_ReadSVG checks that a cached SVG source is read as
|
||||||
|
// SVG, and a cached SVG variant (orig of an SVG source) as image/svg+xml.
|
||||||
|
func TestSourceAndVariant_ReadSVG(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
cache, _ := newEvictionTestCache(t, 1<<30)
|
||||||
|
|
||||||
|
storeEvictionTestSource(t, cache, testHostCDN, "/drawing.svg", []byte(testSVG))
|
||||||
|
|
||||||
|
reader, _, format, err := cache.Source(t.Context(),
|
||||||
|
imagerequest.Source{Host: testHostCDN, Path: "/drawing.svg"})
|
||||||
|
t.Logf("Source() format = %q, error = %v", format, err)
|
||||||
|
|
||||||
|
if err != nil || format != imagerequest.FormatSVG {
|
||||||
|
t.Errorf("Source() = format %q, error %v; want %q", format, err,
|
||||||
|
imagerequest.FormatSVG)
|
||||||
|
}
|
||||||
|
|
||||||
|
if reader != nil {
|
||||||
|
_ = reader.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
key := imagerequest.VariantKey("aabbccdd3001")
|
||||||
|
|
||||||
|
err = cache.variants.Write(string(key), []byte(testSVG))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("storing the variant file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
variant, err := cache.Variant(key)
|
||||||
|
t.Logf("Variant() content type = %q, error = %v", variant.ContentType, err)
|
||||||
|
|
||||||
|
if err != nil || variant.ContentType != "image/svg+xml" {
|
||||||
|
t.Errorf("Variant() = content type %q, error %v; want image/svg+xml",
|
||||||
|
variant.ContentType, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if variant.Content != nil {
|
||||||
|
_ = variant.Content.Close()
|
||||||
|
}
|
||||||
|
}
|
||||||
+27
-1
@@ -20,6 +20,10 @@ var (
|
|||||||
// MinMagicBytes is the minimum number of bytes needed to detect format.
|
// MinMagicBytes is the minimum number of bytes needed to detect format.
|
||||||
const MinMagicBytes = 12
|
const MinMagicBytes = 12
|
||||||
|
|
||||||
|
// MaxMagicBytes is the most bytes DetectFormat looks at. An SVG's root
|
||||||
|
// element must start within them, as libvips requires.
|
||||||
|
const MaxMagicBytes = 1024
|
||||||
|
|
||||||
// Magic byte signatures for supported formats.
|
// Magic byte signatures for supported formats.
|
||||||
// These are effectively constants but Go doesn't support const slices.
|
// These are effectively constants but Go doesn't support const slices.
|
||||||
//
|
//
|
||||||
@@ -39,6 +43,12 @@ var (
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// utf8BOM is the UTF-8 byte order mark, which a text file such as an SVG may
|
||||||
|
// start with.
|
||||||
|
//
|
||||||
|
//nolint:gochecknoglobals // immutable lookup data
|
||||||
|
var utf8BOM = []byte{0xEF, 0xBB, 0xBF}
|
||||||
|
|
||||||
// WebP identifier appears at offset 8 after RIFF header.
|
// WebP identifier appears at offset 8 after RIFF header.
|
||||||
//
|
//
|
||||||
//nolint:gochecknoglobals // immutable lookup data
|
//nolint:gochecknoglobals // immutable lookup data
|
||||||
@@ -53,7 +63,7 @@ var (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// DetectFormat detects the image format from magic bytes. Data in none of
|
// DetectFormat detects the image format from magic bytes. Data in none of
|
||||||
// the formats pixa reads, SVG included, is ErrUnknownFormat.
|
// the formats pixa reads is ErrUnknownFormat.
|
||||||
func DetectFormat(data []byte) (imagerequest.Format, error) {
|
func DetectFormat(data []byte) (imagerequest.Format, error) {
|
||||||
if len(data) < MinMagicBytes {
|
if len(data) < MinMagicBytes {
|
||||||
return "", ErrNotEnoughData
|
return "", ErrNotEnoughData
|
||||||
@@ -96,9 +106,25 @@ func DetectFormat(data []byte) (imagerequest.Format, error) {
|
|||||||
return imagerequest.FormatJXL, nil
|
return imagerequest.FormatJXL, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if isSVG(data) {
|
||||||
|
return imagerequest.FormatSVG, nil
|
||||||
|
}
|
||||||
|
|
||||||
return "", ErrUnknownFormat
|
return "", ErrUnknownFormat
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// isSVG reports whether data starts as an SVG does: with a "<", after an
|
||||||
|
// optional UTF-8 byte order mark and white space, and with "<svg" within the
|
||||||
|
// first MaxMagicBytes bytes. Before the root element an SVG may have an XML
|
||||||
|
// declaration, comments and a DOCTYPE.
|
||||||
|
func isSVG(data []byte) bool {
|
||||||
|
data = data[:min(len(data), MaxMagicBytes)]
|
||||||
|
text := bytes.TrimLeft(bytes.TrimPrefix(data, utf8BOM), " \t\r\n")
|
||||||
|
|
||||||
|
return bytes.HasPrefix(text, []byte("<")) &&
|
||||||
|
bytes.Contains(text, []byte("<svg"))
|
||||||
|
}
|
||||||
|
|
||||||
// ValidateMagicBytes detects the format of data from its magic bytes, and
|
// ValidateMagicBytes detects the format of data from its magic bytes, and
|
||||||
// returns it when it is the format the declared MIME type names. Data in
|
// returns it when it is the format the declared MIME type names. Data in
|
||||||
// another format is ErrMagicByteMismatch.
|
// another format is ErrMagicByteMismatch.
|
||||||
|
|||||||
@@ -47,9 +47,9 @@ func TestDetectFormat(t *testing.T) {
|
|||||||
{name: "AVIF", data: avif, wantFormat: imagerequest.FormatAVIF},
|
{name: "AVIF", data: avif, wantFormat: imagerequest.FormatAVIF},
|
||||||
{name: "AVIF sequence", data: avis, wantFormat: imagerequest.FormatAVIF},
|
{name: "AVIF sequence", data: avis, wantFormat: imagerequest.FormatAVIF},
|
||||||
{
|
{
|
||||||
name: "SVG",
|
name: "SVG",
|
||||||
data: []byte(`<svg xmlns="http://www.w3.org/2000/svg"></svg>`),
|
data: []byte(`<svg xmlns="http://www.w3.org/2000/svg"></svg>`),
|
||||||
wantErr: ErrUnknownFormat,
|
wantFormat: imagerequest.FormatSVG,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "unknown format",
|
name: "unknown format",
|
||||||
|
|||||||
@@ -0,0 +1,85 @@
|
|||||||
|
package magic
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestDetectFormat_SVG checks that DetectFormat names SVG text with what may
|
||||||
|
// come before the root element, and does not name as SVG text that does not
|
||||||
|
// start with "<" or has no "<svg" within its first MaxMagicBytes bytes.
|
||||||
|
func TestDetectFormat_SVG(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const root = `<svg xmlns="http://www.w3.org/2000/svg" width="4" height="4"/>`
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
data string
|
||||||
|
wantErr error
|
||||||
|
}{
|
||||||
|
{name: "root element only", data: root},
|
||||||
|
{name: "XML declaration", data: `<?xml version="1.0"?>` + "\n" + root},
|
||||||
|
{name: "byte order mark and white space", data: "\xEF\xBB\xBF \n\t" + root},
|
||||||
|
{name: "comment", data: "<!-- a drawing -->" + root},
|
||||||
|
{
|
||||||
|
name: "DOCTYPE",
|
||||||
|
data: `<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" ` +
|
||||||
|
`"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">` + root,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "root element after the first bytes",
|
||||||
|
data: "<!--" + strings.Repeat(" ", MaxMagicBytes) + "-->" + root,
|
||||||
|
wantErr: ErrUnknownFormat,
|
||||||
|
},
|
||||||
|
{name: "text naming <svg", data: "draw it with " + root, wantErr: ErrUnknownFormat},
|
||||||
|
{name: "HTML", data: "<!DOCTYPE html><html></html>", wantErr: ErrUnknownFormat},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
got, err := DetectFormat([]byte(tt.data))
|
||||||
|
t.Logf("DetectFormat() = %q, %v", got, err)
|
||||||
|
|
||||||
|
if tt.wantErr != nil {
|
||||||
|
if !errors.Is(err, tt.wantErr) {
|
||||||
|
t.Errorf("DetectFormat() error = %v, want %v", err, tt.wantErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil || got != imagerequest.FormatSVG {
|
||||||
|
t.Errorf("DetectFormat() = %q, %v; want %q", got, err,
|
||||||
|
imagerequest.FormatSVG)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestValidateMagicBytes_SVG checks that SVG data is SVG for the
|
||||||
|
// Content-Type image/svg+xml, with a charset too, and a mismatch for another.
|
||||||
|
func TestValidateMagicBytes_SVG(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
data := []byte(`<svg xmlns="http://www.w3.org/2000/svg" width="4" height="4"/>`)
|
||||||
|
|
||||||
|
for _, declared := range []string{"image/svg+xml", "image/svg+xml; charset=utf-8"} {
|
||||||
|
got, err := ValidateMagicBytes(data, declared)
|
||||||
|
if err != nil || got != imagerequest.FormatSVG {
|
||||||
|
t.Errorf("ValidateMagicBytes(%q) = %q, %v; want %q", declared, got,
|
||||||
|
err, imagerequest.FormatSVG)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := ValidateMagicBytes(data, "image/png")
|
||||||
|
if !errors.Is(err, ErrMagicByteMismatch) {
|
||||||
|
t.Errorf("ValidateMagicBytes(image/png) error = %v, want %v", err,
|
||||||
|
ErrMagicByteMismatch)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -35,7 +35,7 @@ import (
|
|||||||
|
|
||||||
// Constructors returns the constructors of the parts this package builds,
|
// Constructors returns the constructors of the parts this package builds,
|
||||||
// for fx.Provide. The fx graph must also provide the prometheus.Registerer
|
// for fx.Provide. The fx graph must also provide the prometheus.Registerer
|
||||||
// the cache and the image service register their counters on.
|
// the cache and the image service register their metrics on.
|
||||||
func Constructors() []any {
|
func Constructors() []any {
|
||||||
return []any{
|
return []any{
|
||||||
newKeys,
|
newKeys,
|
||||||
|
|||||||
@@ -4,38 +4,134 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"sneak.berlin/go/pixa/internal/imageprocessor"
|
||||||
|
"sneak.berlin/go/pixa/internal/imageproxy"
|
||||||
|
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||||
|
"sneak.berlin/go/pixa/internal/magic"
|
||||||
|
"sneak.berlin/go/pixa/internal/middleware"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TestImageProxyRefusesSVG requests an image the upstream answers as SVG,
|
// TestMain runs this test binary as an SVG child process when the image
|
||||||
// in its own format (orig) and resized as PNG, through pixa's router,
|
// processor starts it again as one, as the SVG tests below make it do.
|
||||||
// upstream fetcher and image processor. pixa does not accept SVG as a
|
func TestMain(m *testing.M) {
|
||||||
// source, so each request is answered 415, never with an image.
|
if imageprocessor.IsSVGChildProcess(os.Args[1:]) {
|
||||||
func TestImageProxyRefusesSVG(t *testing.T) {
|
os.Exit(imageprocessor.RunAsSVGChildProcess(os.Args[1:]))
|
||||||
t.Parallel()
|
}
|
||||||
|
|
||||||
|
os.Exit(m.Run())
|
||||||
|
}
|
||||||
|
|
||||||
|
// testSVG is the SVG the upstream serves in the SVG tests below.
|
||||||
|
const testSVG = `<svg xmlns="http://www.w3.org/2000/svg" width="64" height="48">` +
|
||||||
|
`<rect width="64" height="48"/></svg>`
|
||||||
|
|
||||||
|
// startSVGUpstream starts an upstream that answers every request with
|
||||||
|
// testSVG as image/svg+xml, and returns it with its count of requests.
|
||||||
|
func startSVGUpstream(t *testing.T) (*httptest.Server, *atomic.Int64) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
var requests atomic.Int64
|
||||||
|
|
||||||
upstream := httptest.NewServer(http.HandlerFunc(
|
upstream := httptest.NewServer(http.HandlerFunc(
|
||||||
func(w http.ResponseWriter, _ *http.Request) {
|
func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
requests.Add(1)
|
||||||
w.Header().Set("Content-Type", "image/svg+xml")
|
w.Header().Set("Content-Type", "image/svg+xml")
|
||||||
_, _ = io.WriteString(w, `<svg xmlns="http://www.w3.org/2000/svg" `+
|
_, _ = io.WriteString(w, testSVG)
|
||||||
`width="64" height="48"><rect width="64" height="48"/></svg>`)
|
|
||||||
}))
|
}))
|
||||||
t.Cleanup(upstream.Close)
|
t.Cleanup(upstream.Close)
|
||||||
|
|
||||||
|
return upstream, &requests
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestImageProxyServesSVGAsItIs requests an SVG source in its own format
|
||||||
|
// (orig) at the orig size through pixa's router, upstream fetcher, image
|
||||||
|
// processor and cache, twice. Both answers are the SVG itself, never another
|
||||||
|
// format, the second from the cache; the upstream is asked once. Both carry
|
||||||
|
// the Content-Security-Policy of an SVG response in place of the
|
||||||
|
// middleware's. orig at another size is refused with 400.
|
||||||
|
func TestImageProxyServesSVGAsItIs(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const wantPolicy = "default-src 'none'; style-src 'unsafe-inline'; " +
|
||||||
|
"img-src data:; frame-ancestors 'none'; sandbox"
|
||||||
|
|
||||||
|
upstream, requests := startSVGUpstream(t)
|
||||||
|
s, _, _ := startImageProxy(t, upstream)
|
||||||
|
target := "/v1/image/" + upstreamHost + "/drawing.svg/orig.orig"
|
||||||
|
|
||||||
|
for _, wantCache := range []imageproxy.CacheStatus{
|
||||||
|
imageproxy.CacheMiss, imageproxy.CacheHit,
|
||||||
|
} {
|
||||||
|
rec := getImage(t, s, target)
|
||||||
|
|
||||||
|
if rec.Header().Get("Content-Type") != "image/svg+xml" ||
|
||||||
|
rec.Body.String() != testSVG {
|
||||||
|
t.Errorf("GET %s: %s %q, want image/svg+xml %q", target,
|
||||||
|
rec.Header().Get("Content-Type"), rec.Body.String(), testSVG)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := rec.Header().Get("X-Pixa-Cache"); got != string(wantCache) {
|
||||||
|
t.Errorf("GET %s: X-Pixa-Cache = %q, want %q", target, got, wantCache)
|
||||||
|
}
|
||||||
|
|
||||||
|
policy := rec.Header().Get("Content-Security-Policy")
|
||||||
|
if policy != wantPolicy {
|
||||||
|
t.Errorf("GET %s: Content-Security-Policy = %q, want %q", target,
|
||||||
|
policy, wantPolicy)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if requests.Load() != 1 {
|
||||||
|
t.Errorf("upstream requests = %d, want 1", requests.Load())
|
||||||
|
}
|
||||||
|
|
||||||
|
resized := "/v1/image/" + upstreamHost + "/drawing.svg/32x24.orig"
|
||||||
|
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
s.ServeHTTP(rec, httptest.NewRequestWithContext(
|
||||||
|
t.Context(), http.MethodGet, resized, nil))
|
||||||
|
t.Logf("GET %s: %d %s", resized, rec.Code, rec.Body.String())
|
||||||
|
|
||||||
|
if rec.Code != http.StatusBadRequest {
|
||||||
|
t.Errorf("GET %s: %d %s, want 400", resized, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestImageProxyRendersSVG requests an SVG source as each raster format
|
||||||
|
// through pixa's router, upstream fetcher and image processor. Each answer
|
||||||
|
// is an image in that format, with the middleware's Content-Security-Policy.
|
||||||
|
func TestImageProxyRendersSVG(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
upstream, _ := startSVGUpstream(t)
|
||||||
s, _, _ := startImageProxy(t, upstream)
|
s, _, _ := startImageProxy(t, upstream)
|
||||||
|
|
||||||
for _, sizeFormat := range []string{"64x48.orig", "32x24.png"} {
|
for _, format := range []imagerequest.Format{
|
||||||
target := "/v1/image/" + upstreamHost + "/drawing.svg/" + sizeFormat
|
imagerequest.FormatJPEG, imagerequest.FormatPNG, imagerequest.FormatGIF,
|
||||||
|
imagerequest.FormatWebP, imagerequest.FormatAVIF, imagerequest.FormatJXL,
|
||||||
|
} {
|
||||||
|
target := "/v1/image/" + upstreamHost + "/drawing.svg/32x24." +
|
||||||
|
string(format)
|
||||||
|
|
||||||
rec := httptest.NewRecorder()
|
rec := getImage(t, s, target)
|
||||||
s.ServeHTTP(rec, httptest.NewRequestWithContext(
|
detected, err := magic.DetectFormat(rec.Body.Bytes())
|
||||||
t.Context(), http.MethodGet, target, nil))
|
t.Logf("GET %s: %s, body in %q (%v)", target,
|
||||||
t.Logf("GET %s: %d, Content-Type %s, body %s",
|
rec.Header().Get("Content-Type"), detected, err)
|
||||||
target, rec.Code, rec.Header().Get("Content-Type"), rec.Body.String())
|
|
||||||
|
|
||||||
if rec.Code != http.StatusUnsupportedMediaType {
|
if rec.Header().Get("Content-Type") != format.MIMEType() ||
|
||||||
t.Errorf("GET %s status = %d, want %d",
|
err != nil || detected != format {
|
||||||
target, rec.Code, http.StatusUnsupportedMediaType)
|
t.Errorf("GET %s: %s, body in %q (%v); want %s", target,
|
||||||
|
rec.Header().Get("Content-Type"), detected, err, format.MIMEType())
|
||||||
|
}
|
||||||
|
|
||||||
|
policy := rec.Header().Get("Content-Security-Policy")
|
||||||
|
if policy != middleware.ContentSecurityPolicyValue {
|
||||||
|
t.Errorf("GET %s: Content-Security-Policy = %q, want %q", target,
|
||||||
|
policy, middleware.ContentSecurityPolicyValue)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
|
<title>Pixa - Form Refused</title>
|
||||||
|
<link rel="stylesheet" href="/static/style.css">
|
||||||
|
</head>
|
||||||
|
<body class="login">
|
||||||
|
<div class="card">
|
||||||
|
<h1>Form Refused</h1>
|
||||||
|
|
||||||
|
<div class="error">
|
||||||
|
{{.Cause}}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<p>
|
||||||
|
<a href="/">Back to the start page</a>
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
Reference in new issue
Block a user