Compare commits
6
Commits
0dfb2e9651
...
next
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bdde021b45 | ||
|
|
db91ab29e6 | ||
|
|
8597253ffd | ||
|
|
99e4aa3bb2 | ||
|
|
d2944aa891 | ||
|
|
54328377d0 |
+15
-10
@@ -17,13 +17,14 @@ COPY . .
|
||||
RUN golangci-lint run --config .golangci.yml ./...
|
||||
|
||||
# Test phase. script/test builds it alone.
|
||||
# golang:1.25.4-alpine, 2026-02-25
|
||||
FROM golang:1.25.4-alpine@sha256:d3f0cf7723f3429e3f9ed846243970b20a2de7bae6a5b66fc5914e228d831bbb AS test
|
||||
# golang:1.25.4-alpine3.22, 2026-02-25; the runtime stage uses Alpine 3.22 too
|
||||
FROM golang:1.25.4-alpine3.22@sha256:d3f0cf7723f3429e3f9ed846243970b20a2de7bae6a5b66fc5914e228d831bbb AS test
|
||||
|
||||
WORKDIR /src
|
||||
|
||||
# script/bootstrap --cgo installs the build dependencies (a C compiler
|
||||
# and the libvips and libheif headers) and downloads the Go modules.
|
||||
# script/bootstrap --cgo installs the build dependencies (a C compiler,
|
||||
# the libvips and libheif headers, and libvips' JPEG XL support, which
|
||||
# the tests need) and downloads the Go modules.
|
||||
COPY script/ ./script/
|
||||
COPY go.mod go.sum ./
|
||||
RUN script/bootstrap --cgo
|
||||
@@ -39,8 +40,8 @@ RUN go test -count=1 -timeout 90s -race -cover ./... || \
|
||||
# Build stage. Nothing is wanted from the two phases above: these copies
|
||||
# make BuildKit build them first, so this stage runs only when lint and
|
||||
# test passed.
|
||||
# golang:1.25.4-alpine, 2026-02-25
|
||||
FROM golang:1.25.4-alpine@sha256:d3f0cf7723f3429e3f9ed846243970b20a2de7bae6a5b66fc5914e228d831bbb AS builder
|
||||
# golang:1.25.4-alpine3.22, 2026-02-25; the runtime stage uses Alpine 3.22 too
|
||||
FROM golang:1.25.4-alpine3.22@sha256:d3f0cf7723f3429e3f9ed846243970b20a2de7bae6a5b66fc5914e228d831bbb AS builder
|
||||
|
||||
COPY --from=lint /src/go.sum /dev/null
|
||||
COPY --from=test /src/go.sum /dev/null
|
||||
@@ -76,13 +77,17 @@ RUN version="${VERSION:-$(git describe --tags --always)}"; \
|
||||
-o /pixad ./cmd/pixad
|
||||
|
||||
# Runtime stage, and the last one: a plain `docker build .` builds this
|
||||
# stage and what it depends on, and nothing else.
|
||||
# alpine:3.21, 2026-02-25
|
||||
FROM alpine:3.21@sha256:c3f8e73fdb79deaebaa2037150150191b9dcbfba68b4a46d70103204c53f4709
|
||||
# stage and what it depends on, and nothing else. It must use the Alpine
|
||||
# release the golang image above is based on, so that pixad runs against
|
||||
# the libvips and musl it was built with.
|
||||
# alpine:3.22, 2026-10-08
|
||||
FROM alpine:3.22@sha256:5291449c3df73caf6ed85e649dec1b9e818b39a5d8c871e97afc13e9cd5e8fa8
|
||||
|
||||
# Install runtime dependencies only
|
||||
# Install runtime dependencies only. vips-jxl is libvips' JPEG XL
|
||||
# support, without which pixad does not start.
|
||||
RUN apk add --no-cache \
|
||||
vips \
|
||||
vips-jxl \
|
||||
libheif \
|
||||
ca-certificates \
|
||||
tzdata \
|
||||
|
||||
@@ -88,8 +88,11 @@ or with 1 when images were still being processed after those 5 seconds or
|
||||
another part of pixa failed to stop. A request not finished by then is cut off.
|
||||
`docker stop` waits 10 seconds before it kills the container.
|
||||
|
||||
Outside Docker, pixa needs libvips (the image has 8.15) and libheif to run, as
|
||||
it uses libvips through CGO; building it also needs their development files,
|
||||
Outside Docker, pixa needs libvips (the image has 8.16) and libheif to run, as
|
||||
it uses libvips through CGO. pixad does not start unless libvips has its JPEG XL
|
||||
support, which on Alpine is the `vips-jxl` package and which the nix and brew
|
||||
packages of libvips include, as do the apt ones from Debian 12 and Ubuntu 24.04
|
||||
on. Building pixa also needs the development files of libvips and libheif,
|
||||
`pkg-config` and a C compiler. `script/bootstrap --cgo` installs all of these,
|
||||
as the `Dockerfile` does where it compiles pixa. Plain `script/bootstrap`, which
|
||||
`script/setup` and `script/cibuild` run, installs git, make and Go, and Node,
|
||||
@@ -172,20 +175,21 @@ path under `/v1/` answers 200, in maintenance mode too.
|
||||
with the page naming a field that is not valid; 500 when the URL cannot be
|
||||
made.
|
||||
- `GET /logout` — end the login session. Needs: nothing. Answers: 303 to `/`.
|
||||
- `GET` or `HEAD` `/v1/image/<host>/<path>/<size>.<format>` — an image, fetched,
|
||||
resized and converted (below). Needs: a signature, unless the host is
|
||||
allowlisted (see Source Hosts). Answers: 200; 304 when `If-None-Match` matches
|
||||
the image's `ETag`; 400 for a URL or parameter that is not valid, or for the
|
||||
format `auto` an `Accept` header that is not valid; 406 for the format `auto`
|
||||
when `Accept` allows none of the formats it chooses from; 401 for a missing or
|
||||
wrong signature, a missing `exp` or an `exp` in the past; 403 when the
|
||||
request's `Referer` names a host in `referer_blocklist`, checked before the
|
||||
signature, the cache and the upstream fetch; 403 when the upstream host, or a
|
||||
host it redirects to, is `localhost`, ends in `.localhost` or `.local`, or has
|
||||
an address in a blocked network (see `blocked_networks`); 502 when the
|
||||
upstream answered with an error status, and for 5 minutes after that for the
|
||||
same source URL; 503 when pixa is busy or in maintenance mode; 500 for any
|
||||
other failure.
|
||||
- `GET` or `HEAD` `/v1/image/<host>/<path>/<size>.<format>`, or
|
||||
`/v1/image/<host>/<path>/<size>` with no format — an image, fetched, resized
|
||||
and converted (below). Needs: a signature, unless the host is allowlisted (see
|
||||
Source Hosts). Answers: 200; 304 when `If-None-Match` matches the image's
|
||||
`ETag`; 400 for a URL or parameter that is not valid, or for the format `auto`
|
||||
an `Accept` header that is not valid; 406 for the format `auto` when `Accept`
|
||||
allows none of the formats it chooses from; 401 for a missing or wrong
|
||||
signature, a missing `exp` or an `exp` in the past; 403 when the request's
|
||||
`Referer` names a host in `referer_blocklist`, checked before the signature,
|
||||
the cache and the upstream fetch; 403 when the upstream host, or a host it
|
||||
redirects to, is `localhost`, ends in `.localhost` or `.local`, or has an
|
||||
address in a blocked network (see `blocked_networks`); 502 when the upstream
|
||||
answered with an error status, and for 5 minutes after that for the same
|
||||
source URL; 503 when pixa is busy or in maintenance mode; 500 for any other
|
||||
failure.
|
||||
- `GET` or `HEAD` `/v1/e/<token>/<name>` — an image through an encrypted URL
|
||||
(see Encrypted URLs). Needs: nothing but the URL. Answers: 200; 304 when
|
||||
`If-None-Match` matches the image's `ETag`; 400 for a token that does not
|
||||
@@ -228,10 +232,11 @@ proxy in front of pixa must pass that header on unchanged. A form body over 1
|
||||
MiB is refused with 413. The image routes answer the errors listed for them with
|
||||
JSON holding `error`, `status` and `timestamp`.
|
||||
|
||||
An image URL has this form:
|
||||
An image URL has one of these forms, the second with no format:
|
||||
|
||||
```
|
||||
/v1/image/<host>/<path>/<size>.<format>?sig=<signature>&exp=<expiration>&q=<quality>&fit=<fit>
|
||||
/v1/image/<host>/<path>/<size>?sig=<signature>&exp=<expiration>&q=<quality>&fit=<fit>
|
||||
```
|
||||
|
||||
Images are only fetched from origins using TLS with valid certificates, unless
|
||||
@@ -242,7 +247,9 @@ A request whose query string cannot be decoded, or gives any parameter more than
|
||||
once, is refused with 400.
|
||||
|
||||
- `<format>`: one of `orig` (or `original`), `jpeg` (or `jpg`), `png`, `webp`,
|
||||
`avif`, `gif`, or `auto` (below)
|
||||
`avif`, `jxl` (JPEG XL), `gif`, or `auto` (below). A URL with no format (the
|
||||
second form, with no dot after the size) is served as JPEG XL, the default, as
|
||||
with `jxl`
|
||||
- `<size>`: `orig` or `<width>x<height>` (e.g. `800x600`)
|
||||
- `sig` and `exp`: the signature and its expiry, needed unless the host is
|
||||
allowlisted (see Signature Specification)
|
||||
@@ -250,23 +257,29 @@ once, is refused with 400.
|
||||
both optional (values under Signature Specification). Both are part of what is
|
||||
cached, so each value of either is a separate cached image.
|
||||
|
||||
The source image may be JPEG, PNG, GIF, WebP, AVIF, JPEG XL or SVG. The
|
||||
upstream's `Content-Type` must name its format, and the image's first bytes must
|
||||
match it.
|
||||
|
||||
With the format `auto`, pixa chooses the format for each request from its
|
||||
`Accept` header, in this order:
|
||||
|
||||
1. AVIF, when the header names `image/avif`;
|
||||
2. WebP, when it names `image/webp`;
|
||||
3. JPEG, when the first of `image/jpeg`, `image/*` and `*/*` that it names
|
||||
1. JPEG XL, when the header names `image/jxl`;
|
||||
2. AVIF, when it names `image/avif`;
|
||||
3. WebP, when it names `image/webp`;
|
||||
4. JPEG, when the first of `image/jpeg`, `image/*` and `*/*` that it names
|
||||
allows it, or when there is no `Accept` header or it is empty.
|
||||
|
||||
An entry with `q=0` refuses its format; other `q` values do not change the
|
||||
order. AVIF and WebP must be named, as clients that cannot show them also send
|
||||
`image/*` and `*/*`. pixa never sends a format the client refused: when the
|
||||
header allows none of the three, the answer is 406, and a header that does not
|
||||
parse, or has a `q` that is not a number from 0 to 1, is refused with 400. The
|
||||
signature, or the token of an encrypted URL, covers `auto` itself, so one URL
|
||||
serves every client. Each format chosen is cached as a separate image, and every
|
||||
answer that depends on `Accept` (the image, a 304, and the 400 and 406 above)
|
||||
carries `Vary: Accept`, so a shared cache keeps the formats apart too.
|
||||
order. JPEG XL, AVIF and WebP must be named, as clients that cannot show them
|
||||
also send `image/*` and `*/*`. pixa never sends a format the client refused:
|
||||
when the header allows none of the four, the answer is 406, and a header that
|
||||
does not parse, or has a `q` that is not a number from 0 to 1, is refused
|
||||
with 400. The signature, or the token of an encrypted URL, covers `auto` itself,
|
||||
so one URL serves every client. Each format chosen is cached as a separate
|
||||
image, and every answer that depends on `Accept` (the image, a 304, and the 400
|
||||
and 406 above) carries `Vary: Accept`, so a shared cache keeps the formats apart
|
||||
too.
|
||||
|
||||
An image is served with `Cache-Control: public, max-age=<seconds>, immutable`.
|
||||
When the URL has an expiry (an `exp`, or the TTL of an encrypted URL), `max-age`
|
||||
@@ -312,13 +325,15 @@ nor change what it asks for.
|
||||
lasts 30 days, or until `/logout`.
|
||||
2. On the generator page, give the source image's URL, the width and height, the
|
||||
format, quality and fit, and how long the URL lasts, then submit the form
|
||||
(`POST /generate`). Width and height both empty or `0` keep the original
|
||||
size; if only one of them is empty or `0`, that side is scaled to keep the
|
||||
image's proportions.
|
||||
(`POST /generate`). The format is JPEG XL unless another is chosen; a form
|
||||
sent with an empty format, or none, also makes a JPEG XL URL. Width and
|
||||
height both empty or `0` keep the original size; if only one of them is empty
|
||||
or `0`, that side is scaled to keep the image's proportions.
|
||||
3. The page shows the URL, `https://<host>/v1/e/<token>/img.<format>`, and when
|
||||
it expires. `<host>` is the host the page was opened on, and the URL starts
|
||||
with `http` instead while `debug` is on. The name after the token is ignored
|
||||
and only gives the URL a file extension, `jpg` for `orig` and `auto`.
|
||||
and only gives the URL a file extension, `jpg` for `orig` and `auto`, and
|
||||
`jxl` for a form with no format.
|
||||
|
||||
The token holds the source's host, path and query and the size, format, quality,
|
||||
fit and expiry, encrypted with a key derived from `signing_key`. The source
|
||||
@@ -347,6 +362,10 @@ turned off.
|
||||
- An image with an ICC profile is converted to sRGB first, since clients show an
|
||||
image with no profile as sRGB. Colours outside sRGB, such as the most
|
||||
saturated ones in a Display P3 photo, are clipped.
|
||||
- The one exception is JPEG XL with libvips 8.16 and later: libvips then writes
|
||||
an EXIF block of its own into the image, as govips cannot ask libvips to leave
|
||||
it out. It holds the image's size and otherwise fixed values, such as an
|
||||
orientation of 1 and a resolution of 72 dpi, and nothing from the source.
|
||||
|
||||
### Source Hosts
|
||||
|
||||
@@ -374,8 +393,9 @@ Where:
|
||||
- `width` — requested width in pixels, `0` for original
|
||||
- `height` — requested height in pixels, `0` for original
|
||||
- `format` — output format, one of those listed under Routes, with `original`
|
||||
signed as `orig` and `jpg` as `jpeg`; `auto` is signed as `auto`, not as the
|
||||
format chosen for the request
|
||||
signed as `orig`, `jpg` as `jpeg`, and no format as `jxl`, so a URL with no
|
||||
format has the signature of the same URL ending in `.jxl`; `auto` is signed as
|
||||
`auto`, not as the format chosen for the request
|
||||
- `expiration` — the URL's `exp` query parameter, the Unix timestamp when the
|
||||
signature expires; a request whose `exp` is not a whole number, an empty
|
||||
`exp=` included, is refused with 400
|
||||
@@ -530,12 +550,10 @@ Key settings in more detail:
|
||||
- `db_url` — the SQLite database to open; omitted, it is
|
||||
`file:<state_dir>/state.sqlite3?_pragma=journal_mode(WAL)`, which keeps the
|
||||
database in WAL mode. pixa opens one connection to it, so its own reads and
|
||||
writes run one at a time. Requests wait while eviction runs one of its
|
||||
queries, some of which read a whole table. pixa adds
|
||||
`_pragma=busy_timeout(5000)` to any `db_url`, so a write that finds another
|
||||
program writing to the file waits up to five seconds for it instead of
|
||||
failing. WAL mode comes only from the URL: keep `_pragma=journal_mode(WAL)` in
|
||||
one you set
|
||||
writes run one at a time. pixa adds `_pragma=busy_timeout(5000)` to any
|
||||
`db_url`, so a write that finds another program writing to the file waits up
|
||||
to five seconds for it instead of failing. WAL mode comes only from the URL:
|
||||
keep `_pragma=journal_mode(WAL)` in one you set
|
||||
- `cache_max_bytes` — disk cache size limit in bytes; `0` disables the disk
|
||||
cache entirely; omitted defaults to 75% of the sum of the free space on the
|
||||
filesystem containing `<state_dir>/cache/` and the bytes of source and
|
||||
@@ -585,8 +603,8 @@ provide:
|
||||
|
||||
- `script/bootstrap` — install git, make, Go, Node, Yarn and prettier and
|
||||
download the Go modules (idempotent); with `--cgo`, the C compiler and the
|
||||
libvips and libheif libraries that compiling pixa needs instead of Node, Yarn
|
||||
and prettier
|
||||
libvips (with its JPEG XL support) and libheif libraries that compiling and
|
||||
testing pixa need instead of Node, Yarn and prettier
|
||||
- `script/setup` — make a fresh clone ready for development (bootstrap, then
|
||||
install-precommit)
|
||||
- `script/projectname` — output the project name ("pixa")
|
||||
|
||||
@@ -30,6 +30,67 @@ P2: security: per-IP rate limiting on the image routes
|
||||
|
||||
# Completed Steps
|
||||
|
||||
- 2026-10-08 every output format is saved with settings pixa sets on purpose
|
||||
(closes #232): each format has its own govips export, as JPEG XL does, in
|
||||
place of govips' generic `Export`, which sent libvips a zero for some settings
|
||||
it was not given. PNG gets libvips' default compression, 6, where it had none,
|
||||
and WebP libvips' default effort, 4, where it had 0. GIF was already at
|
||||
libvips' default effort, 7, and JPEG output is unchanged. AVIF is saved at
|
||||
effort 1, the lowest govips can set, where it had libvips' default, 4. With
|
||||
one libvips thread, an 8192x8192 image of random pixels, the worst case, takes
|
||||
about 51 seconds as AVIF at effort 1 and 43 as WebP at effort 4, against the
|
||||
default `downstream_timeout` of 60 seconds. On an image of milder noise, which
|
||||
AVIF at effort 1 saves in about 12 seconds, effort 4 takes minutes. AVIF is
|
||||
also saved with 8 bits per sample from a 16-bit source, which libvips would
|
||||
save with 12: a 16-bit 8192x8192 image of milder noise takes about 54 seconds
|
||||
at effort 1 with 12 bits, nearly all of the default `downstream_timeout`, and
|
||||
about 12 with 8.
|
||||
- 2026-10-08 JPEG XL is the default output (closes #222): a `/v1/image/` URL
|
||||
whose last segment is a size with no format, such as `800x600` or `orig`, is
|
||||
served as JPEG XL and signed as `jxl`, so it has the signature of the same URL
|
||||
ending in `.jxl`. An encrypted URL whose token holds no format is served as
|
||||
JPEG XL (`encurl.DefaultFormat`). The generator page selects JPEG XL until
|
||||
another format is chosen, and a form with an empty format, or none, makes a
|
||||
JPEG XL URL whose name ends in `.jxl`. The image processor no longer takes an
|
||||
empty format as `orig`: both routes give every request a format, and it
|
||||
refuses a request with none. `auto` still ends with JPEG.
|
||||
- 2026-10-08 JPEG XL as an input and output format (part of #222): a source
|
||||
whose bytes start with either JPEG XL signature, the bare codestream's `FF 0A`
|
||||
or the container's, is detected as `image/jxl`, which the upstream fetch
|
||||
accepts; `orig` of such a source is JPEG XL. The format `jxl` works in plain
|
||||
and encrypted URLs and on the generator page, served as `image/jxl` and cached
|
||||
like the other formats. `auto` chooses JPEG XL first when `Accept` names
|
||||
`image/jxl`. govips sends libvips a JPEG XL distance with every save, so
|
||||
libvips ignores the quality: pixa turns `q` into a distance with libvips' own
|
||||
formula, keeping 100 lossy, and removes the metadata from the image before
|
||||
saving, as govips cannot have libvips strip it from JPEG XL. libvips 8.16 and
|
||||
later still write an EXIF block of their own into JPEG XL, from the image's
|
||||
size, orientation and resolution, and fixed values; the image is upright and
|
||||
is given 72 dpi before the save, so the block holds nothing from the source.
|
||||
An image with an ICC profile is converted to sRGB before the JPEG XL save too,
|
||||
and a CMYK image with none is converted to sRGB, as libvips cannot save CMYK
|
||||
as JPEG XL. libvips' default effort, 7, is kept. JPEG XL is not yet the
|
||||
default output.
|
||||
- 2026-10-08 pixad runs on the Alpine release it is built on (closes #229): the
|
||||
runtime stage of the `Dockerfile` uses `alpine:3.22`, the release of the
|
||||
`golang:1.25.4-alpine3.22` image that the test phase and the build stage use,
|
||||
so all three have libvips 8.16, where the runtime image had 8.15.
|
||||
- 2026-10-08 requests no longer wait behind eviction queries that read a whole
|
||||
table (closes #227): the new `cache_usage` table holds the total cache usage,
|
||||
kept up to date by triggers on `source_content` and `variant_content` in the
|
||||
statement that adds, removes or resizes a row, and `UsageBytes` reads it. The
|
||||
reconciliation pass reads the content tables 1000 rows per query, sums them
|
||||
and corrects the total when it differs, unless a row changed while it summed.
|
||||
Source rows get `last_accessed_at` when added, so choosing source images to
|
||||
evict reads that column's index instead of sorting the whole table.
|
||||
- 2026-10-08 libvips' JPEG XL support is installed and required (part of #222):
|
||||
`script/bootstrap --cgo` installs `vips-jxl` when its package manager is apk,
|
||||
as Alpine's `vips` package lacks the support, and the runtime stage of the
|
||||
`Dockerfile` installs it too. `imgcache.NewService` fails, naming the fix,
|
||||
when `imageprocessor.CheckJPEGXLSupport` finds that libvips cannot load and
|
||||
save JPEG XL, so pixad does not start without it. A test saves an image as
|
||||
JPEG XL with govips and loads it back. JPEG XL is not yet a format pixa
|
||||
serves.
|
||||
- 2026-10-08 SQLite writes no longer fail with "database is locked" under load
|
||||
(closes #223): `internal/database` opens the database with one connection, so
|
||||
pixa's own reads and writes run on it one at a time instead of competing for
|
||||
|
||||
@@ -3,14 +3,12 @@
|
||||
|
||||
-- Source content blobs
|
||||
-- Files stored at: cache/sources/<ab>/<cd>/<sha256>
|
||||
-- last_accessed_at is NULL until the first LRU touch; eviction falls
|
||||
-- back to fetched_at for rows that have never been touched.
|
||||
CREATE TABLE IF NOT EXISTS source_content (
|
||||
content_hash TEXT PRIMARY KEY,
|
||||
content_type TEXT NOT NULL,
|
||||
size_bytes INTEGER NOT NULL,
|
||||
fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
last_accessed_at DATETIME
|
||||
last_accessed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_source_content_last_accessed
|
||||
ON source_content(last_accessed_at);
|
||||
@@ -42,9 +40,8 @@ CREATE INDEX IF NOT EXISTS idx_source_meta_content_hash ON source_metadata(conte
|
||||
-- Processed variant blobs
|
||||
-- Files stored at: cache/variants/<ab>/<cd>/<cache_key> (plus a .meta
|
||||
-- sidecar with the content type). Tracked here (like source content
|
||||
-- blobs above) so total cache usage can be computed with a SUM query,
|
||||
-- never a directory scan, and so LRU eviction has a timestamp to order
|
||||
-- on.
|
||||
-- blobs above) so total cache usage is known without a directory scan,
|
||||
-- and so LRU eviction has a timestamp to order on.
|
||||
CREATE TABLE IF NOT EXISTS variant_content (
|
||||
cache_key TEXT PRIMARY KEY,
|
||||
size_bytes INTEGER NOT NULL,
|
||||
@@ -55,6 +52,74 @@ CREATE TABLE IF NOT EXISTS variant_content (
|
||||
CREATE INDEX IF NOT EXISTS idx_variant_content_last_accessed
|
||||
ON variant_content(last_accessed_at);
|
||||
|
||||
-- Total cache usage: the sum of size_bytes over source_content and
|
||||
-- variant_content, kept by the triggers below in the same statement
|
||||
-- that adds, removes or resizes a row, so eviction reads this one row
|
||||
-- instead of summing both tables. Each trigger also adds one to
|
||||
-- change_count; the reconciliation pass, which sums both tables a page
|
||||
-- at a time, corrects total_size_bytes only if change_count did not
|
||||
-- move while it summed.
|
||||
CREATE TABLE IF NOT EXISTS cache_usage (
|
||||
id INTEGER PRIMARY KEY CHECK (id = 1),
|
||||
total_size_bytes INTEGER NOT NULL DEFAULT 0,
|
||||
change_count INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
INSERT OR IGNORE INTO cache_usage (id) VALUES (1);
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS source_content_usage_insert
|
||||
AFTER INSERT ON source_content
|
||||
BEGIN
|
||||
UPDATE cache_usage
|
||||
SET total_size_bytes = total_size_bytes + NEW.size_bytes,
|
||||
change_count = change_count + 1
|
||||
WHERE id = 1;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS source_content_usage_delete
|
||||
AFTER DELETE ON source_content
|
||||
BEGIN
|
||||
UPDATE cache_usage
|
||||
SET total_size_bytes = total_size_bytes - OLD.size_bytes,
|
||||
change_count = change_count + 1
|
||||
WHERE id = 1;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS source_content_usage_update
|
||||
AFTER UPDATE OF size_bytes ON source_content
|
||||
BEGIN
|
||||
UPDATE cache_usage
|
||||
SET total_size_bytes = total_size_bytes - OLD.size_bytes + NEW.size_bytes,
|
||||
change_count = change_count + 1
|
||||
WHERE id = 1;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS variant_content_usage_insert
|
||||
AFTER INSERT ON variant_content
|
||||
BEGIN
|
||||
UPDATE cache_usage
|
||||
SET total_size_bytes = total_size_bytes + NEW.size_bytes,
|
||||
change_count = change_count + 1
|
||||
WHERE id = 1;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS variant_content_usage_delete
|
||||
AFTER DELETE ON variant_content
|
||||
BEGIN
|
||||
UPDATE cache_usage
|
||||
SET total_size_bytes = total_size_bytes - OLD.size_bytes,
|
||||
change_count = change_count + 1
|
||||
WHERE id = 1;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS variant_content_usage_update
|
||||
AFTER UPDATE OF size_bytes ON variant_content
|
||||
BEGIN
|
||||
UPDATE cache_usage
|
||||
SET total_size_bytes = total_size_bytes - OLD.size_bytes + NEW.size_bytes,
|
||||
change_count = change_count + 1
|
||||
WHERE id = 1;
|
||||
END;
|
||||
|
||||
-- Output/transformed content blobs
|
||||
-- Not written: transformed images are stored in cache/variants and
|
||||
-- tracked in variant_content above.
|
||||
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
// Default values for optional fields.
|
||||
const (
|
||||
DefaultQuality = 85
|
||||
DefaultFormat = imgcache.FormatOriginal
|
||||
DefaultFormat = imgcache.FormatJXL
|
||||
DefaultFitMode = imgcache.FitCover
|
||||
|
||||
// HKDF salt for URL encryption key derivation
|
||||
@@ -37,7 +37,7 @@ type Payload struct {
|
||||
SourceQuery string `cbor:"q,omitempty"` // optional
|
||||
Width int `cbor:"w,omitempty"` // 0 = original
|
||||
Height int `cbor:"ht,omitempty"` // 0 = original
|
||||
Format imgcache.ImageFormat `cbor:"f,omitempty"` // default: orig
|
||||
Format imgcache.ImageFormat `cbor:"f,omitempty"` // default: jxl
|
||||
Quality int `cbor:"ql,omitempty"` // default: 85
|
||||
FitMode imgcache.FitMode `cbor:"fm,omitempty"` // default: cover
|
||||
ExpiresAt int64 `cbor:"e,omitempty"` // 0 = never expires
|
||||
|
||||
@@ -369,10 +369,15 @@ func (s *Handlers) buildGeneratedURL(r *http.Request, token, format string) stri
|
||||
scheme = "http"
|
||||
}
|
||||
|
||||
// Determine file extension for the trailing filename
|
||||
// Determine file extension for the trailing filename. A form with no
|
||||
// format makes a token with none, which is served as encurl.DefaultFormat.
|
||||
ext := format
|
||||
if ext == "" || ext == "orig" || ext == "auto" {
|
||||
ext = "jpg" // Default extension
|
||||
|
||||
switch format {
|
||||
case "":
|
||||
ext = string(encurl.DefaultFormat)
|
||||
case "orig", "auto":
|
||||
ext = "jpg"
|
||||
}
|
||||
|
||||
return scheme + "://" + r.Host + "/v1/e/" + url.PathEscape(token) + "/img." + ext
|
||||
|
||||
@@ -14,8 +14,8 @@ import (
|
||||
// Errors for an Accept header that an auto URL cannot be served for.
|
||||
var (
|
||||
errInvalidAccept = errors.New("invalid Accept header")
|
||||
errNotAcceptable = errors.New(
|
||||
"not acceptable: auto serves image/avif, image/webp or image/jpeg")
|
||||
errNotAcceptable = errors.New("not acceptable: auto serves " +
|
||||
"image/jxl, image/avif, image/webp or image/jpeg")
|
||||
)
|
||||
|
||||
// chooseAutoFormat replaces the format auto in req with the format
|
||||
@@ -51,11 +51,11 @@ func (s *Handlers) chooseAutoFormat(
|
||||
}
|
||||
|
||||
// formatForAccept returns the format an auto URL is served in for the Accept
|
||||
// header accept: AVIF when it names image/avif, else WebP when it names
|
||||
// image/webp, else JPEG when its most specific entry of image/jpeg, image/*
|
||||
// and */* allows it, or when it names nothing. A q of 0 refuses a format.
|
||||
// AVIF and WebP must be named, as clients that cannot show them send image/*
|
||||
// and */* too.
|
||||
// header accept: JPEG XL when it names image/jxl, else AVIF when it names
|
||||
// image/avif, else WebP when it names image/webp, else JPEG when its most
|
||||
// specific entry of image/jpeg, image/* and */* allows it, or when it names
|
||||
// nothing. A q of 0 refuses a format. JPEG XL, AVIF and WebP must be named, as
|
||||
// clients that cannot show them send image/* and */* too.
|
||||
func formatForAccept(accept string) (imgcache.ImageFormat, error) {
|
||||
qualities, err := parseAccept(accept)
|
||||
if err != nil {
|
||||
@@ -66,6 +66,10 @@ func formatForAccept(accept string) (imgcache.ImageFormat, error) {
|
||||
return imgcache.FormatJPEG, nil
|
||||
}
|
||||
|
||||
if qualities["image/jxl"] > 0 {
|
||||
return imgcache.FormatJXL, nil
|
||||
}
|
||||
|
||||
if qualities["image/avif"] > 0 {
|
||||
return imgcache.FormatAVIF, nil
|
||||
}
|
||||
|
||||
@@ -18,7 +18,8 @@ import (
|
||||
)
|
||||
|
||||
// HandleImage handles the main image proxy route:
|
||||
// /v1/image/<host>/<path>/<width>x<height>.<format>
|
||||
// /v1/image/<host>/<path>/<width>x<height>.<format>, or with no format
|
||||
// /v1/image/<host>/<path>/<width>x<height>
|
||||
func (s *Handlers) HandleImage() http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if s.refuseBlockedReferer(w, r) {
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"maps"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/davidbyttow/govips/v2/vips"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/encurl"
|
||||
"sneak.berlin/go/pixa/internal/imgcache"
|
||||
"sneak.berlin/go/pixa/internal/signature"
|
||||
)
|
||||
|
||||
// requireJPEGXL requires that rec answers 200 with a JPEG XL image.
|
||||
func requireJPEGXL(t *testing.T, rec *httptest.ResponseRecorder) {
|
||||
t.Helper()
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d; body %q",
|
||||
rec.Code, http.StatusOK, rec.Body.String())
|
||||
}
|
||||
|
||||
if got := rec.Header().Get("Content-Type"); got != jxlType {
|
||||
t.Errorf("Content-Type = %q, want %s", got, jxlType)
|
||||
}
|
||||
|
||||
if got := vips.DetermineImageType(rec.Body.Bytes()); got != vips.ImageTypeJXL {
|
||||
t.Errorf("body is %s, want jxl", vips.ImageTypes[got])
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageWithoutFormat_ServesJPEGXL verifies that a /v1/image/ URL whose
|
||||
// last segment is a size with no format, 50x50 or orig, answers JPEG XL.
|
||||
func TestImageWithoutFormat_ServesJPEGXL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
route := newImageRoute(t, newPhotoFetcher(t, allowlistedHost))
|
||||
|
||||
for _, size := range []string{"50x50", "orig"} {
|
||||
target := "/v1/image/" + allowlistedHost + photoPath + "/" + size
|
||||
requireJPEGXL(t, sendGet(t, route, target))
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageWithoutFormat_SignedAsJXL verifies that a /v1/image/ URL with no
|
||||
// format is signed as jxl: the signature made for the URL ending in .jxl is
|
||||
// accepted for the same URL without .jxl.
|
||||
func TestImageWithoutFormat_SignedAsJXL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
route := newImageRoute(t, newPhotoFetcher(t, signedHost))
|
||||
expires := time.Now().Add(time.Hour)
|
||||
|
||||
sig := signature.New(testSigningKey).Sign(&signature.Request{
|
||||
SourceHost: signedHost,
|
||||
SourcePath: photoPath,
|
||||
Width: 50,
|
||||
Height: 50,
|
||||
Format: string(imgcache.FormatJXL),
|
||||
Quality: encurl.DefaultQuality,
|
||||
FitMode: string(imgcache.FitCover),
|
||||
Expires: expires,
|
||||
})
|
||||
query := fmt.Sprintf("?sig=%s&exp=%d", sig, expires.Unix())
|
||||
|
||||
for _, size := range []string{"50x50.jxl", "50x50"} {
|
||||
target := "/v1/image/" + signedHost + photoPath + "/" + size + query
|
||||
requireJPEGXL(t, sendGet(t, route, target))
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageEncWithoutFormat_ServesJPEGXL verifies that an encrypted URL whose
|
||||
// token holds no format answers JPEG XL.
|
||||
func TestImageEncWithoutFormat_ServesJPEGXL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
||||
|
||||
token, err := h.encGen.Generate(&encurl.Payload{
|
||||
SourceHost: signedHost,
|
||||
SourcePath: photoPath,
|
||||
Width: 50,
|
||||
Height: 50,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Generate() error = %v", err)
|
||||
}
|
||||
|
||||
requireJPEGXL(t, getEncToken(srv, token))
|
||||
}
|
||||
|
||||
// TestGeneratorPage_SelectsJPEGXL verifies that the generator page's format
|
||||
// choice is JPEG XL until another is chosen.
|
||||
func TestGeneratorPage_SelectsJPEGXL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
h, srv := newCSRFTestRouter(t)
|
||||
|
||||
req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil)
|
||||
req.AddCookie(newSessionCookie(t, h))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rec, req)
|
||||
|
||||
if !strings.Contains(rec.Body.String(), `<option value="jxl" selected>`) {
|
||||
t.Errorf("generator page does not select JPEG XL: %s", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// TestGeneratePost_NoFormat_MakesJPEGXLURL verifies that the generator form
|
||||
// sent with an empty format field, or with none, makes a URL whose name ends
|
||||
// in .jxl and which answers JPEG XL.
|
||||
func TestGeneratePost_NoFormat_MakesJPEGXLURL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
photo := url.Values{
|
||||
sourceURLField: {"https://" + signedHost + photoPath},
|
||||
widthField: {"50"},
|
||||
heightField: {"50"},
|
||||
}
|
||||
|
||||
emptyFormat := maps.Clone(photo)
|
||||
emptyFormat.Set(formatField, "")
|
||||
|
||||
for name, form := range map[string]url.Values{
|
||||
"empty format field": emptyFormat,
|
||||
"no format field": photo,
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, imageSrv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
||||
|
||||
rec := generatePost(t, form)
|
||||
|
||||
match := generatedURLPattern.FindStringSubmatch(rec.Body.String())
|
||||
if match == nil {
|
||||
t.Fatalf("generator page shows no URL: %d %s",
|
||||
rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
t.Logf("generated URL path: %s", match[1])
|
||||
|
||||
if !strings.HasSuffix(match[1], "/img.jxl") {
|
||||
t.Errorf("generated URL %s does not end in /img.jxl", match[1])
|
||||
}
|
||||
|
||||
imageRec := httptest.NewRecorder()
|
||||
imageSrv.ServeHTTP(imageRec, httptest.NewRequestWithContext(
|
||||
t.Context(), http.MethodGet, match[1], nil))
|
||||
|
||||
requireJPEGXL(t, imageRec)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/davidbyttow/govips/v2/vips"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/imgcache"
|
||||
)
|
||||
|
||||
// jxlType is the content type of JPEG XL.
|
||||
const jxlType = "image/jxl"
|
||||
|
||||
// TestFormatForAccept_JPEGXL verifies that the format auto chooses JPEG XL
|
||||
// when Accept names image/jxl, ahead of AVIF whatever their q, and AVIF when
|
||||
// Accept refuses JPEG XL with q=0.
|
||||
func TestFormatForAccept_JPEGXL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
accept string
|
||||
want imgcache.ImageFormat
|
||||
}{
|
||||
{"JPEG XL-capable browser",
|
||||
"image/jxl,image/avif,image/webp,image/*,*/*;q=0.8", imgcache.FormatJXL},
|
||||
{"JPEG XL only", jxlType, imgcache.FormatJXL},
|
||||
{"JPEG XL with a lower q than AVIF", "image/avif,image/jxl;q=0.5",
|
||||
imgcache.FormatJXL},
|
||||
{"q=0 on JPEG XL", "image/jxl;q=0,image/avif,*/*", imgcache.FormatAVIF},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
got, err := formatForAccept(tt.accept)
|
||||
if got != tt.want || err != nil {
|
||||
t.Errorf("formatForAccept(%q) = %q, %v, want %q",
|
||||
tt.accept, got, err, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestFormatAuto_JPEGXL requests an auto URL on each image route with an
|
||||
// Accept header that names image/jxl, and checks that the answer is a JPEG XL
|
||||
// image.
|
||||
func TestFormatAuto_JPEGXL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
||||
signedURL, encryptedURL := autoPhotoURLs(t, h)
|
||||
|
||||
for _, target := range []string{signedURL, encryptedURL} {
|
||||
rec := requestImage(t, srv, http.MethodGet, target,
|
||||
"image/jxl,image/avif,image/webp,*/*;q=0.8")
|
||||
gotType := rec.Header().Get("Content-Type")
|
||||
|
||||
if rec.Code != http.StatusOK || gotType != jxlType {
|
||||
t.Errorf("%s: %d %s, want 200 %s; body %s",
|
||||
target, rec.Code, gotType, jxlType, rec.Body)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
if got := vips.DetermineImageType(rec.Body.Bytes()); got != vips.ImageTypeJXL {
|
||||
t.Errorf("%s: body is %s, want jxl", target, vips.ImageTypes[got])
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -45,6 +45,7 @@ const (
|
||||
contentTypeGIF = "image/gif"
|
||||
contentTypeWebP = "image/webp"
|
||||
contentTypeAVIF = "image/avif"
|
||||
contentTypeJXL = "image/jxl"
|
||||
contentTypeSVG = "image/svg+xml"
|
||||
contentTypeOctetStream = "application/octet-stream"
|
||||
)
|
||||
@@ -156,6 +157,7 @@ func DefaultConfig() *Config {
|
||||
contentTypeGIF,
|
||||
contentTypeWebP,
|
||||
contentTypeAVIF,
|
||||
contentTypeJXL,
|
||||
contentTypeSVG,
|
||||
},
|
||||
AllowHTTP: false,
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
package httpfetcher
|
||||
|
||||
import "testing"
|
||||
|
||||
// TestJPEGXLContentType verifies that the fetcher accepts an upstream answer
|
||||
// of type image/jxl by default, and that the mock fetcher serves a .jxl file
|
||||
// as image/jxl.
|
||||
func TestJPEGXLContentType(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const jxlType = "image/jxl"
|
||||
|
||||
if !New(DefaultConfig()).isAllowedContentType(jxlType) {
|
||||
t.Errorf("isAllowedContentType(%q) = false, want true", jxlType)
|
||||
}
|
||||
|
||||
if got := detectContentTypeFromPath("images/photo.jxl"); got != jxlType {
|
||||
t.Errorf("detectContentTypeFromPath(photo.jxl) = %q, want %q", got, jxlType)
|
||||
}
|
||||
}
|
||||
@@ -109,6 +109,8 @@ func detectContentTypeFromPath(path string) string {
|
||||
return contentTypeWebP
|
||||
case strings.HasSuffix(path, ".avif"):
|
||||
return contentTypeAVIF
|
||||
case strings.HasSuffix(path, ".jxl"):
|
||||
return contentTypeJXL
|
||||
case strings.HasSuffix(path, ".svg"):
|
||||
return contentTypeSVG
|
||||
default:
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
package imageprocessor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestImageProcessor_EmptyFormatRefused verifies that a request with no format
|
||||
// is refused, as both image routes give every request a format before it is
|
||||
// processed.
|
||||
func TestImageProcessor_EmptyFormatRefused(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, err := New(Params{}).Process(
|
||||
t.Context(), bytes.NewReader(createTestJPEG(t, 20, 20)), &Request{},
|
||||
)
|
||||
if !errors.Is(err, ErrUnsupportedOutputFormat) {
|
||||
t.Errorf("Process() error = %v, want %v", err, ErrUnsupportedOutputFormat)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
package imageprocessor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/png"
|
||||
"testing"
|
||||
|
||||
"github.com/davidbyttow/govips/v2/vips"
|
||||
)
|
||||
|
||||
// processedSize runs input through Process and returns the output's size in
|
||||
// bytes.
|
||||
func processedSize(t *testing.T, input []byte, req *Request) int64 {
|
||||
t.Helper()
|
||||
|
||||
result, err := New(Params{}).Process(t.Context(), bytes.NewReader(input), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Process() error = %v", err)
|
||||
}
|
||||
|
||||
_ = result.Content.Close()
|
||||
|
||||
return result.ContentLength
|
||||
}
|
||||
|
||||
// encodePNG encodes img as PNG with Go's encoder.
|
||||
func encodePNG(t *testing.T, img image.Image) []byte {
|
||||
t.Helper()
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
err := png.Encode(&buf, img)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to encode test PNG: %v", err)
|
||||
}
|
||||
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// decode decodes input with vips, as Process does.
|
||||
func decode(t *testing.T, input []byte) *vips.ImageRef {
|
||||
t.Helper()
|
||||
|
||||
img, err := vips.NewImageFromBuffer(input)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to decode input: %v", err)
|
||||
}
|
||||
|
||||
t.Cleanup(img.Close)
|
||||
|
||||
return img
|
||||
}
|
||||
|
||||
// TestImageProcessor_PNGIsCompressed verifies that a PNG output is
|
||||
// compressed: a flat 200x150 image, 90,000 bytes of raw pixels, comes out at
|
||||
// a small fraction of that.
|
||||
func TestImageProcessor_PNGIsCompressed(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const width, height = 200, 150
|
||||
|
||||
flat := image.NewRGBA(image.Rect(0, 0, width, height))
|
||||
for y := range height {
|
||||
for x := range width {
|
||||
flat.Set(x, y, color.RGBA{R: 40, G: 120, B: 200, A: 255})
|
||||
}
|
||||
}
|
||||
|
||||
size := processedSize(t, encodePNG(t, flat), &Request{Format: FormatPNG})
|
||||
|
||||
const rawBytes = width * height * 3
|
||||
if size > rawBytes/10 {
|
||||
t.Errorf("PNG output is %d bytes, want under a tenth of its %d raw",
|
||||
size, rawBytes)
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageProcessor_WebPAtDefaultEffort verifies that WebP is saved at
|
||||
// libvips' default effort, 4: the output is the size of the same image saved
|
||||
// at that effort.
|
||||
func TestImageProcessor_WebPAtDefaultEffort(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
input := createTestJPEG(t, 200, 150)
|
||||
|
||||
size := processedSize(t, input, &Request{Format: FormatWebP, Quality: 85})
|
||||
|
||||
want, _, err := decode(t, input).ExportWebp(&vips.WebpExportParams{
|
||||
StripMetadata: true,
|
||||
Quality: 85,
|
||||
ReductionEffort: 4,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ExportWebp() error = %v", err)
|
||||
}
|
||||
|
||||
if size != int64(len(want)) {
|
||||
t.Errorf("WebP output is %d bytes, want %d, the size at effort 4",
|
||||
size, len(want))
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageProcessor_AVIFAtEffort1 verifies that AVIF is saved at effort 1
|
||||
// with 8 bits per sample: the output is the size of the same image saved
|
||||
// with those settings. The source is 16-bit, which libvips saves with 12
|
||||
// bits when it is not given a bit depth, and 640x480: on a small image,
|
||||
// such as 64x48, efforts 1 and 2 give the same output.
|
||||
func TestImageProcessor_AVIFAtEffort1(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const width, height = 640, 480
|
||||
|
||||
source := image.NewRGBA64(image.Rect(0, 0, width, height))
|
||||
for y := range height {
|
||||
for x := range width {
|
||||
source.Set(x, y, color.RGBA64{
|
||||
R: uint16((x * 65535 / width) & 0xffff),
|
||||
G: uint16((y * 65535 / height) & 0xffff),
|
||||
B: 32768,
|
||||
A: 65535,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
input := encodePNG(t, source)
|
||||
|
||||
size := processedSize(t, input, &Request{Format: FormatAVIF, Quality: 85})
|
||||
|
||||
want, _, err := decode(t, input).ExportAvif(&vips.AvifExportParams{
|
||||
StripMetadata: true,
|
||||
Quality: 85,
|
||||
Effort: 1,
|
||||
Bitdepth: 8,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ExportAvif() error = %v", err)
|
||||
}
|
||||
|
||||
if size != int64(len(want)) {
|
||||
t.Errorf("AVIF output is %d bytes, want %d, the size at effort 1 "+
|
||||
"and 8 bits", size, len(want))
|
||||
}
|
||||
}
|
||||
@@ -37,6 +37,24 @@ func initVips() {
|
||||
})
|
||||
}
|
||||
|
||||
// errNoJPEGXL is returned by CheckJPEGXLSupport.
|
||||
var errNoJPEGXL = errors.New("libvips lacks JPEG XL support: install " +
|
||||
"vips-jxl on Alpine, or use a libvips built with libjxl")
|
||||
|
||||
// CheckJPEGXLSupport returns an error, naming the fix, when libvips
|
||||
// cannot load and save JPEG XL.
|
||||
func CheckJPEGXLSupport() error {
|
||||
initVips()
|
||||
|
||||
// govips counts a format as supported when libvips has its loader;
|
||||
// libvips builds the JPEG XL loader and saver together.
|
||||
if !vips.IsTypeSupported(vips.ImageTypeJXL) {
|
||||
return errNoJPEGXL
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Format represents supported output image formats.
|
||||
type Format string
|
||||
|
||||
@@ -47,6 +65,7 @@ const (
|
||||
FormatPNG Format = "png"
|
||||
FormatWebP Format = "webp"
|
||||
FormatAVIF Format = "avif"
|
||||
FormatJXL Format = "jxl"
|
||||
FormatGIF Format = "gif"
|
||||
)
|
||||
|
||||
@@ -237,9 +256,9 @@ func (p *ImageProcessor) Process(
|
||||
}
|
||||
}
|
||||
|
||||
// Determine output format
|
||||
// orig is the source's own format; encode refuses an empty format
|
||||
outputFormat := req.Format
|
||||
if outputFormat == FormatOriginal || outputFormat == "" {
|
||||
if outputFormat == FormatOriginal {
|
||||
outputFormat = p.formatFromString(inputFormat)
|
||||
}
|
||||
|
||||
@@ -287,6 +306,7 @@ const (
|
||||
mimeGIF = "image/gif"
|
||||
mimeWebP = "image/webp"
|
||||
mimeAVIF = "image/avif"
|
||||
mimeJXL = "image/jxl"
|
||||
)
|
||||
|
||||
// SupportedInputFormats returns MIME types this processor can read.
|
||||
@@ -297,6 +317,7 @@ func (p *ImageProcessor) SupportedInputFormats() []string {
|
||||
mimeGIF,
|
||||
mimeWebP,
|
||||
mimeAVIF,
|
||||
mimeJXL,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -308,6 +329,7 @@ func (p *ImageProcessor) SupportedOutputFormats() []Format {
|
||||
FormatGIF,
|
||||
FormatWebP,
|
||||
FormatAVIF,
|
||||
FormatJXL,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -324,6 +346,8 @@ func FormatToMIME(format Format) string {
|
||||
return mimeGIF
|
||||
case FormatAVIF:
|
||||
return mimeAVIF
|
||||
case FormatJXL:
|
||||
return mimeJXL
|
||||
case FormatOriginal:
|
||||
return "application/octet-stream"
|
||||
default:
|
||||
@@ -393,9 +417,11 @@ func (p *ImageProcessor) detectFormat(img *vips.ImageRef) string {
|
||||
return "webp"
|
||||
case vips.ImageTypeAVIF, vips.ImageTypeHEIF:
|
||||
return string(FormatAVIF)
|
||||
case vips.ImageTypeJXL:
|
||||
return string(FormatJXL)
|
||||
case vips.ImageTypeUnknown, vips.ImageTypeMagick, vips.ImageTypePDF,
|
||||
vips.ImageTypeSVG, vips.ImageTypeTIFF, vips.ImageTypeBMP,
|
||||
vips.ImageTypeJP2K, vips.ImageTypeJXL:
|
||||
vips.ImageTypeJP2K:
|
||||
return "unknown"
|
||||
default:
|
||||
return "unknown"
|
||||
@@ -467,44 +493,6 @@ func (p *ImageProcessor) encode(
|
||||
quality = defaultQuality
|
||||
}
|
||||
|
||||
var params vips.ExportParams
|
||||
|
||||
switch format {
|
||||
case FormatJPEG:
|
||||
params = vips.ExportParams{
|
||||
Format: vips.ImageTypeJPEG,
|
||||
Quality: quality,
|
||||
}
|
||||
|
||||
case FormatPNG:
|
||||
params = vips.ExportParams{
|
||||
Format: vips.ImageTypePNG,
|
||||
}
|
||||
|
||||
case FormatGIF:
|
||||
params = vips.ExportParams{
|
||||
Format: vips.ImageTypeGIF,
|
||||
}
|
||||
|
||||
case FormatWebP:
|
||||
params = vips.ExportParams{
|
||||
Format: vips.ImageTypeWEBP,
|
||||
Quality: quality,
|
||||
}
|
||||
|
||||
case FormatAVIF:
|
||||
params = vips.ExportParams{
|
||||
Format: vips.ImageTypeAVIF,
|
||||
Quality: quality,
|
||||
}
|
||||
|
||||
case FormatOriginal:
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnsupportedOutputFormat, format)
|
||||
|
||||
default:
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnsupportedOutputFormat, format)
|
||||
}
|
||||
|
||||
// Stripping drops the ICC profile as well, and clients show an image
|
||||
// with no profile as sRGB, so convert to sRGB first. "srgb" names
|
||||
// libvips' built-in profile; govips' own sRGB path variable is set on
|
||||
@@ -516,11 +504,166 @@ func (p *ImageProcessor) encode(
|
||||
}
|
||||
}
|
||||
|
||||
// Drop EXIF, XMP, IPTC and the ICC profile. govips ignores this for
|
||||
// GIF, which carries none of them.
|
||||
params.StripMetadata = true
|
||||
switch format {
|
||||
case FormatJPEG:
|
||||
return exportJPEG(img, quality)
|
||||
|
||||
output, _, err := img.Export(¶ms)
|
||||
case FormatPNG:
|
||||
return exportPNG(img)
|
||||
|
||||
case FormatGIF:
|
||||
return exportGIF(img)
|
||||
|
||||
case FormatWebP:
|
||||
return exportWebP(img, quality)
|
||||
|
||||
case FormatAVIF:
|
||||
return exportAVIF(img, quality)
|
||||
|
||||
case FormatJXL:
|
||||
return exportJXL(img, quality)
|
||||
|
||||
case FormatOriginal:
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnsupportedOutputFormat, format)
|
||||
|
||||
default:
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnsupportedOutputFormat, format)
|
||||
}
|
||||
}
|
||||
|
||||
// govips sends libvips Go's zero value for some settings an export leaves
|
||||
// out, such as no compression at all for PNG, so each export below sets
|
||||
// every setting whose zero value is not what pixa wants. Stripping metadata
|
||||
// drops EXIF, XMP, IPTC and the ICC profile.
|
||||
|
||||
// exportJPEG encodes img as JPEG at quality, without metadata. The settings
|
||||
// it leaves out are at libvips' defaults.
|
||||
func exportJPEG(img *vips.ImageRef, quality int) ([]byte, error) {
|
||||
output, _, err := img.ExportJpeg(&vips.JpegExportParams{
|
||||
StripMetadata: true,
|
||||
Quality: quality,
|
||||
})
|
||||
|
||||
return output, err
|
||||
}
|
||||
|
||||
// pngCompression is libvips' default PNG compression, from 0 (none) to 9.
|
||||
const pngCompression = 6
|
||||
|
||||
// exportPNG encodes img as PNG at libvips' default compression and row
|
||||
// filter, without metadata.
|
||||
func exportPNG(img *vips.ImageRef) ([]byte, error) {
|
||||
output, _, err := img.ExportPng(&vips.PngExportParams{
|
||||
StripMetadata: true,
|
||||
Compression: pngCompression,
|
||||
Filter: vips.PngFilterNone,
|
||||
})
|
||||
|
||||
return output, err
|
||||
}
|
||||
|
||||
// gifEffort is libvips' default GIF effort, from 1 to 10.
|
||||
const gifEffort = 7
|
||||
|
||||
// exportGIF encodes img as GIF at libvips' default effort. govips cannot
|
||||
// have libvips strip metadata from GIF, which carries none.
|
||||
func exportGIF(img *vips.ImageRef) ([]byte, error) {
|
||||
output, _, err := img.ExportGIF(&vips.GifExportParams{Effort: gifEffort})
|
||||
|
||||
return output, err
|
||||
}
|
||||
|
||||
// webpEffort is libvips' default WebP effort, from 0 (fastest) to 6.
|
||||
const webpEffort = 4
|
||||
|
||||
// exportWebP encodes img as lossy WebP at quality and libvips' default
|
||||
// effort, without metadata.
|
||||
func exportWebP(img *vips.ImageRef, quality int) ([]byte, error) {
|
||||
output, _, err := img.ExportWebp(&vips.WebpExportParams{
|
||||
StripMetadata: true,
|
||||
Quality: quality,
|
||||
ReductionEffort: webpEffort,
|
||||
})
|
||||
|
||||
return output, err
|
||||
}
|
||||
|
||||
// avifEffort is the AVIF effort, from 0 (fastest) to 9; 1 is the lowest
|
||||
// govips can set. With one thread, as pixad runs libvips, 1 takes about 51
|
||||
// seconds to save an 8192x8192 image of random pixels, the worst case,
|
||||
// against the default downstream_timeout of 60 seconds. On an image of
|
||||
// milder noise, which 1 saves in about 12 seconds, 2 takes nearly a minute
|
||||
// and libvips' default, 4, takes minutes.
|
||||
const avifEffort = 1
|
||||
|
||||
// avifBitdepth is the AVIF bit depth, 8 bits per sample for every image.
|
||||
// libvips would save a 16-bit image with 12, but at avifEffort that takes
|
||||
// about 54 seconds for a 16-bit 8192x8192 image of milder noise, nearly all
|
||||
// of the default downstream_timeout, and about 12 seconds with 8.
|
||||
const avifBitdepth = 8
|
||||
|
||||
// exportAVIF encodes img as lossy AVIF at quality, avifEffort and
|
||||
// avifBitdepth, without metadata.
|
||||
func exportAVIF(img *vips.ImageRef, quality int) ([]byte, error) {
|
||||
output, _, err := img.ExportAvif(&vips.AvifExportParams{
|
||||
StripMetadata: true,
|
||||
Quality: quality,
|
||||
Effort: avifEffort,
|
||||
Bitdepth: avifBitdepth,
|
||||
})
|
||||
|
||||
return output, err
|
||||
}
|
||||
|
||||
// jxlResolution is the resolution every JPEG XL image is saved with, in
|
||||
// pixels per millimetre as libvips counts it: 72 dpi, what libvips gives a
|
||||
// JPEG that names none.
|
||||
const jxlResolution = 72 / 25.4
|
||||
|
||||
// exportJXL encodes img as JPEG XL at quality, with libvips' default effort
|
||||
// and without metadata. govips sends libvips a distance, the JPEG XL
|
||||
// encoder's own measure of quality, along with the quality, and libvips then
|
||||
// uses the distance alone, so the quality is also given as a distance.
|
||||
func exportJXL(img *vips.ImageRef, quality int) ([]byte, error) {
|
||||
// libvips converts a CMYK image to sRGB before it saves WebP, AVIF or
|
||||
// PNG, but cannot save one as JPEG XL. encode has already converted
|
||||
// any image with an ICC profile to sRGB, so this is CMYK with none.
|
||||
if img.Interpretation() == vips.InterpretationCMYK {
|
||||
err := img.ToColorSpace(vips.InterpretationSRGB)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to convert CMYK to sRGB: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// govips cannot make libvips strip metadata from JPEG XL, so it is
|
||||
// removed from the image itself. RemoveMetadata removes EXIF, XMP and
|
||||
// IPTC but keeps the ICC profile.
|
||||
err := img.RemoveMetadata()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = img.RemoveICCProfile()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// libvips 8.16 and later still write an EXIF block of their own, from
|
||||
// the image's size, orientation and resolution, and fixed values. The
|
||||
// image is upright, so its orientation is 1, but its resolution is still
|
||||
// the source's.
|
||||
toSave, err := img.CopyChangingResolution(jxlResolution, jxlResolution)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer toSave.Close()
|
||||
|
||||
params := vips.NewJxlExportParams()
|
||||
params.Quality = quality
|
||||
params.Distance = jxlDistance(quality)
|
||||
|
||||
output, _, err := toSave.ExportJxl(params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -528,6 +671,22 @@ func (p *ImageProcessor) encode(
|
||||
return output, nil
|
||||
}
|
||||
|
||||
// jxlDistance turns a quality from 1 to 100 into the JPEG XL encoder's
|
||||
// distance, with the formula libvips and libjxl use for their own quality
|
||||
// setting, except that 100 stays lossy (distance 0.1) where libjxl makes it
|
||||
// lossless.
|
||||
//
|
||||
//nolint:mnd // the constants of that formula
|
||||
func jxlDistance(quality int) float64 {
|
||||
q := float64(quality)
|
||||
|
||||
if quality >= 30 {
|
||||
return 0.1 + (100-q)*0.09
|
||||
}
|
||||
|
||||
return 53.0/3000.0*q*q - 23.0/20.0*q + 25.0
|
||||
}
|
||||
|
||||
// formatFromString converts a format string to Format.
|
||||
func (p *ImageProcessor) formatFromString(format string) Format {
|
||||
switch format {
|
||||
@@ -541,6 +700,8 @@ func (p *ImageProcessor) formatFromString(format string) Format {
|
||||
return FormatWebP
|
||||
case string(FormatAVIF):
|
||||
return FormatAVIF
|
||||
case string(FormatJXL):
|
||||
return FormatJXL
|
||||
default:
|
||||
return FormatJPEG
|
||||
}
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
package imageprocessor
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/davidbyttow/govips/v2/vips"
|
||||
)
|
||||
|
||||
// TestCheckJPEGXLSupport fails when libvips lacks JPEG XL support, as on
|
||||
// Alpine without the vips-jxl package.
|
||||
func TestCheckJPEGXLSupport(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
err := CheckJPEGXLSupport()
|
||||
if err != nil {
|
||||
t.Fatalf("CheckJPEGXLSupport() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLibvipsSavesAndLoadsJPEGXL saves an image as JPEG XL with govips and
|
||||
// loads it back. It fails when libvips lacks JPEG XL support, as on Alpine
|
||||
// without the vips-jxl package.
|
||||
func TestLibvipsSavesAndLoadsJPEGXL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
img, err := vips.NewImageFromBuffer(createTestJPEG(t, 64, 48))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to load test JPEG: %v", err)
|
||||
}
|
||||
|
||||
defer img.Close()
|
||||
|
||||
jxl, _, err := img.ExportJxl(vips.NewJxlExportParams())
|
||||
if err != nil {
|
||||
t.Fatalf("ExportJxl() error = %v", err)
|
||||
}
|
||||
|
||||
loaded, err := vips.NewImageFromBuffer(jxl)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to load the JPEG XL image: %v", err)
|
||||
}
|
||||
|
||||
defer loaded.Close()
|
||||
|
||||
if loaded.Format() != vips.ImageTypeJXL {
|
||||
t.Errorf("loaded format = %s, want jxl", vips.ImageTypes[loaded.Format()])
|
||||
}
|
||||
|
||||
if loaded.Width() != 64 || loaded.Height() != 48 {
|
||||
t.Errorf("loaded size = %dx%d, want 64x48", loaded.Width(), loaded.Height())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
package imageprocessor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/davidbyttow/govips/v2/vips"
|
||||
)
|
||||
|
||||
// TestImageProcessor_EncodeJPEGXL converts a JPEG to a smaller JPEG XL and
|
||||
// checks the content type, and the format and size the output loads as.
|
||||
func TestImageProcessor_EncodeJPEGXL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
req := &Request{Size: Size{Width: 100, Height: 75}, Format: FormatJXL}
|
||||
|
||||
result, err := New(Params{}).Process(
|
||||
t.Context(), bytes.NewReader(createTestJPEG(t, 200, 150)), req,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("Process() error = %v", err)
|
||||
}
|
||||
|
||||
defer func() { _ = result.Content.Close() }()
|
||||
|
||||
if result.ContentType != "image/jxl" {
|
||||
t.Errorf("ContentType = %q, want image/jxl", result.ContentType)
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(result.Content)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read result: %v", err)
|
||||
}
|
||||
|
||||
output, err := vips.NewImageFromBuffer(data)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to load the output: %v", err)
|
||||
}
|
||||
|
||||
defer output.Close()
|
||||
|
||||
if output.Format() != vips.ImageTypeJXL {
|
||||
t.Errorf("output format = %s, want jxl", vips.ImageTypes[output.Format()])
|
||||
}
|
||||
|
||||
if output.Width() != 100 || output.Height() != 75 {
|
||||
t.Errorf("output size = %dx%d, want 100x75", output.Width(), output.Height())
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageProcessor_JPEGXLQuality verifies that the quality reaches the JPEG
|
||||
// XL encoder: the same image comes out smaller at quality 30 than at 90.
|
||||
func TestImageProcessor_JPEGXLQuality(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
input := createTestJPEG(t, 400, 300)
|
||||
outputBytes := make(map[int]int64)
|
||||
|
||||
for _, quality := range []int{30, 90} {
|
||||
result, err := New(Params{}).Process(t.Context(),
|
||||
bytes.NewReader(input), &Request{Format: FormatJXL, Quality: quality})
|
||||
if err != nil {
|
||||
t.Fatalf("Process() at quality %d error = %v", quality, err)
|
||||
}
|
||||
|
||||
_ = result.Content.Close()
|
||||
outputBytes[quality] = result.ContentLength
|
||||
}
|
||||
|
||||
if outputBytes[30] >= outputBytes[90] {
|
||||
t.Errorf("%d bytes at quality 30, %d at 90, want fewer at 30",
|
||||
outputBytes[30], outputBytes[90])
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageProcessor_JPEGXLDropsSourceEXIF verifies that none of the source's
|
||||
// EXIF reaches a JPEG XL output. libvips 8.16 and later write an EXIF block of
|
||||
// their own into JPEG XL (orientation, resolution, size, colour space and
|
||||
// fixed defaults), and govips cannot ask them to leave it out, so the test
|
||||
// looks for the source's fields rather than for no EXIF at all.
|
||||
func TestImageProcessor_JPEGXLDropsSourceEXIF(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
input, err := os.ReadFile("testdata/gps-exif.jpg")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read test JPEG: %v", err)
|
||||
}
|
||||
|
||||
exif := processAndDecode(t, input, &Request{Format: FormatJXL}).GetExif()
|
||||
|
||||
for _, field := range []string{
|
||||
"exif-ifd0-Make", "exif-ifd0-Model", "exif-ifd2-BodySerialNumber",
|
||||
"exif-ifd2-DateTimeOriginal", "exif-ifd3-GPSLatitude",
|
||||
"exif-ifd3-GPSLongitude",
|
||||
} {
|
||||
if value, found := exif[field]; found {
|
||||
t.Errorf("output has %s: %s", field, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageProcessor_JPEGXLDropsSourceResolution verifies that the source's
|
||||
// resolution does not reach the EXIF block libvips 8.16 and later write into
|
||||
// JPEG XL. A JPEG XL image holds its resolution in that block alone, so the
|
||||
// resolution the output loads with is the block's.
|
||||
func TestImageProcessor_JPEGXLDropsSourceResolution(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// dpi-300.jpg is a flat 8x8 grey image with a resolution of 300 dpi.
|
||||
input, err := os.ReadFile("testdata/dpi-300.jpg")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read test JPEG: %v", err)
|
||||
}
|
||||
|
||||
output := processAndDecode(t, input, &Request{Format: FormatJXL})
|
||||
|
||||
// libvips gives the resolution in pixels per millimetre.
|
||||
xDPI := math.Round(output.ResX() * 25.4)
|
||||
yDPI := math.Round(output.ResY() * 25.4)
|
||||
|
||||
if xDPI == 300 || yDPI == 300 {
|
||||
t.Errorf("output resolution = %vx%v dpi, the source's", xDPI, yDPI)
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageProcessor_JPEGXLAppliesEXIFOrientation verifies that a JPEG XL
|
||||
// output is turned upright, as TestImageProcessor_AppliesEXIFOrientation does
|
||||
// for PNG.
|
||||
func TestImageProcessor_JPEGXLAppliesEXIFOrientation(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// orientation-6.jpg is stored 16x8, red on the left and blue on the
|
||||
// right, with EXIF orientation 6 (turn 90 degrees clockwise to view).
|
||||
// Upright it is 8x16, red on top and blue below.
|
||||
input, err := os.ReadFile("testdata/orientation-6.jpg")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read test JPEG: %v", err)
|
||||
}
|
||||
|
||||
output := processAndDecode(t, input, &Request{Format: FormatJXL})
|
||||
|
||||
if output.Width() != 8 || output.Height() != 16 {
|
||||
t.Fatalf("output is %dx%d, want 8x16", output.Width(), output.Height())
|
||||
}
|
||||
|
||||
top, err := output.GetPoint(4, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("GetPoint() error = %v", err)
|
||||
}
|
||||
|
||||
bottom, err := output.GetPoint(4, 15)
|
||||
if err != nil {
|
||||
t.Fatalf("GetPoint() error = %v", err)
|
||||
}
|
||||
|
||||
if top[0] <= top[2] || bottom[2] <= bottom[0] {
|
||||
t.Errorf("top pixel = %v, bottom pixel = %v, want red above blue",
|
||||
top, bottom)
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageProcessor_JPEGXLConvertsWideGamutToSRGB verifies that a JPEG XL
|
||||
// output is converted to sRGB, as TestImageProcessor_ConvertsWideGamutToSRGB
|
||||
// does for PNG. It does not check for an ICC profile, as libvips reports one
|
||||
// for every JPEG XL image it loads.
|
||||
func TestImageProcessor_JPEGXLConvertsWideGamutToSRGB(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// display-p3.jpg is a flat 8x8 image with the Display P3 profile
|
||||
// embedded, filled with Display P3 (234, 51, 35), which is sRGB red.
|
||||
input, err := os.ReadFile("testdata/display-p3.jpg")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read test JPEG: %v", err)
|
||||
}
|
||||
|
||||
output := processAndDecode(t, input, &Request{Format: FormatJXL})
|
||||
|
||||
pixel, err := output.GetPoint(4, 4)
|
||||
if err != nil {
|
||||
t.Fatalf("GetPoint() error = %v", err)
|
||||
}
|
||||
|
||||
want := []float64{255, 0, 0}
|
||||
for i := range want {
|
||||
if math.Abs(pixel[i]-want[i]) > 5 {
|
||||
t.Fatalf("pixel = %v, want within 5 of %v", pixel, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageProcessor_JPEGXLFromCMYK verifies that a CMYK JPEG with no ICC
|
||||
// profile can be served as JPEG XL, in sRGB.
|
||||
func TestImageProcessor_JPEGXLFromCMYK(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// cmyk.jpg is a flat 8x8 CMYK image with no ICC profile, filled with
|
||||
// full cyan and no magenta, yellow or black.
|
||||
input, err := os.ReadFile("testdata/cmyk.jpg")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read test JPEG: %v", err)
|
||||
}
|
||||
|
||||
output := processAndDecode(t, input, &Request{Format: FormatJXL})
|
||||
|
||||
if output.Bands() != 3 {
|
||||
t.Fatalf("output has %d bands, want 3", output.Bands())
|
||||
}
|
||||
|
||||
pixel, err := output.GetPoint(4, 4)
|
||||
if err != nil {
|
||||
t.Fatalf("GetPoint() error = %v", err)
|
||||
}
|
||||
|
||||
// Cyan in sRGB: little red, much green and blue.
|
||||
if pixel[0] > 50 || pixel[1] < 100 || pixel[2] < 200 {
|
||||
t.Errorf("pixel = %v, want cyan", pixel)
|
||||
}
|
||||
}
|
||||
|
||||
// TestJXLDistance verifies the JPEG XL distance for a few qualities: 90 is
|
||||
// distance 1, the encoder's own default, and 100 stays lossy.
|
||||
func TestJXLDistance(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
quality int
|
||||
want float64
|
||||
}{
|
||||
{quality: 100, want: 0.1},
|
||||
{quality: 90, want: 1},
|
||||
{quality: 30, want: 6.4},
|
||||
{quality: 20, want: 9.0667},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
got := jxlDistance(tt.quality)
|
||||
if math.Abs(got-tt.want) > 0.0001 {
|
||||
t.Errorf("jxlDistance(%d) = %v, want %v", tt.quality, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
BIN
Binary file not shown.
|
After Width: | Height: | Size: 352 B |
BIN
Binary file not shown.
|
After Width: | Height: | Size: 799 B |
@@ -96,6 +96,17 @@ type Cache struct {
|
||||
// deterministically pause inside that window to exercise
|
||||
// concurrent stores against it; production code leaves it nil.
|
||||
evictSourceBlobTestHook func(ContentHash)
|
||||
|
||||
// reconciliationPageSize is the most rows one read of a content table
|
||||
// returns in the reconciliation pass and in Stats. newCache sets it to
|
||||
// defaultReconciliationPageSize; tests set it smaller.
|
||||
reconciliationPageSize int
|
||||
|
||||
// reconciliationReadTestHook, when set, is called after each of those
|
||||
// reads with the number of rows the read covered, so tests can check
|
||||
// that no read covers more than one page; production code leaves it
|
||||
// nil.
|
||||
reconciliationReadTestHook func(rows int)
|
||||
}
|
||||
|
||||
// NewCache creates a new cache instance.
|
||||
@@ -127,6 +138,8 @@ func newCache(
|
||||
evictionDone: make(chan struct{}),
|
||||
metaCache: metaCache,
|
||||
contentLocks: newContentLock(),
|
||||
|
||||
reconciliationPageSize: defaultReconciliationPageSize,
|
||||
}
|
||||
|
||||
if c.disabled {
|
||||
@@ -471,8 +484,9 @@ func (c *Cache) Stats(ctx context.Context) (*CacheStats, error) {
|
||||
return nil, fmt.Errorf("failed to get cache stats: %w", err)
|
||||
}
|
||||
|
||||
// Count and size the cached source images and processed variants. A
|
||||
// disabled cache holds none, whatever rows an earlier run left.
|
||||
// Count and size the cached source images and processed variants from
|
||||
// their tables. A disabled cache holds none, whatever rows an earlier
|
||||
// run left.
|
||||
if !c.disabled {
|
||||
err = c.db.QueryRowContext(ctx, `
|
||||
SELECT (SELECT COUNT(*) FROM source_content)
|
||||
@@ -482,7 +496,7 @@ func (c *Cache) Stats(ctx context.Context) (*CacheStats, error) {
|
||||
c.log.Warn("failed to count cache items for stats", "error", err)
|
||||
}
|
||||
|
||||
stats.TotalSizeBytes, err = c.UsageBytes(ctx)
|
||||
stats.TotalSizeBytes, err = c.sumContentSizeBytes(ctx)
|
||||
if err != nil {
|
||||
c.log.Warn("failed to sum cache size for stats", "error", err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,265 @@
|
||||
package imgcache
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestEvictionReadsTheUsageTotal adds, stores again, resizes and evicts
|
||||
// cache content, then drops both content tables. UsageBytes must still
|
||||
// report what the tables held, and an eviction pass under the limit must
|
||||
// still succeed: neither may sum the tables.
|
||||
func TestEvictionReadsTheUsageTotal(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cache, _ := newEvictionTestCache(t, 1<<30)
|
||||
ctx := t.Context()
|
||||
|
||||
kept := storeEvictionTestSource(t, cache, "total.example.com", "/kept.jpg",
|
||||
bytes.Repeat([]byte{0x71}, 1000))
|
||||
evicted := storeEvictionTestSource(t, cache, "total.example.com", "/evicted.jpg",
|
||||
bytes.Repeat([]byte{0x72}, 700))
|
||||
|
||||
storeEvictionTestVariant(t, cache, testVariantKeyOne,
|
||||
bytes.Repeat([]byte{0x73}, 500))
|
||||
storeEvictionTestVariant(t, cache, testVariantKeyOne,
|
||||
bytes.Repeat([]byte{0x74}, 300))
|
||||
storeEvictionTestVariant(t, cache, testVariantKeyTwo,
|
||||
bytes.Repeat([]byte{0x75}, 200))
|
||||
|
||||
// pixa never changes a source's size, but a statement that does must
|
||||
// change the total too.
|
||||
_, err := cache.db.ExecContext(ctx,
|
||||
`UPDATE source_content SET size_bytes = 900 WHERE content_hash = ?`,
|
||||
string(kept))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to change the source's size: %v", err)
|
||||
}
|
||||
|
||||
err = cache.evictSourceBlob(ctx, evicted)
|
||||
if err != nil {
|
||||
t.Fatalf("evictSourceBlob failed: %v", err)
|
||||
}
|
||||
|
||||
err = cache.evictVariant(ctx, testVariantKeyTwo)
|
||||
if err != nil {
|
||||
t.Fatalf("evictVariant failed: %v", err)
|
||||
}
|
||||
|
||||
_, err = cache.db.ExecContext(ctx,
|
||||
`DROP TABLE source_content; DROP TABLE variant_content`)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to drop the content tables: %v", err)
|
||||
}
|
||||
|
||||
usage, err := cache.UsageBytes(ctx)
|
||||
t.Logf("UsageBytes() = %d, error = %v", usage, err)
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("UsageBytes() error = %v, want nil: it read a content table", err)
|
||||
}
|
||||
|
||||
if usage != 1200 {
|
||||
t.Errorf("UsageBytes() = %d, want 1200 (900 + 300)", usage)
|
||||
}
|
||||
|
||||
err = cache.EvictToLimit(ctx)
|
||||
if err != nil {
|
||||
t.Errorf("EvictToLimit() error = %v, want nil: it read a content table", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReconciliationCorrectsTheUsageTotal sets the total cache usage to a
|
||||
// wrong value and checks that a reconciliation pass, which sums both
|
||||
// content tables, puts it right.
|
||||
func TestReconciliationCorrectsTheUsageTotal(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cache, _ := newEvictionTestCache(t, 1<<30)
|
||||
ctx := t.Context()
|
||||
|
||||
storeEvictionTestSource(t, cache, "total.example.com", "/a.jpg",
|
||||
bytes.Repeat([]byte{0x76}, 1000))
|
||||
storeEvictionTestVariant(t, cache, testVariantKeyOne,
|
||||
bytes.Repeat([]byte{0x77}, 500))
|
||||
|
||||
_, err := cache.db.ExecContext(ctx,
|
||||
`UPDATE cache_usage SET total_size_bytes = 1 WHERE id = 1`)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to set a wrong total: %v", err)
|
||||
}
|
||||
|
||||
err = cache.reconcileAccounting(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("reconcileAccounting failed: %v", err)
|
||||
}
|
||||
|
||||
usage, err := cache.UsageBytes(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("UsageBytes failed: %v", err)
|
||||
}
|
||||
|
||||
if usage != 1500 {
|
||||
t.Errorf("UsageBytes() after reconciliation = %d, want 1500 (1000 + 500)",
|
||||
usage)
|
||||
}
|
||||
}
|
||||
|
||||
// TestUsageTotalNotCorrectedFromAnOlderSum stores a variant after the
|
||||
// change count was read, then offers a sum taken before the store as the
|
||||
// correction: the total must keep the stored variant, as that sum may
|
||||
// have missed it.
|
||||
func TestUsageTotalNotCorrectedFromAnOlderSum(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cache, _ := newEvictionTestCache(t, 1<<30)
|
||||
ctx := t.Context()
|
||||
|
||||
var changeCount int64
|
||||
|
||||
err := cache.db.QueryRowContext(ctx,
|
||||
`SELECT change_count FROM cache_usage WHERE id = 1`,
|
||||
).Scan(&changeCount)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read the change count: %v", err)
|
||||
}
|
||||
|
||||
storeEvictionTestVariant(t, cache, testVariantKeyOne,
|
||||
bytes.Repeat([]byte{0x78}, 500))
|
||||
|
||||
corrected, err := cache.correctUsageTotal(ctx, 0, changeCount)
|
||||
if err != nil {
|
||||
t.Fatalf("correctUsageTotal failed: %v", err)
|
||||
}
|
||||
|
||||
if corrected {
|
||||
t.Error("correctUsageTotal() = true, want false: content changed since the sum")
|
||||
}
|
||||
|
||||
usage, err := cache.UsageBytes(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("UsageBytes failed: %v", err)
|
||||
}
|
||||
|
||||
if usage != 500 {
|
||||
t.Errorf("UsageBytes() = %d, want 500", usage)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReconciliationReadsAPageAtATime stores five source images and five
|
||||
// variants, sets the page size to two rows and runs a reconciliation
|
||||
// pass. No read the pass makes of a content table, to check its rows or
|
||||
// to sum them, may cover more than two rows, so a request's query waits
|
||||
// for one page at most. Between them the reads must still cover every
|
||||
// row of both tables twice, once to check it and once to sum it, and the
|
||||
// sum must put a wrong total right.
|
||||
func TestReconciliationReadsAPageAtATime(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cache, _ := newEvictionTestCache(t, 1<<30)
|
||||
ctx := t.Context()
|
||||
|
||||
const pageSize = 2
|
||||
|
||||
cache.reconciliationPageSize = pageSize
|
||||
|
||||
// Sources of 100 bytes and variants of 10, five of each.
|
||||
for i := range 5 {
|
||||
content := []byte(strconv.Itoa(i))
|
||||
|
||||
storeEvictionTestSource(t, cache, "pages.example.com",
|
||||
"/"+strconv.Itoa(i)+".jpg", bytes.Repeat(content, 100))
|
||||
storeEvictionTestVariant(t, cache, VariantKey("aabbccdd000"+strconv.Itoa(i)),
|
||||
bytes.Repeat(content, 10))
|
||||
}
|
||||
|
||||
_, err := cache.db.ExecContext(ctx,
|
||||
`UPDATE cache_usage SET total_size_bytes = 1 WHERE id = 1`)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to set a wrong total: %v", err)
|
||||
}
|
||||
|
||||
var rowsPerRead []int
|
||||
|
||||
cache.reconciliationReadTestHook = func(rows int) {
|
||||
rowsPerRead = append(rowsPerRead, rows)
|
||||
}
|
||||
|
||||
err = cache.reconcileAccounting(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("reconcileAccounting failed: %v", err)
|
||||
}
|
||||
|
||||
t.Logf("rows covered by each read, in order: %v", rowsPerRead)
|
||||
|
||||
coveredRows := 0
|
||||
|
||||
for _, rows := range rowsPerRead {
|
||||
if rows > pageSize {
|
||||
t.Errorf("a read covered %d rows, want at most %d", rows, pageSize)
|
||||
}
|
||||
|
||||
coveredRows += rows
|
||||
}
|
||||
|
||||
// Ten rows, each read once to check it and once to sum it.
|
||||
if coveredRows != 20 {
|
||||
t.Errorf("the reads covered %d rows in all, want 20", coveredRows)
|
||||
}
|
||||
|
||||
usage, err := cache.UsageBytes(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("UsageBytes failed: %v", err)
|
||||
}
|
||||
|
||||
if usage != 550 {
|
||||
t.Errorf("UsageBytes() after reconciliation = %d, want 550 (5*100 + 5*10)",
|
||||
usage)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSourceEvictionCandidatesComeFromTheIndex checks that the query
|
||||
// choosing source images to evict reads source_content in last access
|
||||
// order from its index, instead of reading and sorting the whole table.
|
||||
func TestSourceEvictionCandidatesComeFromTheIndex(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cache, _ := newEvictionTestCache(t, 1<<30)
|
||||
|
||||
rows, err := cache.db.QueryContext(t.Context(),
|
||||
`EXPLAIN QUERY PLAN `+sourceCandidatesQuery, evictionBatchSize)
|
||||
if err != nil {
|
||||
t.Fatalf("EXPLAIN QUERY PLAN failed: %v", err)
|
||||
}
|
||||
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
var plan []string
|
||||
|
||||
for rows.Next() {
|
||||
var id, parent, unused int
|
||||
|
||||
var detail string
|
||||
|
||||
err := rows.Scan(&id, &parent, &unused, &detail)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to scan the query plan: %v", err)
|
||||
}
|
||||
|
||||
plan = append(plan, detail)
|
||||
}
|
||||
|
||||
err = rows.Err()
|
||||
if err != nil {
|
||||
t.Fatalf("query plan iteration failed: %v", err)
|
||||
}
|
||||
|
||||
t.Logf("query plan: %q", plan)
|
||||
|
||||
if !strings.Contains(strings.Join(plan, "\n"), "idx_source_content_last_accessed") {
|
||||
t.Errorf("the source candidate query does not use "+
|
||||
"idx_source_content_last_accessed; plan: %q", plan)
|
||||
}
|
||||
}
|
||||
@@ -72,7 +72,7 @@ func (c *Cache) computeDefaultMaxBytes(
|
||||
}
|
||||
|
||||
// Both terms are at most math.MaxInt64, so the sum cannot overflow.
|
||||
//nolint:gosec // G115: UsageBytes sums file sizes, never negative
|
||||
//nolint:gosec // G115: UsageBytes returns the total cache usage, never negative
|
||||
spaceBytes := min(freeBytes, math.MaxInt64) + uint64(usedBytes)
|
||||
|
||||
computed := spaceBytes / freeSpaceFractionDenominator * freeSpaceFractionNumerator
|
||||
|
||||
+246
-72
@@ -21,6 +21,12 @@ const DefaultEvictionInterval = 5 * time.Minute
|
||||
// and source blobs) one eviction pass fetches from the database.
|
||||
const evictionBatchSize = 100
|
||||
|
||||
// defaultReconciliationPageSize is the most rows one read of the
|
||||
// reconciliation pass returns, unless a test sets
|
||||
// Cache.reconciliationPageSize smaller. Each read is a query of its own,
|
||||
// so a request waits for one page at most, however large the cache is.
|
||||
const defaultReconciliationPageSize = 1000
|
||||
|
||||
// staleTempFileAge is how old an orphaned temp file (left behind by a
|
||||
// crashed write) must be before reconciliation removes it. Fresh temp
|
||||
// files may still belong to an in-flight store.
|
||||
@@ -45,7 +51,9 @@ const fallbackContentType = "application/octet-stream"
|
||||
|
||||
// UsageBytes returns the total number of bytes of cache content
|
||||
// tracked in the database (source content blobs plus processed
|
||||
// variants). It never scans the cache directories.
|
||||
// variants). It reads the total the database keeps up to date as rows
|
||||
// are added and removed, so it neither scans the cache directories nor
|
||||
// sums the tables.
|
||||
func (c *Cache) UsageBytes(ctx context.Context) (int64, error) {
|
||||
if c.disabled {
|
||||
return 0, nil
|
||||
@@ -53,12 +61,11 @@ func (c *Cache) UsageBytes(ctx context.Context) (int64, error) {
|
||||
|
||||
var total int64
|
||||
|
||||
err := c.db.QueryRowContext(ctx, `
|
||||
SELECT (SELECT COALESCE(SUM(size_bytes), 0) FROM source_content)
|
||||
+ (SELECT COALESCE(SUM(size_bytes), 0) FROM variant_content)
|
||||
`).Scan(&total)
|
||||
err := c.db.QueryRowContext(ctx,
|
||||
`SELECT total_size_bytes FROM cache_usage WHERE id = 1`,
|
||||
).Scan(&total)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to compute cache usage: %w", err)
|
||||
return 0, fmt.Errorf("failed to read cache usage: %w", err)
|
||||
}
|
||||
|
||||
return total, nil
|
||||
@@ -236,16 +243,19 @@ func (c *Cache) variantCandidates(ctx context.Context) ([]evictionCandidate, err
|
||||
return candidates, nil
|
||||
}
|
||||
|
||||
// sourceCandidates returns the least recently used source blobs. Rows
|
||||
// written before the LRU column existed fall back to fetched_at.
|
||||
// sourceCandidatesQuery selects the least recently used source blobs. It
|
||||
// orders by the last_accessed_at column itself, not by an expression, so
|
||||
// SQLite reads the rows in order from that column's index instead of
|
||||
// sorting the whole table.
|
||||
const sourceCandidatesQuery = `
|
||||
SELECT content_hash, size_bytes, last_accessed_at
|
||||
FROM source_content
|
||||
ORDER BY last_accessed_at ASC, content_hash ASC
|
||||
LIMIT ?`
|
||||
|
||||
// sourceCandidates returns the least recently used source blobs.
|
||||
func (c *Cache) sourceCandidates(ctx context.Context) ([]evictionCandidate, error) {
|
||||
rows, err := c.db.QueryContext(ctx, `
|
||||
SELECT content_hash, size_bytes,
|
||||
COALESCE(last_accessed_at, fetched_at, '1970-01-01 00:00:00') AS lru
|
||||
FROM source_content
|
||||
ORDER BY lru ASC, content_hash ASC
|
||||
LIMIT ?
|
||||
`, evictionBatchSize)
|
||||
rows, err := c.db.QueryContext(ctx, sourceCandidatesQuery, evictionBatchSize)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query source eviction candidates: %w", err)
|
||||
}
|
||||
@@ -532,11 +542,13 @@ func (c *Cache) runReconciliationPass(ctx context.Context) {
|
||||
// (or whose accounting insert failed, e.g. StoreVariant's best-effort
|
||||
// insert under transient DB contention), drops accounting rows whose
|
||||
// files are missing, removes source blob files the database does not
|
||||
// know (and rows whose files are gone), and sweeps stale temp files
|
||||
// left behind by crashed writes. Running it periodically, not just
|
||||
// once, bounds how long such drift can accumulate unaccounted for on a
|
||||
// long-running process to one eviction interval. Once ctx is cancelled,
|
||||
// it stops at the next file or row and returns ctx's error.
|
||||
// know (and rows whose files are gone), sweeps stale temp files left
|
||||
// behind by crashed writes, and last checks the total cache usage
|
||||
// against the tables. It reads the tables a page at a time. Running it
|
||||
// periodically, not just once, bounds how long such drift can
|
||||
// accumulate unaccounted for on a long-running process to one eviction
|
||||
// interval. Once ctx is cancelled, it stops at the next file or row and
|
||||
// returns ctx's error.
|
||||
func (c *Cache) reconcileAccounting(ctx context.Context) error {
|
||||
if c.disabled {
|
||||
return nil
|
||||
@@ -562,7 +574,7 @@ func (c *Cache) reconcileAccounting(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
return c.reconcileUsageTotal(ctx)
|
||||
}
|
||||
|
||||
// reconcileVariantFiles walks the variant storage directory, adopting
|
||||
@@ -657,46 +669,63 @@ func (c *Cache) variantContentTypeFromSidecar(variantPath string) string {
|
||||
// reconcileVariantRows drops accounting rows whose variant files are
|
||||
// missing, so the database never references deleted content.
|
||||
func (c *Cache) reconcileVariantRows(ctx context.Context) error {
|
||||
keys, err := c.allVariantKeys(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var after VariantKey
|
||||
|
||||
for _, key := range keys {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
if c.variants.Exists(key) {
|
||||
continue
|
||||
}
|
||||
|
||||
_, err := c.db.ExecContext(ctx,
|
||||
`DELETE FROM variant_content WHERE cache_key = ?`, string(key))
|
||||
for {
|
||||
keys, err := c.variantKeysAfter(ctx, after)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to drop stale variant accounting row: %w", err)
|
||||
return err
|
||||
}
|
||||
|
||||
c.log.Info("dropped accounting row for missing variant file", "cache_key", key)
|
||||
if c.reconciliationReadTestHook != nil {
|
||||
c.reconciliationReadTestHook(len(keys))
|
||||
}
|
||||
|
||||
if len(keys) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, key := range keys {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
if c.variants.Exists(key) {
|
||||
continue
|
||||
}
|
||||
|
||||
_, err := c.db.ExecContext(ctx,
|
||||
`DELETE FROM variant_content WHERE cache_key = ?`, string(key))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to drop stale variant accounting row: %w", err)
|
||||
}
|
||||
|
||||
c.log.Info("dropped accounting row for missing variant file", "cache_key", key)
|
||||
}
|
||||
|
||||
after = keys[len(keys)-1]
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// allVariantKeys returns every tracked variant cache key.
|
||||
func (c *Cache) allVariantKeys(ctx context.Context) ([]VariantKey, error) {
|
||||
return queryStringColumn[VariantKey](ctx, c.db,
|
||||
`SELECT cache_key FROM variant_content`, "variant keys", "variant key")
|
||||
// variantKeysAfter returns, in order, up to c.reconciliationPageSize
|
||||
// tracked variant cache keys that sort after the given one.
|
||||
func (c *Cache) variantKeysAfter(
|
||||
ctx context.Context, after VariantKey,
|
||||
) ([]VariantKey, error) {
|
||||
return queryStringColumn[VariantKey](ctx, c.db, `
|
||||
SELECT cache_key FROM variant_content
|
||||
WHERE cache_key > ? ORDER BY cache_key LIMIT ?
|
||||
`, "variant keys", "variant key", string(after), c.reconciliationPageSize)
|
||||
}
|
||||
|
||||
// queryStringColumn runs a single-column query and returns the column
|
||||
// values as T. plural names the set for the query and scan failure
|
||||
// messages; singular names one row for the scan and iteration failure
|
||||
// messages.
|
||||
// queryStringColumn runs a single-column query with args and returns the
|
||||
// column values as T. plural names the set for the query and scan
|
||||
// failure messages; singular names one row for the scan and iteration
|
||||
// failure messages.
|
||||
func queryStringColumn[T ~string](
|
||||
ctx context.Context, db *sql.DB, query, plural, singular string,
|
||||
ctx context.Context, db *sql.DB, query, plural, singular string, args ...any,
|
||||
) ([]T, error) {
|
||||
rows, err := db.QueryContext(ctx, query)
|
||||
rows, err := db.QueryContext(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query %s: %w", plural, err)
|
||||
}
|
||||
@@ -791,38 +820,183 @@ func (c *Cache) removeUntrackedSourceFile(
|
||||
// reconcileSourceRows removes source_content rows (and their metadata
|
||||
// references and sidecars) whose blob files are missing on disk.
|
||||
func (c *Cache) reconcileSourceRows(ctx context.Context) error {
|
||||
hashes, err := c.allSourceContentHashes(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var after ContentHash
|
||||
|
||||
for _, hash := range hashes {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
if c.srcContent.Exists(hash) {
|
||||
continue
|
||||
}
|
||||
|
||||
// The blob file is already gone; evictSourceBlob removes the
|
||||
// rows and sidecars and tolerates the missing file.
|
||||
err := c.evictSourceBlob(ctx, hash)
|
||||
for {
|
||||
hashes, err := c.sourceContentHashesAfter(ctx, after)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
c.log.Info("dropped rows for missing source content file", "content_hash", hash)
|
||||
if c.reconciliationReadTestHook != nil {
|
||||
c.reconciliationReadTestHook(len(hashes))
|
||||
}
|
||||
|
||||
if len(hashes) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, hash := range hashes {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
if c.srcContent.Exists(hash) {
|
||||
continue
|
||||
}
|
||||
|
||||
// The blob file is already gone; evictSourceBlob removes the
|
||||
// rows and sidecars and tolerates the missing file.
|
||||
err := c.evictSourceBlob(ctx, hash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
c.log.Info("dropped rows for missing source content file", "content_hash", hash)
|
||||
}
|
||||
|
||||
after = hashes[len(hashes)-1]
|
||||
}
|
||||
}
|
||||
|
||||
// sourceContentHashesAfter returns, in order, up to
|
||||
// c.reconciliationPageSize tracked source content hashes that sort after
|
||||
// the given one.
|
||||
func (c *Cache) sourceContentHashesAfter(
|
||||
ctx context.Context, after ContentHash,
|
||||
) ([]ContentHash, error) {
|
||||
return queryStringColumn[ContentHash](ctx, c.db, `
|
||||
SELECT content_hash FROM source_content
|
||||
WHERE content_hash > ? ORDER BY content_hash LIMIT ?
|
||||
`, "source content hashes", "content hash", string(after),
|
||||
c.reconciliationPageSize)
|
||||
}
|
||||
|
||||
// Each of these queries sums size_bytes over the next page of rows of
|
||||
// one content table, the rows that sort after a key, and returns the
|
||||
// page's last key, the sum and the number of rows in the page. Past the
|
||||
// last row the page has no rows and the key is NULL.
|
||||
const (
|
||||
sourceSizePageQuery = `
|
||||
SELECT MAX(content_hash), COALESCE(SUM(size_bytes), 0), COUNT(*)
|
||||
FROM (
|
||||
SELECT content_hash, size_bytes FROM source_content
|
||||
WHERE content_hash > ? ORDER BY content_hash LIMIT ?
|
||||
)`
|
||||
variantSizePageQuery = `
|
||||
SELECT MAX(cache_key), COALESCE(SUM(size_bytes), 0), COUNT(*)
|
||||
FROM (
|
||||
SELECT cache_key, size_bytes FROM variant_content
|
||||
WHERE cache_key > ? ORDER BY cache_key LIMIT ?
|
||||
)`
|
||||
)
|
||||
|
||||
// sumContentSizeBytes sums size_bytes over both content tables, a page
|
||||
// of rows per query.
|
||||
func (c *Cache) sumContentSizeBytes(ctx context.Context) (int64, error) {
|
||||
sourceBytes, err := c.sumSizeBytesInPages(ctx, sourceSizePageQuery)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
variantBytes, err := c.sumSizeBytesInPages(ctx, variantSizePageQuery)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return sourceBytes + variantBytes, nil
|
||||
}
|
||||
|
||||
// sumSizeBytesInPages runs pageQuery, one of the size page queries
|
||||
// above, from the first page to the last and adds up the page sums.
|
||||
func (c *Cache) sumSizeBytesInPages(
|
||||
ctx context.Context, pageQuery string,
|
||||
) (int64, error) {
|
||||
var total int64
|
||||
|
||||
after := ""
|
||||
|
||||
for {
|
||||
var lastKey sql.NullString
|
||||
|
||||
var pageBytes int64
|
||||
|
||||
var pageRows int
|
||||
|
||||
err := c.db.QueryRowContext(ctx, pageQuery, after, c.reconciliationPageSize).
|
||||
Scan(&lastKey, &pageBytes, &pageRows)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to sum cache content sizes: %w", err)
|
||||
}
|
||||
|
||||
if c.reconciliationReadTestHook != nil {
|
||||
c.reconciliationReadTestHook(pageRows)
|
||||
}
|
||||
|
||||
if pageRows == 0 {
|
||||
return total, nil
|
||||
}
|
||||
|
||||
total += pageBytes
|
||||
after = lastKey.String
|
||||
}
|
||||
}
|
||||
|
||||
// reconcileUsageTotal checks the total cache usage the database keeps
|
||||
// against size_bytes summed over both content tables, and corrects the
|
||||
// total when they differ.
|
||||
func (c *Cache) reconcileUsageTotal(ctx context.Context) error {
|
||||
var totalBytes, changeCount int64
|
||||
|
||||
err := c.db.QueryRowContext(ctx,
|
||||
`SELECT total_size_bytes, change_count FROM cache_usage WHERE id = 1`,
|
||||
).Scan(&totalBytes, &changeCount)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read cache usage: %w", err)
|
||||
}
|
||||
|
||||
sumBytes, err := c.sumContentSizeBytes(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if sumBytes == totalBytes {
|
||||
return nil
|
||||
}
|
||||
|
||||
corrected, err := c.correctUsageTotal(ctx, sumBytes, changeCount)
|
||||
if err != nil || !corrected {
|
||||
return err
|
||||
}
|
||||
|
||||
c.log.Warn("corrected total cache usage to the sum of the content tables",
|
||||
"previous_usage_bytes", totalBytes, "usage_bytes", sumBytes)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// allSourceContentHashes returns every tracked source content hash.
|
||||
func (c *Cache) allSourceContentHashes(ctx context.Context) ([]ContentHash, error) {
|
||||
return queryStringColumn[ContentHash](ctx, c.db,
|
||||
`SELECT content_hash FROM source_content`,
|
||||
"source content hashes", "content hash")
|
||||
// correctUsageTotal sets the total cache usage to sumBytes, a sum of the
|
||||
// content tables taken when the change count was changeCount, and
|
||||
// reports whether it did. If a row was added, removed or resized since,
|
||||
// the count has moved and the total is left alone: the sum may have
|
||||
// missed that change, and the next pass checks again.
|
||||
func (c *Cache) correctUsageTotal(
|
||||
ctx context.Context, sumBytes, changeCount int64,
|
||||
) (bool, error) {
|
||||
result, err := c.db.ExecContext(ctx, `
|
||||
UPDATE cache_usage SET total_size_bytes = ?
|
||||
WHERE id = 1 AND change_count = ?
|
||||
`, sumBytes, changeCount)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to correct cache usage: %w", err)
|
||||
}
|
||||
|
||||
affected, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to read cache usage correction: %w", err)
|
||||
}
|
||||
|
||||
return affected > 0, nil
|
||||
}
|
||||
|
||||
// sweepStaleTempFile removes a temp file left behind by a crashed
|
||||
|
||||
@@ -21,11 +21,12 @@ const (
|
||||
FormatPNG ImageFormat = "png"
|
||||
FormatWebP ImageFormat = "webp"
|
||||
FormatAVIF ImageFormat = "avif"
|
||||
FormatJXL ImageFormat = "jxl"
|
||||
FormatGIF ImageFormat = "gif"
|
||||
|
||||
// FormatAuto stands for AVIF, WebP or JPEG, chosen for each request
|
||||
// from its Accept header once the URL's signature or token has been
|
||||
// checked; it is never processed or cached as itself.
|
||||
// FormatAuto stands for JPEG XL, AVIF, WebP or JPEG, chosen for each
|
||||
// request from its Accept header once the URL's signature or token has
|
||||
// been checked; it is never processed or cached as itself.
|
||||
FormatAuto ImageFormat = "auto"
|
||||
)
|
||||
|
||||
|
||||
@@ -100,6 +100,13 @@ func NewService(cfg *ServiceConfig) (*Service, error) {
|
||||
allowHTTP = cfg.FetcherConfig.AllowHTTP
|
||||
}
|
||||
|
||||
// JPEG XL is the default output format, so pixad does not start
|
||||
// without it.
|
||||
err := imageprocessor.CheckJPEGXLSupport()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
maxResponseSize := fetcherCfg.MaxResponseSize
|
||||
processor := imageprocessor.New(imageprocessor.Params{
|
||||
MaxInputBytes: maxResponseSize,
|
||||
|
||||
@@ -38,8 +38,10 @@ func ValidateDimension(name string, value int) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// sizeFormatRegex matches patterns like "800x600.webp", "0x0.jpeg", "orig.png"
|
||||
var sizeFormatRegex = regexp.MustCompile(`^(\d+)x(\d+)\.(\w+)$|^(orig)\.(\w+)$`)
|
||||
// sizeFormatRegex matches patterns like "800x600.webp", "0x0.jpeg", "orig.png",
|
||||
// and a size with no format, such as "800x600" or "orig"
|
||||
var sizeFormatRegex = regexp.MustCompile(
|
||||
`^(\d+)x(\d+)(?:\.(\w+))?$|^(orig)(?:\.(\w+))?$`)
|
||||
|
||||
// ParsedURL contains the parsed components of an image proxy URL.
|
||||
type ParsedURL struct {
|
||||
@@ -56,12 +58,13 @@ type ParsedURL struct {
|
||||
}
|
||||
|
||||
// ParseImagePath parses the path captured by chi's wildcard:
|
||||
// <host>/<path>/<size>.<format>
|
||||
// <host>/<path>/<size>.<format>, or <host>/<path>/<size> for JPEG XL
|
||||
// This is the primary entry point when using chi routing.
|
||||
// Examples:
|
||||
// - cdn.example.com/photos/cat.jpg/800x600.webp
|
||||
// - cdn.example.com/photos/cat.jpg/0x0.jpeg
|
||||
// - cdn.example.com/photos/cat.jpg/orig.png
|
||||
// - cdn.example.com/photos/cat.jpg/800x600
|
||||
func ParseImagePath(path string) (*ParsedURL, error) {
|
||||
// Strip leading slash if present (chi may include it)
|
||||
path = strings.TrimPrefix(path, "/")
|
||||
@@ -72,7 +75,8 @@ func ParseImagePath(path string) (*ParsedURL, error) {
|
||||
return parseImageComponents(path)
|
||||
}
|
||||
|
||||
// ParseImageURL parses a full URL path like /v1/image/<host>/<path>/<size>.<format>
|
||||
// ParseImageURL parses a full URL path like /v1/image/<host>/<path>/<size>.<format>,
|
||||
// or /v1/image/<host>/<path>/<size> for JPEG XL
|
||||
// Use ParseImagePath instead when working with chi's wildcard capture.
|
||||
func ParseImageURL(urlPath string) (*ParsedURL, error) {
|
||||
// Remove the /v1/image/ prefix
|
||||
@@ -89,7 +93,8 @@ func ParseImageURL(urlPath string) (*ParsedURL, error) {
|
||||
return parseImageComponents(remainder)
|
||||
}
|
||||
|
||||
// parseImageComponents parses <host>/<path>/<size>.<format> structure.
|
||||
// parseImageComponents parses <host>/<path>/<size>.<format>, or
|
||||
// <host>/<path>/<size> for JPEG XL.
|
||||
func parseImageComponents(remainder string) (*ParsedURL, error) {
|
||||
// Check for path traversal before any other processing
|
||||
err := checkPathTraversal(remainder)
|
||||
@@ -97,7 +102,7 @@ func parseImageComponents(remainder string) (*ParsedURL, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Find the last path segment which contains size.format
|
||||
// Find the last path segment, which holds "size" or "size.format"
|
||||
lastSlash := strings.LastIndex(remainder, "/")
|
||||
if lastSlash == -1 {
|
||||
return nil, ErrMissingSize
|
||||
@@ -212,7 +217,7 @@ func checkPathTraversal(path string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// parseSizeFormat parses strings like "800x600.webp" or "orig.png"
|
||||
// parseSizeFormat parses strings like "800x600.webp", "orig.png" or "800x600"
|
||||
func parseSizeFormat(s string) (Size, ImageFormat, error) {
|
||||
matches := sizeFormatRegex.FindStringSubmatch(s)
|
||||
if matches == nil {
|
||||
@@ -225,11 +230,11 @@ func parseSizeFormat(s string) (Size, ImageFormat, error) {
|
||||
)
|
||||
|
||||
if matches[4] == "orig" {
|
||||
// "orig.format" pattern
|
||||
// "orig" or "orig.format" pattern
|
||||
size = Size{Width: 0, Height: 0}
|
||||
formatStr = matches[5]
|
||||
} else {
|
||||
// "WxH.format" pattern
|
||||
// "WxH" or "WxH.format" pattern
|
||||
width, err := strconv.Atoi(matches[1])
|
||||
if err != nil {
|
||||
return Size{}, "", ErrInvalidSize
|
||||
@@ -254,6 +259,11 @@ func parseSizeFormat(s string) (Size, ImageFormat, error) {
|
||||
return Size{}, "", err
|
||||
}
|
||||
|
||||
// A URL that names no format is served, and signed, as JPEG XL
|
||||
if formatStr == "" {
|
||||
return size, FormatJXL, nil
|
||||
}
|
||||
|
||||
format, err := parseFormat(formatStr)
|
||||
if err != nil {
|
||||
return Size{}, "", err
|
||||
@@ -275,6 +285,8 @@ func parseFormat(s string) (ImageFormat, error) {
|
||||
return FormatWebP, nil
|
||||
case "avif":
|
||||
return FormatAVIF, nil
|
||||
case "jxl":
|
||||
return FormatJXL, nil
|
||||
case "gif":
|
||||
return FormatGIF, nil
|
||||
case "auto":
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
package magic
|
||||
|
||||
import "testing"
|
||||
|
||||
// testMIMEJXL is the MIME type of JPEG XL.
|
||||
const testMIMEJXL = "image/jxl"
|
||||
|
||||
// TestDetectFormatJPEGXL verifies that both JPEG XL signatures, that of a
|
||||
// bare codestream and that of the container, are detected as image/jxl.
|
||||
func TestDetectFormatJPEGXL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
data []byte
|
||||
}{
|
||||
{"codestream", pad(0xFF, 0x0A)},
|
||||
{"container", pad(
|
||||
0x00, 0x00, 0x00, 0x0C, 0x4A, 0x58, 0x4C, 0x20, 0x0D, 0x0A, 0x87, 0x0A,
|
||||
)},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
got, err := DetectFormat(tt.data)
|
||||
if err != nil || got != MIMETypeJXL {
|
||||
t.Errorf("DetectFormat() = %q, %v, want %q", got, err, testMIMEJXL)
|
||||
}
|
||||
|
||||
err = ValidateMagicBytes(tt.data, testMIMEJXL)
|
||||
if err != nil {
|
||||
t.Errorf("ValidateMagicBytes(%q) error = %v", testMIMEJXL, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestJPEGXLMIMEType verifies that image/jxl is a supported input type and
|
||||
// maps to and from the format jxl.
|
||||
func TestJPEGXLMIMEType(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if !IsSupportedMIMEType(testMIMEJXL) {
|
||||
t.Errorf("IsSupportedMIMEType(%q) = false", testMIMEJXL)
|
||||
}
|
||||
|
||||
format, ok := MIMEToImageFormat(testMIMEJXL)
|
||||
if format != FormatJXL || !ok {
|
||||
t.Errorf("MIMEToImageFormat(%q) = %q, %v, want %q, true",
|
||||
testMIMEJXL, format, ok, FormatJXL)
|
||||
}
|
||||
|
||||
if got := ImageFormatToMIME(FormatJXL); got != testMIMEJXL {
|
||||
t.Errorf("ImageFormatToMIME(%q) = %q, want %q", FormatJXL, got, testMIMEJXL)
|
||||
}
|
||||
}
|
||||
+19
-1
@@ -26,6 +26,7 @@ const (
|
||||
MIMETypeWebP = MIMEType("image/webp")
|
||||
MIMETypeGIF = MIMEType("image/gif")
|
||||
MIMETypeAVIF = MIMEType("image/avif")
|
||||
MIMETypeJXL = MIMEType("image/jxl")
|
||||
MIMETypeSVG = MIMEType("image/svg+xml")
|
||||
)
|
||||
|
||||
@@ -40,6 +41,7 @@ const (
|
||||
FormatPNG ImageFormat = "png"
|
||||
FormatWebP ImageFormat = "webp"
|
||||
FormatAVIF ImageFormat = "avif"
|
||||
FormatJXL ImageFormat = "jxl"
|
||||
FormatGIF ImageFormat = "gif"
|
||||
)
|
||||
|
||||
@@ -62,6 +64,11 @@ var (
|
||||
// AVIF uses the ftyp box with brand "avif" or "avis"
|
||||
// Format: size(4 bytes) + "ftyp" + brand(4 bytes)
|
||||
magicFtyp = []byte{0x66, 0x74, 0x79, 0x70} // "ftyp"
|
||||
// JPEG XL is a bare codestream or a codestream in a container
|
||||
magicJXLCodestream = []byte{0xFF, 0x0A}
|
||||
magicJXLContainer = []byte{
|
||||
0x00, 0x00, 0x00, 0x0C, 0x4A, 0x58, 0x4C, 0x20, 0x0D, 0x0A, 0x87, 0x0A,
|
||||
}
|
||||
)
|
||||
|
||||
// WebP identifier appears at offset 8 after RIFF header.
|
||||
@@ -115,6 +122,12 @@ func DetectFormat(data []byte) (MIMEType, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// Check JPEG XL (FF0A, or the 12-byte container signature)
|
||||
if bytes.HasPrefix(data, magicJXLCodestream) ||
|
||||
bytes.HasPrefix(data, magicJXLContainer) {
|
||||
return MIMETypeJXL, nil
|
||||
}
|
||||
|
||||
// Check SVG - look for XML declaration or SVG tag
|
||||
if detectSVG(data) {
|
||||
return MIMETypeSVG, nil
|
||||
@@ -181,7 +194,8 @@ func normalizeMIMEType(mimeType string) string {
|
||||
func IsSupportedMIMEType(mimeType string) bool {
|
||||
normalized := normalizeMIMEType(mimeType)
|
||||
switch MIMEType(normalized) {
|
||||
case MIMETypeJPEG, MIMETypePNG, MIMETypeWebP, MIMETypeGIF, MIMETypeAVIF, MIMETypeSVG:
|
||||
case MIMETypeJPEG, MIMETypePNG, MIMETypeWebP, MIMETypeGIF, MIMETypeAVIF,
|
||||
MIMETypeJXL, MIMETypeSVG:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
@@ -226,6 +240,8 @@ func MIMEToImageFormat(mimeType string) (ImageFormat, bool) {
|
||||
return FormatGIF, true
|
||||
case MIMETypeAVIF:
|
||||
return FormatAVIF, true
|
||||
case MIMETypeJXL:
|
||||
return FormatJXL, true
|
||||
case MIMETypeSVG:
|
||||
// SVG has no corresponding output format.
|
||||
return "", false
|
||||
@@ -247,6 +263,8 @@ func ImageFormatToMIME(format ImageFormat) string {
|
||||
return string(MIMETypeGIF)
|
||||
case FormatAVIF:
|
||||
return string(MIMETypeAVIF)
|
||||
case FormatJXL:
|
||||
return string(MIMETypeJXL)
|
||||
case FormatOriginal:
|
||||
// Original format passes content through unchanged.
|
||||
return mimeOctetStream
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/davidbyttow/govips/v2/vips"
|
||||
)
|
||||
|
||||
// TestImageProxyFlowJPEGXL requests the JPEG XL testdata/red.jxl, 64x48 and
|
||||
// made with libvips, through pixa's router, upstream fetcher, image processor
|
||||
// and disk cache: resized as PNG, resized as JPEG XL, and at its own size and
|
||||
// format, which is JPEG XL. The second request for each URL is a cache hit.
|
||||
func TestImageProxyFlowJPEGXL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
source, err := os.ReadFile("testdata/red.jxl")
|
||||
if err != nil {
|
||||
t.Fatalf("reading the test image: %v", err)
|
||||
}
|
||||
|
||||
upstream := httptest.NewServer(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "image/jxl")
|
||||
_, _ = w.Write(source)
|
||||
}))
|
||||
t.Cleanup(upstream.Close)
|
||||
|
||||
s, _, _ := startImageProxy(t, upstream)
|
||||
|
||||
tests := []struct {
|
||||
sizeFormat string // the <size>.<format> part of the image URL
|
||||
contentType string
|
||||
imageType vips.ImageType
|
||||
width, height int
|
||||
}{
|
||||
{"32x24.png", "image/png", vips.ImageTypePNG, 32, 24},
|
||||
{"32x24.jxl", "image/jxl", vips.ImageTypeJXL, 32, 24},
|
||||
{"orig.orig", "image/jxl", vips.ImageTypeJXL, 64, 48},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
target := "/v1/image/" + upstreamHost + "/red.jxl/" + tt.sizeFormat
|
||||
|
||||
for _, wantCache := range []string{"MISS", "HIT"} {
|
||||
rec := getImage(t, s, target)
|
||||
gotType := rec.Header().Get("Content-Type")
|
||||
gotCache := rec.Header().Get("X-Pixa-Cache")
|
||||
|
||||
if gotType != tt.contentType || gotCache != wantCache {
|
||||
t.Errorf("%s: %s, X-Pixa-Cache %s, want %s, %s",
|
||||
target, gotType, gotCache, tt.contentType, wantCache)
|
||||
}
|
||||
|
||||
img, err := vips.NewImageFromBuffer(rec.Body.Bytes())
|
||||
if err != nil {
|
||||
t.Fatalf("%s: loading the image: %v", target, err)
|
||||
}
|
||||
|
||||
if img.Format() != tt.imageType ||
|
||||
img.Width() != tt.width || img.Height() != tt.height {
|
||||
t.Errorf("%s: %s %dx%d, want %s %dx%d", target,
|
||||
vips.ImageTypes[img.Format()], img.Width(), img.Height(),
|
||||
vips.ImageTypes[tt.imageType], tt.width, tt.height)
|
||||
}
|
||||
|
||||
img.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -89,7 +89,8 @@ func (s *Server) SetupRoutes() {
|
||||
r.Use(s.refuseDuringMaintenance)
|
||||
|
||||
// Main image proxy route
|
||||
// /v1/image/<host>/<path>/<width>x<height>.<format>
|
||||
// /v1/image/<host>/<path>/<width>x<height>.<format>, or with no
|
||||
// format /v1/image/<host>/<path>/<width>x<height>
|
||||
r.Get("/image/*", s.h.HandleImage())
|
||||
r.Head("/image/*", s.h.HandleImage())
|
||||
|
||||
|
||||
Vendored
BIN
Binary file not shown.
@@ -95,11 +95,12 @@
|
||||
</label>
|
||||
<select id="format" name="format">
|
||||
<option value="orig" {{if eq .FormFormat "orig"}}selected{{end}}>Original</option>
|
||||
<option value="auto" {{if eq .FormFormat "auto"}}selected{{end}}>Auto (AVIF, WebP or JPEG)</option>
|
||||
<option value="auto" {{if eq .FormFormat "auto"}}selected{{end}}>Auto (JPEG XL, AVIF, WebP or JPEG)</option>
|
||||
<option value="jpeg" {{if eq .FormFormat "jpeg"}}selected{{end}}>JPEG</option>
|
||||
<option value="png" {{if eq .FormFormat "png"}}selected{{end}}>PNG</option>
|
||||
<option value="webp" {{if eq .FormFormat "webp"}}selected{{end}}>WebP</option>
|
||||
<option value="avif" {{if eq .FormFormat "avif"}}selected{{end}}>AVIF</option>
|
||||
<option value="jxl" {{if or (eq .FormFormat "jxl") (eq .FormFormat "")}}selected{{end}}>JPEG XL</option>
|
||||
<option value="gif" {{if eq .FormFormat "gif"}}selected{{end}}>GIF</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
+16
-5
@@ -14,11 +14,12 @@
|
||||
# script/fmt-check: all the host needs, as
|
||||
# the checks compile pixa in Docker
|
||||
# script/bootstrap --cgo git, make, Go, and a C compiler and the
|
||||
# CGO image libraries (pkg-config, vips,
|
||||
# libheif) for the govips bindings instead
|
||||
# of Node: to compile pixa, in the
|
||||
# Dockerfile's test phase and build stage,
|
||||
# which format nothing
|
||||
# CGO image libraries (pkg-config, vips
|
||||
# with its JPEG XL support, libheif) for
|
||||
# the govips bindings instead of Node: to
|
||||
# compile pixa, in the Dockerfile's test
|
||||
# phase and build stage, which format
|
||||
# nothing
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
@@ -150,6 +151,16 @@ ensure_cgo_deps() {
|
||||
if ! pkg-config --exists vips; then
|
||||
pkg_install vips libvips-dev vips vips-dev
|
||||
fi
|
||||
# libvips' JPEG XL loader and saver are in the nix and brew vips
|
||||
# packages, and in apt's from Debian 12 and Ubuntu 24.04 on, but in
|
||||
# the package vips-jxl on Alpine. detect_pkgmgr is called only where
|
||||
# apk exists, as on apt it updates the package lists.
|
||||
if ! missing apk; then
|
||||
detect_pkgmgr
|
||||
fi
|
||||
if [ "$PKGMGR" = "apk" ] && ! apk info -e vips-jxl >/dev/null; then
|
||||
apk add --no-cache vips-jxl
|
||||
fi
|
||||
if ! pkg-config --exists libheif; then
|
||||
pkg_install libheif libheif-dev libheif libheif-dev
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user