Compare commits
15
Commits
f743930f71
...
next
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
957ec85f97 | ||
|
|
930e43961e | ||
|
|
9eacfa6406 | ||
|
|
0ab15324f7 | ||
|
|
faad959bea | ||
|
|
3a6b3bc202 | ||
|
|
08d6c3c459 | ||
|
|
efa431ba18 | ||
|
|
8b4dcda549 | ||
|
|
6d271d4617 | ||
|
|
4a4bc9758f | ||
|
|
9a96740463 | ||
|
|
1fd5cf8bd0 | ||
|
|
62b4f4791e | ||
|
|
22bfc432b6 |
+31
-22
@@ -1,12 +1,16 @@
|
|||||||
# Every stage declares ARG CACHEBUST first: script/ passes a new value
|
# In every stage ARG CACHEBUST comes right after the steps that install
|
||||||
# on every build, which reruns every RUN step after it. The steps that
|
# system packages and download Go's modules, or right after FROM in a
|
||||||
# compile mount Go's build cache, which survives from build to build.
|
# stage with neither, and every step that lints, tests or compiles
|
||||||
|
# comes after it. script/ passes a new value on every build, which
|
||||||
|
# reruns every RUN step after it. Docker keeps the layers above it
|
||||||
|
# until the base image, their own lines or go.mod and go.sum change.
|
||||||
|
# The steps that compile mount Go's build cache, and golangci-lint run
|
||||||
|
# also mounts golangci-lint's; both survive from build to build.
|
||||||
|
|
||||||
# Lint phase. script/lint builds it alone. The linter is run directly:
|
# Lint phase. script/lint builds it alone. The linter is run directly:
|
||||||
# `make lint` and script/lint are themselves a docker build.
|
# `make lint` and script/lint are themselves a docker build.
|
||||||
# golangci/golangci-lint:v2.14.0, 2026-10-15
|
# golangci/golangci-lint:v2.14.0, 2026-10-15
|
||||||
FROM golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint
|
FROM golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint
|
||||||
ARG CACHEBUST
|
|
||||||
|
|
||||||
# The linter compiles every package, and govips needs the libvips
|
# The linter compiles every package, and govips needs the libvips
|
||||||
# headers for that. REPO_POLICIES.md has the lint phase install them
|
# headers for that. REPO_POLICIES.md has the lint phase install them
|
||||||
@@ -18,25 +22,29 @@ RUN apt-get update \
|
|||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
RUN go mod download
|
RUN go mod download
|
||||||
|
ARG CACHEBUST
|
||||||
COPY . .
|
COPY . .
|
||||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
RUN --mount=type=cache,target=/root/.cache/go-build \
|
||||||
|
--mount=type=cache,target=/root/.cache/golangci-lint \
|
||||||
golangci-lint run --config .golangci.yml ./...
|
golangci-lint run --config .golangci.yml ./...
|
||||||
|
|
||||||
# Test phase. script/test builds it alone. -race needs cgo and so a C
|
# Test phase. script/test builds it alone. -race needs cgo and so a C
|
||||||
# compiler, which the Debian Go image ships and the alpine one does not.
|
# compiler, which the Debian Go image ships and the alpine one does not.
|
||||||
# golang:1.25.4-trixie, 2026-10-15
|
# golang:1.25.4-trixie, 2026-10-15
|
||||||
FROM golang@sha256:a02d35efc036053fdf0da8c15919276bf777a80cbfda6a35c5e9f087e652adfc AS test
|
FROM golang@sha256:a02d35efc036053fdf0da8c15919276bf777a80cbfda6a35c5e9f087e652adfc AS test
|
||||||
ARG CACHEBUST
|
|
||||||
|
# govips compiles against the libvips and libheif headers, and the tests
|
||||||
|
# need libvips' JPEG XL support; libvips-dev brings all three. The tests
|
||||||
|
# also encode AVIF, which needs libheif's AVIF encoder,
|
||||||
|
# libheif-plugin-aomenc: Debian only recommends it, so it is named here.
|
||||||
|
RUN apt-get update \
|
||||||
|
&& apt-get install -y --no-install-recommends libvips-dev libheif-plugin-aomenc \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
|
|
||||||
# script/bootstrap --cgo installs the build dependencies (the libvips
|
|
||||||
# and libheif headers, and libvips' JPEG XL support, which the tests
|
|
||||||
# need) and downloads the Go modules.
|
|
||||||
COPY script/ ./script/
|
|
||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
RUN script/bootstrap --cgo
|
RUN go mod download
|
||||||
|
ARG CACHEBUST
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
# Without -v first; on a failure, again with -v for the details, and
|
# Without -v first; on a failure, again with -v for the details, and
|
||||||
@@ -52,22 +60,23 @@ RUN --mount=type=cache,target=/root/.cache/go-build \
|
|||||||
go test -count=1 -timeout 90s -race -parallel 4 -v ./...; exit 1; }
|
go test -count=1 -timeout 90s -race -parallel 4 -v ./...; exit 1; }
|
||||||
|
|
||||||
# Build stage. Nothing is wanted from the two phases above: these copies
|
# Build stage. Nothing is wanted from the two phases above: these copies
|
||||||
# make BuildKit build them first, so this stage runs only when lint and
|
# make BuildKit build them first, so nothing after them runs unless lint
|
||||||
# test passed.
|
# and test passed.
|
||||||
# golang:1.25.4-alpine3.22, 2026-02-25; the runtime stage uses Alpine 3.22 too
|
# golang:1.25.4-alpine3.22, 2026-02-25; the runtime stage uses Alpine 3.22 too
|
||||||
FROM golang:1.25.4-alpine3.22@sha256:d3f0cf7723f3429e3f9ed846243970b20a2de7bae6a5b66fc5914e228d831bbb AS builder
|
FROM golang:1.25.4-alpine3.22@sha256:d3f0cf7723f3429e3f9ed846243970b20a2de7bae6a5b66fc5914e228d831bbb AS builder
|
||||||
|
|
||||||
|
# git for the version, and for govips a C compiler, pkg-config and the
|
||||||
|
# libvips and libheif headers
|
||||||
|
RUN apk add --no-cache git build-base pkgconfig vips-dev libheif-dev
|
||||||
|
|
||||||
|
WORKDIR /src
|
||||||
|
COPY go.mod go.sum ./
|
||||||
|
RUN go mod download
|
||||||
ARG CACHEBUST
|
ARG CACHEBUST
|
||||||
|
|
||||||
COPY --from=lint /src/go.sum /dev/null
|
COPY --from=lint /src/go.sum /dev/null
|
||||||
COPY --from=test /src/go.sum /dev/null
|
COPY --from=test /src/go.sum /dev/null
|
||||||
|
|
||||||
WORKDIR /src
|
|
||||||
|
|
||||||
# Build dependencies, git among them, and Go modules, as in the test
|
|
||||||
# phase
|
|
||||||
COPY script/ ./script/
|
|
||||||
COPY go.mod go.sum ./
|
|
||||||
RUN script/bootstrap --cgo
|
|
||||||
# A tar-stream context keeps the sender's file owners, which git refuses.
|
# A tar-stream context keeps the sender's file owners, which git refuses.
|
||||||
RUN git config --system --add safe.directory /src
|
RUN git config --system --add safe.directory /src
|
||||||
|
|
||||||
@@ -102,7 +111,6 @@ RUN --mount=type=cache,target=/root/.cache/go-build \
|
|||||||
# the libvips and musl it was built with.
|
# the libvips and musl it was built with.
|
||||||
# alpine:3.22, 2026-10-08
|
# alpine:3.22, 2026-10-08
|
||||||
FROM alpine:3.22@sha256:5291449c3df73caf6ed85e649dec1b9e818b39a5d8c871e97afc13e9cd5e8fa8
|
FROM alpine:3.22@sha256:5291449c3df73caf6ed85e649dec1b9e818b39a5d8c871e97afc13e9cd5e8fa8
|
||||||
ARG CACHEBUST
|
|
||||||
|
|
||||||
# Install runtime dependencies only. vips-jxl is libvips' JPEG XL
|
# Install runtime dependencies only. vips-jxl is libvips' JPEG XL
|
||||||
# support, without which pixad does not start.
|
# support, without which pixad does not start.
|
||||||
@@ -113,6 +121,7 @@ RUN apk add --no-cache \
|
|||||||
ca-certificates \
|
ca-certificates \
|
||||||
tzdata \
|
tzdata \
|
||||||
su-exec
|
su-exec
|
||||||
|
ARG CACHEBUST
|
||||||
|
|
||||||
# Copy binary from builder
|
# Copy binary from builder
|
||||||
COPY --from=builder /pixad /usr/local/bin/pixad
|
COPY --from=builder /pixad /usr/local/bin/pixad
|
||||||
|
|||||||
@@ -102,8 +102,8 @@ it uses libvips through CGO. pixad does not start unless libvips has its JPEG XL
|
|||||||
support, which on Alpine is the `vips-jxl` package and which the nix and brew
|
support, which on Alpine is the `vips-jxl` package and which the nix and brew
|
||||||
packages of libvips include, as do the apt ones from Debian 12 and Ubuntu 24.04
|
packages of libvips include, as do the apt ones from Debian 12 and Ubuntu 24.04
|
||||||
on. Building pixa also needs the development files of libvips and libheif,
|
on. Building pixa also needs the development files of libvips and libheif,
|
||||||
`pkg-config` and a C compiler. `script/bootstrap --cgo` installs all of these,
|
`pkg-config` and a C compiler. `script/bootstrap --cgo` installs all of these;
|
||||||
as the `Dockerfile` does where it compiles pixa. Plain `script/bootstrap`, which
|
the `Dockerfile` installs its own packages. Plain `script/bootstrap`, which
|
||||||
`script/setup` and `script/cibuild` run, installs git, make and Go, and Node,
|
`script/setup` and `script/cibuild` run, installs git, make and Go, and Node,
|
||||||
Yarn and the prettier pinned in `yarn.lock` for formatting the markdown, but
|
Yarn and the prettier pinned in `yarn.lock` for formatting the markdown, but
|
||||||
none of the C libraries: the checks compile pixa in Docker. Docker itself must
|
none of the C libraries: the checks compile pixa in Docker. Docker itself must
|
||||||
@@ -199,7 +199,11 @@ into place, so a crash leaves no file under its own name with part of its data.
|
|||||||
When pixa starts, and every five minutes after, it removes each cached source or
|
When pixa starts, and every five minutes after, it removes each cached source or
|
||||||
transformed image whose file is empty, or not the size the database records for
|
transformed image whose file is empty, or not the size the database records for
|
||||||
it, with a warning naming the file; the next request for it fetches the source
|
it, with a warning naming the file; the next request for it fetches the source
|
||||||
or converts the image again.
|
or converts the image again. Looking for those files reads the database once for
|
||||||
|
every cached file, so pixa skips each five-minute look that would start less
|
||||||
|
than ten times as long after the last look ended as that look took. After a look
|
||||||
|
that took one minute, the next one starts 10 to 15 minutes after that look
|
||||||
|
ended.
|
||||||
|
|
||||||
pixa's target is 1-5k r/s, which has not been measured at that rate (see Load
|
pixa's target is 1-5k r/s, which has not been measured at that rate (see Load
|
||||||
Test). Toward it, pixa keeps in memory the content types of the 10,000
|
Test). Toward it, pixa keeps in memory the content types of the 10,000
|
||||||
@@ -316,6 +320,35 @@ once, is refused with 400.
|
|||||||
both optional (values under Signature Specification). Both are part of what is
|
both optional (values under Signature Specification). Both are part of what is
|
||||||
cached, so each value of either is a separate cached image.
|
cached, so each value of either is a separate cached image.
|
||||||
|
|
||||||
|
A source URL's query follows its path after `%3F`, the encoded `?`:
|
||||||
|
`/v1/image/<host>/<path>%3F<query>/<size>.<format>`. pixa decodes the
|
||||||
|
percent-escapes in the image URL's path once, then reads the host up to the
|
||||||
|
first `/`, the size and format after the last `/`, and between them the source
|
||||||
|
path up to the first `?` and the source query after it. So the `?` before the
|
||||||
|
query, any `?` in the query, and every `%` in the source path or query must be
|
||||||
|
percent-encoded, as `%3F` and `%25`. Any other character may be written as
|
||||||
|
itself or percent-encoded: `/`, `;`, `,`, `=` and `&` work either way. Neither
|
||||||
|
the source path nor the source query may hold a `#`, and an image URL that
|
||||||
|
writes one as `%23` is refused with 400. For example,
|
||||||
|
`https://cdn.example.com/img?url=https://origin.example.com/my%20cat.jpg` at
|
||||||
|
800x600 WebP is
|
||||||
|
`/v1/image/cdn.example.com/img%3Furl=https://origin.example.com/my%2520cat.jpg/800x600.webp`,
|
||||||
|
signed as shown under Signature Specification.
|
||||||
|
|
||||||
|
The source path is the path with its percent-escapes decoded, and pixa fetches
|
||||||
|
it percent-encoded again: every character other than letters, digits, `-`, `.`,
|
||||||
|
`_`, `~`, `/`, `$`, `&`, `+`, `,`, `:`, `;`, `=` and `@` is sent as a
|
||||||
|
percent-escape. So `my%20cat.jpg` in the image URL is the source path
|
||||||
|
`/my cat.jpg`, fetched as `/my%20cat.jpg`; `my%2520cat.jpg` is the source path
|
||||||
|
`/my%20cat.jpg`, fetched as `/my%2520cat.jpg`; and `100%25.png` is fetched as
|
||||||
|
`/100%25.png`. The source query is the source URL's query as that URL writes it,
|
||||||
|
its percent-escapes included, and is sent as it is, except that the characters a
|
||||||
|
browser percent-encodes in a query (a space, `"`, `'`, `<`, `>`, control
|
||||||
|
characters and characters outside ASCII) and a `%` that does not begin a
|
||||||
|
percent-escape are percent-encoded. pixa signs and caches the query in that
|
||||||
|
form, so `img%3Fname=my%20cat` and `img%3Fname=my%2520cat` in the image URL are
|
||||||
|
the same source query, `name=my%20cat`.
|
||||||
|
|
||||||
A size with one side `0` has that side scaled to keep the source's proportions,
|
A size with one side `0` has that side scaled to keep the source's proportions,
|
||||||
and is refused with 400 when that side would come out above 8192, the most
|
and is refused with 400 when that side would come out above 8192, the most
|
||||||
either side may be; so is a size with `fit=outside`, which scales the image to
|
either side may be; so is a size with `fit=outside`, which scales the image to
|
||||||
@@ -430,7 +463,13 @@ nor change what it asks for.
|
|||||||
sent with an empty format, or none, also makes a JPEG XL URL. Width and
|
sent with an empty format, or none, also makes a JPEG XL URL. Width and
|
||||||
height both empty or `0` keep the original size; if only one of them is empty
|
height both empty or `0` keep the original size; if only one of them is empty
|
||||||
or `0`, that side is scaled to keep the image's proportions. A URL whose
|
or `0`, that side is scaled to keep the image's proportions. A URL whose
|
||||||
scaled side would come out above 8192 answers 400.
|
scaled side would come out above 8192 answers 400, and so does a source URL
|
||||||
|
whose path holds a `..` segment. So does a source URL that the image URL
|
||||||
|
would fetch written another way, and the page then shows the URL pixa would
|
||||||
|
fetch: one with a user name and password, which pixa does not send, or one
|
||||||
|
whose path holds `%2F`, fetched as `/`, or `(`, fetched as `%28`, as pixa
|
||||||
|
keeps the source path decoded and fetches it percent-encoded again (see
|
||||||
|
Routes).
|
||||||
3. The page shows the URL, `https://<host>/v1/e/<token>/img.<format>`, and when
|
3. The page shows the URL, `https://<host>/v1/e/<token>/img.<format>`, and when
|
||||||
it expires. `<host>` is the host the page was opened on, and the URL starts
|
it expires. `<host>` is the host the page was opened on, and the URL starts
|
||||||
with `http` instead while `debug` is on. The name after the token is ignored
|
with `http` instead while `debug` is on. The name after the token is ignored
|
||||||
@@ -438,9 +477,11 @@ nor change what it asks for.
|
|||||||
`jxl` for a form with no format.
|
`jxl` for a form with no format.
|
||||||
|
|
||||||
The token holds the source's host, path and query and the size, format, quality,
|
The token holds the source's host, path and query and the size, format, quality,
|
||||||
fit and expiry, encrypted with a key derived from `signing_key`. The source
|
fit and expiry, encrypted with a key derived from `signing_key`. The host is
|
||||||
URL's scheme is not kept: the image is fetched like any other (see Routes), and
|
kept in lower case and an empty path as `/`, as for `/v1/image/`, so
|
||||||
the blocked networks still apply.
|
`https://CDN.Example.com` and `https://cdn.example.com/` are one source image,
|
||||||
|
fetched and cached once. The source URL's scheme is not kept: the image is
|
||||||
|
fetched like any other (see Routes), and the blocked networks still apply.
|
||||||
|
|
||||||
How long the URL lasts is chosen on the page, from 1 minute to 1 year, or never.
|
How long the URL lasts is chosen on the page, from 1 minute to 1 year, or never.
|
||||||
The expiry is fixed in the token when the URL is made and cannot be changed or
|
The expiry is fixed in the token when the URL is made and cannot be changed or
|
||||||
@@ -492,13 +533,20 @@ Where:
|
|||||||
- `host` — source host as the URL writes it up to the first `/`, in lower case,
|
- `host` — source host as the URL writes it up to the first `/`, in lower case,
|
||||||
keeping an IPv6 address's brackets and any `:` and port after it (e.g.
|
keeping an IPv6 address's brackets and any `:` and port after it (e.g.
|
||||||
`cdn.example.com`, `cdn.example.com:8443`, `[2001:db8::1]:8443`)
|
`cdn.example.com`, `cdn.example.com:8443`, `[2001:db8::1]:8443`)
|
||||||
- `path` — source path (e.g. `/photos/cat.jpg`), with each `%` written as `%25`
|
- `path` — source path, with its percent-escapes decoded as under Routes (e.g.
|
||||||
and each `:` as `%3A`, so `/photos/a:b.jpg` is signed as `/photos/a%3Ab.jpg`
|
`/photos/cat.jpg`, or `/photos/my cat.jpg` for `/photos/my%20cat.jpg`), with
|
||||||
- `query` — source query string, empty string if none, with `%` and `:` written
|
each `%` written as `%25` and each `:` as `%3A`, so `/photos/a:b.jpg` is
|
||||||
as in `path`; neither then holds a `:`, so path `/a:b` with no query and path
|
signed as `/photos/a%3Ab.jpg`
|
||||||
`/a` with query `b:` are signed differently
|
- `query` — source query string as under Routes, empty string if none, with `%`
|
||||||
- `width` — requested width in pixels, `0` for original
|
and `:` written as in `path`; neither then holds a `:`, so path `/a:b` with no
|
||||||
- `height` — requested height in pixels, `0` for original
|
query and path `/a` with query `b:` are signed differently
|
||||||
|
- `width` — requested width in pixels, or `0`: the width is then worked out from
|
||||||
|
the source's proportions and the requested height, rounded down but at least
|
||||||
|
1, or is the source's own width when the height is `0` too (the `orig` size)
|
||||||
|
- `height` — requested height in pixels, or `0`: the height is then worked out
|
||||||
|
from the source's proportions and the requested width, rounded down but at
|
||||||
|
least 1, or is the source's own height when the width is `0` too (the `orig`
|
||||||
|
size)
|
||||||
- `format` — output format, one of those listed under Routes, with `original`
|
- `format` — output format, one of those listed under Routes, with `original`
|
||||||
signed as `orig`, `jpg` as `jpeg`, and no format as `jxl`, so a URL with no
|
signed as `orig`, `jpg` as `jpeg`, and no format as `jxl`, so a URL with no
|
||||||
format has the signature of the same URL ending in `.jxl`; `auto` is signed as
|
format has the signature of the same URL ending in `.jxl`; `auto` is signed as
|
||||||
@@ -511,7 +559,10 @@ Where:
|
|||||||
with 400
|
with 400
|
||||||
- `fit` — the URL's `fit` query parameter (cover, contain, fill, inside,
|
- `fit` — the URL's `fit` query parameter (cover, contain, fill, inside,
|
||||||
outside), or `cover` when the URL has no `fit`; a request whose `fit` is
|
outside), or `cover` when the URL has no `fit`; a request whose `fit` is
|
||||||
anything else, an empty `fit=` included, is refused with 400
|
anything else, an empty `fit=` included, is refused with 400. `outside` scales
|
||||||
|
the image to cover the requested size, keeping its proportions, so it can come
|
||||||
|
out larger than that size on one side: an 8192x1 source at `100x0`, which is
|
||||||
|
100x1 by its proportions, comes out at 8192x1
|
||||||
|
|
||||||
The URL's `sig` is the HMAC-SHA256 result in base64url (the URL-safe alphabet of
|
The URL's `sig` is the HMAC-SHA256 result in base64url (the URL-safe alphabet of
|
||||||
RFC 4648) with the trailing `=` padding kept, 44 characters in all. pixa
|
RFC 4648) with the trailing `=` padding kept, 44 characters in all. pixa
|
||||||
@@ -535,6 +586,17 @@ For the same image at quality 40 with fit `contain`, the input ends in
|
|||||||
and the URL is
|
and the URL is
|
||||||
`/v1/image/cdn.example.com/photos/cat.jpg/800x600.webp?sig=5IwXUx6vf7yefhaUvFzgXZvG2o0Df4RJxPTK3pKq5VU=&exp=1704067200&q=40&fit=contain`.
|
`/v1/image/cdn.example.com/photos/cat.jpg/800x600.webp?sig=5IwXUx6vf7yefhaUvFzgXZvG2o0Df4RJxPTK3pKq5VU=&exp=1704067200&q=40&fit=contain`.
|
||||||
|
|
||||||
|
For a source URL with a query,
|
||||||
|
`https://cdn.example.com/img?url=https://origin.example.com/my%20cat.jpg`, at
|
||||||
|
800x600 WebP with the same key and expiration and the default quality and fit:
|
||||||
|
|
||||||
|
1. Build input, with the query's `:` written as `%3A` and its `%` as `%25`:
|
||||||
|
`cdn.example.com:/img:url=https%3A//origin.example.com/my%2520cat.jpg:800:600:webp:1704067200:85:cover`
|
||||||
|
2. The signature is `37aFa9GSVCZM1kRY9SzMOk2fOg_M7Wt5gKNR4xM-FYA=`
|
||||||
|
3. URL, with the `?` before the query written as `%3F` and the query's `%` as
|
||||||
|
`%25` (see Routes):
|
||||||
|
`/v1/image/cdn.example.com/img%3Furl=https://origin.example.com/my%2520cat.jpg/800x600.webp?sig=37aFa9GSVCZM1kRY9SzMOk2fOg_M7Wt5gKNR4xM-FYA=&exp=1704067200`
|
||||||
|
|
||||||
**Allowlist patterns:**
|
**Allowlist patterns:**
|
||||||
|
|
||||||
- **Exact match**: `cdn.example.com` — matches only that host
|
- **Exact match**: `cdn.example.com` — matches only that host
|
||||||
@@ -550,7 +612,11 @@ pixa lower-cases the host of a `/v1/image/` URL before anything else uses it, so
|
|||||||
`CDN.Example.com` is matched, fetched, cached and counted against
|
`CDN.Example.com` is matched, fetched, cached and counted against
|
||||||
`upstream_connections_per_host` as `cdn.example.com`. Such a URL whose host has
|
`upstream_connections_per_host` as `cdn.example.com`. Such a URL whose host has
|
||||||
a port, as in `cdn.example.com:8443`, or a `:` with no port, needs a signature
|
a port, as in `cdn.example.com:8443`, or a `:` with no port, needs a signature
|
||||||
even when the host is allowlisted. A host holding `@` is refused with 400.
|
even when the host is allowlisted. The host must be a host name or an IPv4
|
||||||
|
address (letters, digits, hyphens, underscores and dots) or an IPv6 address in
|
||||||
|
brackets, either one optionally followed by `:` and a port. Any other host, such
|
||||||
|
as one holding `@`, or a `?`, `#`, `%`, `\` or space written percent-encoded, is
|
||||||
|
refused with 400.
|
||||||
|
|
||||||
### Configuration
|
### Configuration
|
||||||
|
|
||||||
@@ -632,7 +698,12 @@ Key settings in more detail:
|
|||||||
and generator pages. Default: empty
|
and generator pages. Default: empty
|
||||||
- `blocked_networks` — list of CIDR ranges to refuse for SSRF protection, added
|
- `blocked_networks` — list of CIDR ranges to refuse for SSRF protection, added
|
||||||
to the always-enforced built-in ranges (loopback, private, link-local, CGNAT,
|
to the always-enforced built-in ranges (loopback, private, link-local, CGNAT,
|
||||||
benchmark, NAT64, and the like); an invalid CIDR aborts startup
|
benchmark, NAT64, and the like); an invalid CIDR aborts startup. An IPv6
|
||||||
|
address in the IPv4-mapped form (`::ffff:a.b.c.d`) is matched only as the IPv4
|
||||||
|
address it holds, so an IPv6 range such as `::/0` refuses neither it nor a
|
||||||
|
plain IPv4 address. An entry written in that form is read as the IPv4 range it
|
||||||
|
covers, so `::ffff:10.0.0.0/104` refuses what `10.0.0.0/8` refuses; such an
|
||||||
|
entry shorter than `/96` covers more than IPv4 addresses and aborts startup
|
||||||
- `trusted_proxies` — list of CIDR ranges of the reverse proxies in front of
|
- `trusted_proxies` — list of CIDR ranges of the reverse proxies in front of
|
||||||
pixa. `X-Forwarded-For` is believed only when the direct peer falls inside one
|
pixa. `X-Forwarded-For` is believed only when the direct peer falls inside one
|
||||||
of these ranges; the logged and login-recorded client address is then the
|
of these ranges; the logged and login-recorded client address is then the
|
||||||
@@ -642,16 +713,22 @@ Key settings in more detail:
|
|||||||
omitted key defaults to the RFC 1918 private ranges (`10.0.0.0/8`,
|
omitted key defaults to the RFC 1918 private ranges (`10.0.0.0/8`,
|
||||||
`172.16.0.0/12`, `192.168.0.0/16`), since pixa is deployed behind a proxy on a
|
`172.16.0.0/12`, `192.168.0.0/16`), since pixa is deployed behind a proxy on a
|
||||||
private network; an explicitly empty list (`[]`) trusts no one, and an
|
private network; an explicitly empty list (`[]`) trusts no one, and an
|
||||||
explicit list replaces the default. An invalid CIDR aborts startup. Set this
|
explicit list replaces the default. An invalid CIDR aborts startup. An IPv6
|
||||||
to the address pixa sees for requests that come through your proxy, such as
|
address in the IPv4-mapped form (`::ffff:a.b.c.d`) is matched only as the IPv4
|
||||||
`172.17.0.1/32`, when the defaults do not cover it, or to trust nothing else
|
address it holds, so an IPv6 range such as `::/0` trusts neither it nor a
|
||||||
(see the login limit and `image_requests_per_minute` under Routes). For a
|
plain IPv4 address. An entry written in that form is read as the IPv4 range it
|
||||||
proxy on the Docker host that connects to pixa over `127.0.0.1`, that address
|
covers, so `::ffff:10.0.0.0/104` trusts what `10.0.0.0/8` trusts; such an
|
||||||
is the gateway of the container's Docker network (`172.17.0.1` on the default
|
entry shorter than `/96` covers more than IPv4 addresses and aborts startup.
|
||||||
bridge), not the proxy's own address; a proxy that connects through another of
|
Set this to the address pixa sees for requests that come through your proxy,
|
||||||
the host's addresses is seen with that address. To be sure which address it
|
such as `172.17.0.1/32`, when the defaults do not cover it, or to trust
|
||||||
is, set this to `[]` (or `PIXA_TRUSTED_PROXIES` to empty), send a request
|
nothing else (see the login limit and `image_requests_per_minute` under
|
||||||
through the proxy, and read `remoteIP` in pixa's request log line for it
|
Routes). For a proxy on the Docker host that connects to pixa over
|
||||||
|
`127.0.0.1`, that address is the gateway of the container's Docker network
|
||||||
|
(`172.17.0.1` on the default bridge), not the proxy's own address; a proxy
|
||||||
|
that connects through another of the host's addresses is seen with that
|
||||||
|
address. To be sure which address it is, set this to `[]` (or
|
||||||
|
`PIXA_TRUSTED_PROXIES` to empty), send a request through the proxy, and read
|
||||||
|
`remoteIP` in pixa's request log line for it
|
||||||
- `upstream_fetch_timeout` — time allowed for one fetch from an upstream host,
|
- `upstream_fetch_timeout` — time allowed for one fetch from an upstream host,
|
||||||
as a duration such as `30s` (the default) or `2m`
|
as a duration such as `30s` (the default) or `2m`
|
||||||
- `upstream_max_response_size` — largest upstream response accepted, in bytes;
|
- `upstream_max_response_size` — largest upstream response accepted, in bytes;
|
||||||
@@ -663,11 +740,14 @@ Key settings in more detail:
|
|||||||
- `signing_key` — HMAC secret for URL signatures
|
- `signing_key` — HMAC secret for URL signatures
|
||||||
- `db_url` — the SQLite database to open; omitted, it is
|
- `db_url` — the SQLite database to open; omitted, it is
|
||||||
`file:<state_dir>/state.sqlite3?_pragma=journal_mode(WAL)`, which keeps the
|
`file:<state_dir>/state.sqlite3?_pragma=journal_mode(WAL)`, which keeps the
|
||||||
database in WAL mode. pixa opens one connection to it, so its own reads and
|
database in WAL mode. `state_dir` is percent-encoded in it, so a `%`, `?` or
|
||||||
writes run one at a time. pixa adds `_pragma=busy_timeout(5000)` to any
|
`#` in the name is written as `%25`, `%3F` or `%23`. For a `state_dir` of `/`
|
||||||
`db_url`, so a write that finds another program writing to the file waits up
|
or one starting with `//` it starts `file:////`, so SQLite does not read the
|
||||||
to five seconds for it instead of failing. WAL mode comes only from the URL:
|
first name in the path as a host. pixa opens one connection to it, so its own
|
||||||
keep `_pragma=journal_mode(WAL)` in one you set
|
reads and writes run one at a time. pixa adds `_pragma=busy_timeout(5000)` to
|
||||||
|
any `db_url`, so a write that finds another program writing to the file waits
|
||||||
|
up to five seconds for it instead of failing. WAL mode comes only from the
|
||||||
|
URL: keep `_pragma=journal_mode(WAL)` in one you set
|
||||||
- `cache_max_bytes` — disk cache size limit in bytes; `0` disables the disk
|
- `cache_max_bytes` — disk cache size limit in bytes; `0` disables the disk
|
||||||
cache entirely; omitted defaults to 75% of the sum of the free space on the
|
cache entirely; omitted defaults to 75% of the sum of the free space on the
|
||||||
filesystem containing `<state_dir>/cache/` and the bytes of source and
|
filesystem containing `<state_dir>/cache/` and the bytes of source and
|
||||||
@@ -757,19 +837,25 @@ provide:
|
|||||||
`script/precommit`
|
`script/precommit`
|
||||||
|
|
||||||
Every `docker build` in these scripts passes a new value of the `CACHEBUST`
|
Every `docker build` in these scripts passes a new value of the `CACHEBUST`
|
||||||
build argument, which every stage of the `Dockerfile` declares first, so every
|
build argument, which every stage of the `Dockerfile` declares, so every `RUN`
|
||||||
`RUN` step, the lint and test phases included, runs on every build instead of
|
step after that line, the lint and test steps included, runs on every build
|
||||||
coming from the build cache. Go's build cache is kept from build to build: the
|
instead of coming from the build cache. Above the line are the steps that
|
||||||
steps that compile mount it, so a package whose inputs are unchanged is not
|
install system packages and download the Go modules; Docker keeps their layers
|
||||||
compiled again, and both `go test` runs pass `-count=1` so that no stored test
|
until the base image, the step itself, `go.mod` or `go.sum` changes. Go's build
|
||||||
result is replayed. The `test` phase runs on the Debian Go image, with Debian's
|
cache is kept from build to build: the steps that compile mount it, so a package
|
||||||
libvips and libheif; the image pixa ships is built and runs on Alpine.
|
whose inputs are unchanged is not compiled again, and both `go test` runs pass
|
||||||
`script/check`, `script/cibuild`, `script/docker`, `script/lint`, `script/test`,
|
`-count=1` so that no stored test result is replayed. The lint step also mounts
|
||||||
`script/setup` and `script/install-precommit` are the standard copies from
|
golangci-lint's cache, which reuses a package's findings only when its files,
|
||||||
`sneak/prompts`, kept identical to them. `script/fmt` and `script/fmt-check` are
|
those of the packages it imports, `go.mod`, the linters and their settings in
|
||||||
the standard copies with pixa's `gofmt` step kept before prettier. prettier
|
`.golangci.yml`, and the golangci-lint version are all unchanged. The `test`
|
||||||
formats the markdown only, not the HTML templates, as it cannot parse a Go
|
phase runs on the Debian Go image, with Debian's libvips and libheif; the image
|
||||||
template action inside a tag.
|
pixa ships is built and runs on Alpine. `script/check`, `script/cibuild`,
|
||||||
|
`script/docker`, `script/lint`, `script/test`, `script/setup` and
|
||||||
|
`script/install-precommit` are the standard copies from `sneak/prompts`, kept
|
||||||
|
identical to them. `script/fmt` and `script/fmt-check` are the standard copies
|
||||||
|
with pixa's `gofmt` step kept before prettier. prettier formats the markdown
|
||||||
|
only, not the HTML templates, as it cannot parse a Go template action inside a
|
||||||
|
tag.
|
||||||
|
|
||||||
## Load Test
|
## Load Test
|
||||||
|
|
||||||
|
|||||||
+68
-35
@@ -1,6 +1,6 @@
|
|||||||
---
|
---
|
||||||
title: Repository Policies
|
title: Repository Policies
|
||||||
last_modified: 2026-10-15
|
last_modified: 2026-10-18
|
||||||
---
|
---
|
||||||
|
|
||||||
This document covers repository structure, tooling, and workflow standards. Code
|
This document covers repository structure, tooling, and workflow standards. Code
|
||||||
@@ -159,8 +159,11 @@ style conventions are in separate documents:
|
|||||||
one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs
|
one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs
|
||||||
exit non-zero with `parallel golangci-lint is running` — a status a caller
|
exit non-zero with `parallel golangci-lint is running` — a status a caller
|
||||||
cannot tell from real findings. Both have produced wrong verdicts in this
|
cannot tell from real findings. Both have produced wrong verdicts in this
|
||||||
org, in both directions. A container has its own golangci-lint cache, its
|
org, in both directions. A container has its own `TMPDIR`, so the lock is
|
||||||
own `TMPDIR` and a digest-pinned binary, so neither is reachable.
|
out of reach. The lint phase does keep golangci-lint's cache between check
|
||||||
|
builds (below), but it lints every checkout at `/src`, and a finding is
|
||||||
|
taken from that cache only for the same files, settings and golangci-lint
|
||||||
|
version.
|
||||||
|
|
||||||
- **Any build that runs checks is given a new `CACHEBUST` value.** Docker
|
- **Any build that runs checks is given a new `CACHEBUST` value.** Docker
|
||||||
invalidates a `COPY` layer only when the copied content changes, so on an
|
invalidates a `COPY` layer only when the copied content changes, so on an
|
||||||
@@ -170,15 +173,21 @@ style conventions are in separate documents:
|
|||||||
id, so no two builds share a value, not even the three in one
|
id, so no two builds share a value, not even the three in one
|
||||||
`script/cibuild`. `script/lint`, `script/test`, `script/cibuild` and
|
`script/cibuild`. `script/lint`, `script/test`, `script/cibuild` and
|
||||||
`script/docker` are the four, and there is no fifth — `script/check` runs the
|
`script/docker` are the four, and there is no fifth — `script/check` runs the
|
||||||
two gate phases and `script/fmt-check`, and builds no image of its own. Every
|
two gate phases and `script/fmt-check`, and builds no image of its own. In
|
||||||
stage of the `Dockerfile` declares `ARG CACHEBUST` as its first line after
|
every stage of the `Dockerfile`, `ARG CACHEBUST` comes right after the steps
|
||||||
`FROM`, and a new value reruns every `RUN` step after that line; `COPY` steps
|
that install system packages and download Go's modules, or right after `FROM`
|
||||||
are compared by content and need nothing. A stage without the `ARG` line is
|
in a stage with neither, and every step that lints, tests or compiles comes
|
||||||
served from cache on an unchanged tree, so a repository re-vendors the scripts
|
after it. A step that installs anything else as well, such as a
|
||||||
and adds the `ARG` lines in one commit. The scripts do not pass `--no-cache`:
|
`script/bootstrap` that also installs node packages, comes after it too. A new
|
||||||
it reruns every step too, but it also empties every cache mount, and Go's
|
value reruns every `RUN` step after that line; `COPY` steps are compared by
|
||||||
build cache, which the canonical Go `Dockerfile` below mounts so that compiled
|
content and need nothing. Docker keeps the layers above the line until the
|
||||||
packages survive between check builds, would start empty every time. A bare
|
base image, their own lines or the files they copy change. A stage without the
|
||||||
|
`ARG` line is served from cache on an unchanged tree, so a repository
|
||||||
|
re-vendors the scripts and adds the `ARG` lines in one commit. The scripts do
|
||||||
|
not pass `--no-cache`: it reruns every step too, but it also builds the layers
|
||||||
|
above the line again and empties every cache mount, so Go's build cache and
|
||||||
|
golangci-lint's, which the canonical Go `Dockerfile` below mounts so that they
|
||||||
|
survive between check builds, would start empty every time. A bare
|
||||||
`docker build .` is not evidence that anything ran: a sub-second build
|
`docker build .` is not evidence that anything ran: a sub-second build
|
||||||
reporting success is a cache hit, not a result. Never invalidate by pruning —
|
reporting success is a cache hit, not a result. Never invalidate by pruning —
|
||||||
`docker builder prune` and friends destroy a build cache shared with every
|
`docker builder prune` and friends destroy a build cache shared with every
|
||||||
@@ -194,19 +203,25 @@ style conventions are in separate documents:
|
|||||||
`Dockerfile`:
|
`Dockerfile`:
|
||||||
|
|
||||||
```dockerfile
|
```dockerfile
|
||||||
# Every stage declares ARG CACHEBUST first: script/ passes a new value
|
# In every stage ARG CACHEBUST comes right after the steps that install
|
||||||
# on every build, which reruns every RUN step after it. The steps that
|
# system packages and download Go's modules, or right after FROM in a
|
||||||
# compile mount Go's build cache, which survives from build to build.
|
# stage with neither, and every step that lints, tests or compiles
|
||||||
|
# comes after it. script/ passes a new value on every build, which
|
||||||
|
# reruns every RUN step after it. Docker keeps the layers above it
|
||||||
|
# until the base image, their own lines or go.mod and go.sum change.
|
||||||
|
# The steps that compile mount Go's build cache, and golangci-lint run
|
||||||
|
# also mounts golangci-lint's; both survive from build to build.
|
||||||
|
|
||||||
# Lint phase
|
# Lint phase
|
||||||
# golangci/golangci-lint:v2.x.x, YYYY-MM-DD
|
# golangci/golangci-lint:v2.x.x, YYYY-MM-DD
|
||||||
FROM golangci/golangci-lint@sha256:... AS lint
|
FROM golangci/golangci-lint@sha256:... AS lint
|
||||||
ARG CACHEBUST
|
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
RUN go mod download
|
RUN go mod download
|
||||||
|
ARG CACHEBUST
|
||||||
COPY . .
|
COPY . .
|
||||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
RUN --mount=type=cache,target=/root/.cache/go-build \
|
||||||
|
--mount=type=cache,target=/root/.cache/golangci-lint \
|
||||||
golangci-lint run --config .golangci.yml ./...
|
golangci-lint run --config .golangci.yml ./...
|
||||||
|
|
||||||
# Test phase. -race needs cgo and so a C compiler, which the Debian Go
|
# Test phase. -race needs cgo and so a C compiler, which the Debian Go
|
||||||
@@ -214,10 +229,10 @@ style conventions are in separate documents:
|
|||||||
# cache also holds test results, which go test would otherwise replay.
|
# cache also holds test results, which go test would otherwise replay.
|
||||||
# golang:1.x, YYYY-MM-DD
|
# golang:1.x, YYYY-MM-DD
|
||||||
FROM golang@sha256:... AS test
|
FROM golang@sha256:... AS test
|
||||||
ARG CACHEBUST
|
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
RUN go mod download
|
RUN go mod download
|
||||||
|
ARG CACHEBUST
|
||||||
COPY . .
|
COPY . .
|
||||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
RUN --mount=type=cache,target=/root/.cache/go-build \
|
||||||
go test -count=1 -timeout 90s -race -cover ./... || \
|
go test -count=1 -timeout 90s -race -cover ./... || \
|
||||||
@@ -225,19 +240,19 @@ style conventions are in separate documents:
|
|||||||
go test -count=1 -timeout 90s -race -v ./...; exit 1; }
|
go test -count=1 -timeout 90s -race -v ./...; exit 1; }
|
||||||
|
|
||||||
# Build stage. Nothing is wanted from either phase above; the copies
|
# Build stage. Nothing is wanted from either phase above; the copies
|
||||||
# are what make BuildKit build them first, so this stage cannot run
|
# are what make BuildKit build them first, so nothing after them can
|
||||||
# unless lint and test passed.
|
# run unless lint and test passed.
|
||||||
# golang:1.x-alpine, YYYY-MM-DD
|
# golang:1.x-alpine, YYYY-MM-DD
|
||||||
FROM golang@sha256:... AS builder
|
FROM golang@sha256:... AS builder
|
||||||
ARG CACHEBUST
|
|
||||||
COPY --from=lint /src/go.sum /dev/null
|
|
||||||
COPY --from=test /src/go.sum /dev/null
|
|
||||||
RUN apk add --no-cache git
|
RUN apk add --no-cache git
|
||||||
# A tar-stream context keeps the sender's file owners, which git refuses.
|
|
||||||
RUN git config --system --add safe.directory /src
|
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
RUN go mod download
|
RUN go mod download
|
||||||
|
ARG CACHEBUST
|
||||||
|
COPY --from=lint /src/go.sum /dev/null
|
||||||
|
COPY --from=test /src/go.sum /dev/null
|
||||||
|
# A tar-stream context keeps the sender's file owners, which git refuses.
|
||||||
|
RUN git config --system --add safe.directory /src
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
# The VERSION build arg when one is given, otherwise
|
# The VERSION build arg when one is given, otherwise
|
||||||
@@ -274,16 +289,26 @@ style conventions are in separate documents:
|
|||||||
- Keep the runtime stage last, and if you add a stage after it, give it the
|
- Keep the runtime stage last, and if you add a stage after it, give it the
|
||||||
same two copies. A plain `docker build .` builds the last stage's chain
|
same two copies. A plain `docker build .` builds the last stage's chain
|
||||||
and nothing else.
|
and nothing else.
|
||||||
- Only Go's build cache is mounted (`/root/.cache/go-build`, where Go keeps
|
- Check builds keep these, and none of them can hand back a stale result:
|
||||||
it for root), and only on the steps that compile: `golangci-lint run`,
|
- The layers above `ARG CACHEBUST`: the system packages and Go's
|
||||||
both `go test` runs and `go build`. It keeps compiled packages between
|
modules. Docker installs the packages again when the base image or the
|
||||||
check builds, and Go reuses one only when all its inputs are unchanged, so
|
install line changes, and downloads the modules again when `go.mod` or
|
||||||
it saves the compile without skipping a check. Nothing else is kept:
|
`go.sum` changes. Go checks every module against `go.sum` as it
|
||||||
golangci-lint's own cache would replay stored findings in place of a lint
|
downloads it, and no later build can change a kept layer. The modules
|
||||||
run, and Go builds from whatever files a kept module cache holds without
|
are kept in a layer rather than a cache mount because a cache mount is
|
||||||
checking them again (only `go mod verify` does), so a file changed there
|
shared by every build on the host, and Go builds from whatever files
|
||||||
would go unnoticed. Go's build cache also holds test results, which is why
|
it holds without checking them again (only `go mod verify` does), so a
|
||||||
both `go test` runs carry `-count=1`.
|
file changed there would go unnoticed.
|
||||||
|
- Go's build cache (`/root/.cache/go-build`, where Go keeps it for
|
||||||
|
root), mounted on the steps that compile: `golangci-lint run`, both
|
||||||
|
`go test` runs and `go build`. Go reuses a compiled package only when
|
||||||
|
all its inputs are unchanged. The cache also holds test results, which
|
||||||
|
is why both `go test` runs carry `-count=1`.
|
||||||
|
- golangci-lint's cache (`/root/.cache/golangci-lint`, where the pinned
|
||||||
|
image keeps it), mounted on `golangci-lint run`. golangci-lint reuses
|
||||||
|
a package's findings only when the package's files and those of every
|
||||||
|
package it imports, `go.mod`, the linters and their settings in
|
||||||
|
`.golangci.yml`, and the golangci-lint version are all unchanged.
|
||||||
- If the project uses `//go:embed` directives that reference build artifacts
|
- If the project uses `//go:embed` directives that reference build artifacts
|
||||||
(e.g. a web frontend compiled in a separate stage), the lint phase must
|
(e.g. a web frontend compiled in a separate stage), the lint phase must
|
||||||
create placeholder files so the embed directives resolve. Example:
|
create placeholder files so the embed directives resolve. Example:
|
||||||
@@ -292,12 +317,20 @@ style conventions are in separate documents:
|
|||||||
in the lint phase. The `golangci/golangci-lint` image is Debian-based and
|
in the lint phase. The `golangci/golangci-lint` image is Debian-based and
|
||||||
has no `apk`, so install with `apt-get` under the Debian package name
|
has no `apk`, so install with `apt-get` under the Debian package name
|
||||||
(`libvips-dev`, where alpine says `vips-dev`), and delete the package
|
(`libvips-dev`, where alpine says `vips-dev`), and delete the package
|
||||||
lists in the same `RUN`, so the layer does not keep them:
|
lists in the same `RUN`, so the layer does not keep them. The test phase,
|
||||||
|
on the Debian Go image, installs the libraries its tests need the same
|
||||||
|
way. Either install goes right after `FROM`, above `ARG CACHEBUST`, so
|
||||||
|
Docker keeps it until the base image or the install line changes:
|
||||||
|
|
||||||
```dockerfile
|
```dockerfile
|
||||||
|
FROM golangci/golangci-lint@sha256:... AS lint
|
||||||
RUN apt-get update \
|
RUN apt-get update \
|
||||||
&& apt-get install -y --no-install-recommends libvips-dev \
|
&& apt-get install -y --no-install-recommends libvips-dev \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
WORKDIR /src
|
||||||
|
COPY go.mod go.sum ./
|
||||||
|
RUN go mod download
|
||||||
|
ARG CACHEBUST
|
||||||
```
|
```
|
||||||
|
|
||||||
- `.dockerignore` lets `.git` into the build context. It keeps out every git
|
- `.dockerignore` lets `.git` into the build context. It keeps out every git
|
||||||
|
|||||||
@@ -30,6 +30,128 @@ P2: security: per-origin rate limiting
|
|||||||
|
|
||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
|
- 2026-10-18 re-vendored from `sneak/prompts` commit `0653579` (closes #221):
|
||||||
|
each `Dockerfile` stage installs its system packages and downloads the Go
|
||||||
|
modules above `ARG CACHEBUST`, so check builds keep them, and the lint step
|
||||||
|
keeps golangci-lint's cache.
|
||||||
|
- 2026-10-19 a `trusted_proxies` entry in the IPv4-mapped form trusted nothing;
|
||||||
|
it is now read as the IPv4 range it covers by the same code that reads
|
||||||
|
`blocked_networks`, and such an entry shorter than `/96` aborts startup
|
||||||
|
(closes #303).
|
||||||
|
- 2026-10-18 the next reconciliation pass waits at least ten times as long as
|
||||||
|
the last one took, so passes on a large cache no longer run back to back
|
||||||
|
(closes #250).
|
||||||
|
- 2026-10-18 a `blocked_networks` entry in the IPv4-mapped form refused nothing;
|
||||||
|
`::ffff:10.0.0.0/104` is now read as `10.0.0.0/8`, and such an entry shorter
|
||||||
|
than `/96` aborts startup (closes #301).
|
||||||
|
- 2026-10-16 P1 security bug: the check that keeps pixa from fetching internal
|
||||||
|
addresses refuses the local-use NAT64 range (`64:ff9b:1::/48`, RFC 8215) as it
|
||||||
|
refuses `64:ff9b::/96`, and refuses an IPv6 address in the IPv4-compatible
|
||||||
|
form (`::/96`) when `blocked_networks` contains either the address as written
|
||||||
|
or the IPv4 address it holds (closes #299). A source host such as
|
||||||
|
`[64:ff9b:1::a00:1]` was fetched, which on a network that runs NAT64 on that
|
||||||
|
range reaches the IPv4 address it holds, here `10.0.0.1`; and with
|
||||||
|
`203.0.113.0/24` in `blocked_networks`, `[::ffff:203.0.113.5]` was refused but
|
||||||
|
`[::203.0.113.5]` was fetched. Both now answer 403. The mapped form
|
||||||
|
(`::ffff:a.b.c.d`) is still matched only as the IPv4 address it holds: Go
|
||||||
|
keeps a plain IPv4 address in that form, and an IPv6 entry such as `::/0` must
|
||||||
|
not refuse every IPv4 address.
|
||||||
|
- 2026-10-16 P1 security bug: the check that keeps pixa from fetching internal
|
||||||
|
addresses judges an IPv6 address in the IPv4-compatible form (`::/96`), such
|
||||||
|
as `::127.0.0.1` or `::10.0.0.1`, by the IPv4 address in its last four bytes
|
||||||
|
(closes #297). It read only the mapped form (`::ffff:a.b.c.d`) as IPv4, so a
|
||||||
|
source host such as `[::127.0.0.1]` was checked as a public IPv6 address and
|
||||||
|
fetched, which on a network that routes it reaches the internal IPv4 host; it
|
||||||
|
is now refused with 403. The NAT64 form (`64:ff9b::/96`) was already refused
|
||||||
|
as a whole, as one of the built-in blocked ranges, and still is.
|
||||||
|
- 2026-10-16 a source host that is an IPv6 address in brackets with no port,
|
||||||
|
such as `[2001:db8::a]`, is fetched like one with a port, through `/v1/image/`
|
||||||
|
and through an encrypted URL (closes #295). The fetcher's check of the source
|
||||||
|
URL kept the brackets when no port followed them, could not look the host up,
|
||||||
|
and the request answered 500. It now checks the address without brackets, so a
|
||||||
|
private one is refused with 403, as with a port.
|
||||||
|
- 2026-10-16 a `state_dir` whose name holds `%`, `?` or `#` keeps its database
|
||||||
|
at `state.sqlite3` in that directory (closes #290): the `db_url` derived from
|
||||||
|
it put the name into a `file:` URL as it was, so SQLite read `%41` as `A`, a
|
||||||
|
`?` or `#` ended the path early, and the journal mode parameter was lost. The
|
||||||
|
derived `db_url` is now built with Go's `net/url`, which writes those
|
||||||
|
characters as `%25`, `%3F` and `%23`. For a `state_dir` of `/` or one starting
|
||||||
|
with `//` it starts `file:////`, so SQLite no longer reads the first name in
|
||||||
|
the path as a host.
|
||||||
|
- 2026-10-16 a size with one side `0` keeps the side computed from the source's
|
||||||
|
proportions at 1 or more (closes #283): it was rounded down, so for a thin
|
||||||
|
source it came out at 0, and libvips then made the size square. An 8192x1
|
||||||
|
source at `100x0` came out at 100x100, and at `1300x0` with `fit=cover` made
|
||||||
|
libvips ask for about 39 GB at once; an 8192x2 source at `100x0` answered 500.
|
||||||
|
`fit=contain` and `fit=inside` work both sides out again from the smaller of
|
||||||
|
the two scales and `fit=outside` from the larger, and each keeps both sides at
|
||||||
|
1 or more too, so a 1x8192 source at `0x100` with `fit=contain`, and a 49x8192
|
||||||
|
source at `0x100` or a 49x49 source at `1x1` with `fit=outside`, no longer
|
||||||
|
answer 500. An 8192x1 source at `1300x0` now comes out at 1300x1, and the
|
||||||
|
process running it alone peaked at about 34 MiB resident. With `fit=outside`
|
||||||
|
the image is scaled to cover the size, so that source at `100x0` comes out at
|
||||||
|
8192x1. `README.md` now says this under Signature Specification, and says how
|
||||||
|
a `0` side is worked out where it said `0` meant the original.
|
||||||
|
- 2026-10-16 the generator page lower-cases the source host and writes an empty
|
||||||
|
source path as `/` before it makes the encrypted URL, as `/v1/image/` does
|
||||||
|
(closes #293). A host written in upper and in lower case, or a source URL with
|
||||||
|
no path and with the path `/`, made two cache keys and two upstream requests
|
||||||
|
for one image.
|
||||||
|
- 2026-10-16 the source URL a request fetches is built with `net/url` from the
|
||||||
|
source host, path and query, not joined as text (closes #288). The source
|
||||||
|
path, decoded, is fetched percent-encoded again, so `100%25.png` fetches
|
||||||
|
`/100%25.png` where it answered 401, and `my%20cat.png` and `my%2520cat.png`
|
||||||
|
fetch different paths where both fetched `/my%20cat.png` under two cache keys.
|
||||||
|
The source query stays as the source URL writes it; the characters a browser
|
||||||
|
percent-encodes in a query, such as `'` or a space, which was sent raw and
|
||||||
|
answered 502, are percent-encoded when the image URL is read, so the query is
|
||||||
|
signed and cached as it is sent. The generator page answers 400, showing the
|
||||||
|
URL pixa would fetch, for a source URL its image URL would fetch written
|
||||||
|
another way, such as with `%2F` in the path, whatever else the path holds, or
|
||||||
|
with a user name and password. It also answers 400 for a source path holding a
|
||||||
|
`..` segment, as `/v1/image/` refuses one.
|
||||||
|
- 2026-10-15 a thin source requested with `fit=cover` no longer makes libvips
|
||||||
|
build a huge image (closes #282): libvips' thumbnail scaled the whole source
|
||||||
|
to cover the requested size and copied that into memory before it cut out the
|
||||||
|
centre, so an 8192x64 source at 8192x8192 made it allocate about 24 GiB, and
|
||||||
|
an 8192x1 source answered 500. The image processor now first cuts the centre
|
||||||
|
of the source, widened to whole source pixels, and thumbnail scales and cuts
|
||||||
|
only that, by the same amount on both sides as before. What it copies into
|
||||||
|
memory is then larger than the output by less than one scaled source pixel on
|
||||||
|
each side. With the runtime image's libvips and JPEG output at 8192x8192,
|
||||||
|
resident memory peaked at about 242 MiB for the 8192x64 source and 431 MiB for
|
||||||
|
the 8192x1 one. The output can sit a fraction of an output pixel from where it
|
||||||
|
was.
|
||||||
|
- 2026-10-15 a `/v1/image/` URL whose source path or source query holds a `#`,
|
||||||
|
written as `%23`, is refused with 400 (closes #286): the source URL built from
|
||||||
|
it ended at the `#` when fetched, so each text after it fetched the same
|
||||||
|
upstream image, without a signature for an allowlisted host, and cached it
|
||||||
|
under keys of its own. `README.md` says neither may hold a `#`.
|
||||||
|
- 2026-10-15 the host of a `/v1/image/` URL must be a host name or an IPv4
|
||||||
|
address (letters, digits, hyphens, underscores and dots) or an IPv6 address in
|
||||||
|
brackets, either one optionally followed by `:` and a port, and any other host
|
||||||
|
is refused with 400 (closes #275): only `@` was refused, so a `?` written as
|
||||||
|
`%3F` gave the host `cdn.example.com?x=1`, which Go's URL parser reads as
|
||||||
|
`cdn.example.com` with a query. Such a URL passed the allowlist, and each text
|
||||||
|
after the `?` fetched the host's root path and cached the answer under keys of
|
||||||
|
its own. A `#`, `%`, `\` or space written percent-encoded is refused the same
|
||||||
|
way, and so is an IPv6 address without brackets, which needed a signature
|
||||||
|
before. Underscores are kept because `allowlist_hosts` accepts host names
|
||||||
|
holding them.
|
||||||
|
- 2026-10-15 the README says how a source URL with a query is written into an
|
||||||
|
image URL, and the image route accepts it (closes #255): the query follows the
|
||||||
|
source path after `%3F`. pixa now reads the image URL's path decoded once, so
|
||||||
|
the `?` before the query, any `?` in it and every `%` in the source path or
|
||||||
|
query must be percent-encoded, and any other character may be. The route used
|
||||||
|
to parse chi's wildcard, which holds the path as the client wrote it whenever
|
||||||
|
that is not Go's own encoding of it; a `/`, `;` or `,` in the query written as
|
||||||
|
`%2F`, `%3B` or `%2C`, as `GenerateSignedURL` wrote them, kept the `%3F` from
|
||||||
|
being read as `?`, and the request failed with 401. `GenerateSignedURL` now
|
||||||
|
writes Go's own encoding of the path. The README has an example with a query
|
||||||
|
and its signature. The path traversal check no longer reads the source query,
|
||||||
|
and keeps a `%` that does not start a percent-escape as it is instead of
|
||||||
|
refusing the URL, so a source path or query holding `%25` is accepted, while a
|
||||||
|
double-encoded `..` next to such a `%` is still refused.
|
||||||
- 2026-10-15 `max_concurrent_processing` defaults to half the CPUs pixa sees,
|
- 2026-10-15 `max_concurrent_processing` defaults to half the CPUs pixa sees,
|
||||||
rounded down, and at least 1 (closes #242): it defaulted to one image per CPU,
|
rounded down, and at least 1 (closes #242): it defaulted to one image per CPU,
|
||||||
and a container with no CPU limit sees every CPU of the host, so on a 48-CPU
|
and a container with no CPU limit sees every CPU of the host, so on a 48-CPU
|
||||||
|
|||||||
@@ -88,3 +88,56 @@ func TestBlockedNetworksInvalidAbortsStartup(t *testing.T) {
|
|||||||
},
|
},
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestBlockedNetworksMappedEntryIsIPv4Range checks that an entry written in
|
||||||
|
// the IPv4-mapped form (::ffff:a.b.c.d) is kept as the IPv4 range it covers,
|
||||||
|
// since the fetcher matches an IPv4-mapped address as the IPv4 address it
|
||||||
|
// holds. /96, the shortest such entry, covers every IPv4 address.
|
||||||
|
func TestBlockedNetworksMappedEntryIsIPv4Range(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
yamlContent := signingKeyLine + `blocked_networks:
|
||||||
|
- "::ffff:10.0.0.0/104"
|
||||||
|
- "::ffff:0:0/96"
|
||||||
|
`
|
||||||
|
|
||||||
|
c, err := configFromYAML(t, yamlContent)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("IPv4-mapped blocked_networks entries should load, got error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
want := []string{"10.0.0.0/8", "0.0.0.0/0"}
|
||||||
|
if len(c.BlockedNetworks) != len(want) {
|
||||||
|
t.Fatalf("BlockedNetworks = %v, want %d entries", c.BlockedNetworks, len(want))
|
||||||
|
}
|
||||||
|
|
||||||
|
for i, w := range want {
|
||||||
|
if got := c.BlockedNetworks[i].String(); got != w {
|
||||||
|
t.Errorf("BlockedNetworks[%d] = %q, want %q", i, got, w)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestBlockedNetworksMappedEntryShorterThan96AbortsStartup checks that an
|
||||||
|
// entry in the IPv4-mapped form shorter than /96, which covers more than the
|
||||||
|
// IPv4 addresses, aborts startup with an error naming the key and the value.
|
||||||
|
func TestBlockedNetworksMappedEntryShorterThan96AbortsStartup(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
runAbortCases(t, []abortCase{
|
||||||
|
{
|
||||||
|
name: "mapped-shorter-than-96",
|
||||||
|
yaml: signingKeyLine + `blocked_networks:
|
||||||
|
- "::ffff:0.0.0.0/80"
|
||||||
|
`,
|
||||||
|
wantErrSubstrings: []string{keyBlockedNetworks, "::ffff:0.0.0.0/80"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "mapped-95",
|
||||||
|
yaml: signingKeyLine + `blocked_networks:
|
||||||
|
- "::ffff:0:0/95"
|
||||||
|
`,
|
||||||
|
wantErrSubstrings: []string{keyBlockedNetworks},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|||||||
@@ -91,6 +91,7 @@ var (
|
|||||||
errNotABoolean = errors.New("not a boolean")
|
errNotABoolean = errors.New("not a boolean")
|
||||||
errNotAStringList = errors.New("not a list of strings")
|
errNotAStringList = errors.New("not a list of strings")
|
||||||
errNotAValidCIDR = errors.New("not a valid CIDR network")
|
errNotAValidCIDR = errors.New("not a valid CIDR network")
|
||||||
|
errMappedTooShort = errors.New("in the IPv4-mapped form but shorter than /96")
|
||||||
errNotAMetricsMap = errors.New("not a map of metrics settings")
|
errNotAMetricsMap = errors.New("not a map of metrics settings")
|
||||||
errEmptyListEntry = errors.New("list contains an empty entry")
|
errEmptyListEntry = errors.New("list contains an empty entry")
|
||||||
errEmptyEntry = errors.New("contains an empty entry")
|
errEmptyEntry = errors.New("contains an empty entry")
|
||||||
@@ -338,8 +339,7 @@ func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) {
|
|||||||
settingName(keyDBURL), errValueEmpty)
|
settingName(keyDBURL), errValueEmpty)
|
||||||
}
|
}
|
||||||
|
|
||||||
// The driver sets the journal mode only through a _pragma parameter.
|
c.DBURL = defaultDBURL(c.StateDir)
|
||||||
c.DBURL = fmt.Sprintf("file:%s/state.sqlite3?_pragma=journal_mode(WAL)", c.StateDir)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if loader.err != nil {
|
if loader.err != nil {
|
||||||
@@ -354,6 +354,30 @@ func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) {
|
|||||||
return c, nil
|
return c, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// defaultDBURL returns the db_url used when db_url is omitted: the file
|
||||||
|
// state.sqlite3 in stateDir, in WAL mode, which the driver sets only
|
||||||
|
// through a _pragma parameter. String() percent-encodes any %, ? or # in
|
||||||
|
// the directory name, which SQLite decodes back, so the URL names exactly
|
||||||
|
// this file. OmitHost writes file:/var/lib/pixa/... instead of
|
||||||
|
// file:///var/lib/pixa/..., and keeps a relative stateDir from being read
|
||||||
|
// as a host name. The path written into the URL is stateDir followed by
|
||||||
|
// /state.sqlite3; when it starts with //, as it does for a stateDir of /
|
||||||
|
// or one starting with //, the empty host is kept, so the URL starts
|
||||||
|
// file://// and SQLite does not read the first name in the path as a
|
||||||
|
// host.
|
||||||
|
func defaultDBURL(stateDir string) string {
|
||||||
|
dbPath := stateDir + "/state.sqlite3"
|
||||||
|
|
||||||
|
dbURL := url.URL{
|
||||||
|
Scheme: "file",
|
||||||
|
OmitHost: !strings.HasPrefix(dbPath, "//"),
|
||||||
|
Path: dbPath,
|
||||||
|
RawQuery: "_pragma=journal_mode(WAL)",
|
||||||
|
}
|
||||||
|
|
||||||
|
return dbURL.String()
|
||||||
|
}
|
||||||
|
|
||||||
// validateKnownKeys rejects configuration files containing keys the
|
// validateKnownKeys rejects configuration files containing keys the
|
||||||
// application does not understand, so typos fail at startup instead of
|
// application does not understand, so typos fail at startup instead of
|
||||||
// being silently ignored, and rejects keys that are explicitly set to
|
// being silently ignored, and rejects keys that are explicitly set to
|
||||||
@@ -1225,6 +1249,13 @@ func defaultTrustedProxies() []netip.Prefix {
|
|||||||
// wrong type, an empty entry, a non-string entry, or an unparseable CIDR
|
// wrong type, an empty entry, a non-string entry, or an unparseable CIDR
|
||||||
// aborts startup naming the key and the offending value; the default
|
// aborts startup naming the key and the offending value; the default
|
||||||
// (an empty list) applies only to an omitted key.
|
// (an empty list) applies only to an omitted key.
|
||||||
|
//
|
||||||
|
// An entry written in the IPv4-mapped form (::ffff:a.b.c.d) is kept as the
|
||||||
|
// IPv4 range it covers: ::ffff:10.0.0.0/104 is kept as 10.0.0.0/8. Both
|
||||||
|
// blocked_networks and trusted_proxies are matched against an IPv4-mapped
|
||||||
|
// address as the IPv4 address it holds, so the entry as written would match
|
||||||
|
// nothing. Such an entry shorter than /96 covers more than the IPv4
|
||||||
|
// addresses and aborts startup.
|
||||||
func parseCIDRList(sc *smartconfig.Config, key string) ([]netip.Prefix, error) {
|
func parseCIDRList(sc *smartconfig.Config, key string) ([]netip.Prefix, error) {
|
||||||
raw, ok := lookupValue(sc, key)
|
raw, ok := lookupValue(sc, key)
|
||||||
if !ok {
|
if !ok {
|
||||||
@@ -1240,6 +1271,10 @@ func parseCIDRList(sc *smartconfig.Config, key string) ([]netip.Prefix, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// An IPv4-mapped address is the 96 bits of ::ffff:0:0/96 followed by
|
||||||
|
// the 32 bits of the IPv4 address it holds.
|
||||||
|
const mappedPrefixBits = 96
|
||||||
|
|
||||||
prefixes := make([]netip.Prefix, 0, len(entries))
|
prefixes := make([]netip.Prefix, 0, len(entries))
|
||||||
|
|
||||||
for _, entry := range entries {
|
for _, entry := range entries {
|
||||||
@@ -1249,6 +1284,16 @@ func parseCIDRList(sc *smartconfig.Config, key string) ([]netip.Prefix, error) {
|
|||||||
settingName(key), entry, errNotAValidCIDR)
|
settingName(key), entry, errNotAValidCIDR)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if prefix.Addr().Is4In6() && prefix.Bits() < mappedPrefixBits {
|
||||||
|
return nil, fmt.Errorf("%s: value %q is %w",
|
||||||
|
settingName(key), entry, errMappedTooShort)
|
||||||
|
}
|
||||||
|
|
||||||
|
if prefix.Addr().Is4In6() {
|
||||||
|
prefix = netip.PrefixFrom(prefix.Addr().Unmap(),
|
||||||
|
prefix.Bits()-mappedPrefixBits)
|
||||||
|
}
|
||||||
|
|
||||||
prefixes = append(prefixes, prefix)
|
prefixes = append(prefixes, prefix)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -135,6 +135,104 @@ func TestDefaultDBURLOpensTheDatabaseInWALMode(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestDerivedDBURLOpensStateDirHoldingURLCharacters opens the db_url
|
||||||
|
// derived from a state_dir whose name holds %41, ? and #, and checks that
|
||||||
|
// the database is state.sqlite3 in that directory, in WAL mode. Put into
|
||||||
|
// a file: URL as it is, the name would lose its meaning: %41 is read as
|
||||||
|
// A, and the path ends at the ?, which also loses the journal mode.
|
||||||
|
func TestDerivedDBURLOpensStateDirHoldingURLCharacters(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
stateDir := filepath.Join(t.TempDir(), "a%41b?c#d")
|
||||||
|
|
||||||
|
err := os.Mkdir(stateDir, 0o750)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to create %q: %v", stateDir, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Quoted so YAML takes the whole name as it is.
|
||||||
|
c, err := configFromYAML(t, signingKeyLine+"state_dir: '"+stateDir+"'\n")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config with state_dir %q should be valid, got: %v", stateDir, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("state_dir %q gives db_url %q", stateDir, c.DBURL)
|
||||||
|
|
||||||
|
checkDBURLOpensFileInWALMode(t, c.DBURL, filepath.Join(stateDir, "state.sqlite3"))
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDerivedDBURLOpensStateDirStartingWithTwoSlashes opens the db_url
|
||||||
|
// derived from a state_dir that starts with //, and checks that the
|
||||||
|
// database is state.sqlite3 in that directory, in WAL mode. Written right
|
||||||
|
// after file:, such a name gives a URL starting file://, and SQLite reads
|
||||||
|
// the first directory name as a host and refuses to open it.
|
||||||
|
func TestDerivedDBURLOpensStateDirStartingWithTwoSlashes(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
stateDir := "/" + dir
|
||||||
|
|
||||||
|
c, err := configFromYAML(t, signingKeyLine+"state_dir: "+stateDir+"\n")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config with state_dir %q should be valid, got: %v", stateDir, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("state_dir %q gives db_url %q", stateDir, c.DBURL)
|
||||||
|
|
||||||
|
checkDBURLOpensFileInWALMode(t, c.DBURL, filepath.Join(dir, "state.sqlite3"))
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDerivedDBURLForRootStateDir checks the db_url derived from a
|
||||||
|
// state_dir of /. The path written into the URL is then
|
||||||
|
// //state.sqlite3, and written right after file: it gives a URL starting
|
||||||
|
// file://, where SQLite reads state.sqlite3 as a host and refuses to open
|
||||||
|
// it. The test only checks the URL, since it cannot write to /.
|
||||||
|
func TestDerivedDBURLForRootStateDir(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
c, err := configFromYAML(t, signingKeyLine+"state_dir: /\n")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config with state_dir / should be valid, got: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("state_dir / gives db_url %q", c.DBURL)
|
||||||
|
|
||||||
|
wantDBURL := "file:////state.sqlite3?_pragma=journal_mode(WAL)"
|
||||||
|
if c.DBURL != wantDBURL {
|
||||||
|
t.Errorf("DBURL = %q, want %q", c.DBURL, wantDBURL)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkDBURLOpensFileInWALMode opens dbURL with the SQLite driver pixad
|
||||||
|
// uses, and checks that the database is in WAL mode and is the file
|
||||||
|
// dbFile.
|
||||||
|
func checkDBURLOpensFileInWALMode(t *testing.T, dbURL, dbFile string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
db, err := sql.Open("sqlite", dbURL)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to open %q: %v", dbURL, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Cleanup(func() { _ = db.Close() })
|
||||||
|
|
||||||
|
var journalMode string
|
||||||
|
|
||||||
|
err = db.QueryRowContext(t.Context(), "PRAGMA journal_mode").Scan(&journalMode)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to read the journal mode of %q: %v", dbURL, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if journalMode != "wal" {
|
||||||
|
t.Errorf("journal mode of %q = %q, want wal", dbURL, journalMode)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = os.Stat(dbFile)
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("opening %q did not create %q: %v", dbURL, dbFile, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExplicitValidValuesAreUsed(t *testing.T) {
|
func TestExplicitValidValuesAreUsed(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -83,3 +83,41 @@ func TestTrustedProxiesInvalidAbortsStartup(t *testing.T) {
|
|||||||
},
|
},
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestTrustedProxiesMappedEntryIsIPv4Range checks that an entry written in
|
||||||
|
// the IPv4-mapped form (::ffff:a.b.c.d) is kept as the IPv4 range it covers,
|
||||||
|
// since the client address is matched as the IPv4 address it holds. /96, the
|
||||||
|
// shortest such entry, covers every IPv4 address.
|
||||||
|
func TestTrustedProxiesMappedEntryIsIPv4Range(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
c, err := configFromYAML(t, signingKeyLine+
|
||||||
|
`trusted_proxies: ["::ffff:10.0.0.0/104", "::ffff:0:0/96"]`+"\n")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("IPv4-mapped trusted_proxies entries should load: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got := make([]string, len(c.TrustedProxies))
|
||||||
|
for i, p := range c.TrustedProxies {
|
||||||
|
got[i] = p.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
if joined := strings.Join(got, ","); joined != "10.0.0.0/8,0.0.0.0/0" {
|
||||||
|
t.Errorf("TrustedProxies = %v, want 10.0.0.0/8 and 0.0.0.0/0", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestTrustedProxiesMappedEntryShorterThan96AbortsStartup checks that an
|
||||||
|
// entry in the IPv4-mapped form shorter than /96, which covers more than the
|
||||||
|
// IPv4 addresses, aborts startup with an error naming the key and the value.
|
||||||
|
func TestTrustedProxiesMappedEntryShorterThan96AbortsStartup(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
runAbortCases(t, []abortCase{
|
||||||
|
{
|
||||||
|
name: "mapped-95",
|
||||||
|
yaml: signingKeyLine + `trusted_proxies: ["::ffff:0:0/95"]` + "\n",
|
||||||
|
wantErrSubstrings: []string{keyTrustedProxies, "::ffff:0:0/95"},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|||||||
@@ -9,7 +9,9 @@ import (
|
|||||||
"math"
|
"math"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
|
"slices"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/pixa/internal/clientip"
|
"sneak.berlin/go/pixa/internal/clientip"
|
||||||
@@ -125,7 +127,7 @@ func (s *Handlers) HandleGenerateURL() http.HandlerFunc {
|
|||||||
sourceURL := r.FormValue("url")
|
sourceURL := r.FormValue("url")
|
||||||
|
|
||||||
// Validate source URL
|
// Validate source URL
|
||||||
parsed, err := url.Parse(sourceURL)
|
parsed, err := parseSourceURL(sourceURL)
|
||||||
if err != nil || parsed.Host == "" {
|
if err != nil || parsed.Host == "" {
|
||||||
s.renderGeneratorWithForm(w, r, "Invalid source URL", r.Form,
|
s.renderGeneratorWithForm(w, r, "Invalid source URL", r.Form,
|
||||||
http.StatusBadRequest)
|
http.StatusBadRequest)
|
||||||
@@ -133,6 +135,16 @@ func (s *Handlers) HandleGenerateURL() http.HandlerFunc {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// /v1/image/ refuses a ".." segment in the source path, and so does
|
||||||
|
// the generator. The path is decoded, so %2e%2e counts too
|
||||||
|
if slices.Contains(strings.Split(parsed.Path, "/"), "..") {
|
||||||
|
s.renderGeneratorWithForm(w, r,
|
||||||
|
"Source URL path cannot hold a .. segment", r.Form,
|
||||||
|
http.StatusBadRequest)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
payload, expiresAt, ttl, err := buildGeneratePayload(parsed, r.Form)
|
payload, expiresAt, ttl, err := buildGeneratePayload(parsed, r.Form)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.renderGeneratorWithForm(w, r, err.Error(), r.Form,
|
s.renderGeneratorWithForm(w, r, err.Error(), r.Form,
|
||||||
@@ -141,6 +153,19 @@ func (s *Handlers) HandleGenerateURL() http.HandlerFunc {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The image URL must fetch the source URL as it was written
|
||||||
|
sourceRequest := payload.ToImageRequest()
|
||||||
|
sourceRequest.AllowHTTP = s.config.AllowHTTP
|
||||||
|
fetchURL := sourceRequest.SourceURL()
|
||||||
|
|
||||||
|
if !fetchedAsWritten(parsed, fetchURL) {
|
||||||
|
s.renderGeneratorWithForm(w, r, "Source URL cannot be fetched as "+
|
||||||
|
"written: pixa would fetch "+fetchURL,
|
||||||
|
r.Form, http.StatusBadRequest)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
// Generate encrypted token
|
// Generate encrypted token
|
||||||
token, err := s.encGen.Generate(payload)
|
token, err := s.encGen.Generate(payload)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -173,6 +198,48 @@ func (s *Handlers) HandleGenerateURL() http.HandlerFunc {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// parseSourceURL parses the source URL entered on the generator page with its
|
||||||
|
// host lower-cased and an empty path written as "/", as /v1/image/ reads a
|
||||||
|
// source URL, so that each spelling of one source URL makes one cache key and
|
||||||
|
// one upstream request.
|
||||||
|
func parseSourceURL(sourceURL string) (*url.URL, error) {
|
||||||
|
parsed, err := url.Parse(sourceURL)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
parsed.Host = strings.ToLower(parsed.Host)
|
||||||
|
|
||||||
|
if parsed.Path == "" {
|
||||||
|
parsed.Path = "/"
|
||||||
|
}
|
||||||
|
|
||||||
|
return parsed, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// fetchedAsWritten reports whether fetchURL, the URL pixa fetches for an
|
||||||
|
// image URL made from the source URL source, is source as it was written,
|
||||||
|
// apart from the scheme, which the token does not keep. It is not when source
|
||||||
|
// holds a user name and password, which the token does not keep either, or a
|
||||||
|
// "?" with no query after it. The token keeps the source path decoded and
|
||||||
|
// pixa fetches it percent-encoded again, so a path written another way, such
|
||||||
|
// as with %2F, is fetched as another path. url.URL keeps the path as written
|
||||||
|
// in RawPath when it is not the decoded path percent-encoded again.
|
||||||
|
// EscapedPath ignores RawPath when it also holds a character net/url encodes,
|
||||||
|
// such as a space, so RawPath is compared itself.
|
||||||
|
func fetchedAsWritten(source *url.URL, fetchURL string) bool {
|
||||||
|
writtenPath := source.RawPath
|
||||||
|
if writtenPath == "" {
|
||||||
|
writtenPath = source.EscapedPath()
|
||||||
|
}
|
||||||
|
|
||||||
|
fetched, err := url.Parse(fetchURL)
|
||||||
|
|
||||||
|
return err == nil && source.User == nil && !source.ForceQuery &&
|
||||||
|
fetched.EscapedPath() == writtenPath &&
|
||||||
|
fetched.RawQuery == source.RawQuery
|
||||||
|
}
|
||||||
|
|
||||||
// buildGeneratePayload parses the numeric form fields and assembles the
|
// buildGeneratePayload parses the numeric form fields and assembles the
|
||||||
// encrypted URL payload. ttl=0 means never expires (ExpiresAt stays 0). A
|
// encrypted URL payload. ttl=0 means never expires (ExpiresAt stays 0). A
|
||||||
// non-numeric or out-of-range width, height, quality or ttl, or an
|
// non-numeric or out-of-range width, height, quality or ttl, or an
|
||||||
|
|||||||
@@ -9,7 +9,6 @@ import (
|
|||||||
"strconv"
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/go-chi/chi/v5"
|
|
||||||
"github.com/go-chi/chi/v5/middleware"
|
"github.com/go-chi/chi/v5/middleware"
|
||||||
"sneak.berlin/go/pixa/internal/encurl"
|
"sneak.berlin/go/pixa/internal/encurl"
|
||||||
"sneak.berlin/go/pixa/internal/httpfetcher"
|
"sneak.berlin/go/pixa/internal/httpfetcher"
|
||||||
@@ -80,20 +79,21 @@ func (s *Handlers) HandleRobotsTxt() http.HandlerFunc {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// parseImageRequest parses the wildcard path and query parameters into
|
// parseImageRequest parses the image URL's path and query parameters into
|
||||||
// an ImageRequest. On invalid input it writes an error response and
|
// an ImageRequest. On invalid input it writes an error response and
|
||||||
// returns false.
|
// returns false.
|
||||||
func (s *Handlers) parseImageRequest(
|
func (s *Handlers) parseImageRequest(
|
||||||
w http.ResponseWriter, r *http.Request,
|
w http.ResponseWriter, r *http.Request,
|
||||||
) (*imgcache.ImageRequest, bool) {
|
) (*imgcache.ImageRequest, bool) {
|
||||||
// Get the wildcard path from chi
|
// The path is parsed decoded once, as r.URL.Path holds it, and not from
|
||||||
pathParam := chi.URLParam(r, "*")
|
// chi's wildcard: chi routes on the path as the client wrote it whenever
|
||||||
|
// that is not Go's own encoding of it, so its wildcard would still hold
|
||||||
// Parse the URL path
|
// the %3F before a source query when, say, a "/" in the query is written
|
||||||
parsed, err := imgcache.ParseImagePath(pathParam)
|
// as %2F
|
||||||
|
parsed, err := imgcache.ParseImageURL(r.URL.Path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.log.Warn("failed to parse image URL",
|
s.log.Warn("failed to parse image URL",
|
||||||
"path", pathParam,
|
"path", r.URL.Path,
|
||||||
"error", err,
|
"error", err,
|
||||||
)
|
)
|
||||||
s.respondError(w, "invalid image URL: "+err.Error(), http.StatusBadRequest)
|
s.respondError(w, "invalid image URL: "+err.Error(), http.StatusBadRequest)
|
||||||
|
|||||||
@@ -264,3 +264,127 @@ func TestHandleImage_AllowlistOrSignature(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestHandleImage_InvalidHost_Returns400 sends image URLs whose host no URL
|
||||||
|
// host can be, and checks that each is refused with 400 naming the host as
|
||||||
|
// decoded: allowlistedHost with a character after it that a URL host cannot
|
||||||
|
// hold, percent-encoded, an IPv6 address without brackets, and brackets
|
||||||
|
// around something that is not an IPv6 address. A "?" or "#" after
|
||||||
|
// allowlistedHost would otherwise end the host when the source URL is read,
|
||||||
|
// so the request would pass the allowlist and be fetched and cached under a
|
||||||
|
// host that is not the one fetched.
|
||||||
|
func TestHandleImage_InvalidHost_Returns400(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
photos := newPhotoFetcher(t, allowlistedHost)
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
writtenHost string
|
||||||
|
decodedHost string
|
||||||
|
}{
|
||||||
|
{"? written as %3F", allowlistedHost + "%3Fx=1", allowlistedHost + "?x=1"},
|
||||||
|
{"# written as %23", allowlistedHost + "%23x", allowlistedHost + "#x"},
|
||||||
|
{"% written as %25", allowlistedHost + "%253F", allowlistedHost + "%3F"},
|
||||||
|
{`\ written as %5C`, allowlistedHost + "%5Cx", allowlistedHost + `\x`},
|
||||||
|
{"space written as %20", allowlistedHost + "%20", allowlistedHost + " "},
|
||||||
|
{"IPv6 address without brackets", "2001:db8::1", "2001:db8::1"},
|
||||||
|
{"IPv4 address in brackets", "[203.0.113.5]", "[203.0.113.5]"},
|
||||||
|
{"no address in brackets", "[abc]", "[abc]"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
rec := sendGet(t, newImageRoute(t, photos), photoURL(tt.writtenHost))
|
||||||
|
checkErrorBody(t, rec, http.StatusBadRequest,
|
||||||
|
fmt.Sprintf("invalid image URL: invalid source host: %q",
|
||||||
|
tt.decodedHost))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestHandleImage_HostForms_Served checks that the image route serves, with
|
||||||
|
// a valid signature, the image at photoPath on a host name, an IPv4 address,
|
||||||
|
// an IPv6 address in brackets, and a host followed by ":" with or without a
|
||||||
|
// port.
|
||||||
|
func TestHandleImage_HostForms_Served(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
hosts := []string{
|
||||||
|
allowlistedHost,
|
||||||
|
"img_1.example.com",
|
||||||
|
"203.0.113.5",
|
||||||
|
"[2001:db8::1]",
|
||||||
|
allowlistedHost + ":8443",
|
||||||
|
"[2001:db8::1]:8443",
|
||||||
|
allowlistedHost + ":",
|
||||||
|
}
|
||||||
|
|
||||||
|
photos := newPhotoFetcher(t, hosts...)
|
||||||
|
exp := time.Now().Add(time.Hour)
|
||||||
|
|
||||||
|
for _, host := range hosts {
|
||||||
|
t.Run(host, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
sig := photoSignature(testSigningKey, host, exp)
|
||||||
|
|
||||||
|
rec := sendGet(t, newImageRoute(t, photos),
|
||||||
|
photoURLWithSig(host, sig, exp))
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Errorf("status = %d, want %d: %s",
|
||||||
|
rec.Code, http.StatusOK, rec.Body)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestHandleImage_HashInSourcePathOrQuery_Returns400 sends image URLs whose
|
||||||
|
// source path or source query holds a "#", written as %23, and checks that
|
||||||
|
// each is refused with 400, the one signed for that source path included. The
|
||||||
|
// source URL would otherwise end at the "#" when it is fetched, so each text
|
||||||
|
// after it would fetch the image at photoPath and cache it under keys of its
|
||||||
|
// own.
|
||||||
|
func TestHandleImage_HashInSourcePathOrQuery_Returns400(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
photos := newPhotoFetcher(t, allowlistedHost, signedHost)
|
||||||
|
exp := time.Now().Add(time.Hour)
|
||||||
|
|
||||||
|
sig := signature.New(testSigningKey).Sign(&signature.Request{
|
||||||
|
SourceHost: signedHost,
|
||||||
|
SourcePath: photoPath + "#x",
|
||||||
|
Width: 50,
|
||||||
|
Height: 50,
|
||||||
|
Format: string(imgcache.FormatJPEG),
|
||||||
|
Quality: 85,
|
||||||
|
FitMode: string(imgcache.FitCover),
|
||||||
|
Expires: exp,
|
||||||
|
})
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
target string
|
||||||
|
}{
|
||||||
|
{"# in the source path",
|
||||||
|
"/v1/image/" + allowlistedHost + photoPath + "%23x/50x50.jpeg"},
|
||||||
|
{"# in the source query",
|
||||||
|
"/v1/image/" + allowlistedHost + photoPath + "%3Fv=1%23x/50x50.jpeg"},
|
||||||
|
{"# in the source path, signed",
|
||||||
|
fmt.Sprintf("/v1/image/%s%s%%23x/50x50.jpeg?sig=%s&exp=%d",
|
||||||
|
signedHost, photoPath, sig, exp.Unix())},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
rec := sendGet(t, newImageRoute(t, photos), tt.target)
|
||||||
|
checkErrorBody(t, rec, http.StatusBadRequest,
|
||||||
|
`invalid image URL: invalid image path: `+
|
||||||
|
`the source path or query holds a "#"`)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -169,3 +169,166 @@ func TestHandleImage_InvalidExp_Returns400(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestHandleImage_GeneratedSignedURLWithSourceQueryVerifies sends URLs built
|
||||||
|
// by the service's signed-URL generator for source URLs with a query to the
|
||||||
|
// route, through a chi router: chi routes on the path as the client wrote it
|
||||||
|
// whenever that is not Go's own encoding of it. A 502 means the signature
|
||||||
|
// passed, so the route read the source path and query that were signed.
|
||||||
|
func TestHandleImage_GeneratedSignedURLWithSourceQueryVerifies(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
query string
|
||||||
|
}{
|
||||||
|
{"slash and colon", "url=https://origin.example.com/cat.jpg"},
|
||||||
|
{"semicolon", "a=1;b=2"},
|
||||||
|
{"comma", "crop=0,0,50,50"},
|
||||||
|
{"equals and ampersand", "a=1&b=2"},
|
||||||
|
{"percent-escape", "name=my%20cat"},
|
||||||
|
{"question mark", "next=/a?b=1"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
fix := setupTestHandler(t)
|
||||||
|
|
||||||
|
signedURL, err := fix.service.GenerateSignedURL("", &imgcache.ImageRequest{
|
||||||
|
SourceHost: signedHost,
|
||||||
|
SourcePath: photoPath,
|
||||||
|
SourceQuery: tt.query,
|
||||||
|
Size: imgcache.Size{Width: 50, Height: 50},
|
||||||
|
Format: imgcache.FormatJPEG,
|
||||||
|
}, time.Hour)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GenerateSignedURL() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
status := getImage(t, fix, signedURL)
|
||||||
|
if status != http.StatusBadGateway {
|
||||||
|
t.Errorf("status = %d, want %d", status, http.StatusBadGateway)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestHandleImage_SourceQueryEncodedAnyWay signs one source URL with a query
|
||||||
|
// and writes it into the image URL in ways the README allows: the "?" before
|
||||||
|
// the query and each "%" in it percent-encoded, and every other character as
|
||||||
|
// itself or percent-encoded. A 502 means the signature passed.
|
||||||
|
func TestHandleImage_SourceQueryEncodedAnyWay(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
expires := time.Now().Add(time.Hour)
|
||||||
|
signer := signature.New("test-signing-key-must-be-32-chars")
|
||||||
|
sig := signer.Sign(&signature.Request{
|
||||||
|
SourceHost: signedHost,
|
||||||
|
SourcePath: photoPath,
|
||||||
|
SourceQuery: "url=https://origin.example.com/my%20cat.jpg",
|
||||||
|
Width: 50,
|
||||||
|
Height: 50,
|
||||||
|
Format: string(imgcache.FormatJPEG),
|
||||||
|
Quality: 85,
|
||||||
|
FitMode: string(imgcache.FitCover),
|
||||||
|
Expires: expires,
|
||||||
|
})
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
writtenQuery string
|
||||||
|
}{
|
||||||
|
{"only ? and % encoded",
|
||||||
|
"%3Furl=https://origin.example.com/my%2520cat.jpg"},
|
||||||
|
{"every reserved character encoded",
|
||||||
|
"%3Furl%3Dhttps%3A%2F%2Forigin.example.com%2Fmy%2520cat.jpg"},
|
||||||
|
{"lower-case hex digits",
|
||||||
|
"%3furl=https://origin.example.com/my%2520cat.jpg"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
target := fmt.Sprintf("/v1/image/%s%s%s/50x50.jpeg?sig=%s&exp=%d",
|
||||||
|
signedHost, photoPath, tt.writtenQuery, sig, expires.Unix())
|
||||||
|
|
||||||
|
status := getImage(t, setupTestHandler(t), target)
|
||||||
|
if status != http.StatusBadGateway {
|
||||||
|
t.Errorf("status = %d, want %d", status, http.StatusBadGateway)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestHandleImage_SourceURLHoldingPercent25Verifies signs a source path and a
|
||||||
|
// source query that each hold %25, the percent-escape of a literal "%", and
|
||||||
|
// sends each to the route written as the README says, with that "%" written as
|
||||||
|
// %25, and as GenerateSignedURL writes it. A 502 means the signature passed.
|
||||||
|
func TestHandleImage_SourceURLHoldingPercent25Verifies(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
sourcePath string
|
||||||
|
sourceQuery string
|
||||||
|
// writtenPathAndQuery is the source path and query as the README says
|
||||||
|
// to write them into the image URL
|
||||||
|
writtenPathAndQuery string
|
||||||
|
}{
|
||||||
|
{"path", "/sale/50%25off.jpg", "", "/sale/50%2525off.jpg"},
|
||||||
|
{"query", photoPath, "discount=50%25", photoPath + "%3Fdiscount=50%2525"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name+" written by the README rule", func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
expires := time.Now().Add(time.Hour)
|
||||||
|
signer := signature.New("test-signing-key-must-be-32-chars")
|
||||||
|
sig := signer.Sign(&signature.Request{
|
||||||
|
SourceHost: signedHost,
|
||||||
|
SourcePath: tt.sourcePath,
|
||||||
|
SourceQuery: tt.sourceQuery,
|
||||||
|
Width: 50,
|
||||||
|
Height: 50,
|
||||||
|
Format: string(imgcache.FormatJPEG),
|
||||||
|
Quality: 85,
|
||||||
|
FitMode: string(imgcache.FitCover),
|
||||||
|
Expires: expires,
|
||||||
|
})
|
||||||
|
|
||||||
|
target := fmt.Sprintf("/v1/image/%s%s/50x50.jpeg?sig=%s&exp=%d",
|
||||||
|
signedHost, tt.writtenPathAndQuery, sig, expires.Unix())
|
||||||
|
|
||||||
|
status := getImage(t, setupTestHandler(t), target)
|
||||||
|
if status != http.StatusBadGateway {
|
||||||
|
t.Errorf("status = %d, want %d", status, http.StatusBadGateway)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run(tt.name+" written by GenerateSignedURL", func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
fix := setupTestHandler(t)
|
||||||
|
|
||||||
|
signedURL, err := fix.service.GenerateSignedURL("", &imgcache.ImageRequest{
|
||||||
|
SourceHost: signedHost,
|
||||||
|
SourcePath: tt.sourcePath,
|
||||||
|
SourceQuery: tt.sourceQuery,
|
||||||
|
Size: imgcache.Size{Width: 50, Height: 50},
|
||||||
|
Format: imgcache.FormatJPEG,
|
||||||
|
}, time.Hour)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GenerateSignedURL() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
status := getImage(t, fix, signedURL)
|
||||||
|
if status != http.StatusBadGateway {
|
||||||
|
t.Errorf("status = %d, want %d", status, http.StatusBadGateway)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -59,15 +59,17 @@ const (
|
|||||||
// builtinBlockedPrefixes are internal or special-use ranges that Go's
|
// builtinBlockedPrefixes are internal or special-use ranges that Go's
|
||||||
// net.IP predicates (IsPrivate, IsLinkLocalUnicast, and the like) do not
|
// net.IP predicates (IsPrivate, IsLinkLocalUnicast, and the like) do not
|
||||||
// already cover. They are always blocked, in addition to any
|
// already cover. They are always blocked, in addition to any
|
||||||
// operator-supplied networks. IPv4-mapped IPv6 addresses are unmapped
|
// operator-supplied networks. IPv4-mapped and IPv4-compatible IPv6
|
||||||
// before matching, so these IPv4 ranges are caught in both forms.
|
// addresses are read as the IPv4 address they hold before matching, so
|
||||||
|
// these IPv4 ranges are caught in all three forms.
|
||||||
//
|
//
|
||||||
//nolint:gochecknoglobals // immutable built-in blocklist
|
//nolint:gochecknoglobals // immutable built-in blocklist
|
||||||
var builtinBlockedPrefixes = []netip.Prefix{
|
var builtinBlockedPrefixes = []netip.Prefix{
|
||||||
netip.MustParsePrefix("100.64.0.0/10"), // RFC 6598 CGNAT / carrier-grade NAT
|
netip.MustParsePrefix("100.64.0.0/10"), // RFC 6598 CGNAT / carrier-grade NAT
|
||||||
netip.MustParsePrefix("192.0.0.0/24"), // RFC 6890 IETF protocol assignments
|
netip.MustParsePrefix("192.0.0.0/24"), // RFC 6890 IETF protocol assignments
|
||||||
netip.MustParsePrefix("198.18.0.0/15"), // RFC 2544 benchmarking range
|
netip.MustParsePrefix("198.18.0.0/15"), // RFC 2544 benchmarking range
|
||||||
netip.MustParsePrefix("64:ff9b::/96"), // RFC 6052 NAT64 (maps onto IPv4)
|
netip.MustParsePrefix("64:ff9b::/96"), // RFC 6052 NAT64 (maps onto IPv4)
|
||||||
|
netip.MustParsePrefix("64:ff9b:1::/48"), // RFC 8215 local-use NAT64 (maps onto IPv4)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fetcher errors.
|
// Fetcher errors.
|
||||||
@@ -473,16 +475,16 @@ func validateURL(ctx context.Context, rawURL string, allowHTTP bool) error {
|
|||||||
return ErrUnsupportedScheme
|
return ErrUnsupportedScheme
|
||||||
}
|
}
|
||||||
|
|
||||||
// Parse to extract host
|
parsedURL, err := neturl.Parse(rawURL)
|
||||||
host := extractHost(rawURL)
|
if err != nil {
|
||||||
if host == "" {
|
return fmt.Errorf("%w: %w", ErrInvalidHost, err)
|
||||||
return ErrInvalidHost
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Remove port if present
|
// Hostname drops any port, and the brackets of an IPv6 address whether
|
||||||
h, _, err := net.SplitHostPort(host)
|
// or not a port follows them
|
||||||
if err == nil {
|
host := parsedURL.Hostname()
|
||||||
host = h
|
if host == "" {
|
||||||
|
return ErrInvalidHost
|
||||||
}
|
}
|
||||||
|
|
||||||
// Block obvious localhost patterns
|
// Block obvious localhost patterns
|
||||||
@@ -543,6 +545,9 @@ func isPrivateIP(ip net.IP) bool {
|
|||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// An IPv4-compatible IPv6 address is judged as the IPv4 address it holds
|
||||||
|
ip = ipv4CompatibleToIPv4(ip)
|
||||||
|
|
||||||
// Check for loopback
|
// Check for loopback
|
||||||
if ip.IsLoopback() {
|
if ip.IsLoopback() {
|
||||||
return true
|
return true
|
||||||
@@ -594,7 +599,12 @@ func isPrivateIP(ip net.IP) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// isBlockedIP reports whether ip is refused, either by the built-in
|
// isBlockedIP reports whether ip is refused, either by the built-in
|
||||||
// internal-range check or by one of the operator-supplied prefixes.
|
// internal-range check or by one of the operator-supplied prefixes. An
|
||||||
|
// IPv4-compatible address (in ::/96) is refused when a prefix contains the
|
||||||
|
// address as written or the IPv4 address it holds. An IPv4-mapped address (in
|
||||||
|
// ::ffff:0:0/96) is matched only as the IPv4 address it holds: Go keeps a
|
||||||
|
// plain IPv4 address in that form too, and an IPv6 prefix such as ::/0 must
|
||||||
|
// not refuse every IPv4 address.
|
||||||
func isBlockedIP(ip net.IP, blocked []netip.Prefix) bool {
|
func isBlockedIP(ip net.IP, blocked []netip.Prefix) bool {
|
||||||
if isPrivateIP(ip) {
|
if isPrivateIP(ip) {
|
||||||
return true
|
return true
|
||||||
@@ -607,11 +617,28 @@ func isBlockedIP(ip net.IP, blocked []netip.Prefix) bool {
|
|||||||
|
|
||||||
addr = addr.Unmap()
|
addr = addr.Unmap()
|
||||||
|
|
||||||
|
// ip read as IPv4 where it is or holds an IPv4 address, otherwise the
|
||||||
|
// zero netip.Addr, which no prefix contains. Matching asIPv4 as well as
|
||||||
|
// addr refuses more only for an IPv4-compatible address.
|
||||||
|
asIPv4, _ := netip.AddrFromSlice(ipv4CompatibleToIPv4(ip).To4())
|
||||||
|
|
||||||
return slices.ContainsFunc(blocked, func(prefix netip.Prefix) bool {
|
return slices.ContainsFunc(blocked, func(prefix netip.Prefix) bool {
|
||||||
return prefix.Contains(addr)
|
return prefix.Contains(addr) || prefix.Contains(asIPv4)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ipv4CompatibleToIPv4 returns the IPv4 address that an IPv4-compatible IPv6
|
||||||
|
// address (in ::/96, such as ::127.0.0.1) holds in its last four bytes, and
|
||||||
|
// any other address unchanged.
|
||||||
|
func ipv4CompatibleToIPv4(ip net.IP) net.IP {
|
||||||
|
addr, ok := netip.AddrFromSlice(ip)
|
||||||
|
if ok && netip.MustParsePrefix("::/96").Contains(addr) {
|
||||||
|
return ip[12:]
|
||||||
|
}
|
||||||
|
|
||||||
|
return ip
|
||||||
|
}
|
||||||
|
|
||||||
// ssrfSafeDialer validates IP addresses against the built-in blocked ranges
|
// ssrfSafeDialer validates IP addresses against the built-in blocked ranges
|
||||||
// before connecting. New wraps dialSSRFSafe with the operator-supplied
|
// before connecting. New wraps dialSSRFSafe with the operator-supplied
|
||||||
// blocklist; this entry point enforces the built-in ranges alone.
|
// blocklist; this entry point enforces the built-in ranges alone.
|
||||||
|
|||||||
@@ -186,6 +186,44 @@ func TestIsPrivateIP(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestIsPrivateIPJudgesEmbeddedIPv4Address checks IPv6 addresses that hold an
|
||||||
|
// IPv4 address in their last four bytes: the IPv4-compatible form (::/96) and
|
||||||
|
// the NAT64 form (64:ff9b::/96). One holding a loopback, private or
|
||||||
|
// link-local IPv4 address is refused as that address is, and one holding a
|
||||||
|
// public IPv4 address in the IPv4-compatible form passes.
|
||||||
|
func TestIsPrivateIPJudgesEmbeddedIPv4Address(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
ip string
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{"::127.0.0.1", true}, // loopback
|
||||||
|
{"::10.0.0.1", true}, // private
|
||||||
|
{"::169.254.169.254", true}, // link-local
|
||||||
|
{"64:ff9b::7f00:1", true}, // NAT64 form of 127.0.0.1
|
||||||
|
{"::8.8.8.8", false}, // public
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.ip, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
ip := net.ParseIP(tc.ip)
|
||||||
|
if ip == nil {
|
||||||
|
t.Fatalf("failed to parse IP %q", tc.ip)
|
||||||
|
}
|
||||||
|
|
||||||
|
got := isPrivateIP(ip)
|
||||||
|
t.Logf("isPrivateIP(%s) = %v", ip, got)
|
||||||
|
|
||||||
|
if got != tc.want {
|
||||||
|
t.Errorf("isPrivateIP(%q) = %v, want %v", tc.ip, got, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestValidateURL_RejectsNonHTTPS(t *testing.T) {
|
func TestValidateURL_RejectsNonHTTPS(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -50,3 +50,69 @@ func TestImageProcessor_RefusesComputedSideAboveMaximum(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestImageProcessor_ComputedSideIsAtLeastOne verifies that, for a size with
|
||||||
|
// one side 0, the side computed from the input's proportions is 1 where it
|
||||||
|
// would round down to 0, and that Process makes that size in every fit mode
|
||||||
|
// but FitOutside, which scales the input to cover the size and so can make
|
||||||
|
// it longer. Each case checks the size targetDimensions gives before it
|
||||||
|
// calls Process: with that side at 0, an 8192x1 input at 1300x0 makes
|
||||||
|
// libvips ask for about 39 GB with FitCover.
|
||||||
|
func TestImageProcessor_ComputedSideIsAtLeastOne(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
inputWidth, inputHeight int
|
||||||
|
size Size
|
||||||
|
wantWidth, wantHeight int
|
||||||
|
}{
|
||||||
|
{"8192x1 at 100x0", 8192, 1, Size{Width: 100}, 100, 1},
|
||||||
|
{"8192x1 at 1300x0", 8192, 1, Size{Width: 1300}, 1300, 1},
|
||||||
|
{"8192x2 at 100x0", 8192, 2, Size{Width: 100}, 100, 1},
|
||||||
|
{"1x8192 at 0x100", 1, 8192, Size{Height: 100}, 1, 100},
|
||||||
|
{"1x8192 at 0x1300", 1, 8192, Size{Height: 1300}, 1, 1300},
|
||||||
|
{"2x8192 at 0x100", 2, 8192, Size{Height: 100}, 1, 100},
|
||||||
|
}
|
||||||
|
|
||||||
|
fits := []FitMode{FitCover, FitContain, FitFill, FitInside, FitOutside}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
for _, fit := range fits {
|
||||||
|
t.Run(tt.name+" "+string(fit), func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
width, height, err := targetDimensions(tt.size,
|
||||||
|
tt.inputWidth, tt.inputHeight)
|
||||||
|
if err != nil || width != tt.wantWidth || height != tt.wantHeight {
|
||||||
|
t.Fatalf("targetDimensions() = %dx%d, %v, want %dx%d",
|
||||||
|
width, height, err, tt.wantWidth, tt.wantHeight)
|
||||||
|
}
|
||||||
|
|
||||||
|
input := createTestJPEG(t, tt.inputWidth, tt.inputHeight)
|
||||||
|
|
||||||
|
result, err := New(Params{}).Process(t.Context(),
|
||||||
|
bytes.NewReader(input),
|
||||||
|
&Request{Size: tt.size, Format: FormatJPEG, FitMode: fit})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Process() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
defer func() { _ = result.Content.Close() }()
|
||||||
|
|
||||||
|
t.Logf("Process() made %dx%d", result.Width, result.Height)
|
||||||
|
|
||||||
|
ok := result.Width == tt.wantWidth && result.Height == tt.wantHeight
|
||||||
|
if fit == FitOutside {
|
||||||
|
ok = result.Width >= tt.wantWidth && result.Height >= tt.wantHeight
|
||||||
|
}
|
||||||
|
|
||||||
|
if !ok {
|
||||||
|
t.Errorf("Process() made %dx%d, want %dx%d (at least, for %s)",
|
||||||
|
result.Width, result.Height, tt.wantWidth, tt.wantHeight,
|
||||||
|
FitOutside)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,268 @@
|
|||||||
|
package imageprocessor
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"image"
|
||||||
|
"image/color"
|
||||||
|
"image/png"
|
||||||
|
"math"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestImageProcessor_FitCoverServesThinSource verifies that FitCover serves
|
||||||
|
// a thin source at the requested size. An 8192x1 source scaled to cover
|
||||||
|
// 8192x8192 is 8192 times as long as the output on its long side. The centre
|
||||||
|
// of an 8192x1 source with the proportions of 1x8192 is 1/8192 of a pixel
|
||||||
|
// wide. Each case is also turned on its side.
|
||||||
|
func TestImageProcessor_FitCoverServesThinSource(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
inputWidth, inputHeight int
|
||||||
|
size Size
|
||||||
|
}{
|
||||||
|
{"8192x1 at 8192x8192", 8192, 1, Size{Width: 8192, Height: 8192}},
|
||||||
|
{"8192x1 at 1x8192", 8192, 1, Size{Width: 1, Height: 8192}},
|
||||||
|
{"1x8192 at 8192x8192", 1, 8192, Size{Width: 8192, Height: 8192}},
|
||||||
|
{"1x8192 at 8192x1", 1, 8192, Size{Width: 8192, Height: 1}},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
input := createTestJPEG(t, tt.inputWidth, tt.inputHeight)
|
||||||
|
|
||||||
|
result, err := New(Params{}).Process(t.Context(),
|
||||||
|
bytes.NewReader(input),
|
||||||
|
&Request{Size: tt.size, Format: FormatJPEG, FitMode: FitCover})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Process() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
defer func() { _ = result.Content.Close() }()
|
||||||
|
|
||||||
|
t.Logf("Process() made %dx%d", result.Width, result.Height)
|
||||||
|
|
||||||
|
if result.Width != tt.size.Width || result.Height != tt.size.Height {
|
||||||
|
t.Errorf("Process() made %dx%d, want %dx%d",
|
||||||
|
result.Width, result.Height, tt.size.Width, tt.size.Height)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// createBandsPNG creates a PNG of three equal bands, red, green and blue,
|
||||||
|
// side by side when it is wider than tall and top to bottom otherwise.
|
||||||
|
func createBandsPNG(t *testing.T, width, height int) []byte {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
bands := []color.RGBA{
|
||||||
|
{R: 255, A: 255},
|
||||||
|
{G: 255, A: 255},
|
||||||
|
{B: 255, A: 255},
|
||||||
|
}
|
||||||
|
|
||||||
|
img := image.NewRGBA(image.Rect(0, 0, width, height))
|
||||||
|
for y := range height {
|
||||||
|
for x := range width {
|
||||||
|
band := 3 * y / height
|
||||||
|
if width > height {
|
||||||
|
band = 3 * x / width
|
||||||
|
}
|
||||||
|
|
||||||
|
img.Set(x, y, bands[band])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return encodePNG(t, img)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestImageProcessor_FitCoverCropsTheCentre verifies that FitCover keeps the
|
||||||
|
// centre of the source: a 300x100 source of red, green and blue bands side
|
||||||
|
// by side, or a 100x300 one with them top to bottom, comes out at 50x50 as
|
||||||
|
// its green band.
|
||||||
|
func TestImageProcessor_FitCoverCropsTheCentre(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
inputWidth, inputHeight int
|
||||||
|
}{
|
||||||
|
{"bands side by side", 300, 100},
|
||||||
|
{"bands top to bottom", 100, 300},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
input := createBandsPNG(t, tt.inputWidth, tt.inputHeight)
|
||||||
|
|
||||||
|
result, err := New(Params{}).Process(t.Context(),
|
||||||
|
bytes.NewReader(input),
|
||||||
|
&Request{
|
||||||
|
Size: Size{Width: 50, Height: 50},
|
||||||
|
Format: FormatPNG,
|
||||||
|
FitMode: FitCover,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Process() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
defer func() { _ = result.Content.Close() }()
|
||||||
|
|
||||||
|
output, err := png.Decode(result.Content)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to decode output: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
bounds := output.Bounds()
|
||||||
|
t.Logf("Process() made %dx%d", bounds.Dx(), bounds.Dy())
|
||||||
|
|
||||||
|
if bounds.Dx() != 50 || bounds.Dy() != 50 {
|
||||||
|
t.Fatalf("Process() made %dx%d, want 50x50",
|
||||||
|
bounds.Dx(), bounds.Dy())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Resampling may blend a little of the bands beside the
|
||||||
|
// green one into the output's edges.
|
||||||
|
for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
|
||||||
|
for x := bounds.Min.X; x < bounds.Max.X; x++ {
|
||||||
|
r, g, b, _ := output.At(x, y).RGBA()
|
||||||
|
if r>>8 > 64 || g>>8 < 192 || b>>8 > 64 {
|
||||||
|
t.Fatalf("output pixel %d,%d is %d,%d,%d, want green",
|
||||||
|
x, y, r>>8, g>>8, b>>8)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The greys of createSquarePNG, and the grey halfway between them.
|
||||||
|
const (
|
||||||
|
backgroundGrey = 192
|
||||||
|
squareGrey = 64
|
||||||
|
halfwayGrey = (backgroundGrey + squareGrey) / 2
|
||||||
|
)
|
||||||
|
|
||||||
|
// createSquarePNG creates a grey PNG side pixels square with a darker grey
|
||||||
|
// square squareSide pixels wide in its centre.
|
||||||
|
func createSquarePNG(t *testing.T, side, squareSide int) []byte {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
first := (side - squareSide) / 2
|
||||||
|
last := first + squareSide - 1
|
||||||
|
|
||||||
|
img := image.NewGray(image.Rect(0, 0, side, side))
|
||||||
|
for y := range side {
|
||||||
|
for x := range side {
|
||||||
|
grey := uint8(backgroundGrey)
|
||||||
|
if x >= first && x <= last && y >= first && y <= last {
|
||||||
|
grey = squareGrey
|
||||||
|
}
|
||||||
|
|
||||||
|
img.SetGray(x, y, color.Gray{Y: grey})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return encodePNG(t, img)
|
||||||
|
}
|
||||||
|
|
||||||
|
// squareLength returns how long the square is along greys, the greys of a
|
||||||
|
// row or column of the output that crosses it: the distance between the
|
||||||
|
// first and the last point where greys passes halfwayGrey, or 0 when there
|
||||||
|
// is no such point. Resampling blurs the square's edges over several
|
||||||
|
// pixels, and each point is placed between the two pixels on either side
|
||||||
|
// of it in proportion to their greys, so the length is measured to a
|
||||||
|
// fraction of a pixel.
|
||||||
|
func squareLength(greys []float64) float64 {
|
||||||
|
first, last := -1.0, -1.0
|
||||||
|
|
||||||
|
for i := 1; i < len(greys); i++ {
|
||||||
|
before, after := greys[i-1], greys[i]
|
||||||
|
if (before > halfwayGrey) == (after > halfwayGrey) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
point := float64(i-1) + (before-halfwayGrey)/(before-after)
|
||||||
|
if first < 0 {
|
||||||
|
first = point
|
||||||
|
}
|
||||||
|
|
||||||
|
last = point
|
||||||
|
}
|
||||||
|
|
||||||
|
return last - first
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestImageProcessor_FitCoverKeepsProportions verifies that FitCover scales
|
||||||
|
// both sides of the source by the same amount: a square in the centre of
|
||||||
|
// the source comes out square. Its width and height are measured along the
|
||||||
|
// output's middle row and column.
|
||||||
|
func TestImageProcessor_FitCoverKeepsProportions(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
side, squareSide int
|
||||||
|
size Size
|
||||||
|
}{
|
||||||
|
{"16x16 at 100x60", 16, 8, Size{Width: 100, Height: 60}},
|
||||||
|
{"16x16 at 60x100", 16, 8, Size{Width: 60, Height: 100}},
|
||||||
|
{"10x10 at 100x35", 10, 2, Size{Width: 100, Height: 35}},
|
||||||
|
{"10x10 at 35x100", 10, 2, Size{Width: 35, Height: 100}},
|
||||||
|
{"64x64 at 300x100", 64, 16, Size{Width: 300, Height: 100}},
|
||||||
|
{"64x64 at 100x300", 64, 16, Size{Width: 100, Height: 300}},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
input := createSquarePNG(t, tt.side, tt.squareSide)
|
||||||
|
|
||||||
|
result, err := New(Params{}).Process(t.Context(),
|
||||||
|
bytes.NewReader(input),
|
||||||
|
&Request{Size: tt.size, Format: FormatPNG, FitMode: FitCover})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Process() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
defer func() { _ = result.Content.Close() }()
|
||||||
|
|
||||||
|
output, err := png.Decode(result.Content)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to decode output: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
bounds := output.Bounds()
|
||||||
|
if bounds.Dx() != tt.size.Width || bounds.Dy() != tt.size.Height {
|
||||||
|
t.Fatalf("Process() made %dx%d, want %dx%d",
|
||||||
|
bounds.Dx(), bounds.Dy(), tt.size.Width, tt.size.Height)
|
||||||
|
}
|
||||||
|
|
||||||
|
row := make([]float64, bounds.Dx())
|
||||||
|
for x := range row {
|
||||||
|
r, _, _, _ := output.At(x, bounds.Dy()/2).RGBA()
|
||||||
|
row[x] = float64(r >> 8)
|
||||||
|
}
|
||||||
|
|
||||||
|
column := make([]float64, bounds.Dy())
|
||||||
|
for y := range column {
|
||||||
|
r, _, _, _ := output.At(bounds.Dx()/2, y).RGBA()
|
||||||
|
column[y] = float64(r >> 8)
|
||||||
|
}
|
||||||
|
|
||||||
|
width, height := squareLength(row), squareLength(column)
|
||||||
|
t.Logf("square is %.2f wide and %.2f high", width, height)
|
||||||
|
|
||||||
|
if width == 0 || math.Abs(width-height) > 0.5 {
|
||||||
|
t.Errorf("square is %.2f wide and %.2f high, want it square",
|
||||||
|
width, height)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -60,3 +60,45 @@ func TestImageProcessor_RefusesFitOutsideAboveMaximum(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestImageProcessor_FitOutsideSideIsAtLeastOne verifies that FitOutside
|
||||||
|
// keeps each side at 1 or more when it works both sides out again from the
|
||||||
|
// larger scale. A side of 49 scaled to 1 is 49 times 1/49, which in floating
|
||||||
|
// point is just under 1 and so rounded down to 0, a width libvips refuses.
|
||||||
|
func TestImageProcessor_FitOutsideSideIsAtLeastOne(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
inputWidth, inputHeight int
|
||||||
|
size Size
|
||||||
|
wantWidth, wantHeight int
|
||||||
|
}{
|
||||||
|
{"49x8192 at 0x100", 49, 8192, Size{Height: 100}, 1, 167},
|
||||||
|
{"49x49 at 1x1", 49, 49, Size{Width: 1, Height: 1}, 1, 1},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
input := createTestJPEG(t, tt.inputWidth, tt.inputHeight)
|
||||||
|
|
||||||
|
result, err := New(Params{}).Process(t.Context(),
|
||||||
|
bytes.NewReader(input),
|
||||||
|
&Request{Size: tt.size, Format: FormatJPEG, FitMode: FitOutside})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Process() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
defer func() { _ = result.Content.Close() }()
|
||||||
|
|
||||||
|
t.Logf("Process() made %dx%d", result.Width, result.Height)
|
||||||
|
|
||||||
|
if result.Width != tt.wantWidth || result.Height != tt.wantHeight {
|
||||||
|
t.Errorf("Process() made %dx%d, want %dx%d",
|
||||||
|
result.Width, result.Height, tt.wantWidth, tt.wantHeight)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -311,8 +311,10 @@ func (p *ImageProcessor) Process(
|
|||||||
|
|
||||||
// targetDimensions calculates the output dimensions for a requested size,
|
// targetDimensions calculates the output dimensions for a requested size,
|
||||||
// scaling proportionally when only one dimension is given and keeping the
|
// scaling proportionally when only one dimension is given and keeping the
|
||||||
// original dimensions when both are zero. A side scaled proportionally can
|
// original dimensions when both are zero. A side scaled proportionally is
|
||||||
// come out above MaxOutputDimension, which is ErrOutputTooLarge.
|
// rounded down but kept at 1 or more, as for a thin image it would round
|
||||||
|
// down to 0, a side of no pixels. It can come out above MaxOutputDimension,
|
||||||
|
// which is ErrOutputTooLarge.
|
||||||
func targetDimensions(size Size, origWidth, origHeight int) (int, int, error) {
|
func targetDimensions(size Size, origWidth, origHeight int) (int, int, error) {
|
||||||
width, height := size.Width, size.Height
|
width, height := size.Width, size.Height
|
||||||
|
|
||||||
@@ -322,10 +324,10 @@ func targetDimensions(size Size, origWidth, origHeight int) (int, int, error) {
|
|||||||
return origWidth, origHeight, nil
|
return origWidth, origHeight, nil
|
||||||
case width == 0:
|
case width == 0:
|
||||||
// Only height specified: calculate width proportionally
|
// Only height specified: calculate width proportionally
|
||||||
width = origWidth * height / origHeight
|
width = max(1, origWidth*height/origHeight)
|
||||||
case height == 0:
|
case height == 0:
|
||||||
// Only width specified: calculate height proportionally
|
// Only width specified: calculate height proportionally
|
||||||
height = origHeight * width / origWidth
|
height = max(1, origHeight*width/origWidth)
|
||||||
}
|
}
|
||||||
|
|
||||||
if width > MaxOutputDimension || height > MaxOutputDimension {
|
if width > MaxOutputDimension || height > MaxOutputDimension {
|
||||||
@@ -475,17 +477,20 @@ func (p *ImageProcessor) resize(
|
|||||||
) error {
|
) error {
|
||||||
switch fit {
|
switch fit {
|
||||||
case FitCover, "":
|
case FitCover, "":
|
||||||
// Resize and crop to fill exact dimensions (default)
|
// Crop the centre and resize to fill exact dimensions (default)
|
||||||
return img.Thumbnail(width, height, vips.InterestingCentre)
|
return cropCentreAndResize(img, width, height)
|
||||||
|
|
||||||
case FitContain:
|
case FitContain:
|
||||||
// Resize to fit within dimensions, maintaining aspect ratio
|
// Resize to fit within dimensions, maintaining aspect ratio. Scaled
|
||||||
|
// by the smaller scale, the short side of a thin image can round
|
||||||
|
// down to 0, which libvips refuses as a width, so each side is at
|
||||||
|
// least 1.
|
||||||
imgW, imgH := img.Width(), img.Height()
|
imgW, imgH := img.Width(), img.Height()
|
||||||
scaleW := float64(width) / float64(imgW)
|
scaleW := float64(width) / float64(imgW)
|
||||||
scaleH := float64(height) / float64(imgH)
|
scaleH := float64(height) / float64(imgH)
|
||||||
scale := min(scaleW, scaleH)
|
scale := min(scaleW, scaleH)
|
||||||
newW := int(float64(imgW) * scale)
|
newW := max(1, int(float64(imgW)*scale))
|
||||||
newH := int(float64(imgH) * scale)
|
newH := max(1, int(float64(imgH)*scale))
|
||||||
|
|
||||||
return img.Thumbnail(newW, newH, vips.InterestingNone)
|
return img.Thumbnail(newW, newH, vips.InterestingNone)
|
||||||
|
|
||||||
@@ -503,19 +508,22 @@ func (p *ImageProcessor) resize(
|
|||||||
scaleW := float64(width) / float64(imgW)
|
scaleW := float64(width) / float64(imgW)
|
||||||
scaleH := float64(height) / float64(imgH)
|
scaleH := float64(height) / float64(imgH)
|
||||||
scale := min(scaleW, scaleH)
|
scale := min(scaleW, scaleH)
|
||||||
newW := int(float64(imgW) * scale)
|
newW := max(1, int(float64(imgW)*scale))
|
||||||
newH := int(float64(imgH) * scale)
|
newH := max(1, int(float64(imgH)*scale))
|
||||||
|
|
||||||
return img.Thumbnail(newW, newH, vips.InterestingNone)
|
return img.Thumbnail(newW, newH, vips.InterestingNone)
|
||||||
|
|
||||||
case FitOutside:
|
case FitOutside:
|
||||||
// Resize so smallest dimension fits, may exceed target on other dimension
|
// Resize so smallest dimension fits, may exceed target on other
|
||||||
|
// dimension. A side scaled to 1 can come out just under 1 in
|
||||||
|
// floating point, as 49 times 1/49 does, and round down to 0, which
|
||||||
|
// libvips refuses as a width, so each side is at least 1.
|
||||||
imgW, imgH := img.Width(), img.Height()
|
imgW, imgH := img.Width(), img.Height()
|
||||||
scaleW := float64(width) / float64(imgW)
|
scaleW := float64(width) / float64(imgW)
|
||||||
scaleH := float64(height) / float64(imgH)
|
scaleH := float64(height) / float64(imgH)
|
||||||
scale := max(scaleW, scaleH)
|
scale := max(scaleW, scaleH)
|
||||||
newW := int(float64(imgW) * scale)
|
newW := max(1, int(float64(imgW)*scale))
|
||||||
newH := int(float64(imgH) * scale)
|
newH := max(1, int(float64(imgH)*scale))
|
||||||
|
|
||||||
// The dimension that exceeds the target is held to the maximum too
|
// The dimension that exceeds the target is held to the maximum too
|
||||||
if newW > MaxOutputDimension || newH > MaxOutputDimension {
|
if newW > MaxOutputDimension || newH > MaxOutputDimension {
|
||||||
@@ -530,6 +538,38 @@ func (p *ImageProcessor) resize(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// cropCentreAndResize scales img by the same amount on both sides, the
|
||||||
|
// least that makes it cover width x height, and cuts the centre width x
|
||||||
|
// height out of it, as libvips' thumbnail does with InterestingCentre.
|
||||||
|
// thumbnail alone scales the whole image and copies the result into memory
|
||||||
|
// before it cuts, and for a thin image that is far larger than the image or
|
||||||
|
// the output: 8192x64 at 8192x8192 is 1048576x8192, about 24 GiB. So the
|
||||||
|
// centre is cut from img first, widened to whole pixels of img, which adds
|
||||||
|
// less than one pixel of img on each side, and thumbnail scales and cuts
|
||||||
|
// only that.
|
||||||
|
func cropCentreAndResize(img *vips.ImageRef, width, height int) error {
|
||||||
|
imgW, imgH := img.Width(), img.Height()
|
||||||
|
|
||||||
|
// An image wider than the requested proportions keeps its height and
|
||||||
|
// loses width down to imgH*width/height, any other keeps its width and
|
||||||
|
// loses height down to imgW*height/width. Half of what is lost comes
|
||||||
|
// off each side, rounded down to whole pixels, so the part kept holds
|
||||||
|
// all of the centre.
|
||||||
|
left, top := 0, 0
|
||||||
|
if imgW*height > imgH*width {
|
||||||
|
left = (imgW*height - imgH*width) / (2 * height) //nolint:mnd // each side
|
||||||
|
} else {
|
||||||
|
top = (imgH*width - imgW*height) / (2 * width) //nolint:mnd // each side
|
||||||
|
}
|
||||||
|
|
||||||
|
err := img.ExtractArea(left, top, imgW-2*left, imgH-2*top)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return img.Thumbnail(width, height, vips.InterestingCentre)
|
||||||
|
}
|
||||||
|
|
||||||
const defaultQuality = 85
|
const defaultQuality = 85
|
||||||
|
|
||||||
// encode encodes an image to the specified format. govips gives pixa no way
|
// encode encodes an image to the specified format. govips gives pixa no way
|
||||||
|
|||||||
@@ -18,6 +18,14 @@ import (
|
|||||||
// write-pressure wakeups triggered by stores.
|
// write-pressure wakeups triggered by stores.
|
||||||
const DefaultEvictionInterval = 5 * time.Minute
|
const DefaultEvictionInterval = 5 * time.Minute
|
||||||
|
|
||||||
|
// reconciliationWaitFactor is how many times as long as a
|
||||||
|
// reconciliation pass took the eviction loop waits at least, after that
|
||||||
|
// pass ends, before it starts the next one. A pass reads a row and stats a
|
||||||
|
// file for every cached file, so its time grows with the cache; waiting
|
||||||
|
// ten times as long keeps reconciliation to at most a tenth of the time,
|
||||||
|
// however large the cache is.
|
||||||
|
const reconciliationWaitFactor = 10
|
||||||
|
|
||||||
// evictionBatchSize is how many LRU candidates of each class (variants
|
// evictionBatchSize is how many LRU candidates of each class (variants
|
||||||
// and source blobs) one eviction pass fetches from the database.
|
// and source blobs) one eviction pass fetches from the database.
|
||||||
const evictionBatchSize = 100
|
const evictionBatchSize = 100
|
||||||
@@ -438,10 +446,12 @@ func (c *Cache) notifyWritePressure() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// StartEviction launches the background eviction goroutine, which
|
// StartEviction launches the background eviction goroutine, which
|
||||||
// reconciles the database accounting with the cache directories and
|
|
||||||
// removes the expired negative cache rows at startup and again on every
|
// removes the expired negative cache rows at startup and again on every
|
||||||
// periodic tick thereafter, and evicts to the configured limit on the
|
// periodic tick thereafter, reconciles the database accounting with the
|
||||||
// given periodic interval and on write-pressure notifications. On a
|
// cache directories at startup and again on the first periodic tick
|
||||||
|
// that comes at least ten times as long after a reconciliation pass
|
||||||
|
// ended as that pass took, and evicts to the configured limit on the given
|
||||||
|
// periodic interval and on write-pressure notifications. On a
|
||||||
// disabled cache it only removes the expired negative cache rows, as
|
// disabled cache it only removes the expired negative cache rows, as
|
||||||
// the negative cache is used either way. It is a no-op when already
|
// the negative cache is used either way. It is a no-op when already
|
||||||
// started. The goroutine outlives the caller, so it runs with its own
|
// started. The goroutine outlives the caller, so it runs with its own
|
||||||
@@ -482,7 +492,7 @@ func (c *Cache) StopEviction(ctx context.Context) error {
|
|||||||
func (c *Cache) evictionLoop(ctx context.Context, interval time.Duration) {
|
func (c *Cache) evictionLoop(ctx context.Context, interval time.Duration) {
|
||||||
defer close(c.evictionDone)
|
defer close(c.evictionDone)
|
||||||
|
|
||||||
c.runReconciliationPass(ctx)
|
nextReconciliation := c.runReconciliationPass(ctx)
|
||||||
c.runCleanExpired(ctx)
|
c.runCleanExpired(ctx)
|
||||||
c.runEvictionPass(ctx)
|
c.runEvictionPass(ctx)
|
||||||
|
|
||||||
@@ -494,16 +504,20 @@ func (c *Cache) evictionLoop(ctx context.Context, interval time.Duration) {
|
|||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return
|
return
|
||||||
case <-ticker.C:
|
case <-ticker.C:
|
||||||
// Reconciliation walks the cache directories, so it only
|
// Reconciliation walks the cache directories and reads a
|
||||||
// runs on the periodic ticker rather than on every
|
// row for every file, so it runs only on the periodic
|
||||||
// write-pressure wakeup, keeping it off the per-store hot
|
// ticker rather than on every write-pressure wakeup, and
|
||||||
// path. Reusing the eviction interval itself (rather than a
|
// only on the first tick at or after nextReconciliation,
|
||||||
// separate, longer one) is a deliberate choice: it is the
|
// which is ten times as long after the last pass ended as
|
||||||
// simplest option that still bounds how long a store's
|
// that pass took. Passes then never run back to back,
|
||||||
// best-effort accounting insert can stay silently
|
// however long one takes. A file whose best-effort
|
||||||
// unaccounted for to one interval, on a process that is
|
// accounting insert failed in a store stays unaccounted for
|
||||||
// already running this loop regardless.
|
// until the next pass, up to about one eviction interval
|
||||||
c.runReconciliationPass(ctx)
|
// plus ten times the last pass's time.
|
||||||
|
if !time.Now().Before(nextReconciliation) {
|
||||||
|
nextReconciliation = c.runReconciliationPass(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
c.runCleanExpired(ctx)
|
c.runCleanExpired(ctx)
|
||||||
case <-c.evictionPressure:
|
case <-c.evictionPressure:
|
||||||
}
|
}
|
||||||
@@ -527,17 +541,25 @@ func (c *Cache) runEvictionPass(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// runReconciliationPass runs one reconciliation pass, logging failures
|
// runReconciliationPass runs one reconciliation pass, logging failures
|
||||||
// instead of propagating them (the loop must keep running). It does
|
// instead of propagating them (the loop must keep running), and returns
|
||||||
// nothing once ctx is cancelled.
|
// the earliest time the next pass may start: reconciliationWaitFactor
|
||||||
func (c *Cache) runReconciliationPass(ctx context.Context) {
|
// times as long after this pass ended as it took. It does nothing once
|
||||||
|
// ctx is cancelled.
|
||||||
|
func (c *Cache) runReconciliationPass(ctx context.Context) time.Time {
|
||||||
if ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
return
|
return time.Now()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
started := time.Now()
|
||||||
|
|
||||||
err := c.reconcileAccounting(ctx)
|
err := c.reconcileAccounting(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
c.log.Warn("cache accounting reconciliation failed", "error", err)
|
c.log.Warn("cache accounting reconciliation failed", "error", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
ended := time.Now()
|
||||||
|
|
||||||
|
return ended.Add(reconciliationWaitFactor * ended.Sub(started))
|
||||||
}
|
}
|
||||||
|
|
||||||
// runCleanExpired runs CleanExpired, logging a failure instead of
|
// runCleanExpired runs CleanExpired, logging a failure instead of
|
||||||
@@ -556,7 +578,8 @@ func (c *Cache) runCleanExpired(ctx context.Context) {
|
|||||||
|
|
||||||
// reconcileAccounting synchronizes the database size accounting with
|
// reconcileAccounting synchronizes the database size accounting with
|
||||||
// the actual contents of the cache directories. It runs at startup and
|
// the actual contents of the cache directories. It runs at startup and
|
||||||
// again on every periodic eviction tick thereafter, off the request
|
// then on the first periodic eviction tick that comes at least ten times
|
||||||
|
// as long after the last pass ended as that pass took, off the request
|
||||||
// hot path: it adopts variant files that predate the accounting table
|
// hot path: it adopts variant files that predate the accounting table
|
||||||
// (or whose accounting insert failed, e.g. StoreVariant's best-effort
|
// (or whose accounting insert failed, e.g. StoreVariant's best-effort
|
||||||
// insert under transient DB contention), drops accounting rows whose
|
// insert under transient DB contention), drops accounting rows whose
|
||||||
@@ -566,9 +589,10 @@ func (c *Cache) runCleanExpired(ctx context.Context) {
|
|||||||
// files left behind by crashed writes, and last checks the total cache
|
// files left behind by crashed writes, and last checks the total cache
|
||||||
// usage against the tables. It reads the tables a page at a time.
|
// usage against the tables. It reads the tables a page at a time.
|
||||||
// Running it periodically, not just once, bounds how long such drift can
|
// Running it periodically, not just once, bounds how long such drift can
|
||||||
// accumulate unaccounted for on a long-running process to one eviction
|
// accumulate unaccounted for on a long-running process to about one
|
||||||
// interval. Once ctx is cancelled, it stops at the next file or row and
|
// eviction interval plus ten times the last pass's time.
|
||||||
// returns ctx's error.
|
// Once ctx is cancelled, it stops at the next file or row and returns
|
||||||
|
// ctx's error.
|
||||||
func (c *Cache) reconcileAccounting(ctx context.Context) error {
|
func (c *Cache) reconcileAccounting(ctx context.Context) error {
|
||||||
if c.disabled {
|
if c.disabled {
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -918,6 +918,89 @@ func TestPeriodicReconciliationAdoptsFileThatAppearsAfterStartup(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestReconciliationPassesAreSpacedByTenTimesTheirTime makes the hook
|
||||||
|
// called after every read of the reconciliation pass sleep 10 ms, runs
|
||||||
|
// the eviction loop on a 10 ms interval, and records when each of those
|
||||||
|
// calls starts and ends. A pass starts before its first call and ends
|
||||||
|
// after its last, so the time from the first call's start to the last
|
||||||
|
// call's end is at most the pass's time. The next pass's first call must
|
||||||
|
// start at least ten times that long after the last call ended.
|
||||||
|
func TestReconciliationPassesAreSpacedByTenTimesTheirTime(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
cache, _ := newEvictionTestCache(t, 1<<30)
|
||||||
|
|
||||||
|
// A pass made here, before the loop starts, counts the reads one
|
||||||
|
// pass makes on this empty cache, so the reads recorded below can be
|
||||||
|
// split into passes.
|
||||||
|
readsPerPass := 0
|
||||||
|
cache.reconciliationReadTestHook = func(int) { readsPerPass++ }
|
||||||
|
|
||||||
|
err := cache.reconcileAccounting(t.Context())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("reconcileAccounting failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("one reconciliation pass makes %d reads", readsPerPass)
|
||||||
|
|
||||||
|
if readsPerPass == 0 {
|
||||||
|
t.Fatal("a reconciliation pass made no read")
|
||||||
|
}
|
||||||
|
|
||||||
|
// The pass at startup and two after it.
|
||||||
|
const passes = 3
|
||||||
|
|
||||||
|
type read struct{ start, end time.Time }
|
||||||
|
|
||||||
|
// The channel holds every read the test looks at, so none of those
|
||||||
|
// is dropped; reads after them are, and never block the loop.
|
||||||
|
reads := make(chan read, passes*readsPerPass)
|
||||||
|
|
||||||
|
cache.reconciliationReadTestHook = func(int) {
|
||||||
|
start := time.Now()
|
||||||
|
|
||||||
|
time.Sleep(10 * time.Millisecond)
|
||||||
|
|
||||||
|
select {
|
||||||
|
case reads <- read{start: start, end: time.Now()}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
cache.StartEviction(10 * time.Millisecond)
|
||||||
|
defer func() { _ = cache.StopEviction(t.Context()) }()
|
||||||
|
|
||||||
|
recorded := make([]read, 0, passes*readsPerPass)
|
||||||
|
timeout := time.After(15 * time.Second)
|
||||||
|
|
||||||
|
for len(recorded) < passes*readsPerPass {
|
||||||
|
select {
|
||||||
|
case r := <-reads:
|
||||||
|
recorded = append(recorded, r)
|
||||||
|
case <-timeout:
|
||||||
|
t.Fatalf("saw %d reconciliation reads in 15 s, want %d (%d passes)",
|
||||||
|
len(recorded), passes*readsPerPass, passes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for pass := range passes - 1 {
|
||||||
|
first := recorded[pass*readsPerPass]
|
||||||
|
last := recorded[(pass+1)*readsPerPass-1]
|
||||||
|
next := recorded[(pass+1)*readsPerPass]
|
||||||
|
|
||||||
|
passTime := last.end.Sub(first.start)
|
||||||
|
wait := next.start.Sub(last.end)
|
||||||
|
|
||||||
|
t.Logf("pass %d took at least %v, and the next pass started %v after it",
|
||||||
|
pass+1, passTime, wait)
|
||||||
|
|
||||||
|
if wait < 10*passTime {
|
||||||
|
t.Errorf("pass %d took at least %v, and the next pass started %v "+
|
||||||
|
"after it, want at least %v", pass+1, passTime, wait, 10*passTime)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestStopEvictionInterruptsPassInProgress holds the test database's
|
// TestStopEvictionInterruptsPassInProgress holds the test database's
|
||||||
// only connection, so the startup reconciliation pass waits for it, and
|
// only connection, so the startup reconciliation pass waits for it, and
|
||||||
// checks that StopEviction stops that pass instead of waiting for the
|
// checks that StopEviction stops that pass instead of waiting for the
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
|
"net/url"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -86,9 +87,11 @@ func ValidateImageRequest(req *ImageRequest) error {
|
|||||||
type ImageRequest struct {
|
type ImageRequest struct {
|
||||||
// SourceHost is the origin host (e.g., "cdn.example.com")
|
// SourceHost is the origin host (e.g., "cdn.example.com")
|
||||||
SourceHost string
|
SourceHost string
|
||||||
// SourcePath is the path on the origin (e.g., "/photos/cat.jpg")
|
// SourcePath is the path on the origin with its percent-escapes decoded,
|
||||||
|
// as url.URL's Path holds it (e.g., "/photos/my cat.jpg")
|
||||||
SourcePath string
|
SourcePath string
|
||||||
// SourceQuery is the optional query string for the origin URL
|
// SourceQuery is the optional query string for the origin URL as the URL
|
||||||
|
// writes it, percent-escapes included, as url.URL's RawQuery holds it
|
||||||
SourceQuery string
|
SourceQuery string
|
||||||
// Size is the requested output dimensions
|
// Size is the requested output dimensions
|
||||||
Size Size
|
Size Size
|
||||||
@@ -107,20 +110,25 @@ type ImageRequest struct {
|
|||||||
AllowHTTP bool
|
AllowHTTP bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// SourceURL returns the full upstream URL to fetch.
|
// SourceURL returns the full upstream URL to fetch: the source path
|
||||||
// Uses http:// scheme when AllowHTTP is true, otherwise https://.
|
// percent-encoded again, as net/url writes a path, so each source path is one
|
||||||
|
// upstream path, and the source query as it is, with the characters
|
||||||
|
// escapeQuery encodes percent-encoded. Uses http:// scheme when AllowHTTP is
|
||||||
|
// true, otherwise https://.
|
||||||
func (r *ImageRequest) SourceURL() string {
|
func (r *ImageRequest) SourceURL() string {
|
||||||
scheme := "https"
|
scheme := "https"
|
||||||
if r.AllowHTTP {
|
if r.AllowHTTP {
|
||||||
scheme = "http"
|
scheme = "http"
|
||||||
}
|
}
|
||||||
|
|
||||||
url := scheme + "://" + r.SourceHost + r.SourcePath
|
sourceURL := url.URL{
|
||||||
if r.SourceQuery != "" {
|
Scheme: scheme,
|
||||||
url += "?" + r.SourceQuery
|
Host: r.SourceHost,
|
||||||
|
Path: r.SourcePath,
|
||||||
|
RawQuery: escapeQuery(r.SourceQuery),
|
||||||
}
|
}
|
||||||
|
|
||||||
return url
|
return sourceURL.String()
|
||||||
}
|
}
|
||||||
|
|
||||||
// ImageResponse represents a processed image ready to serve
|
// ImageResponse represents a processed image ready to serve
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package imgcache
|
|||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"net/netip"
|
||||||
"net/url"
|
"net/url"
|
||||||
"regexp"
|
"regexp"
|
||||||
"strconv"
|
"strconv"
|
||||||
@@ -28,7 +29,9 @@ var (
|
|||||||
const MaxDimension = imageprocessor.MaxOutputDimension
|
const MaxDimension = imageprocessor.MaxOutputDimension
|
||||||
|
|
||||||
// ValidateDimension checks one requested width or height; name ("width" or
|
// ValidateDimension checks one requested width or height; name ("width" or
|
||||||
// "height") appears in the error. 0 means "original size" and is valid.
|
// "height") appears in the error. 0 is valid: that side is then worked out from
|
||||||
|
// the source's proportions and the other requested side, rounded down but at
|
||||||
|
// least 1, or is the source's own side when the other side is 0 too.
|
||||||
func ValidateDimension(name string, value int) error {
|
func ValidateDimension(name string, value int) error {
|
||||||
if value < 0 {
|
if value < 0 {
|
||||||
return fmt.Errorf("%w: %s is negative", ErrInvalidSize, name)
|
return fmt.Errorf("%w: %s is negative", ErrInvalidSize, name)
|
||||||
@@ -47,6 +50,24 @@ func ValidateDimension(name string, value int) error {
|
|||||||
var sizeFormatRegex = regexp.MustCompile(
|
var sizeFormatRegex = regexp.MustCompile(
|
||||||
`^(\d+)x(\d+)(?:\.(\w+))?$|^(orig)(?:\.(\w+))?$`)
|
`^(\d+)x(\d+)(?:\.(\w+))?$|^(orig)(?:\.(\w+))?$`)
|
||||||
|
|
||||||
|
// percentEscapeRegex matches one percent-escape, such as "%2e"
|
||||||
|
var percentEscapeRegex = regexp.MustCompile(`%[0-9A-Fa-f]{2}`)
|
||||||
|
|
||||||
|
// queryEscapeRegex matches a percent-escape, or one character that a browser
|
||||||
|
// percent-encodes in the query of an http or https URL: one outside the
|
||||||
|
// printable ASCII characters "!" to "~", such as a space, a control character
|
||||||
|
// or a character outside ASCII, or one of `"#'<>`. A "%" that does not begin a
|
||||||
|
// percent-escape matches too.
|
||||||
|
var queryEscapeRegex = regexp.MustCompile(`%[0-9A-Fa-f]{2}|[^!-~]|["#'<>%]`)
|
||||||
|
|
||||||
|
// hostRegex matches the forms the host of an image URL may take: a host name
|
||||||
|
// or an IPv4 address (letters, digits, "-", "_" and "."), or an IPv6 address
|
||||||
|
// in brackets (hex digits, ":" and "."), either one optionally followed by ":"
|
||||||
|
// and a port, which may be empty. Its one group holds the text in brackets,
|
||||||
|
// which parseImageComponents also checks is an IPv6 address.
|
||||||
|
var hostRegex = regexp.MustCompile(
|
||||||
|
`^(?:[A-Za-z0-9_.-]+|\[([0-9A-Fa-f:.]+)\])(?::[0-9]*)?$`)
|
||||||
|
|
||||||
// ParsedURL contains the parsed components of an image proxy URL.
|
// ParsedURL contains the parsed components of an image proxy URL.
|
||||||
type ParsedURL struct {
|
type ParsedURL struct {
|
||||||
// Host is the source origin host (e.g., "cdn.example.com")
|
// Host is the source origin host (e.g., "cdn.example.com")
|
||||||
@@ -61,9 +82,8 @@ type ParsedURL struct {
|
|||||||
Format ImageFormat
|
Format ImageFormat
|
||||||
}
|
}
|
||||||
|
|
||||||
// ParseImagePath parses the path captured by chi's wildcard:
|
// ParseImagePath parses an image URL's path after /v1/image/, decoded once:
|
||||||
// <host>/<path>/<size>.<format>, or <host>/<path>/<size> for JPEG XL
|
// <host>/<path>/<size>.<format>, or <host>/<path>/<size> for JPEG XL.
|
||||||
// This is the primary entry point when using chi routing.
|
|
||||||
// Examples:
|
// Examples:
|
||||||
// - cdn.example.com/photos/cat.jpg/800x600.webp
|
// - cdn.example.com/photos/cat.jpg/800x600.webp
|
||||||
// - cdn.example.com/photos/cat.jpg/0x0.jpeg
|
// - cdn.example.com/photos/cat.jpg/0x0.jpeg
|
||||||
@@ -79,9 +99,10 @@ func ParseImagePath(path string) (*ParsedURL, error) {
|
|||||||
return parseImageComponents(path)
|
return parseImageComponents(path)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ParseImageURL parses a full URL path like /v1/image/<host>/<path>/<size>.<format>,
|
// ParseImageURL parses an image URL's path, decoded once as r.URL.Path holds
|
||||||
// or /v1/image/<host>/<path>/<size> for JPEG XL
|
// it: /v1/image/<host>/<path>/<size>.<format>, or /v1/image/<host>/<path>/<size>
|
||||||
// Use ParseImagePath instead when working with chi's wildcard capture.
|
// for JPEG XL. A source query follows the source path after a "?", which the
|
||||||
|
// URL writes as %3F.
|
||||||
func ParseImageURL(urlPath string) (*ParsedURL, error) {
|
func ParseImageURL(urlPath string) (*ParsedURL, error) {
|
||||||
// Remove the /v1/image/ prefix
|
// Remove the /v1/image/ prefix
|
||||||
const prefix = "/v1/image/"
|
const prefix = "/v1/image/"
|
||||||
@@ -100,12 +121,6 @@ func ParseImageURL(urlPath string) (*ParsedURL, error) {
|
|||||||
// parseImageComponents parses <host>/<path>/<size>.<format>, or
|
// parseImageComponents parses <host>/<path>/<size>.<format>, or
|
||||||
// <host>/<path>/<size> for JPEG XL.
|
// <host>/<path>/<size> for JPEG XL.
|
||||||
func parseImageComponents(remainder string) (*ParsedURL, error) {
|
func parseImageComponents(remainder string) (*ParsedURL, error) {
|
||||||
// Check for path traversal before any other processing
|
|
||||||
err := checkPathTraversal(remainder)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Find the last path segment, which holds "size" or "size.format"
|
// Find the last path segment, which holds "size" or "size.format"
|
||||||
lastSlash := strings.LastIndex(remainder, "/")
|
lastSlash := strings.LastIndex(remainder, "/")
|
||||||
if lastSlash == -1 {
|
if lastSlash == -1 {
|
||||||
@@ -144,10 +159,20 @@ func parseImageComponents(remainder string) (*ParsedURL, error) {
|
|||||||
return nil, ErrMissingHost
|
return nil, ErrMissingHost
|
||||||
}
|
}
|
||||||
|
|
||||||
// A URL reads what comes before "@" as a user name and password, and the
|
// Only the forms hostRegex describes are accepted. A "?", "#" or "@" in
|
||||||
// rest as the host
|
// the host would make Go's URL parser, which the allowlist check and the
|
||||||
if strings.Contains(host, "@") {
|
// fetcher use, read a different host from the one allowlisted and cached
|
||||||
return nil, fmt.Errorf("%w: %q holds @", ErrInvalidHost, host)
|
hostMatch := hostRegex.FindStringSubmatch(host)
|
||||||
|
if hostMatch == nil {
|
||||||
|
return nil, fmt.Errorf("%w: %q", ErrInvalidHost, host)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Go's URL parser refuses brackets around anything but an IPv6 address
|
||||||
|
if inBrackets := hostMatch[1]; inBrackets != "" {
|
||||||
|
addr, err := netip.ParseAddr(inBrackets)
|
||||||
|
if err != nil || !addr.Is6() {
|
||||||
|
return nil, fmt.Errorf("%w: %q", ErrInvalidHost, host)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Host names do not depend on letter case, so the allowlist, the upstream
|
// Host names do not depend on letter case, so the allowlist, the upstream
|
||||||
@@ -155,12 +180,30 @@ func parseImageComponents(remainder string) (*ParsedURL, error) {
|
|||||||
// host
|
// host
|
||||||
host = strings.ToLower(host)
|
host = strings.ToLower(host)
|
||||||
|
|
||||||
// Extract query string if present in path
|
// Go's URL parser, which the fetcher uses, reads a "#" as the start of a
|
||||||
|
// fragment and drops the rest, so each text after it would be another
|
||||||
|
// spelling, cached under its own keys, of one upstream request
|
||||||
|
if strings.Contains(path, "#") {
|
||||||
|
return nil, fmt.Errorf(
|
||||||
|
`%w: the source path or query holds a "#"`, ErrInvalidPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract query string if present in path. It is the source URL's query
|
||||||
|
// as that URL writes it. A character a URL's query cannot hold as it is,
|
||||||
|
// such as a space written %20 in the image URL, is percent-encoded here,
|
||||||
|
// so the query is signed and cached as it is sent upstream
|
||||||
if qIndex := strings.Index(path, "?"); qIndex != -1 {
|
if qIndex := strings.Index(path, "?"); qIndex != -1 {
|
||||||
query = path[qIndex+1:]
|
query = escapeQuery(path[qIndex+1:])
|
||||||
path = path[:qIndex]
|
path = path[:qIndex]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The source query names no file on the upstream, so only the host and
|
||||||
|
// the source path are checked for path traversal
|
||||||
|
err = checkPathTraversal(host + path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
// Ensure path starts with /
|
// Ensure path starts with /
|
||||||
if !strings.HasPrefix(path, "/") {
|
if !strings.HasPrefix(path, "/") {
|
||||||
path = "/" + path
|
path = "/" + path
|
||||||
@@ -186,12 +229,7 @@ func checkPathTraversal(path string) error {
|
|||||||
// Decode multiple times to catch double-encoding
|
// Decode multiple times to catch double-encoding
|
||||||
decoded := path
|
decoded := path
|
||||||
for range 3 {
|
for range 3 {
|
||||||
newDecoded, err := url.PathUnescape(decoded)
|
newDecoded := decodePercentEscapes(decoded)
|
||||||
if err != nil {
|
|
||||||
// Malformed encoding is suspicious
|
|
||||||
return ErrPathTraversal
|
|
||||||
}
|
|
||||||
|
|
||||||
if newDecoded == decoded {
|
if newDecoded == decoded {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
@@ -232,6 +270,32 @@ func checkPathTraversal(path string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// decodePercentEscapes decodes every percent-escape in s, such as %2e, and
|
||||||
|
// keeps a "%" that does not start one as it is, where url.PathUnescape would
|
||||||
|
// refuse the whole string. A source path holding %25, a literal "%", has such
|
||||||
|
// a "%" after one more decoding.
|
||||||
|
func decodePercentEscapes(s string) string {
|
||||||
|
return percentEscapeRegex.ReplaceAllStringFunc(s, func(escape string) string {
|
||||||
|
decoded, _ := url.PathUnescape(escape)
|
||||||
|
|
||||||
|
return decoded
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// escapeQuery returns a source query as a URL's query holds it: each
|
||||||
|
// character that queryEscapeRegex names, such as a space, percent-encoded,
|
||||||
|
// and the rest, the query's percent-escapes included, kept as they are. The
|
||||||
|
// result is unchanged by a second escapeQuery.
|
||||||
|
func escapeQuery(query string) string {
|
||||||
|
return queryEscapeRegex.ReplaceAllStringFunc(query, func(match string) string {
|
||||||
|
if percentEscapeRegex.MatchString(match) {
|
||||||
|
return match
|
||||||
|
}
|
||||||
|
|
||||||
|
return url.PathEscape(match)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
// parseSizeFormat parses strings like "800x600.webp", "orig.png" or "800x600"
|
// parseSizeFormat parses strings like "800x600.webp", "orig.png" or "800x600"
|
||||||
func parseSizeFormat(s string) (Size, ImageFormat, error) {
|
func parseSizeFormat(s string) (Size, ImageFormat, error) {
|
||||||
matches := sizeFormatRegex.FindStringSubmatch(s)
|
matches := sizeFormatRegex.FindStringSubmatch(s)
|
||||||
|
|||||||
@@ -407,6 +407,71 @@ func TestParseImagePath_PathTraversal(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestParseImageURL_SourceURLHoldingPercent25 verifies that a source path and
|
||||||
|
// a source query holding %25, the percent-escape of a literal "%", are
|
||||||
|
// accepted as they are, although decoding them once more leaves a "%" that
|
||||||
|
// does not start a percent-escape.
|
||||||
|
func TestParseImageURL_SourceURLHoldingPercent25(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
input string
|
||||||
|
want *ParsedURL
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "path",
|
||||||
|
input: "/v1/image/cdn.example.com/sale/50%25off.jpg/800x600.webp",
|
||||||
|
want: &ParsedURL{
|
||||||
|
Host: testHostCDN, Path: "/sale/50%25off.jpg",
|
||||||
|
Size: Size{Width: 800, Height: 600}, Format: FormatWebP,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "query",
|
||||||
|
input: "/v1/image/cdn.example.com/photos/cat.jpg?discount=50%25/800x600.webp",
|
||||||
|
want: &ParsedURL{
|
||||||
|
Host: testHostCDN, Path: testPathCat, Query: "discount=50%25",
|
||||||
|
Size: Size{Width: 800, Height: 600}, Format: FormatWebP,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
got, err := ParseImageURL(tt.input)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ParseImageURL() unexpected error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertParsedURL(t, got, tt.want)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestParseImageURL_DoubleEncodedTraversalNextToPercent verifies that a
|
||||||
|
// double-encoded ".." is still refused when the path also holds a "%" that
|
||||||
|
// does not start a percent-escape, either as written or after one more
|
||||||
|
// decoding.
|
||||||
|
func TestParseImageURL_DoubleEncodedTraversalNextToPercent(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
for _, input := range []string{
|
||||||
|
"/v1/image/cdn.example.com/photos/%252e%252e/secret/50%25off.jpg/800x600.jpeg",
|
||||||
|
"/v1/image/cdn.example.com/photos/%252e%252e/secret/100%/800x600.jpeg",
|
||||||
|
} {
|
||||||
|
_, err := ParseImageURL(input)
|
||||||
|
t.Logf("ParseImageURL(%q) error = %v", input, err)
|
||||||
|
|
||||||
|
if !errors.Is(err, ErrPathTraversal) {
|
||||||
|
t.Errorf("ParseImageURL(%q) error = %v, want ErrPathTraversal",
|
||||||
|
input, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// errorIs checks if err matches target (handles wrapped errors).
|
// errorIs checks if err matches target (handles wrapped errors).
|
||||||
func errorIs(err, target error) bool {
|
func errorIs(err, target error) bool {
|
||||||
if errors.Is(err, target) {
|
if errors.Is(err, target) {
|
||||||
|
|||||||
@@ -14,10 +14,12 @@ import (
|
|||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
|
"net/netip"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"go.uber.org/fx/fxtest"
|
"go.uber.org/fx/fxtest"
|
||||||
@@ -183,6 +185,57 @@ func startImageProxy(
|
|||||||
return s, db.DB(), stateDir
|
return s, db.DB(), stateDir
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// startImageProxyWithBlockedNetworks starts the components pixad's fx app
|
||||||
|
// builds, from a config with a fresh state directory and blockedNetworks as
|
||||||
|
// blocked_networks. Unlike startImageProxy it provides no upstream fetcher, so
|
||||||
|
// the handlers build the one pixad uses, which checks every address it
|
||||||
|
// connects to against the built-in blocked ranges and blocked_networks. It
|
||||||
|
// returns the server with its routes.
|
||||||
|
func startImageProxyWithBlockedNetworks(
|
||||||
|
t *testing.T, blockedNetworks []netip.Prefix,
|
||||||
|
) *Server {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
stateDir := t.TempDir()
|
||||||
|
cfg := &config.Config{
|
||||||
|
SigningKey: testSigningKey,
|
||||||
|
StateDir: stateDir,
|
||||||
|
DBURL: "file:" + filepath.Join(stateDir, "state.sqlite3"),
|
||||||
|
BlockedNetworks: blockedNetworks,
|
||||||
|
// A fetch that gets past the checks and finds nothing at the address
|
||||||
|
// fails within this time at the latest.
|
||||||
|
UpstreamFetchTimeout: 2 * time.Second,
|
||||||
|
// With no connection slots, the fetch would fail before connecting.
|
||||||
|
UpstreamConnections: config.DefaultUpstreamConnections,
|
||||||
|
DownstreamTimeout: config.DefaultDownstreamTimeout,
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
h *handlers.Handlers
|
||||||
|
mw *middleware.Middleware
|
||||||
|
)
|
||||||
|
|
||||||
|
app := fxtest.New(t,
|
||||||
|
fx.Supply(cfg),
|
||||||
|
fx.Provide(
|
||||||
|
globals.New,
|
||||||
|
logger.New,
|
||||||
|
database.New,
|
||||||
|
healthcheck.New,
|
||||||
|
handlers.New,
|
||||||
|
middleware.New,
|
||||||
|
),
|
||||||
|
fx.Populate(&h, &mw),
|
||||||
|
)
|
||||||
|
app.RequireStart()
|
||||||
|
t.Cleanup(app.RequireStop)
|
||||||
|
|
||||||
|
s := &Server{config: cfg, mw: mw, h: h}
|
||||||
|
s.SetupRoutes()
|
||||||
|
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
// getImage sends a GET for target to s and fails unless it answers 200.
|
// getImage sends a GET for target to s and fails unless it answers 200.
|
||||||
func getImage(t *testing.T, s *Server, target string) *httptest.ResponseRecorder {
|
func getImage(t *testing.T, s *Server, target string) *httptest.ResponseRecorder {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|||||||
@@ -0,0 +1,665 @@
|
|||||||
|
package server
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"net/netip"
|
||||||
|
"net/url"
|
||||||
|
"regexp"
|
||||||
|
"slices"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"sneak.berlin/go/pixa/internal/encurl"
|
||||||
|
"sneak.berlin/go/pixa/internal/imgcache"
|
||||||
|
"sneak.berlin/go/pixa/internal/session"
|
||||||
|
"sneak.berlin/go/pixa/internal/signature"
|
||||||
|
)
|
||||||
|
|
||||||
|
// percentSignPath is a source path holding a "%", which a URL writes %25.
|
||||||
|
const percentSignPath = "/100%25.png"
|
||||||
|
|
||||||
|
// generateFormPattern extracts the CSRF token rendered into the generator
|
||||||
|
// page's form.
|
||||||
|
var generateFormPattern = regexp.MustCompile(
|
||||||
|
`(?s)action="/generate".*?name="gorilla\.csrf\.Token" value="([^"]+)"`)
|
||||||
|
|
||||||
|
// generatedURLPattern extracts the path of the encrypted URL the generator
|
||||||
|
// page shows.
|
||||||
|
var generatedURLPattern = regexp.MustCompile(`value="[^"]*(/v1/e/[^"]+)"`)
|
||||||
|
|
||||||
|
// recordingUpstream is a test upstream that answers every request with a PNG
|
||||||
|
// and records the request target, the path and query as sent, of each request
|
||||||
|
// it receives.
|
||||||
|
type recordingUpstream struct {
|
||||||
|
*httptest.Server
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
received []string
|
||||||
|
}
|
||||||
|
|
||||||
|
// startRecordingUpstream starts a recordingUpstream, which is closed when the
|
||||||
|
// test ends.
|
||||||
|
func startRecordingUpstream(t *testing.T) *recordingUpstream {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
source := encodeTestPNG(t, 64, 48)
|
||||||
|
upstream := &recordingUpstream{}
|
||||||
|
|
||||||
|
upstream.Server = httptest.NewServer(http.HandlerFunc(
|
||||||
|
func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
upstream.mu.Lock()
|
||||||
|
upstream.received = append(upstream.received, r.RequestURI)
|
||||||
|
upstream.mu.Unlock()
|
||||||
|
|
||||||
|
w.Header().Set("Content-Type", "image/png")
|
||||||
|
_, _ = w.Write(source)
|
||||||
|
}))
|
||||||
|
t.Cleanup(upstream.Close)
|
||||||
|
|
||||||
|
return upstream
|
||||||
|
}
|
||||||
|
|
||||||
|
// requestTargets returns the request targets the upstream has received, in
|
||||||
|
// the order it received them.
|
||||||
|
func (u *recordingUpstream) requestTargets() []string {
|
||||||
|
u.mu.Lock()
|
||||||
|
defer u.mu.Unlock()
|
||||||
|
|
||||||
|
return slices.Clone(u.received)
|
||||||
|
}
|
||||||
|
|
||||||
|
// pageOrigin is the origin of the login and generator pages below, opened
|
||||||
|
// over HTTPS; httptest gives every request the host example.com.
|
||||||
|
const pageOrigin = "https://example.com"
|
||||||
|
|
||||||
|
// openRoot sends GET / to s over HTTPS, with cookie when it is not nil, and
|
||||||
|
// returns the page: the login page, or the generator page for a login session.
|
||||||
|
func openRoot(
|
||||||
|
t *testing.T, s *Server, cookie *http.Cookie,
|
||||||
|
) *httptest.ResponseRecorder {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
req := httptest.NewRequestWithContext(t.Context(), http.MethodGet,
|
||||||
|
pageOrigin+"/", nil)
|
||||||
|
if cookie != nil {
|
||||||
|
req.AddCookie(cookie)
|
||||||
|
}
|
||||||
|
|
||||||
|
page := httptest.NewRecorder()
|
||||||
|
s.ServeHTTP(page, req)
|
||||||
|
|
||||||
|
return page
|
||||||
|
}
|
||||||
|
|
||||||
|
// submitForm posts form to path on s as a browser submits it from page over
|
||||||
|
// HTTPS: with the CSRF token tokenPattern finds in page, the cookies page set,
|
||||||
|
// cookie when it is not nil, and a Referer naming the page, which pixa needs
|
||||||
|
// while debug is off, as it is in startImageProxy's config.
|
||||||
|
func submitForm(
|
||||||
|
t *testing.T, s *Server, page *httptest.ResponseRecorder,
|
||||||
|
tokenPattern *regexp.Regexp, path string, form url.Values,
|
||||||
|
cookie *http.Cookie,
|
||||||
|
) *httptest.ResponseRecorder {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
match := tokenPattern.FindStringSubmatch(page.Body.String())
|
||||||
|
if match == nil {
|
||||||
|
t.Fatalf("the page has no form for %s with a CSRF token", path)
|
||||||
|
}
|
||||||
|
|
||||||
|
form.Set("gorilla.csrf.Token", match[1])
|
||||||
|
|
||||||
|
req := httptest.NewRequestWithContext(t.Context(), http.MethodPost,
|
||||||
|
pageOrigin+path, strings.NewReader(form.Encode()))
|
||||||
|
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||||
|
req.Header.Set("Referer", pageOrigin+"/")
|
||||||
|
|
||||||
|
for _, c := range page.Result().Cookies() {
|
||||||
|
req.AddCookie(c)
|
||||||
|
}
|
||||||
|
|
||||||
|
if cookie != nil {
|
||||||
|
req.AddCookie(cookie)
|
||||||
|
}
|
||||||
|
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
s.ServeHTTP(rec, req)
|
||||||
|
|
||||||
|
return rec
|
||||||
|
}
|
||||||
|
|
||||||
|
// postGenerate logs in to s with the signing key, submits the generator page's
|
||||||
|
// form with sourceURL, and returns the response.
|
||||||
|
func postGenerate(
|
||||||
|
t *testing.T, s *Server, sourceURL string,
|
||||||
|
) *httptest.ResponseRecorder {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
login := submitForm(t, s, openRoot(t, s, nil), csrfFieldPattern, "/",
|
||||||
|
url.Values{"key": {testSigningKey}}, nil)
|
||||||
|
|
||||||
|
var sessionCookie *http.Cookie
|
||||||
|
|
||||||
|
for _, c := range login.Result().Cookies() {
|
||||||
|
if c.Name == session.CookieName {
|
||||||
|
sessionCookie = c
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if sessionCookie == nil {
|
||||||
|
t.Fatalf("logging in answered %d and set no session cookie", login.Code)
|
||||||
|
}
|
||||||
|
|
||||||
|
rec := submitForm(t, s, openRoot(t, s, sessionCookie), generateFormPattern,
|
||||||
|
"/generate", url.Values{"url": {sourceURL}}, sessionCookie)
|
||||||
|
t.Logf("POST /generate with url %s: %d", sourceURL, rec.Code)
|
||||||
|
|
||||||
|
return rec
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestImageURLFetchesItsSourcePathAndQuery requests image URLs whose source
|
||||||
|
// query holds a space, written percent-encoded, or a "'", and whose source
|
||||||
|
// path holds a "%", written percent-encoded, and checks the one request the
|
||||||
|
// upstream receives: the source path percent-encoded again, and the source
|
||||||
|
// query with the space written %20 and the "'" written %27, as a browser
|
||||||
|
// writes them in a query.
|
||||||
|
func TestImageURLFetchesItsSourcePathAndQuery(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
// pathAndQuery is the source path and query as the image URL writes
|
||||||
|
// them
|
||||||
|
pathAndQuery string
|
||||||
|
// wantRequest is the path and query the upstream must receive
|
||||||
|
wantRequest string
|
||||||
|
}{
|
||||||
|
{"space in the query", "/photo.png%3Fname=my%20cat",
|
||||||
|
"/photo.png?name=my%20cat"},
|
||||||
|
{"apostrophe in the query", "/photo.png%3Fname=it's",
|
||||||
|
"/photo.png?name=it%27s"},
|
||||||
|
{"percent sign in the path", percentSignPath, percentSignPath},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
upstream := startRecordingUpstream(t)
|
||||||
|
s, _, _ := startImageProxy(t, upstream.Server)
|
||||||
|
|
||||||
|
getImage(t, s, "/v1/image/"+upstreamHost+tt.pathAndQuery+"/orig.png")
|
||||||
|
|
||||||
|
got := upstream.requestTargets()
|
||||||
|
t.Logf("upstream received %q", got)
|
||||||
|
|
||||||
|
if !slices.Equal(got, []string{tt.wantRequest}) {
|
||||||
|
t.Errorf("upstream received %q, want [%q]", got, tt.wantRequest)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSourcePathsHoldingSpaceAndPercent20AreFetchedApart requests twice each
|
||||||
|
// of the source paths /my cat.png and /my%20cat.png, which the image URL
|
||||||
|
// writes my%20cat.png and my%2520cat.png. Each is fetched once, as a request
|
||||||
|
// of its own, and its second request is served from the cache.
|
||||||
|
func TestSourcePathsHoldingSpaceAndPercent20AreFetchedApart(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
upstream := startRecordingUpstream(t)
|
||||||
|
s, _, _ := startImageProxy(t, upstream.Server)
|
||||||
|
|
||||||
|
imageURLs := []string{
|
||||||
|
"/v1/image/" + upstreamHost + "/my%20cat.png/orig.png",
|
||||||
|
"/v1/image/" + upstreamHost + "/my%2520cat.png/orig.png",
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, imageURL := range imageURLs {
|
||||||
|
getImage(t, s, imageURL)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, imageURL := range imageURLs {
|
||||||
|
cacheStatus := getImage(t, s, imageURL).Header().Get("X-Pixa-Cache")
|
||||||
|
if cacheStatus != string(imgcache.CacheHit) {
|
||||||
|
t.Errorf("second GET %s: X-Pixa-Cache = %q, want HIT",
|
||||||
|
imageURL, cacheStatus)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
got := upstream.requestTargets()
|
||||||
|
t.Logf("upstream received %q", got)
|
||||||
|
|
||||||
|
want := []string{"/my%20cat.png", "/my%2520cat.png"}
|
||||||
|
if !slices.Equal(got, want) {
|
||||||
|
t.Errorf("upstream received %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSourceQueryWrittenTwoWaysIsOneUpstreamRequest requests the source query
|
||||||
|
// name=my%20cat written as the README says, name=my%2520cat, and with the
|
||||||
|
// space written %20, name=my%20cat. Both are one source query: the upstream
|
||||||
|
// receives one request, and the second image URL is served from the cache.
|
||||||
|
func TestSourceQueryWrittenTwoWaysIsOneUpstreamRequest(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
upstream := startRecordingUpstream(t)
|
||||||
|
s, _, _ := startImageProxy(t, upstream.Server)
|
||||||
|
|
||||||
|
getImage(t, s, "/v1/image/"+upstreamHost+"/photo.png%3Fname=my%2520cat/orig.png")
|
||||||
|
|
||||||
|
second := getImage(t, s,
|
||||||
|
"/v1/image/"+upstreamHost+"/photo.png%3Fname=my%20cat/orig.png")
|
||||||
|
if got := second.Header().Get("X-Pixa-Cache"); got != string(imgcache.CacheHit) {
|
||||||
|
t.Errorf("second X-Pixa-Cache = %q, want HIT", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
got := upstream.requestTargets()
|
||||||
|
t.Logf("upstream received %q", got)
|
||||||
|
|
||||||
|
want := []string{"/photo.png?name=my%20cat"}
|
||||||
|
if !slices.Equal(got, want) {
|
||||||
|
t.Errorf("upstream received %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestGeneratedURLFetchesTheSourceURLAsEntered makes an encrypted URL on the
|
||||||
|
// generator page for source URLs holding %20, %25 and %23 in the path or the
|
||||||
|
// query, requests it, and checks that the upstream receives the path and
|
||||||
|
// query exactly as entered.
|
||||||
|
func TestGeneratedURLFetchesTheSourceURLAsEntered(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
// The names hold no "%": each test's state directory is named after it
|
||||||
|
// and goes into the database URL, which decodes percent-escapes
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
pathAndQuery string
|
||||||
|
}{
|
||||||
|
{"space in the path", "/my%20cat.png"},
|
||||||
|
{"percent sign in the path", percentSignPath},
|
||||||
|
{"number sign in the path", "/a%23b.png"},
|
||||||
|
{"all three in the query", "/photo.png?name=my%20cat%25%23"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
upstream := startRecordingUpstream(t)
|
||||||
|
s, _, _ := startImageProxy(t, upstream.Server)
|
||||||
|
|
||||||
|
rec := postGenerate(t, s, "https://"+upstreamHost+tt.pathAndQuery)
|
||||||
|
|
||||||
|
match := generatedURLPattern.FindStringSubmatch(rec.Body.String())
|
||||||
|
if rec.Code != http.StatusOK || match == nil {
|
||||||
|
t.Fatalf("POST /generate: %d with no encrypted URL; body %s",
|
||||||
|
rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
getImage(t, s, match[1])
|
||||||
|
|
||||||
|
got := upstream.requestTargets()
|
||||||
|
t.Logf("upstream received %q", got)
|
||||||
|
|
||||||
|
if !slices.Equal(got, []string{tt.pathAndQuery}) {
|
||||||
|
t.Errorf("upstream received %q, want [%q]", got, tt.pathAndQuery)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestGeneratorMakesOneImageURLForTwoSpellingsOfASourceURL enters two
|
||||||
|
// spellings of one source URL on the generator page, requests the encrypted
|
||||||
|
// URL made for each, and checks that the upstream receives one request and
|
||||||
|
// that the second encrypted URL is served from the cache. One pair writes the
|
||||||
|
// host in upper and in lower case, which /v1/image/ lower-cases. The host is
|
||||||
|
// an IPv6 address, whose hex digits have letter case, because the fetcher
|
||||||
|
// would look a host name up in DNS. It has a port, as the fetcher refuses an
|
||||||
|
// IPv6 address written without one. The other pair is a source URL with no
|
||||||
|
// path and with the path "/", both of which fetch "/".
|
||||||
|
func TestGeneratorMakesOneImageURLForTwoSpellingsOfASourceURL(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
// sourceURLs are the two spellings, after "https://"
|
||||||
|
sourceURLs []string
|
||||||
|
// wantRequest is the path the upstream must receive
|
||||||
|
wantRequest string
|
||||||
|
}{
|
||||||
|
{"host in upper and in lower case",
|
||||||
|
[]string{"[2001:DB8::A]:8443/a.png", "[2001:db8::a]:8443/a.png"},
|
||||||
|
"/a.png"},
|
||||||
|
{"no path and the path written as a slash",
|
||||||
|
[]string{upstreamHost, upstreamHost + "/"}, "/"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
upstream := startRecordingUpstream(t)
|
||||||
|
s, _, _ := startImageProxy(t, upstream.Server)
|
||||||
|
|
||||||
|
var imageURLs []string
|
||||||
|
|
||||||
|
for _, sourceURL := range tt.sourceURLs {
|
||||||
|
rec := postGenerate(t, s, "https://"+sourceURL)
|
||||||
|
|
||||||
|
match := generatedURLPattern.FindStringSubmatch(rec.Body.String())
|
||||||
|
if rec.Code != http.StatusOK || match == nil {
|
||||||
|
t.Fatalf("POST /generate: %d with no encrypted URL; body %s",
|
||||||
|
rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
imageURLs = append(imageURLs, match[1])
|
||||||
|
}
|
||||||
|
|
||||||
|
getImage(t, s, imageURLs[0])
|
||||||
|
|
||||||
|
second := getImage(t, s, imageURLs[1])
|
||||||
|
if got := second.Header().Get("X-Pixa-Cache"); got != string(imgcache.CacheHit) {
|
||||||
|
t.Errorf("second X-Pixa-Cache = %q, want HIT", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
got := upstream.requestTargets()
|
||||||
|
t.Logf("upstream received %q", got)
|
||||||
|
|
||||||
|
if !slices.Equal(got, []string{tt.wantRequest}) {
|
||||||
|
t.Errorf("upstream received %q, want [%q]", got, tt.wantRequest)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestGeneratorRefusesSourceURLItCannotFetchAsWritten submits source URLs the
|
||||||
|
// generator page must make no encrypted URL for, and checks that it answers
|
||||||
|
// 400 and shows why. pixa keeps the source path decoded and fetches it
|
||||||
|
// percent-encoded again, so a path holding %2F would be fetched with a "/"
|
||||||
|
// there, whatever else the path holds. pixa sends no user name or password,
|
||||||
|
// and no "?" with no query after it. For these the page shows the URL pixa
|
||||||
|
// would fetch. A source path holding a ".." segment is refused, as
|
||||||
|
// /v1/image/ refuses it.
|
||||||
|
func TestGeneratorRefusesSourceURLItCannotFetchAsWritten(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
// The test config sets allow_http, so pixa would fetch over plain HTTP
|
||||||
|
const fetchedFrom = "http://" + upstreamHost
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
// sourceURL is the source URL after "https://"
|
||||||
|
sourceURL string
|
||||||
|
// wantShown is what the page must show: the URL pixa would fetch, or
|
||||||
|
// why it makes no encrypted URL
|
||||||
|
wantShown string
|
||||||
|
}{
|
||||||
|
{"encoded slash", upstreamHost + "/a%2Fb.png",
|
||||||
|
fetchedFrom + "/a/b.png"},
|
||||||
|
{"encoded slash and a space", upstreamHost + "/a%2Fb c.png",
|
||||||
|
fetchedFrom + "/a/b%20c.png"},
|
||||||
|
{"encoded slash and a bar", upstreamHost + "/a%2Fb|c.png",
|
||||||
|
fetchedFrom + "/a/b%7Cc.png"},
|
||||||
|
{"encoded slash and a letter outside ASCII", upstreamHost + "/café%2Fx.png",
|
||||||
|
fetchedFrom + "/caf%C3%A9/x.png"},
|
||||||
|
{"user name and password", "user:pass@" + upstreamHost + "/a.png",
|
||||||
|
fetchedFrom + "/a.png"},
|
||||||
|
{"question mark with no query", upstreamHost + "/a.png?",
|
||||||
|
fetchedFrom + "/a.png"},
|
||||||
|
{"two-dot segment", upstreamHost + "/a/../b.png",
|
||||||
|
"cannot hold a .. segment"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
s, _, _ := startImageProxy(t, startRecordingUpstream(t).Server)
|
||||||
|
|
||||||
|
rec := postGenerate(t, s, "https://"+tt.sourceURL)
|
||||||
|
body := rec.Body.String()
|
||||||
|
|
||||||
|
if rec.Code != http.StatusBadRequest {
|
||||||
|
t.Errorf("status = %d, want %d", rec.Code, http.StatusBadRequest)
|
||||||
|
}
|
||||||
|
|
||||||
|
if strings.Contains(body, "/v1/e/") {
|
||||||
|
t.Error("the page shows an encrypted URL")
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.Contains(body, tt.wantShown) {
|
||||||
|
t.Errorf("the page does not show %s; body %s", tt.wantShown, body)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ipv6SourcePath is the source path of the image URLs signedImageURL and
|
||||||
|
// encryptedImageURL make.
|
||||||
|
const ipv6SourcePath = "/ipv6.png"
|
||||||
|
|
||||||
|
// TestImageRoutesFetchIPv6SourceHostWithOrWithoutPort requests ipv6SourcePath
|
||||||
|
// on an IPv6 source host, written with and without a port, through a
|
||||||
|
// /v1/image/ URL with a signature and through an encrypted URL. The public
|
||||||
|
// address 2001:db8::a, a documentation address (RFC 3849) the fetcher accepts
|
||||||
|
// as public, answers 200, and the upstream receives the request for
|
||||||
|
// ipv6SourcePath. The private address fd00::1 answers 403, and the upstream
|
||||||
|
// receives no request: the test's fetcher connects every address to the test
|
||||||
|
// upstream, so only the fetcher's check of the source URL can refuse it.
|
||||||
|
func TestImageRoutesFetchIPv6SourceHostWithOrWithoutPort(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
sourceHost string
|
||||||
|
wantStatus int
|
||||||
|
// wantRequests are the request targets the upstream must receive
|
||||||
|
wantRequests []string
|
||||||
|
}{
|
||||||
|
{"public address without a port", "[2001:db8::a]",
|
||||||
|
http.StatusOK, []string{ipv6SourcePath}},
|
||||||
|
{"public address with a port", "[2001:db8::a]:8443",
|
||||||
|
http.StatusOK, []string{ipv6SourcePath}},
|
||||||
|
{"private address without a port", "[fd00::1]",
|
||||||
|
http.StatusForbidden, nil},
|
||||||
|
{"private address with a port", "[fd00::1]:8443",
|
||||||
|
http.StatusForbidden, nil},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
imageURLs := map[string]string{
|
||||||
|
"image URL with a signature": signedImageURL(t, tt.sourceHost),
|
||||||
|
"encrypted URL": encryptedImageURL(t, tt.sourceHost),
|
||||||
|
}
|
||||||
|
|
||||||
|
for route, imageURL := range imageURLs {
|
||||||
|
t.Run(tt.name+" "+route, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
upstream := startRecordingUpstream(t)
|
||||||
|
s, _, _ := startImageProxy(t, upstream.Server)
|
||||||
|
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
s.ServeHTTP(rec, httptest.NewRequestWithContext(
|
||||||
|
t.Context(), http.MethodGet, imageURL, nil))
|
||||||
|
t.Logf("GET %s: %d", imageURL, rec.Code)
|
||||||
|
|
||||||
|
if rec.Code != tt.wantStatus {
|
||||||
|
t.Errorf("status = %d, want %d; body %s",
|
||||||
|
rec.Code, tt.wantStatus, rec.Body.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
got := upstream.requestTargets()
|
||||||
|
t.Logf("upstream received %q", got)
|
||||||
|
|
||||||
|
if !slices.Equal(got, tt.wantRequests) {
|
||||||
|
t.Errorf("upstream received %q, want %q", got, tt.wantRequests)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// signedImageURL returns the /v1/image/ URL of ipv6SourcePath on sourceHost at
|
||||||
|
// its original size as PNG, with a signature made with the test signing key,
|
||||||
|
// which a host not on the allowlist needs. The URL has no q or fit, so it is
|
||||||
|
// signed with the quality 85 and the fit cover.
|
||||||
|
func signedImageURL(t *testing.T, sourceHost string) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
path, sig, exp := signature.New(testSigningKey).GenerateSignedURL(
|
||||||
|
&signature.Request{
|
||||||
|
SourceHost: sourceHost,
|
||||||
|
SourcePath: ipv6SourcePath,
|
||||||
|
Format: string(imgcache.FormatPNG),
|
||||||
|
Quality: 85,
|
||||||
|
FitMode: string(imgcache.FitCover),
|
||||||
|
}, time.Hour)
|
||||||
|
|
||||||
|
return path + "?sig=" + sig + "&exp=" + strconv.FormatInt(exp, 10)
|
||||||
|
}
|
||||||
|
|
||||||
|
// encryptedImageURL returns the encrypted URL of ipv6SourcePath on sourceHost
|
||||||
|
// at its original size as PNG, made with the test signing key.
|
||||||
|
func encryptedImageURL(t *testing.T, sourceHost string) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
generator, err := encurl.NewGenerator(testSigningKey)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("encurl.NewGenerator: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
token, err := generator.Generate(&encurl.Payload{
|
||||||
|
SourceHost: sourceHost, SourcePath: ipv6SourcePath,
|
||||||
|
Format: imgcache.FormatPNG,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("encurl Generate: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return "/v1/e/" + token + "/img.png"
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestImageRoutesRefuseIPv6SourceHostHoldingInternalIPv4Address requests
|
||||||
|
// ipv6SourcePath on IPv6 source hosts that hold a loopback or private IPv4
|
||||||
|
// address in their last four bytes, in the IPv4-compatible form (::/96) or the
|
||||||
|
// NAT64 form (64:ff9b::/96), through a /v1/image/ URL with a signature and
|
||||||
|
// through an encrypted URL. Each answers 403, and the upstream receives no
|
||||||
|
// request: the test's fetcher connects every address to the test upstream, so
|
||||||
|
// only the fetcher's check of the source URL can refuse it.
|
||||||
|
func TestImageRoutesRefuseIPv6SourceHostHoldingInternalIPv4Address(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
upstream := startRecordingUpstream(t)
|
||||||
|
s, _, _ := startImageProxy(t, upstream.Server)
|
||||||
|
|
||||||
|
sourceHosts := []string{
|
||||||
|
"[::127.0.0.1]", // IPv4-compatible form of a loopback address
|
||||||
|
"[::10.0.0.1]", // IPv4-compatible form of a private address
|
||||||
|
"[64:ff9b::7f00:1]", // NAT64 form of 127.0.0.1
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, sourceHost := range sourceHosts {
|
||||||
|
imageURLs := []string{
|
||||||
|
signedImageURL(t, sourceHost),
|
||||||
|
encryptedImageURL(t, sourceHost),
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, imageURL := range imageURLs {
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
s.ServeHTTP(rec, httptest.NewRequestWithContext(
|
||||||
|
t.Context(), http.MethodGet, imageURL, nil))
|
||||||
|
t.Logf("GET %s (source host %s): %d", imageURL, sourceHost, rec.Code)
|
||||||
|
|
||||||
|
if rec.Code != http.StatusForbidden {
|
||||||
|
t.Errorf("source host %s: status = %d, want %d; body %s",
|
||||||
|
sourceHost, rec.Code, http.StatusForbidden, rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := upstream.requestTargets(); len(got) != 0 {
|
||||||
|
t.Errorf("upstream received %q, want no request", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestImageRoutesCheckIPv4AddressInIPv6SourceHost requests ipv6SourcePath on
|
||||||
|
// source hosts that hold an IPv4 address, through a /v1/image/ URL with a
|
||||||
|
// signature and through an encrypted URL, with the fetcher pixad uses.
|
||||||
|
// [64:ff9b:1::a00:1], in the local-use NAT64 range (64:ff9b:1::/48, RFC 8215),
|
||||||
|
// answers 403. [::203.0.113.5] holds the public address 203.0.113.5 in the
|
||||||
|
// IPv4-compatible form (::/96) and answers 403 when blocked_networks lists
|
||||||
|
// 203.0.113.0/24 or ::/96. [::ffff:203.0.113.5] holds it in the IPv4-mapped
|
||||||
|
// form (::ffff:0:0/96) and answers 403 when blocked_networks lists
|
||||||
|
// 203.0.113.0/24. Go keeps the plain address 203.0.113.5 in the mapped form
|
||||||
|
// too, and an IPv6 range must not refuse it: with ::/0 listed it is not
|
||||||
|
// refused. Neither is [::203.0.113.5] with blocked_networks empty. Where a
|
||||||
|
// host is not refused, the fetcher tries to connect to it, finds nothing
|
||||||
|
// there, and the request fails with another status.
|
||||||
|
func TestImageRoutesCheckIPv4AddressInIPv6SourceHost(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const (
|
||||||
|
ipv4CompatibleHost = "[::203.0.113.5]"
|
||||||
|
ipv4MappedHost = "[::ffff:203.0.113.5]"
|
||||||
|
)
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
sourceHost string
|
||||||
|
blockedNetworks []netip.Prefix
|
||||||
|
wantForbidden bool
|
||||||
|
}{
|
||||||
|
{"local-use NAT64 form of 10.0.0.1", "[64:ff9b:1::a00:1]", nil, true},
|
||||||
|
{"IPv4-compatible form of an address in blocked_networks",
|
||||||
|
ipv4CompatibleHost,
|
||||||
|
[]netip.Prefix{netip.MustParsePrefix("203.0.113.0/24")}, true},
|
||||||
|
{"IPv4-compatible form with ::/96 in blocked_networks",
|
||||||
|
ipv4CompatibleHost,
|
||||||
|
[]netip.Prefix{netip.MustParsePrefix("::/96")}, true},
|
||||||
|
{"IPv4-mapped form of an address in blocked_networks",
|
||||||
|
ipv4MappedHost,
|
||||||
|
[]netip.Prefix{netip.MustParsePrefix("203.0.113.0/24")}, true},
|
||||||
|
{"plain IPv4 address with ::/0 in blocked_networks",
|
||||||
|
"203.0.113.5",
|
||||||
|
[]netip.Prefix{netip.MustParsePrefix("::/0")}, false},
|
||||||
|
{"IPv4-compatible form of an address with blocked_networks empty",
|
||||||
|
ipv4CompatibleHost, nil, false},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
imageURLs := map[string]string{
|
||||||
|
"image URL with a signature": signedImageURL(t, tt.sourceHost),
|
||||||
|
"encrypted URL": encryptedImageURL(t, tt.sourceHost),
|
||||||
|
}
|
||||||
|
|
||||||
|
for route, imageURL := range imageURLs {
|
||||||
|
t.Run(tt.name+" "+route, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
s := startImageProxyWithBlockedNetworks(t, tt.blockedNetworks)
|
||||||
|
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
s.ServeHTTP(rec, httptest.NewRequestWithContext(
|
||||||
|
t.Context(), http.MethodGet, imageURL, nil))
|
||||||
|
t.Logf("GET %s: %d", imageURL, rec.Code)
|
||||||
|
|
||||||
|
forbidden := rec.Code == http.StatusForbidden
|
||||||
|
if tt.wantForbidden && !forbidden {
|
||||||
|
t.Errorf("status = %d, want %d; body %s",
|
||||||
|
rec.Code, http.StatusForbidden, rec.Body.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
if !tt.wantForbidden && forbidden {
|
||||||
|
t.Errorf("status = %d, want any other status; body %s",
|
||||||
|
rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -126,31 +126,20 @@ func (s *Signer) GenerateSignedURL(
|
|||||||
sizeStr = fmt.Sprintf("%dx%d", req.Width, req.Height)
|
sizeStr = fmt.Sprintf("%dx%d", req.Width, req.Height)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Build the path.
|
// Build the path. A source query follows the source path after a "?".
|
||||||
// When a source query is present, it is embedded as a path segment
|
// The image route reads the path decoded once, so the whole path is
|
||||||
// (e.g. /host/path?query/size.fmt) so that the URL parser can extract
|
// written in Go's own percent-encoding of a URL path, which writes "?"
|
||||||
// it from the last-slash split. The "?" inside a path segment is
|
// as %3F and "%" as %25, and leaves "/", ";", ",", "=" and "&" as they
|
||||||
// percent-encoded by clients but chi delivers it decoded, which is
|
// are.
|
||||||
// exactly what the URL parser expects.
|
sourcePathAndQuery := req.SourcePath
|
||||||
var path string
|
|
||||||
if req.SourceQuery != "" {
|
if req.SourceQuery != "" {
|
||||||
path = fmt.Sprintf("/v1/image/%s%s%%3F%s/%s.%s",
|
sourcePathAndQuery += "?" + req.SourceQuery
|
||||||
req.SourceHost,
|
|
||||||
req.SourcePath,
|
|
||||||
url.PathEscape(req.SourceQuery),
|
|
||||||
sizeStr,
|
|
||||||
req.Format,
|
|
||||||
)
|
|
||||||
} else {
|
|
||||||
path = fmt.Sprintf("/v1/image/%s%s/%s.%s",
|
|
||||||
req.SourceHost,
|
|
||||||
req.SourcePath,
|
|
||||||
sizeStr,
|
|
||||||
req.Format,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return path, sig, exp
|
unescapedPath := fmt.Sprintf("/v1/image/%s%s/%s.%s",
|
||||||
|
req.SourceHost, sourcePathAndQuery, sizeStr, req.Format)
|
||||||
|
|
||||||
|
return (&url.URL{Path: unescapedPath}).EscapedPath(), sig, exp
|
||||||
}
|
}
|
||||||
|
|
||||||
// buildSignatureData creates the string to be signed.
|
// buildSignatureData creates the string to be signed.
|
||||||
|
|||||||
@@ -475,6 +475,44 @@ func TestGenerateSignedURL_WithoutQueryString(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestGenerateSignedURL_ReadmeQueryExample checks the README's example of a
|
||||||
|
// source URL with a query, the image at
|
||||||
|
// https://cdn.example.com/img?url=https://origin.example.com/my%20cat.jpg: its
|
||||||
|
// signature with the README's key, and the path GenerateSignedURL writes for
|
||||||
|
// it, with the "?" written as %3F and the "%" as %25.
|
||||||
|
func TestGenerateSignedURL_ReadmeQueryExample(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
signer := signature.New("example-signing-key-for-documentation")
|
||||||
|
|
||||||
|
req := signature.Request{
|
||||||
|
SourceHost: testHost,
|
||||||
|
SourcePath: "/img",
|
||||||
|
SourceQuery: "url=https://origin.example.com/my%20cat.jpg",
|
||||||
|
Width: 800,
|
||||||
|
Height: 600,
|
||||||
|
Format: testFormatWebP,
|
||||||
|
Quality: 85,
|
||||||
|
FitMode: testFitCover,
|
||||||
|
Expires: time.Unix(1704067200, 0),
|
||||||
|
}
|
||||||
|
|
||||||
|
const wantSig = "37aFa9GSVCZM1kRY9SzMOk2fOg_M7Wt5gKNR4xM-FYA="
|
||||||
|
|
||||||
|
gotSig := signer.Sign(&req)
|
||||||
|
if gotSig != wantSig {
|
||||||
|
t.Errorf("Sign() = %q, want %q", gotSig, wantSig)
|
||||||
|
}
|
||||||
|
|
||||||
|
const wantPath = "/v1/image/cdn.example.com" +
|
||||||
|
"/img%3Furl=https://origin.example.com/my%2520cat.jpg/800x600.webp"
|
||||||
|
|
||||||
|
gotPath, _, _ := signer.GenerateSignedURL(&req, time.Hour)
|
||||||
|
if gotPath != wantPath {
|
||||||
|
t.Errorf("GenerateSignedURL() path = %q, want %q", gotPath, wantPath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestParseParams(t *testing.T) {
|
func TestParseParams(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
+2
-3
@@ -19,9 +19,8 @@
|
|||||||
# CGO image libraries (pkg-config, vips
|
# CGO image libraries (pkg-config, vips
|
||||||
# with its JPEG XL support, libheif) for
|
# with its JPEG XL support, libheif) for
|
||||||
# the govips bindings instead of Node: to
|
# the govips bindings instead of Node: to
|
||||||
# compile pixa, in the Dockerfile's test
|
# compile pixa outside Docker, as
|
||||||
# phase and build stage, which format
|
# `make build` does
|
||||||
# nothing
|
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
|
|||||||
+5
-4
@@ -2,11 +2,12 @@
|
|||||||
# script/cibuild: run the CI build. It bootstraps first: a CI runner
|
# script/cibuild: run the CI build. It bootstraps first: a CI runner
|
||||||
# checks out and runs this and nothing else, and script/fmt-check runs
|
# checks out and runs this and nothing else, and script/fmt-check runs
|
||||||
# the formatter on the host, which a pristine checkout cannot do.
|
# the formatter on the host, which a pristine checkout cannot do.
|
||||||
# A new CACHEBUST value reruns every RUN step, since a cached gate phase
|
# A new CACHEBUST value reruns every RUN step after the ARG CACHEBUST
|
||||||
# is a check that did not run; unlike --no-cache, it keeps Go's build cache.
|
# line, since a cached gate phase is a check that did not run; unlike
|
||||||
|
# --no-cache, it keeps the layers above that line and the cache mounts.
|
||||||
# This relies on every stage of the Dockerfile declaring ARG CACHEBUST
|
# This relies on every stage of the Dockerfile declaring ARG CACHEBUST
|
||||||
# first; a stage without that line is served from cache on an unchanged
|
# before every step that lints, tests or compiles; a stage without that
|
||||||
# tree, with no warning.
|
# line is served from cache on an unchanged tree, with no warning.
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||||
|
|||||||
+5
-4
@@ -1,11 +1,12 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
# script/docker: build the Docker image tagged with the project name.
|
# script/docker: build the Docker image tagged with the project name.
|
||||||
# Identical in all repos; the tag comes from script/projectname.
|
# Identical in all repos; the tag comes from script/projectname.
|
||||||
# A new CACHEBUST value reruns every RUN step, since a cached gate phase
|
# A new CACHEBUST value reruns every RUN step after the ARG CACHEBUST
|
||||||
# is a check that did not run; unlike --no-cache, it keeps Go's build cache.
|
# line, since a cached gate phase is a check that did not run; unlike
|
||||||
|
# --no-cache, it keeps the layers above that line and the cache mounts.
|
||||||
# This relies on every stage of the Dockerfile declaring ARG CACHEBUST
|
# This relies on every stage of the Dockerfile declaring ARG CACHEBUST
|
||||||
# first; a stage without that line is served from cache on an unchanged
|
# before every step that lints, tests or compiles; a stage without that
|
||||||
# tree, with no warning.
|
# line is served from cache on an unchanged tree, with no warning.
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||||
|
|||||||
+7
-5
@@ -5,11 +5,13 @@
|
|||||||
# make its answer untrustworthy.
|
# make its answer untrustworthy.
|
||||||
#
|
#
|
||||||
# The phase is not the last stage in the file, so it is built only when
|
# The phase is not the last stage in the file, so it is built only when
|
||||||
# --target names it. A new CACHEBUST value reruns every RUN step, since
|
# --target names it. A new CACHEBUST value reruns every RUN step after
|
||||||
# a cached lint layer is a lint that did not run; unlike --no-cache, it
|
# the ARG CACHEBUST line, since a cached lint layer is a lint that did
|
||||||
# keeps Go's build cache. This relies on every stage of the Dockerfile
|
# not run; unlike --no-cache, it keeps the layers above that line and
|
||||||
# declaring ARG CACHEBUST first; a stage without that line is served
|
# the cache mounts. This relies on every stage of the Dockerfile
|
||||||
# from cache on an unchanged tree, with no warning.
|
# declaring ARG CACHEBUST before every step that lints, tests or
|
||||||
|
# compiles; a stage without that line is served from cache on an
|
||||||
|
# unchanged tree, with no warning.
|
||||||
# --output type=cacheonly writes no image, since nothing uses one.
|
# --output type=cacheonly writes no image, since nothing uses one.
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
|
|||||||
+6
-5
@@ -3,11 +3,12 @@
|
|||||||
# and this builds that phase alone, on the same terms as script/lint:
|
# and this builds that phase alone, on the same terms as script/lint:
|
||||||
# --target because a phase that is not the last stage is built only when
|
# --target because a phase that is not the last stage is built only when
|
||||||
# named, and a new CACHEBUST value because a cached test layer is a test
|
# named, and a new CACHEBUST value because a cached test layer is a test
|
||||||
# that did not run; unlike --no-cache, it keeps Go's build cache. This
|
# that did not run; unlike --no-cache, it keeps the layers above the ARG
|
||||||
# relies on every stage of the Dockerfile declaring ARG CACHEBUST first;
|
# CACHEBUST line and the cache mounts. This relies on every stage of the
|
||||||
# a stage without that line is served from cache on an unchanged tree,
|
# Dockerfile declaring ARG CACHEBUST before every step that lints, tests
|
||||||
# with no warning. --output type=cacheonly writes no image, since
|
# or compiles; a stage without that line is served from cache on an
|
||||||
# nothing uses one.
|
# unchanged tree, with no warning. --output type=cacheonly writes no
|
||||||
|
# image, since nothing uses one.
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||||
|
|||||||
Reference in New Issue
Block a user