Compare commits
15
Commits
d98d4dc119
...
next
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
233a9c05ad | ||
|
|
842372250f | ||
|
|
58d601ea48 | ||
|
|
48f21d4ecf | ||
|
|
f8c437b83f | ||
|
|
04093f53ad | ||
|
|
be6c715b36 | ||
|
|
604b51eea6 | ||
|
|
ba5a716223 | ||
|
|
c7173c47d8 | ||
|
|
363774c058 | ||
|
|
1616e91a6a | ||
|
|
6830bdc5de | ||
|
|
5b17d1f555 | ||
|
|
3a274aaa44 |
@@ -0,0 +1,4 @@
|
|||||||
|
# Every PR adds an entry at the top of TODO.md's Completed Steps; union keeps
|
||||||
|
# both sides instead of conflicting. Git never reports a conflict here: read
|
||||||
|
# the merged entries after every merge or rebase.
|
||||||
|
TODO.md merge=union
|
||||||
+66
-2
@@ -10,14 +10,20 @@ run:
|
|||||||
|
|
||||||
linters:
|
linters:
|
||||||
default: all
|
default: all
|
||||||
|
enable:
|
||||||
|
# Successor to the deprecated gomodguard. Named explicitly, rather than
|
||||||
|
# left to `default: all`, because it carries the module policy below.
|
||||||
|
- gomodguard_v2
|
||||||
disable:
|
disable:
|
||||||
# Genuinely incompatible with project patterns
|
# Genuinely incompatible with project patterns
|
||||||
- exhaustruct # Requires all struct fields
|
- exhaustruct # Requires all struct fields
|
||||||
- depguard # Dependency allow/block lists
|
|
||||||
- godot # Requires comments to end with periods
|
- godot # Requires comments to end with periods
|
||||||
- wsl # Deprecated, replaced by wsl_v5
|
|
||||||
- wrapcheck # Too verbose for internal packages
|
- wrapcheck # Too verbose for internal packages
|
||||||
- varnamelen # Short names like db, id are idiomatic Go
|
- varnamelen # Short names like db, id are idiomatic Go
|
||||||
|
# Deprecated: the warning is attached to the old name, so it is
|
||||||
|
# silenced by disabling that name, not by enabling the successor.
|
||||||
|
- wsl # Deprecated, replaced by wsl_v5
|
||||||
|
- gomodguard # Deprecated, replaced by gomodguard_v2
|
||||||
settings:
|
settings:
|
||||||
lll:
|
lll:
|
||||||
line-length: 88
|
line-length: 88
|
||||||
@@ -28,6 +34,64 @@ linters:
|
|||||||
max-complexity: 15
|
max-complexity: 15
|
||||||
dupl:
|
dupl:
|
||||||
threshold: 100
|
threshold: 100
|
||||||
|
depguard:
|
||||||
|
# Test-support code must not be compiled into the shipped binary. A
|
||||||
|
# test-support package exists to hand a test privileges the program
|
||||||
|
# itself must never have, so a file that is not a test must not import
|
||||||
|
# one. Test files, and the files inside a package whose directory name
|
||||||
|
# ends in `test`, are where that code belongs, and are exempt.
|
||||||
|
#
|
||||||
|
# The deny list below is the one part of this file a repository is
|
||||||
|
# expected to extend, and the only part it may. depguard matches an
|
||||||
|
# import path against a list of prefixes, so it cannot be told "any path
|
||||||
|
# whose last segment ends in test"; a repository's own test-support
|
||||||
|
# packages have to be named here one at a time, by full import path,
|
||||||
|
# under a module path that differs from repository to repository. Add
|
||||||
|
# them; change nothing else.
|
||||||
|
rules:
|
||||||
|
test-support:
|
||||||
|
list-mode: lax
|
||||||
|
files:
|
||||||
|
- "$all"
|
||||||
|
- "!$test"
|
||||||
|
- "!**/*test/**"
|
||||||
|
deny:
|
||||||
|
- pkg: net/http/httptest
|
||||||
|
desc: >-
|
||||||
|
Test-support code belongs in test files and in packages whose
|
||||||
|
directory name ends in test, not in the shipped binary.
|
||||||
|
# Only decisions already recorded in the Go package defaults are
|
||||||
|
# listed here. Every entry matches the module path exactly.
|
||||||
|
gomodguard_v2:
|
||||||
|
blocked:
|
||||||
|
- module: github.com/rs/zerolog
|
||||||
|
recommendations:
|
||||||
|
- log/slog
|
||||||
|
reason: "Structured logging is stdlib log/slog."
|
||||||
|
# One entry per pre-fork module path, because the later releases
|
||||||
|
# are separate paths. A prefix match would be shorter but would
|
||||||
|
# also reach github.com/go-redis/redismock, the test double for
|
||||||
|
# the successor these entries recommend.
|
||||||
|
- module: github.com/go-redis/redis
|
||||||
|
recommendations:
|
||||||
|
- github.com/redis/go-redis/v9
|
||||||
|
reason: "Pre-fork module; use the maintained go-redis v9."
|
||||||
|
- module: github.com/go-redis/redis/v7
|
||||||
|
recommendations:
|
||||||
|
- github.com/redis/go-redis/v9
|
||||||
|
reason: "Pre-fork module; use the maintained go-redis v9."
|
||||||
|
- module: github.com/go-redis/redis/v8
|
||||||
|
recommendations:
|
||||||
|
- github.com/redis/go-redis/v9
|
||||||
|
reason: "Pre-fork module; use the maintained go-redis v9."
|
||||||
|
- module: github.com/sergi/go-diff
|
||||||
|
recommendations:
|
||||||
|
- github.com/aymanbagabas/go-udiff
|
||||||
|
reason: "No unified diff output; use go-udiff."
|
||||||
|
- module: github.com/hexops/gotextdiff
|
||||||
|
recommendations:
|
||||||
|
- github.com/aymanbagabas/go-udiff
|
||||||
|
reason: "Unmaintained fork; use go-udiff."
|
||||||
|
|
||||||
issues:
|
issues:
|
||||||
max-issues-per-linter: 0
|
max-issues-per-linter: 0
|
||||||
|
|||||||
@@ -34,6 +34,65 @@ else has a built-in default. A config file mounted at `/etc/pixa/config.yml`
|
|||||||
is optional: it is read when present, and an environment variable wins over
|
is optional: it is read when present, and an environment variable wins over
|
||||||
the same setting in it.
|
the same setting in it.
|
||||||
|
|
||||||
|
## Deployment
|
||||||
|
|
||||||
|
pixa listens on plain HTTP and runs behind a reverse proxy that terminates TLS.
|
||||||
|
[`configs/Caddyfile`](configs/Caddyfile) is an example for Caddy, chosen because
|
||||||
|
it is the smallest correct one: Caddy gets the TLS certificate itself and does
|
||||||
|
everything in this list without further settings. The reverse proxy must:
|
||||||
|
|
||||||
|
- terminate TLS, as the login and generator pages work only over HTTPS (see
|
||||||
|
Routes);
|
||||||
|
- pass the `Host`, `Origin` and `Referer` headers on unchanged, as pixa refuses
|
||||||
|
a form from those pages unless `Origin` or `Referer` names the host in `Host`,
|
||||||
|
and builds encrypted URLs from `Host`;
|
||||||
|
- set `X-Forwarded-For` to the client's address, with `trusted_proxies` set to
|
||||||
|
the address pixa sees the proxy's requests come from, so the login limit
|
||||||
|
counts each client by its own address (see `trusted_proxies` under
|
||||||
|
Configuration);
|
||||||
|
- wait for pixa's answer for at least `downstream_timeout` (default `60s`), the
|
||||||
|
longest pixa takes to fetch, convert and send an image.
|
||||||
|
|
||||||
|
It may also refuse `/metrics`, as the example does, so that only a scraper that
|
||||||
|
reaches pixa directly can read it; pixa itself asks for the metrics username and
|
||||||
|
password there.
|
||||||
|
|
||||||
|
pixa does the rest itself: it checks signatures and encrypted URLs, applies the
|
||||||
|
allowlist, refuses upstream hosts with private or local addresses, limits login
|
||||||
|
attempts, upstream response size and image dimensions, and sends the security
|
||||||
|
headers, `Strict-Transport-Security` included, with every response.
|
||||||
|
|
||||||
|
The state directory (`state_dir`, `/var/lib/pixa` in the container) holds the
|
||||||
|
database and the disk cache:
|
||||||
|
|
||||||
|
- It needs a persistent volume: without one, every restart starts with an empty
|
||||||
|
cache. In the container, the startup script gives the directory to the user
|
||||||
|
pixa runs as (uid 65532) and sets its mode to `750`; outside it, that user
|
||||||
|
must be able to write the directory.
|
||||||
|
- `cache_max_bytes` limits the source and transformed images together. The
|
||||||
|
database, the metadata files, the `.meta` file beside each transformed image
|
||||||
|
and files still being written come on top, and eviction runs in the
|
||||||
|
background, so the cache can pass the limit for a while: leave room on the
|
||||||
|
volume beyond it.
|
||||||
|
- Set `cache_max_bytes` for a lasting deployment. Its default is 75% of the
|
||||||
|
space free when pixa starts, which the cache's own files reduce, so a fuller
|
||||||
|
cache gives a smaller limit after a restart.
|
||||||
|
|
||||||
|
A load balancer's health check can request `/.well-known/healthcheck.json`,
|
||||||
|
which answers 200 whenever pixa is running, in maintenance mode too (see
|
||||||
|
`maintenance_mode`).
|
||||||
|
|
||||||
|
On SIGTERM or SIGINT pixa stops accepting connections, gives the requests in
|
||||||
|
progress and the images being processed 5 seconds to finish, and exits: with 0,
|
||||||
|
or with 1 when images were still being processed after those 5 seconds or
|
||||||
|
another part of pixa failed to stop. A request not finished by then is cut off.
|
||||||
|
`docker stop` waits 10 seconds before it kills the container.
|
||||||
|
|
||||||
|
Outside Docker, pixa needs libvips (the image has 8.15) and libheif to run, as
|
||||||
|
it uses libvips through CGO; building it also needs their development files,
|
||||||
|
`pkg-config` and a C compiler. `script/bootstrap` installs all of these with
|
||||||
|
nix, apt, brew or apk.
|
||||||
|
|
||||||
## Running under upaas
|
## Running under upaas
|
||||||
|
|
||||||
What the [upaas](https://git.eeqj.de/sneak/upaas) app for pixa needs:
|
What the [upaas](https://git.eeqj.de/sneak/upaas) app for pixa needs:
|
||||||
@@ -71,13 +130,17 @@ prevent abuse, and allowlisted source hosts for open access.
|
|||||||
### Storage
|
### Storage
|
||||||
|
|
||||||
- **Source content**:
|
- **Source content**:
|
||||||
`<statedir>/cache/src-content/<ab>/<cd>/<sha256 of source content>`
|
`<state_dir>/cache/sources/<ab>/<cd>/<sha256 of source content>`
|
||||||
- **Source metadata**:
|
- **Source metadata**:
|
||||||
`<statedir>/cache/src-metadata/<hostname>/<sha256 of path>.json`
|
`<state_dir>/cache/metadata/<hostname>/<sha256 of path and query>.json`
|
||||||
(fetch time, original headers, request, content hash)
|
(host, path and query, content hash, upstream status and headers, fetch time)
|
||||||
- **Database**: `<statedir>/state.sqlite3` (SQLite)
|
- **Database**: `<state_dir>/state.sqlite3` (SQLite)
|
||||||
- **Output documents**:
|
- **Transformed images**:
|
||||||
`<statedir>/cache/dst-content/<ab>/<cd>/<sha256 of output content>`
|
`<state_dir>/cache/variants/<ab>/<cd>/<sha256 of host, path, query, size, format, quality and fit>`,
|
||||||
|
each with a `.meta` file beside it holding its content type
|
||||||
|
|
||||||
|
`<ab>` and `<cd>` are the first and second pairs of characters of the file's
|
||||||
|
name.
|
||||||
|
|
||||||
Multiple source paths may reference the same content blob; the
|
Multiple source paths may reference the same content blob; the
|
||||||
database tracks references rather than using filesystem refcounting.
|
database tracks references rather than using filesystem refcounting.
|
||||||
@@ -88,17 +151,97 @@ the metadata file stored beside it.
|
|||||||
|
|
||||||
### Routes
|
### Routes
|
||||||
|
|
||||||
|
pixa answers these routes; any other path answers 404. A path in this list asked
|
||||||
|
with a method the list does not give answers 405, except `/static/<file>`, which
|
||||||
|
answers any method as it answers `GET`. A browser's CORS preflight request
|
||||||
|
(`OPTIONS` with `Origin` and `Access-Control-Request-Method` headers) to any
|
||||||
|
path under `/v1/` answers 200, in maintenance mode too.
|
||||||
|
|
||||||
|
- `GET /` — the login page, or the URL generator page with a login session
|
||||||
|
(see Encrypted URLs). Needs: nothing. Answers: 200.
|
||||||
|
- `POST /` — log in with the signing key typed into the login page. Needs: the
|
||||||
|
login page's form (below). Answers: 303 to `/` with a login session cookie
|
||||||
|
that lasts 30 days for the right key; 200 with the login page and an error for
|
||||||
|
a wrong key; 429 over the login limit (below).
|
||||||
|
- `POST /generate` — make an encrypted URL from the generator page's form.
|
||||||
|
Needs: a login session and the generator page's form (below); without a login
|
||||||
|
session it answers 303 to `/`. Answers: 200 with the page showing the URL; 400
|
||||||
|
with the page naming a field that is not valid; 500 when the URL cannot be
|
||||||
|
made.
|
||||||
|
- `GET /logout` — end the login session. Needs: nothing. Answers: 303 to `/`.
|
||||||
|
- `GET` or `HEAD` `/v1/image/<host>/<path>/<size>.<format>` — an image, fetched,
|
||||||
|
resized and converted (below). Needs: a signature, unless the host is
|
||||||
|
allowlisted (see Source Hosts). Answers: 200; 304 when `If-None-Match` matches
|
||||||
|
the image's `ETag`; 400 for a URL or parameter that is not valid; 401 for a
|
||||||
|
missing or wrong signature, a missing `exp` or an `exp` in the past; 403 when
|
||||||
|
the upstream host, or a host it redirects to, is `localhost`, ends in
|
||||||
|
`.localhost` or `.local`, or has an address in a blocked network (see
|
||||||
|
`blocked_networks`); 502 when the upstream answered with an error status, and
|
||||||
|
for 5 minutes after that for the same source URL; 503 when pixa is busy or in
|
||||||
|
maintenance mode; 500 for any other failure.
|
||||||
|
- `GET` or `HEAD` `/v1/e/<token>/<name>` — an image through an encrypted URL
|
||||||
|
(see Encrypted URLs). Needs: nothing but the URL. Answers: 200; 304 when
|
||||||
|
`If-None-Match` matches the image's `ETag`; 400 for a token that does not
|
||||||
|
decrypt, or that asks for a size or fit that is not valid; 410 once it has
|
||||||
|
expired; 504 when the upstream has not sent its response headers within
|
||||||
|
`upstream_fetch_timeout`, but 500 when that time runs out while the image
|
||||||
|
itself is still arriving; 403, 502, 503 and 500 as for `/v1/image/`.
|
||||||
|
- `GET /robots.txt` — asks every crawler to stay away (`Disallow: /`). Needs:
|
||||||
|
nothing. Answers: 200.
|
||||||
|
- `GET /.well-known/healthcheck.json` — JSON with `status` (`ok`), `now`,
|
||||||
|
`uptime_seconds`, `uptime_human`, `version`, `appname` and
|
||||||
|
`maintenance_mode`. Needs: nothing. Answers: 200, always.
|
||||||
|
- `GET /static/<file>` — the stylesheet and script the login and generator
|
||||||
|
pages load. Needs: nothing. Answers: 200, or 404 for a file that does not
|
||||||
|
exist.
|
||||||
|
- `GET /metrics` — Prometheus metrics (see Architecture). Needs: HTTP basic
|
||||||
|
authentication with `metrics.username` and `metrics.password`. Answers: 200;
|
||||||
|
401 without them; 404 when they are not set, as the route then does not exist.
|
||||||
|
|
||||||
|
Every response carries an `X-Request-ID` header holding the request's ID, which
|
||||||
|
a client can quote when reporting a problem: the request's own `X-Request-ID`,
|
||||||
|
as a reverse proxy in front of pixa may send, when it is at most 64 letters,
|
||||||
|
digits, `-`, `_` or `.`; otherwise a random one pixa makes for the request,
|
||||||
|
which tells nothing about the machine or the other requests. pixa's log line for
|
||||||
|
the request carries the same ID as `request_id`, and so do the lines it logs
|
||||||
|
when it fetches, converts and serves an image; the fetch sends it to the
|
||||||
|
upstream host as `X-Request-ID`.
|
||||||
|
|
||||||
|
Both `POST` routes accept only a form that pixa's own page served: the page puts
|
||||||
|
a token in the form and sets a cookie to match, and a request without both is
|
||||||
|
refused with 403, so another site cannot submit the form from a visitor's
|
||||||
|
browser. The login and generator pages are meant to be opened over HTTPS: while
|
||||||
|
`debug` is off, a form sent from a page opened over plain HTTP is refused with
|
||||||
|
403, and while it is on, so is one sent from a page opened over HTTPS. Plain
|
||||||
|
HTTP is for development on the browser's own machine: the login session cookie
|
||||||
|
is always marked `Secure`, and over plain HTTP a browser keeps such a cookie
|
||||||
|
only for its own machine (`localhost`), if at all. A form is also refused with
|
||||||
|
403 when the page's host is not the `Host` header pixa receives, so a reverse
|
||||||
|
proxy in front of pixa must pass that header on unchanged. A form body over
|
||||||
|
1 MiB is refused with 413. The image routes answer the errors listed for them
|
||||||
|
with JSON holding `error`, `status` and `timestamp`.
|
||||||
|
|
||||||
|
An image URL has this form:
|
||||||
|
|
||||||
```
|
```
|
||||||
/v1/image/<host>/<path>/<size>.<format>?sig=<signature>&exp=<expiration>
|
/v1/image/<host>/<path>/<size>.<format>?sig=<signature>&exp=<expiration>&q=<quality>&fit=<fit>
|
||||||
```
|
```
|
||||||
|
|
||||||
Images are only fetched from origins using TLS with valid certificates.
|
Images are only fetched from origins using TLS with valid certificates, unless
|
||||||
|
`allow_http` is set: then pixa fetches every image over plain HTTP, which is for
|
||||||
|
testing only.
|
||||||
|
|
||||||
A request whose query string cannot be decoded, or gives any parameter more
|
A request whose query string cannot be decoded, or gives any parameter more
|
||||||
than once, is refused with 400.
|
than once, is refused with 400.
|
||||||
|
|
||||||
- `<format>`: one of `orig`, `png`, `jpeg`, `webp`
|
- `<format>`: one of `orig` (or `original`), `jpeg` (or `jpg`), `png`, `webp`,
|
||||||
|
`avif`, `gif`
|
||||||
- `<size>`: `orig` or `<width>x<height>` (e.g. `800x600`)
|
- `<size>`: `orig` or `<width>x<height>` (e.g. `800x600`)
|
||||||
|
- `sig` and `exp`: the signature and its expiry, needed unless the host is
|
||||||
|
allowlisted (see Signature Specification)
|
||||||
|
- `q` and `fit`: the output quality and how the image is fitted to `<size>`,
|
||||||
|
both optional (values under Signature Specification). Both are part of what
|
||||||
|
is cached, so each value of either is a separate cached image.
|
||||||
|
|
||||||
An image is served with `Cache-Control: public, max-age=<seconds>, immutable`.
|
An image is served with `Cache-Control: public, max-age=<seconds>, immutable`.
|
||||||
When the URL has an expiry (an `exp`, or the TTL of an encrypted URL),
|
When the URL has an expiry (an `exp`, or the TTL of an encrypted URL),
|
||||||
@@ -113,7 +256,9 @@ source) and one transcode: the first request does the work, and the others wait
|
|||||||
for its image or its error, holding no upstream connection or processing slot
|
for its image or its error, holding no upstream connection or processing slot
|
||||||
of their own. A waiting request stops waiting when its own client goes away.
|
of their own. A waiting request stops waiting when its own client goes away.
|
||||||
The work goes on for the others even if the first request's client goes away,
|
The work goes on for the others even if the first request's client goes away,
|
||||||
until that request's `downstream_timeout` ends.
|
until that request's `downstream_timeout` ends. The shared fetch sends the first
|
||||||
|
request's ID upstream, and the lines logged for the fetch and the transcode
|
||||||
|
carry that ID.
|
||||||
|
|
||||||
The login form (`POST /`) is limited to 5 attempts per minute per client
|
The login form (`POST /`) is limited to 5 attempts per minute per client
|
||||||
address, counting an IPv6 client by its /64; an attempt over the limit is
|
address, counting an IPv6 client by its /64; an attempt over the limit is
|
||||||
@@ -132,6 +277,39 @@ its own `X-Forwarded-For`, whether it connects directly or through the proxy,
|
|||||||
because its own address is trusted too. Setting `trusted_proxies` to only the
|
because its own address is trusted too. Setting `trusted_proxies` to only the
|
||||||
address pixa sees for requests that come through the proxy closes this.
|
address pixa sees for requests that come through the proxy closes this.
|
||||||
|
|
||||||
|
### Encrypted URLs
|
||||||
|
|
||||||
|
An encrypted URL is an image URL made on pixa's own web page by someone who
|
||||||
|
knows the signing key. It works for any upstream host, allowlisted or not,
|
||||||
|
without a signature, and whoever gets it can neither read the source URL from it
|
||||||
|
nor change what it asks for.
|
||||||
|
|
||||||
|
1. Open `/` in a browser over HTTPS (or over plain HTTP while `debug` is on, see
|
||||||
|
Routes) and log in with the signing key (`signing_key`). The login session
|
||||||
|
lasts 30 days, or until `/logout`.
|
||||||
|
2. On the generator page, give the source image's URL, the width and height, the
|
||||||
|
format, quality and fit, and how long the URL lasts, then submit the form
|
||||||
|
(`POST /generate`). Width and height both empty or `0` keep the original
|
||||||
|
size; if only one of them is empty or `0`, that side is scaled to keep the
|
||||||
|
image's proportions.
|
||||||
|
3. The page shows the URL, `https://<host>/v1/e/<token>/img.<format>`, and when
|
||||||
|
it expires. `<host>` is the host the page was opened on, and the URL starts
|
||||||
|
with `http` instead while `debug` is on. The name after the token is ignored
|
||||||
|
and only gives the URL a file extension, `jpg` for `orig`.
|
||||||
|
|
||||||
|
The token holds the source's host, path and query and the size, format,
|
||||||
|
quality, fit and expiry, encrypted with a key derived from `signing_key`. The
|
||||||
|
source URL's scheme is not kept: the image is fetched like any other (see
|
||||||
|
Routes), and the blocked networks still apply.
|
||||||
|
|
||||||
|
How long the URL lasts is chosen on the page, from 1 minute to 1 year, or
|
||||||
|
never. The expiry is fixed in the token when the URL is made and cannot be
|
||||||
|
changed or revoked afterwards. Until then the image is served with a `max-age`
|
||||||
|
that ends at the expiry (see Routes); after it the URL answers 410
|
||||||
|
`URL has expired`. A URL made to last forever stops working only when
|
||||||
|
`signing_key` changes: changing it makes every encrypted URL already handed out
|
||||||
|
answer 400, and ends every login session.
|
||||||
|
|
||||||
### Image Metadata
|
### Image Metadata
|
||||||
|
|
||||||
pixa decodes and re-encodes every image it serves, and removes all metadata from
|
pixa decodes and re-encodes every image it serves, and removes all metadata from
|
||||||
@@ -173,7 +351,8 @@ Where:
|
|||||||
- `query` — source query string, empty string if none
|
- `query` — source query string, empty string if none
|
||||||
- `width` — requested width in pixels, `0` for original
|
- `width` — requested width in pixels, `0` for original
|
||||||
- `height` — requested height in pixels, `0` for original
|
- `height` — requested height in pixels, `0` for original
|
||||||
- `format` — output format (jpeg, png, webp, avif, gif, orig)
|
- `format` — output format, one of those listed under Routes, with `original`
|
||||||
|
signed as `orig` and `jpg` as `jpeg`
|
||||||
- `expiration` — the URL's `exp` query parameter, the Unix timestamp when
|
- `expiration` — the URL's `exp` query parameter, the Unix timestamp when
|
||||||
the signature expires; a request whose `exp` is not a whole number, an
|
the signature expires; a request whose `exp` is not a whole number, an
|
||||||
empty `exp=` included, is refused with 400
|
empty `exp=` included, is refused with 400
|
||||||
@@ -229,6 +408,17 @@ startup naming it, as an unknown config key does. The one other accepted
|
|||||||
name is `PIXA_CONFIG_PATH`, the config file's path (like `--config`). The
|
name is `PIXA_CONFIG_PATH`, the config file's path (like `--config`). The
|
||||||
variables set by the file's `env:` section are checked the same way.
|
variables set by the file's `env:` section are checked the same way.
|
||||||
|
|
||||||
|
pixa reads at most one config file: the one given with `--config` (or `-c`),
|
||||||
|
otherwise the one `PIXA_CONFIG_PATH` names, otherwise the first of these that
|
||||||
|
pixa finds: `/etc/pixa/config.yml`, `/etc/pixa/config.yaml`,
|
||||||
|
`~/.config/pixa/config.yml`, `~/.config/pixa/config.yaml`, then `config.yml`
|
||||||
|
and `config.yaml` in the working directory. A named file that does not exist,
|
||||||
|
cannot be read or does not parse aborts startup. Of the files pixa looks for on
|
||||||
|
its own, only one that does not exist is passed over, without a message. One
|
||||||
|
that pixa cannot read or parse aborts startup, naming the file. So does one in a
|
||||||
|
directory pixa may not enter, whether or not it is there, since pixa cannot
|
||||||
|
tell. With no file, pixa uses the environment and the defaults.
|
||||||
|
|
||||||
| Variable | Config key | Meaning |
|
| Variable | Config key | Meaning |
|
||||||
| ------------------------------------ | ------------------------------- | ---------------------------------------------------------------------------- |
|
| ------------------------------------ | ------------------------------- | ---------------------------------------------------------------------------- |
|
||||||
| `PIXA_SIGNING_KEY` | `signing_key` | Required: secret for signed and encrypted URLs and login, 32+ characters |
|
| `PIXA_SIGNING_KEY` | `signing_key` | Required: secret for signed and encrypted URLs and login, 32+ characters |
|
||||||
@@ -314,12 +504,12 @@ Key settings in more detail:
|
|||||||
seconds for one to free up; if none does, and `downstream_timeout` has not
|
seconds for one to free up; if none does, and `downstream_timeout` has not
|
||||||
ended first, it is answered 503 the same way
|
ended first, it is answered 503 the same way
|
||||||
- `maintenance_mode` — while `true`, the image routes (`/v1/image/` and
|
- `maintenance_mode` — while `true`, the image routes (`/v1/image/` and
|
||||||
`/v1/e/`) answer every request with 503, a `Retry-After` header and a JSON
|
`/v1/e/`) answer every request for an image with 503, a `Retry-After` header
|
||||||
error body. The health check (`/.well-known/healthcheck.json`) still answers
|
and a JSON error body. The health check (`/.well-known/healthcheck.json`)
|
||||||
200 and reports `"maintenance_mode": true`. It stays 200 because the image's
|
still answers 200 and reports `"maintenance_mode": true`. It stays 200
|
||||||
Docker `HEALTHCHECK` requests it: a 503 there would make the container
|
because the image's Docker `HEALTHCHECK` requests it: a 503 there would make
|
||||||
unhealthy, and upaas marks a deploy failed when its container is unhealthy.
|
the container unhealthy, and upaas marks a deploy failed when its container
|
||||||
The login and URL generator pages and `/metrics` keep working
|
is unhealthy. The login and URL generator pages and `/metrics` keep working
|
||||||
|
|
||||||
See `config.example.yml` for all options with defaults.
|
See `config.example.yml` for all options with defaults.
|
||||||
|
|
||||||
@@ -330,7 +520,10 @@ See `config.example.yml` for all options with defaults.
|
|||||||
- **Image processing**: govips (CGO wrapper for libvips)
|
- **Image processing**: govips (CGO wrapper for libvips)
|
||||||
- **Database**: SQLite via modernc.org/sqlite
|
- **Database**: SQLite via modernc.org/sqlite
|
||||||
- **Static assets**: embedded via `//go:embed`
|
- **Static assets**: embedded via `//go:embed`
|
||||||
- **Metrics**: Prometheus
|
- **Metrics**: Prometheus, at `/metrics`: generic HTTP request metrics
|
||||||
|
(duration, response size, requests in flight) and the Go runtime and process
|
||||||
|
metrics; requests are measured and `/metrics` is served only when
|
||||||
|
`metrics.username` and `metrics.password` are set
|
||||||
- **Logging**: stdlib slog
|
- **Logging**: stdlib slog
|
||||||
|
|
||||||
## Entrypoints
|
## Entrypoints
|
||||||
|
|||||||
@@ -3,6 +3,8 @@
|
|||||||
* branch per issue from `next`
|
* branch per issue from `next`
|
||||||
* do the work in Next Step
|
* do the work in Next Step
|
||||||
* move Next Step to the top of Completed Steps
|
* move Next Step to the top of Completed Steps
|
||||||
|
* `TODO.md` merges with git's union merge (`.gitattributes`), which never
|
||||||
|
reports a conflict: read the merged entries after every merge or rebase
|
||||||
* move the top item of Future Steps into Next Step
|
* move the top item of Future Steps into Next Step
|
||||||
* commit (`TODO.md` changes in the same commit as the work)
|
* commit (`TODO.md` changes in the same commit as the work)
|
||||||
* open a PR based on `next`
|
* open a PR based on `next`
|
||||||
@@ -29,6 +31,137 @@ P2: security: referer blacklist
|
|||||||
|
|
||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
|
- 2026-10-04 logging in, logging out, the URL generator and `/v1/e/` have
|
||||||
|
handler tests (closes #77): new tests in `internal/handlers`, with no
|
||||||
|
network, check that `GET /` without a login session shows the login form; a
|
||||||
|
wrong key shows it again with an error and sets no session cookie; the right
|
||||||
|
key answers 303 to `/` with a session cookie marked `Secure`, `HttpOnly` and
|
||||||
|
`SameSite=Strict`, with which `GET /` shows the generator page; `GET /logout`
|
||||||
|
answers 303 to `/` with an empty session cookie sent with `Max-Age=0`;
|
||||||
|
`POST /generate` without a login session answers 303 to `/`; `/v1/e/` serves
|
||||||
|
the image for a valid token, answers 410 for an expired one and 400 for one
|
||||||
|
with a character changed, cut short or made with another signing key; and a
|
||||||
|
URL made on the generator page is served by `/v1/e/`. No code changes.
|
||||||
|
- 2026-10-04 `TODO.md` merges with git's union merge (closes #190): a root
|
||||||
|
`.gitattributes`, copied from `sneak/prompts`, marks it `merge=union`, so two
|
||||||
|
branches that each add an entry at the top of Completed Steps merge without a
|
||||||
|
conflict and keep both entries. Git now never reports a conflict in
|
||||||
|
`TODO.md`: a real one keeps both versions of the lines, and two entries that
|
||||||
|
share an identical line can end up one inside the other, which a rebase can
|
||||||
|
do to an entry already on `next`. The Workflow above says to read the merged
|
||||||
|
entries after every merge or rebase.
|
||||||
|
- 2026-10-04 `TestEvictionRunsOnPeriodicSchedule` no longer races the evictor
|
||||||
|
(closes #183): it wrote each variant file and then inserted its accounting row
|
||||||
|
by hand, and a reconciliation pass between the two adopted the file first, so
|
||||||
|
the insert failed. It now writes the files only, while holding the test
|
||||||
|
database's only connection so the evictor's startup pass waits after walking
|
||||||
|
the empty variant directory; a periodic reconciliation pass then adopts the
|
||||||
|
files and the eviction pass after it evicts them. No other test in
|
||||||
|
`internal/imgcache` inserts a row by hand after starting the evictor. Test
|
||||||
|
only.
|
||||||
|
- 2026-10-04 a config file pixa cannot read aborts startup (closes #176): of the
|
||||||
|
places pixa looks for its config file on its own, only one where the file does
|
||||||
|
not exist is passed over; any other error, such as a directory on the path
|
||||||
|
that pixa may not enter, aborts startup naming the file, as a file that does
|
||||||
|
not parse already did.
|
||||||
|
- 2026-10-04 `.golangci.yml` re-vendored from the canonical copy (closes #57):
|
||||||
|
the deprecated `gomodguard` is switched off, so lint runs print no
|
||||||
|
deprecation warning; its successor `gomodguard_v2` runs with the shared
|
||||||
|
module block list, and `depguard` keeps `net/http/httptest` out of files that
|
||||||
|
are not tests. The tree needed no code changes.
|
||||||
|
- 2026-10-04 the Content-Security-Policy allows no inline script or style
|
||||||
|
(closes #125): `script-src` and `style-src` are `'self'` only. The generator
|
||||||
|
page's two inline `onclick` handlers moved into
|
||||||
|
`internal/static/generator.js`, attached with `addEventListener`; the bundled
|
||||||
|
Tailwind script, which built styles in the browser, is replaced by a small
|
||||||
|
hand-written `internal/static/style.css` with only the rules the login and
|
||||||
|
generator pages use, the templates carrying a few plain class names in place
|
||||||
|
of Tailwind's. No build step. The pages keep their layout, not every pixel of
|
||||||
|
it.
|
||||||
|
- 2026-10-04 deployment guide and example Caddy config (closes #89):
|
||||||
|
"Deployment" in `README.md` says what the reverse proxy in front of pixa must
|
||||||
|
do (terminate TLS; pass `Host`, `Origin` and `Referer` on unchanged; set
|
||||||
|
`X-Forwarded-For`, with `trusted_proxies` to match; wait at least
|
||||||
|
`downstream_timeout`; optionally refuse `/metrics`) and what pixa does itself,
|
||||||
|
that the state directory needs a persistent volume and what `cache_max_bytes`
|
||||||
|
counts, the health check for a load balancer, what a stop does and its exit
|
||||||
|
codes, and what running outside Docker needs; `configs/Caddyfile` is the
|
||||||
|
example, checked with `caddy validate`.
|
||||||
|
- 2026-10-04 the metrics basic auth, CORS preflight, request logging and
|
||||||
|
metrics recording have tests (closes #79): `MetricsAuth` on its own answers
|
||||||
|
401 with a challenge without credentials or with a wrong username or password
|
||||||
|
and lets the configured ones through; a preflight request gets `*` for any
|
||||||
|
origin when `access_control_allow_origin` is `*` and no
|
||||||
|
`Access-Control-Allow-Origin` from another origin than the configured one; a
|
||||||
|
`POST /` carrying the signing key leaves no trace of it in the request log
|
||||||
|
line, and the login handler's own log lines leave out the submitted key; the
|
||||||
|
metrics middleware on its own records a request it served, and the router
|
||||||
|
records nothing while no metrics username is set. Not tested: that the router
|
||||||
|
puts the basic auth in front of `/metrics` and records requests when a
|
||||||
|
metrics username is set. Only one test per package can set up `/metrics`, and
|
||||||
|
in `internal/server` that is `TestMaintenanceModeKeepsOtherRoutes`, which
|
||||||
|
needs the owner's approval to change; #180 holds it. Tests only; the basic
|
||||||
|
auth library already compares the password in constant time.
|
||||||
|
- 2026-10-04 the image route's signature check and error answers are tested
|
||||||
|
(closes #76): new tests in `internal/handlers`, with no network, check the
|
||||||
|
status and JSON error body for a missing, wrong, unpadded, upper-case or
|
||||||
|
expired signature on a host not on the allowlist, or a valid one sent for
|
||||||
|
its parent domain, a sibling host, a subdomain or the host with another
|
||||||
|
domain appended (401), an unparseable path (400), `localhost` as the
|
||||||
|
upstream host (403) and an upstream error (502); that an allowlisted host is
|
||||||
|
served without a signature, another host only with a valid one; and the
|
||||||
|
answers of `/robots.txt` and the health check. No code changes.
|
||||||
|
- 2026-10-04 request IDs returned and passed on, and `/v1/e/` revalidates
|
||||||
|
(closes #84): pixa's own `RequestID` middleware, in place of chi's, gives each
|
||||||
|
request an ID, its own `X-Request-ID` when that is at most 64 letters, digits,
|
||||||
|
`-`, `_` or `.` and a random one otherwise, stores it where chi's did and
|
||||||
|
sends it back as `X-Request-ID` on every response; the upstream fetch sends
|
||||||
|
that ID, and the "upstream fetched", "image converted" and "image served" log
|
||||||
|
lines carry it as `request_id`, a fetch shared by several requests carrying
|
||||||
|
the first request's; `/v1/e/` sets `ETag`, answers a matching `If-None-Match`
|
||||||
|
with 304 and is routed for `HEAD`, the `ETag` and 304 code being
|
||||||
|
`notModified`, which `/v1/image/` calls too; its token checks moved unchanged
|
||||||
|
into `parseImageEncRequest` to keep `HandleImageEnc` within the line limit; no
|
||||||
|
`Vary` is added, as no response depends on a request header except the image
|
||||||
|
routes' CORS headers, for which `go-chi/cors` already sends `Vary: Origin`;
|
||||||
|
`Vary: Accept` is left to #88.
|
||||||
|
- 2026-10-04 routes, encrypted URLs and config file documented (closes #75):
|
||||||
|
"Routes" in `README.md` lists every route with its method, purpose, what it
|
||||||
|
needs and the status codes it answers with, and says `q` and `fit` are part
|
||||||
|
of what is cached; "Encrypted URLs" covers logging in, making one on the
|
||||||
|
generator page, how long it lasts and the 410 once it has expired;
|
||||||
|
"Configuration" gives the order in which pixa looks for its config file;
|
||||||
|
`config.example.yml` lists `db_url` and `env` and gives every key's default;
|
||||||
|
`scripts/manual-test.sh` is left to #97.
|
||||||
|
- 2026-10-04 shutdown stops cache eviction in progress (closes #102):
|
||||||
|
`StartEviction` runs the eviction goroutine with its own context, which
|
||||||
|
`StopEviction` cancels, so a pass in progress stops at its next database
|
||||||
|
call, file, row or eviction candidate instead of running to completion, and
|
||||||
|
no pass starts after it, so a stop logs at most one warning;
|
||||||
|
`StopEviction` takes a context and, when that context ends before the
|
||||||
|
goroutine exits, stops waiting and returns its error; the handlers' stop hook
|
||||||
|
passes fx's stop context, so an eviction still running when fx's stop
|
||||||
|
deadline ends fails the stop and makes the exit code 1.
|
||||||
|
- 2026-10-04 dead code in `internal/imgcache` is gone (closes #73): `Purge`,
|
||||||
|
which only returned an error and which nothing called, is no longer part of
|
||||||
|
the `ImageCache` interface or `Service`; the `SignatureValidator`,
|
||||||
|
`Allowlist` and `Storage` interfaces, which nothing implemented or used, are
|
||||||
|
deleted. Nothing else changes.
|
||||||
|
- 2026-10-04 upstream host semaphores and variant `.meta` files no longer
|
||||||
|
outlive their use (closes #87): the fetcher counts the fetches holding or
|
||||||
|
waiting for a slot of each upstream host's semaphore and removes the host's
|
||||||
|
semaphore once none is left, so fetches from many hosts no longer leave one
|
||||||
|
semaphore each until restart; `VariantStorage.Delete` removes the variant's
|
||||||
|
`.meta` file along with it, a missing `.meta` file not being an error, and
|
||||||
|
`DeleteWithMeta`, which eviction called for that, is gone.
|
||||||
|
- 2026-10-04 `README.md` matches the code (closes #74): "Storage" names the
|
||||||
|
cache directories pixa uses (`cache/sources`, `cache/metadata`,
|
||||||
|
`cache/variants`) and how files are named in each, and the comments in
|
||||||
|
`001_schema.sql` name the same paths; the routes and the signature section
|
||||||
|
list the same output formats, `jpg` and `original` included; the TLS
|
||||||
|
sentence names `allow_http` as its exception; "Metrics" says only generic
|
||||||
|
HTTP and Go runtime metrics exist, measured and served only when the metrics
|
||||||
|
username and password are set.
|
||||||
- 2026-10-03 shutdown sets the exit code and waits for image processing
|
- 2026-10-03 shutdown sets the exit code and waits for image processing
|
||||||
(closes #86): fx alone handles SIGINT and SIGTERM, and the server's own
|
(closes #86): fx alone handles SIGINT and SIGTERM, and the server's own
|
||||||
signal handler is gone; fx's `Run` in `cmd/pixad` exits with the shutdown's
|
signal handler is gone; fx's `Run` in `cmd/pixad` exits with the shutdown's
|
||||||
@@ -401,7 +534,6 @@ P2: security: referer blacklist
|
|||||||
- P2: HTTP response handling
|
- P2: HTTP response handling
|
||||||
- Last-Modified headers
|
- Last-Modified headers
|
||||||
- Vary header for content negotiation
|
- Vary header for content negotiation
|
||||||
- X-Request-ID propagation
|
|
||||||
- P2: auto format selection (format=auto based on Accept header)
|
- P2: auto format selection (format=auto based on Accept header)
|
||||||
- P2: configuration
|
- P2: configuration
|
||||||
- YAML config file support
|
- YAML config file support
|
||||||
@@ -411,8 +543,3 @@ P2: security: referer blacklist
|
|||||||
- Prometheus performance metrics
|
- Prometheus performance metrics
|
||||||
- integration tests for the image proxy flow
|
- integration tests for the image proxy flow
|
||||||
- load tests to verify the 1k to 5k req/s target
|
- load tests to verify the 1k to 5k req/s target
|
||||||
- P2: documentation
|
|
||||||
- configuration options
|
|
||||||
- API endpoints
|
|
||||||
- deployment guide
|
|
||||||
- example nginx or caddy reverse proxy config
|
|
||||||
|
|||||||
+27
-9
@@ -12,27 +12,38 @@
|
|||||||
# Durations are Go duration strings such as 30s or 2m and must be
|
# Durations are Go duration strings such as 30s or 2m and must be
|
||||||
# positive; a bare number has no unit and aborts startup. Sizes are a
|
# positive; a bare number has no unit and aborts startup. Sizes are a
|
||||||
# whole number of bytes.
|
# whole number of bytes.
|
||||||
|
#
|
||||||
|
# A key left out takes the default its comment gives.
|
||||||
|
|
||||||
# Server settings
|
# Port to listen on (default: 8080)
|
||||||
port: 8080
|
port: 8080
|
||||||
|
|
||||||
|
# Debug logging and plain-HTTP local development (default: false)
|
||||||
debug: false
|
debug: false
|
||||||
|
|
||||||
# While true, the image routes (/v1/image/ and /v1/e/) answer every request
|
# While true, the image routes (/v1/image/ and /v1/e/) answer every request
|
||||||
# with 503 and a Retry-After header. The health check keeps answering 200 and
|
# for an image with 503 and a Retry-After header. The health check keeps
|
||||||
# reports maintenance_mode as true. It stays 200 because the image's Docker
|
# answering 200 and reports maintenance_mode as true. It stays 200 because
|
||||||
# HEALTHCHECK requests it: a 503 there would make the container unhealthy, and
|
# the image's Docker HEALTHCHECK requests it: a 503 there would make the
|
||||||
# upaas marks a deploy failed when its container is unhealthy.
|
# container unhealthy, and upaas marks a deploy failed when its container is
|
||||||
|
# unhealthy. (default: false)
|
||||||
maintenance_mode: false
|
maintenance_mode: false
|
||||||
|
|
||||||
# Data directory for SQLite database and cache files
|
# Data directory for SQLite database and cache files
|
||||||
|
# (default: /var/lib/pixa)
|
||||||
state_dir: ./data
|
state_dir: ./data
|
||||||
|
|
||||||
|
# SQLite database URL (default:
|
||||||
|
# file:<state_dir>/state.sqlite3?_journal_mode=WAL). An empty value aborts
|
||||||
|
# startup; leave the key out to use the default.
|
||||||
|
# db_url: "file:./data/state.sqlite3?_journal_mode=WAL"
|
||||||
|
|
||||||
# Image proxy settings
|
# Image proxy settings
|
||||||
# HMAC signing key for URL signatures (required, at least 32 characters)
|
# HMAC signing key for URL signatures (required, at least 32 characters)
|
||||||
# Generate with: openssl rand -base64 32
|
# Generate with: openssl rand -base64 32
|
||||||
signing_key: "CHANGE_ME_generate_with_openssl_rand_base64_32"
|
signing_key: "CHANGE_ME_generate_with_openssl_rand_base64_32"
|
||||||
|
|
||||||
# Hosts that don't require signatures
|
# Hosts that don't require signatures (default: none)
|
||||||
# Use "." prefix for wildcard subdomain matching (e.g., ".example.com" matches "cdn.example.com")
|
# Use "." prefix for wildcard subdomain matching (e.g., ".example.com" matches "cdn.example.com")
|
||||||
allowlist_hosts:
|
allowlist_hosts:
|
||||||
- s3.sneak.cloud
|
- s3.sneak.cloud
|
||||||
@@ -45,7 +56,7 @@ allowlist_hosts:
|
|||||||
# SSRF protection. These are added to the always-enforced built-in ranges
|
# SSRF protection. These are added to the always-enforced built-in ranges
|
||||||
# (loopback, RFC 1918 private, link-local, CGNAT, benchmark, NAT64, and
|
# (loopback, RFC 1918 private, link-local, CGNAT, benchmark, NAT64, and
|
||||||
# similar), never replacing them. Each entry must be a valid CIDR in IPv4
|
# similar), never replacing them. Each entry must be a valid CIDR in IPv4
|
||||||
# or IPv6 form; an invalid entry aborts startup.
|
# or IPv6 form; an invalid entry aborts startup. (default: none)
|
||||||
# blocked_networks:
|
# blocked_networks:
|
||||||
# - 100.64.0.0/10
|
# - 100.64.0.0/10
|
||||||
# - 2001:db8::/32
|
# - 2001:db8::/32
|
||||||
@@ -72,6 +83,7 @@ allowlist_hosts:
|
|||||||
# - 2001:db8::/32
|
# - 2001:db8::/32
|
||||||
|
|
||||||
# Allow HTTP upstream (only for testing, always use HTTPS in production)
|
# Allow HTTP upstream (only for testing, always use HTTPS in production)
|
||||||
|
# (default: false)
|
||||||
allow_http: false
|
allow_http: false
|
||||||
|
|
||||||
# Maximum concurrent connections per upstream host (default: 20)
|
# Maximum concurrent connections per upstream host (default: 20)
|
||||||
@@ -121,10 +133,16 @@ access_control_allow_origin: "*"
|
|||||||
# with a minimum of 500 MiB.
|
# with a minimum of 500 MiB.
|
||||||
# cache_max_bytes: 10737418240
|
# cache_max_bytes: 10737418240
|
||||||
|
|
||||||
# Sentry error reporting (optional)
|
# Sentry DSN for error reporting (default: empty, which turns it off)
|
||||||
sentry_dsn: ""
|
sentry_dsn: ""
|
||||||
|
|
||||||
# Metrics endpoint authentication (optional)
|
# Username and password for /metrics, set together (default: unset). Metrics
|
||||||
|
# are measured and /metrics is served only when both are set.
|
||||||
# metrics:
|
# metrics:
|
||||||
# username: "admin"
|
# username: "admin"
|
||||||
# password: "secret"
|
# password: "secret"
|
||||||
|
|
||||||
|
# Environment variables set while this file loads, as described at the top
|
||||||
|
# (default: none)
|
||||||
|
# env:
|
||||||
|
# PIXA_DEBUG: "true"
|
||||||
|
|||||||
@@ -0,0 +1,17 @@
|
|||||||
|
# Example Caddy config for running pixa behind Caddy; see "Deployment" in
|
||||||
|
# README.md. Replace images.example.com with pixa's public host name, and
|
||||||
|
# 127.0.0.1:8080 with the address Caddy reaches pixa on.
|
||||||
|
#
|
||||||
|
# Caddy gets and renews the TLS certificate for the host name, passes the
|
||||||
|
# Host, Origin and Referer headers on unchanged, sets X-Forwarded-For to the
|
||||||
|
# client's address, and waits for pixa's answer with no time limit of its
|
||||||
|
# own, so pixa's downstream_timeout is what ends a slow request.
|
||||||
|
|
||||||
|
images.example.com
|
||||||
|
|
||||||
|
# pixa asks for metrics.username and metrics.password on /metrics. This
|
||||||
|
# line also keeps it off the public address, for a scraper that reaches
|
||||||
|
# pixa directly; remove it to read /metrics through Caddy.
|
||||||
|
respond /metrics 404
|
||||||
|
|
||||||
|
reverse_proxy 127.0.0.1:8080
|
||||||
+21
-11
@@ -4,6 +4,7 @@ package config
|
|||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"math"
|
"math"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
@@ -14,6 +15,7 @@ import (
|
|||||||
"sort"
|
"sort"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.eeqj.de/sneak/smartconfig"
|
"git.eeqj.de/sneak/smartconfig"
|
||||||
@@ -778,19 +780,27 @@ func loadConfigFile(log *slog.Logger, appName string) (*smartconfig.Config, erro
|
|||||||
for _, path := range configPaths {
|
for _, path := range configPaths {
|
||||||
cleanPath := filepath.Clean(path)
|
cleanPath := filepath.Clean(path)
|
||||||
|
|
||||||
|
// Only a config file that does not exist is skipped, including
|
||||||
|
// one whose path runs through a file, such as under a HOME of
|
||||||
|
// /dev/null. One that cannot be read or does not parse is a
|
||||||
|
// fatal startup error.
|
||||||
_, statErr := os.Stat(cleanPath)
|
_, statErr := os.Stat(cleanPath)
|
||||||
if statErr == nil {
|
if errors.Is(statErr, fs.ErrNotExist) || errors.Is(statErr, syscall.ENOTDIR) {
|
||||||
// A config file that exists but does not parse is a fatal
|
continue
|
||||||
// startup error, never something to skip over.
|
|
||||||
sc, err := smartconfig.NewFromConfigPath(path)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to parse config file %s: %w", path, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Info("loaded config file", "path", path)
|
|
||||||
|
|
||||||
return sc, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if statErr != nil {
|
||||||
|
return nil, fmt.Errorf("failed to read config file %s: %w", path, statErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
sc, err := smartconfig.NewFromConfigPath(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to parse config file %s: %w", path, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info("loaded config file", "path", path)
|
||||||
|
|
||||||
|
return sc, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil, nil //nolint:nilnil // nil config is valid (use defaults)
|
return nil, nil //nolint:nilnil // nil config is valid (use defaults)
|
||||||
|
|||||||
@@ -564,6 +564,116 @@ func TestMalformedConfigFileAbortsStartup(t *testing.T) {
|
|||||||
t.Logf("got expected error: %v", err)
|
t.Logf("got expected error: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestConfigFileInDirectoryPixaMayNotEnterAbortsStartup checks that a
|
||||||
|
// config file pixa cannot read because it may not enter its directory
|
||||||
|
// aborts startup instead of being passed over.
|
||||||
|
func TestConfigFileInDirectoryPixaMayNotEnterAbortsStartup(t *testing.T) {
|
||||||
|
if os.Geteuid() == 0 {
|
||||||
|
t.Skip("root may enter any directory")
|
||||||
|
}
|
||||||
|
|
||||||
|
home := t.TempDir()
|
||||||
|
configDir := filepath.Join(home, ".config", "pixa-test-nonexistent-app")
|
||||||
|
configPath := filepath.Join(configDir, "config.yml")
|
||||||
|
|
||||||
|
err := os.MkdirAll(configDir, 0o700)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to create config directory: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = os.WriteFile(configPath, []byte(signingKeyLine), 0o600)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to write config: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = os.Chmod(configDir, 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to remove the config directory's permissions: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Give the directory back its permissions so t.TempDir can remove it.
|
||||||
|
t.Cleanup(func() {
|
||||||
|
//nolint:gosec // G302: a directory needs its execute bit to be removed
|
||||||
|
_ = os.Chmod(configDir, 0o700)
|
||||||
|
})
|
||||||
|
|
||||||
|
// The ~/.config candidate is the only one that exists: the appname
|
||||||
|
// rules out /etc, and the working directory is empty.
|
||||||
|
t.Setenv("PIXA_CONFIG_PATH", "")
|
||||||
|
t.Setenv("HOME", home)
|
||||||
|
t.Chdir(t.TempDir())
|
||||||
|
|
||||||
|
log := slog.New(slog.DiscardHandler)
|
||||||
|
|
||||||
|
sc, err := loadConfigFile(log, "pixa-test-nonexistent-app")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("config file pixa cannot read must abort startup, got config: %v",
|
||||||
|
sc)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("got expected error: %v", err)
|
||||||
|
|
||||||
|
if !strings.Contains(err.Error(), configPath) {
|
||||||
|
t.Errorf("error %q does not name the config file %s", err.Error(), configPath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestConfigFileLinkingToItselfAbortsStartup checks that a config file
|
||||||
|
// pixa cannot read for a reason other than not existing aborts startup,
|
||||||
|
// as root too: a symbolic link to itself fails with "too many levels of
|
||||||
|
// symbolic links".
|
||||||
|
func TestConfigFileLinkingToItselfAbortsStartup(t *testing.T) {
|
||||||
|
workDir := t.TempDir()
|
||||||
|
|
||||||
|
err := os.Symlink("config.yml", filepath.Join(workDir, "config.yml"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to create symbolic link: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Only the working directory's config.yml is there: the appname rules
|
||||||
|
// out /etc, and HOME is empty.
|
||||||
|
t.Setenv("PIXA_CONFIG_PATH", "")
|
||||||
|
t.Setenv("HOME", t.TempDir())
|
||||||
|
t.Chdir(workDir)
|
||||||
|
|
||||||
|
log := slog.New(slog.DiscardHandler)
|
||||||
|
|
||||||
|
sc, err := loadConfigFile(log, "pixa-test-nonexistent-app")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("config file pixa cannot read must abort startup, got config: %v",
|
||||||
|
sc)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("got expected error: %v", err)
|
||||||
|
|
||||||
|
if !strings.Contains(err.Error(), "config.yml") {
|
||||||
|
t.Errorf("error %q does not name the config file config.yml", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestConfigPathThroughFileIsPassedOver checks that a config file path
|
||||||
|
// that runs through a file, such as one under a HOME of /dev/null, is
|
||||||
|
// passed over like one that does not exist, since no file can be there.
|
||||||
|
func TestConfigPathThroughFileIsPassedOver(t *testing.T) {
|
||||||
|
// No config file is there: the appname rules out /etc, HOME is
|
||||||
|
// /dev/null, and the working directory is empty.
|
||||||
|
t.Setenv("PIXA_CONFIG_PATH", "")
|
||||||
|
t.Setenv("HOME", os.DevNull)
|
||||||
|
t.Chdir(t.TempDir())
|
||||||
|
|
||||||
|
log := slog.New(slog.DiscardHandler)
|
||||||
|
|
||||||
|
sc, err := loadConfigFile(log, "pixa-test-nonexistent-app")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("a config path through a file must be passed over, got error: %v",
|
||||||
|
err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if sc != nil {
|
||||||
|
t.Errorf("expected no config file, got config: %v", sc)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestEnsureStateDirCreatesDirectory(t *testing.T) {
|
func TestEnsureStateDirCreatesDirectory(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
-- Creates all tables for the pixa caching image proxy
|
-- Creates all tables for the pixa caching image proxy
|
||||||
|
|
||||||
-- Source content blobs
|
-- Source content blobs
|
||||||
-- Files stored at: cache/src-content/<ab>/<cd>/<sha256>
|
-- Files stored at: cache/sources/<ab>/<cd>/<sha256>
|
||||||
-- last_accessed_at is NULL until the first LRU touch; eviction falls
|
-- last_accessed_at is NULL until the first LRU touch; eviction falls
|
||||||
-- back to fetched_at for rows that have never been touched.
|
-- back to fetched_at for rows that have never been touched.
|
||||||
CREATE TABLE IF NOT EXISTS source_content (
|
CREATE TABLE IF NOT EXISTS source_content (
|
||||||
@@ -16,7 +16,7 @@ CREATE INDEX IF NOT EXISTS idx_source_content_last_accessed
|
|||||||
ON source_content(last_accessed_at);
|
ON source_content(last_accessed_at);
|
||||||
|
|
||||||
-- Source URL metadata - maps URLs to content hashes
|
-- Source URL metadata - maps URLs to content hashes
|
||||||
-- JSON stored at: cache/src-metadata/<hostname>/<path_hash>.json
|
-- JSON stored at: cache/metadata/<hostname>/<path_hash>.json
|
||||||
CREATE TABLE IF NOT EXISTS source_metadata (
|
CREATE TABLE IF NOT EXISTS source_metadata (
|
||||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
source_host TEXT NOT NULL,
|
source_host TEXT NOT NULL,
|
||||||
@@ -56,7 +56,8 @@ CREATE INDEX IF NOT EXISTS idx_variant_content_last_accessed
|
|||||||
ON variant_content(last_accessed_at);
|
ON variant_content(last_accessed_at);
|
||||||
|
|
||||||
-- Output/transformed content blobs
|
-- Output/transformed content blobs
|
||||||
-- Files stored at: cache/dst-content/<ab>/<cd>/<sha256>
|
-- Not written: transformed images are stored in cache/variants and
|
||||||
|
-- tracked in variant_content above.
|
||||||
CREATE TABLE IF NOT EXISTS output_content (
|
CREATE TABLE IF NOT EXISTS output_content (
|
||||||
content_hash TEXT PRIMARY KEY,
|
content_hash TEXT PRIMARY KEY,
|
||||||
content_type TEXT NOT NULL,
|
content_type TEXT NOT NULL,
|
||||||
|
|||||||
@@ -0,0 +1,59 @@
|
|||||||
|
package handlers
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"sneak.berlin/go/pixa/internal/config"
|
||||||
|
"sneak.berlin/go/pixa/internal/session"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestLoginLogLeavesOutSubmittedKey verifies that the log lines for a
|
||||||
|
// failed and for a successful login do not contain the submitted key.
|
||||||
|
func TestLoginLogLeavesOutSubmittedKey(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const wrongKey = "wrong-signing-key-fedcba9876543210"
|
||||||
|
|
||||||
|
var buf bytes.Buffer
|
||||||
|
|
||||||
|
sessMgr, err := session.NewManager(testSigningKey)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("session.NewManager() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
h := &Handlers{
|
||||||
|
log: slog.New(slog.NewJSONHandler(&buf, nil)),
|
||||||
|
config: &config.Config{SigningKey: testSigningKey},
|
||||||
|
sessMgr: sessMgr,
|
||||||
|
}
|
||||||
|
|
||||||
|
submittedKeys := []string{wrongKey, testSigningKey}
|
||||||
|
|
||||||
|
for _, key := range submittedKeys {
|
||||||
|
form := url.Values{loginKeyField: {key}}
|
||||||
|
req := httptest.NewRequestWithContext(
|
||||||
|
t.Context(), http.MethodPost, "/",
|
||||||
|
strings.NewReader(form.Encode()))
|
||||||
|
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||||
|
|
||||||
|
h.handleLoginPost(httptest.NewRecorder(), req)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, msg := range []string{"failed login attempt", "successful login"} {
|
||||||
|
if !strings.Contains(buf.String(), msg) {
|
||||||
|
t.Fatalf("log missing %q; got %q", msg, buf.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, key := range submittedKeys {
|
||||||
|
if strings.Contains(buf.String(), key) {
|
||||||
|
t.Errorf("log contains submitted key %q; got %q", key, buf.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,243 @@
|
|||||||
|
package handlers
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"net/url"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"sneak.berlin/go/pixa/internal/imgcache"
|
||||||
|
"sneak.berlin/go/pixa/internal/session"
|
||||||
|
)
|
||||||
|
|
||||||
|
// formatField is the generator form's format field name.
|
||||||
|
const formatField = "format"
|
||||||
|
|
||||||
|
// Markers telling the login page from the generator page.
|
||||||
|
const (
|
||||||
|
loginForm = `action="/"`
|
||||||
|
loginKeyInput = `name="key"`
|
||||||
|
generatorForm = `action="/generate"`
|
||||||
|
)
|
||||||
|
|
||||||
|
// generatedURLPattern extracts the path of the URL the generator page shows.
|
||||||
|
// The test router runs with debug on, so the URL starts with http, and its
|
||||||
|
// host is httptest's default request host.
|
||||||
|
var generatedURLPattern = regexp.MustCompile(
|
||||||
|
`value="http://example\.com(/v1/e/[^"]+)"`)
|
||||||
|
|
||||||
|
// findSessionCookie returns the session cookie rec sets, or nil if it sets
|
||||||
|
// none.
|
||||||
|
func findSessionCookie(rec *httptest.ResponseRecorder) *http.Cookie {
|
||||||
|
for _, c := range rec.Result().Cookies() {
|
||||||
|
if c.Name == session.CookieName {
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestHandleRoot_NoSession_ShowsLoginForm verifies that GET / without a
|
||||||
|
// login session shows the login form.
|
||||||
|
func TestHandleRoot_NoSession_ShowsLoginForm(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
_, srv := newCSRFTestRouter(t)
|
||||||
|
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
srv.ServeHTTP(rec, httptest.NewRequestWithContext(
|
||||||
|
t.Context(), http.MethodGet, "/", nil))
|
||||||
|
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK)
|
||||||
|
}
|
||||||
|
|
||||||
|
body := rec.Body.String()
|
||||||
|
if !strings.Contains(body, loginForm) || !strings.Contains(body, loginKeyInput) {
|
||||||
|
t.Errorf("page is not the login form: %s", body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestLoginPost_WrongKey_ShowsErrorWithoutSession verifies that a wrong key
|
||||||
|
// shows the login form again with an error, and sets no session cookie.
|
||||||
|
func TestLoginPost_WrongKey_ShowsErrorWithoutSession(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
_, srv := newCSRFTestRouter(t)
|
||||||
|
|
||||||
|
cookies, token := csrfCredentials(t, srv, nil)
|
||||||
|
|
||||||
|
rec := postForm(srv, "/", cookies, url.Values{
|
||||||
|
loginKeyField: {"wrong-signing-key-fedcba9876543210"},
|
||||||
|
csrfTokenField: {token},
|
||||||
|
})
|
||||||
|
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK)
|
||||||
|
}
|
||||||
|
|
||||||
|
body := rec.Body.String()
|
||||||
|
if !strings.Contains(body, loginForm) || !strings.Contains(body, loginKeyInput) {
|
||||||
|
t.Errorf("page is not the login form: %s", body)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.Contains(body, "Invalid signing key") {
|
||||||
|
t.Error("login form does not show the error")
|
||||||
|
}
|
||||||
|
|
||||||
|
if c := findSessionCookie(rec); c != nil {
|
||||||
|
t.Errorf("wrong key set a session cookie: %s", c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestLoginPost_RightKey_SetsSessionCookie verifies that the right key answers
|
||||||
|
// 303 to / with a session cookie marked Secure, HttpOnly and SameSite=Strict,
|
||||||
|
// and that GET / with that cookie shows the generator page.
|
||||||
|
func TestLoginPost_RightKey_SetsSessionCookie(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
_, srv := newCSRFTestRouter(t)
|
||||||
|
|
||||||
|
cookies, token := csrfCredentials(t, srv, nil)
|
||||||
|
|
||||||
|
rec := postForm(srv, "/", cookies, url.Values{
|
||||||
|
loginKeyField: {testSigningKey},
|
||||||
|
csrfTokenField: {token},
|
||||||
|
})
|
||||||
|
|
||||||
|
if rec.Code != http.StatusSeeOther || rec.Header().Get("Location") != "/" {
|
||||||
|
t.Fatalf("status = %d, Location = %q, want %d to /",
|
||||||
|
rec.Code, rec.Header().Get("Location"), http.StatusSeeOther)
|
||||||
|
}
|
||||||
|
|
||||||
|
sessionCookie := findSessionCookie(rec)
|
||||||
|
if sessionCookie == nil {
|
||||||
|
t.Fatal("right key set no session cookie")
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("Set-Cookie: %s", sessionCookie)
|
||||||
|
|
||||||
|
if !sessionCookie.Secure {
|
||||||
|
t.Error("session cookie is not Secure")
|
||||||
|
}
|
||||||
|
|
||||||
|
if !sessionCookie.HttpOnly {
|
||||||
|
t.Error("session cookie is not HttpOnly")
|
||||||
|
}
|
||||||
|
|
||||||
|
if sessionCookie.SameSite != http.SameSiteStrictMode {
|
||||||
|
t.Errorf("session cookie SameSite = %v, want Strict", sessionCookie.SameSite)
|
||||||
|
}
|
||||||
|
|
||||||
|
req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil)
|
||||||
|
req.AddCookie(sessionCookie)
|
||||||
|
|
||||||
|
rec = httptest.NewRecorder()
|
||||||
|
srv.ServeHTTP(rec, req)
|
||||||
|
|
||||||
|
if rec.Code != http.StatusOK ||
|
||||||
|
!strings.Contains(rec.Body.String(), generatorForm) {
|
||||||
|
t.Errorf("GET / with the session cookie: status = %d, "+
|
||||||
|
"want %d and the generator page", rec.Code, http.StatusOK)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestHandleLogout_ClearsSessionCookie verifies that GET /logout answers 303
|
||||||
|
// to / and replaces the session cookie with an empty one sent with
|
||||||
|
// Max-Age=0, which makes the browser delete it.
|
||||||
|
func TestHandleLogout_ClearsSessionCookie(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
h, _ := newCSRFTestRouter(t)
|
||||||
|
|
||||||
|
req := httptest.NewRequestWithContext(
|
||||||
|
t.Context(), http.MethodGet, "/logout", nil)
|
||||||
|
req.AddCookie(newSessionCookie(t, h))
|
||||||
|
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
h.HandleLogout().ServeHTTP(rec, req)
|
||||||
|
|
||||||
|
if rec.Code != http.StatusSeeOther || rec.Header().Get("Location") != "/" {
|
||||||
|
t.Fatalf("status = %d, Location = %q, want %d to /",
|
||||||
|
rec.Code, rec.Header().Get("Location"), http.StatusSeeOther)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("Set-Cookie: %s", rec.Header().Get("Set-Cookie"))
|
||||||
|
|
||||||
|
sessionCookie := findSessionCookie(rec)
|
||||||
|
if sessionCookie == nil {
|
||||||
|
t.Fatal("logout did not set the session cookie")
|
||||||
|
}
|
||||||
|
|
||||||
|
if sessionCookie.Value != "" {
|
||||||
|
t.Errorf("session cookie value = %q, want empty", sessionCookie.Value)
|
||||||
|
}
|
||||||
|
|
||||||
|
// net/http reads a Max-Age=0 attribute back as MaxAge -1.
|
||||||
|
if sessionCookie.MaxAge != -1 {
|
||||||
|
t.Errorf("session cookie MaxAge = %d, want -1 (Max-Age=0)",
|
||||||
|
sessionCookie.MaxAge)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestGeneratePost_NoSession_RedirectsToLogin verifies that POST /generate
|
||||||
|
// with a valid CSRF token but no login session answers 303 to / and makes no
|
||||||
|
// URL.
|
||||||
|
func TestGeneratePost_NoSession_RedirectsToLogin(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
_, srv := newCSRFTestRouter(t)
|
||||||
|
|
||||||
|
cookies, token := csrfCredentials(t, srv, nil)
|
||||||
|
|
||||||
|
rec := postForm(srv, "/generate", cookies, url.Values{
|
||||||
|
sourceURLField: {testSourceURL},
|
||||||
|
csrfTokenField: {token},
|
||||||
|
})
|
||||||
|
|
||||||
|
if rec.Code != http.StatusSeeOther || rec.Header().Get("Location") != "/" {
|
||||||
|
t.Fatalf("status = %d, Location = %q, want %d to /",
|
||||||
|
rec.Code, rec.Header().Get("Location"), http.StatusSeeOther)
|
||||||
|
}
|
||||||
|
|
||||||
|
if strings.Contains(rec.Body.String(), "/v1/e/") {
|
||||||
|
t.Error("a URL was made without a login session")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestGeneratePost_URLServesImage verifies that the URL the generator page
|
||||||
|
// makes is served by /v1/e/. The image route runs on handlers of its own,
|
||||||
|
// made with the same signing key.
|
||||||
|
func TestGeneratePost_URLServesImage(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
_, imageSrv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
||||||
|
|
||||||
|
rec := generatePost(t, url.Values{
|
||||||
|
sourceURLField: {"https://" + signedHost + photoPath},
|
||||||
|
widthField: {"50"},
|
||||||
|
heightField: {"50"},
|
||||||
|
formatField: {string(imgcache.FormatJPEG)},
|
||||||
|
})
|
||||||
|
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("POST /generate status = %d, want %d", rec.Code, http.StatusOK)
|
||||||
|
}
|
||||||
|
|
||||||
|
match := generatedURLPattern.FindStringSubmatch(rec.Body.String())
|
||||||
|
if match == nil {
|
||||||
|
t.Fatalf("generator page shows no URL: %s", rec.Body.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("generated URL path: %s", match[1])
|
||||||
|
|
||||||
|
imageRec := httptest.NewRecorder()
|
||||||
|
imageSrv.ServeHTTP(imageRec, httptest.NewRequestWithContext(
|
||||||
|
t.Context(), http.MethodGet, match[1], nil))
|
||||||
|
|
||||||
|
requireServedPhoto(t, imageRec)
|
||||||
|
}
|
||||||
@@ -58,23 +58,16 @@ func New(lc fx.Lifecycle, params Params) (*Handlers, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
lc.Append(fx.Hook{
|
lc.Append(fx.Hook{
|
||||||
// The eviction goroutine must outlive OnStart, so it cannot
|
//nolint:contextcheck // the eviction loop outlives OnStart; OnStop cancels it
|
||||||
// inherit this hook's context. It makes its own instead, which
|
|
||||||
// leaves it uncancellable: an in-flight pass runs to completion
|
|
||||||
// during OnStop regardless of the shutdown deadline. Making the
|
|
||||||
// loop cancellable changes shutdown semantics and is tracked
|
|
||||||
// separately in issue #102, rather than being folded into the
|
|
||||||
// lint-conformance change that surfaced it.
|
|
||||||
//nolint:contextcheck // see issue #102
|
|
||||||
OnStart: func(_ context.Context) error {
|
OnStart: func(_ context.Context) error {
|
||||||
return s.initImageService()
|
return s.initImageService()
|
||||||
},
|
},
|
||||||
OnStop: func(_ context.Context) error {
|
OnStop: func(ctx context.Context) error {
|
||||||
if s.imgCache != nil {
|
if s.imgCache == nil {
|
||||||
s.imgCache.StopEviction()
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return s.imgCache.StopEviction(ctx)
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|||||||
+23
-11
@@ -10,6 +10,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/go-chi/chi/v5"
|
"github.com/go-chi/chi/v5"
|
||||||
|
"github.com/go-chi/chi/v5/middleware"
|
||||||
"sneak.berlin/go/pixa/internal/encurl"
|
"sneak.berlin/go/pixa/internal/encurl"
|
||||||
"sneak.berlin/go/pixa/internal/httpfetcher"
|
"sneak.berlin/go/pixa/internal/httpfetcher"
|
||||||
"sneak.berlin/go/pixa/internal/imageprocessor"
|
"sneak.berlin/go/pixa/internal/imageprocessor"
|
||||||
@@ -247,6 +248,25 @@ func cacheControl(expires time.Time) string {
|
|||||||
return fmt.Sprintf("public, max-age=%d, immutable", int64(maxAge/time.Second))
|
return fmt.Sprintf("public, max-age=%d, immutable", int64(maxAge/time.Second))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// notModified sets the ETag header to etag and, when the request's
|
||||||
|
// If-None-Match is that ETag, answers 304 Not Modified. It reports whether it
|
||||||
|
// answered. An empty etag sets no header and never answers.
|
||||||
|
func notModified(w http.ResponseWriter, r *http.Request, etag string) bool {
|
||||||
|
if etag == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
w.Header().Set("ETag", etag)
|
||||||
|
|
||||||
|
if r.Header.Get("If-None-Match") != etag {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
w.WriteHeader(http.StatusNotModified)
|
||||||
|
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
// writeImageResponse writes headers and streams the image content,
|
// writeImageResponse writes headers and streams the image content,
|
||||||
// handling conditional and HEAD requests.
|
// handling conditional and HEAD requests.
|
||||||
func (s *Handlers) writeImageResponse(
|
func (s *Handlers) writeImageResponse(
|
||||||
@@ -265,17 +285,8 @@ func (s *Handlers) writeImageResponse(
|
|||||||
w.Header().Set("Cache-Control", cacheControl(req.Expires))
|
w.Header().Set("Cache-Control", cacheControl(req.Expires))
|
||||||
w.Header().Set("X-Pixa-Cache", string(resp.CacheStatus))
|
w.Header().Set("X-Pixa-Cache", string(resp.CacheStatus))
|
||||||
|
|
||||||
if resp.ETag != "" {
|
if notModified(w, r, resp.ETag) {
|
||||||
w.Header().Set("ETag", resp.ETag)
|
return
|
||||||
|
|
||||||
// Check for conditional request (If-None-Match)
|
|
||||||
if ifNoneMatch := r.Header.Get("If-None-Match"); ifNoneMatch != "" {
|
|
||||||
if ifNoneMatch == resp.ETag {
|
|
||||||
w.WriteHeader(http.StatusNotModified)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Handle HEAD request - return headers only
|
// Handle HEAD request - return headers only
|
||||||
@@ -298,6 +309,7 @@ func (s *Handlers) writeImageResponse(
|
|||||||
// Log cache status and timing after serving
|
// Log cache status and timing after serving
|
||||||
duration := time.Since(startTime)
|
duration := time.Since(startTime)
|
||||||
s.log.Info("image served",
|
s.log.Info("image served",
|
||||||
|
"request_id", middleware.GetReqID(r.Context()),
|
||||||
"cache_key", cacheKey,
|
"cache_key", cacheKey,
|
||||||
"cache_status", resp.CacheStatus,
|
"cache_status", resp.CacheStatus,
|
||||||
"duration_ms", duration.Milliseconds(),
|
"duration_ms", duration.Milliseconds(),
|
||||||
|
|||||||
@@ -23,8 +23,10 @@ const photoPath = "/images/photo.jpg"
|
|||||||
// newSignedHostServer returns a router for both image routes, and the Handlers
|
// newSignedHostServer returns a router for both image routes, and the Handlers
|
||||||
// behind it, whose fetcher serves a JPEG at photoPath on signedHost. signedHost
|
// behind it, whose fetcher serves a JPEG at photoPath on signedHost. signedHost
|
||||||
// is not on the allowlist, so a /v1/image/ URL for it is served only with a
|
// is not on the allowlist, so a /v1/image/ URL for it is served only with a
|
||||||
// valid signature.
|
// valid signature. The handlers and the image service log to log.
|
||||||
func newSignedHostServer(t *testing.T) (*Handlers, http.Handler) {
|
func newSignedHostServer(
|
||||||
|
t *testing.T, log *slog.Logger,
|
||||||
|
) (*Handlers, http.Handler) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
cache, err := imgcache.NewCache(setupTestDB(t), imgcache.CacheConfig{
|
cache, err := imgcache.NewCache(setupTestDB(t), imgcache.CacheConfig{
|
||||||
@@ -44,6 +46,7 @@ func newSignedHostServer(t *testing.T) (*Handlers, http.Handler) {
|
|||||||
signedHost + photoPath: &fstest.MapFile{Data: jpegData},
|
signedHost + photoPath: &fstest.MapFile{Data: jpegData},
|
||||||
}),
|
}),
|
||||||
SigningKey: testSigningKey,
|
SigningKey: testSigningKey,
|
||||||
|
Logger: log,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("imgcache.NewService() error = %v", err)
|
t.Fatalf("imgcache.NewService() error = %v", err)
|
||||||
@@ -55,7 +58,7 @@ func newSignedHostServer(t *testing.T) (*Handlers, http.Handler) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
h := &Handlers{
|
h := &Handlers{
|
||||||
log: slog.New(slog.DiscardHandler),
|
log: log,
|
||||||
imgSvc: svc,
|
imgSvc: svc,
|
||||||
encGen: encGen,
|
encGen: encGen,
|
||||||
}
|
}
|
||||||
@@ -103,7 +106,7 @@ func getMaxAge(t *testing.T, srv http.Handler, target string) int {
|
|||||||
func TestHandleImage_SignedURL_MaxAgeEndsAtExp(t *testing.T) {
|
func TestHandleImage_SignedURL_MaxAgeEndsAtExp(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
h, srv := newSignedHostServer(t)
|
h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
||||||
|
|
||||||
signedURL, err := h.imgSvc.GenerateSignedURL("", &imgcache.ImageRequest{
|
signedURL, err := h.imgSvc.GenerateSignedURL("", &imgcache.ImageRequest{
|
||||||
SourceHost: signedHost,
|
SourceHost: signedHost,
|
||||||
@@ -179,7 +182,7 @@ func TestHandleImageEnc_MaxAge(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
h, srv := newSignedHostServer(t)
|
h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
||||||
|
|
||||||
token, err := h.encGen.Generate(&encurl.Payload{
|
token, err := h.encGen.Generate(&encurl.Payload{
|
||||||
SourceHost: signedHost,
|
SourceHost: signedHost,
|
||||||
|
|||||||
@@ -0,0 +1,267 @@
|
|||||||
|
package handlers
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"image/color"
|
||||||
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/go-chi/chi/v5"
|
||||||
|
"sneak.berlin/go/pixa/internal/httpfetcher"
|
||||||
|
"sneak.berlin/go/pixa/internal/imgcache"
|
||||||
|
"sneak.berlin/go/pixa/internal/signature"
|
||||||
|
)
|
||||||
|
|
||||||
|
// allowlistedHost is the only host on the allowlist of the image route
|
||||||
|
// newImageRoute builds.
|
||||||
|
const allowlistedHost = "allowed.example.com"
|
||||||
|
|
||||||
|
// newImageRoute returns the image route of a Handlers whose service fetches
|
||||||
|
// with fetcher and checks signatures with testSigningKey.
|
||||||
|
func newImageRoute(t *testing.T, fetcher httpfetcher.Fetcher) http.Handler {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
cache, err := imgcache.NewCache(setupTestDB(t), imgcache.CacheConfig{
|
||||||
|
StateDir: t.TempDir(),
|
||||||
|
CacheTTL: time.Hour,
|
||||||
|
NegativeTTL: 5 * time.Minute,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to create cache: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
svc, err := imgcache.NewService(&imgcache.ServiceConfig{
|
||||||
|
Cache: cache,
|
||||||
|
Fetcher: fetcher,
|
||||||
|
SigningKey: testSigningKey,
|
||||||
|
Allowlist: []string{allowlistedHost},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to create service: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
h := &Handlers{imgSvc: svc, log: slog.New(slog.DiscardHandler)}
|
||||||
|
|
||||||
|
r := chi.NewRouter()
|
||||||
|
r.Get("/v1/image/*", h.HandleImage())
|
||||||
|
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
// newPhotoFetcher returns a mock fetcher that serves a JPEG at photoPath on
|
||||||
|
// each of hosts, and answers any other URL with an upstream error.
|
||||||
|
func newPhotoFetcher(t *testing.T, hosts ...string) *httpfetcher.MockFetcher {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
photo := &fstest.MapFile{
|
||||||
|
Data: generateTestJPEG(t, 100, 100, color.RGBA{255, 0, 0, 255}),
|
||||||
|
}
|
||||||
|
|
||||||
|
files := fstest.MapFS{}
|
||||||
|
for _, host := range hosts {
|
||||||
|
files[host+photoPath] = photo
|
||||||
|
}
|
||||||
|
|
||||||
|
return httpfetcher.NewMock(files)
|
||||||
|
}
|
||||||
|
|
||||||
|
// photoURL returns the image route URL of photoPath on host, as a 50x50 JPEG.
|
||||||
|
func photoURL(host string) string {
|
||||||
|
return "/v1/image/" + host + photoPath + "/50x50.jpeg"
|
||||||
|
}
|
||||||
|
|
||||||
|
// photoURLWithSig returns photoURL(host) with sig and expires as its sig and
|
||||||
|
// exp.
|
||||||
|
func photoURLWithSig(host, sig string, expires time.Time) string {
|
||||||
|
return fmt.Sprintf("%s?sig=%s&exp=%d", photoURL(host), sig, expires.Unix())
|
||||||
|
}
|
||||||
|
|
||||||
|
// photoSignature returns the signature of photoURL(host) at the default
|
||||||
|
// quality and fit, made with key and expiring at expires.
|
||||||
|
func photoSignature(key, host string, expires time.Time) string {
|
||||||
|
return signature.New(key).Sign(&signature.Request{
|
||||||
|
SourceHost: host,
|
||||||
|
SourcePath: photoPath,
|
||||||
|
Width: 50,
|
||||||
|
Height: 50,
|
||||||
|
Format: string(imgcache.FormatJPEG),
|
||||||
|
Quality: 85,
|
||||||
|
FitMode: string(imgcache.FitCover),
|
||||||
|
Expires: expires,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// sendGet sends a GET for target to route and returns the response.
|
||||||
|
func sendGet(
|
||||||
|
t *testing.T, route http.Handler, target string,
|
||||||
|
) *httptest.ResponseRecorder {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, target, nil)
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
|
||||||
|
route.ServeHTTP(rec, req)
|
||||||
|
t.Logf("GET %s: %d", target, rec.Code)
|
||||||
|
|
||||||
|
return rec
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkErrorBody checks that rec has status wantStatus and the JSON error body
|
||||||
|
// the image route sends: wantError, wantStatus and the time in RFC 3339.
|
||||||
|
func checkErrorBody(
|
||||||
|
t *testing.T, rec *httptest.ResponseRecorder, wantStatus int, wantError string,
|
||||||
|
) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
if rec.Code != wantStatus {
|
||||||
|
t.Errorf("status = %d, want %d", rec.Code, wantStatus)
|
||||||
|
}
|
||||||
|
|
||||||
|
if ct := rec.Header().Get("Content-Type"); ct != "application/json" {
|
||||||
|
t.Errorf("Content-Type = %q, want application/json", ct)
|
||||||
|
}
|
||||||
|
|
||||||
|
var body struct {
|
||||||
|
Error string `json:"error"`
|
||||||
|
Status int `json:"status"`
|
||||||
|
Timestamp string `json:"timestamp"`
|
||||||
|
}
|
||||||
|
|
||||||
|
err := json.NewDecoder(rec.Body).Decode(&body)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("decoding response body: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if body.Error != wantError || body.Status != wantStatus {
|
||||||
|
t.Errorf("body error and status = %q %d, want %q %d",
|
||||||
|
body.Error, body.Status, wantError, wantStatus)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = time.Parse(time.RFC3339, body.Timestamp)
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("body timestamp: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestHandleImage_ErrorAnswers checks the status and the JSON error body the
|
||||||
|
// image route answers each request below with. The JPEG at photoPath exists on
|
||||||
|
// signedHost and on each host below that differs from it, so a request refused
|
||||||
|
// with 401 would otherwise be served.
|
||||||
|
func TestHandleImage_ErrorAnswers(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
// A signature for signedHost must not verify for any of these.
|
||||||
|
parentHost := "example.com"
|
||||||
|
siblingHost := "other.example.com"
|
||||||
|
subdomainHost := "img." + signedHost
|
||||||
|
appendedHost := signedHost + ".example.net"
|
||||||
|
|
||||||
|
photos := newPhotoFetcher(t,
|
||||||
|
signedHost, parentHost, siblingHost, subdomainHost, appendedHost)
|
||||||
|
// The real fetcher refuses localhost before any lookup or connection.
|
||||||
|
realFetcher := httpfetcher.New(httpfetcher.DefaultConfig())
|
||||||
|
|
||||||
|
exp := time.Now().Add(time.Hour)
|
||||||
|
expired := time.Now().Add(-time.Hour)
|
||||||
|
sig := photoSignature(testSigningKey, signedHost, exp)
|
||||||
|
otherKeySig := photoSignature("another-signing-key", signedHost, exp)
|
||||||
|
expiredSig := photoSignature(testSigningKey, signedHost, expired)
|
||||||
|
localhostSig := photoSignature(testSigningKey, "localhost", exp)
|
||||||
|
|
||||||
|
// The error every request refused for its signature gets.
|
||||||
|
const unauthorized = "unauthorized"
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
fetcher httpfetcher.Fetcher
|
||||||
|
target string
|
||||||
|
wantStatus int
|
||||||
|
wantError string
|
||||||
|
}{
|
||||||
|
{"no sig or exp", photos, photoURL(signedHost),
|
||||||
|
http.StatusUnauthorized, unauthorized},
|
||||||
|
{"exp but no sig", photos,
|
||||||
|
fmt.Sprintf("%s?exp=%d", photoURL(signedHost), exp.Unix()),
|
||||||
|
http.StatusUnauthorized, unauthorized},
|
||||||
|
{"sig made with another key", photos,
|
||||||
|
photoURLWithSig(signedHost, otherKeySig, exp),
|
||||||
|
http.StatusUnauthorized, unauthorized},
|
||||||
|
{"sig without its = padding", photos,
|
||||||
|
photoURLWithSig(signedHost, strings.TrimRight(sig, "="), exp),
|
||||||
|
http.StatusUnauthorized, unauthorized},
|
||||||
|
{"sig in upper case", photos,
|
||||||
|
photoURLWithSig(signedHost, strings.ToUpper(sig), exp),
|
||||||
|
http.StatusUnauthorized, unauthorized},
|
||||||
|
{"expired sig", photos, photoURLWithSig(signedHost, expiredSig, expired),
|
||||||
|
http.StatusUnauthorized, unauthorized},
|
||||||
|
{"sig sent for the parent domain", photos,
|
||||||
|
photoURLWithSig(parentHost, sig, exp),
|
||||||
|
http.StatusUnauthorized, unauthorized},
|
||||||
|
{"sig sent for a sibling host", photos,
|
||||||
|
photoURLWithSig(siblingHost, sig, exp),
|
||||||
|
http.StatusUnauthorized, unauthorized},
|
||||||
|
{"sig sent for a subdomain", photos,
|
||||||
|
photoURLWithSig(subdomainHost, sig, exp),
|
||||||
|
http.StatusUnauthorized, unauthorized},
|
||||||
|
{"sig sent with another domain appended", photos,
|
||||||
|
photoURLWithSig(appendedHost, sig, exp),
|
||||||
|
http.StatusUnauthorized, unauthorized},
|
||||||
|
{"unparseable path", photos,
|
||||||
|
"/v1/image/" + allowlistedHost + photoPath + "/big.jpeg",
|
||||||
|
http.StatusBadRequest, "invalid image URL: invalid size format"},
|
||||||
|
{"blocked upstream address", realFetcher,
|
||||||
|
photoURLWithSig("localhost", localhostSig, exp),
|
||||||
|
http.StatusForbidden, "forbidden"},
|
||||||
|
{"upstream error", photos,
|
||||||
|
"/v1/image/" + allowlistedHost + "/images/missing.jpg/50x50.jpeg",
|
||||||
|
http.StatusBadGateway, "upstream error"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
rec := sendGet(t, newImageRoute(t, tt.fetcher), tt.target)
|
||||||
|
checkErrorBody(t, rec, tt.wantStatus, tt.wantError)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestHandleImage_AllowlistOrSignature checks that the image route serves an
|
||||||
|
// image without a signature for a host on the allowlist only, and for another
|
||||||
|
// host only with a valid signature.
|
||||||
|
func TestHandleImage_AllowlistOrSignature(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
photos := newPhotoFetcher(t, allowlistedHost, signedHost)
|
||||||
|
exp := time.Now().Add(time.Hour)
|
||||||
|
sig := photoSignature(testSigningKey, signedHost, exp)
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
target string
|
||||||
|
wantStatus int
|
||||||
|
}{
|
||||||
|
{"allowlisted host, no sig", photoURL(allowlistedHost), http.StatusOK},
|
||||||
|
{"other host, no sig", photoURL(signedHost), http.StatusUnauthorized},
|
||||||
|
{"other host, valid sig", photoURLWithSig(signedHost, sig, exp),
|
||||||
|
http.StatusOK},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
rec := sendGet(t, newImageRoute(t, photos), tt.target)
|
||||||
|
if rec.Code != tt.wantStatus {
|
||||||
|
t.Errorf("status = %d, want %d", rec.Code, tt.wantStatus)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/go-chi/chi/v5"
|
"github.com/go-chi/chi/v5"
|
||||||
|
"github.com/go-chi/chi/v5/middleware"
|
||||||
|
|
||||||
"sneak.berlin/go/pixa/internal/encurl"
|
"sneak.berlin/go/pixa/internal/encurl"
|
||||||
"sneak.berlin/go/pixa/internal/httpfetcher"
|
"sneak.berlin/go/pixa/internal/httpfetcher"
|
||||||
@@ -24,43 +25,8 @@ func (s *Handlers) HandleImageEnc() http.HandlerFunc {
|
|||||||
ctx := r.Context()
|
ctx := r.Context()
|
||||||
start := time.Now()
|
start := time.Now()
|
||||||
|
|
||||||
// Extract token from URL
|
req, ok := s.parseImageEncRequest(w, r)
|
||||||
token := chi.URLParam(r, "token")
|
if !ok {
|
||||||
if token == "" {
|
|
||||||
s.respondError(w, "missing token", http.StatusBadRequest)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Decrypt and validate the payload
|
|
||||||
payload, err := s.encGen.Parse(token)
|
|
||||||
if err != nil {
|
|
||||||
if errors.Is(err, encurl.ErrExpired) {
|
|
||||||
s.log.Debug("encrypted URL expired", "error", err)
|
|
||||||
s.respondError(w, "URL has expired", http.StatusGone)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
s.log.Debug("failed to decrypt URL", "error", err)
|
|
||||||
s.respondError(w, "invalid encrypted URL", http.StatusBadRequest)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Convert payload to ImageRequest
|
|
||||||
req := payload.ToImageRequest()
|
|
||||||
|
|
||||||
// Apply the same dimension and fit-mode bounds as the plain image
|
|
||||||
// route: a sealed payload is trusted for its origin, not for staying
|
|
||||||
// within limits, so an over-limit size or unknown fit mode is a 400
|
|
||||||
// here rather than an out-of-memory or a 500 from the processor.
|
|
||||||
err = imgcache.ValidateImageRequest(req)
|
|
||||||
if err != nil {
|
|
||||||
s.log.Debug("encrypted URL failed validation", "error", err)
|
|
||||||
s.respondError(w, "invalid encrypted URL: "+err.Error(),
|
|
||||||
http.StatusBadRequest)
|
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -94,6 +60,17 @@ func (s *Handlers) HandleImageEnc() http.HandlerFunc {
|
|||||||
w.Header().Set("Cache-Control", cacheControl(req.Expires))
|
w.Header().Set("Cache-Control", cacheControl(req.Expires))
|
||||||
w.Header().Set("X-Pixa-Cache", string(resp.CacheStatus))
|
w.Header().Set("X-Pixa-Cache", string(resp.CacheStatus))
|
||||||
|
|
||||||
|
if notModified(w, r, resp.ETag) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// A HEAD request gets the headers only
|
||||||
|
if r.Method == http.MethodHead {
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
// Stream the response
|
// Stream the response
|
||||||
written, err := io.Copy(w, resp.Content)
|
written, err := io.Copy(w, resp.Content)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -105,6 +82,7 @@ func (s *Handlers) HandleImageEnc() http.HandlerFunc {
|
|||||||
// Log completion
|
// Log completion
|
||||||
duration := time.Since(start)
|
duration := time.Since(start)
|
||||||
s.log.Info("image served",
|
s.log.Info("image served",
|
||||||
|
"request_id", middleware.GetReqID(ctx),
|
||||||
"cache_key", imgcache.CacheKey(req),
|
"cache_key", imgcache.CacheKey(req),
|
||||||
"host", req.SourceHost,
|
"host", req.SourceHost,
|
||||||
"path", req.SourcePath,
|
"path", req.SourcePath,
|
||||||
@@ -116,6 +94,56 @@ func (s *Handlers) HandleImageEnc() http.HandlerFunc {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// parseImageEncRequest decrypts the token of an encrypted image URL into an
|
||||||
|
// ImageRequest and checks it. On a token that is missing, does not decrypt,
|
||||||
|
// has expired or asks for something not valid, it writes an error response
|
||||||
|
// and returns false.
|
||||||
|
func (s *Handlers) parseImageEncRequest(
|
||||||
|
w http.ResponseWriter, r *http.Request,
|
||||||
|
) (*imgcache.ImageRequest, bool) {
|
||||||
|
// Extract token from URL
|
||||||
|
token := chi.URLParam(r, "token")
|
||||||
|
if token == "" {
|
||||||
|
s.respondError(w, "missing token", http.StatusBadRequest)
|
||||||
|
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// Decrypt and validate the payload
|
||||||
|
payload, err := s.encGen.Parse(token)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, encurl.ErrExpired) {
|
||||||
|
s.log.Debug("encrypted URL expired", "error", err)
|
||||||
|
s.respondError(w, "URL has expired", http.StatusGone)
|
||||||
|
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
|
||||||
|
s.log.Debug("failed to decrypt URL", "error", err)
|
||||||
|
s.respondError(w, "invalid encrypted URL", http.StatusBadRequest)
|
||||||
|
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convert payload to ImageRequest
|
||||||
|
req := payload.ToImageRequest()
|
||||||
|
|
||||||
|
// Apply the same dimension and fit-mode bounds as the plain image
|
||||||
|
// route: a sealed payload is trusted for its origin, not for staying
|
||||||
|
// within limits, so an over-limit size or unknown fit mode is a 400
|
||||||
|
// here rather than an out-of-memory or a 500 from the processor.
|
||||||
|
err = imgcache.ValidateImageRequest(req)
|
||||||
|
if err != nil {
|
||||||
|
s.log.Debug("encrypted URL failed validation", "error", err)
|
||||||
|
s.respondError(w, "invalid encrypted URL: "+err.Error(),
|
||||||
|
http.StatusBadRequest)
|
||||||
|
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
|
||||||
|
return req, true
|
||||||
|
}
|
||||||
|
|
||||||
// handleImageError converts image service errors to HTTP responses.
|
// handleImageError converts image service errors to HTTP responses.
|
||||||
func (s *Handlers) handleImageError(w http.ResponseWriter, err error) {
|
func (s *Handlers) handleImageError(w http.ResponseWriter, err error) {
|
||||||
switch {
|
switch {
|
||||||
|
|||||||
@@ -96,3 +96,70 @@ func TestHandleImageEnc_InvalidFitMode_Returns400(t *testing.T) {
|
|||||||
t.Fatalf("status = %d, want %d", rec.Code, http.StatusBadRequest)
|
t.Fatalf("status = %d, want %d", rec.Code, http.StatusBadRequest)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestHandleImageEnc_IfNoneMatch_Returns304 verifies that an image served
|
||||||
|
// through an encrypted URL carries an ETag, and that a request whose
|
||||||
|
// If-None-Match is that ETag is answered 304 Not Modified with no body.
|
||||||
|
func TestHandleImageEnc_IfNoneMatch_Returns304(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
||||||
|
target := encPhotoURL(t, h)
|
||||||
|
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
srv.ServeHTTP(rec, httptest.NewRequestWithContext(
|
||||||
|
t.Context(), http.MethodGet, target, nil))
|
||||||
|
|
||||||
|
etag := rec.Header().Get("ETag")
|
||||||
|
t.Logf("GET: %d, ETag %q", rec.Code, etag)
|
||||||
|
|
||||||
|
if rec.Code != http.StatusOK || etag == "" {
|
||||||
|
t.Fatalf("GET: status = %d, ETag = %q, want %d and an ETag",
|
||||||
|
rec.Code, etag, http.StatusOK)
|
||||||
|
}
|
||||||
|
|
||||||
|
req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, target, nil)
|
||||||
|
req.Header.Set("If-None-Match", etag)
|
||||||
|
|
||||||
|
rec = httptest.NewRecorder()
|
||||||
|
srv.ServeHTTP(rec, req)
|
||||||
|
t.Logf("GET with If-None-Match: %d, %d body bytes", rec.Code, rec.Body.Len())
|
||||||
|
|
||||||
|
if rec.Code != http.StatusNotModified || rec.Body.Len() != 0 {
|
||||||
|
t.Errorf("status = %d with %d body bytes, want %d with none",
|
||||||
|
rec.Code, rec.Body.Len(), http.StatusNotModified)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestHandleImageEnc_HEAD_ReturnsHeadersOnly verifies that HEAD on an
|
||||||
|
// encrypted URL is answered 200 with the headers GET sends and no body.
|
||||||
|
func TestHandleImageEnc_HEAD_ReturnsHeadersOnly(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
h, _ := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
||||||
|
|
||||||
|
r := chi.NewRouter()
|
||||||
|
r.Head("/v1/e/{token}/*", h.HandleImageEnc())
|
||||||
|
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
r.ServeHTTP(rec, httptest.NewRequestWithContext(
|
||||||
|
t.Context(), http.MethodHead, encPhotoURL(t, h), nil))
|
||||||
|
t.Logf("HEAD: %d, headers %v, %d body bytes",
|
||||||
|
rec.Code, rec.Header(), rec.Body.Len())
|
||||||
|
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, name := range []string{
|
||||||
|
"Content-Type", "Content-Length", "Cache-Control", "ETag",
|
||||||
|
} {
|
||||||
|
if rec.Header().Get(name) == "" {
|
||||||
|
t.Errorf("HEAD response has no %s", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if rec.Body.Len() != 0 {
|
||||||
|
t.Errorf("HEAD response body has %d bytes, want none", rec.Body.Len())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,126 @@
|
|||||||
|
package handlers
|
||||||
|
|
||||||
|
import (
|
||||||
|
"image/jpeg"
|
||||||
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"sneak.berlin/go/pixa/internal/encurl"
|
||||||
|
)
|
||||||
|
|
||||||
|
// requireServedPhoto requires that rec answers 200 with the JPEG at photoPath
|
||||||
|
// on signedHost at the 50x50 that encPhotoURL and the generator tests ask for.
|
||||||
|
func requireServedPhoto(t *testing.T, rec *httptest.ResponseRecorder) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("status = %d, want %d; body %q",
|
||||||
|
rec.Code, http.StatusOK, rec.Body.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
contentType := rec.Header().Get("Content-Type")
|
||||||
|
if contentType != "image/jpeg" {
|
||||||
|
t.Errorf("Content-Type = %q, want image/jpeg", contentType)
|
||||||
|
}
|
||||||
|
|
||||||
|
img, err := jpeg.DecodeConfig(rec.Body)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("body is not a JPEG: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if img.Width != 50 || img.Height != 50 {
|
||||||
|
t.Errorf("image is %dx%d, want 50x50", img.Width, img.Height)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestHandleImageEnc_ValidToken_ServesImage verifies that a token made with
|
||||||
|
// the signing key serves the image it asks for.
|
||||||
|
func TestHandleImageEnc_ValidToken_ServesImage(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
||||||
|
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
srv.ServeHTTP(rec, httptest.NewRequestWithContext(
|
||||||
|
t.Context(), http.MethodGet, encPhotoURL(t, h), nil))
|
||||||
|
|
||||||
|
requireServedPhoto(t, rec)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestHandleImageEnc_RejectedToken verifies that a token that has expired
|
||||||
|
// answers 410, and that a token with one character changed, a token cut
|
||||||
|
// short, and a token made with another signing key answer 400. The server
|
||||||
|
// would serve the photo for a token it accepted.
|
||||||
|
func TestHandleImageEnc_RejectedToken(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
||||||
|
|
||||||
|
photo := encurl.Payload{
|
||||||
|
SourceHost: signedHost,
|
||||||
|
SourcePath: photoPath,
|
||||||
|
Width: 50,
|
||||||
|
Height: 50,
|
||||||
|
}
|
||||||
|
|
||||||
|
valid, err := h.encGen.Generate(&photo)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Generate() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
expiredPhoto := photo
|
||||||
|
expiredPhoto.ExpiresAt = time.Now().Add(-time.Minute).Unix()
|
||||||
|
|
||||||
|
expired, err := h.encGen.Generate(&expiredPhoto)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Generate() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
otherGen, err := encurl.NewGenerator("another-signing-key-fedcba9876543210")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("encurl.NewGenerator() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
otherKey, err := otherGen.Generate(&photo)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Generate() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Changing a character in the middle always changes the decoded bytes;
|
||||||
|
// the last character of unpadded base64 can carry unused bits.
|
||||||
|
middle := len(valid) / 2
|
||||||
|
|
||||||
|
replacement := "A"
|
||||||
|
if valid[middle] == 'A' {
|
||||||
|
replacement = "B"
|
||||||
|
}
|
||||||
|
|
||||||
|
changed := valid[:middle] + replacement + valid[middle+1:]
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
token string
|
||||||
|
wantStatus int
|
||||||
|
}{
|
||||||
|
{"expired", expired, http.StatusGone},
|
||||||
|
{"one character changed", changed, http.StatusBadRequest},
|
||||||
|
{"cut short", valid[:middle], http.StatusBadRequest},
|
||||||
|
{"another signing key", otherKey, http.StatusBadRequest},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
rec := getEncToken(srv, tt.token)
|
||||||
|
t.Logf("GET /v1/e/%s/img.jpg: %d %s", tt.token, rec.Code, rec.Body)
|
||||||
|
|
||||||
|
if rec.Code != tt.wantStatus {
|
||||||
|
t.Errorf("status = %d, want %d", rec.Code, tt.wantStatus)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,131 @@
|
|||||||
|
package handlers
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/go-chi/chi/v5/middleware"
|
||||||
|
|
||||||
|
"sneak.berlin/go/pixa/internal/encurl"
|
||||||
|
"sneak.berlin/go/pixa/internal/imgcache"
|
||||||
|
)
|
||||||
|
|
||||||
|
// signedPhotoURL returns a signed /v1/image/ URL, valid for a minute, for the
|
||||||
|
// JPEG at photoPath on signedHost at 50x50, made with h's image service.
|
||||||
|
func signedPhotoURL(t *testing.T, h *Handlers) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
signedURL, err := h.imgSvc.GenerateSignedURL("", &imgcache.ImageRequest{
|
||||||
|
SourceHost: signedHost,
|
||||||
|
SourcePath: photoPath,
|
||||||
|
Size: imgcache.Size{Width: 50, Height: 50},
|
||||||
|
Format: imgcache.FormatJPEG,
|
||||||
|
}, time.Minute)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GenerateSignedURL() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return signedURL
|
||||||
|
}
|
||||||
|
|
||||||
|
// encPhotoURL returns an encrypted /v1/e/ URL, which never expires, for the
|
||||||
|
// JPEG at photoPath on signedHost at 50x50, made with h's generator.
|
||||||
|
func encPhotoURL(t *testing.T, h *Handlers) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
token, err := h.encGen.Generate(&encurl.Payload{
|
||||||
|
SourceHost: signedHost,
|
||||||
|
SourcePath: photoPath,
|
||||||
|
Width: 50,
|
||||||
|
Height: 50,
|
||||||
|
Format: imgcache.FormatJPEG,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Generate() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return "/v1/e/" + token + "/img.jpg"
|
||||||
|
}
|
||||||
|
|
||||||
|
// requestIDByMessage reads the JSON log lines in logs and returns the
|
||||||
|
// request_id of each line, by its message.
|
||||||
|
func requestIDByMessage(t *testing.T, logs io.Reader) map[string]string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
logged := make(map[string]string)
|
||||||
|
|
||||||
|
dec := json.NewDecoder(logs)
|
||||||
|
for dec.More() {
|
||||||
|
var line map[string]any
|
||||||
|
|
||||||
|
err := dec.Decode(&line)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("decoding log line: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
msg, _ := line["msg"].(string)
|
||||||
|
requestID, _ := line["request_id"].(string)
|
||||||
|
logged[msg] = requestID
|
||||||
|
}
|
||||||
|
|
||||||
|
return logged
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestImageLogLinesCarryRequestID verifies that the lines logged when an image
|
||||||
|
// is fetched, converted and served through either image route carry the
|
||||||
|
// request's ID as request_id, as the request log line does, so they can be
|
||||||
|
// found from it.
|
||||||
|
func TestImageLogLinesCarryRequestID(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const requestID = "test-request-id"
|
||||||
|
|
||||||
|
imageURLs := map[string]func(*testing.T, *Handlers) string{
|
||||||
|
"/v1/image/": signedPhotoURL,
|
||||||
|
"/v1/e/": encPhotoURL,
|
||||||
|
}
|
||||||
|
|
||||||
|
for route, imageURL := range imageURLs {
|
||||||
|
t.Run(route, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var logs bytes.Buffer
|
||||||
|
|
||||||
|
h, srv := newSignedHostServer(t,
|
||||||
|
slog.New(slog.NewJSONHandler(&logs, nil)))
|
||||||
|
|
||||||
|
ctx := context.WithValue(t.Context(),
|
||||||
|
middleware.RequestIDKey, requestID)
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
srv.ServeHTTP(rec, httptest.NewRequestWithContext(
|
||||||
|
ctx, http.MethodGet, imageURL(t, h), nil))
|
||||||
|
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("logged:\n%s", logs.String())
|
||||||
|
|
||||||
|
logged := requestIDByMessage(t, &logs)
|
||||||
|
|
||||||
|
for _, msg := range []string{
|
||||||
|
"upstream fetched", "image converted", "image served",
|
||||||
|
} {
|
||||||
|
got, ok := logged[msg]
|
||||||
|
if !ok {
|
||||||
|
t.Errorf("no %q line logged", msg)
|
||||||
|
} else if got != requestID {
|
||||||
|
t.Errorf("%q line has request_id %q, want %q",
|
||||||
|
msg, got, requestID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
package handlers
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.uber.org/fx/fxtest"
|
||||||
|
"sneak.berlin/go/pixa/internal/config"
|
||||||
|
"sneak.berlin/go/pixa/internal/globals"
|
||||||
|
"sneak.berlin/go/pixa/internal/healthcheck"
|
||||||
|
"sneak.berlin/go/pixa/internal/logger"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestHandleRobotsTxt checks that /robots.txt asks every crawler to stay off
|
||||||
|
// the whole site.
|
||||||
|
func TestHandleRobotsTxt(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
h := &Handlers{log: slog.New(slog.DiscardHandler)}
|
||||||
|
rec := sendGet(t, h.HandleRobotsTxt(), "/robots.txt")
|
||||||
|
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Errorf("status = %d, want %d", rec.Code, http.StatusOK)
|
||||||
|
}
|
||||||
|
|
||||||
|
if ct := rec.Header().Get("Content-Type"); ct != "text/plain" {
|
||||||
|
t.Errorf("Content-Type = %q, want text/plain", ct)
|
||||||
|
}
|
||||||
|
|
||||||
|
want := "User-agent: *\nDisallow: /\n"
|
||||||
|
if rec.Body.String() != want {
|
||||||
|
t.Errorf("body = %q, want %q", rec.Body.String(), want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestHandleHealthCheck checks that the health check answers 200 with status
|
||||||
|
// ok, the app's name and version, now, uptime_seconds, uptime_human and
|
||||||
|
// maintenance_mode, which is true here: the health check stays 200 while
|
||||||
|
// maintenance mode is on.
|
||||||
|
func TestHandleHealthCheck(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
lc := fxtest.NewLifecycle(t)
|
||||||
|
|
||||||
|
log, err := logger.New(lc, logger.Params{Globals: &globals.Globals{}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("logger.New() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
hc, err := healthcheck.New(lc, healthcheck.Params{
|
||||||
|
Globals: &globals.Globals{Appname: "pixad", Version: "v1.2.3"},
|
||||||
|
Config: &config.Config{MaintenanceMode: true},
|
||||||
|
Logger: log,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("healthcheck.New() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
h := &Handlers{hc: hc, log: slog.New(slog.DiscardHandler)}
|
||||||
|
rec := sendGet(t, h.HandleHealthCheck(), "/.well-known/healthcheck.json")
|
||||||
|
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Errorf("status = %d, want %d", rec.Code, http.StatusOK)
|
||||||
|
}
|
||||||
|
|
||||||
|
if ct := rec.Header().Get("Content-Type"); ct != "application/json" {
|
||||||
|
t.Errorf("Content-Type = %q, want application/json", ct)
|
||||||
|
}
|
||||||
|
|
||||||
|
var body map[string]any
|
||||||
|
|
||||||
|
err = json.NewDecoder(rec.Body).Decode(&body)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("decoding response body: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if body["status"] != "ok" || body["appname"] != "pixad" ||
|
||||||
|
body["version"] != "v1.2.3" || body["maintenance_mode"] != true {
|
||||||
|
t.Errorf("body = %v, want status ok, appname pixad, version v1.2.3 "+
|
||||||
|
"and maintenance_mode true", body)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, key := range []string{"now", "uptime_seconds", "uptime_human"} {
|
||||||
|
if _, ok := body[key]; !ok {
|
||||||
|
t.Errorf("body = %v, has no %s", body, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -191,7 +191,23 @@ func fetchBody(t *testing.T, f *HTTPFetcher, path string) string {
|
|||||||
|
|
||||||
// semLen reports how many per-host semaphore slots are currently held.
|
// semLen reports how many per-host semaphore slots are currently held.
|
||||||
func semLen(f *HTTPFetcher, host string) int {
|
func semLen(f *HTTPFetcher, host string) int {
|
||||||
return len(f.getHostSemaphore(host))
|
f.hostSemMu.Lock()
|
||||||
|
defer f.hostSemMu.Unlock()
|
||||||
|
|
||||||
|
sem, ok := f.hostSems[host]
|
||||||
|
if !ok {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
return len(sem.slots)
|
||||||
|
}
|
||||||
|
|
||||||
|
// hostSemCount reports how many hosts have a semaphore in hostSems.
|
||||||
|
func hostSemCount(f *HTTPFetcher) int {
|
||||||
|
f.hostSemMu.Lock()
|
||||||
|
defer f.hostSemMu.Unlock()
|
||||||
|
|
||||||
|
return len(f.hostSems)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFetchRedirectToPrivateIPBlocked(t *testing.T) {
|
func TestFetchRedirectToPrivateIPBlocked(t *testing.T) {
|
||||||
|
|||||||
@@ -18,6 +18,8 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/go-chi/chi/v5/middleware"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Fetcher configuration constants.
|
// Fetcher configuration constants.
|
||||||
@@ -160,10 +162,13 @@ func DefaultConfig() *Config {
|
|||||||
// HTTPFetcher implements Fetcher with SSRF protection and connection limits
|
// HTTPFetcher implements Fetcher with SSRF protection and connection limits
|
||||||
// per host and for all hosts together.
|
// per host and for all hosts together.
|
||||||
type HTTPFetcher struct {
|
type HTTPFetcher struct {
|
||||||
client *http.Client
|
client *http.Client
|
||||||
config *Config
|
config *Config
|
||||||
hostSems map[string]chan struct{} // per-host semaphores
|
// hostSems holds the semaphore of each host with a fetch holding or
|
||||||
hostSemMu sync.Mutex // protects hostSems map
|
// waiting for one of its slots; the entry is removed when the host's
|
||||||
|
// last such fetch gives its slot back or stops waiting.
|
||||||
|
hostSems map[string]*hostSemaphore
|
||||||
|
hostSemMu sync.Mutex // protects hostSems and each entry's count
|
||||||
// allHostsSemaphore has one slot per connection allowed to all hosts
|
// allHostsSemaphore has one slot per connection allowed to all hosts
|
||||||
// together (config.MaxConnections).
|
// together (config.MaxConnections).
|
||||||
allHostsSemaphore chan struct{}
|
allHostsSemaphore chan struct{}
|
||||||
@@ -171,6 +176,14 @@ type HTTPFetcher struct {
|
|||||||
connectionWaitTimeout time.Duration
|
connectionWaitTimeout time.Duration
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// hostSemaphore is one host's connection slots
|
||||||
|
// (config.MaxConnectionsPerHost) and the number of fetches holding or
|
||||||
|
// waiting for one of them.
|
||||||
|
type hostSemaphore struct {
|
||||||
|
slots chan struct{}
|
||||||
|
count int
|
||||||
|
}
|
||||||
|
|
||||||
// New creates a new HTTPFetcher with SSRF protection.
|
// New creates a new HTTPFetcher with SSRF protection.
|
||||||
func New(config *Config) *HTTPFetcher {
|
func New(config *Config) *HTTPFetcher {
|
||||||
if config == nil {
|
if config == nil {
|
||||||
@@ -211,7 +224,7 @@ func New(config *Config) *HTTPFetcher {
|
|||||||
return &HTTPFetcher{
|
return &HTTPFetcher{
|
||||||
client: client,
|
client: client,
|
||||||
config: config,
|
config: config,
|
||||||
hostSems: make(map[string]chan struct{}),
|
hostSems: make(map[string]*hostSemaphore),
|
||||||
allHostsSemaphore: make(chan struct{}, config.MaxConnections),
|
allHostsSemaphore: make(chan struct{}, config.MaxConnections),
|
||||||
connectionWaitTimeout: ConnectionWaitTimeout,
|
connectionWaitTimeout: ConnectionWaitTimeout,
|
||||||
}
|
}
|
||||||
@@ -256,6 +269,13 @@ func (f *HTTPFetcher) Fetch(ctx context.Context, url string) (*FetchResult, erro
|
|||||||
req.Header.Set("User-Agent", f.config.UserAgent)
|
req.Header.Set("User-Agent", f.config.UserAgent)
|
||||||
req.Header.Set("Accept", strings.Join(f.config.AllowedContentTypes, ", "))
|
req.Header.Set("Accept", strings.Join(f.config.AllowedContentTypes, ", "))
|
||||||
|
|
||||||
|
// The ID of the request this fetch serves, so the fetch can be found in
|
||||||
|
// the upstream host's logs
|
||||||
|
requestID := middleware.GetReqID(ctx)
|
||||||
|
if requestID != "" {
|
||||||
|
req.Header.Set(middleware.RequestIDHeader, requestID)
|
||||||
|
}
|
||||||
|
|
||||||
// Use httptrace to capture connection details
|
// Use httptrace to capture connection details
|
||||||
var remoteAddr string
|
var remoteAddr string
|
||||||
|
|
||||||
@@ -307,6 +327,8 @@ func (f *HTTPFetcher) acquireConnection(
|
|||||||
select {
|
select {
|
||||||
case hostSem <- struct{}{}:
|
case hostSem <- struct{}{}:
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
|
f.putHostSemaphore(host)
|
||||||
|
|
||||||
return nil, ctx.Err()
|
return nil, ctx.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -314,32 +336,55 @@ func (f *HTTPFetcher) acquireConnection(
|
|||||||
case f.allHostsSemaphore <- struct{}{}:
|
case f.allHostsSemaphore <- struct{}{}:
|
||||||
case <-time.After(f.connectionWaitTimeout):
|
case <-time.After(f.connectionWaitTimeout):
|
||||||
<-hostSem
|
<-hostSem
|
||||||
|
f.putHostSemaphore(host)
|
||||||
|
|
||||||
return nil, ErrTooManyConnections
|
return nil, ErrTooManyConnections
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
<-hostSem
|
<-hostSem
|
||||||
|
f.putHostSemaphore(host)
|
||||||
|
|
||||||
return nil, ctx.Err()
|
return nil, ctx.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
return func() {
|
return func() {
|
||||||
<-hostSem
|
<-hostSem
|
||||||
|
f.putHostSemaphore(host)
|
||||||
<-f.allHostsSemaphore
|
<-f.allHostsSemaphore
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// getHostSemaphore returns the semaphore for a host, creating it if necessary.
|
// getHostSemaphore returns the semaphore for a host, creating it if
|
||||||
|
// necessary, and counts the caller among the fetches using it. The caller
|
||||||
|
// calls putHostSemaphore once it holds no slot and waits for none.
|
||||||
func (f *HTTPFetcher) getHostSemaphore(host string) chan struct{} {
|
func (f *HTTPFetcher) getHostSemaphore(host string) chan struct{} {
|
||||||
f.hostSemMu.Lock()
|
f.hostSemMu.Lock()
|
||||||
defer f.hostSemMu.Unlock()
|
defer f.hostSemMu.Unlock()
|
||||||
|
|
||||||
sem, ok := f.hostSems[host]
|
sem, ok := f.hostSems[host]
|
||||||
if !ok {
|
if !ok {
|
||||||
sem = make(chan struct{}, f.config.MaxConnectionsPerHost)
|
sem = &hostSemaphore{
|
||||||
|
slots: make(chan struct{}, f.config.MaxConnectionsPerHost),
|
||||||
|
}
|
||||||
f.hostSems[host] = sem
|
f.hostSems[host] = sem
|
||||||
}
|
}
|
||||||
|
|
||||||
return sem
|
sem.count++
|
||||||
|
|
||||||
|
return sem.slots
|
||||||
|
}
|
||||||
|
|
||||||
|
// putHostSemaphore stops counting the caller among the fetches using the
|
||||||
|
// host's semaphore, and removes the semaphore when no fetch uses it.
|
||||||
|
func (f *HTTPFetcher) putHostSemaphore(host string) {
|
||||||
|
f.hostSemMu.Lock()
|
||||||
|
defer f.hostSemMu.Unlock()
|
||||||
|
|
||||||
|
sem := f.hostSems[host]
|
||||||
|
|
||||||
|
sem.count--
|
||||||
|
if sem.count == 0 {
|
||||||
|
delete(f.hostSems, host)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// buildResult validates the upstream response and assembles a FetchResult
|
// buildResult validates the upstream response and assembles a FetchResult
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"net"
|
"net"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
@@ -119,6 +120,98 @@ func TestFetchFreesHostSlotWhenContextEndsWaitingForConnection(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestFetchRemovesIdleHostSemaphores checks that a host's semaphore is
|
||||||
|
// removed once no fetch holds or waits for one of its slots: after 100
|
||||||
|
// concurrent fetches from 50 hosts have all finished, no semaphore is left.
|
||||||
|
func TestFetchRemovesIdleHostSemaphores(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
srv := startUpstream(t)
|
||||||
|
f, _ := newServerFetcher(t, srv, nil)
|
||||||
|
ctx := testContext(t)
|
||||||
|
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
|
||||||
|
for i := range 100 {
|
||||||
|
wg.Go(func() {
|
||||||
|
res, err := f.Fetch(ctx, imageURLOnPort(1+i%50))
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("Fetch() error = %v", err)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
_ = res.Content.Close()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
if n := hostSemCount(f); n != 0 {
|
||||||
|
t.Errorf("%d host semaphores left after every fetch finished, want 0", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestFetchRemovesHostSemaphoreWhenNoConnection checks that a fetch that
|
||||||
|
// ends without a connection leaves no semaphore behind: when it is refused
|
||||||
|
// after waiting for a connection shared by all hosts, when its context ends
|
||||||
|
// while it waits for its host's slot, and when its context ends while it
|
||||||
|
// waits for a connection shared by all hosts, long before the 10 second
|
||||||
|
// wait timeout.
|
||||||
|
func TestFetchRemovesHostSemaphoreWhenNoConnection(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
srv := startUpstream(t)
|
||||||
|
|
||||||
|
cfg := DefaultConfig()
|
||||||
|
cfg.MaxConnections = 1
|
||||||
|
cfg.MaxConnectionsPerHost = 1
|
||||||
|
|
||||||
|
f, _ := newServerFetcher(t, srv, cfg)
|
||||||
|
f.connectionWaitTimeout = 100 * time.Millisecond
|
||||||
|
|
||||||
|
open, err := f.Fetch(testContext(t), imageURLOnPort(81))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("first Fetch() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = f.Fetch(testContext(t), imageURLOnPort(82))
|
||||||
|
if !errors.Is(err, ErrTooManyConnections) {
|
||||||
|
t.Fatalf("Fetch() from another host: error = %v, "+
|
||||||
|
"want ErrTooManyConnections", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
_, err = f.Fetch(ctx, imageURLOnPort(81))
|
||||||
|
if !errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
t.Fatalf("Fetch() from the busy host: error = %v, "+
|
||||||
|
"want context.DeadlineExceeded", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Back to the 10 second wait, so the next fetch's context ends first.
|
||||||
|
f.connectionWaitTimeout = ConnectionWaitTimeout
|
||||||
|
|
||||||
|
ctx, cancel = context.WithTimeout(t.Context(), 100*time.Millisecond)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
_, err = f.Fetch(ctx, imageURLOnPort(83))
|
||||||
|
if !errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
t.Fatalf("Fetch() from a host with nothing open: error = %v, "+
|
||||||
|
"want context.DeadlineExceeded", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = open.Content.Close()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("close first body: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if n := hostSemCount(f); n != 0 {
|
||||||
|
t.Errorf("%d host semaphores left after every fetch finished, want 0", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestFetchReleasesConnectionOnError checks that a fetch that fails after
|
// TestFetchReleasesConnectionOnError checks that a fetch that fails after
|
||||||
// taking its connection gives it back: with MaxConnections at 1, the slot
|
// taking its connection gives it back: with MaxConnections at 1, the slot
|
||||||
// must be free after the failure and the next fetch must succeed.
|
// must be free after the failure and the next fetch must succeed.
|
||||||
|
|||||||
@@ -0,0 +1,50 @@
|
|||||||
|
package httpfetcher
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/go-chi/chi/v5/middleware"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestFetchSendsRequestID verifies that a fetch sends the ID of the request
|
||||||
|
// it serves, which the RequestID middleware stores in the request context,
|
||||||
|
// to the upstream host as X-Request-Id, so the fetch can be found in that
|
||||||
|
// host's logs.
|
||||||
|
func TestFetchSendsRequestID(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const requestID = "test-request-id"
|
||||||
|
|
||||||
|
received := make(chan string, 1)
|
||||||
|
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(
|
||||||
|
func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
received <- r.Header.Get("X-Request-Id")
|
||||||
|
|
||||||
|
w.Header().Set("Content-Type", contentTypeJPEG)
|
||||||
|
_, _ = io.WriteString(w, imagePayload)
|
||||||
|
}))
|
||||||
|
t.Cleanup(srv.Close)
|
||||||
|
|
||||||
|
f, _ := newServerFetcher(t, srv, nil)
|
||||||
|
|
||||||
|
ctx := context.WithValue(testContext(t), middleware.RequestIDKey, requestID)
|
||||||
|
|
||||||
|
res, err := f.Fetch(ctx, upstreamURL("/image"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Fetch() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_ = res.Content.Close()
|
||||||
|
|
||||||
|
got := <-received
|
||||||
|
t.Logf("upstream received X-Request-Id %q", got)
|
||||||
|
|
||||||
|
if got != requestID {
|
||||||
|
t.Errorf("upstream X-Request-Id = %q, want %q", got, requestID)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -11,7 +11,6 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sync"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
lru "github.com/hashicorp/golang-lru/v2"
|
lru "github.com/hashicorp/golang-lru/v2"
|
||||||
@@ -69,11 +68,11 @@ type Cache struct {
|
|||||||
|
|
||||||
// Eviction machinery. The channels are created in NewCache so
|
// Eviction machinery. The channels are created in NewCache so
|
||||||
// stores can signal write pressure without racing StartEviction.
|
// stores can signal write pressure without racing StartEviction.
|
||||||
|
// evictionCancel, set by StartEviction, cancels the eviction
|
||||||
|
// goroutine's context.
|
||||||
evictionPressure chan struct{}
|
evictionPressure chan struct{}
|
||||||
evictionStop chan struct{}
|
|
||||||
evictionDone chan struct{}
|
evictionDone chan struct{}
|
||||||
evictionStarted bool
|
evictionCancel context.CancelFunc
|
||||||
evictionStopOnce sync.Once
|
|
||||||
|
|
||||||
// metaCache holds the content types of the variants most recently
|
// metaCache holds the content types of the variants most recently
|
||||||
// stored or served, so a hit does not read the variant's .meta file.
|
// stored or served, so a hit does not read the variant's .meta file.
|
||||||
@@ -112,7 +111,6 @@ func NewCache(db *sql.DB, config CacheConfig) (*Cache, error) {
|
|||||||
log: log,
|
log: log,
|
||||||
disabled: config.DisableDiskCache,
|
disabled: config.DisableDiskCache,
|
||||||
evictionPressure: make(chan struct{}, 1),
|
evictionPressure: make(chan struct{}, 1),
|
||||||
evictionStop: make(chan struct{}),
|
|
||||||
evictionDone: make(chan struct{}),
|
evictionDone: make(chan struct{}),
|
||||||
metaCache: metaCache,
|
metaCache: metaCache,
|
||||||
contentLocks: newContentLock(),
|
contentLocks: newContentLock(),
|
||||||
|
|||||||
@@ -117,7 +117,10 @@ func (c *Cache) EvictToLimit(ctx context.Context) error {
|
|||||||
|
|
||||||
// evictBatch fetches one batch of LRU candidates across variants and
|
// evictBatch fetches one batch of LRU candidates across variants and
|
||||||
// source blobs and evicts them oldest-first until excessBytes are
|
// source blobs and evicts them oldest-first until excessBytes are
|
||||||
// freed or the batch is exhausted. It returns the bytes freed.
|
// freed or the batch is exhausted. It returns the bytes freed. A
|
||||||
|
// candidate that fails once ctx is cancelled (every one started after
|
||||||
|
// that fails at its first database call) ends the batch with ctx's
|
||||||
|
// error, without a warning.
|
||||||
func (c *Cache) evictBatch(ctx context.Context, excessBytes int64) (int64, error) {
|
func (c *Cache) evictBatch(ctx context.Context, excessBytes int64) (int64, error) {
|
||||||
candidates, err := c.evictionCandidates(ctx)
|
candidates, err := c.evictionCandidates(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -133,6 +136,10 @@ func (c *Cache) evictBatch(ctx context.Context, excessBytes int64) (int64, error
|
|||||||
|
|
||||||
err := c.evictCandidate(ctx, candidate)
|
err := c.evictCandidate(ctx, candidate)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return freed, ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
c.log.Warn("failed to evict cache entry",
|
c.log.Warn("failed to evict cache entry",
|
||||||
"cache_key", candidate.cacheKey,
|
"cache_key", candidate.cacheKey,
|
||||||
"content_hash", candidate.contentHash,
|
"content_hash", candidate.contentHash,
|
||||||
@@ -283,7 +290,7 @@ func (c *Cache) evictVariant(ctx context.Context, cacheKey VariantKey) error {
|
|||||||
|
|
||||||
c.metaCache.Remove(cacheKey)
|
c.metaCache.Remove(cacheKey)
|
||||||
|
|
||||||
err = c.variants.DeleteWithMeta(cacheKey)
|
err = c.variants.Delete(cacheKey)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -424,37 +431,44 @@ func (c *Cache) notifyWritePressure() {
|
|||||||
// startup and again on every periodic tick thereafter, and evicts to
|
// startup and again on every periodic tick thereafter, and evicts to
|
||||||
// the configured limit on the given periodic interval and on
|
// the configured limit on the given periodic interval and on
|
||||||
// write-pressure notifications. It is a no-op on a disabled cache or
|
// write-pressure notifications. It is a no-op on a disabled cache or
|
||||||
// when already started.
|
// when already started. The goroutine outlives the caller, so it runs
|
||||||
|
// with its own context, which StopEviction cancels.
|
||||||
func (c *Cache) StartEviction(interval time.Duration) {
|
func (c *Cache) StartEviction(interval time.Duration) {
|
||||||
if c.disabled || c.evictionStarted {
|
if c.disabled || c.evictionCancel != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
c.evictionStarted = true
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
c.evictionCancel = cancel
|
||||||
|
|
||||||
go c.evictionLoop(interval)
|
go c.evictionLoop(ctx, interval)
|
||||||
}
|
}
|
||||||
|
|
||||||
// StopEviction stops the background eviction goroutine and waits for
|
// StopEviction cancels the background eviction goroutine, which
|
||||||
// it to exit. It is safe to call when eviction was never started, and
|
// interrupts a pass in progress, and waits for it to exit or for ctx to
|
||||||
// safe to call more than once.
|
// end, whichever comes first. In the second case it returns an error
|
||||||
func (c *Cache) StopEviction() {
|
// wrapping ctx's error. It is safe to call when eviction was never
|
||||||
if !c.evictionStarted {
|
// started, and safe to call more than once.
|
||||||
return
|
func (c *Cache) StopEviction(ctx context.Context) error {
|
||||||
|
if c.evictionCancel == nil {
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
c.evictionStopOnce.Do(func() {
|
c.evictionCancel()
|
||||||
close(c.evictionStop)
|
|
||||||
<-c.evictionDone
|
select {
|
||||||
})
|
case <-c.evictionDone:
|
||||||
|
return nil
|
||||||
|
case <-ctx.Done():
|
||||||
|
return fmt.Errorf("cache eviction still running: %w", ctx.Err())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// evictionLoop is the body of the background eviction goroutine.
|
// evictionLoop is the body of the background eviction goroutine. It
|
||||||
func (c *Cache) evictionLoop(interval time.Duration) {
|
// returns when ctx is cancelled, and starts no pass after that.
|
||||||
|
func (c *Cache) evictionLoop(ctx context.Context, interval time.Duration) {
|
||||||
defer close(c.evictionDone)
|
defer close(c.evictionDone)
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
c.runReconciliationPass(ctx)
|
c.runReconciliationPass(ctx)
|
||||||
c.runEvictionPass(ctx)
|
c.runEvictionPass(ctx)
|
||||||
|
|
||||||
@@ -463,7 +477,7 @@ func (c *Cache) evictionLoop(interval time.Duration) {
|
|||||||
|
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-c.evictionStop:
|
case <-ctx.Done():
|
||||||
return
|
return
|
||||||
case <-ticker.C:
|
case <-ticker.C:
|
||||||
// Reconciliation walks the cache directories, so it only
|
// Reconciliation walks the cache directories, so it only
|
||||||
@@ -484,8 +498,13 @@ func (c *Cache) evictionLoop(interval time.Duration) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// runEvictionPass runs one eviction pass, logging failures instead of
|
// runEvictionPass runs one eviction pass, logging failures instead of
|
||||||
// propagating them (the loop must keep running).
|
// propagating them (the loop must keep running). It does nothing once
|
||||||
|
// ctx is cancelled.
|
||||||
func (c *Cache) runEvictionPass(ctx context.Context) {
|
func (c *Cache) runEvictionPass(ctx context.Context) {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
err := c.EvictToLimit(ctx)
|
err := c.EvictToLimit(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
c.log.Warn("cache eviction pass failed", "error", err)
|
c.log.Warn("cache eviction pass failed", "error", err)
|
||||||
@@ -493,8 +512,13 @@ func (c *Cache) runEvictionPass(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// runReconciliationPass runs one reconciliation pass, logging failures
|
// runReconciliationPass runs one reconciliation pass, logging failures
|
||||||
// instead of propagating them (the loop must keep running).
|
// instead of propagating them (the loop must keep running). It does
|
||||||
|
// nothing once ctx is cancelled.
|
||||||
func (c *Cache) runReconciliationPass(ctx context.Context) {
|
func (c *Cache) runReconciliationPass(ctx context.Context) {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
err := c.reconcileAccounting(ctx)
|
err := c.reconcileAccounting(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
c.log.Warn("cache accounting reconciliation failed", "error", err)
|
c.log.Warn("cache accounting reconciliation failed", "error", err)
|
||||||
@@ -511,7 +535,8 @@ func (c *Cache) runReconciliationPass(ctx context.Context) {
|
|||||||
// know (and rows whose files are gone), and sweeps stale temp files
|
// know (and rows whose files are gone), and sweeps stale temp files
|
||||||
// left behind by crashed writes. Running it periodically, not just
|
// left behind by crashed writes. Running it periodically, not just
|
||||||
// once, bounds how long such drift can accumulate unaccounted for on a
|
// once, bounds how long such drift can accumulate unaccounted for on a
|
||||||
// long-running process to one eviction interval.
|
// long-running process to one eviction interval. Once ctx is cancelled,
|
||||||
|
// it stops at the next file or row and returns ctx's error.
|
||||||
func (c *Cache) reconcileAccounting(ctx context.Context) error {
|
func (c *Cache) reconcileAccounting(ctx context.Context) error {
|
||||||
if c.disabled {
|
if c.disabled {
|
||||||
return nil
|
return nil
|
||||||
@@ -546,6 +571,10 @@ func (c *Cache) reconcileVariantFiles(ctx context.Context) error {
|
|||||||
return filepath.WalkDir(
|
return filepath.WalkDir(
|
||||||
c.variants.baseDir,
|
c.variants.baseDir,
|
||||||
func(path string, entry fs.DirEntry, err error) error {
|
func(path string, entry fs.DirEntry, err error) error {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
if err != nil || entry.IsDir() {
|
if err != nil || entry.IsDir() {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -634,6 +663,10 @@ func (c *Cache) reconcileVariantRows(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, key := range keys {
|
for _, key := range keys {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
if c.variants.Exists(key) {
|
if c.variants.Exists(key) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -699,6 +732,10 @@ func (c *Cache) reconcileSourceFiles(ctx context.Context) error {
|
|||||||
return filepath.WalkDir(
|
return filepath.WalkDir(
|
||||||
c.srcContent.baseDir,
|
c.srcContent.baseDir,
|
||||||
func(path string, entry fs.DirEntry, err error) error {
|
func(path string, entry fs.DirEntry, err error) error {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
if err != nil || entry.IsDir() {
|
if err != nil || entry.IsDir() {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -760,6 +797,10 @@ func (c *Cache) reconcileSourceRows(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, hash := range hashes {
|
for _, hash := range hashes {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
if c.srcContent.Exists(hash) {
|
if c.srcContent.Exists(hash) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,9 +4,12 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
|
"errors"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
|
"log/slog"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -657,7 +660,7 @@ func TestEvictionRunsUnderWritePressure(t *testing.T) {
|
|||||||
// An interval far longer than the test ensures only write
|
// An interval far longer than the test ensures only write
|
||||||
// pressure can trigger eviction here.
|
// pressure can trigger eviction here.
|
||||||
cache.StartEviction(time.Hour)
|
cache.StartEviction(time.Hour)
|
||||||
defer cache.StopEviction()
|
defer func() { _ = cache.StopEviction(t.Context()) }()
|
||||||
|
|
||||||
keys := []VariantKey{
|
keys := []VariantKey{
|
||||||
testVariantKeyOne, testVariantKeyTwo, testVariantKeyThree,
|
testVariantKeyOne, testVariantKeyTwo, testVariantKeyThree,
|
||||||
@@ -678,6 +681,11 @@ func TestEvictionRunsUnderWritePressure(t *testing.T) {
|
|||||||
assertNoDanglingReferences(t, cache)
|
assertNoDanglingReferences(t, cache)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestEvictionRunsOnPeriodicSchedule writes three variant files straight
|
||||||
|
// to disk, bypassing StoreVariant, so they have no accounting rows and no
|
||||||
|
// write-pressure notification fires. Only a periodic reconciliation pass
|
||||||
|
// can then adopt them, and only the eviction pass that follows it can
|
||||||
|
// evict them.
|
||||||
func TestEvictionRunsOnPeriodicSchedule(t *testing.T) {
|
func TestEvictionRunsOnPeriodicSchedule(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -685,12 +693,28 @@ func TestEvictionRunsOnPeriodicSchedule(t *testing.T) {
|
|||||||
|
|
||||||
cache, _ := newEvictionTestCache(t, limit)
|
cache, _ := newEvictionTestCache(t, limit)
|
||||||
|
|
||||||
// Start the evictor while the cache is empty, then create tracked
|
// Hold the test database's only connection, so the startup pass
|
||||||
// over-limit state WITHOUT going through the store methods, so no
|
// waits for it after walking the still empty variant directory: the
|
||||||
// write-pressure notification fires and only the periodic ticker
|
// files written while it waits are first seen by a periodic pass.
|
||||||
// can trigger eviction.
|
conn, err := cache.db.Conn(t.Context())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to take the database connection: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
defer func() { _ = conn.Close() }()
|
||||||
|
|
||||||
cache.StartEviction(100 * time.Millisecond)
|
cache.StartEviction(100 * time.Millisecond)
|
||||||
defer cache.StopEviction()
|
defer func() { _ = cache.StopEviction(t.Context()) }()
|
||||||
|
|
||||||
|
deadline := time.Now().Add(5 * time.Second)
|
||||||
|
|
||||||
|
for cache.db.Stats().WaitCount == 0 {
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatal("the startup pass never waited for the database")
|
||||||
|
}
|
||||||
|
|
||||||
|
time.Sleep(10 * time.Millisecond)
|
||||||
|
}
|
||||||
|
|
||||||
keys := []VariantKey{
|
keys := []VariantKey{
|
||||||
testVariantKeyOne, testVariantKeyTwo, testVariantKeyThree,
|
testVariantKeyOne, testVariantKeyTwo, testVariantKeyThree,
|
||||||
@@ -700,25 +724,43 @@ func TestEvictionRunsOnPeriodicSchedule(t *testing.T) {
|
|||||||
for i, key := range keys {
|
for i, key := range keys {
|
||||||
content := bytes.Repeat([]byte{fills[i]}, 1000)
|
content := bytes.Repeat([]byte{fills[i]}, 1000)
|
||||||
|
|
||||||
_, err := cache.variants.Store(key, bytes.NewReader(content), "image/webp")
|
_, err = cache.variants.Store(key, bytes.NewReader(content), "image/webp")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to store variant file: %v", err)
|
t.Fatalf("failed to store variant file: %v", err)
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
_, err = cache.db.ExecContext(t.Context(),
|
_ = conn.Close()
|
||||||
`INSERT INTO variant_content (cache_key, size_bytes, content_type)
|
|
||||||
VALUES (?, ?, ?)`,
|
// Only one of the 1000-byte files fits under the limit: wait until
|
||||||
string(key), len(content), "image/webp",
|
// the evictor has removed the other two.
|
||||||
)
|
stored := len(keys)
|
||||||
if err != nil {
|
deadline = time.Now().Add(5 * time.Second)
|
||||||
t.Fatalf("failed to insert variant accounting row: %v", err)
|
|
||||||
|
for stored > 1 && time.Now().Before(deadline) {
|
||||||
|
time.Sleep(25 * time.Millisecond)
|
||||||
|
|
||||||
|
stored = 0
|
||||||
|
|
||||||
|
for _, key := range keys {
|
||||||
|
if cache.variants.Exists(key) {
|
||||||
|
stored++
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
usage := waitForUsageAtOrBelow(t, cache, limit, 5*time.Second)
|
if stored > 1 {
|
||||||
|
t.Fatalf("periodic schedule did not trigger eviction: %d of %d "+
|
||||||
|
"variant files still on disk, want at most 1", stored, len(keys))
|
||||||
|
}
|
||||||
|
|
||||||
|
usage, err := cache.UsageBytes(t.Context())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("UsageBytes failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
if usage > limit {
|
if usage > limit {
|
||||||
t.Errorf("periodic schedule did not trigger eviction: usage = %d, want <= %d",
|
t.Errorf("usage after eviction = %d, want <= %d", usage, limit)
|
||||||
usage, limit)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
assertNoDanglingReferences(t, cache)
|
assertNoDanglingReferences(t, cache)
|
||||||
@@ -751,7 +793,7 @@ func TestStartEvictionReconcilesAccountingWithDisk(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
cache.StartEviction(time.Hour)
|
cache.StartEviction(time.Hour)
|
||||||
defer cache.StopEviction()
|
defer func() { _ = cache.StopEviction(t.Context()) }()
|
||||||
|
|
||||||
deadline := time.Now().Add(5 * time.Second)
|
deadline := time.Now().Add(5 * time.Second)
|
||||||
|
|
||||||
@@ -808,7 +850,7 @@ func TestPeriodicReconciliationAdoptsFileThatAppearsAfterStartup(t *testing.T) {
|
|||||||
const interval = 100 * time.Millisecond
|
const interval = 100 * time.Millisecond
|
||||||
|
|
||||||
cache.StartEviction(interval)
|
cache.StartEviction(interval)
|
||||||
defer cache.StopEviction()
|
defer func() { _ = cache.StopEviction(t.Context()) }()
|
||||||
|
|
||||||
// Let startup reconciliation run and settle on an empty cache
|
// Let startup reconciliation run and settle on an empty cache
|
||||||
// before introducing the untracked file, so the adoption we assert
|
// before introducing the untracked file, so the adoption we assert
|
||||||
@@ -862,6 +904,242 @@ func TestPeriodicReconciliationAdoptsFileThatAppearsAfterStartup(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestStopEvictionInterruptsPassInProgress holds the test database's
|
||||||
|
// only connection, so the startup reconciliation pass waits for it, and
|
||||||
|
// checks that StopEviction stops that pass instead of waiting for the
|
||||||
|
// connection to come free, and that the stop logs one warning: the
|
||||||
|
// interrupted reconciliation's, with no eviction pass started after it.
|
||||||
|
func TestStopEvictionInterruptsPassInProgress(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
cache, _ := newEvictionTestCache(t, 1<<30)
|
||||||
|
|
||||||
|
var logBuf bytes.Buffer
|
||||||
|
|
||||||
|
cache.log = slog.New(slog.NewJSONHandler(&logBuf, nil))
|
||||||
|
|
||||||
|
conn, err := cache.db.Conn(t.Context())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to take the database connection: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
defer func() { _ = conn.Close() }()
|
||||||
|
|
||||||
|
cache.StartEviction(time.Hour)
|
||||||
|
|
||||||
|
// The pass is in progress once it waits for the connection.
|
||||||
|
deadline := time.Now().Add(5 * time.Second)
|
||||||
|
|
||||||
|
for cache.db.Stats().WaitCount == 0 {
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatal("the reconciliation pass never waited for the database")
|
||||||
|
}
|
||||||
|
|
||||||
|
time.Sleep(10 * time.Millisecond)
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
err = cache.StopEviction(ctx)
|
||||||
|
t.Logf("StopEviction() error = %v", err)
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("StopEviction() error = %v, want nil: the pass waiting for "+
|
||||||
|
"the database did not stop", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("log output: %s", logBuf.String())
|
||||||
|
|
||||||
|
warnings := strings.Count(logBuf.String(), `"level":"WARN"`)
|
||||||
|
if warnings != 1 {
|
||||||
|
t.Errorf("the stop logged %d warnings, want 1", warnings)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestEvictToLimitStopsAtNextCandidateOnceCancelled cancels the context
|
||||||
|
// while the oldest of three source blobs is being evicted, and checks that
|
||||||
|
// EvictToLimit then returns context.Canceled without evicting the other
|
||||||
|
// two or logging a warning for either of them.
|
||||||
|
func TestEvictToLimitStopsAtNextCandidateOnceCancelled(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
cache, _ := newEvictionTestCache(t, 1)
|
||||||
|
|
||||||
|
var logBuf bytes.Buffer
|
||||||
|
|
||||||
|
cache.log = slog.New(slog.NewJSONHandler(&logBuf, nil))
|
||||||
|
|
||||||
|
hashes := []ContentHash{
|
||||||
|
storeEvictionTestSource(t, cache, "cancel.example.com", "/a.jpg",
|
||||||
|
bytes.Repeat([]byte{0x61}, 1000)),
|
||||||
|
storeEvictionTestSource(t, cache, "cancel.example.com", "/b.jpg",
|
||||||
|
bytes.Repeat([]byte{0x62}, 1000)),
|
||||||
|
storeEvictionTestSource(t, cache, "cancel.example.com", "/c.jpg",
|
||||||
|
bytes.Repeat([]byte{0x63}, 1000)),
|
||||||
|
}
|
||||||
|
|
||||||
|
base := time.Now().Add(-time.Hour)
|
||||||
|
|
||||||
|
for i, hash := range hashes {
|
||||||
|
setSourceLastAccessed(t, cache, hash, base.Add(time.Duration(i)*time.Minute))
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(t.Context())
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
cache.evictSourceBlobTestHook = func(ContentHash) { cancel() }
|
||||||
|
|
||||||
|
err := cache.EvictToLimit(ctx)
|
||||||
|
t.Logf("EvictToLimit() error = %v", err)
|
||||||
|
t.Logf("log output: %s", logBuf.String())
|
||||||
|
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Errorf("EvictToLimit() error = %v, want context.Canceled", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if cache.srcContent.Exists(hashes[0]) {
|
||||||
|
t.Errorf("source blob %s, evicted when the context was cancelled, "+
|
||||||
|
"is still on disk", hashes[0])
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, hash := range hashes[1:] {
|
||||||
|
if !cache.srcContent.Exists(hash) {
|
||||||
|
t.Errorf("source blob %s was evicted after the context was cancelled", hash)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if strings.Contains(logBuf.String(), `"level":"WARN"`) {
|
||||||
|
t.Errorf("EvictToLimit logged a warning after the context was cancelled")
|
||||||
|
}
|
||||||
|
|
||||||
|
assertNoDanglingReferences(t, cache)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestStopEvictionReturnsWhenItsContextEnds pauses an eviction pass where
|
||||||
|
// cancellation cannot reach it, after a source blob's rows are deleted and
|
||||||
|
// before its file is removed, and checks that StopEviction returns its
|
||||||
|
// context's error when that context ends instead of waiting for the pass.
|
||||||
|
// Once the pass goes on, the goroutine exits and no row points at a
|
||||||
|
// missing file.
|
||||||
|
func TestStopEvictionReturnsWhenItsContextEnds(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
cache, _ := newEvictionTestCache(t, 1)
|
||||||
|
|
||||||
|
paused := make(chan struct{})
|
||||||
|
resume := make(chan struct{})
|
||||||
|
|
||||||
|
cache.evictSourceBlobTestHook = func(ContentHash) {
|
||||||
|
close(paused)
|
||||||
|
<-resume
|
||||||
|
}
|
||||||
|
|
||||||
|
hash := storeEvictionTestSource(t, cache, "stop.example.com", "/a.jpg",
|
||||||
|
bytes.Repeat([]byte{0x61}, 1000))
|
||||||
|
|
||||||
|
cache.StartEviction(time.Hour)
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-paused:
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("the eviction pass never reached the source blob")
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
err := cache.StopEviction(ctx)
|
||||||
|
t.Logf("StopEviction() error = %v", err)
|
||||||
|
|
||||||
|
if !errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
t.Errorf("StopEviction() error = %v, want context.DeadlineExceeded", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
close(resume)
|
||||||
|
|
||||||
|
err = cache.StopEviction(t.Context())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("second StopEviction() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertNoDanglingReferences(t, cache)
|
||||||
|
|
||||||
|
if cache.srcContent.Exists(hash) {
|
||||||
|
t.Errorf("source blob %s is still on disk after its rows were deleted", hash)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestReconciliationWalksStopOnceCancelled checks that both directory
|
||||||
|
// walks of a reconciliation pass return the context's error once it is
|
||||||
|
// cancelled, leaving in place a stale temp file they would otherwise
|
||||||
|
// remove.
|
||||||
|
func TestReconciliationWalksStopOnceCancelled(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
cache, _ := newEvictionTestCache(t, 1<<30)
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(t.Context())
|
||||||
|
cancel()
|
||||||
|
|
||||||
|
staleTime := time.Now().Add(-2 * staleTempFileAge)
|
||||||
|
|
||||||
|
walks := map[string]func(context.Context) error{
|
||||||
|
cache.variants.baseDir: cache.reconcileVariantFiles,
|
||||||
|
cache.srcContent.baseDir: cache.reconcileSourceFiles,
|
||||||
|
}
|
||||||
|
|
||||||
|
for dir, walk := range walks {
|
||||||
|
tempFile := filepath.Join(dir, tempFilePrefix+"stale")
|
||||||
|
|
||||||
|
err := os.WriteFile(tempFile, []byte("partial"), 0o600)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to write temp file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = os.Chtimes(tempFile, staleTime, staleTime)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to backdate temp file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = walk(ctx)
|
||||||
|
t.Logf("walk of %s: error = %v", dir, err)
|
||||||
|
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Errorf("walk of %s: error = %v, want context.Canceled", dir, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = os.Stat(tempFile)
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("walk of %s went on after cancellation: %v", dir, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestReconciliationPassLogsNoWarningOnceCancelled checks that a
|
||||||
|
// reconciliation pass run with an already cancelled context logs no
|
||||||
|
// warning, so a periodic tick the loop takes after a stop adds no
|
||||||
|
// warning to the one from the pass the stop interrupted.
|
||||||
|
func TestReconciliationPassLogsNoWarningOnceCancelled(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
cache, _ := newEvictionTestCache(t, 1<<30)
|
||||||
|
|
||||||
|
var logBuf bytes.Buffer
|
||||||
|
|
||||||
|
cache.log = slog.New(slog.NewJSONHandler(&logBuf, nil))
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(t.Context())
|
||||||
|
cancel()
|
||||||
|
|
||||||
|
cache.runReconciliationPass(ctx)
|
||||||
|
t.Logf("log output: %s", logBuf.String())
|
||||||
|
|
||||||
|
if strings.Contains(logBuf.String(), `"level":"WARN"`) {
|
||||||
|
t.Errorf("runReconciliationPass logged a warning with a cancelled context")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestEvictSourceBlobExcludesConcurrentStoreOfIdenticalContent exercises
|
// TestEvictSourceBlobExcludesConcurrentStoreOfIdenticalContent exercises
|
||||||
// the exact TOCTOU window between evictSourceBlob's row-deletion
|
// the exact TOCTOU window between evictSourceBlob's row-deletion
|
||||||
// transaction commit and its content file unlink: a concurrent
|
// transaction commit and its content file unlink: a concurrent
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
"net/url"
|
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -154,9 +153,6 @@ type ImageCache interface {
|
|||||||
// Warm pre-fetches and caches an image without returning it
|
// Warm pre-fetches and caches an image without returning it
|
||||||
Warm(ctx context.Context, req *ImageRequest) error
|
Warm(ctx context.Context, req *ImageRequest) error
|
||||||
|
|
||||||
// Purge removes a cached image
|
|
||||||
Purge(ctx context.Context, req *ImageRequest) error
|
|
||||||
|
|
||||||
// Stats returns cache statistics
|
// Stats returns cache statistics
|
||||||
Stats(ctx context.Context) (*CacheStats, error)
|
Stats(ctx context.Context) (*CacheStats, error)
|
||||||
}
|
}
|
||||||
@@ -176,29 +172,3 @@ type CacheStats struct {
|
|||||||
// HitRate is HitCount / (HitCount + MissCount)
|
// HitRate is HitCount / (HitCount + MissCount)
|
||||||
HitRate float64
|
HitRate float64
|
||||||
}
|
}
|
||||||
|
|
||||||
// SignatureValidator validates request signatures
|
|
||||||
type SignatureValidator interface {
|
|
||||||
// Validate checks if the signature is valid for the request
|
|
||||||
Validate(req *ImageRequest) error
|
|
||||||
// Generate creates a signature for a request
|
|
||||||
Generate(req *ImageRequest) string
|
|
||||||
}
|
|
||||||
|
|
||||||
// Allowlist checks if a URL is allowlisted (no signature required)
|
|
||||||
type Allowlist interface {
|
|
||||||
// IsAllowlisted returns true if the URL doesn't require a signature
|
|
||||||
IsAllowlisted(u *url.URL) bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// Storage handles persistent storage of cached content
|
|
||||||
type Storage interface {
|
|
||||||
// Store saves content and returns its hash
|
|
||||||
Store(ctx context.Context, content io.Reader) (hash string, err error)
|
|
||||||
// Load retrieves content by hash
|
|
||||||
Load(ctx context.Context, hash string) (io.ReadCloser, error)
|
|
||||||
// Delete removes content by hash
|
|
||||||
Delete(ctx context.Context, hash string) error
|
|
||||||
// Exists checks if content exists
|
|
||||||
Exists(ctx context.Context, hash string) (bool, error)
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import (
|
|||||||
|
|
||||||
"github.com/dustin/go-humanize"
|
"github.com/dustin/go-humanize"
|
||||||
"github.com/getsentry/sentry-go"
|
"github.com/getsentry/sentry-go"
|
||||||
|
"github.com/go-chi/chi/v5/middleware"
|
||||||
"golang.org/x/sync/singleflight"
|
"golang.org/x/sync/singleflight"
|
||||||
"sneak.berlin/go/pixa/internal/allowlist"
|
"sneak.berlin/go/pixa/internal/allowlist"
|
||||||
"sneak.berlin/go/pixa/internal/httpfetcher"
|
"sneak.berlin/go/pixa/internal/httpfetcher"
|
||||||
@@ -56,11 +57,10 @@ type ServiceConfig struct {
|
|||||||
Logger *slog.Logger
|
Logger *slog.Logger
|
||||||
}
|
}
|
||||||
|
|
||||||
// Static errors for service construction and unimplemented operations.
|
// Static errors for service construction.
|
||||||
var (
|
var (
|
||||||
errCacheRequired = errors.New("cache is required")
|
errCacheRequired = errors.New("cache is required")
|
||||||
errSigningKeyRequired = errors.New("signing key is required")
|
errSigningKeyRequired = errors.New("signing key is required")
|
||||||
errPurgeNotImplemented = errors.New("purge not implemented")
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// NewService creates a new image service.
|
// NewService creates a new image service.
|
||||||
@@ -189,11 +189,6 @@ func (s *Service) Warm(ctx context.Context, req *ImageRequest) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Purge removes a cached image. Purging is not implemented yet.
|
|
||||||
func (s *Service) Purge(_ context.Context, _ *ImageRequest) error {
|
|
||||||
return errPurgeNotImplemented
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stats returns cache statistics.
|
// Stats returns cache statistics.
|
||||||
func (s *Service) Stats(ctx context.Context) (*CacheStats, error) {
|
func (s *Service) Stats(ctx context.Context) (*CacheStats, error) {
|
||||||
return s.cache.Stats(ctx)
|
return s.cache.Stats(ctx)
|
||||||
@@ -467,6 +462,7 @@ func (s *Service) fetchAndProcess(
|
|||||||
|
|
||||||
// Log upstream fetch details
|
// Log upstream fetch details
|
||||||
s.log.Info("upstream fetched",
|
s.log.Info("upstream fetched",
|
||||||
|
"request_id", middleware.GetReqID(ctx),
|
||||||
"host", req.SourceHost,
|
"host", req.SourceHost,
|
||||||
"path", req.SourcePath,
|
"path", req.SourcePath,
|
||||||
"bytes", fetchBytes,
|
"bytes", fetchBytes,
|
||||||
@@ -551,6 +547,7 @@ func (s *Service) processAndStore(
|
|||||||
}
|
}
|
||||||
|
|
||||||
s.log.Info("image converted",
|
s.log.Info("image converted",
|
||||||
|
"request_id", middleware.GetReqID(ctx),
|
||||||
"host", req.SourceHost,
|
"host", req.SourceHost,
|
||||||
"path", req.SourcePath,
|
"path", req.SourcePath,
|
||||||
"src_format", processResult.InputFormat,
|
"src_format", processResult.InputFormat,
|
||||||
|
|||||||
@@ -564,7 +564,8 @@ func (s *VariantStorage) Exists(key VariantKey) bool {
|
|||||||
return err == nil
|
return err == nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Delete removes content at the given key.
|
// Delete removes the content at the given key together with its .meta
|
||||||
|
// sidecar file. A missing file is not an error.
|
||||||
func (s *VariantStorage) Delete(key VariantKey) error {
|
func (s *VariantStorage) Delete(key VariantKey) error {
|
||||||
path := s.keyToPath(key)
|
path := s.keyToPath(key)
|
||||||
|
|
||||||
@@ -573,18 +574,7 @@ func (s *VariantStorage) Delete(key VariantKey) error {
|
|||||||
return fmt.Errorf("failed to delete content: %w", err)
|
return fmt.Errorf("failed to delete content: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
metaPath := path + ".meta"
|
||||||
}
|
|
||||||
|
|
||||||
// DeleteWithMeta removes the content at the given key together with
|
|
||||||
// its .meta sidecar file. A missing file is not an error.
|
|
||||||
func (s *VariantStorage) DeleteWithMeta(key VariantKey) error {
|
|
||||||
err := s.Delete(key)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
metaPath := s.keyToPath(key) + ".meta"
|
|
||||||
|
|
||||||
err = os.Remove(metaPath)
|
err = os.Remove(metaPath)
|
||||||
if err != nil && !os.IsNotExist(err) {
|
if err != nil && !os.IsNotExist(err) {
|
||||||
|
|||||||
@@ -438,3 +438,73 @@ func TestVariantStorage_StoreLogsFailedMetaWrite(t *testing.T) {
|
|||||||
t.Errorf("log missing %s; got %q", want, logBuf.String())
|
t.Errorf("log missing %s; got %q", want, logBuf.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// storeTestVariant stores one variant, with its .meta file, in a new
|
||||||
|
// VariantStorage and returns the storage and the variant's key.
|
||||||
|
func storeTestVariant(t *testing.T) (*VariantStorage, VariantKey) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
storage, err := NewVariantStorage(t.TempDir(), slog.New(slog.DiscardHandler))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewVariantStorage() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
key := CacheKey(&ImageRequest{SourceHost: testHostCDN, SourcePath: testPathCat})
|
||||||
|
|
||||||
|
_, err = storage.Store(key, bytes.NewReader([]byte("variant data")), "image/webp")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Store() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return storage, key
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestVariantStorage_DeleteRemovesMeta verifies that Delete removes the
|
||||||
|
// variant's .meta file along with the variant file.
|
||||||
|
func TestVariantStorage_DeleteRemovesMeta(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
storage, key := storeTestVariant(t)
|
||||||
|
metaPath := storage.keyToPath(key) + ".meta"
|
||||||
|
|
||||||
|
_, err := os.Stat(metaPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Store() wrote no .meta file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = storage.Delete(key)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Delete() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if storage.Exists(key) {
|
||||||
|
t.Error("Exists() = true after delete, want false")
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = os.Stat(metaPath)
|
||||||
|
if !os.IsNotExist(err) {
|
||||||
|
t.Errorf(".meta file left after Delete() (stat err=%v)", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestVariantStorage_DeleteWithoutMeta verifies that Delete succeeds for a
|
||||||
|
// variant whose .meta file is missing.
|
||||||
|
func TestVariantStorage_DeleteWithoutMeta(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
storage, key := storeTestVariant(t)
|
||||||
|
|
||||||
|
err := os.Remove(storage.keyToPath(key) + ".meta")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("removing .meta file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = storage.Delete(key)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Delete() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if storage.Exists(key) {
|
||||||
|
t.Error("Exists() = true after delete, want false")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -2,9 +2,12 @@
|
|||||||
package middleware
|
package middleware
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/rand"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
|
"regexp"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
basicauth "github.com/99designs/basicauth-go"
|
basicauth "github.com/99designs/basicauth-go"
|
||||||
@@ -32,13 +35,10 @@ const HSTSValue = "max-age=31536000; includeSubDomains"
|
|||||||
|
|
||||||
// ContentSecurityPolicyValue is the Content-Security-Policy header value.
|
// ContentSecurityPolicyValue is the Content-Security-Policy header value.
|
||||||
// default-src 'self' is the baseline and frame-ancestors 'none' is the primary
|
// default-src 'self' is the baseline and frame-ancestors 'none' is the primary
|
||||||
// clickjacking control. 'unsafe-inline' is required in script-src and style-src
|
// clickjacking control.
|
||||||
// because the served templates carry inline onclick handlers (generator page)
|
|
||||||
// and the bundled Tailwind asset injects a runtime <style> element; dropping it
|
|
||||||
// needs template changes outside this issue's scope.
|
|
||||||
const ContentSecurityPolicyValue = "default-src 'self'; " +
|
const ContentSecurityPolicyValue = "default-src 'self'; " +
|
||||||
"script-src 'self' 'unsafe-inline'; " +
|
"script-src 'self'; " +
|
||||||
"style-src 'self' 'unsafe-inline'; " +
|
"style-src 'self'; " +
|
||||||
"object-src 'none'; " +
|
"object-src 'none'; " +
|
||||||
"base-uri 'self'; " +
|
"base-uri 'self'; " +
|
||||||
"form-action 'self'; " +
|
"form-action 'self'; " +
|
||||||
@@ -115,6 +115,32 @@ func (s *Middleware) RateLimit(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// requestIDPattern is what a request's own X-Request-Id must look like to be
|
||||||
|
// kept as its ID: 1 to 64 letters, digits, '-', '_' or '.'.
|
||||||
|
var requestIDPattern = regexp.MustCompile(`^[A-Za-z0-9._-]{1,64}$`)
|
||||||
|
|
||||||
|
// RequestID returns a middleware that gives each request an ID and sends it as
|
||||||
|
// the X-Request-Id response header, so a client can quote it when reporting a
|
||||||
|
// problem. The ID is the request's own X-Request-Id when that matches
|
||||||
|
// requestIDPattern, and otherwise a random one, which tells nothing about the
|
||||||
|
// machine or the traffic. It is stored in the request context under chi's
|
||||||
|
// RequestIDKey, where the logging middleware, the handlers and the upstream
|
||||||
|
// fetch read it.
|
||||||
|
func (s *Middleware) RequestID() func(http.Handler) http.Handler {
|
||||||
|
return func(next http.Handler) http.Handler {
|
||||||
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
id := r.Header.Get(middleware.RequestIDHeader)
|
||||||
|
if !requestIDPattern.MatchString(id) {
|
||||||
|
id = rand.Text()
|
||||||
|
}
|
||||||
|
|
||||||
|
w.Header().Set(middleware.RequestIDHeader, id)
|
||||||
|
ctx := context.WithValue(r.Context(), middleware.RequestIDKey, id)
|
||||||
|
next.ServeHTTP(w, r.WithContext(ctx))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type loggingResponseWriter struct {
|
type loggingResponseWriter struct {
|
||||||
http.ResponseWriter
|
http.ResponseWriter
|
||||||
|
|
||||||
|
|||||||
@@ -1,11 +1,16 @@
|
|||||||
package middleware
|
package middleware
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||||
|
|
||||||
"sneak.berlin/go/pixa/internal/config"
|
"sneak.berlin/go/pixa/internal/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -56,6 +61,203 @@ func TestCORSAnswersWithConfiguredOrigin(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestCORSAnswersPreflightWithConfiguredOrigin checks that the CORS
|
||||||
|
// middleware answers a preflight request, which the CORS library handles
|
||||||
|
// apart from other requests, the same way: "*" lets any origin read
|
||||||
|
// responses and a single origin lets only that origin read them.
|
||||||
|
func TestCORSAnswersPreflightWithConfiguredOrigin(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const appOrigin = "https://app.example.com"
|
||||||
|
|
||||||
|
cases := []struct {
|
||||||
|
configured string
|
||||||
|
requestOrigin string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{"*", "https://any.example.com", "*"},
|
||||||
|
{appOrigin, appOrigin, appOrigin},
|
||||||
|
{appOrigin, "https://other.example.com", ""},
|
||||||
|
}
|
||||||
|
|
||||||
|
testHandler := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
})
|
||||||
|
|
||||||
|
for _, tc := range cases {
|
||||||
|
mw := &Middleware{
|
||||||
|
log: slog.Default(),
|
||||||
|
config: &config.Config{AccessControlAllowOrigin: tc.configured},
|
||||||
|
}
|
||||||
|
|
||||||
|
handler := mw.CORS()(testHandler)
|
||||||
|
|
||||||
|
// An OPTIONS request naming the method it asks about is the
|
||||||
|
// preflight a browser sends before some cross-origin requests.
|
||||||
|
req := httptest.NewRequestWithContext(
|
||||||
|
t.Context(), http.MethodOptions, "/v1/image/example.com/a.jpg/1x1.png", nil)
|
||||||
|
req.Header.Set("Origin", tc.requestOrigin)
|
||||||
|
req.Header.Set("Access-Control-Request-Method", http.MethodGet)
|
||||||
|
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
|
||||||
|
handler.ServeHTTP(rec, req)
|
||||||
|
|
||||||
|
got := rec.Header().Get("Access-Control-Allow-Origin")
|
||||||
|
if got != tc.want {
|
||||||
|
t.Errorf("configured %q, preflight from %q: "+
|
||||||
|
"Access-Control-Allow-Origin = %q, want %q",
|
||||||
|
tc.configured, tc.requestOrigin, got, tc.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestMetricsAuthRequiresConfiguredCredentials checks that MetricsAuth on
|
||||||
|
// its own answers 401 with a challenge to a request without credentials or
|
||||||
|
// with a wrong username or password, and lets a request with the configured
|
||||||
|
// username and password through. That the router puts it in front of
|
||||||
|
// /metrics is not tested.
|
||||||
|
func TestMetricsAuthRequiresConfiguredCredentials(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const (
|
||||||
|
username = "metricsuser"
|
||||||
|
password = "metricspass"
|
||||||
|
challenge = `Basic realm="metrics"`
|
||||||
|
)
|
||||||
|
|
||||||
|
// An empty username stands for a request sent without credentials.
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
username string
|
||||||
|
password string
|
||||||
|
wantReached bool
|
||||||
|
}{
|
||||||
|
{"no credentials", "", "", false},
|
||||||
|
{"wrong username", "someone", password, false},
|
||||||
|
{"wrong password", username, "wrongpass", false},
|
||||||
|
{"configured credentials", username, password, true},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
mw := &Middleware{
|
||||||
|
log: slog.Default(),
|
||||||
|
config: &config.Config{
|
||||||
|
MetricsUsername: username,
|
||||||
|
MetricsPassword: password,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
reached := false
|
||||||
|
handler := mw.MetricsAuth()(http.HandlerFunc(
|
||||||
|
func(http.ResponseWriter, *http.Request) {
|
||||||
|
reached = true
|
||||||
|
}))
|
||||||
|
|
||||||
|
req := httptest.NewRequestWithContext(
|
||||||
|
t.Context(), http.MethodGet, "/metrics", nil)
|
||||||
|
|
||||||
|
if tc.username != "" {
|
||||||
|
req.SetBasicAuth(tc.username, tc.password)
|
||||||
|
}
|
||||||
|
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
|
||||||
|
handler.ServeHTTP(rec, req)
|
||||||
|
|
||||||
|
if reached != tc.wantReached {
|
||||||
|
t.Fatalf("request reached /metrics = %v, want %v",
|
||||||
|
reached, tc.wantReached)
|
||||||
|
}
|
||||||
|
|
||||||
|
if tc.wantReached {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if rec.Code != http.StatusUnauthorized {
|
||||||
|
t.Errorf("status = %d, want %d",
|
||||||
|
rec.Code, http.StatusUnauthorized)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := rec.Header().Get("WWW-Authenticate"); got != challenge {
|
||||||
|
t.Errorf("WWW-Authenticate = %q, want %q", got, challenge)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestMetricsRecordsServedRequest checks that the metrics middleware
|
||||||
|
// records a request it served, so /metrics reports it. It is the only test
|
||||||
|
// in this package that sets up the metrics middleware, which registers with
|
||||||
|
// the process-wide Prometheus registry and can do so only once.
|
||||||
|
func TestMetricsRecordsServedRequest(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
// The line /metrics shows once one GET /test has been served.
|
||||||
|
const want = `http_request_duration_seconds_count{` +
|
||||||
|
`code="200",handler="/test",method="GET",service=""} 1`
|
||||||
|
|
||||||
|
mw := &Middleware{log: slog.Default(), config: &config.Config{}}
|
||||||
|
|
||||||
|
handler := mw.Metrics()(http.HandlerFunc(
|
||||||
|
func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
}))
|
||||||
|
|
||||||
|
handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequestWithContext(
|
||||||
|
t.Context(), http.MethodGet, "/test", nil))
|
||||||
|
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
promhttp.Handler().ServeHTTP(rec, httptest.NewRequestWithContext(
|
||||||
|
t.Context(), http.MethodGet, "/metrics", nil))
|
||||||
|
|
||||||
|
if !strings.Contains(rec.Body.String(), want) {
|
||||||
|
t.Errorf("/metrics does not report the GET /test served; "+
|
||||||
|
"want the line %q in:\n%s", want, rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestLoggingLeavesOutSubmittedSigningKey checks that a login, a POST /
|
||||||
|
// whose form carries the signing key, leaves no trace of the key in the
|
||||||
|
// request's log line.
|
||||||
|
func TestLoggingLeavesOutSubmittedSigningKey(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const signingKey = "test-signing-key-0123456789abcdef"
|
||||||
|
|
||||||
|
var buf bytes.Buffer
|
||||||
|
|
||||||
|
mw := newTestMiddleware(t, &buf)
|
||||||
|
|
||||||
|
// The handler reads the key from the form, as the login handler does.
|
||||||
|
handler := mw.Logging()(http.HandlerFunc(
|
||||||
|
func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if got := r.FormValue("key"); got != signingKey {
|
||||||
|
t.Errorf("key in form = %q, want %q", got, signingKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
w.WriteHeader(http.StatusSeeOther)
|
||||||
|
}))
|
||||||
|
|
||||||
|
form := url.Values{"key": {signingKey}}
|
||||||
|
req := httptest.NewRequestWithContext(t.Context(), http.MethodPost, "/",
|
||||||
|
strings.NewReader(form.Encode()))
|
||||||
|
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||||
|
|
||||||
|
handler.ServeHTTP(httptest.NewRecorder(), req)
|
||||||
|
|
||||||
|
if !strings.Contains(buf.String(), `"method":"POST"`) {
|
||||||
|
t.Fatalf("no log line for the request; got %q", buf.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
if strings.Contains(buf.String(), signingKey) {
|
||||||
|
t.Errorf("log output contains the signing key; got %q", buf.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestSecurityHeaders(t *testing.T) {
|
func TestSecurityHeaders(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -123,6 +325,13 @@ func TestSecurityHeaders_PolicyHeaders(t *testing.T) {
|
|||||||
|
|
||||||
handler.ServeHTTP(rec, req)
|
handler.ServeHTTP(rec, req)
|
||||||
|
|
||||||
|
// The login and generator pages load their script and stylesheet from
|
||||||
|
// /static, so the policy allows no inline script or style.
|
||||||
|
csp := rec.Header().Get("Content-Security-Policy")
|
||||||
|
if strings.Contains(csp, "unsafe-inline") {
|
||||||
|
t.Errorf("Content-Security-Policy allows unsafe-inline: %q", csp)
|
||||||
|
}
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
header string
|
header string
|
||||||
want string
|
want string
|
||||||
@@ -131,8 +340,8 @@ func TestSecurityHeaders_PolicyHeaders(t *testing.T) {
|
|||||||
{
|
{
|
||||||
"Content-Security-Policy",
|
"Content-Security-Policy",
|
||||||
"default-src 'self'; " +
|
"default-src 'self'; " +
|
||||||
"script-src 'self' 'unsafe-inline'; " +
|
"script-src 'self'; " +
|
||||||
"style-src 'self' 'unsafe-inline'; " +
|
"style-src 'self'; " +
|
||||||
"object-src 'none'; " +
|
"object-src 'none'; " +
|
||||||
"base-uri 'self'; " +
|
"base-uri 'self'; " +
|
||||||
"form-action 'self'; " +
|
"form-action 'self'; " +
|
||||||
|
|||||||
@@ -0,0 +1,118 @@
|
|||||||
|
package middleware
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/go-chi/chi/v5/middleware"
|
||||||
|
|
||||||
|
"sneak.berlin/go/pixa/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
// sendRequestID sends a request through the RequestID middleware, carrying
|
||||||
|
// incoming as its X-Request-Id unless that is empty. It returns the ID the
|
||||||
|
// next handler found in the request context, which the upstream fetch sends
|
||||||
|
// and the log lines carry, and the X-Request-Id of the response.
|
||||||
|
func sendRequestID(t *testing.T, incoming string) (string, string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
mw := &Middleware{log: slog.Default(), config: &config.Config{}}
|
||||||
|
|
||||||
|
var inContext string
|
||||||
|
|
||||||
|
handler := mw.RequestID()(http.HandlerFunc(
|
||||||
|
func(_ http.ResponseWriter, r *http.Request) {
|
||||||
|
inContext = middleware.GetReqID(r.Context())
|
||||||
|
}))
|
||||||
|
|
||||||
|
req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil)
|
||||||
|
if incoming != "" {
|
||||||
|
req.Header.Set("X-Request-Id", incoming)
|
||||||
|
}
|
||||||
|
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
handler.ServeHTTP(rec, req)
|
||||||
|
|
||||||
|
return inContext, rec.Header().Get("X-Request-Id")
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRequestIDKeepsShortPlainID verifies that a request's own X-Request-Id of
|
||||||
|
// at most 64 letters, digits, '-', '_' or '.' is kept as its ID.
|
||||||
|
func TestRequestIDKeepsShortPlainID(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
for _, incoming := range []string{
|
||||||
|
"client-request-id",
|
||||||
|
"A1_b2.c3-d4",
|
||||||
|
strings.Repeat("a", 64),
|
||||||
|
} {
|
||||||
|
inContext, inResponse := sendRequestID(t, incoming)
|
||||||
|
|
||||||
|
if inContext != incoming || inResponse != incoming {
|
||||||
|
t.Errorf("incoming %q: context has %q, response %q, want both %q",
|
||||||
|
incoming, inContext, inResponse, incoming)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRequestIDReplacesLongOrUnusualID verifies that a request's own
|
||||||
|
// X-Request-Id that is over 64 characters or holds anything but letters,
|
||||||
|
// digits, '-', '_' or '.' is neither sent back nor sent upstream: the request
|
||||||
|
// gets a fresh ID instead.
|
||||||
|
func TestRequestIDReplacesLongOrUnusualID(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
for _, incoming := range []string{
|
||||||
|
strings.Repeat("a", 65),
|
||||||
|
strings.Repeat("a", 9000),
|
||||||
|
"has space",
|
||||||
|
"a/b",
|
||||||
|
"a,b",
|
||||||
|
"<script>",
|
||||||
|
"ünicode",
|
||||||
|
} {
|
||||||
|
inContext, inResponse := sendRequestID(t, incoming)
|
||||||
|
t.Logf("incoming %.20q: made up %q", incoming, inResponse)
|
||||||
|
|
||||||
|
if inResponse == "" || inResponse == incoming {
|
||||||
|
t.Errorf("incoming %.20q: response has %q, want a fresh ID",
|
||||||
|
incoming, inResponse)
|
||||||
|
}
|
||||||
|
|
||||||
|
if inContext != inResponse {
|
||||||
|
t.Errorf("incoming %.20q: context has %q, want the response's %q",
|
||||||
|
incoming, inContext, inResponse)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRequestIDMadeUpTellsNothing verifies that the ID made up for a request
|
||||||
|
// that sent none differs for every request and does not hold the host name.
|
||||||
|
func TestRequestIDMadeUpTellsNothing(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
hostname, err := os.Hostname()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("os.Hostname() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
firstInContext, first := sendRequestID(t, "")
|
||||||
|
_, second := sendRequestID(t, "")
|
||||||
|
t.Logf("host %q, made up %q and %q", hostname, first, second)
|
||||||
|
|
||||||
|
if first == "" || first == second {
|
||||||
|
t.Errorf("made up %q and %q, want two different IDs", first, second)
|
||||||
|
}
|
||||||
|
|
||||||
|
if firstInContext != first {
|
||||||
|
t.Errorf("context has %q, want the response's %q", firstInContext, first)
|
||||||
|
}
|
||||||
|
|
||||||
|
if strings.Contains(first, hostname) {
|
||||||
|
t.Errorf("made-up ID %q holds the host name %q", first, hostname)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
package server
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestNoMetricsRecordedWithoutMetricsUsername checks that with no metrics
|
||||||
|
// username set the router records nothing about the requests it serves.
|
||||||
|
// /metrics is not served then, so the process-wide Prometheus registry is
|
||||||
|
// read directly. TestMaintenanceModeKeepsOtherRoutes records into the same
|
||||||
|
// registry, but never a GET /robots.txt.
|
||||||
|
func TestNoMetricsRecordedWithoutMetricsUsername(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
s := newTestServer(t)
|
||||||
|
s.ServeHTTP(httptest.NewRecorder(), httptest.NewRequestWithContext(
|
||||||
|
t.Context(), http.MethodGet, "/robots.txt", nil))
|
||||||
|
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
promhttp.Handler().ServeHTTP(rec, httptest.NewRequestWithContext(
|
||||||
|
t.Context(), http.MethodGet, "/metrics", nil))
|
||||||
|
|
||||||
|
if strings.Contains(rec.Body.String(), `handler="/robots.txt"`) {
|
||||||
|
t.Errorf("with no metrics username GET /robots.txt was recorded:\n%s",
|
||||||
|
rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,58 @@
|
|||||||
|
package server
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// requestIDHeader is the header that carries a request's ID.
|
||||||
|
const requestIDHeader = "X-Request-Id"
|
||||||
|
|
||||||
|
// TestResponsesCarryRequestID verifies that every response, whatever its route
|
||||||
|
// and status, carries the request's ID as X-Request-Id, so a client can quote
|
||||||
|
// it when reporting a problem: one pixa made up when the request brought none,
|
||||||
|
// and the request's own X-Request-Id when it brought one.
|
||||||
|
func TestResponsesCarryRequestID(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const clientRequestID = "client-request-id"
|
||||||
|
|
||||||
|
s := newTestServer(t)
|
||||||
|
|
||||||
|
paths := []string{
|
||||||
|
"/robots.txt",
|
||||||
|
"/no-such-path",
|
||||||
|
unsignedImagePath,
|
||||||
|
encryptedImagePath,
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, path := range paths {
|
||||||
|
t.Run(path, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
s.ServeHTTP(rec, httptest.NewRequestWithContext(
|
||||||
|
t.Context(), http.MethodGet, path, nil))
|
||||||
|
t.Logf("status %d, %s %q",
|
||||||
|
rec.Code, requestIDHeader, rec.Header().Get(requestIDHeader))
|
||||||
|
|
||||||
|
if rec.Header().Get(requestIDHeader) == "" {
|
||||||
|
t.Errorf("response has no %s", requestIDHeader)
|
||||||
|
}
|
||||||
|
|
||||||
|
req := httptest.NewRequestWithContext(
|
||||||
|
t.Context(), http.MethodGet, path, nil)
|
||||||
|
req.Header.Set(requestIDHeader, clientRequestID)
|
||||||
|
|
||||||
|
rec = httptest.NewRecorder()
|
||||||
|
s.ServeHTTP(rec, req)
|
||||||
|
|
||||||
|
got := rec.Header().Get(requestIDHeader)
|
||||||
|
if got != clientRequestID {
|
||||||
|
t.Errorf("%s = %q, want the request's own %q",
|
||||||
|
requestIDHeader, got, clientRequestID)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -28,7 +28,7 @@ func (s *Server) SetupRoutes() {
|
|||||||
s.router = chi.NewRouter()
|
s.router = chi.NewRouter()
|
||||||
|
|
||||||
s.router.Use(middleware.Recoverer)
|
s.router.Use(middleware.Recoverer)
|
||||||
s.router.Use(middleware.RequestID)
|
s.router.Use(s.mw.RequestID())
|
||||||
s.router.Use(s.mw.ClientIP())
|
s.router.Use(s.mw.ClientIP())
|
||||||
s.router.Use(s.mw.SecurityHeaders())
|
s.router.Use(s.mw.SecurityHeaders())
|
||||||
s.router.Use(s.mw.Logging())
|
s.router.Use(s.mw.Logging())
|
||||||
@@ -53,7 +53,7 @@ func (s *Server) SetupRoutes() {
|
|||||||
// Robots.txt
|
// Robots.txt
|
||||||
s.router.Get("/robots.txt", s.h.HandleRobotsTxt())
|
s.router.Get("/robots.txt", s.h.HandleRobotsTxt())
|
||||||
|
|
||||||
// Static files (Tailwind CSS, etc.)
|
// The login and generator pages' stylesheet and script
|
||||||
s.router.Handle("/static/*", http.StripPrefix("/static/", static.Handler()))
|
s.router.Handle("/static/*", http.StripPrefix("/static/", static.Handler()))
|
||||||
|
|
||||||
// Login/generator UI. The form routes carry CSRF protection; the
|
// Login/generator UI. The form routes carry CSRF protection; the
|
||||||
@@ -97,6 +97,7 @@ func (s *Server) SetupRoutes() {
|
|||||||
// The trailing filename (e.g., /img.jpg) is ignored but helps
|
// The trailing filename (e.g., /img.jpg) is ignored but helps
|
||||||
// browsers with content type
|
// browsers with content type
|
||||||
r.Get("/e/{token}/*", s.h.HandleImageEnc())
|
r.Get("/e/{token}/*", s.h.HandleImageEnc())
|
||||||
|
r.Head("/e/{token}/*", s.h.HandleImageEnc())
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,27 @@
|
|||||||
|
package server
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestEncryptedImageRouteAnswersHEAD verifies that HEAD on the encrypted image
|
||||||
|
// route reaches its handler, as GET does, instead of being answered 405 Method
|
||||||
|
// Not Allowed. The handler refuses a token it cannot decrypt with 400, so that
|
||||||
|
// status shows the request got through.
|
||||||
|
func TestEncryptedImageRouteAnswersHEAD(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
s := newTestServer(t)
|
||||||
|
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
s.ServeHTTP(rec, httptest.NewRequestWithContext(
|
||||||
|
t.Context(), http.MethodHead, encryptedImagePath, nil))
|
||||||
|
t.Logf("status %d", rec.Code)
|
||||||
|
|
||||||
|
if rec.Code != http.StatusBadRequest {
|
||||||
|
t.Errorf("status = %d, want %d from the encrypted image handler",
|
||||||
|
rec.Code, http.StatusBadRequest)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
// Generator page: a click on the generated URL selects it, and the Copy
|
||||||
|
// button copies it. Both are on the page only once a URL has been generated.
|
||||||
|
const generatedURL = document.getElementById("generated-url");
|
||||||
|
|
||||||
|
if (generatedURL) {
|
||||||
|
generatedURL.addEventListener("click", () => generatedURL.select());
|
||||||
|
document.getElementById("copy-url").addEventListener("click", () => {
|
||||||
|
navigator.clipboard.writeText(generatedURL.value);
|
||||||
|
});
|
||||||
|
}
|
||||||
@@ -7,7 +7,7 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
)
|
)
|
||||||
|
|
||||||
//go:embed *.js
|
//go:embed *.css *.js
|
||||||
var files embed.FS
|
var files embed.FS
|
||||||
|
|
||||||
// FS returns the embedded filesystem containing static files.
|
// FS returns the embedded filesystem containing static files.
|
||||||
|
|||||||
@@ -0,0 +1,190 @@
|
|||||||
|
/* The login and generator pages. */
|
||||||
|
|
||||||
|
* {
|
||||||
|
box-sizing: border-box;
|
||||||
|
}
|
||||||
|
|
||||||
|
body {
|
||||||
|
margin: 0;
|
||||||
|
min-height: 100vh;
|
||||||
|
background: #f3f4f6;
|
||||||
|
font-family: system-ui, sans-serif;
|
||||||
|
line-height: 1.5;
|
||||||
|
}
|
||||||
|
|
||||||
|
h1 {
|
||||||
|
margin: 0;
|
||||||
|
font-size: 1.5rem;
|
||||||
|
line-height: 2rem;
|
||||||
|
font-weight: 700;
|
||||||
|
color: #1f2937;
|
||||||
|
}
|
||||||
|
|
||||||
|
label {
|
||||||
|
display: block;
|
||||||
|
margin-bottom: 0.25rem;
|
||||||
|
font-size: 0.875rem;
|
||||||
|
font-weight: 500;
|
||||||
|
color: #374151;
|
||||||
|
}
|
||||||
|
|
||||||
|
input,
|
||||||
|
select {
|
||||||
|
width: 100%;
|
||||||
|
padding: 0.5rem 0.75rem;
|
||||||
|
border: 1px solid #d1d5db;
|
||||||
|
border-radius: 0.375rem;
|
||||||
|
box-shadow: 0 1px 2px rgb(0 0 0 / 5%);
|
||||||
|
font: inherit;
|
||||||
|
}
|
||||||
|
|
||||||
|
input:focus,
|
||||||
|
select:focus {
|
||||||
|
outline: none;
|
||||||
|
border-color: #3b82f6;
|
||||||
|
box-shadow: 0 0 0 2px #3b82f6;
|
||||||
|
}
|
||||||
|
|
||||||
|
button {
|
||||||
|
width: 100%;
|
||||||
|
padding: 0.5rem 1rem;
|
||||||
|
border: none;
|
||||||
|
border-radius: 0.375rem;
|
||||||
|
background: #2563eb;
|
||||||
|
color: #fff;
|
||||||
|
font: inherit;
|
||||||
|
cursor: pointer;
|
||||||
|
transition: background-color 0.15s;
|
||||||
|
}
|
||||||
|
|
||||||
|
button:hover {
|
||||||
|
background: #1d4ed8;
|
||||||
|
}
|
||||||
|
|
||||||
|
button:focus {
|
||||||
|
outline: 2px solid #3b82f6;
|
||||||
|
outline-offset: 2px;
|
||||||
|
}
|
||||||
|
|
||||||
|
form > * + * {
|
||||||
|
margin-top: 1rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.card {
|
||||||
|
padding: 1.5rem;
|
||||||
|
border-radius: 0.5rem;
|
||||||
|
background: #fff;
|
||||||
|
box-shadow:
|
||||||
|
0 4px 6px -1px rgb(0 0 0 / 10%),
|
||||||
|
0 2px 4px -2px rgb(0 0 0 / 10%);
|
||||||
|
}
|
||||||
|
|
||||||
|
.error {
|
||||||
|
margin-bottom: 1rem;
|
||||||
|
padding: 0.75rem 1rem;
|
||||||
|
border: 1px solid #f87171;
|
||||||
|
border-radius: 0.25rem;
|
||||||
|
background: #fee2e2;
|
||||||
|
color: #b91c1c;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Login page: the card centred on the screen. */
|
||||||
|
|
||||||
|
.login {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: center;
|
||||||
|
}
|
||||||
|
|
||||||
|
.login .card {
|
||||||
|
width: 100%;
|
||||||
|
max-width: 28rem;
|
||||||
|
padding: 2rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.login h1 {
|
||||||
|
margin-bottom: 1.5rem;
|
||||||
|
text-align: center;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Generator page. */
|
||||||
|
|
||||||
|
.page {
|
||||||
|
max-width: 42rem;
|
||||||
|
margin: 0 auto;
|
||||||
|
padding: 2rem 1rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
header {
|
||||||
|
display: flex;
|
||||||
|
justify-content: space-between;
|
||||||
|
align-items: center;
|
||||||
|
margin-bottom: 2rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
header a {
|
||||||
|
font-size: 0.875rem;
|
||||||
|
color: #4b5563;
|
||||||
|
}
|
||||||
|
|
||||||
|
header a:hover {
|
||||||
|
color: #1f2937;
|
||||||
|
}
|
||||||
|
|
||||||
|
.result {
|
||||||
|
margin-bottom: 1.5rem;
|
||||||
|
padding: 1rem;
|
||||||
|
border: 1px solid #bbf7d0;
|
||||||
|
border-radius: 0.5rem;
|
||||||
|
background: #f0fdf4;
|
||||||
|
}
|
||||||
|
|
||||||
|
.result h2 {
|
||||||
|
margin: 0 0 0.5rem;
|
||||||
|
font-size: 0.875rem;
|
||||||
|
font-weight: 500;
|
||||||
|
color: #166534;
|
||||||
|
}
|
||||||
|
|
||||||
|
.result div {
|
||||||
|
display: flex;
|
||||||
|
gap: 0.5rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.result input {
|
||||||
|
flex: 1;
|
||||||
|
border-color: #86efac;
|
||||||
|
box-shadow: none;
|
||||||
|
font-family: ui-monospace, monospace;
|
||||||
|
font-size: 0.875rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.result button {
|
||||||
|
width: auto;
|
||||||
|
padding: 0.5rem 0.75rem;
|
||||||
|
background: #16a34a;
|
||||||
|
font-size: 0.875rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.result button:hover {
|
||||||
|
background: #15803d;
|
||||||
|
}
|
||||||
|
|
||||||
|
.result p {
|
||||||
|
margin: 0.5rem 0 0;
|
||||||
|
font-size: 0.75rem;
|
||||||
|
color: #16a34a;
|
||||||
|
}
|
||||||
|
|
||||||
|
.columns {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||||
|
gap: 1rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.note {
|
||||||
|
margin-top: 1rem;
|
||||||
|
font-size: 0.75rem;
|
||||||
|
color: #6b7280;
|
||||||
|
text-align: center;
|
||||||
|
}
|
||||||
File diff suppressed because one or more lines are too long
@@ -4,52 +4,47 @@
|
|||||||
<meta charset="UTF-8">
|
<meta charset="UTF-8">
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
<title>Pixa - URL Generator</title>
|
<title>Pixa - URL Generator</title>
|
||||||
<script src="/static/tailwind.js"></script>
|
<link rel="stylesheet" href="/static/style.css">
|
||||||
</head>
|
</head>
|
||||||
<body class="bg-gray-100 min-h-screen">
|
<body>
|
||||||
<div class="max-w-2xl mx-auto py-8 px-4">
|
<div class="page">
|
||||||
<div class="flex justify-between items-center mb-8">
|
<header>
|
||||||
<h1 class="text-2xl font-bold text-gray-800">Pixa URL Generator</h1>
|
<h1>Pixa URL Generator</h1>
|
||||||
<a href="/logout" class="text-sm text-gray-600 hover:text-gray-800 underline">
|
<a href="/logout">
|
||||||
Logout
|
Logout
|
||||||
</a>
|
</a>
|
||||||
</div>
|
</header>
|
||||||
|
|
||||||
{{if .GeneratedURL}}
|
{{if .GeneratedURL}}
|
||||||
<div class="bg-green-50 border border-green-200 rounded-lg p-4 mb-6">
|
<div class="result">
|
||||||
<h2 class="text-sm font-medium text-green-800 mb-2">Generated URL</h2>
|
<h2>Generated URL</h2>
|
||||||
<div class="flex gap-2">
|
<div>
|
||||||
<input
|
<input
|
||||||
type="text"
|
type="text"
|
||||||
readonly
|
readonly
|
||||||
value="{{.GeneratedURL}}"
|
value="{{.GeneratedURL}}"
|
||||||
id="generated-url"
|
id="generated-url"
|
||||||
class="flex-1 px-3 py-2 bg-white border border-green-300 rounded-md text-sm font-mono"
|
|
||||||
onclick="this.select()"
|
|
||||||
>
|
|
||||||
<button
|
|
||||||
onclick="navigator.clipboard.writeText(document.getElementById('generated-url').value)"
|
|
||||||
class="px-3 py-2 bg-green-600 text-white rounded-md hover:bg-green-700 text-sm"
|
|
||||||
>
|
>
|
||||||
|
<button id="copy-url">
|
||||||
Copy
|
Copy
|
||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
<p class="text-xs text-green-600 mt-2">
|
<p>
|
||||||
Expires: {{.ExpiresAt}}
|
Expires: {{.ExpiresAt}}
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
{{end}}
|
{{end}}
|
||||||
|
|
||||||
{{if .Error}}
|
{{if .Error}}
|
||||||
<div class="bg-red-100 border border-red-400 text-red-700 px-4 py-3 rounded mb-6">
|
<div class="error">
|
||||||
{{.Error}}
|
{{.Error}}
|
||||||
</div>
|
</div>
|
||||||
{{end}}
|
{{end}}
|
||||||
|
|
||||||
<form method="POST" action="/generate" class="bg-white rounded-lg shadow-md p-6 space-y-4">
|
<form method="POST" action="/generate" class="card">
|
||||||
{{ .CSRFField }}
|
{{ .CSRFField }}
|
||||||
<div>
|
<div>
|
||||||
<label for="url" class="block text-sm font-medium text-gray-700 mb-1">
|
<label for="url">
|
||||||
Source URL
|
Source URL
|
||||||
</label>
|
</label>
|
||||||
<input
|
<input
|
||||||
@@ -59,13 +54,12 @@
|
|||||||
required
|
required
|
||||||
placeholder="https://example.com/image.jpg"
|
placeholder="https://example.com/image.jpg"
|
||||||
value="{{.FormURL}}"
|
value="{{.FormURL}}"
|
||||||
class="w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500"
|
|
||||||
>
|
>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="grid grid-cols-2 gap-4">
|
<div class="columns">
|
||||||
<div>
|
<div>
|
||||||
<label for="width" class="block text-sm font-medium text-gray-700 mb-1">
|
<label for="width">
|
||||||
Width
|
Width
|
||||||
</label>
|
</label>
|
||||||
<input
|
<input
|
||||||
@@ -76,11 +70,10 @@
|
|||||||
max="8192"
|
max="8192"
|
||||||
value="{{if .FormWidth}}{{.FormWidth}}{{else}}0{{end}}"
|
value="{{if .FormWidth}}{{.FormWidth}}{{else}}0{{end}}"
|
||||||
placeholder="0 = original"
|
placeholder="0 = original"
|
||||||
class="w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500"
|
|
||||||
>
|
>
|
||||||
</div>
|
</div>
|
||||||
<div>
|
<div>
|
||||||
<label for="height" class="block text-sm font-medium text-gray-700 mb-1">
|
<label for="height">
|
||||||
Height
|
Height
|
||||||
</label>
|
</label>
|
||||||
<input
|
<input
|
||||||
@@ -91,21 +84,16 @@
|
|||||||
max="8192"
|
max="8192"
|
||||||
value="{{if .FormHeight}}{{.FormHeight}}{{else}}0{{end}}"
|
value="{{if .FormHeight}}{{.FormHeight}}{{else}}0{{end}}"
|
||||||
placeholder="0 = original"
|
placeholder="0 = original"
|
||||||
class="w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500"
|
|
||||||
>
|
>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="grid grid-cols-2 gap-4">
|
<div class="columns">
|
||||||
<div>
|
<div>
|
||||||
<label for="format" class="block text-sm font-medium text-gray-700 mb-1">
|
<label for="format">
|
||||||
Format
|
Format
|
||||||
</label>
|
</label>
|
||||||
<select
|
<select id="format" name="format">
|
||||||
id="format"
|
|
||||||
name="format"
|
|
||||||
class="w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500"
|
|
||||||
>
|
|
||||||
<option value="orig" {{if eq .FormFormat "orig"}}selected{{end}}>Original</option>
|
<option value="orig" {{if eq .FormFormat "orig"}}selected{{end}}>Original</option>
|
||||||
<option value="jpeg" {{if eq .FormFormat "jpeg"}}selected{{end}}>JPEG</option>
|
<option value="jpeg" {{if eq .FormFormat "jpeg"}}selected{{end}}>JPEG</option>
|
||||||
<option value="png" {{if eq .FormFormat "png"}}selected{{end}}>PNG</option>
|
<option value="png" {{if eq .FormFormat "png"}}selected{{end}}>PNG</option>
|
||||||
@@ -115,14 +103,10 @@
|
|||||||
</select>
|
</select>
|
||||||
</div>
|
</div>
|
||||||
<div>
|
<div>
|
||||||
<label for="quality" class="block text-sm font-medium text-gray-700 mb-1">
|
<label for="quality">
|
||||||
Quality
|
Quality
|
||||||
</label>
|
</label>
|
||||||
<select
|
<select id="quality" name="quality">
|
||||||
id="quality"
|
|
||||||
name="quality"
|
|
||||||
class="w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500"
|
|
||||||
>
|
|
||||||
<option value="25" {{if eq .FormQuality "25"}}selected{{end}}>Potato</option>
|
<option value="25" {{if eq .FormQuality "25"}}selected{{end}}>Potato</option>
|
||||||
<option value="50" {{if eq .FormQuality "50"}}selected{{end}}>Low</option>
|
<option value="50" {{if eq .FormQuality "50"}}selected{{end}}>Low</option>
|
||||||
<option value="70" {{if eq .FormQuality "70"}}selected{{end}}>Medium</option>
|
<option value="70" {{if eq .FormQuality "70"}}selected{{end}}>Medium</option>
|
||||||
@@ -132,16 +116,12 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="grid grid-cols-2 gap-4">
|
<div class="columns">
|
||||||
<div>
|
<div>
|
||||||
<label for="fit" class="block text-sm font-medium text-gray-700 mb-1">
|
<label for="fit">
|
||||||
Fit Mode
|
Fit Mode
|
||||||
</label>
|
</label>
|
||||||
<select
|
<select id="fit" name="fit">
|
||||||
id="fit"
|
|
||||||
name="fit"
|
|
||||||
class="w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500"
|
|
||||||
>
|
|
||||||
<option value="cover" {{if eq .FormFit "cover"}}selected{{end}}>Cover</option>
|
<option value="cover" {{if eq .FormFit "cover"}}selected{{end}}>Cover</option>
|
||||||
<option value="contain" {{if eq .FormFit "contain"}}selected{{end}}>Contain</option>
|
<option value="contain" {{if eq .FormFit "contain"}}selected{{end}}>Contain</option>
|
||||||
<option value="fill" {{if eq .FormFit "fill"}}selected{{end}}>Fill</option>
|
<option value="fill" {{if eq .FormFit "fill"}}selected{{end}}>Fill</option>
|
||||||
@@ -150,14 +130,10 @@
|
|||||||
</select>
|
</select>
|
||||||
</div>
|
</div>
|
||||||
<div>
|
<div>
|
||||||
<label for="ttl" class="block text-sm font-medium text-gray-700 mb-1">
|
<label for="ttl">
|
||||||
Expires In
|
Expires In
|
||||||
</label>
|
</label>
|
||||||
<select
|
<select id="ttl" name="ttl">
|
||||||
id="ttl"
|
|
||||||
name="ttl"
|
|
||||||
class="w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500"
|
|
||||||
>
|
|
||||||
<option value="0" {{if or (eq .FormTTL "0") (eq .FormTTL "")}}selected{{end}}>Never</option>
|
<option value="0" {{if or (eq .FormTTL "0") (eq .FormTTL "")}}selected{{end}}>Never</option>
|
||||||
<option value="60" {{if eq .FormTTL "60"}}selected{{end}}>1 minute</option>
|
<option value="60" {{if eq .FormTTL "60"}}selected{{end}}>1 minute</option>
|
||||||
<option value="3600" {{if eq .FormTTL "3600"}}selected{{end}}>1 hour</option>
|
<option value="3600" {{if eq .FormTTL "3600"}}selected{{end}}>1 hour</option>
|
||||||
@@ -169,17 +145,15 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<button
|
<button type="submit">
|
||||||
type="submit"
|
|
||||||
class="w-full bg-blue-600 text-white py-2 px-4 rounded-md hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-blue-500 focus:ring-offset-2 transition-colors"
|
|
||||||
>
|
|
||||||
Generate Encrypted URL
|
Generate Encrypted URL
|
||||||
</button>
|
</button>
|
||||||
</form>
|
</form>
|
||||||
|
|
||||||
<p class="text-xs text-gray-500 mt-4 text-center">
|
<p class="note">
|
||||||
Generated URLs are encrypted and cannot be modified. They will expire at the specified time.
|
Generated URLs are encrypted and cannot be modified. They will expire at the specified time.
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
|
<script src="/static/generator.js"></script>
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
|
|||||||
@@ -4,22 +4,22 @@
|
|||||||
<meta charset="UTF-8">
|
<meta charset="UTF-8">
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
<title>Pixa - Login</title>
|
<title>Pixa - Login</title>
|
||||||
<script src="/static/tailwind.js"></script>
|
<link rel="stylesheet" href="/static/style.css">
|
||||||
</head>
|
</head>
|
||||||
<body class="bg-gray-100 min-h-screen flex items-center justify-center">
|
<body class="login">
|
||||||
<div class="bg-white p-8 rounded-lg shadow-md w-full max-w-md">
|
<div class="card">
|
||||||
<h1 class="text-2xl font-bold text-gray-800 mb-6 text-center">Pixa Image Proxy</h1>
|
<h1>Pixa Image Proxy</h1>
|
||||||
|
|
||||||
{{if .Error}}
|
{{if .Error}}
|
||||||
<div class="bg-red-100 border border-red-400 text-red-700 px-4 py-3 rounded mb-4">
|
<div class="error">
|
||||||
{{.Error}}
|
{{.Error}}
|
||||||
</div>
|
</div>
|
||||||
{{end}}
|
{{end}}
|
||||||
|
|
||||||
<form method="POST" action="/" class="space-y-4">
|
<form method="POST" action="/">
|
||||||
{{ .CSRFField }}
|
{{ .CSRFField }}
|
||||||
<div>
|
<div>
|
||||||
<label for="key" class="block text-sm font-medium text-gray-700 mb-1">
|
<label for="key">
|
||||||
Signing Key
|
Signing Key
|
||||||
</label>
|
</label>
|
||||||
<input
|
<input
|
||||||
@@ -28,15 +28,11 @@
|
|||||||
name="key"
|
name="key"
|
||||||
required
|
required
|
||||||
autocomplete="current-password"
|
autocomplete="current-password"
|
||||||
class="w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500"
|
|
||||||
placeholder="Enter your signing key"
|
placeholder="Enter your signing key"
|
||||||
>
|
>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<button
|
<button type="submit">
|
||||||
type="submit"
|
|
||||||
class="w-full bg-blue-600 text-white py-2 px-4 rounded-md hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-blue-500 focus:ring-offset-2 transition-colors"
|
|
||||||
>
|
|
||||||
Login
|
Login
|
||||||
</button>
|
</button>
|
||||||
</form>
|
</form>
|
||||||
|
|||||||
Reference in New Issue
Block a user