Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e09b1a51dd | ||
|
|
48827f0eb5 | ||
|
|
a7aa967890 | ||
|
|
46b3c74846 | ||
|
|
047d53e66a | ||
|
|
9192a06389 | ||
|
|
e216252a57 |
No files matched your search
@@ -71,10 +71,10 @@ The state directory (`state_dir`, `/var/lib/pixa` in the container) holds the
|
|||||||
database and the disk cache:
|
database and the disk cache:
|
||||||
|
|
||||||
- It needs a persistent volume: without one, every restart starts with an empty
|
- It needs a persistent volume: without one, every restart starts with an empty
|
||||||
cache. Losing the directory also ends every login session (see Encrypted
|
cache and forgets the count of logouts, so login sessions that a logout ended
|
||||||
URLs). In the container, the startup script gives the directory to the user
|
can work again (see Encrypted URLs). In the container, the startup script
|
||||||
pixa runs as (uid 65532) and sets its mode to `750`; outside it, that user
|
gives the directory to the user pixa runs as (uid 65532) and sets its mode to
|
||||||
must be able to write the directory.
|
`750`; outside it, that user must be able to write the directory.
|
||||||
- `cache_max_bytes` limits the source and transformed images together. The
|
- `cache_max_bytes` limits the source and transformed images together. The
|
||||||
database, the metadata files, the `.meta` file beside each transformed image
|
database, the metadata files, the `.meta` file beside each transformed image
|
||||||
and files still being written come on top, and eviction runs in the
|
and files still being written come on top, and eviction runs in the
|
||||||
@@ -323,26 +323,9 @@ cache at once, they share one upstream fetch (or one read of the cached source)
|
|||||||
and one transcode: the first request does the work, and the others wait for its
|
and one transcode: the first request does the work, and the others wait for its
|
||||||
image or its error, holding no upstream connection or processing slot of their
|
image or its error, holding no upstream connection or processing slot of their
|
||||||
own. A waiting request stops waiting when its own client goes away. The work
|
own. A waiting request stops waiting when its own client goes away. The work
|
||||||
goes on for the others even if the first request's client goes away. The shared
|
goes on for the others even if the first request's client goes away, until that
|
||||||
fetch sends the first request's ID upstream, and the lines logged for the fetch
|
request's `downstream_timeout` ends. The shared fetch sends the first request's
|
||||||
and the transcode carry that ID.
|
ID upstream, and the lines logged for the fetch and the transcode carry that ID.
|
||||||
|
|
||||||
A transcode whose request's `downstream_timeout` has ended (for a shared one,
|
|
||||||
the first request's) stops waiting for a processing slot, or, once it has one,
|
|
||||||
gives it back without beginning to encode. A request waiting for a shared
|
|
||||||
transcode that stops this way is answered 503 with the error
|
|
||||||
`server busy, try again later`, though its own `downstream_timeout` may not have
|
|
||||||
ended. govips gives pixa no way to stop libvips once an encode has begun, so an
|
|
||||||
encode keeps its slot until it finishes, which can be after
|
|
||||||
`downstream_timeout`. The limit of 8192 on each side of the source and of the
|
|
||||||
output bounds how long that takes. With one libvips thread, as pixad runs it,
|
|
||||||
encoding 8192x8192 random pixels takes up to about 1 second as JPEG, 15 as PNG,
|
|
||||||
16 as GIF, 63 as WebP, 110 as AVIF and 96 as JPEG XL, the slowest of 8 or 16
|
|
||||||
bits per sample, with or without an alpha channel. WebP, AVIF and JPEG XL are
|
|
||||||
that slow only with an alpha channel: 8-bit RGB takes about 44, 53 and 28
|
|
||||||
seconds. Decoding such a source adds up to about 11 seconds. A WebP, AVIF or
|
|
||||||
JPEG XL encode can therefore hold its slot past the default `downstream_timeout`
|
|
||||||
of 60 seconds.
|
|
||||||
|
|
||||||
The login form (`POST /`) is limited to 5 attempts per minute per client
|
The login form (`POST /`) is limited to 5 attempts per minute per client
|
||||||
address, counting an IPv6 client by its /64; an attempt over the limit is
|
address, counting an IPv6 client by its /64; an attempt over the limit is
|
||||||
@@ -382,12 +365,11 @@ nor change what it asks for.
|
|||||||
lasts 30 days, or until someone logs out with the generator page's Logout
|
lasts 30 days, or until someone logs out with the generator page's Logout
|
||||||
button. Every login is made with the one signing key, so logging out ends
|
button. Every login is made with the one signing key, so logging out ends
|
||||||
every login session, in every browser, a copy of the session cookie included.
|
every login session, in every browser, a copy of the session cookie included.
|
||||||
pixa keeps a random value in its database, which it makes when the database
|
pixa keeps a count of logouts in its database: each logout adds one, a login
|
||||||
has none and replaces at each logout. A login session carries the value
|
session carries the count at its login, and a login session whose count is
|
||||||
current at its login and is refused once the value has changed, also after a
|
not the current one is refused, also after a restart that keeps the state
|
||||||
restart (see Deployment). The check for a logout compares no time, so a login
|
directory (see Deployment). No time is compared, so a login after a logout
|
||||||
after a logout works whatever the host's clock says; the 30 days are still
|
works whatever the host's clock says.
|
||||||
measured by the host's clock.
|
|
||||||
2. On the generator page, give the source image's URL, the width and height, the
|
2. On the generator page, give the source image's URL, the width and height, the
|
||||||
format, quality and fit, and how long the URL lasts, then submit the form
|
format, quality and fit, and how long the URL lasts, then submit the form
|
||||||
(`POST /generate`). The format is JPEG XL unless another is chosen; a form
|
(`POST /generate`). The format is JPEG XL unless another is chosen; a form
|
||||||
|
|||||||
@@ -30,41 +30,6 @@ P2: security: per-origin rate limiting
|
|||||||
|
|
||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
- 2026-10-15 a transcode stops once its request's `downstream_timeout` has
|
|
||||||
ended, unless it has begun encoding (closes #243): the image processor looked
|
|
||||||
at its context only while it waited for a processing slot, so an image was
|
|
||||||
decoded and encoded, holding the slot, however long after its request's
|
|
||||||
deadline that took. `Process` now returns the context's error, and gives the
|
|
||||||
slot back, when the context has ended by the time it has the slot or before it
|
|
||||||
encodes. A request that waited for a shared transcode stopped at the first
|
|
||||||
request's deadline is answered 503 `server busy, try again later`, not 500,
|
|
||||||
though its own `downstream_timeout` may not have ended. govips gives pixa no
|
|
||||||
way to stop libvips once an encode has begun, so that encode keeps its slot
|
|
||||||
until it finishes; the limit of 8192 on each side of the source and of the
|
|
||||||
output bounds how long that takes. With one libvips thread, 8192x8192 random
|
|
||||||
pixels take up to about 1 second to encode as JPEG, 15 as PNG, 16 as GIF, 63
|
|
||||||
as WebP, 110 as AVIF and 96 as JPEG XL, the last three with an alpha channel,
|
|
||||||
plus up to about 11 seconds to decode the source; `README.md` gives these
|
|
||||||
times and no longer says the work stops when `downstream_timeout` ends. The
|
|
||||||
settings of the WebP, AVIF and JPEG XL encoders, which can hold a slot past
|
|
||||||
the default `downstream_timeout` of 60 seconds, are unchanged.
|
|
||||||
- 2026-10-15 logging out ends every login session (closes #256): a login session
|
|
||||||
was a cookie with nothing kept on the server, so `/logout` removed it from one
|
|
||||||
browser and a copy of it kept working for its 30 days. Every login is made
|
|
||||||
with the one signing key, so logging out now ends them all: the session
|
|
||||||
manager keeps a random value, made with `crypto/rand` when the new `logout`
|
|
||||||
table has none and replaced by each logout, saves it there so that it outlasts
|
|
||||||
a restart, puts the value at login in each session, and refuses a session
|
|
||||||
carrying another value. The check for a logout compares no time, so a login
|
|
||||||
after a logout works whatever the host's clock does; a session's 30 days are
|
|
||||||
still checked against the clock. Losing the state directory ends every login
|
|
||||||
session, and no logout can make an ended one work again. Logging out is now
|
|
||||||
`POST /logout` from a form on the generator page, behind the CSRF check the
|
|
||||||
other forms have, so a link or image elsewhere on the same site cannot end the
|
|
||||||
sessions. A logout without a valid login session ends none and only removes
|
|
||||||
the cookie, as anyone can take a CSRF token from the login page. A logout that
|
|
||||||
cannot be saved answers 500. `session.NewManager`, which the tests use, keeps
|
|
||||||
the value in memory only; pixad uses `session.NewManagerWithDatabase`.
|
|
||||||
- 2026-10-15 "Running under upaas" in `README.md` tells the operator to set
|
- 2026-10-15 "Running under upaas" in `README.md` tells the operator to set
|
||||||
`PIXA_CACHE_MAX_BYTES` (closes #251): it listed the variable as optional, but
|
`PIXA_CACHE_MAX_BYTES` (closes #251): it listed the variable as optional, but
|
||||||
under upaas the volume is a directory on the host's filesystem. The default,
|
under upaas the volume is a directory on the host's filesystem. The default,
|
||||||
@@ -81,6 +46,22 @@ P2: security: per-origin rate limiting
|
|||||||
405, or a CORS preflight. The metrics library takes the label before the
|
405, or a CORS preflight. The metrics library takes the label before the
|
||||||
request is routed, so the metrics middleware asks the router which route
|
request is routed, so the metrics middleware asks the router which route
|
||||||
matches.
|
matches.
|
||||||
|
- 2026-10-15 logging out ends every login session (closes #256): a login session
|
||||||
|
was a cookie with nothing kept on the server, so `/logout` removed it from one
|
||||||
|
browser and a copy of it kept working for its 30 days. Every login is made
|
||||||
|
with the one signing key, so logging out now ends them all: the session
|
||||||
|
manager counts logouts, saves the count in the new `logout` table so that it
|
||||||
|
outlasts a restart, puts the count at login in each session, and refuses a
|
||||||
|
session whose count is not the current one. No time is compared, so a login
|
||||||
|
after a logout works whatever the host's clock does. A restart without a
|
||||||
|
persistent volume for the state directory forgets the count. Logging out is
|
||||||
|
now `POST /logout` from a form on the generator page, behind the CSRF check
|
||||||
|
the other forms have, so a link or image elsewhere on the same site cannot end
|
||||||
|
the sessions. A logout without a valid login session ends none and only
|
||||||
|
removes the cookie, as anyone can take a CSRF token from the login page. A
|
||||||
|
logout that cannot be saved answers 500. `session.NewManager`, which the tests
|
||||||
|
use, keeps the count in memory only; pixad uses
|
||||||
|
`session.NewManagerWithDatabase`.
|
||||||
- 2026-10-15 expired negative cache rows are removed on a schedule (closes
|
- 2026-10-15 expired negative cache rows are removed on a schedule (closes
|
||||||
#244): the row of a failed fetch, which makes pixa answer 502 for that source
|
#244): the row of a failed fetch, which makes pixa answer 502 for that source
|
||||||
URL for 5 minutes, was deleted only when the same URL was requested again
|
URL for 5 minutes, was deleted only when the same URL was requested again
|
||||||
@@ -584,11 +565,11 @@ P2: security: per-origin rate limiting
|
|||||||
cancellation but keeps its deadline, and the others wait for its image or
|
cancellation but keeps its deadline, and the others wait for its image or
|
||||||
error holding no upstream connection or processing slot, and stop waiting when
|
error holding no upstream connection or processing slot, and stop waiting when
|
||||||
their own context ends; the request doing the processing waits for it even
|
their own context ends; the request doing the processing waits for it even
|
||||||
then, until it ends, which can be after its deadline; a request whose context
|
then, up to its deadline; a request whose context has already ended starts
|
||||||
has already ended starts nothing; each request counts one miss, and the
|
nothing; each request counts one miss, and the processing counts its fetch and
|
||||||
processing counts its fetch and transcode once; a panic while processing is
|
transcode once; a panic while processing is reported to Sentry when
|
||||||
reported to Sentry when `sentry_dsn` is set and becomes an error for every
|
`sentry_dsn` is set and becomes an error for every waiting request instead of
|
||||||
waiting request instead of stopping pixad; documented in `README.md`.
|
stopping pixad; documented in `README.md`.
|
||||||
- 2026-09-29 only the image routes send CORS headers (closes #98): the CORS
|
- 2026-09-29 only the image routes send CORS headers (closes #98): the CORS
|
||||||
middleware, with the `access_control_allow_origin` origin, moved from the
|
middleware, with the `access_control_allow_origin` origin, moved from the
|
||||||
router root onto a `/v1` subrouter holding `/v1/image/` and `/v1/e/`, where it
|
router root onto a `/v1` subrouter holding `/v1/image/` and `/v1/e/`, where it
|
||||||
|
|||||||
@@ -177,10 +177,11 @@ CREATE TABLE IF NOT EXISTS cache_stats (
|
|||||||
);
|
);
|
||||||
INSERT OR IGNORE INTO cache_stats (id) VALUES (1);
|
INSERT OR IGNORE INTO cache_stats (id) VALUES (1);
|
||||||
|
|
||||||
-- A random value that a login session carries from its login. pixa makes it
|
-- Count of logouts. Each logout adds one; a login session carries the count
|
||||||
-- when the table has no row, and each logout replaces it; a login session
|
-- at its login and is refused once the count has changed, so logging out ends
|
||||||
-- carrying another value is refused, so logging out ends every login session.
|
-- every login session.
|
||||||
CREATE TABLE IF NOT EXISTS logout (
|
CREATE TABLE IF NOT EXISTS logout (
|
||||||
id INTEGER PRIMARY KEY CHECK (id = 1),
|
id INTEGER PRIMARY KEY CHECK (id = 1),
|
||||||
logout_value TEXT NOT NULL
|
logout_count INTEGER NOT NULL DEFAULT 0
|
||||||
);
|
);
|
||||||
|
INSERT OR IGNORE INTO logout (id) VALUES (1);
|
||||||
@@ -164,8 +164,8 @@ func (s *Handlers) initImageService() error {
|
|||||||
|
|
||||||
// Initialize session manager (signing key is validated at config load
|
// Initialize session manager (signing key is validated at config load
|
||||||
// time). Session cookies are always Secure/HttpOnly/SameSite=Strict.
|
// time). Session cookies are always Secure/HttpOnly/SameSite=Strict.
|
||||||
// The random value that each logout replaces is saved in the database,
|
// The count of logouts is saved in the database, so it outlasts a
|
||||||
// so it outlasts a restart.
|
// restart.
|
||||||
sessMgr, err := session.NewManagerWithDatabase(
|
sessMgr, err := session.NewManagerWithDatabase(
|
||||||
context.Background(), s.config.SigningKey, s.db.DB())
|
context.Background(), s.config.SigningKey, s.db.DB())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -240,8 +240,7 @@ func (s *Handlers) respondImageError(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if errors.Is(err, httpfetcher.ErrTooManyConnections) ||
|
if errors.Is(err, httpfetcher.ErrTooManyConnections) ||
|
||||||
errors.Is(err, imageprocessor.ErrTooManyImages) ||
|
errors.Is(err, imageprocessor.ErrTooManyImages) {
|
||||||
errors.Is(err, imgcache.ErrProcessingTimedOut) {
|
|
||||||
s.respondError(w, "server busy, try again later",
|
s.respondError(w, "server busy, try again later",
|
||||||
http.StatusServiceUnavailable)
|
http.StatusServiceUnavailable)
|
||||||
|
|
||||||
|
|||||||
@@ -158,8 +158,7 @@ func (s *Handlers) handleImageError(w http.ResponseWriter, err error) {
|
|||||||
case errors.Is(err, httpfetcher.ErrUpstreamTimeout):
|
case errors.Is(err, httpfetcher.ErrUpstreamTimeout):
|
||||||
s.respondError(w, "upstream timeout", http.StatusGatewayTimeout)
|
s.respondError(w, "upstream timeout", http.StatusGatewayTimeout)
|
||||||
case errors.Is(err, httpfetcher.ErrTooManyConnections),
|
case errors.Is(err, httpfetcher.ErrTooManyConnections),
|
||||||
errors.Is(err, imageprocessor.ErrTooManyImages),
|
errors.Is(err, imageprocessor.ErrTooManyImages):
|
||||||
errors.Is(err, imgcache.ErrProcessingTimedOut):
|
|
||||||
s.respondError(w, "server busy, try again later",
|
s.respondError(w, "server busy, try again later",
|
||||||
http.StatusServiceUnavailable)
|
http.StatusServiceUnavailable)
|
||||||
case errors.Is(err, httpfetcher.ErrInvalidContentType),
|
case errors.Is(err, httpfetcher.ErrInvalidContentType),
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
package handlers
|
package handlers
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/http"
|
"net/http"
|
||||||
@@ -48,40 +47,3 @@ func TestServerBusyAnswers503(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestProcessingTimedOutAnswers503 checks that respondImageError and
|
|
||||||
// handleImageError, which answer the errors of the two image routes, answer
|
|
||||||
// 503 with the server busy error when the image service reports that
|
|
||||||
// processing stopped at the deadline of the request doing it, wrapped as the
|
|
||||||
// service wraps it. A request that waited for that processing gets this error
|
|
||||||
// while its own deadline may have time left. The request's source only goes
|
|
||||||
// into a log line, so it is left empty.
|
|
||||||
func TestProcessingTimedOutAnswers503(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
h := &Handlers{log: slog.New(slog.DiscardHandler)}
|
|
||||||
err := fmt.Errorf("%w: failed to encode: %w",
|
|
||||||
imgcache.ErrProcessingTimedOut, context.DeadlineExceeded)
|
|
||||||
|
|
||||||
plain := httptest.NewRecorder()
|
|
||||||
h.respondImageError(plain, &imgcache.ImageRequest{}, err)
|
|
||||||
|
|
||||||
encrypted := httptest.NewRecorder()
|
|
||||||
h.handleImageError(encrypted, err)
|
|
||||||
|
|
||||||
for name, rec := range map[string]*httptest.ResponseRecorder{
|
|
||||||
"respondImageError": plain, "handleImageError": encrypted,
|
|
||||||
} {
|
|
||||||
t.Logf("%s: status = %d, body = %q", name, rec.Code, rec.Body.String())
|
|
||||||
|
|
||||||
if rec.Code != http.StatusServiceUnavailable {
|
|
||||||
t.Errorf("%s: status = %d, want %d",
|
|
||||||
name, rec.Code, http.StatusServiceUnavailable)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !strings.Contains(rec.Body.String(), "server busy, try again later") {
|
|
||||||
t.Errorf("%s: body = %q, want the server busy error",
|
|
||||||
name, rec.Body.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,106 +0,0 @@
|
|||||||
package imageprocessor
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"io"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// readerAfterDeadline is a Process input whose first Read waits until done
|
|
||||||
// is closed, so the deadline passes after Process has taken its slot and
|
|
||||||
// before it encodes.
|
|
||||||
type readerAfterDeadline struct {
|
|
||||||
data *bytes.Reader
|
|
||||||
done <-chan struct{}
|
|
||||||
waited bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *readerAfterDeadline) Read(p []byte) (int, error) {
|
|
||||||
if !r.waited {
|
|
||||||
r.waited = true
|
|
||||||
|
|
||||||
<-r.done
|
|
||||||
}
|
|
||||||
|
|
||||||
return r.data.Read(p)
|
|
||||||
}
|
|
||||||
|
|
||||||
// inputNotToBeRead is a Process input that fails the test when it is read.
|
|
||||||
type inputNotToBeRead struct {
|
|
||||||
t *testing.T
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r inputNotToBeRead) Read([]byte) (int, error) {
|
|
||||||
r.t.Error("Process read its input although its deadline had passed")
|
|
||||||
|
|
||||||
return 0, io.EOF
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestProcessStopsWhenDeadlinePassed checks that Process returns
|
|
||||||
// context.DeadlineExceeded, and holds no slot afterwards, when its context's
|
|
||||||
// deadline has passed before it starts, then without reading its input, or
|
|
||||||
// while it reads its input. govips gives pixa no way to stop libvips once an
|
|
||||||
// encode has begun, so Process must not begin one for a request that has
|
|
||||||
// ended.
|
|
||||||
func TestProcessStopsWhenDeadlinePassed(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
input := createTestJPEG(t, 50, 50)
|
|
||||||
|
|
||||||
t.Run("before Process starts", func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
ctx, cancel := context.WithDeadline(t.Context(),
|
|
||||||
time.Now().Add(-time.Second))
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
// With a slot free, the select in acquireSlot takes the slot or sees
|
|
||||||
// the ended context at random, so Process is called many times
|
|
||||||
for range 100 {
|
|
||||||
checkProcessStopsAtDeadline(ctx, t, inputNotToBeRead{t: t})
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("while Process reads its input", func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
checkProcessStopsAtDeadline(ctx, t, &readerAfterDeadline{
|
|
||||||
data: bytes.NewReader(input), done: ctx.Done(),
|
|
||||||
})
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkProcessStopsAtDeadline runs Process on input with ctx, whose deadline
|
|
||||||
// has passed by the time Process could encode, and fails the test unless
|
|
||||||
// Process returns context.DeadlineExceeded and gives its slot back.
|
|
||||||
func checkProcessStopsAtDeadline(
|
|
||||||
ctx context.Context, t *testing.T, input io.Reader,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
proc := New(Params{MaxConcurrentProcessing: 1})
|
|
||||||
|
|
||||||
result, err := proc.Process(ctx, input, smallJPEGRequest())
|
|
||||||
if result != nil {
|
|
||||||
t.Logf("Process() made a %dx%d image after its deadline passed",
|
|
||||||
result.Width, result.Height)
|
|
||||||
|
|
||||||
_ = result.Content.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Logf("Process() error = %v", err)
|
|
||||||
|
|
||||||
if !errors.Is(err, context.DeadlineExceeded) {
|
|
||||||
t.Errorf("Process() error = %v, want context.DeadlineExceeded", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if held := len(proc.processingSemaphore); held != 0 {
|
|
||||||
t.Errorf("%d slots still held after Process() returned", held)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -208,10 +208,7 @@ func New(params Params) *ImageProcessor {
|
|||||||
// Process transforms an image according to the request. When
|
// Process transforms an image according to the request. When
|
||||||
// MaxConcurrentProcessing images are already being processed, it waits up
|
// MaxConcurrentProcessing images are already being processed, it waits up
|
||||||
// to ProcessingWaitTimeout for one to finish, then fails with
|
// to ProcessingWaitTimeout for one to finish, then fails with
|
||||||
// ErrTooManyImages. When ctx has ended once Process has a slot, or before it
|
// ErrTooManyImages.
|
||||||
// encodes, it returns ctx's error and gives the slot back. govips gives pixa
|
|
||||||
// no way to stop libvips once an encode has begun, so an encode keeps its
|
|
||||||
// slot until it finishes.
|
|
||||||
func (p *ImageProcessor) Process(
|
func (p *ImageProcessor) Process(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
input io.Reader,
|
input io.Reader,
|
||||||
@@ -293,7 +290,7 @@ func (p *ImageProcessor) Process(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Encode to target format
|
// Encode to target format
|
||||||
output, err := p.encode(ctx, img, outputFormat, req.Quality)
|
output, err := p.encode(img, outputFormat, req.Quality)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to encode: %w", err)
|
return nil, fmt.Errorf("failed to encode: %w", err)
|
||||||
}
|
}
|
||||||
@@ -420,28 +417,25 @@ func (p *ImageProcessor) WaitForProcessing(ctx context.Context) int {
|
|||||||
|
|
||||||
// acquireSlot takes a slot in processingSemaphore, waiting at most
|
// acquireSlot takes a slot in processingSemaphore, waiting at most
|
||||||
// processingWaitTimeout for one to free up, and returns the func that gives
|
// processingWaitTimeout for one to free up, and returns the func that gives
|
||||||
// it back. When ctx has ended by the time it has a slot, it gives the slot
|
// it back. A free slot is taken even when ctx has ended; only the wait for
|
||||||
// back and returns ctx's error.
|
// one stops when ctx ends, as the rest of Process does not check ctx.
|
||||||
func (p *ImageProcessor) acquireSlot(ctx context.Context) (func(), error) {
|
func (p *ImageProcessor) acquireSlot(ctx context.Context) (func(), error) {
|
||||||
|
release := func() { <-p.processingSemaphore }
|
||||||
|
|
||||||
select {
|
select {
|
||||||
case p.processingSemaphore <- struct{}{}:
|
case p.processingSemaphore <- struct{}{}:
|
||||||
|
return release, nil
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
select {
|
||||||
|
case p.processingSemaphore <- struct{}{}:
|
||||||
|
return release, nil
|
||||||
case <-time.After(p.processingWaitTimeout):
|
case <-time.After(p.processingWaitTimeout):
|
||||||
return nil, ErrTooManyImages
|
return nil, ErrTooManyImages
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return nil, ctx.Err()
|
return nil, ctx.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
release := func() { <-p.processingSemaphore }
|
|
||||||
|
|
||||||
// select takes the slot even when ctx has ended at the same moment
|
|
||||||
err := ctx.Err()
|
|
||||||
if err != nil {
|
|
||||||
release()
|
|
||||||
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return release, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// detectFormat returns the format string for a govips image type.
|
// detectFormat returns the format string for a govips image type.
|
||||||
@@ -533,17 +527,10 @@ func (p *ImageProcessor) resize(
|
|||||||
|
|
||||||
const defaultQuality = 85
|
const defaultQuality = 85
|
||||||
|
|
||||||
// encode encodes an image to the specified format. govips gives pixa no way
|
// encode encodes an image to the specified format.
|
||||||
// to stop libvips once an encode has begun, so when ctx has ended encode
|
|
||||||
// returns ctx's error without beginning one.
|
|
||||||
func (p *ImageProcessor) encode(
|
func (p *ImageProcessor) encode(
|
||||||
ctx context.Context, img *vips.ImageRef, format Format, quality int,
|
img *vips.ImageRef, format Format, quality int,
|
||||||
) ([]byte, error) {
|
) ([]byte, error) {
|
||||||
err := ctx.Err()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if quality <= 0 {
|
if quality <= 0 {
|
||||||
quality = defaultQuality
|
quality = defaultQuality
|
||||||
}
|
}
|
||||||
@@ -553,7 +540,7 @@ func (p *ImageProcessor) encode(
|
|||||||
// libvips' built-in profile; govips' own sRGB path variable is set on
|
// libvips' built-in profile; govips' own sRGB path variable is set on
|
||||||
// first use but read without a lock, so concurrent requests race on it.
|
// first use but read without a lock, so concurrent requests race on it.
|
||||||
if img.HasICCProfile() {
|
if img.HasICCProfile() {
|
||||||
err = img.TransformICCProfileWithFallback("srgb", "srgb")
|
err := img.TransformICCProfileWithFallback("srgb", "srgb")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to convert to sRGB: %w", err)
|
return nil, fmt.Errorf("failed to convert to sRGB: %w", err)
|
||||||
}
|
}
|
||||||
@@ -645,11 +632,10 @@ func exportWebP(img *vips.ImageRef, quality int) ([]byte, error) {
|
|||||||
|
|
||||||
// avifEffort is the AVIF effort, from 0 (fastest) to 9; 1 is the lowest
|
// avifEffort is the AVIF effort, from 0 (fastest) to 9; 1 is the lowest
|
||||||
// govips can set. With one thread, as pixad runs libvips, 1 takes about 51
|
// govips can set. With one thread, as pixad runs libvips, 1 takes about 51
|
||||||
// seconds to save an 8192x8192 RGB image of random pixels, and about 110
|
// seconds to save an 8192x8192 image of random pixels, the worst case,
|
||||||
// with an alpha channel, the worst case, against the default
|
// against the default downstream_timeout of 60 seconds. On an image of
|
||||||
// downstream_timeout of 60 seconds. On an image of milder noise, which 1
|
// milder noise, which 1 saves in about 12 seconds, 2 takes nearly a minute
|
||||||
// saves in about 12 seconds, 2 takes nearly a minute and libvips' default,
|
// and libvips' default, 4, takes minutes.
|
||||||
// 4, takes minutes.
|
|
||||||
const avifEffort = 1
|
const avifEffort = 1
|
||||||
|
|
||||||
// avifBitdepth is the AVIF bit depth, 8 bits per sample for every image.
|
// avifBitdepth is the AVIF bit depth, 8 bits per sample for every image.
|
||||||
|
|||||||
@@ -15,7 +15,6 @@ import (
|
|||||||
|
|
||||||
"github.com/getsentry/sentry-go"
|
"github.com/getsentry/sentry-go"
|
||||||
"sneak.berlin/go/pixa/internal/httpfetcher"
|
"sneak.berlin/go/pixa/internal/httpfetcher"
|
||||||
"sneak.berlin/go/pixa/internal/imageprocessor"
|
|
||||||
"sneak.berlin/go/pixa/internal/magic"
|
"sneak.berlin/go/pixa/internal/magic"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -436,47 +435,6 @@ func TestService_Get_ReturnsByItsDeadline(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestService_Get_WaiterGetsProcessingTimedOut has a request with no deadline
|
|
||||||
// join the held fetch of a first request with a short one. With the only
|
|
||||||
// processing slot held, the first request's transcode waits for it until the
|
|
||||||
// first request's deadline ends, and stops then. The waiting request, whose
|
|
||||||
// own context has not ended, must get ErrProcessingTimedOut, which the image
|
|
||||||
// routes answer 503 as when pixa is busy.
|
|
||||||
func TestService_Get_WaiterGetsProcessingTimedOut(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
svc, fixtures, fetcher := setupHeldFetchService(t)
|
|
||||||
svc.processor = imageprocessor.New(
|
|
||||||
imageprocessor.Params{MaxConcurrentProcessing: 1},
|
|
||||||
)
|
|
||||||
|
|
||||||
holdProcessingSlot(t, svc.processor)
|
|
||||||
|
|
||||||
const firstTimeout = 500 * time.Millisecond
|
|
||||||
|
|
||||||
firstCtx, cancelFirst := context.WithTimeout(t.Context(), firstTimeout)
|
|
||||||
defer cancelFirst()
|
|
||||||
|
|
||||||
first := startGet(firstCtx, svc, photoVariant(fixtures, 85, FitCover))
|
|
||||||
|
|
||||||
<-fetcher.started
|
|
||||||
|
|
||||||
waiter := startGet(t.Context(), svc, photoVariant(fixtures, 85, FitCover))
|
|
||||||
|
|
||||||
time.Sleep(arrivalWait)
|
|
||||||
fetcher.releaseFetches()
|
|
||||||
|
|
||||||
t.Logf("first request: Get() error = %v", (<-first).err)
|
|
||||||
|
|
||||||
got := <-waiter
|
|
||||||
t.Logf("waiting request: Get() error = %v", got.err)
|
|
||||||
|
|
||||||
if !errors.Is(got.err, ErrProcessingTimedOut) {
|
|
||||||
t.Errorf("waiting request: Get() error = %v, want %v",
|
|
||||||
got.err, ErrProcessingTimedOut)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// panickingFetcher panics on every fetch.
|
// panickingFetcher panics on every fetch.
|
||||||
type panickingFetcher struct{}
|
type panickingFetcher struct{}
|
||||||
|
|
||||||
|
|||||||
@@ -129,12 +129,6 @@ func NewService(cfg *ServiceConfig) (*Service, error) {
|
|||||||
// ErrNegativeCached is returned when a URL is in the negative cache (recently failed).
|
// ErrNegativeCached is returned when a URL is in the negative cache (recently failed).
|
||||||
var ErrNegativeCached = errors.New("request is in negative cache (recently failed)")
|
var ErrNegativeCached = errors.New("request is in negative cache (recently failed)")
|
||||||
|
|
||||||
// ErrProcessingTimedOut is returned when processing an image stopped because
|
|
||||||
// the downstream_timeout of the request doing it ended, to that request and to
|
|
||||||
// every request that waited for its image.
|
|
||||||
var ErrProcessingTimedOut = errors.New(
|
|
||||||
"image processing stopped at the deadline of the request doing it")
|
|
||||||
|
|
||||||
// Get retrieves a processed image, fetching and processing if necessary.
|
// Get retrieves a processed image, fetching and processing if necessary.
|
||||||
// While its source URL is in the negative cache, a cached variant is still
|
// While its source URL is in the negative cache, a cached variant is still
|
||||||
// served and a variant not in the cache is still made from a cached source;
|
// served and a variant not in the cache is still made from a cached source;
|
||||||
@@ -555,13 +549,7 @@ func (s *Service) processAndStore(
|
|||||||
FitMode: imageprocessor.FitMode(req.FitMode),
|
FitMode: imageprocessor.FitMode(req.FitMode),
|
||||||
}
|
}
|
||||||
|
|
||||||
// ctx has the deadline of the request doing the processing, which can
|
|
||||||
// end while a request waiting for its image has time left
|
|
||||||
processResult, err := s.processor.Process(ctx, source, processReq)
|
processResult, err := s.processor.Process(ctx, source, processReq)
|
||||||
if errors.Is(err, context.DeadlineExceeded) {
|
|
||||||
return nil, fmt.Errorf("%w: %w", ErrProcessingTimedOut, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("image processing failed: %w", err)
|
return nil, fmt.Errorf("image processing failed: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
+33
-48
@@ -1,11 +1,10 @@
|
|||||||
// Package session provides encrypted session cookie management, and keeps in
|
// Package session provides encrypted session cookie management, and keeps a
|
||||||
// the database a random value that each logout replaces, so that a logout
|
// count of logouts in the database so that a logout ends every session made
|
||||||
// ends every session made before it.
|
// before it.
|
||||||
package session
|
package session
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/rand"
|
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -40,35 +39,33 @@ type Data struct {
|
|||||||
CreatedAt time.Time `json:"created"`
|
CreatedAt time.Time `json:"created"`
|
||||||
ExpiresAt time.Time `json:"expires"`
|
ExpiresAt time.Time `json:"expires"`
|
||||||
|
|
||||||
// LogoutValue is the Manager's random value, which each logout
|
// LogoutCount is the count of logouts when the session was made.
|
||||||
// replaces, when the session was made.
|
LogoutCount int64 `json:"logoutCount"`
|
||||||
LogoutValue string `json:"logoutValue"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Manager handles session creation and validation using encrypted cookies.
|
// Manager handles session creation and validation using encrypted cookies.
|
||||||
//
|
//
|
||||||
// Every login is made with the one signing key, so a logout ends every
|
// Every login is made with the one signing key, so a logout ends every
|
||||||
// session made before it, in every browser. The Manager keeps a random
|
// session made before it, in every browser. The Manager counts logouts: each
|
||||||
// value, which each logout replaces: a session carries the value current at
|
// logout adds one, a session carries the count at its login, and a session
|
||||||
// its login, and a session carrying another value is refused. The check for
|
// whose count is not the current one is refused. No clock is compared, so a
|
||||||
// a logout compares no time, so a login after a logout works whatever the
|
// login after a logout works whatever the host's clock does. The Manager
|
||||||
// host's clock does; the session's expiry is still checked against the
|
// keeps the count in memory, so checking a session reads no database; a
|
||||||
// clock. The Manager keeps the value in memory, so checking a session reads
|
// Manager from NewManagerWithDatabase also saves it in the database, so that
|
||||||
// no database; a Manager from NewManagerWithDatabase also saves it in the
|
// it outlasts a restart.
|
||||||
// database, so that it outlasts a restart.
|
|
||||||
type Manager struct {
|
type Manager struct {
|
||||||
sc *securecookie.SecureCookie
|
sc *securecookie.SecureCookie
|
||||||
db *sql.DB
|
db *sql.DB
|
||||||
|
|
||||||
// mu guards logoutValue, and is held while a logout is saved, so that
|
// mu guards logoutCount, and is held while a logout is saved, so that
|
||||||
// the database and logoutValue agree.
|
// the database and logoutCount agree.
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
logoutValue string
|
logoutCount int64
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewManager creates a session manager with keys derived from the signing key.
|
// NewManager creates a session manager with keys derived from the signing key.
|
||||||
// It makes a new random value and keeps it in memory only, so a restart ends
|
// It keeps the count of logouts in memory only, so a restart forgets it;
|
||||||
// every session; NewManagerWithDatabase keeps the value in the database.
|
// NewManagerWithDatabase keeps it in the database.
|
||||||
//
|
//
|
||||||
// Session cookies always carry the Secure, HttpOnly, and SameSite=Strict
|
// Session cookies always carry the Secure, HttpOnly, and SameSite=Strict
|
||||||
// attributes; this cannot be configured. Browsers treat http://localhost as a
|
// attributes; this cannot be configured. Browsers treat http://localhost as a
|
||||||
@@ -92,15 +89,13 @@ func NewManager(signingKey string) (*Manager, error) {
|
|||||||
sc.MaxAge(int(SessionTTL.Seconds()))
|
sc.MaxAge(int(SessionTTL.Seconds()))
|
||||||
|
|
||||||
return &Manager{
|
return &Manager{
|
||||||
sc: sc,
|
sc: sc,
|
||||||
logoutValue: rand.Text(),
|
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewManagerWithDatabase creates a session manager like NewManager that keeps
|
// NewManagerWithDatabase creates a session manager like NewManager that saves
|
||||||
// its random value in db, which must have pixa's schema: it saves the value
|
// the count of logouts in db, which must have pixa's schema, and starts from
|
||||||
// NewManager made when db has none, and otherwise starts from the one saved
|
// the count saved there.
|
||||||
// there.
|
|
||||||
func NewManagerWithDatabase(
|
func NewManagerWithDatabase(
|
||||||
ctx context.Context, signingKey string, db *sql.DB,
|
ctx context.Context, signingKey string, db *sql.DB,
|
||||||
) (*Manager, error) {
|
) (*Manager, error) {
|
||||||
@@ -109,20 +104,11 @@ func NewManagerWithDatabase(
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = db.ExecContext(ctx,
|
|
||||||
"INSERT OR IGNORE INTO logout (id, logout_value) VALUES (1, ?)",
|
|
||||||
m.logoutValue)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf(
|
|
||||||
"failed to save the random value in the logout table: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = db.QueryRowContext(ctx,
|
err = db.QueryRowContext(ctx,
|
||||||
"SELECT logout_value FROM logout WHERE id = 1",
|
"SELECT logout_count FROM logout WHERE id = 1",
|
||||||
).Scan(&m.logoutValue)
|
).Scan(&m.logoutCount)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf(
|
return nil, fmt.Errorf("failed to read the count of logouts: %w", err)
|
||||||
"failed to read the random value in the logout table: %w", err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
m.db = db
|
m.db = db
|
||||||
@@ -137,7 +123,7 @@ func (m *Manager) CreateSession(w http.ResponseWriter) error {
|
|||||||
Authenticated: true,
|
Authenticated: true,
|
||||||
CreatedAt: now,
|
CreatedAt: now,
|
||||||
ExpiresAt: now.Add(SessionTTL),
|
ExpiresAt: now.Add(SessionTTL),
|
||||||
LogoutValue: m.currentLogoutValue(),
|
LogoutCount: m.currentLogoutCount(),
|
||||||
}
|
}
|
||||||
|
|
||||||
encoded, err := m.sc.Encode(CookieName, data)
|
encoded, err := m.sc.Encode(CookieName, data)
|
||||||
@@ -187,7 +173,7 @@ func (m *Manager) ValidateSession(r *http.Request) (*Data, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// A logout since the session was made has ended it.
|
// A logout since the session was made has ended it.
|
||||||
if data.LogoutValue != m.currentLogoutValue() {
|
if data.LogoutCount != m.currentLogoutCount() {
|
||||||
return nil, ErrInvalidSession
|
return nil, ErrInvalidSession
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -231,31 +217,30 @@ func (m *Manager) IsAuthenticated(r *http.Request) bool {
|
|||||||
return err == nil && data != nil && data.Authenticated
|
return err == nil && data != nil && data.Authenticated
|
||||||
}
|
}
|
||||||
|
|
||||||
// saveLogout replaces the random value with a new one, saving it in the
|
// saveLogout adds one to the count of logouts, saving the new count in the
|
||||||
// database first when the Manager has one.
|
// database first when the Manager has one.
|
||||||
func (m *Manager) saveLogout(ctx context.Context) error {
|
func (m *Manager) saveLogout(ctx context.Context) error {
|
||||||
value := rand.Text()
|
|
||||||
|
|
||||||
m.mu.Lock()
|
m.mu.Lock()
|
||||||
defer m.mu.Unlock()
|
defer m.mu.Unlock()
|
||||||
|
|
||||||
if m.db != nil {
|
if m.db != nil {
|
||||||
_, err := m.db.ExecContext(ctx,
|
_, err := m.db.ExecContext(ctx,
|
||||||
"UPDATE logout SET logout_value = ? WHERE id = 1", value)
|
"UPDATE logout SET logout_count = ? WHERE id = 1",
|
||||||
|
m.logoutCount+1)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to save the logout: %w", err)
|
return fmt.Errorf("failed to save the logout: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
m.logoutValue = value
|
m.logoutCount++
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// currentLogoutValue returns the Manager's random value.
|
// currentLogoutCount returns the count of logouts.
|
||||||
func (m *Manager) currentLogoutValue() string {
|
func (m *Manager) currentLogoutCount() int64 {
|
||||||
m.mu.Lock()
|
m.mu.Lock()
|
||||||
defer m.mu.Unlock()
|
defer m.mu.Unlock()
|
||||||
|
|
||||||
return m.logoutValue
|
return m.logoutCount
|
||||||
}
|
}
|
||||||
Reference in new issue
Block a user