Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d178113b45 |
@@ -26,11 +26,6 @@ make docker
|
||||
docker run -p 8080:8080 -e PIXA_SIGNING_KEY="$(openssl rand -base64 32)" pixa:latest
|
||||
```
|
||||
|
||||
The example config allowlists no host (`allowlist_hosts`), so pixa serves an
|
||||
image only for a signed or encrypted URL. Every image on an allowlisted host is
|
||||
served to anyone without a signature, at any size and in any format, so
|
||||
allowlist only hosts you control, where nobody else can put images.
|
||||
|
||||
A container takes its settings from environment variables (see Configuration
|
||||
below for the list). Only `PIXA_SIGNING_KEY` is required; if it is unset the
|
||||
container exits at startup naming the variable. Everything else has a built-in
|
||||
@@ -120,9 +115,7 @@ What the [upaas](https://git.eeqj.de/sneak/upaas) app for pixa needs:
|
||||
- `PIXA_SIGNING_KEY` (required): secret for signed and encrypted URLs and
|
||||
login, 32+ characters, for example from `openssl rand -base64 32`
|
||||
- `PIXA_ALLOWLIST_HOSTS`: upstream hosts served without a signature,
|
||||
comma-separated; default none. Every image on a listed host is served to
|
||||
anyone without a signature, so list only hosts you control, where nobody
|
||||
else can put images.
|
||||
comma-separated
|
||||
- `PIXA_CACHE_MAX_BYTES`: disk cache limit in bytes; `0` disables it;
|
||||
default 75% of (free space + what the cache holds)
|
||||
- the rest are in the table under Configuration below
|
||||
@@ -160,14 +153,6 @@ name.
|
||||
Multiple source paths may reference the same content blob; the database tracks
|
||||
references rather than using filesystem refcounting.
|
||||
|
||||
Source and transformed image files, and the `.meta` file beside each transformed
|
||||
image, are written under a temporary name, synced to disk, and only then renamed
|
||||
into place, so a crash leaves no file under its own name with part of its data.
|
||||
When pixa starts, and every five minutes after, it removes each cached source or
|
||||
transformed image whose file is empty, or not the size the database records for
|
||||
it, with a warning naming the file; the next request for it fetches the source
|
||||
or converts the image again.
|
||||
|
||||
pixa's target is 1-5k r/s, which has not been measured at that rate (see Load
|
||||
Test). Toward it, pixa keeps in memory the content types of the 10,000
|
||||
transformed images most recently cached or served, so a cache hit on one of them
|
||||
@@ -205,12 +190,9 @@ path under `/v1/` answers 200, in maintenance mode too.
|
||||
the cache and the upstream fetch; 403 when the upstream host, or a host it
|
||||
redirects to, is `localhost`, ends in `.localhost` or `.local`, or has an
|
||||
address in a blocked network (see `blocked_networks`); 429 over
|
||||
`image_requests_per_minute` (below); 415 when the upstream answers with a
|
||||
`Content-Type` that is not one of the source formats (below), or with an image
|
||||
whose first bytes libvips reads as another format; 502 when the upstream
|
||||
answered with an error status, and for 5 minutes after that for the same
|
||||
source URL; 503 when pixa is busy or in maintenance mode; 500 for any other
|
||||
failure.
|
||||
`image_requests_per_minute` (below); 502 when the upstream answered with an
|
||||
error status, and for 5 minutes after that for the same source URL; 503 when
|
||||
pixa is busy or in maintenance mode; 500 for any other failure.
|
||||
- `GET` or `HEAD` `/v1/e/<token>/<name>` — an image through an encrypted URL
|
||||
(see Encrypted URLs). Needs: nothing but the URL. Answers: 200; 304 when
|
||||
`If-None-Match` matches the image's `ETag`; 400 for a token that does not
|
||||
@@ -218,7 +200,7 @@ path under `/v1/` answers 200, in maintenance mode too.
|
||||
expired; 504 when the upstream has not sent its response headers within
|
||||
`upstream_fetch_timeout`, but 500 when that time runs out while the image
|
||||
itself is still arriving; 400 for an `Accept` header that is not valid, and
|
||||
406, 403, 415, 429, 502, 503 and 500, as for `/v1/image/`.
|
||||
406, 403, 429, 502, 503 and 500, as for `/v1/image/`.
|
||||
- `GET /robots.txt` — asks every crawler to stay away (`Disallow: /`). Needs:
|
||||
nothing. Answers: 200.
|
||||
- `GET /.well-known/healthcheck.json` — JSON with `status` (`ok`), `now`,
|
||||
@@ -278,17 +260,9 @@ once, is refused with 400.
|
||||
both optional (values under Signature Specification). Both are part of what is
|
||||
cached, so each value of either is a separate cached image.
|
||||
|
||||
A size with one side `0` has that side scaled to keep the source's proportions,
|
||||
and is refused with 400 when that side would come out above 8192, the most
|
||||
either side may be; so is a size with `fit=outside`, which scales the image to
|
||||
cover the size and so past it on one side, when that side would come out
|
||||
above 8192.
|
||||
|
||||
The source image may be JPEG, PNG, GIF, WebP, AVIF or JPEG XL. The upstream's
|
||||
`Content-Type` must name its format, and the image's first bytes must match it.
|
||||
An upstream answering with any other `Content-Type`, SVG included, is refused
|
||||
with 415, and so is an image whose first bytes libvips reads as another format,
|
||||
such as an SVG whose first bytes pass the check for AVIF.
|
||||
The source image may be JPEG, PNG, GIF, WebP, AVIF, JPEG XL or SVG. The
|
||||
upstream's `Content-Type` must name its format, and the image's first bytes must
|
||||
match it.
|
||||
|
||||
With the format `auto`, pixa chooses the format for each request from its
|
||||
`Accept` header, in this order:
|
||||
@@ -366,8 +340,7 @@ nor change what it asks for.
|
||||
(`POST /generate`). The format is JPEG XL unless another is chosen; a form
|
||||
sent with an empty format, or none, also makes a JPEG XL URL. Width and
|
||||
height both empty or `0` keep the original size; if only one of them is empty
|
||||
or `0`, that side is scaled to keep the image's proportions. A URL whose
|
||||
scaled side would come out above 8192 answers 400.
|
||||
or `0`, that side is scaled to keep the image's proportions.
|
||||
3. The page shows the URL, `https://<host>/v1/e/<token>/img.<format>`, and when
|
||||
it expires. `<host>` is the host the page was opened on, and the URL starts
|
||||
with `http` instead while `debug` is on. The name after the token is ignored
|
||||
|
||||
@@ -30,63 +30,11 @@ P2: security: per-origin rate limiting
|
||||
|
||||
# Completed Steps
|
||||
|
||||
- 2026-10-15 a cached file left empty or partial is no longer served, or fetched
|
||||
again on every request (closes #252). Source and transformed image files, and
|
||||
each transformed image's `.meta` file, are synced to disk before they are
|
||||
renamed into place, and their directory after; the `.meta` file is written
|
||||
before the image, and a failed directory sync removes the renamed file. The
|
||||
reconciliation pass, at startup and every five minutes, removes a cached
|
||||
source or transformed image whose file is empty, or not the size the database
|
||||
records for it, with its rows and a warning naming the file, so the next
|
||||
request fetches the source or converts the image again; requests compare no
|
||||
sizes. Storing a source over a file of another size replaces that file. A
|
||||
source that fails to decode is not removed for that alone: a damaged upstream
|
||||
image would then be fetched on every request.
|
||||
- 2026-10-15 the example config allowlists no host (closes #249): it allowlisted
|
||||
`github.com` and `user-images.githubusercontent.com`, where anyone can put an
|
||||
image, so a deployment set up as Getting Started says served those images to
|
||||
anyone without a signature, at any size and in any format.
|
||||
`configs/config.example.yml` now leaves `allowlist_hosts` out, with one
|
||||
commented entry under `example.com`. Getting Started and "Running under upaas"
|
||||
in `README.md` say that every image on an allowlisted host is served without a
|
||||
signature, so an allowlisted host must be one the operator controls.
|
||||
- 2026-10-13 `fit=outside` no longer makes an output above 8192 on a side
|
||||
(closes #257): it scales the image to cover the requested size, and the side
|
||||
scaled past that size was held to no maximum, so `1x8192?fit=outside` of a
|
||||
3000x600 source was resized to 40960x8192 and encoded. The image processor now
|
||||
refuses such a size with `ErrOutputTooLarge` before resizing, and both image
|
||||
routes answer it 400, as for a size with one side `0`. The other fit modes
|
||||
make the requested size or one within it, so none can go above 8192.
|
||||
- 2026-10-13 a size with one side `0` no longer makes an output above 8192 on a
|
||||
side (closes #241): that side is computed from the source's proportions, and
|
||||
nothing held it to the 8192 a requested side is held to, so `0x8192` of a
|
||||
3000x600 source was resized to 40960x8192 and encoded. The image processor now
|
||||
refuses such a size with `ErrOutputTooLarge` once it knows the source's
|
||||
dimensions, before resizing, and both image routes answer it 400 with the JSON
|
||||
error body. The refusal is not cached as a failed fetch, so the same source is
|
||||
still served at other sizes. `imgcache.MaxDimension` is now the image
|
||||
processor's `MaxOutputDimension`, so the two cannot differ.
|
||||
- 2026-10-13 the unused cache lifetime is removed (closes #69):
|
||||
`CacheConfig.CacheTTL`, which pixad set to `DefaultCacheTTL` (24 hours), was
|
||||
never read, so cached images never expired. The field and the constant are
|
||||
removed. Cached source and transformed images stay until eviction removes them
|
||||
to keep the cache within `cache_max_bytes`.
|
||||
- 2026-10-13 SVG sources are refused, and `orig` is never saved as JPEG in place
|
||||
of the source's own format (closes #68): the upstream fetch no longer accepts
|
||||
`image/svg+xml`, which libvips would render. Both image routes answer 415 for
|
||||
an upstream `Content-Type` that is not one of the source formats, SVG
|
||||
included, where they answered 500. Before decoding, the image processor
|
||||
refuses with `ErrUnsupportedInputFormat`, also answered 415, any input that
|
||||
govips' `DetermineImageType`, which picks the libvips loader from the first
|
||||
bytes, does not name as a source format. That catches an SVG whose first bytes
|
||||
pass the first-bytes check for another type, as an XML comment can make them
|
||||
do for AVIF; an SVG whose first bytes do not pass that check answers 500, as
|
||||
before, and so does a damaged image in a source format, whose first bytes
|
||||
`DetermineImageType` names. `orig` is always the source's own format, where a
|
||||
source the processor could not name was saved as JPEG. `README.md` no longer
|
||||
lists SVG. The SVG detection in `internal/magic` stays: the request path does
|
||||
not call it, and it goes with the unused code of
|
||||
https://git.eeqj.de/sneak/pixa/issues/168.
|
||||
- 2026-10-12 per-client limit on the image routes (closes #236): the new setting
|
||||
`image_requests_per_minute` (`PIXA_IMAGE_REQUESTS_PER_MINUTE`), default 600,
|
||||
is how many `GET` and `HEAD` requests one client may send to `/v1/image/` and
|
||||
|
||||
@@ -45,14 +45,15 @@ state_dir: ./data
|
||||
signing_key: "CHANGE_ME_generate_with_openssl_rand_base64_32"
|
||||
|
||||
# Hosts that don't require signatures (default: none)
|
||||
# Every image on an allowlisted host is served to anyone without a
|
||||
# signature, at any size and in any format, so list only hosts you
|
||||
# control, where nobody else can put images.
|
||||
# Use "." prefix for wildcard subdomain matching (e.g., ".example.com" matches "cdn.example.com")
|
||||
# An entry that is neither a host name nor an IP address (IPv6 without
|
||||
# brackets), such as one with a port or a "*." wildcard, aborts startup.
|
||||
# allowlist_hosts:
|
||||
# - images.example.com
|
||||
allowlist_hosts:
|
||||
- s3.sneak.cloud
|
||||
- static.sneak.cloud
|
||||
- sneak.berlin
|
||||
- github.com
|
||||
- user-images.githubusercontent.com
|
||||
|
||||
# Hosts whose pages may not show pixa's images, written as for
|
||||
# allowlist_hosts. A request to /v1/image/ or /v1/e/ whose Referer header
|
||||
|
||||
@@ -92,32 +92,6 @@ func TestGetStringSlice_Empty(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestExampleConfigAllowlistsNoHost loads configs/config.example.yml
|
||||
// with a real signing key, as Getting Started in README.md has it copied
|
||||
// and edited, and checks that it allowlists no host: every image on an
|
||||
// allowlisted host is served to anyone without a signature.
|
||||
func TestExampleConfigAllowlistsNoHost(t *testing.T) {
|
||||
t.Setenv("PIXA_SIGNING_KEY", validTestSigningKey)
|
||||
|
||||
examplePath := filepath.Join("..", "..", "configs", "config.example.yml")
|
||||
|
||||
sc, err := loadTestConfig(examplePath)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to load %s: %v", examplePath, err)
|
||||
}
|
||||
|
||||
c, err := newFromSmartConfig(sc)
|
||||
if err != nil {
|
||||
t.Fatalf("%s is not a valid config: %v", examplePath, err)
|
||||
}
|
||||
|
||||
t.Logf("allowlist_hosts in %s: %v", examplePath, c.AllowlistHosts)
|
||||
|
||||
if len(c.AllowlistHosts) != 0 {
|
||||
t.Errorf("%s allowlists %v, want no host", examplePath, c.AllowlistHosts)
|
||||
}
|
||||
}
|
||||
|
||||
// loadTestConfig is a helper to load a config file for testing.
|
||||
func loadTestConfig(path string) (*smartconfig.Config, error) {
|
||||
return smartconfig.NewFromConfigPath(path)
|
||||
|
||||
@@ -247,22 +247,6 @@ func (s *Handlers) respondImageError(
|
||||
return
|
||||
}
|
||||
|
||||
if errors.Is(err, httpfetcher.ErrInvalidContentType) ||
|
||||
errors.Is(err, imageprocessor.ErrUnsupportedInputFormat) {
|
||||
s.respondError(w, "unsupported source format",
|
||||
http.StatusUnsupportedMediaType)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if errors.Is(err, imageprocessor.ErrOutputTooLarge) {
|
||||
s.respondError(w, fmt.Sprintf("size too large for the source: the side "+
|
||||
"computed from its proportions is above %d",
|
||||
imageprocessor.MaxOutputDimension), http.StatusBadRequest)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
s.respondError(w, "internal error", http.StatusInternalServerError)
|
||||
}
|
||||
|
||||
|
||||
@@ -161,14 +161,6 @@ func (s *Handlers) handleImageError(w http.ResponseWriter, err error) {
|
||||
errors.Is(err, imageprocessor.ErrTooManyImages):
|
||||
s.respondError(w, "server busy, try again later",
|
||||
http.StatusServiceUnavailable)
|
||||
case errors.Is(err, httpfetcher.ErrInvalidContentType),
|
||||
errors.Is(err, imageprocessor.ErrUnsupportedInputFormat):
|
||||
s.respondError(w, "unsupported source format",
|
||||
http.StatusUnsupportedMediaType)
|
||||
case errors.Is(err, imageprocessor.ErrOutputTooLarge):
|
||||
s.respondError(w, fmt.Sprintf("size too large for the source: the side "+
|
||||
"computed from its proportions is above %d",
|
||||
imageprocessor.MaxOutputDimension), http.StatusBadRequest)
|
||||
default:
|
||||
s.log.Error("image request failed", "error", err)
|
||||
s.respondError(w, "internal error", http.StatusInternalServerError)
|
||||
|
||||
@@ -1,54 +0,0 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/httpfetcher"
|
||||
"sneak.berlin/go/pixa/internal/imageprocessor"
|
||||
"sneak.berlin/go/pixa/internal/imgcache"
|
||||
)
|
||||
|
||||
// TestUnsupportedSourceFormatAnswers415 checks that both image routes answer
|
||||
// 415 with a clear error when the upstream answers with a content type pixa
|
||||
// does not accept, such as SVG, and when orig is asked of a source in a
|
||||
// format the processor cannot name, wrapped as the service wraps them.
|
||||
func TestUnsupportedSourceFormatAnswers415(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
h := &Handlers{log: slog.New(slog.DiscardHandler)}
|
||||
req := &imgcache.ImageRequest{SourceHost: "img.example.com", SourcePath: "/a.svg"}
|
||||
|
||||
for _, err := range []error{
|
||||
fmt.Errorf("upstream fetch failed: %w: image/svg+xml",
|
||||
httpfetcher.ErrInvalidContentType),
|
||||
fmt.Errorf("image processing failed: %w: unknown",
|
||||
imageprocessor.ErrUnsupportedInputFormat),
|
||||
} {
|
||||
plain := httptest.NewRecorder()
|
||||
h.respondImageError(plain, req, err)
|
||||
|
||||
encrypted := httptest.NewRecorder()
|
||||
h.handleImageError(encrypted, err)
|
||||
|
||||
for route, rec := range map[string]*httptest.ResponseRecorder{
|
||||
"HandleImage": plain, "HandleImageEnc": encrypted,
|
||||
} {
|
||||
t.Logf("%s for %v: %d %s", route, err, rec.Code, rec.Body.String())
|
||||
|
||||
if rec.Code != http.StatusUnsupportedMediaType {
|
||||
t.Errorf("%s for %v: status = %d, want %d",
|
||||
route, err, rec.Code, http.StatusUnsupportedMediaType)
|
||||
}
|
||||
|
||||
if !strings.Contains(rec.Body.String(), "unsupported source format") {
|
||||
t.Errorf("%s for %v: body = %q, want the unsupported source format error",
|
||||
route, err, rec.Body.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -158,6 +158,7 @@ func DefaultConfig() *Config {
|
||||
contentTypeWebP,
|
||||
contentTypeAVIF,
|
||||
contentTypeJXL,
|
||||
contentTypeSVG,
|
||||
},
|
||||
AllowHTTP: false,
|
||||
MaxConnectionsPerHost: DefaultMaxConnectionsPerHost,
|
||||
|
||||
@@ -1,52 +0,0 @@
|
||||
package imageprocessor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestImageProcessor_RefusesComputedSideAboveMaximum verifies that a size
|
||||
// with one side 0, which Process computes from the input's proportions, is
|
||||
// served when that side comes out at MaxOutputDimension and refused with
|
||||
// ErrOutputTooLarge when it would be above. A 4096x1 or 1x4096 input given
|
||||
// 2 for its short side has 8192 for its long side.
|
||||
func TestImageProcessor_RefusesComputedSideAboveMaximum(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
inputWidth, inputHeight int
|
||||
size Size
|
||||
wantErr error
|
||||
}{
|
||||
{"width at the maximum", 4096, 1, Size{Height: 2}, nil},
|
||||
{"width above the maximum", 4096, 1, Size{Height: 3}, ErrOutputTooLarge},
|
||||
{"height at the maximum", 1, 4096, Size{Width: 2}, nil},
|
||||
{"height above the maximum", 1, 4096, Size{Width: 3}, ErrOutputTooLarge},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
input := createTestJPEG(t, tt.inputWidth, tt.inputHeight)
|
||||
|
||||
result, err := New(Params{}).Process(t.Context(),
|
||||
bytes.NewReader(input),
|
||||
&Request{Size: tt.size, Format: FormatJPEG, FitMode: FitCover})
|
||||
if result != nil {
|
||||
t.Logf("Process() made %dx%d", result.Width, result.Height)
|
||||
|
||||
if max(result.Width, result.Height) != MaxOutputDimension {
|
||||
t.Errorf("Process() made %dx%d, want a side of %d",
|
||||
result.Width, result.Height, MaxOutputDimension)
|
||||
}
|
||||
}
|
||||
|
||||
if !errors.Is(err, tt.wantErr) {
|
||||
t.Errorf("Process() error = %v, want %v", err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
package imageprocessor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestImageProcessor_RefusesFitOutsideAboveMaximum verifies that FitOutside,
|
||||
// which scales the input to cover the requested size and so past it on one
|
||||
// side, is refused with ErrOutputTooLarge when that side would come out above
|
||||
// MaxOutputDimension, and otherwise makes the scaled size. A 1024x256 or
|
||||
// 256x1024 input scaled by 8 has 8192 for its long side.
|
||||
func TestImageProcessor_RefusesFitOutsideAboveMaximum(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
inputWidth, inputHeight int
|
||||
size Size
|
||||
// wantWidth and wantHeight are the output's, 0 for a refusal
|
||||
wantWidth, wantHeight int
|
||||
}{
|
||||
{"width above the maximum", 3000, 600, Size{Width: 1, Height: 8192}, 0, 0},
|
||||
{"height above the maximum", 600, 3000, Size{Width: 8192, Height: 1}, 0, 0},
|
||||
{"width at the maximum", 1024, 256, Size{Width: 1, Height: 2048}, 8192, 2048},
|
||||
{"height at the maximum", 256, 1024, Size{Width: 2048, Height: 1}, 2048, 8192},
|
||||
{"within the maximum", 3000, 600, Size{Width: 1, Height: 1200}, 6000, 1200},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
input := createTestJPEG(t, tt.inputWidth, tt.inputHeight)
|
||||
|
||||
result, err := New(Params{}).Process(t.Context(),
|
||||
bytes.NewReader(input),
|
||||
&Request{Size: tt.size, Format: FormatJPEG, FitMode: FitOutside})
|
||||
if result != nil {
|
||||
t.Logf("Process() made %dx%d", result.Width, result.Height)
|
||||
}
|
||||
|
||||
if tt.wantWidth == 0 {
|
||||
if !errors.Is(err, ErrOutputTooLarge) {
|
||||
t.Errorf("Process() error = %v, want %v", err, ErrOutputTooLarge)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("Process() error = %v", err)
|
||||
}
|
||||
|
||||
if result.Width != tt.wantWidth || result.Height != tt.wantHeight {
|
||||
t.Errorf("Process() made %dx%d, want %dx%d",
|
||||
result.Width, result.Height, tt.wantWidth, tt.wantHeight)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -133,26 +133,10 @@ var ErrInputTooLarge = errors.New("input image dimensions exceed maximum")
|
||||
// configured byte limit.
|
||||
var ErrInputDataTooLarge = errors.New("input data exceeds maximum allowed size")
|
||||
|
||||
// MaxOutputDimension is the maximum width or height of an output image, and
|
||||
// so of a requested size (imgcache.MaxDimension).
|
||||
const MaxOutputDimension = 8192
|
||||
|
||||
// ErrOutputTooLarge is returned, before resizing, when the output would have a
|
||||
// side above MaxOutputDimension: the side of a size with one side 0, computed
|
||||
// from the input's proportions, or with FitOutside, the side scaled past the
|
||||
// requested size.
|
||||
var ErrOutputTooLarge = errors.New("output image dimensions exceed maximum")
|
||||
|
||||
// ErrUnsupportedOutputFormat is returned when the requested output format is
|
||||
// not supported.
|
||||
var ErrUnsupportedOutputFormat = errors.New("unsupported output format")
|
||||
|
||||
// ErrUnsupportedInputFormat is returned, before the input is decoded, when
|
||||
// libvips reads the input's first bytes as another format than the source
|
||||
// formats, or as none: for example an SVG whose first bytes pass the
|
||||
// first-bytes check for AVIF.
|
||||
var ErrUnsupportedInputFormat = errors.New("unsupported input format")
|
||||
|
||||
// ErrTooManyImages is returned when MaxConcurrentProcessing images are being
|
||||
// processed and none finishes within ProcessingWaitTimeout.
|
||||
var ErrTooManyImages = errors.New("too many images being processed at once")
|
||||
@@ -235,17 +219,6 @@ func (p *ImageProcessor) Process(
|
||||
return nil, ErrInputDataTooLarge
|
||||
}
|
||||
|
||||
// govips picks the libvips loader from the input's first bytes with
|
||||
// DetermineImageType. Input whose first bytes it reads as no source
|
||||
// format, such as SVG, which librsvg would render, is refused before
|
||||
// libvips reads it.
|
||||
inputFormat := p.detectFormat(vips.DetermineImageType(data))
|
||||
|
||||
sourceFormat, err := p.formatFromString(inputFormat)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Decode image
|
||||
img, err := vips.NewImageFromBuffer(data)
|
||||
if err != nil {
|
||||
@@ -264,16 +237,16 @@ func (p *ImageProcessor) Process(
|
||||
origWidth := img.Width()
|
||||
origHeight := img.Height()
|
||||
|
||||
// Detect input format
|
||||
inputFormat := p.detectFormat(img)
|
||||
|
||||
// Validate input dimensions to prevent DoS via decompression bombs
|
||||
if origWidth > MaxInputDimension || origHeight > MaxInputDimension {
|
||||
return nil, ErrInputTooLarge
|
||||
}
|
||||
|
||||
// Determine target dimensions; too large ones are refused before resizing
|
||||
targetWidth, targetHeight, err := targetDimensions(req.Size, origWidth, origHeight)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Determine target dimensions
|
||||
targetWidth, targetHeight := targetDimensions(req.Size, origWidth, origHeight)
|
||||
|
||||
// Resize if needed
|
||||
if targetWidth != origWidth || targetHeight != origHeight {
|
||||
@@ -286,7 +259,7 @@ func (p *ImageProcessor) Process(
|
||||
// orig is the source's own format; encode refuses an empty format
|
||||
outputFormat := req.Format
|
||||
if outputFormat == FormatOriginal {
|
||||
outputFormat = sourceFormat
|
||||
outputFormat = p.formatFromString(inputFormat)
|
||||
}
|
||||
|
||||
// Encode to target format
|
||||
@@ -309,29 +282,21 @@ func (p *ImageProcessor) Process(
|
||||
|
||||
// targetDimensions calculates the output dimensions for a requested size,
|
||||
// scaling proportionally when only one dimension is given and keeping the
|
||||
// original dimensions when both are zero. A side scaled proportionally can
|
||||
// come out above MaxOutputDimension, which is ErrOutputTooLarge.
|
||||
func targetDimensions(size Size, origWidth, origHeight int) (int, int, error) {
|
||||
width, height := size.Width, size.Height
|
||||
|
||||
// original dimensions when both are zero.
|
||||
func targetDimensions(size Size, origWidth, origHeight int) (int, int) {
|
||||
switch {
|
||||
case width == 0 && height == 0:
|
||||
case size.Width == 0 && size.Height == 0:
|
||||
// Both are 0: keep original size
|
||||
return origWidth, origHeight, nil
|
||||
case width == 0:
|
||||
return origWidth, origHeight
|
||||
case size.Width == 0:
|
||||
// Only height specified: calculate width proportionally
|
||||
width = origWidth * height / origHeight
|
||||
case height == 0:
|
||||
return origWidth * size.Height / origHeight, size.Height
|
||||
case size.Height == 0:
|
||||
// Only width specified: calculate height proportionally
|
||||
height = origHeight * width / origWidth
|
||||
return size.Width, origHeight * size.Width / origWidth
|
||||
default:
|
||||
return size.Width, size.Height
|
||||
}
|
||||
|
||||
if width > MaxOutputDimension || height > MaxOutputDimension {
|
||||
return 0, 0, fmt.Errorf("%w: %dx%d is above %d", ErrOutputTooLarge,
|
||||
width, height, MaxOutputDimension)
|
||||
}
|
||||
|
||||
return width, height, nil
|
||||
}
|
||||
|
||||
// MIME types for the supported image formats.
|
||||
@@ -438,9 +403,10 @@ func (p *ImageProcessor) acquireSlot(ctx context.Context) (func(), error) {
|
||||
}
|
||||
}
|
||||
|
||||
// detectFormat returns the format string for a govips image type.
|
||||
func (p *ImageProcessor) detectFormat(imageType vips.ImageType) string {
|
||||
switch imageType {
|
||||
// detectFormat returns the format string from a vips image.
|
||||
func (p *ImageProcessor) detectFormat(img *vips.ImageRef) string {
|
||||
format := img.Format()
|
||||
switch format {
|
||||
case vips.ImageTypeJPEG:
|
||||
return "jpeg"
|
||||
case vips.ImageTypePNG:
|
||||
@@ -462,9 +428,7 @@ func (p *ImageProcessor) detectFormat(imageType vips.ImageType) string {
|
||||
}
|
||||
}
|
||||
|
||||
// resize resizes the image according to the fit mode. With FitOutside one
|
||||
// side can come out larger than requested; when it would be above
|
||||
// MaxOutputDimension, resize returns ErrOutputTooLarge without resizing.
|
||||
// resize resizes the image according to the fit mode.
|
||||
func (p *ImageProcessor) resize(
|
||||
img *vips.ImageRef, width, height int, fit FitMode,
|
||||
) error {
|
||||
@@ -512,12 +476,6 @@ func (p *ImageProcessor) resize(
|
||||
newW := int(float64(imgW) * scale)
|
||||
newH := int(float64(imgH) * scale)
|
||||
|
||||
// The dimension that exceeds the target is held to the maximum too
|
||||
if newW > MaxOutputDimension || newH > MaxOutputDimension {
|
||||
return fmt.Errorf("%w: %dx%d is above %d", ErrOutputTooLarge,
|
||||
newW, newH, MaxOutputDimension)
|
||||
}
|
||||
|
||||
return img.Thumbnail(newW, newH, vips.InterestingNone)
|
||||
|
||||
default:
|
||||
@@ -729,23 +687,22 @@ func jxlDistance(quality int) float64 {
|
||||
return 53.0/3000.0*q*q - 23.0/20.0*q + 25.0
|
||||
}
|
||||
|
||||
// formatFromString converts a format string from detectFormat to Format. A
|
||||
// format detectFormat could not name is ErrUnsupportedInputFormat.
|
||||
func (p *ImageProcessor) formatFromString(format string) (Format, error) {
|
||||
// formatFromString converts a format string to Format.
|
||||
func (p *ImageProcessor) formatFromString(format string) Format {
|
||||
switch format {
|
||||
case "jpeg":
|
||||
return FormatJPEG, nil
|
||||
return FormatJPEG
|
||||
case "png":
|
||||
return FormatPNG, nil
|
||||
return FormatPNG
|
||||
case "gif":
|
||||
return FormatGIF, nil
|
||||
return FormatGIF
|
||||
case "webp":
|
||||
return FormatWebP, nil
|
||||
return FormatWebP
|
||||
case string(FormatAVIF):
|
||||
return FormatAVIF, nil
|
||||
return FormatAVIF
|
||||
case string(FormatJXL):
|
||||
return FormatJXL, nil
|
||||
return FormatJXL
|
||||
default:
|
||||
return "", fmt.Errorf("%w: %s", ErrUnsupportedInputFormat, format)
|
||||
return FormatJPEG
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
package imageprocessor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/davidbyttow/govips/v2/vips"
|
||||
)
|
||||
|
||||
// TestImageProcessor_OrigOfUnnamedFormatRefused verifies that orig of a TIFF
|
||||
// input, which is not a source format, is refused with
|
||||
// ErrUnsupportedInputFormat before it is decoded, and not saved as JPEG.
|
||||
func TestImageProcessor_OrigOfUnnamedFormatRefused(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
img, err := vips.NewImageFromBuffer(createTestPNG(t, 20, 20))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to decode test PNG: %v", err)
|
||||
}
|
||||
|
||||
t.Cleanup(img.Close)
|
||||
|
||||
tiff, _, err := img.ExportTiff(vips.NewTiffExportParams())
|
||||
if err != nil {
|
||||
t.Fatalf("failed to save test TIFF: %v", err)
|
||||
}
|
||||
|
||||
result, err := New(Params{}).Process(
|
||||
t.Context(), bytes.NewReader(tiff), &Request{Format: FormatOriginal},
|
||||
)
|
||||
if result != nil {
|
||||
t.Logf("Process() returned %s", result.ContentType)
|
||||
}
|
||||
|
||||
if !errors.Is(err, ErrUnsupportedInputFormat) {
|
||||
t.Errorf("Process() error = %v, want %v", err, ErrUnsupportedInputFormat)
|
||||
}
|
||||
}
|
||||
@@ -1,46 +0,0 @@
|
||||
package imageprocessor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestImageProcessor_LoadsEachSourceFormat verifies that an image in each of
|
||||
// the six source formats, saved by the processor itself, is loaded and saved
|
||||
// again in its own format (orig).
|
||||
func TestImageProcessor_LoadsEachSourceFormat(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
proc := New(Params{})
|
||||
|
||||
for _, format := range []Format{
|
||||
FormatJPEG, FormatPNG, FormatGIF, FormatWebP, FormatAVIF, FormatJXL,
|
||||
} {
|
||||
saved, err := proc.Process(t.Context(),
|
||||
bytes.NewReader(createTestPNG(t, 20, 20)), &Request{Format: format})
|
||||
if err != nil {
|
||||
t.Fatalf("saving the test image as %s: %v", format, err)
|
||||
}
|
||||
|
||||
source, err := io.ReadAll(saved.Content)
|
||||
if err != nil {
|
||||
t.Fatalf("reading the %s image: %v", format, err)
|
||||
}
|
||||
|
||||
result, err := proc.Process(t.Context(), bytes.NewReader(source),
|
||||
&Request{Format: FormatOriginal})
|
||||
if err != nil {
|
||||
t.Errorf("orig of the %s image: %v", format, err)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
t.Logf("orig of the %s image: %s", format, result.ContentType)
|
||||
|
||||
if result.ContentType != FormatToMIME(format) {
|
||||
t.Errorf("orig of the %s image is %s, want %s",
|
||||
format, result.ContentType, FormatToMIME(format))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -643,10 +643,10 @@ func (c *Cache) readHitAndMissCounts(ctx context.Context, stats *CacheStats) err
|
||||
// hold: it reads the content type from the variant's .meta file and
|
||||
// keeps it in metaCache, unless a StoreVariant has put one there
|
||||
// meanwhile, as the store's is newer. A read that finds no .meta file,
|
||||
// as one can when the store's .meta write failed or an eviction removed
|
||||
// the file after the read opened the variant, serves
|
||||
// application/octet-stream and keeps nothing, so metaCache only ever
|
||||
// holds a type read from a .meta file or passed to StoreVariant.
|
||||
// as one can between a store's writing of the variant file and of its
|
||||
// .meta file, serves application/octet-stream and keeps nothing, so
|
||||
// metaCache only ever holds a type read from a .meta file or passed to
|
||||
// StoreVariant.
|
||||
func (c *Cache) loadVariantWithMeta(
|
||||
cacheKey VariantKey,
|
||||
) (io.ReadCloser, int64, string, error) {
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
@@ -543,11 +542,10 @@ func (c *Cache) runReconciliationPass(ctx context.Context) {
|
||||
// (or whose accounting insert failed, e.g. StoreVariant's best-effort
|
||||
// insert under transient DB contention), drops accounting rows whose
|
||||
// files are missing, removes source blob files the database does not
|
||||
// know (and rows whose files are gone), removes source and variant files
|
||||
// that are empty or not the size their rows record, sweeps stale temp
|
||||
// files left behind by crashed writes, and last checks the total cache
|
||||
// usage against the tables. It reads the tables a page at a time.
|
||||
// Running it periodically, not just once, bounds how long such drift can
|
||||
// know (and rows whose files are gone), sweeps stale temp files left
|
||||
// behind by crashed writes, and last checks the total cache usage
|
||||
// against the tables. It reads the tables a page at a time. Running it
|
||||
// periodically, not just once, bounds how long such drift can
|
||||
// accumulate unaccounted for on a long-running process to one eviction
|
||||
// interval. Once ctx is cancelled, it stops at the next file or row and
|
||||
// returns ctx's error.
|
||||
@@ -579,9 +577,8 @@ func (c *Cache) reconcileAccounting(ctx context.Context) error {
|
||||
return c.reconcileUsageTotal(ctx)
|
||||
}
|
||||
|
||||
// reconcileVariantFiles walks the variant storage directory, checking
|
||||
// each variant file against its accounting row (reconcileVariantFile)
|
||||
// and sweeping stale temp files.
|
||||
// reconcileVariantFiles walks the variant storage directory, adopting
|
||||
// files without accounting rows and sweeping stale temp files.
|
||||
func (c *Cache) reconcileVariantFiles(ctx context.Context) error {
|
||||
return filepath.WalkDir(
|
||||
c.variants.baseDir,
|
||||
@@ -606,55 +603,38 @@ func (c *Cache) reconcileVariantFiles(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
return c.reconcileVariantFile(ctx, path, entry, VariantKey(name))
|
||||
return c.adoptVariantFile(ctx, path, entry, VariantKey(name))
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
// reconcileVariantFile checks one variant file against its accounting
|
||||
// row. A file without a row is adopted into the accounting, unless it is
|
||||
// empty. A file that is empty, or not the size its row records, as a
|
||||
// crash or a restored snapshot can leave it, is removed with its row and
|
||||
// a warning naming it, so the variant is converted again.
|
||||
func (c *Cache) reconcileVariantFile(
|
||||
ctx context.Context, path string, entry fs.DirEntry, cacheKey VariantKey,
|
||||
) error {
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to stat variant file: %w", err)
|
||||
}
|
||||
|
||||
var recordedSize int64
|
||||
|
||||
err = c.db.QueryRowContext(ctx,
|
||||
`SELECT size_bytes FROM variant_content WHERE cache_key = ?`, string(cacheKey),
|
||||
).Scan(&recordedSize)
|
||||
|
||||
switch {
|
||||
case errors.Is(err, sql.ErrNoRows) && info.Size() > 0:
|
||||
return c.adoptVariantFile(ctx, path, info, cacheKey)
|
||||
case err != nil && !errors.Is(err, sql.ErrNoRows):
|
||||
return fmt.Errorf("failed to check variant accounting row: %w", err)
|
||||
case info.Size() > 0 && info.Size() == recordedSize:
|
||||
return nil
|
||||
}
|
||||
|
||||
// The file is empty, or not the size its row records
|
||||
c.log.Warn("removing damaged cached variant file", "path", path,
|
||||
"file_bytes", info.Size(), "recorded_bytes", recordedSize)
|
||||
|
||||
return c.evictVariant(ctx, cacheKey)
|
||||
}
|
||||
|
||||
// adoptVariantFile inserts an accounting row for a variant file that
|
||||
// has none, using the file's size and modification time.
|
||||
func (c *Cache) adoptVariantFile(
|
||||
ctx context.Context, path string, info fs.FileInfo, cacheKey VariantKey,
|
||||
ctx context.Context, path string, entry fs.DirEntry, cacheKey VariantKey,
|
||||
) error {
|
||||
var rowExists int
|
||||
|
||||
err := c.db.QueryRowContext(ctx,
|
||||
`SELECT COUNT(*) FROM variant_content WHERE cache_key = ?`, string(cacheKey),
|
||||
).Scan(&rowExists)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to check variant accounting row: %w", err)
|
||||
}
|
||||
|
||||
if rowExists > 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to stat variant file: %w", err)
|
||||
}
|
||||
|
||||
modTime := info.ModTime().UTC().Format(sqliteTimestampLayout)
|
||||
contentType := c.variantContentTypeFromSidecar(path)
|
||||
|
||||
_, err := c.db.ExecContext(ctx, `
|
||||
_, err = c.db.ExecContext(ctx, `
|
||||
INSERT INTO variant_content
|
||||
(cache_key, size_bytes, content_type, created_at, last_accessed_at)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
@@ -773,9 +753,10 @@ func queryStringColumn[T ~string](
|
||||
return values, nil
|
||||
}
|
||||
|
||||
// reconcileSourceFiles walks the source content directory, checking
|
||||
// each blob file against its source_content row (reconcileSourceFile)
|
||||
// and sweeping stale temp files.
|
||||
// reconcileSourceFiles walks the source content directory, removing
|
||||
// blob files the database does not track (they are unreachable: source
|
||||
// lookups always go through source_metadata) and sweeping stale temp
|
||||
// files.
|
||||
func (c *Cache) reconcileSourceFiles(ctx context.Context) error {
|
||||
return filepath.WalkDir(
|
||||
c.srcContent.baseDir,
|
||||
@@ -796,55 +777,31 @@ func (c *Cache) reconcileSourceFiles(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
return c.reconcileSourceFile(ctx, path, entry, ContentHash(name))
|
||||
return c.removeUntrackedSourceFile(ctx, path, ContentHash(name))
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
// reconcileSourceFile checks one source blob file against its
|
||||
// source_content row. A file without a row is removed: it is
|
||||
// unreachable, as source lookups always go through source_metadata. A
|
||||
// file that is empty, or not the size its row records, as a crash or a
|
||||
// restored snapshot can leave it, is removed with its rows and a warning
|
||||
// naming it, so the source is fetched again.
|
||||
func (c *Cache) reconcileSourceFile(
|
||||
ctx context.Context, path string, entry fs.DirEntry, contentHash ContentHash,
|
||||
) error {
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to stat source content file: %w", err)
|
||||
}
|
||||
|
||||
var recordedSize int64
|
||||
|
||||
err = c.db.QueryRowContext(ctx,
|
||||
`SELECT size_bytes FROM source_content WHERE content_hash = ?`,
|
||||
string(contentHash),
|
||||
).Scan(&recordedSize)
|
||||
|
||||
switch {
|
||||
case errors.Is(err, sql.ErrNoRows):
|
||||
return c.removeUntrackedSourceFile(ctx, path, contentHash)
|
||||
case err != nil:
|
||||
return fmt.Errorf("failed to check source content row: %w", err)
|
||||
case info.Size() > 0 && info.Size() == recordedSize:
|
||||
return nil
|
||||
}
|
||||
|
||||
// The file is empty, or not the size its row records
|
||||
c.log.Warn("removing damaged cached source file", "path", path,
|
||||
"file_bytes", info.Size(), "recorded_bytes", recordedSize)
|
||||
|
||||
return c.evictSourceBlob(ctx, contentHash)
|
||||
}
|
||||
|
||||
// removeUntrackedSourceFile deletes a source blob file that has no
|
||||
// source_content row. Any source_metadata rows referencing the hash
|
||||
// are removed first so no row ever points at a deleted file.
|
||||
func (c *Cache) removeUntrackedSourceFile(
|
||||
ctx context.Context, path string, contentHash ContentHash,
|
||||
) error {
|
||||
_, err := c.db.ExecContext(ctx,
|
||||
var rowExists int
|
||||
|
||||
err := c.db.QueryRowContext(ctx,
|
||||
`SELECT COUNT(*) FROM source_content WHERE content_hash = ?`, string(contentHash),
|
||||
).Scan(&rowExists)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to check source content row: %w", err)
|
||||
}
|
||||
|
||||
if rowExists > 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
_, err = c.db.ExecContext(ctx,
|
||||
`DELETE FROM source_metadata WHERE content_hash = ?`, string(contentHash))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete metadata rows for untracked blob: %w", err)
|
||||
|
||||
@@ -1,359 +0,0 @@
|
||||
package imgcache
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// getImage calls Get for req and returns the image it answered with and
|
||||
// its cache status.
|
||||
func getImage(t *testing.T, svc *Service, req *ImageRequest) ([]byte, CacheStatus) {
|
||||
t.Helper()
|
||||
|
||||
resp, err := svc.Get(t.Context(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Get() of quality %d error = %v", req.Quality, err)
|
||||
}
|
||||
|
||||
defer func() { _ = resp.Content.Close() }()
|
||||
|
||||
body, err := io.ReadAll(resp.Content)
|
||||
if err != nil {
|
||||
t.Fatalf("reading the image of quality %d: %v", req.Quality, err)
|
||||
}
|
||||
|
||||
return body, resp.CacheStatus
|
||||
}
|
||||
|
||||
// fileSize returns the size of the file at path.
|
||||
func fileSize(t *testing.T, path string) int64 {
|
||||
t.Helper()
|
||||
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatalf("stat %s: %v", path, err)
|
||||
}
|
||||
|
||||
return info.Size()
|
||||
}
|
||||
|
||||
// truncateFile cuts the file at path down to size bytes.
|
||||
func truncateFile(t *testing.T, path string, size int64) {
|
||||
t.Helper()
|
||||
|
||||
t.Logf("cutting %s from %d to %d bytes",
|
||||
filepath.Base(path), fileSize(t, path), size)
|
||||
|
||||
err := os.Truncate(path, size)
|
||||
if err != nil {
|
||||
t.Fatalf("truncating %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
// cachedSourcePath returns the path of the cached source file req uses.
|
||||
func cachedSourcePath(t *testing.T, svc *Service, req *ImageRequest) string {
|
||||
t.Helper()
|
||||
|
||||
contentHash, _, err := svc.cache.LookupSource(t.Context(), req)
|
||||
if err != nil || contentHash == "" {
|
||||
t.Fatalf("LookupSource() = %q, %v; want the cached source", contentHash, err)
|
||||
}
|
||||
|
||||
return svc.cache.srcContent.hashToPath(contentHash)
|
||||
}
|
||||
|
||||
// reconcileAsAtStartup runs the cache's reconciliation pass, as pixa does
|
||||
// when it starts, and returns what the pass logged.
|
||||
func reconcileAsAtStartup(t *testing.T, cache *Cache) string {
|
||||
t.Helper()
|
||||
|
||||
var logBuf bytes.Buffer
|
||||
|
||||
cache.log = slog.New(slog.NewJSONHandler(&logBuf, nil))
|
||||
|
||||
err := cache.reconcileAccounting(t.Context())
|
||||
if err != nil {
|
||||
t.Fatalf("reconcileAccounting() error = %v", err)
|
||||
}
|
||||
|
||||
t.Logf("reconciliation log: %s", logBuf.String())
|
||||
|
||||
return logBuf.String()
|
||||
}
|
||||
|
||||
// assertWarningNames checks that logs hold a warning naming the file at
|
||||
// path.
|
||||
func assertWarningNames(t *testing.T, logs, path string) {
|
||||
t.Helper()
|
||||
|
||||
for line := range strings.Lines(logs) {
|
||||
if strings.Contains(line, `"level":"WARN"`) &&
|
||||
strings.Contains(line, `"path":"`+path+`"`) {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
t.Errorf("reconciliation logged no warning naming %s", path)
|
||||
}
|
||||
|
||||
// TestService_Get_FetchesAgainForEmptyCachedSource empties the cached source
|
||||
// file, as a crash or a restored snapshot could, and runs the reconciliation
|
||||
// pass, which must remove it with a warning naming it. It then asks for two
|
||||
// new sizes of the source: the first must fetch it again and cache it whole,
|
||||
// so that the second is made from the cache without a third fetch.
|
||||
func TestService_Get_FetchesAgainForEmptyCachedSource(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
svc, fixtures := SetupTestService(t)
|
||||
|
||||
getImage(t, svc, photoVariant(fixtures, 85, FitCover))
|
||||
|
||||
path := cachedSourcePath(t, svc, photoVariant(fixtures, 85, FitCover))
|
||||
wholeSize := fileSize(t, path)
|
||||
|
||||
truncateFile(t, path, 0)
|
||||
assertWarningNames(t, reconcileAsAtStartup(t, svc.cache), path)
|
||||
|
||||
getImage(t, svc, photoVariant(fixtures, 40, FitCover))
|
||||
getImage(t, svc, photoVariant(fixtures, 30, FitCover))
|
||||
|
||||
fetches := readCacheStatsCounters(t, svc.cache).upstreamFetchCount
|
||||
t.Logf("upstream fetches = %d", fetches)
|
||||
|
||||
if fetches != 2 {
|
||||
t.Errorf("upstream fetches = %d, want 2: the empty cached source "+
|
||||
"was kept and fetched again on every new size", fetches)
|
||||
}
|
||||
|
||||
size := fileSize(t, path)
|
||||
if size != wholeSize {
|
||||
t.Errorf("cached source is %d bytes, want %d", size, wholeSize)
|
||||
}
|
||||
}
|
||||
|
||||
// TestService_Get_FetchesAgainForPartialCachedSource cuts the cached source
|
||||
// file to half its size, as a crash or a restored snapshot could, and runs
|
||||
// the reconciliation pass, which must remove it with a warning naming it.
|
||||
// It then asks for two new sizes of the source: the first must fetch it
|
||||
// again rather than fail to decode the partial file, and the second is
|
||||
// made from the cache.
|
||||
func TestService_Get_FetchesAgainForPartialCachedSource(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
svc, fixtures := SetupTestService(t)
|
||||
|
||||
getImage(t, svc, photoVariant(fixtures, 85, FitCover))
|
||||
|
||||
path := cachedSourcePath(t, svc, photoVariant(fixtures, 85, FitCover))
|
||||
wholeSize := fileSize(t, path)
|
||||
|
||||
truncateFile(t, path, wholeSize/2)
|
||||
assertWarningNames(t, reconcileAsAtStartup(t, svc.cache), path)
|
||||
|
||||
getImage(t, svc, photoVariant(fixtures, 40, FitCover))
|
||||
getImage(t, svc, photoVariant(fixtures, 30, FitCover))
|
||||
|
||||
fetches := readCacheStatsCounters(t, svc.cache).upstreamFetchCount
|
||||
t.Logf("upstream fetches = %d", fetches)
|
||||
|
||||
if fetches != 2 {
|
||||
t.Errorf("upstream fetches = %d, want 2", fetches)
|
||||
}
|
||||
|
||||
size := fileSize(t, path)
|
||||
if size != wholeSize {
|
||||
t.Errorf("cached source is %d bytes, want %d", size, wholeSize)
|
||||
}
|
||||
}
|
||||
|
||||
// assertConvertedAgain caches a variant of the photo, cuts its file to size
|
||||
// bytes and runs the reconciliation pass, which must remove it with a
|
||||
// warning naming it. It then checks that Get converts the image again
|
||||
// instead of serving the file, and that the next Get serves the whole
|
||||
// variant from the cache.
|
||||
func assertConvertedAgain(t *testing.T, size int64) {
|
||||
t.Helper()
|
||||
|
||||
svc, fixtures := SetupTestService(t)
|
||||
req := photoVariant(fixtures, 85, FitCover)
|
||||
|
||||
want, _ := getImage(t, svc, req)
|
||||
|
||||
path := svc.cache.variants.keyToPath(CacheKey(req))
|
||||
|
||||
truncateFile(t, path, size)
|
||||
assertWarningNames(t, reconcileAsAtStartup(t, svc.cache), path)
|
||||
|
||||
got, status := getImage(t, svc, req)
|
||||
t.Logf("after the cut: %d bytes, %s", len(got), status)
|
||||
|
||||
if status != CacheMiss || !bytes.Equal(got, want) {
|
||||
t.Errorf("after the cut: Get() = %d bytes, %s; want the %d bytes "+
|
||||
"converted again, %s", len(got), status, len(want), CacheMiss)
|
||||
}
|
||||
|
||||
got, status = getImage(t, svc, req)
|
||||
t.Logf("next request: %d bytes, %s", len(got), status)
|
||||
|
||||
if status != CacheHit || !bytes.Equal(got, want) {
|
||||
t.Errorf("next request: Get() = %d bytes, %s; want the %d bytes, %s",
|
||||
len(got), status, len(want), CacheHit)
|
||||
}
|
||||
}
|
||||
|
||||
// TestService_Get_ConvertsAgainForEmptyCachedVariant empties a cached
|
||||
// variant file, as a crash or a restored snapshot could, and checks that it
|
||||
// is converted again rather than served empty.
|
||||
func TestService_Get_ConvertsAgainForEmptyCachedVariant(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
assertConvertedAgain(t, 0)
|
||||
}
|
||||
|
||||
// TestService_Get_ConvertsAgainForPartialCachedVariant cuts a cached
|
||||
// variant file to ten bytes, as a crash or a restored snapshot could, and
|
||||
// checks that it is converted again rather than served cut short.
|
||||
func TestService_Get_ConvertsAgainForPartialCachedVariant(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
assertConvertedAgain(t, 10)
|
||||
}
|
||||
|
||||
// TestReconciliationRemovesEmptyVariantFileWithoutRow checks that the
|
||||
// reconciliation pass removes an empty variant file that has no
|
||||
// accounting row, with a warning naming it, instead of adding a row for
|
||||
// it that would have it served empty.
|
||||
func TestReconciliationRemovesEmptyVariantFileWithoutRow(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cache, _ := newEvictionTestCache(t, 1<<30)
|
||||
|
||||
_, err := cache.variants.Store(testVariantKeyOne, bytes.NewReader(nil),
|
||||
testContentTypeWebP)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to store an empty variant file: %v", err)
|
||||
}
|
||||
|
||||
logs := reconcileAsAtStartup(t, cache)
|
||||
|
||||
assertWarningNames(t, logs, cache.variants.keyToPath(testVariantKeyOne))
|
||||
|
||||
if cache.variants.Exists(testVariantKeyOne) {
|
||||
t.Error("the empty variant file is still on disk")
|
||||
}
|
||||
|
||||
rows := countRows(t, cache,
|
||||
`SELECT COUNT(*) FROM variant_content WHERE cache_key = ?`,
|
||||
string(testVariantKeyOne))
|
||||
if rows != 0 {
|
||||
t.Errorf("the empty variant file got %d accounting rows, want 0", rows)
|
||||
}
|
||||
}
|
||||
|
||||
// TestVariantStorage_StoreReplacesMetaFile stores a variant over one
|
||||
// already stored and checks that Store puts a new .meta file in place of
|
||||
// the old one, as it does the image, rather than writing into the old
|
||||
// file, which a crash could leave empty. The old .meta file is given a
|
||||
// second name before the store, so that it can be told apart from the new
|
||||
// one after it.
|
||||
func TestVariantStorage_StoreReplacesMetaFile(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
storage, key := storeTestVariant(t)
|
||||
metaPath := storage.keyToPath(key) + variantMetaSuffix
|
||||
oldMetaPath := metaPath + ".old"
|
||||
|
||||
err := os.Link(metaPath, oldMetaPath)
|
||||
if err != nil {
|
||||
t.Fatalf("giving the .meta file a second name: %v", err)
|
||||
}
|
||||
|
||||
_, err = storage.Store(key, bytes.NewReader([]byte("new variant data")),
|
||||
testContentTypeJPEG)
|
||||
if err != nil {
|
||||
t.Fatalf("Store() error = %v", err)
|
||||
}
|
||||
|
||||
oldMeta, err := os.Stat(oldMetaPath)
|
||||
if err != nil {
|
||||
t.Fatalf("stat of the old .meta file: %v", err)
|
||||
}
|
||||
|
||||
newMeta, err := os.Stat(metaPath)
|
||||
if err != nil {
|
||||
t.Fatalf("stat of the new .meta file: %v", err)
|
||||
}
|
||||
|
||||
if os.SameFile(oldMeta, newMeta) {
|
||||
t.Error("Store() wrote into the old .meta file instead of putting " +
|
||||
"a new one in its place")
|
||||
}
|
||||
|
||||
reader, _, contentType, err := storage.LoadWithMeta(key)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadWithMeta() error = %v", err)
|
||||
}
|
||||
|
||||
_ = reader.Close()
|
||||
|
||||
t.Logf("content type after the store: %s", contentType)
|
||||
|
||||
if contentType != testContentTypeJPEG {
|
||||
t.Errorf("content type = %q, want %q", contentType, testContentTypeJPEG)
|
||||
}
|
||||
}
|
||||
|
||||
// TestContentStorage_StoreHashedReplacesFileOfAnotherSize checks that a
|
||||
// store over a file of another size at the content's path, such as a
|
||||
// partial file a crash left, replaces it with the content.
|
||||
func TestContentStorage_StoreHashedReplacesFileOfAnotherSize(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
storage, err := NewContentStorage(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("NewContentStorage() error = %v", err)
|
||||
}
|
||||
|
||||
content := []byte("the whole source image")
|
||||
sum := sha256.Sum256(content)
|
||||
hash := ContentHash(hex.EncodeToString(sum[:]))
|
||||
path := storage.hashToPath(hash)
|
||||
|
||||
err = os.MkdirAll(filepath.Dir(path), StorageDirPerm)
|
||||
if err != nil {
|
||||
t.Fatalf("creating the content's directory: %v", err)
|
||||
}
|
||||
|
||||
err = os.WriteFile(path, content[:5], StorageFilePerm)
|
||||
if err != nil {
|
||||
t.Fatalf("writing a partial file: %v", err)
|
||||
}
|
||||
|
||||
_, err = storage.StoreHashed(hash, content)
|
||||
if err != nil {
|
||||
t.Fatalf("StoreHashed() error = %v", err)
|
||||
}
|
||||
|
||||
stored, err := storage.Load(hash)
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
|
||||
defer func() { _ = stored.Close() }()
|
||||
|
||||
got, err := io.ReadAll(stored)
|
||||
if err != nil {
|
||||
t.Fatalf("reading the stored content: %v", err)
|
||||
}
|
||||
|
||||
if !bytes.Equal(got, content) {
|
||||
t.Errorf("stored file = %q, want %q", got, content)
|
||||
}
|
||||
}
|
||||
@@ -2,9 +2,7 @@ package imgcache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -432,172 +430,6 @@ func TestService_Get_Caching(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestService_Get_ServesSourceAndVariantCachedTwoDaysAgo caches a source and
|
||||
// a variant of it and sets back two days every time pixa stored for them.
|
||||
// For the source: fetched_at in source_metadata, fetched_at and
|
||||
// last_accessed_at in source_content, the fetched_at in its metadata file,
|
||||
// and the access and modification times of its file and of its metadata
|
||||
// file. For the variant: created_at and last_accessed_at in variant_content,
|
||||
// the created_at in its .meta file, and the access and modification times of
|
||||
// its file and of its .meta file. It then checks that the variant is still
|
||||
// served from the cache and that a new variant is made from the cached source
|
||||
// without fetching it again.
|
||||
//
|
||||
// Left at the present: the files' status change and birth times, which a
|
||||
// program cannot set, and the times of the directories holding the files,
|
||||
// which other entries share. The source's stored response headers are empty,
|
||||
// as the mock fetcher sends none. expires_at, etag and last_modified in
|
||||
// source_metadata and in the metadata file are never written, nor are the
|
||||
// request_cache and output_content tables.
|
||||
func TestService_Get_ServesSourceAndVariantCachedTwoDaysAgo(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
svc, fixtures := SetupTestService(t)
|
||||
variant := photoVariant(fixtures, 85, FitCover)
|
||||
|
||||
resp, err := svc.Get(t.Context(), variant)
|
||||
if err != nil {
|
||||
t.Fatalf("Get() first request error = %v", err)
|
||||
}
|
||||
|
||||
_ = resp.Content.Close()
|
||||
|
||||
twoDaysAgo := time.Now().AddDate(0, 0, -2)
|
||||
backdateSource(t, svc.cache, variant, twoDaysAgo)
|
||||
backdateVariant(t, svc.cache, variant, twoDaysAgo)
|
||||
|
||||
resp, err = svc.Get(t.Context(), variant)
|
||||
if err != nil {
|
||||
t.Fatalf("Get() of the cached variant error = %v", err)
|
||||
}
|
||||
|
||||
_ = resp.Content.Close()
|
||||
|
||||
if resp.CacheStatus != CacheHit {
|
||||
t.Errorf("cached variant CacheStatus = %q, want %q",
|
||||
resp.CacheStatus, CacheHit)
|
||||
}
|
||||
|
||||
// Another quality is a new variant, made from the cached source
|
||||
resp, err = svc.Get(t.Context(), photoVariant(fixtures, 40, FitCover))
|
||||
if err != nil {
|
||||
t.Fatalf("Get() of a new variant error = %v", err)
|
||||
}
|
||||
|
||||
_ = resp.Content.Close()
|
||||
|
||||
fetches := readCacheStatsCounters(t, svc.cache).upstreamFetchCount
|
||||
if fetches != 1 {
|
||||
t.Errorf("upstream fetches = %d, want 1", fetches)
|
||||
}
|
||||
}
|
||||
|
||||
// backdateSource sets every time pixa stored for req's cached source to when,
|
||||
// as listed on TestService_Get_ServesSourceAndVariantCachedTwoDaysAgo.
|
||||
func backdateSource(
|
||||
t *testing.T, cache *Cache, req *ImageRequest, when time.Time,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
pathHash := HashPath(req.SourcePath + "?" + req.SourceQuery)
|
||||
|
||||
meta, err := cache.srcMetadata.Load(req.SourceHost, pathHash)
|
||||
if err != nil {
|
||||
t.Fatalf("reading the source's metadata file: %v", err)
|
||||
}
|
||||
|
||||
meta.FetchedAt = when.Unix()
|
||||
|
||||
err = cache.srcMetadata.Store(req.SourceHost, pathHash, meta)
|
||||
if err != nil {
|
||||
t.Fatalf("writing the source's metadata file: %v", err)
|
||||
}
|
||||
|
||||
dbTime := when.UTC().Format(sqliteTimestampFormat)
|
||||
|
||||
updateOneRow(t, cache, `
|
||||
UPDATE source_metadata SET fetched_at = ?
|
||||
WHERE source_host = ? AND source_path = ? AND source_query = ?
|
||||
`, dbTime, req.SourceHost, req.SourcePath, req.SourceQuery)
|
||||
updateOneRow(t, cache,
|
||||
`UPDATE source_content SET fetched_at = ? WHERE content_hash = ?`,
|
||||
dbTime, meta.ContentHash)
|
||||
setSourceLastAccessed(t, cache, ContentHash(meta.ContentHash), when)
|
||||
|
||||
for _, path := range []string{
|
||||
cache.srcContent.hashToPath(ContentHash(meta.ContentHash)),
|
||||
cache.srcMetadata.metaPath(req.SourceHost, pathHash),
|
||||
} {
|
||||
err = os.Chtimes(path, when, when)
|
||||
if err != nil {
|
||||
t.Fatalf("setting back the times of %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// backdateVariant sets every time pixa stored for the cached variant req asks
|
||||
// for to when, as listed on
|
||||
// TestService_Get_ServesSourceAndVariantCachedTwoDaysAgo.
|
||||
func backdateVariant(
|
||||
t *testing.T, cache *Cache, req *ImageRequest, when time.Time,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
key := CacheKey(req)
|
||||
|
||||
content, size, contentType, err := cache.variants.LoadWithMeta(key)
|
||||
if err != nil || contentType == "" {
|
||||
t.Fatalf("reading the variant's .meta file: content type %q, error %v",
|
||||
contentType, err)
|
||||
}
|
||||
|
||||
_ = content.Close()
|
||||
|
||||
meta, err := json.Marshal(VariantMeta{
|
||||
ContentType: contentType,
|
||||
Size: size,
|
||||
CreatedAt: when.Unix(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("encoding the variant's .meta file: %v", err)
|
||||
}
|
||||
|
||||
path := cache.variants.keyToPath(key)
|
||||
|
||||
err = os.WriteFile(path+variantMetaSuffix, meta, StorageFilePerm)
|
||||
if err != nil {
|
||||
t.Fatalf("writing the variant's .meta file: %v", err)
|
||||
}
|
||||
|
||||
updateOneRow(t, cache,
|
||||
`UPDATE variant_content SET created_at = ? WHERE cache_key = ?`,
|
||||
when.UTC().Format(sqliteTimestampFormat), string(key))
|
||||
setVariantLastAccessed(t, cache, key, when)
|
||||
|
||||
for _, file := range []string{path, path + variantMetaSuffix} {
|
||||
err = os.Chtimes(file, when, when)
|
||||
if err != nil {
|
||||
t.Fatalf("setting back the times of %s: %v", file, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// updateOneRow runs the UPDATE statement query and fails the test unless it
|
||||
// changed exactly one row.
|
||||
func updateOneRow(t *testing.T, cache *Cache, query string, args ...any) {
|
||||
t.Helper()
|
||||
|
||||
result, err := cache.db.ExecContext(t.Context(), query, args...)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", query, err)
|
||||
}
|
||||
|
||||
rows, err := result.RowsAffected()
|
||||
if err != nil || rows != 1 {
|
||||
t.Fatalf("%s: rows = %d, error = %v, want 1 row", query, rows, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestService_Get_DifferentSizes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -79,8 +79,8 @@ func (s *ContentStorage) Store(r io.Reader) (ContentHash, int64, error) {
|
||||
// from hash, without recomputing it. Callers that already know the
|
||||
// hash before writing (e.g. because they must hold a hash-keyed lock
|
||||
// across the whole store operation) use this instead of Store. Like
|
||||
// Store, it is idempotent: a file of the content's size already at that
|
||||
// path is left untouched, and a file of another size is replaced.
|
||||
// Store, it is idempotent: content already on disk at that path is
|
||||
// left untouched.
|
||||
func (s *ContentStorage) StoreHashed(
|
||||
hash ContentHash, data []byte,
|
||||
) (int64, error) {
|
||||
@@ -151,35 +151,28 @@ func (s *ContentStorage) Exists(hash ContentHash) bool {
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// writeIfAbsent writes data with writeFileSynced to the path derived
|
||||
// from hash, unless a file of the data's size is already there.
|
||||
// writeIfAbsent writes data to the path derived from hash, unless
|
||||
// content already exists there, via a temp-file-plus-rename so
|
||||
// concurrent readers never observe a partial file.
|
||||
func (s *ContentStorage) writeIfAbsent(hash ContentHash, data []byte) error {
|
||||
// Build path: <basedir>/<ab>/<cd>/<hash>
|
||||
path := s.hashToPath(hash)
|
||||
|
||||
// A file of another size there is damaged, and is replaced
|
||||
info, statErr := os.Stat(path)
|
||||
if statErr == nil && info.Size() == int64(len(data)) {
|
||||
// Check if already exists
|
||||
_, statErr := os.Stat(path)
|
||||
if statErr == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
err := os.MkdirAll(filepath.Dir(path), StorageDirPerm)
|
||||
// Create directory structure
|
||||
dir := filepath.Dir(path)
|
||||
|
||||
err := os.MkdirAll(dir, StorageDirPerm)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create directory: %w", err)
|
||||
}
|
||||
|
||||
return writeFileSynced(path, data)
|
||||
}
|
||||
|
||||
// writeFileSynced writes data to path through a temp file in the same
|
||||
// directory: the temp file is synced to disk, renamed to path, and the
|
||||
// directory synced after, so that neither a concurrent reader nor a
|
||||
// crash finds a partial file under path. When the directory sync fails,
|
||||
// the file is removed from path, so that a failed write leaves no file
|
||||
// there.
|
||||
func writeFileSynced(path string, data []byte) error {
|
||||
dir := filepath.Dir(path)
|
||||
|
||||
// Write to temp file first, then rename for atomicity
|
||||
tmpFile, err := os.CreateTemp(dir, ".tmp-*")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create temp file: %w", err)
|
||||
@@ -187,20 +180,18 @@ func writeFileSynced(path string, data []byte) error {
|
||||
|
||||
tmpPath := tmpFile.Name()
|
||||
|
||||
// Each failure path below unlinks the temp file explicitly. This
|
||||
// replaces a deferred cleanup that read a named result, which the
|
||||
// canonical config does not permit; the set of paths that remove
|
||||
// tmpPath, and the order relative to Close, is unchanged. This
|
||||
// mirrors how MetadataStorage.Store and VariantStorage.Store below
|
||||
// already express the same cleanup.
|
||||
_, err = tmpFile.Write(data)
|
||||
if err != nil {
|
||||
_ = tmpFile.Close()
|
||||
_ = os.Remove(tmpPath)
|
||||
|
||||
return fmt.Errorf("failed to write temp file: %w", err)
|
||||
}
|
||||
|
||||
err = tmpFile.Sync()
|
||||
if err != nil {
|
||||
_ = tmpFile.Close()
|
||||
_ = os.Remove(tmpPath)
|
||||
|
||||
return fmt.Errorf("failed to sync temp file: %w", err)
|
||||
return fmt.Errorf("failed to write content: %w", err)
|
||||
}
|
||||
|
||||
err = tmpFile.Close()
|
||||
@@ -210,6 +201,7 @@ func writeFileSynced(path string, data []byte) error {
|
||||
return fmt.Errorf("failed to close temp file: %w", err)
|
||||
}
|
||||
|
||||
// Atomic rename
|
||||
err = os.Rename(filepath.Clean(tmpPath), filepath.Clean(path))
|
||||
if err != nil {
|
||||
_ = os.Remove(tmpPath)
|
||||
@@ -217,31 +209,6 @@ func writeFileSynced(path string, data []byte) error {
|
||||
return fmt.Errorf("failed to rename temp file: %w", err)
|
||||
}
|
||||
|
||||
err = syncDirectory(dir)
|
||||
if err != nil {
|
||||
_ = os.Remove(path)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// syncDirectory syncs the directory dir to disk, so that a file just
|
||||
// renamed into it is still there after a crash.
|
||||
func syncDirectory(dir string) error {
|
||||
d, err := os.Open(filepath.Clean(dir))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open directory: %w", err)
|
||||
}
|
||||
|
||||
err = d.Sync()
|
||||
_ = d.Close()
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to sync directory: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -449,9 +416,7 @@ func NewVariantStorage(baseDir string, log *slog.Logger) (*VariantStorage, error
|
||||
return &VariantStorage{baseDir: baseDir, log: log}, nil
|
||||
}
|
||||
|
||||
// Store writes content and metadata to storage at the given key, each
|
||||
// with writeFileSynced. The .meta file is written first, so that a crash
|
||||
// cannot leave the content in place without it.
|
||||
// Store writes content and metadata to storage at the given key.
|
||||
func (s *VariantStorage) Store(
|
||||
key VariantKey, r io.Reader, contentType string,
|
||||
) (int64, error) {
|
||||
@@ -464,11 +429,46 @@ func (s *VariantStorage) Store(
|
||||
path := s.keyToPath(key)
|
||||
metaPath := path + ".meta"
|
||||
|
||||
err = os.MkdirAll(filepath.Dir(path), StorageDirPerm)
|
||||
// Create directory structure
|
||||
dir := filepath.Dir(path)
|
||||
|
||||
err = os.MkdirAll(dir, StorageDirPerm)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to create directory: %w", err)
|
||||
}
|
||||
|
||||
// Write content to temp file first, then rename for atomicity
|
||||
tmpFile, err := os.CreateTemp(dir, ".tmp-*")
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to create temp file: %w", err)
|
||||
}
|
||||
|
||||
tmpPath := tmpFile.Name()
|
||||
|
||||
_, err = tmpFile.Write(data)
|
||||
if err != nil {
|
||||
_ = tmpFile.Close()
|
||||
_ = os.Remove(tmpPath)
|
||||
|
||||
return 0, fmt.Errorf("failed to write content: %w", err)
|
||||
}
|
||||
|
||||
err = tmpFile.Close()
|
||||
if err != nil {
|
||||
_ = os.Remove(tmpPath)
|
||||
|
||||
return 0, fmt.Errorf("failed to close temp file: %w", err)
|
||||
}
|
||||
|
||||
// Atomic rename content
|
||||
err = os.Rename(filepath.Clean(tmpPath), filepath.Clean(path))
|
||||
if err != nil {
|
||||
_ = os.Remove(tmpPath)
|
||||
|
||||
return 0, fmt.Errorf("failed to rename temp file: %w", err)
|
||||
}
|
||||
|
||||
// Write metadata
|
||||
meta := VariantMeta{
|
||||
ContentType: contentType,
|
||||
Size: size,
|
||||
@@ -480,19 +480,13 @@ func (s *VariantStorage) Store(
|
||||
return 0, fmt.Errorf("failed to marshal metadata: %w", err)
|
||||
}
|
||||
|
||||
// Metadata write failure is non-fatal: the content is then served as
|
||||
// application/octet-stream.
|
||||
err = writeFileSynced(metaPath, metaData)
|
||||
// Metadata write failure is non-fatal; content is already stored.
|
||||
err = os.WriteFile(metaPath, metaData, StorageFilePerm)
|
||||
if err != nil {
|
||||
s.log.Warn("failed to write variant metadata sidecar",
|
||||
"path", metaPath, "error", err)
|
||||
}
|
||||
|
||||
err = writeFileSynced(path, data)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return size, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -7,8 +7,6 @@ import (
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/imageprocessor"
|
||||
)
|
||||
|
||||
// URL parsing errors.
|
||||
@@ -22,9 +20,8 @@ var (
|
||||
ErrPathTraversal = errors.New("path traversal detected")
|
||||
)
|
||||
|
||||
// MaxDimension is the maximum allowed width or height, the same as the image
|
||||
// processor's maximum for an output image.
|
||||
const MaxDimension = imageprocessor.MaxOutputDimension
|
||||
// MaxDimension is the maximum allowed width or height.
|
||||
const MaxDimension = 8192
|
||||
|
||||
// ValidateDimension checks one requested width or height; name ("width" or
|
||||
// "height") appears in the error. 0 means "original size" and is valid.
|
||||
|
||||
@@ -1,153 +0,0 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"image/jpeg"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/encurl"
|
||||
"sneak.berlin/go/pixa/internal/imgcache"
|
||||
)
|
||||
|
||||
// TestImageProxyRefusesComputedSideAboveMaximum requests sizes of a 3000x600
|
||||
// and a 600x3000 PNG through both image routes. A size with one side 0, whose
|
||||
// side computed from the source's proportions would be above 8192, is refused
|
||||
// with 400 and the JSON error body. The sizes after those refusals are served
|
||||
// as before, which also shows that a refusal is not cached as a failed fetch of
|
||||
// the source: orig, a size with both sides given, and a size with one side 0
|
||||
// whose computed side stays within 8192.
|
||||
func TestImageProxyRefusesComputedSideAboveMaximum(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// The upstream paths of the two sources
|
||||
const (
|
||||
widePath = "/wide.png"
|
||||
tallPath = "/tall.png"
|
||||
)
|
||||
|
||||
wide := encodeTestPNG(t, 3000, 600)
|
||||
tall := encodeTestPNG(t, 600, 3000)
|
||||
|
||||
upstream := httptest.NewServer(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "image/png")
|
||||
|
||||
if r.URL.Path == tallPath {
|
||||
_, _ = w.Write(tall)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
_, _ = w.Write(wide)
|
||||
}))
|
||||
t.Cleanup(upstream.Close)
|
||||
|
||||
s, _, _ := startImageProxy(t, upstream)
|
||||
|
||||
generator, err := encurl.NewGenerator(testSigningKey)
|
||||
if err != nil {
|
||||
t.Fatalf("encurl.NewGenerator: %v", err)
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
path string
|
||||
// size is the size of the /v1/image/ URL, and width and height are
|
||||
// those of the encrypted URL
|
||||
size string
|
||||
width, height int
|
||||
// wantWidth and wantHeight are the served image's, 0 for a refusal
|
||||
wantWidth, wantHeight int
|
||||
}{
|
||||
{widePath, "0x8192", 0, 8192, 0, 0},
|
||||
{tallPath, "8192x0", 8192, 0, 0, 0},
|
||||
{widePath, "orig", 0, 0, 3000, 600},
|
||||
{widePath, "300x200", 300, 200, 300, 200},
|
||||
{widePath, "0x300", 0, 300, 1500, 300},
|
||||
{tallPath, "300x0", 300, 0, 300, 1500},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
token, err := generator.Generate(&encurl.Payload{
|
||||
SourceHost: upstreamHost, SourcePath: tt.path,
|
||||
Width: tt.width, Height: tt.height, Format: imgcache.FormatJPEG,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("encurl Generate: %v", err)
|
||||
}
|
||||
|
||||
for _, target := range []string{
|
||||
"/v1/image/" + upstreamHost + tt.path + "/" + tt.size + ".jpeg",
|
||||
"/v1/e/" + token + "/img.jpg",
|
||||
} {
|
||||
rec := httptest.NewRecorder()
|
||||
s.ServeHTTP(rec, httptest.NewRequestWithContext(
|
||||
t.Context(), http.MethodGet, target, nil))
|
||||
t.Logf("GET %s: %d, Content-Type %s",
|
||||
target, rec.Code, rec.Header().Get("Content-Type"))
|
||||
|
||||
if tt.wantWidth == 0 {
|
||||
checkSizeRefused(t, rec)
|
||||
} else {
|
||||
checkServedSize(t, rec, tt.wantWidth, tt.wantHeight)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// checkSizeRefused checks that rec answers 400 with the JSON error body that
|
||||
// refuses a size whose side computed from the source's proportions is above
|
||||
// 8192.
|
||||
func checkSizeRefused(t *testing.T, rec *httptest.ResponseRecorder) {
|
||||
t.Helper()
|
||||
|
||||
const wantError = "size too large for the source: " +
|
||||
"the side computed from its proportions is above 8192"
|
||||
|
||||
var body struct {
|
||||
Error string `json:"error"`
|
||||
Status int `json:"status"`
|
||||
}
|
||||
|
||||
err := json.Unmarshal(rec.Body.Bytes(), &body)
|
||||
if err != nil {
|
||||
t.Errorf("status %d, body is not JSON: %v", rec.Code, err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if rec.Code != http.StatusBadRequest || body.Status != http.StatusBadRequest ||
|
||||
body.Error != wantError {
|
||||
t.Errorf("status %d, body %+v, want %d and the error %q",
|
||||
rec.Code, body, http.StatusBadRequest, wantError)
|
||||
}
|
||||
}
|
||||
|
||||
// checkServedSize checks that rec answers 200 with a JPEG of wantWidth by
|
||||
// wantHeight.
|
||||
func checkServedSize(
|
||||
t *testing.T, rec *httptest.ResponseRecorder, wantWidth, wantHeight int,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Errorf("status %d, want %d; body %s",
|
||||
rec.Code, http.StatusOK, rec.Body.String())
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
served, err := jpeg.DecodeConfig(bytes.NewReader(rec.Body.Bytes()))
|
||||
if err != nil {
|
||||
t.Errorf("decoding the served JPEG: %v", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if served.Width != wantWidth || served.Height != wantHeight {
|
||||
t.Errorf("served %dx%d, want %dx%d",
|
||||
served.Width, served.Height, wantWidth, wantHeight)
|
||||
}
|
||||
}
|
||||
@@ -1,91 +0,0 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/encurl"
|
||||
"sneak.berlin/go/pixa/internal/imgcache"
|
||||
)
|
||||
|
||||
// TestImageProxyRefusesFitOutsideAboveMaximum requests sizes with fit=outside
|
||||
// through both image routes. fit=outside scales the source to cover the size,
|
||||
// and so past it on one side: a size for which that side would be above 8192
|
||||
// is refused with 400 and the JSON error body, whether it is the width or the
|
||||
// height, and a size for which it stays within 8192 is served at the scaled
|
||||
// size.
|
||||
func TestImageProxyRefusesFitOutsideAboveMaximum(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// The upstream paths of the sources
|
||||
const (
|
||||
widePath = "/wide.png"
|
||||
rowPath = "/row.png"
|
||||
columnPath = "/column.png"
|
||||
)
|
||||
|
||||
sources := map[string][]byte{
|
||||
widePath: encodeTestPNG(t, 3000, 600),
|
||||
rowPath: encodeTestPNG(t, 8192, 1),
|
||||
columnPath: encodeTestPNG(t, 1, 8192),
|
||||
}
|
||||
|
||||
upstream := httptest.NewServer(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "image/png")
|
||||
_, _ = w.Write(sources[r.URL.Path])
|
||||
}))
|
||||
t.Cleanup(upstream.Close)
|
||||
|
||||
s, _, _ := startImageProxy(t, upstream)
|
||||
|
||||
generator, err := encurl.NewGenerator(testSigningKey)
|
||||
if err != nil {
|
||||
t.Fatalf("encurl.NewGenerator: %v", err)
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
path string
|
||||
width, height int
|
||||
// wantWidth and wantHeight are the served image's, 0 for a refusal
|
||||
wantWidth, wantHeight int
|
||||
}{
|
||||
{widePath, 1, 8192, 0, 0},
|
||||
{rowPath, 1, 8192, 0, 0},
|
||||
{columnPath, 8192, 1, 0, 0},
|
||||
{widePath, 1, 1200, 6000, 1200},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
token, err := generator.Generate(&encurl.Payload{
|
||||
SourceHost: upstreamHost, SourcePath: tt.path,
|
||||
Width: tt.width, Height: tt.height, Format: imgcache.FormatJPEG,
|
||||
FitMode: imgcache.FitOutside,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("encurl Generate: %v", err)
|
||||
}
|
||||
|
||||
size := fmt.Sprintf("%dx%d", tt.width, tt.height)
|
||||
|
||||
for _, target := range []string{
|
||||
"/v1/image/" + upstreamHost + tt.path + "/" + size +
|
||||
".jpeg?fit=outside",
|
||||
"/v1/e/" + token + "/img.jpg",
|
||||
} {
|
||||
rec := httptest.NewRecorder()
|
||||
s.ServeHTTP(rec, httptest.NewRequestWithContext(
|
||||
t.Context(), http.MethodGet, target, nil))
|
||||
t.Logf("GET %s: %d, Content-Type %s",
|
||||
target, rec.Code, rec.Header().Get("Content-Type"))
|
||||
|
||||
if tt.wantWidth == 0 {
|
||||
checkSizeRefused(t, rec)
|
||||
} else {
|
||||
checkServedSize(t, rec, tt.wantWidth, tt.wantHeight)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,71 +0,0 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/encurl"
|
||||
"sneak.berlin/go/pixa/internal/imgcache"
|
||||
)
|
||||
|
||||
// svgServedAsAVIF is an SVG whose first bytes pass the first-bytes check for
|
||||
// AVIF: its XML comment puts "ftypavis" at bytes 4 to 11, where an AVIF file
|
||||
// has its ftyp box and brand. libvips would load it as SVG.
|
||||
const svgServedAsAVIF = `<!--ftypavis--><svg xmlns="http://www.w3.org/2000/svg" ` +
|
||||
`width="64" height="48"><rect width="64" height="48"/></svg>`
|
||||
|
||||
// TestImageProxyRefusesSVGServedAsAVIF requests svgServedAsAVIF, which the
|
||||
// upstream answers as image/avif, through both image routes, resized as PNG
|
||||
// and in its own format (orig). Each request is answered 415, as for an SVG
|
||||
// source, never with an image.
|
||||
func TestImageProxyRefusesSVGServedAsAVIF(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
upstream := httptest.NewServer(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "image/avif")
|
||||
_, _ = io.WriteString(w, svgServedAsAVIF)
|
||||
}))
|
||||
t.Cleanup(upstream.Close)
|
||||
|
||||
s, _, _ := startImageProxy(t, upstream)
|
||||
|
||||
generator, err := encurl.NewGenerator(testSigningKey)
|
||||
if err != nil {
|
||||
t.Fatalf("encurl.NewGenerator: %v", err)
|
||||
}
|
||||
|
||||
encryptedURL := func(format imgcache.ImageFormat) string {
|
||||
token, err := generator.Generate(&encurl.Payload{
|
||||
SourceHost: upstreamHost, SourcePath: "/drawing.avif",
|
||||
Width: 32, Height: 24, Format: format,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("encurl Generate: %v", err)
|
||||
}
|
||||
|
||||
return "/v1/e/" + token + "/drawing." + string(format)
|
||||
}
|
||||
|
||||
for _, target := range []string{
|
||||
"/v1/image/" + upstreamHost + "/drawing.avif/32x24.png",
|
||||
"/v1/image/" + upstreamHost + "/drawing.avif/64x48.orig",
|
||||
encryptedURL(imgcache.FormatPNG),
|
||||
encryptedURL(imgcache.FormatOriginal),
|
||||
} {
|
||||
rec := httptest.NewRecorder()
|
||||
s.ServeHTTP(rec, httptest.NewRequestWithContext(
|
||||
t.Context(), http.MethodGet, target, nil))
|
||||
t.Logf("GET %s: %d, Content-Type %s, body %q",
|
||||
target, rec.Code, rec.Header().Get("Content-Type"), rec.Body.String())
|
||||
|
||||
if rec.Code != http.StatusUnsupportedMediaType ||
|
||||
!strings.Contains(rec.Body.String(), "unsupported source format") {
|
||||
t.Errorf("GET %s: %d %q, want %d unsupported source format",
|
||||
target, rec.Code, rec.Body.String(), http.StatusUnsupportedMediaType)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestImageProxyRefusesSVG requests an image the upstream answers as SVG,
|
||||
// in its own format (orig) and resized as PNG, through pixa's router,
|
||||
// upstream fetcher and image processor. pixa does not accept SVG as a
|
||||
// source, so each request is answered 415, never with an image.
|
||||
func TestImageProxyRefusesSVG(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
upstream := httptest.NewServer(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "image/svg+xml")
|
||||
_, _ = io.WriteString(w, `<svg xmlns="http://www.w3.org/2000/svg" `+
|
||||
`width="64" height="48"><rect width="64" height="48"/></svg>`)
|
||||
}))
|
||||
t.Cleanup(upstream.Close)
|
||||
|
||||
s, _, _ := startImageProxy(t, upstream)
|
||||
|
||||
for _, sizeFormat := range []string{"64x48.orig", "32x24.png"} {
|
||||
target := "/v1/image/" + upstreamHost + "/drawing.svg/" + sizeFormat
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
s.ServeHTTP(rec, httptest.NewRequestWithContext(
|
||||
t.Context(), http.MethodGet, target, nil))
|
||||
t.Logf("GET %s: %d, Content-Type %s, body %s",
|
||||
target, rec.Code, rec.Header().Get("Content-Type"), rec.Body.String())
|
||||
|
||||
if rec.Code != http.StatusUnsupportedMediaType {
|
||||
t.Errorf("GET %s status = %d, want %d",
|
||||
target, rec.Code, http.StatusUnsupportedMediaType)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user