diff --git a/Dockerfile b/Dockerfile index d811825..edd7c82 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,8 +22,9 @@ FROM golang:1.25.4-alpine@sha256:d3f0cf7723f3429e3f9ed846243970b20a2de7bae6a5b66 WORKDIR /src -# script/bootstrap --cgo installs the build dependencies (a C compiler -# and the libvips and libheif headers) and downloads the Go modules. +# script/bootstrap --cgo installs the build dependencies (a C compiler, +# the libvips and libheif headers, and libvips' JPEG XL support, which +# the tests need) and downloads the Go modules. COPY script/ ./script/ COPY go.mod go.sum ./ RUN script/bootstrap --cgo @@ -80,9 +81,11 @@ RUN version="${VERSION:-$(git describe --tags --always)}"; \ # alpine:3.21, 2026-02-25 FROM alpine:3.21@sha256:c3f8e73fdb79deaebaa2037150150191b9dcbfba68b4a46d70103204c53f4709 -# Install runtime dependencies only +# Install runtime dependencies only. vips-jxl is libvips' JPEG XL +# support, without which pixad does not start. RUN apk add --no-cache \ vips \ + vips-jxl \ libheif \ ca-certificates \ tzdata \ diff --git a/README.md b/README.md index bfa0451..5e3a1fc 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,10 @@ another part of pixa failed to stop. A request not finished by then is cut off. `docker stop` waits 10 seconds before it kills the container. Outside Docker, pixa needs libvips (the image has 8.15) and libheif to run, as -it uses libvips through CGO; building it also needs their development files, +it uses libvips through CGO. pixad does not start unless libvips has its JPEG XL +support, which on Alpine is the `vips-jxl` package and which the nix and brew +packages of libvips include, as do the apt ones from Debian 12 and Ubuntu 24.04 +on. Building pixa also needs the development files of libvips and libheif, `pkg-config` and a C compiler. `script/bootstrap --cgo` installs all of these, as the `Dockerfile` does where it compiles pixa. Plain `script/bootstrap`, which `script/setup` and `script/cibuild` run, installs git, make and Go, and Node, @@ -582,8 +585,8 @@ provide: - `script/bootstrap` — install git, make, Go, Node, Yarn and prettier and download the Go modules (idempotent); with `--cgo`, the C compiler and the - libvips and libheif libraries that compiling pixa needs instead of Node, Yarn - and prettier + libvips (with its JPEG XL support) and libheif libraries that compiling and + testing pixa need instead of Node, Yarn and prettier - `script/setup` — make a fresh clone ready for development (bootstrap, then install-precommit) - `script/projectname` — output the project name ("pixa") diff --git a/TODO.md b/TODO.md index 3cc0295..c5f538b 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,12 @@ P2: security: per-IP rate limiting on the image routes # Completed Steps +- 2026-10-08 libvips' JPEG XL support is installed and required (part of #222): + `script/bootstrap --cgo` installs `vips-jxl` on Alpine, whose `vips` package + lacks it, and the runtime stage of the `Dockerfile` installs it too. + `imageprocessor.New` fails when libvips cannot load and save JPEG XL, so pixad + does not start without it. A test saves an image as JPEG XL with govips and + loads it back. JPEG XL is not yet a format pixa serves. - 2026-10-05 the format `auto` (closes #88): a format in the `/v1/image/` path, an encrypted URL's token and the generator page's format choice, chosen for each request from `Accept` once the signature or token is checked: AVIF when diff --git a/internal/imageprocessor/imageprocessor.go b/internal/imageprocessor/imageprocessor.go index f26ed57..b91dbd6 100644 --- a/internal/imageprocessor/imageprocessor.go +++ b/internal/imageprocessor/imageprocessor.go @@ -19,13 +19,17 @@ import ( //nolint:gochecknoglobals // package-level sync.Once for one-time vips init var vipsOnce sync.Once +// errNoJPEGXL is returned by New when libvips cannot load and save JPEG XL. +var errNoJPEGXL = errors.New("libvips lacks JPEG XL support") + // initVips initializes libvips with quiet logging, one worker thread per // image and no operation cache. Process already works on one image per CPU // by default, so more threads per image would only compete for the CPUs. // Each request decodes different source bytes, so the operation cache // would rarely be hit and would hold memory outside MaxConcurrentProcessing; // repeated requests are served from pixa's disk cache instead. -func initVips() { +// It returns errNoJPEGXL when libvips cannot load and save JPEG XL. +func initVips() error { vipsOnce.Do(func() { vips.LoggingSettings(nil, vips.LogLevelError) vips.Startup(&vips.Config{ @@ -35,6 +39,14 @@ func initVips() { MaxCacheFiles: 0, }) }) + + // govips counts a format as supported when libvips has its loader; + // libvips builds the JPEG XL loader and saver together. + if !vips.IsTypeSupported(vips.ImageTypeJXL) { + return errNoJPEGXL + } + + return nil } // Format represents supported output image formats. @@ -149,9 +161,13 @@ type Params struct { } // New creates a new image processor with the given parameters. -// A zero-value Params{} uses sensible defaults. -func New(params Params) *ImageProcessor { - initVips() +// A zero-value Params{} uses sensible defaults. It fails when libvips +// cannot load and save JPEG XL. +func New(params Params) (*ImageProcessor, error) { + err := initVips() + if err != nil { + return nil, err + } maxInputBytes := params.MaxInputBytes if maxInputBytes <= 0 { @@ -167,7 +183,7 @@ func New(params Params) *ImageProcessor { maxInputBytes: maxInputBytes, processingSemaphore: make(chan struct{}, maxConcurrentProcessing), processingWaitTimeout: ProcessingWaitTimeout, - } + }, nil } // Process transforms an image according to the request. When diff --git a/internal/imageprocessor/imageprocessor_internal_test.go b/internal/imageprocessor/imageprocessor_internal_test.go index 73decd0..f434ac5 100644 --- a/internal/imageprocessor/imageprocessor_internal_test.go +++ b/internal/imageprocessor/imageprocessor_internal_test.go @@ -18,7 +18,10 @@ import ( ) func TestMain(m *testing.M) { - initVips() + err := initVips() + if err != nil { + panic(err) + } code := m.Run() @@ -118,7 +121,11 @@ func detectMIME(data []byte) string { func TestImageProcessor_ResizeJPEG(t *testing.T) { t.Parallel() - proc := New(Params{}) + proc, err := New(Params{}) + if err != nil { + t.Fatalf("New() error = %v", err) + } + ctx := context.Background() input := createTestJPEG(t, 800, 600) @@ -164,7 +171,11 @@ func TestImageProcessor_ResizeJPEG(t *testing.T) { func TestImageProcessor_ConvertToPNG(t *testing.T) { t.Parallel() - proc := New(Params{}) + proc, err := New(Params{}) + if err != nil { + t.Fatalf("New() error = %v", err) + } + ctx := context.Background() input := createTestJPEG(t, 200, 150) @@ -200,7 +211,11 @@ func processAndCheckSize( ) { t.Helper() - proc := New(Params{}) + proc, err := New(Params{}) + if err != nil { + t.Fatalf("New() error = %v", err) + } + ctx := context.Background() input := createTestJPEG(t, inputW, inputH) @@ -238,7 +253,11 @@ func TestImageProcessor_OriginalSize(t *testing.T) { func TestImageProcessor_FitContain(t *testing.T) { t.Parallel() - proc := New(Params{}) + proc, err := New(Params{}) + if err != nil { + t.Fatalf("New() error = %v", err) + } + ctx := context.Background() // 800x400 image (2:1 aspect) into 400x400 box with contain @@ -284,7 +303,11 @@ func TestImageProcessor_ProportionalScale_HeightOnly(t *testing.T) { func TestImageProcessor_ProcessPNG(t *testing.T) { t.Parallel() - proc := New(Params{}) + proc, err := New(Params{}) + if err != nil { + t.Fatalf("New() error = %v", err) + } + ctx := context.Background() input := createTestPNG(t, 400, 300) @@ -314,7 +337,10 @@ func TestImageProcessor_ProcessPNG(t *testing.T) { func TestImageProcessor_SupportedFormats(t *testing.T) { t.Parallel() - proc := New(Params{}) + proc, err := New(Params{}) + if err != nil { + t.Fatalf("New() error = %v", err) + } inputFormats := proc.SupportedInputFormats() if len(inputFormats) == 0 { @@ -345,7 +371,11 @@ func TestImageProcessor_RejectsOversizedInput(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - proc := New(Params{}) + proc, err := New(Params{}) + if err != nil { + t.Fatalf("New() error = %v", err) + } + ctx := context.Background() input := createTestJPEG(t, tt.width, tt.height) @@ -356,7 +386,7 @@ func TestImageProcessor_RejectsOversizedInput(t *testing.T) { FitMode: FitCover, } - _, err := proc.Process(ctx, bytes.NewReader(input), req) + _, err = proc.Process(ctx, bytes.NewReader(input), req) if err == nil { t.Error("Process() should reject oversized input images") } @@ -371,7 +401,11 @@ func TestImageProcessor_RejectsOversizedInput(t *testing.T) { func TestImageProcessor_AcceptsMaxDimensionInput(t *testing.T) { t.Parallel() - proc := New(Params{}) + proc, err := New(Params{}) + if err != nil { + t.Fatalf("New() error = %v", err) + } + ctx := context.Background() // Create an image at exactly MaxInputDimension - should be accepted @@ -400,7 +434,11 @@ func TestImageProcessor_AcceptsMaxDimensionInput(t *testing.T) { func encodeAndCheck(t *testing.T, format Format, quality int, wantMIME string) { t.Helper() - proc := New(Params{}) + proc, err := New(Params{}) + if err != nil { + t.Fatalf("New() error = %v", err) + } + ctx := context.Background() input := createTestJPEG(t, 200, 150) @@ -449,7 +487,11 @@ func TestImageProcessor_EncodeWebP(t *testing.T) { func TestImageProcessor_DecodeAVIF(t *testing.T) { t.Parallel() - proc := New(Params{}) + proc, err := New(Params{}) + if err != nil { + t.Fatalf("New() error = %v", err) + } + ctx := context.Background() // Load test AVIF file @@ -491,7 +533,11 @@ func TestImageProcessor_RejectsOversizedInputData(t *testing.T) { // Create a processor with a very small byte limit const limit = 1024 - proc := New(Params{MaxInputBytes: limit}) + proc, err := New(Params{MaxInputBytes: limit}) + if err != nil { + t.Fatalf("New() error = %v", err) + } + ctx := context.Background() // Create a valid JPEG that exceeds the byte limit @@ -507,7 +553,7 @@ func TestImageProcessor_RejectsOversizedInputData(t *testing.T) { FitMode: FitCover, } - _, err := proc.Process(ctx, bytes.NewReader(input), req) + _, err = proc.Process(ctx, bytes.NewReader(input), req) if err == nil { t.Fatal("Process() should reject input exceeding maxInputBytes") } @@ -524,7 +570,11 @@ func TestImageProcessor_AcceptsInputWithinLimit(t *testing.T) { input := createTestJPEG(t, 10, 10) limit := int64(len(input)) * 10 // 10× headroom - proc := New(Params{MaxInputBytes: limit}) + proc, err := New(Params{MaxInputBytes: limit}) + if err != nil { + t.Fatalf("New() error = %v", err) + } + ctx := context.Background() req := &Request{ @@ -546,13 +596,21 @@ func TestImageProcessor_DefaultMaxInputBytes(t *testing.T) { t.Parallel() // Passing 0 should use the default - proc := New(Params{}) + proc, err := New(Params{}) + if err != nil { + t.Fatalf("New() error = %v", err) + } + if proc.maxInputBytes != DefaultMaxInputBytes { t.Errorf("maxInputBytes = %d, want %d", proc.maxInputBytes, DefaultMaxInputBytes) } // Passing negative should also use the default - proc = New(Params{MaxInputBytes: -1}) + proc, err = New(Params{MaxInputBytes: -1}) + if err != nil { + t.Fatalf("New() error = %v", err) + } + if proc.maxInputBytes != DefaultMaxInputBytes { t.Errorf("maxInputBytes = %d, want %d", proc.maxInputBytes, DefaultMaxInputBytes) } @@ -564,14 +622,51 @@ func TestImageProcessor_EncodeAVIF(t *testing.T) { encodeAndCheck(t, FormatAVIF, 85, mimeAVIF) } +// TestLibvipsSavesAndLoadsJPEGXL saves an image as JPEG XL with govips and +// loads it back. It fails when libvips lacks JPEG XL support, as on Alpine +// without the vips-jxl package. +func TestLibvipsSavesAndLoadsJPEGXL(t *testing.T) { + t.Parallel() + + img, err := vips.NewImageFromBuffer(createTestJPEG(t, 64, 48)) + if err != nil { + t.Fatalf("failed to load test JPEG: %v", err) + } + + defer img.Close() + + jxl, _, err := img.ExportJxl(vips.NewJxlExportParams()) + if err != nil { + t.Fatalf("ExportJxl() error = %v", err) + } + + loaded, err := vips.NewImageFromBuffer(jxl) + if err != nil { + t.Fatalf("failed to load the JPEG XL image: %v", err) + } + + defer loaded.Close() + + if loaded.Format() != vips.ImageTypeJXL { + t.Errorf("loaded format = %s, want jxl", vips.ImageTypes[loaded.Format()]) + } + + if loaded.Width() != 64 || loaded.Height() != 48 { + t.Errorf("loaded size = %dx%d, want 64x48", loaded.Width(), loaded.Height()) + } +} + // processAndDecode runs input through Process and decodes the output with // vips, so a test can inspect the image a client would receive. func processAndDecode(t *testing.T, input []byte, req *Request) *vips.ImageRef { t.Helper() - result, err := New(Params{}).Process( - context.Background(), bytes.NewReader(input), req, - ) + proc, err := New(Params{}) + if err != nil { + t.Fatalf("New() error = %v", err) + } + + result, err := proc.Process(context.Background(), bytes.NewReader(input), req) if err != nil { t.Fatalf("Process() error = %v", err) } diff --git a/internal/imageprocessor/max_concurrent_processing_internal_test.go b/internal/imageprocessor/max_concurrent_processing_internal_test.go index ba5388c..0f94123 100644 --- a/internal/imageprocessor/max_concurrent_processing_internal_test.go +++ b/internal/imageprocessor/max_concurrent_processing_internal_test.go @@ -119,14 +119,22 @@ func TestNewDefaultsMaxConcurrentProcessingToCPUs(t *testing.T) { t.Parallel() for _, limit := range []int{0, -1} { - proc := New(Params{MaxConcurrentProcessing: limit}) + proc, err := New(Params{MaxConcurrentProcessing: limit}) + if err != nil { + t.Fatalf("New() error = %v", err) + } + if got := cap(proc.processingSemaphore); got != runtime.GOMAXPROCS(0) { t.Errorf("MaxConcurrentProcessing %d: %d slots, want %d, one per CPU", limit, got, runtime.GOMAXPROCS(0)) } } - proc := New(Params{MaxConcurrentProcessing: 3}) + proc, err := New(Params{MaxConcurrentProcessing: 3}) + if err != nil { + t.Fatalf("New() error = %v", err) + } + if got := cap(proc.processingSemaphore); got != 3 { t.Errorf("MaxConcurrentProcessing 3: %d slots, want 3", got) } @@ -145,7 +153,11 @@ func TestProcessNeverExceedsMaxConcurrentProcessing(t *testing.T) { calls = 6 ) - proc := New(Params{MaxConcurrentProcessing: limit}) + proc, err := New(Params{MaxConcurrentProcessing: limit}) + if err != nil { + t.Fatalf("New() error = %v", err) + } + input := createTestJPEG(t, 50, 50) counter := &readingCounter{} @@ -192,7 +204,11 @@ func TestProcessNeverExceedsMaxConcurrentProcessing(t *testing.T) { func TestProcessWaitsThenFailsWhenNoSlotFrees(t *testing.T) { t.Parallel() - proc := New(Params{MaxConcurrentProcessing: 1}) + proc, err := New(Params{MaxConcurrentProcessing: 1}) + if err != nil { + t.Fatalf("New() error = %v", err) + } + proc.processingWaitTimeout = 100 * time.Millisecond input := createTestJPEG(t, 10, 10) @@ -212,7 +228,7 @@ func TestProcessWaitsThenFailsWhenNoSlotFrees(t *testing.T) { start := time.Now() - _, err := proc.Process(context.Background(), bytes.NewReader(input), + _, err = proc.Process(context.Background(), bytes.NewReader(input), smallJPEGRequest()) if !errors.Is(err, ErrTooManyImages) { t.Fatalf("Process() error = %v, want ErrTooManyImages", err) @@ -278,10 +294,14 @@ func TestProcessReleasesSlotOnError(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Parallel() - proc := New(Params{MaxInputBytes: 4096, MaxConcurrentProcessing: 1}) + proc, err := New(Params{MaxInputBytes: 4096, MaxConcurrentProcessing: 1}) + if err != nil { + t.Fatalf("New() error = %v", err) + } + proc.processingWaitTimeout = 100 * time.Millisecond - _, err := proc.Process(context.Background(), tc.input, tc.req) + _, err = proc.Process(context.Background(), tc.input, tc.req) if err == nil || (tc.want != nil && !errors.Is(err, tc.want)) { t.Fatalf("Process() error = %v, want %v", err, tc.want) } @@ -308,7 +328,10 @@ func TestProcessReleasesSlotOnError(t *testing.T) { func TestWaitForProcessing(t *testing.T) { t.Parallel() - proc := New(Params{MaxConcurrentProcessing: 2}) + proc, err := New(Params{MaxConcurrentProcessing: 2}) + if err != nil { + t.Fatalf("New() error = %v", err) + } gate := make(chan struct{}) entered := make(chan struct{}, 1) @@ -347,7 +370,7 @@ func TestWaitForProcessing(t *testing.T) { openGate() - err := <-results + err = <-results if err != nil { t.Errorf("Process() error = %v, want nil", err) } diff --git a/internal/imgcache/max_concurrent_processing_internal_test.go b/internal/imgcache/max_concurrent_processing_internal_test.go index e4c310e..96cc885 100644 --- a/internal/imgcache/max_concurrent_processing_internal_test.go +++ b/internal/imgcache/max_concurrent_processing_internal_test.go @@ -60,9 +60,14 @@ func TestService_Get_WaitsForSlotBeforeReadingCachedSource(t *testing.T) { t.Parallel() svc, fixtures := SetupTestService(t) - svc.processor = imageprocessor.New( + processor, err := imageprocessor.New( imageprocessor.Params{MaxConcurrentProcessing: 1}, ) + if err != nil { + t.Fatalf("imageprocessor.New() error = %v", err) + } + + svc.processor = processor // A first request caches the photo as a source. resp, err := svc.Get(t.Context(), widthOnlyRequest(fixtures, 50)) diff --git a/internal/imgcache/service.go b/internal/imgcache/service.go index 2a40c8c..593a39f 100644 --- a/internal/imgcache/service.go +++ b/internal/imgcache/service.go @@ -101,10 +101,13 @@ func NewService(cfg *ServiceConfig) (*Service, error) { } maxResponseSize := fetcherCfg.MaxResponseSize - processor := imageprocessor.New(imageprocessor.Params{ + processor, err := imageprocessor.New(imageprocessor.Params{ MaxInputBytes: maxResponseSize, MaxConcurrentProcessing: cfg.MaxConcurrentProcessing, }) + if err != nil { + return nil, err + } return &Service{ cache: cfg.Cache, diff --git a/script/bootstrap b/script/bootstrap index 31abd17..e44fa29 100755 --- a/script/bootstrap +++ b/script/bootstrap @@ -14,11 +14,12 @@ # script/fmt-check: all the host needs, as # the checks compile pixa in Docker # script/bootstrap --cgo git, make, Go, and a C compiler and the -# CGO image libraries (pkg-config, vips, -# libheif) for the govips bindings instead -# of Node: to compile pixa, in the -# Dockerfile's test phase and build stage, -# which format nothing +# CGO image libraries (pkg-config, vips +# with its JPEG XL support, libheif) for +# the govips bindings instead of Node: to +# compile pixa, in the Dockerfile's test +# phase and build stage, which format +# nothing set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" @@ -150,6 +151,12 @@ ensure_cgo_deps() { if ! pkg-config --exists vips; then pkg_install vips libvips-dev vips vips-dev fi + # libvips' JPEG XL loader and saver are in the nix and brew vips + # packages, and in apt's from Debian 12 and Ubuntu 24.04 on, but in a + # package of their own on Alpine. + if command -v apk >/dev/null 2>&1 && ! apk info -e vips-jxl >/dev/null; then + apk add --no-cache vips-jxl + fi if ! pkg-config --exists libheif; then pkg_install libheif libheif-dev libheif libheif-dev fi