Install libvips JPEG XL support and require it at startup (part of #222)
check / check (push) Waiting to run
check / check (push) Waiting to run
script/bootstrap --cgo installs vips-jxl on Alpine, where libvips' JPEG XL loader and saver are a package of their own; the nix and brew libvips include them, as does apt's from Debian 12 and Ubuntu 24.04 on. The runtime stage of the Dockerfile installs vips-jxl too. imageprocessor.New now returns an error when libvips cannot load and save JPEG XL, and NewService passes it on, so pixad does not start without that support. JPEG XL becomes the default output later in the issue, which a pixad without it could not serve. A new test saves an image as JPEG XL with govips and loads it back. Model: opus-5-5
This commit is contained in:
@@ -19,13 +19,17 @@ import (
|
||||
//nolint:gochecknoglobals // package-level sync.Once for one-time vips init
|
||||
var vipsOnce sync.Once
|
||||
|
||||
// errNoJPEGXL is returned by New when libvips cannot load and save JPEG XL.
|
||||
var errNoJPEGXL = errors.New("libvips lacks JPEG XL support")
|
||||
|
||||
// initVips initializes libvips with quiet logging, one worker thread per
|
||||
// image and no operation cache. Process already works on one image per CPU
|
||||
// by default, so more threads per image would only compete for the CPUs.
|
||||
// Each request decodes different source bytes, so the operation cache
|
||||
// would rarely be hit and would hold memory outside MaxConcurrentProcessing;
|
||||
// repeated requests are served from pixa's disk cache instead.
|
||||
func initVips() {
|
||||
// It returns errNoJPEGXL when libvips cannot load and save JPEG XL.
|
||||
func initVips() error {
|
||||
vipsOnce.Do(func() {
|
||||
vips.LoggingSettings(nil, vips.LogLevelError)
|
||||
vips.Startup(&vips.Config{
|
||||
@@ -35,6 +39,14 @@ func initVips() {
|
||||
MaxCacheFiles: 0,
|
||||
})
|
||||
})
|
||||
|
||||
// govips counts a format as supported when libvips has its loader;
|
||||
// libvips builds the JPEG XL loader and saver together.
|
||||
if !vips.IsTypeSupported(vips.ImageTypeJXL) {
|
||||
return errNoJPEGXL
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Format represents supported output image formats.
|
||||
@@ -149,9 +161,13 @@ type Params struct {
|
||||
}
|
||||
|
||||
// New creates a new image processor with the given parameters.
|
||||
// A zero-value Params{} uses sensible defaults.
|
||||
func New(params Params) *ImageProcessor {
|
||||
initVips()
|
||||
// A zero-value Params{} uses sensible defaults. It fails when libvips
|
||||
// cannot load and save JPEG XL.
|
||||
func New(params Params) (*ImageProcessor, error) {
|
||||
err := initVips()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
maxInputBytes := params.MaxInputBytes
|
||||
if maxInputBytes <= 0 {
|
||||
@@ -167,7 +183,7 @@ func New(params Params) *ImageProcessor {
|
||||
maxInputBytes: maxInputBytes,
|
||||
processingSemaphore: make(chan struct{}, maxConcurrentProcessing),
|
||||
processingWaitTimeout: ProcessingWaitTimeout,
|
||||
}
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Process transforms an image according to the request. When
|
||||
|
||||
@@ -18,7 +18,10 @@ import (
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
initVips()
|
||||
err := initVips()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
code := m.Run()
|
||||
|
||||
@@ -118,7 +121,11 @@ func detectMIME(data []byte) string {
|
||||
func TestImageProcessor_ResizeJPEG(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
proc := New(Params{})
|
||||
proc, err := New(Params{})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
input := createTestJPEG(t, 800, 600)
|
||||
@@ -164,7 +171,11 @@ func TestImageProcessor_ResizeJPEG(t *testing.T) {
|
||||
func TestImageProcessor_ConvertToPNG(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
proc := New(Params{})
|
||||
proc, err := New(Params{})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
input := createTestJPEG(t, 200, 150)
|
||||
@@ -200,7 +211,11 @@ func processAndCheckSize(
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
proc := New(Params{})
|
||||
proc, err := New(Params{})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
input := createTestJPEG(t, inputW, inputH)
|
||||
@@ -238,7 +253,11 @@ func TestImageProcessor_OriginalSize(t *testing.T) {
|
||||
func TestImageProcessor_FitContain(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
proc := New(Params{})
|
||||
proc, err := New(Params{})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// 800x400 image (2:1 aspect) into 400x400 box with contain
|
||||
@@ -284,7 +303,11 @@ func TestImageProcessor_ProportionalScale_HeightOnly(t *testing.T) {
|
||||
func TestImageProcessor_ProcessPNG(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
proc := New(Params{})
|
||||
proc, err := New(Params{})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
input := createTestPNG(t, 400, 300)
|
||||
@@ -314,7 +337,10 @@ func TestImageProcessor_ProcessPNG(t *testing.T) {
|
||||
func TestImageProcessor_SupportedFormats(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
proc := New(Params{})
|
||||
proc, err := New(Params{})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
|
||||
inputFormats := proc.SupportedInputFormats()
|
||||
if len(inputFormats) == 0 {
|
||||
@@ -345,7 +371,11 @@ func TestImageProcessor_RejectsOversizedInput(t *testing.T) {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
proc := New(Params{})
|
||||
proc, err := New(Params{})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
input := createTestJPEG(t, tt.width, tt.height)
|
||||
|
||||
@@ -356,7 +386,7 @@ func TestImageProcessor_RejectsOversizedInput(t *testing.T) {
|
||||
FitMode: FitCover,
|
||||
}
|
||||
|
||||
_, err := proc.Process(ctx, bytes.NewReader(input), req)
|
||||
_, err = proc.Process(ctx, bytes.NewReader(input), req)
|
||||
if err == nil {
|
||||
t.Error("Process() should reject oversized input images")
|
||||
}
|
||||
@@ -371,7 +401,11 @@ func TestImageProcessor_RejectsOversizedInput(t *testing.T) {
|
||||
func TestImageProcessor_AcceptsMaxDimensionInput(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
proc := New(Params{})
|
||||
proc, err := New(Params{})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create an image at exactly MaxInputDimension - should be accepted
|
||||
@@ -400,7 +434,11 @@ func TestImageProcessor_AcceptsMaxDimensionInput(t *testing.T) {
|
||||
func encodeAndCheck(t *testing.T, format Format, quality int, wantMIME string) {
|
||||
t.Helper()
|
||||
|
||||
proc := New(Params{})
|
||||
proc, err := New(Params{})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
input := createTestJPEG(t, 200, 150)
|
||||
@@ -449,7 +487,11 @@ func TestImageProcessor_EncodeWebP(t *testing.T) {
|
||||
func TestImageProcessor_DecodeAVIF(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
proc := New(Params{})
|
||||
proc, err := New(Params{})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Load test AVIF file
|
||||
@@ -491,7 +533,11 @@ func TestImageProcessor_RejectsOversizedInputData(t *testing.T) {
|
||||
// Create a processor with a very small byte limit
|
||||
const limit = 1024
|
||||
|
||||
proc := New(Params{MaxInputBytes: limit})
|
||||
proc, err := New(Params{MaxInputBytes: limit})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a valid JPEG that exceeds the byte limit
|
||||
@@ -507,7 +553,7 @@ func TestImageProcessor_RejectsOversizedInputData(t *testing.T) {
|
||||
FitMode: FitCover,
|
||||
}
|
||||
|
||||
_, err := proc.Process(ctx, bytes.NewReader(input), req)
|
||||
_, err = proc.Process(ctx, bytes.NewReader(input), req)
|
||||
if err == nil {
|
||||
t.Fatal("Process() should reject input exceeding maxInputBytes")
|
||||
}
|
||||
@@ -524,7 +570,11 @@ func TestImageProcessor_AcceptsInputWithinLimit(t *testing.T) {
|
||||
input := createTestJPEG(t, 10, 10)
|
||||
limit := int64(len(input)) * 10 // 10× headroom
|
||||
|
||||
proc := New(Params{MaxInputBytes: limit})
|
||||
proc, err := New(Params{MaxInputBytes: limit})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
req := &Request{
|
||||
@@ -546,13 +596,21 @@ func TestImageProcessor_DefaultMaxInputBytes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Passing 0 should use the default
|
||||
proc := New(Params{})
|
||||
proc, err := New(Params{})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
|
||||
if proc.maxInputBytes != DefaultMaxInputBytes {
|
||||
t.Errorf("maxInputBytes = %d, want %d", proc.maxInputBytes, DefaultMaxInputBytes)
|
||||
}
|
||||
|
||||
// Passing negative should also use the default
|
||||
proc = New(Params{MaxInputBytes: -1})
|
||||
proc, err = New(Params{MaxInputBytes: -1})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
|
||||
if proc.maxInputBytes != DefaultMaxInputBytes {
|
||||
t.Errorf("maxInputBytes = %d, want %d", proc.maxInputBytes, DefaultMaxInputBytes)
|
||||
}
|
||||
@@ -564,14 +622,51 @@ func TestImageProcessor_EncodeAVIF(t *testing.T) {
|
||||
encodeAndCheck(t, FormatAVIF, 85, mimeAVIF)
|
||||
}
|
||||
|
||||
// TestLibvipsSavesAndLoadsJPEGXL saves an image as JPEG XL with govips and
|
||||
// loads it back. It fails when libvips lacks JPEG XL support, as on Alpine
|
||||
// without the vips-jxl package.
|
||||
func TestLibvipsSavesAndLoadsJPEGXL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
img, err := vips.NewImageFromBuffer(createTestJPEG(t, 64, 48))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to load test JPEG: %v", err)
|
||||
}
|
||||
|
||||
defer img.Close()
|
||||
|
||||
jxl, _, err := img.ExportJxl(vips.NewJxlExportParams())
|
||||
if err != nil {
|
||||
t.Fatalf("ExportJxl() error = %v", err)
|
||||
}
|
||||
|
||||
loaded, err := vips.NewImageFromBuffer(jxl)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to load the JPEG XL image: %v", err)
|
||||
}
|
||||
|
||||
defer loaded.Close()
|
||||
|
||||
if loaded.Format() != vips.ImageTypeJXL {
|
||||
t.Errorf("loaded format = %s, want jxl", vips.ImageTypes[loaded.Format()])
|
||||
}
|
||||
|
||||
if loaded.Width() != 64 || loaded.Height() != 48 {
|
||||
t.Errorf("loaded size = %dx%d, want 64x48", loaded.Width(), loaded.Height())
|
||||
}
|
||||
}
|
||||
|
||||
// processAndDecode runs input through Process and decodes the output with
|
||||
// vips, so a test can inspect the image a client would receive.
|
||||
func processAndDecode(t *testing.T, input []byte, req *Request) *vips.ImageRef {
|
||||
t.Helper()
|
||||
|
||||
result, err := New(Params{}).Process(
|
||||
context.Background(), bytes.NewReader(input), req,
|
||||
)
|
||||
proc, err := New(Params{})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
|
||||
result, err := proc.Process(context.Background(), bytes.NewReader(input), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Process() error = %v", err)
|
||||
}
|
||||
|
||||
@@ -119,14 +119,22 @@ func TestNewDefaultsMaxConcurrentProcessingToCPUs(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, limit := range []int{0, -1} {
|
||||
proc := New(Params{MaxConcurrentProcessing: limit})
|
||||
proc, err := New(Params{MaxConcurrentProcessing: limit})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
|
||||
if got := cap(proc.processingSemaphore); got != runtime.GOMAXPROCS(0) {
|
||||
t.Errorf("MaxConcurrentProcessing %d: %d slots, want %d, one per CPU",
|
||||
limit, got, runtime.GOMAXPROCS(0))
|
||||
}
|
||||
}
|
||||
|
||||
proc := New(Params{MaxConcurrentProcessing: 3})
|
||||
proc, err := New(Params{MaxConcurrentProcessing: 3})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
|
||||
if got := cap(proc.processingSemaphore); got != 3 {
|
||||
t.Errorf("MaxConcurrentProcessing 3: %d slots, want 3", got)
|
||||
}
|
||||
@@ -145,7 +153,11 @@ func TestProcessNeverExceedsMaxConcurrentProcessing(t *testing.T) {
|
||||
calls = 6
|
||||
)
|
||||
|
||||
proc := New(Params{MaxConcurrentProcessing: limit})
|
||||
proc, err := New(Params{MaxConcurrentProcessing: limit})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
|
||||
input := createTestJPEG(t, 50, 50)
|
||||
|
||||
counter := &readingCounter{}
|
||||
@@ -192,7 +204,11 @@ func TestProcessNeverExceedsMaxConcurrentProcessing(t *testing.T) {
|
||||
func TestProcessWaitsThenFailsWhenNoSlotFrees(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
proc := New(Params{MaxConcurrentProcessing: 1})
|
||||
proc, err := New(Params{MaxConcurrentProcessing: 1})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
|
||||
proc.processingWaitTimeout = 100 * time.Millisecond
|
||||
|
||||
input := createTestJPEG(t, 10, 10)
|
||||
@@ -212,7 +228,7 @@ func TestProcessWaitsThenFailsWhenNoSlotFrees(t *testing.T) {
|
||||
|
||||
start := time.Now()
|
||||
|
||||
_, err := proc.Process(context.Background(), bytes.NewReader(input),
|
||||
_, err = proc.Process(context.Background(), bytes.NewReader(input),
|
||||
smallJPEGRequest())
|
||||
if !errors.Is(err, ErrTooManyImages) {
|
||||
t.Fatalf("Process() error = %v, want ErrTooManyImages", err)
|
||||
@@ -278,10 +294,14 @@ func TestProcessReleasesSlotOnError(t *testing.T) {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
proc := New(Params{MaxInputBytes: 4096, MaxConcurrentProcessing: 1})
|
||||
proc, err := New(Params{MaxInputBytes: 4096, MaxConcurrentProcessing: 1})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
|
||||
proc.processingWaitTimeout = 100 * time.Millisecond
|
||||
|
||||
_, err := proc.Process(context.Background(), tc.input, tc.req)
|
||||
_, err = proc.Process(context.Background(), tc.input, tc.req)
|
||||
if err == nil || (tc.want != nil && !errors.Is(err, tc.want)) {
|
||||
t.Fatalf("Process() error = %v, want %v", err, tc.want)
|
||||
}
|
||||
@@ -308,7 +328,10 @@ func TestProcessReleasesSlotOnError(t *testing.T) {
|
||||
func TestWaitForProcessing(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
proc := New(Params{MaxConcurrentProcessing: 2})
|
||||
proc, err := New(Params{MaxConcurrentProcessing: 2})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
|
||||
gate := make(chan struct{})
|
||||
entered := make(chan struct{}, 1)
|
||||
@@ -347,7 +370,7 @@ func TestWaitForProcessing(t *testing.T) {
|
||||
|
||||
openGate()
|
||||
|
||||
err := <-results
|
||||
err = <-results
|
||||
if err != nil {
|
||||
t.Errorf("Process() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
@@ -60,9 +60,14 @@ func TestService_Get_WaitsForSlotBeforeReadingCachedSource(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
svc, fixtures := SetupTestService(t)
|
||||
svc.processor = imageprocessor.New(
|
||||
processor, err := imageprocessor.New(
|
||||
imageprocessor.Params{MaxConcurrentProcessing: 1},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("imageprocessor.New() error = %v", err)
|
||||
}
|
||||
|
||||
svc.processor = processor
|
||||
|
||||
// A first request caches the photo as a source.
|
||||
resp, err := svc.Get(t.Context(), widthOnlyRequest(fixtures, 50))
|
||||
|
||||
@@ -101,10 +101,13 @@ func NewService(cfg *ServiceConfig) (*Service, error) {
|
||||
}
|
||||
|
||||
maxResponseSize := fetcherCfg.MaxResponseSize
|
||||
processor := imageprocessor.New(imageprocessor.Params{
|
||||
processor, err := imageprocessor.New(imageprocessor.Params{
|
||||
MaxInputBytes: maxResponseSize,
|
||||
MaxConcurrentProcessing: cfg.MaxConcurrentProcessing,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Service{
|
||||
cache: cfg.Cache,
|
||||
|
||||
Reference in New Issue
Block a user