Bound concurrent image processing and upstream fetches (closes #64)

Nothing bounded total in-flight work, so a burst of cache misses across
hosts could exhaust memory. Two settings now do: max_concurrent_processing
(default the CPUs Go uses) and upstream_connections (default 64, beside
the per-host limit). A request that finds either full waits up to 10
seconds, then gets 503; a slot is released on every path. No request
holds source bytes while it waits: a cached source is read only after
the processing slot is taken, and a fetched one only while it holds its
upstream connection. libvips runs one worker thread per image with its
operation cache off. Both waits count toward downstream_timeout, as the
README says.

Model: opus-5-5

internal/imageprocessor/imageprocessor.go

@@ -7,7 +7,9 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"runtime"
 	"sync"
+	"time"
 
 	"github.com/davidbyttow/govips/v2/vips"
 )
@@ -17,11 +19,21 @@ import (
 //nolint:gochecknoglobals // package-level sync.Once for one-time vips init
 var vipsOnce sync.Once
 
-// initVips initializes libvips with quiet logging.
+// initVips initializes libvips with quiet logging, one worker thread per
+// image and no operation cache. Process already works on one image per CPU
+// by default, so more threads per image would only compete for the CPUs.
+// Each request decodes different source bytes, so the operation cache
+// would rarely be hit and would hold memory outside MaxConcurrentProcessing;
+// repeated requests are served from pixa's disk cache instead.
 func initVips() {
 	vipsOnce.Do(func() {
 		vips.LoggingSettings(nil, vips.LogLevelError)
-		vips.Startup(nil)
+		vips.Startup(&vips.Config{
+			ConcurrencyLevel: 1,
+			MaxCacheSize:     0,
+			MaxCacheMem:      0,
+			MaxCacheFiles:    0,
+		})
 	})
 }
 
@@ -106,9 +118,23 @@ var ErrInputDataTooLarge = errors.New("input data exceeds maximum allowed size")
 // not supported.
 var ErrUnsupportedOutputFormat = errors.New("unsupported output format")
 
+// ErrTooManyImages is returned when MaxConcurrentProcessing images are being
+// processed and none finishes within ProcessingWaitTimeout.
+var ErrTooManyImages = errors.New("too many images being processed at once")
+
+// ProcessingWaitTimeout is how long Process waits for a free slot when
+// MaxConcurrentProcessing images are already being processed.
+const ProcessingWaitTimeout = 10 * time.Second
+
 // ImageProcessor implements image transformation using libvips via govips.
 type ImageProcessor struct {
 	maxInputBytes int64
+	// processingSemaphore has one slot per image that may be processed at
+	// once. Process holds a slot from before it reads its input until it
+	// returns, so the input, the decoded image and the output all count.
+	processingSemaphore chan struct{}
+	// processingWaitTimeout is ProcessingWaitTimeout; tests shorten it.
+	processingWaitTimeout time.Duration
 }
 
 // Params holds configuration for creating an ImageProcessor.
@@ -117,6 +143,9 @@ type Params struct {
 	// MaxInputBytes is the maximum allowed input size in bytes.
 	// If <= 0, DefaultMaxInputBytes is used.
 	MaxInputBytes int64
+	// MaxConcurrentProcessing is the most images processed at once.
+	// If <= 0, the number of CPUs Go uses (runtime.GOMAXPROCS(0)) is used.
+	MaxConcurrentProcessing int
 }
 
 // New creates a new image processor with the given parameters.
@@ -129,17 +158,34 @@ func New(params Params) *ImageProcessor {
 		maxInputBytes = DefaultMaxInputBytes
 	}
 
+	maxConcurrentProcessing := params.MaxConcurrentProcessing
+	if maxConcurrentProcessing <= 0 {
+		maxConcurrentProcessing = runtime.GOMAXPROCS(0)
+	}
+
 	return &ImageProcessor{
-		maxInputBytes: maxInputBytes,
+		maxInputBytes:         maxInputBytes,
+		processingSemaphore:   make(chan struct{}, maxConcurrentProcessing),
+		processingWaitTimeout: ProcessingWaitTimeout,
 	}
 }
 
-// Process transforms an image according to the request.
+// Process transforms an image according to the request. When
+// MaxConcurrentProcessing images are already being processed, it waits up
+// to ProcessingWaitTimeout for one to finish, then fails with
+// ErrTooManyImages.
 func (p *ImageProcessor) Process(
-	_ context.Context,
+	ctx context.Context,
 	input io.Reader,
 	req *Request,
 ) (*Result, error) {
+	release, err := p.acquireSlot(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	defer release()
+
 	// Read input with a size limit to prevent unbounded memory consumption.
 	// We read at most maxInputBytes+1 so we can detect if the input exceeds
 	// the limit without consuming additional memory.
@@ -285,6 +331,29 @@ func FormatToMIME(format Format) string {
 	}
 }
 
+// acquireSlot takes a slot in processingSemaphore, waiting at most
+// processingWaitTimeout for one to free up, and returns the func that gives
+// it back. A free slot is taken even when ctx has ended; only the wait for
+// one stops when ctx ends, as the rest of Process does not check ctx.
+func (p *ImageProcessor) acquireSlot(ctx context.Context) (func(), error) {
+	release := func() { <-p.processingSemaphore }
+
+	select {
+	case p.processingSemaphore <- struct{}{}:
+		return release, nil
+	default:
+	}
+
+	select {
+	case p.processingSemaphore <- struct{}{}:
+		return release, nil
+	case <-time.After(p.processingWaitTimeout):
+		return nil, ErrTooManyImages
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
+}
+
 // detectFormat returns the format string from a vips image.
 func (p *ImageProcessor) detectFormat(img *vips.ImageRef) string {
 	format := img.Format()