Define the default max_concurrent_processing in internal/config

The default is configuration, so it now sits with the other defaults as
config.DefaultMaxConcurrentProcessing, and imageprocessor.New reads it
from there for a value below 1. internal/config no longer imports
internal/imageprocessor, so the packages that import internal/config do
not build against libvips.

Model: opus-5-5
This commit is contained in:
2026-10-15 13:07:16 +00:00
parent 6b184ee2ff
commit 49799bc7b7
3 changed files with 25 additions and 25 deletions
+9 -10
View File
@@ -30,6 +30,15 @@ P2: security: per-origin rate limiting
# Completed Steps # Completed Steps
- 2026-10-15 `max_concurrent_processing` defaults to half the CPUs pixa sees,
rounded down, and at least 1 (closes #242): it defaulted to one image per CPU,
and a container with no CPU limit sees every CPU of the host, so on a 48-CPU
host pixa processed 48 images at once, each of which can take about 1.6 GiB.
`config.DefaultMaxConcurrentProcessing` gives the default to the config and to
`imageprocessor.New`; `PIXA_MAX_CONCURRENT_PROCESSING` still overrides it.
"Running under upaas" in `README.md` gives the worst-case memory, measured,
and tells the operator to set `PIXA_MAX_CONCURRENT_PROCESSING` and
`PIXA_UPSTREAM_CONNECTIONS` to fit the container's memory.
- 2026-10-15 expired negative cache rows are removed on a schedule (closes - 2026-10-15 expired negative cache rows are removed on a schedule (closes
#244): the row of a failed fetch, which makes pixa answer 502 for that source #244): the row of a failed fetch, which makes pixa answer 502 for that source
URL for 5 minutes, was deleted only when the same URL was requested again URL for 5 minutes, was deleted only when the same URL was requested again
@@ -62,16 +71,6 @@ P2: security: per-origin rate limiting
not in `allowlist.IsAllowed`, which also matches `referer_blocklist`, where a not in `allowlist.IsAllowed`, which also matches `referer_blocklist`, where a
`Referer` is blocked whatever its port. A URL is signed over its host as it `Referer` is blocked whatever its port. A URL is signed over its host as it
writes it, in lower case, with any IPv6 brackets, `:` and port kept. writes it, in lower case, with any IPv6 brackets, `:` and port kept.
- 2026-10-15 `max_concurrent_processing` defaults to half the CPUs pixa sees,
rounded down, and at least 1 (closes #242): it defaulted to one image per CPU,
and a container with no CPU limit sees every CPU of the host, so on a 48-CPU
host pixa processed 48 images at once, each of which can take about 1.6 GiB.
`imageprocessor.DefaultMaxConcurrentProcessing` gives the default to the
config and to `imageprocessor.New`; `PIXA_MAX_CONCURRENT_PROCESSING` still
overrides it. "Running under upaas" in `README.md` gives the worst-case
memory, measured, and tells the operator to set
`PIXA_MAX_CONCURRENT_PROCESSING` and `PIXA_UPSTREAM_CONNECTIONS` to fit the
container's memory.
- 2026-10-15 a `:` in the source path or query no longer moves where the path - 2026-10-15 a `:` in the source path or query no longer moves where the path
ends and the query begins (closes #247): the signed data and the cache key ends and the query begins (closes #247): the signed data and the cache key
joined host, path and query with `:` and no escaping, so a URL signed for joined host, path and query with `:` and no escaping, so a URL signed for
+13 -3
View File
@@ -12,6 +12,7 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"regexp" "regexp"
"runtime"
"sort" "sort"
"strconv" "strconv"
"strings" "strings"
@@ -20,7 +21,6 @@ import (
"git.eeqj.de/sneak/smartconfig" "git.eeqj.de/sneak/smartconfig"
"go.uber.org/fx" "go.uber.org/fx"
"sneak.berlin/go/pixa/internal/imageprocessor"
"sneak.berlin/go/pixa/internal/logger" "sneak.berlin/go/pixa/internal/logger"
) )
@@ -37,6 +37,16 @@ const (
DefaultImageRequestsPerMinute = 600 DefaultImageRequestsPerMinute = 600
) )
// DefaultMaxConcurrentProcessing returns the most images processed at once
// when max_concurrent_processing is omitted: half the CPUs Go uses
// (runtime.GOMAXPROCS(0), which follows a container's CPU limit), rounded
// down, and at least 1.
func DefaultMaxConcurrentProcessing() int {
const cpusPerImage = 2
return max(runtime.GOMAXPROCS(0)/cpusPerImage, 1)
}
// Configuration key names. // Configuration key names.
const ( const (
keyDebug = "debug" keyDebug = "debug"
@@ -297,8 +307,8 @@ func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) {
keyUpstreamConnectionsPerHost, DefaultUpstreamConnectionsPerHost), keyUpstreamConnectionsPerHost, DefaultUpstreamConnectionsPerHost),
UpstreamConnections: loader.intVal( UpstreamConnections: loader.intVal(
keyUpstreamConnections, DefaultUpstreamConnections), keyUpstreamConnections, DefaultUpstreamConnections),
MaxConcurrentProcessing: loader.intVal(keyMaxConcurrentProcessing, MaxConcurrentProcessing: loader.intVal(
imageprocessor.DefaultMaxConcurrentProcessing()), keyMaxConcurrentProcessing, DefaultMaxConcurrentProcessing()),
UpstreamFetchTimeout: loader.durationVal( UpstreamFetchTimeout: loader.durationVal(
keyUpstreamFetchTimeout, DefaultUpstreamFetchTimeout), keyUpstreamFetchTimeout, DefaultUpstreamFetchTimeout),
UpstreamMaxResponseSize: loader.int64Val( UpstreamMaxResponseSize: loader.int64Val(
+3 -12
View File
@@ -7,11 +7,11 @@ import (
"errors" "errors"
"fmt" "fmt"
"io" "io"
"runtime"
"sync" "sync"
"time" "time"
"github.com/davidbyttow/govips/v2/vips" "github.com/davidbyttow/govips/v2/vips"
"sneak.berlin/go/pixa/internal/config"
) )
// vipsOnce ensures vips is initialized exactly once. // vipsOnce ensures vips is initialized exactly once.
@@ -179,19 +179,10 @@ type Params struct {
// If <= 0, DefaultMaxInputBytes is used. // If <= 0, DefaultMaxInputBytes is used.
MaxInputBytes int64 MaxInputBytes int64
// MaxConcurrentProcessing is the most images processed at once. // MaxConcurrentProcessing is the most images processed at once.
// If <= 0, DefaultMaxConcurrentProcessing() is used. // If <= 0, config.DefaultMaxConcurrentProcessing() is used.
MaxConcurrentProcessing int MaxConcurrentProcessing int
} }
// DefaultMaxConcurrentProcessing returns the most images processed at once
// when none is set: half the CPUs Go uses (runtime.GOMAXPROCS(0), which
// follows a container's CPU limit), rounded down, and at least 1.
func DefaultMaxConcurrentProcessing() int {
const cpusPerImage = 2
return max(runtime.GOMAXPROCS(0)/cpusPerImage, 1)
}
// New creates a new image processor with the given parameters. // New creates a new image processor with the given parameters.
// A zero-value Params{} uses sensible defaults. // A zero-value Params{} uses sensible defaults.
func New(params Params) *ImageProcessor { func New(params Params) *ImageProcessor {
@@ -204,7 +195,7 @@ func New(params Params) *ImageProcessor {
maxConcurrentProcessing := params.MaxConcurrentProcessing maxConcurrentProcessing := params.MaxConcurrentProcessing
if maxConcurrentProcessing <= 0 { if maxConcurrentProcessing <= 0 {
maxConcurrentProcessing = DefaultMaxConcurrentProcessing() maxConcurrentProcessing = config.DefaultMaxConcurrentProcessing()
} }
return &ImageProcessor{ return &ImageProcessor{