sneak/pixa
1
0
Fork 0
Code Issues 7 Pull Requests 3 Actions 18 Packages Projects Releases Wiki Activity

Build the image pipeline in pixad's fx graph (closes #313)
check / check (push) Canceled after 0s
Details

Browse Source 
The new package internal/providers builds the cache, the fetcher, the
processor, the image service, the session manager and the encrypted URL
generator from the config. pixad's fx graph provides them with
providers.Constructors(), and handlers.New receives them built. The
server tests build the same graph, with only the fetcher's dial function
replaced. database.New opens the database, so these parts can read it
while fx builds them; the cache's stop hook still runs before the
database's.

Each setting's default is declared in internal/config, and
internal/providers works out the default cache_max_bytes. The cache's
MaxBytes is the one value for the disk cache limit, and 0 turns the disk
cache off. The fetcher and the processor take explicit values.

Model: opus-5-5
This commit was merged in pull request #330.
This commit is contained in:
clawbot
2026-10-26 12:28:23 +01:00
parent ac2bae0489
commit 63690a9ec9
54 changed files with 1181 additions and 1102 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/imgcache/cache.go
+12 -53
View File
@@ -32,23 +32,12 @@ type CacheConfig struct {
NegativeTTL time.Duration
// MaxBytes is the disk cache size limit in bytes that eviction
// enforces. Zero means no limit is enforced (no eviction).
// enforces. Zero turns the disk cache off entirely: no cache
// directories are created, lookups always miss, stores are no-ops,
// and the eviction goroutine only removes the expired negative cache
// rows.
MaxBytes int64
// UseDefaultMaxBytes makes NewCache replace MaxBytes with the
// default limit: 75% of the sum of the space free on the filesystem
// holding the cache and the bytes the cache already holds, at least
// DefaultCacheMaxBytesFloor. The config layer sets this when the
// operator omits cache_max_bytes.
UseDefaultMaxBytes bool
// DisableDiskCache turns the disk cache off entirely: no cache
// directories are created, lookups always miss, stores are
// no-ops, and the eviction goroutine only removes the expired
// negative cache rows. The config layer sets this when the
// operator configures cache_max_bytes: 0.
DisableDiskCache bool
// Logger receives accounting and eviction log output. A nil
// Logger means slog.Default().
Logger *slog.Logger
@@ -62,10 +51,6 @@ type Cache struct {
config CacheConfig
log *slog.Logger
// disabled means the disk cache is turned off entirely: lookups
// always miss, stores are no-ops, and no eviction runs.
disabled bool
// Eviction machinery. The channels are created in NewCache so
// stores can signal write pressure without racing StartEviction.
// evictionCancel, set by StartEviction, cancels the eviction
@@ -97,7 +82,7 @@ type Cache struct {
transforms prometheus.Counter
// reconciliationPageSize is the most rows one read of a content table
// returns in the reconciliation pass. newCache sets it to
// returns in the reconciliation pass. NewCache sets it to
// defaultReconciliationPageSize; tests set it smaller.
reconciliationPageSize int
@@ -110,20 +95,9 @@ type Cache struct {
// NewCache creates a new cache instance and registers its counters on
// metricsRegistry. pixad passes Prometheus' default registry, the one
// /metrics serves.
// /metrics serves. It makes no database call.
func NewCache(
db *sql.DB, config CacheConfig, metricsRegistry prometheus.Registerer,
) (*Cache, error) {
return newCache(db, config, metricsRegistry, defaultFreeSpaceProbe)
}
// newCache is NewCache with the free-space probe passed in, so tests
// can fake the free space the default limit is worked out from.
func newCache(
db *sql.DB,
config CacheConfig,
metricsRegistry prometheus.Registerer,
probe FreeSpaceProbeFunc,
) (*Cache, error) {
log := config.Logger
if log == nil {
@@ -134,7 +108,6 @@ func newCache(
db: db,
config: config,
log: log,
disabled: config.DisableDiskCache,
evictionPressure: make(chan struct{}, 1),
evictionDone: make(chan struct{}),
@@ -150,7 +123,7 @@ func newCache(
return nil, err
}
if c.disabled {
if config.MaxBytes == 0 {
return c, nil
}
@@ -171,20 +144,6 @@ func newCache(
c.srcContent = srcContent
c.variants = variants
if config.UseDefaultMaxBytes {
limit, err := c.computeDefaultMaxBytes(context.Background(), probe)
if err != nil {
return nil, err
}
c.config.MaxBytes = limit
log.Info("computed default cache size limit from free space and cache contents",
"cache_max_bytes", limit,
"cache_dir", filepath.Join(config.StateDir, "cache"),
)
}
return c, nil
}
@@ -196,7 +155,7 @@ func newCache(
func (c *Cache) GetVariant(
cacheKey imagerequest.VariantKey,
) (io.ReadCloser, int64, string, error) {
if c.disabled {
if c.config.MaxBytes == 0 {
return nil, 0, "", ErrNotFound
}
@@ -214,7 +173,7 @@ func (c *Cache) StoreSource(
req *imagerequest.Request,
content io.Reader,
) (imagerequest.SourceKey, error) {
if c.disabled {
if c.config.MaxBytes == 0 {
return "", nil
}
@@ -259,7 +218,7 @@ func (c *Cache) StoreSource(
func (c *Cache) StoreVariant(
ctx context.Context, cacheKey imagerequest.VariantKey, content io.Reader,
) error {
if c.disabled {
if c.config.MaxBytes == 0 {
return nil
}
@@ -293,7 +252,7 @@ func (c *Cache) StoreVariant(
func (c *Cache) LookupSource(
ctx context.Context, req *imagerequest.Request,
) (imagerequest.SourceKey, error) {
if c.disabled {
if c.config.MaxBytes == 0 {
return "", nil
}
@@ -345,7 +304,7 @@ func (c *Cache) StoreNegative(ctx context.Context, req *imagerequest.Request) er
func (c *Cache) GetSourceContent(
key imagerequest.SourceKey,
) (io.ReadCloser, int64, error) {
if c.disabled {
if c.config.MaxBytes == 0 {
return nil, 0, ErrNotFound
}
internal/imgcache/cache_hit_internal_test.go
+4 -1
View File
@@ -22,7 +22,10 @@ func TestGetVariant_ContentTypeFromFirstBytes(t *testing.T) {
t.Parallel()
cache, _ := newEvictionTestCache(t, 1<<30)
processor := imageprocessor.New(imageprocessor.Params{})
processor := imageprocessor.New(imageprocessor.Params{
MaxInputBytes: 50 << 20,
MaxConcurrentProcessing: 1,
})
source := generateTestJPEG(t, 40, 30, color.RGBA{200, 40, 40, 255})
for _, format := range []imagerequest.Format{
internal/imgcache/cache_internal_test.go
+2
View File
@@ -53,6 +53,7 @@ func TestCache_CleanExpired(t *testing.T) {
cache, _ := NewCache(db, CacheConfig{
StateDir: tmpDir,
NegativeTTL: 1 * time.Millisecond,
MaxBytes: 1 << 30,
}, prometheus.NewRegistry())
ctx := context.Background()
@@ -104,6 +105,7 @@ func TestCache_StorageDirectoriesCreated(t *testing.T) {
_, err := NewCache(db, CacheConfig{
StateDir: tmpDir,
MaxBytes: 1 << 30,
}, prometheus.NewRegistry())
if err != nil {
t.Fatalf("NewCache() error = %v", err)
internal/imgcache/cachesize.go
-85
View File
@@ -1,85 +0,0 @@
package imgcache
import (
"context"
"errors"
"fmt"
"math"
"path/filepath"
"syscall"
)
// DefaultCacheMaxBytesFloor is the minimum computed default for the
// cache_max_bytes setting: 500 MiB. The floor applies only to the
// computed default (when the key is omitted from the configuration),
// never to explicitly configured values.
const DefaultCacheMaxBytesFloor int64 = 524288000
// freeSpaceFractionNumerator and freeSpaceFractionDenominator express
// the 75% share used for the computed default limit as integer
// arithmetic (dividing before multiplying avoids overflow).
const (
freeSpaceFractionNumerator uint64 = 3
freeSpaceFractionDenominator uint64 = 4
)
var errNegativeBlockSize = errors.New("statfs reported negative block size")
// FreeSpaceProbeFunc reports the number of free bytes available on the
// filesystem containing path. It is a function type so tests can
// inject a fake probe instead of depending on the host disk.
type FreeSpaceProbeFunc func(path string) (uint64, error)
// defaultFreeSpaceProbe reports free filesystem bytes via statfs on
// the given path, as available to unprivileged processes.
func defaultFreeSpaceProbe(path string) (uint64, error) {
var stat syscall.Statfs_t
err := syscall.Statfs(path, &stat)
if err != nil {
return 0, err
}
if stat.Bsize < 0 {
return 0, fmt.Errorf("%w %d for %q", errNegativeBlockSize, stat.Bsize, path)
}
blockSize := uint64(stat.Bsize)
return stat.Bavail * blockSize, nil
}
// computeDefaultMaxBytes returns the default cache size limit: 75% of
// the sum of the free bytes probe reports for <state_dir>/cache/ and
// the bytes the cache already holds, with a floor of
// DefaultCacheMaxBytesFloor. Counting what the cache holds keeps the
// limit from shrinking as the cache fills.
func (c *Cache) computeDefaultMaxBytes(
ctx context.Context, probe FreeSpaceProbeFunc,
) (int64, error) {
cacheDir := filepath.Join(c.config.StateDir, "cache")
freeBytes, err := probe(cacheDir)
if err != nil {
return 0, fmt.Errorf(
"default cache_max_bytes: cannot determine free space for %q: %w",
cacheDir, err)
}
usedBytes, err := c.UsageBytes(ctx)
if err != nil {
return 0, err
}
// Both terms are at most math.MaxInt64, so the sum cannot overflow.
//nolint:gosec // G115: UsageBytes returns the total cache usage, never negative
spaceBytes := min(freeBytes, math.MaxInt64) + uint64(usedBytes)
computed := spaceBytes / freeSpaceFractionDenominator * freeSpaceFractionNumerator
computed = min(computed, math.MaxInt64)
limit := int64(computed)
limit = max(limit, DefaultCacheMaxBytesFloor)
return limit, nil
}
internal/imgcache/cachesize_internal_test.go
-189
View File
@@ -1,189 +0,0 @@
package imgcache
import (
"errors"
"os"
"path/filepath"
"strings"
"testing"
"github.com/prometheus/client_golang/prometheus"
)
// Static errors returned by the stub free-space probes below.
var (
errTestStatfsFailed = errors.New("statfs failed")
errTestProbeNotExpected = errors.New("probe must not be called")
)
// TestComputeDefaultMaxBytesCountsWhatTheCacheHolds verifies that the
// default limit is 75% of the free space plus what the cache already
// holds, so a cache filled to its limit keeps that limit across a
// restart instead of shrinking to 75% of the space left free.
func TestComputeDefaultMaxBytesCountsWhatTheCacheHolds(t *testing.T) {
t.Parallel()
cache, _ := newEvictionTestCache(t, 1<<30)
// Empty cache, 4 GiB free -> 3 GiB default.
got, err := cache.computeDefaultMaxBytes(t.Context(),
func(string) (uint64, error) { return 4294967296, nil })
if err != nil {
t.Fatalf("computeDefaultMaxBytes returned error: %v", err)
}
t.Logf("default for an empty cache with 4 GiB free: %d", got)
if got != 3221225472 {
t.Errorf("default for an empty cache = %d, want 3221225472 (75%% of 4 GiB)",
got)
}
// The cache now holds those 3 GiB, which leaves 1 GiB free.
_, err = cache.db.ExecContext(t.Context(),
`INSERT INTO variant_content (cache_key, size_bytes) VALUES (?, ?)`,
string(testVariantKeyOne), 3221225472,
)
if err != nil {
t.Fatalf("failed to insert variant accounting row: %v", err)
}
got, err = cache.computeDefaultMaxBytes(t.Context(),
func(string) (uint64, error) { return 1073741824, nil })
if err != nil {
t.Fatalf("computeDefaultMaxBytes returned error: %v", err)
}
t.Logf("default for a cache holding 3 GiB with 1 GiB free: %d", got)
if got != 3221225472 {
t.Errorf("default for a cache holding 3 GiB with 1 GiB free = %d, "+
"want 3221225472 (75%% of 1 GiB + 3 GiB)", got)
}
}
// TestComputeDefaultMaxBytesAppliesFloor verifies that when 75% of the
// free space plus what the cache holds is below 500 MiB, the default
// is floored at DefaultCacheMaxBytesFloor.
func TestComputeDefaultMaxBytesAppliesFloor(t *testing.T) {
t.Parallel()
cases := []struct {
name string
freeBytes uint64
}{
{name: "100 MiB free", freeBytes: 104857600},
{name: "zero free", freeBytes: 0},
{name: "just below floor threshold", freeBytes: 699050665},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
cache, _ := newEvictionTestCache(t, 1<<30)
got, err := cache.computeDefaultMaxBytes(t.Context(),
func(string) (uint64, error) { return tc.freeBytes, nil })
if err != nil {
t.Fatalf("computeDefaultMaxBytes returned error: %v", err)
}
if got != DefaultCacheMaxBytesFloor {
t.Errorf("computeDefaultMaxBytes = %d, want floor %d",
got, DefaultCacheMaxBytesFloor)
}
})
}
}
// TestComputeDefaultMaxBytesPropagatesProbeError verifies that a
// failing free-space probe produces an error naming cache_max_bytes,
// instead of a silently wrong default.
func TestComputeDefaultMaxBytesPropagatesProbeError(t *testing.T) {
t.Parallel()
cache, _ := newEvictionTestCache(t, 1<<30)
_, err := cache.computeDefaultMaxBytes(t.Context(),
func(string) (uint64, error) { return 0, errTestStatfsFailed })
if err == nil {
t.Fatal("probe failure must produce an error, got nil")
}
t.Logf("got expected error: %v", err)
if !strings.Contains(err.Error(), "cache_max_bytes") {
t.Errorf("error %q does not name cache_max_bytes", err.Error())
}
}
// TestNewCacheComputesDefaultMaxBytesWhenAsked verifies that with
// UseDefaultMaxBytes set, the cache's limit becomes the computed
// default, and that the probe is pointed at <state_dir>/cache/, which
// must be created first so statfs measures the right filesystem.
func TestNewCacheComputesDefaultMaxBytesWhenAsked(t *testing.T) {
t.Parallel()
stateDir := t.TempDir()
wantCacheDir := filepath.Join(stateDir, "cache")
var probedPath string
// 4 GiB free -> 3 GiB default.
probe := func(path string) (uint64, error) {
probedPath = path
info, err := os.Stat(path)
if err != nil || !info.IsDir() {
t.Errorf("cache directory %q was not created before probing: info=%v err=%v",
path, info, err)
}
return 4294967296, nil
}
cache, err := newCache(evictionTestDB(t), CacheConfig{
StateDir: stateDir,
UseDefaultMaxBytes: true,
}, prometheus.NewRegistry(), probe)
if err != nil {
t.Fatalf("newCache returned error: %v", err)
}
if cache.config.MaxBytes != 3221225472 {
t.Errorf("MaxBytes = %d, want computed default 3221225472",
cache.config.MaxBytes)
}
if probedPath != wantCacheDir {
t.Errorf("free space probed at %q, want cache directory %q",
probedPath, wantCacheDir)
}
}
// TestNewCacheKeepsExplicitMaxBytes verifies that without
// UseDefaultMaxBytes the cache keeps MaxBytes exactly as given and
// never consults the free-space probe.
func TestNewCacheKeepsExplicitMaxBytes(t *testing.T) {
t.Parallel()
probe := func(string) (uint64, error) {
t.Error("free-space probe must not be consulted for explicit values")
return 0, errTestProbeNotExpected
}
cache, err := newCache(evictionTestDB(t), CacheConfig{
StateDir: t.TempDir(),
MaxBytes: 1024,
}, prometheus.NewRegistry(), probe)
if err != nil {
t.Fatalf("newCache returned error: %v", err)
}
if cache.config.MaxBytes != 1024 {
t.Errorf("MaxBytes = %d, want explicit 1024 (no floor, no recompute)",
cache.config.MaxBytes)
}
}
internal/imgcache/concurrent_misses_internal_test.go
+1 -1
View File
@@ -455,7 +455,7 @@ func TestService_Get_WaiterGetsProcessingTimedOut(t *testing.T) {
svc, fixtures, fetcher := setupHeldFetchService(t)
svc.processor = imageprocessor.New(
imageprocessor.Params{MaxConcurrentProcessing: 1},
imageprocessor.Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1},
)
holdProcessingSlot(t, svc.processor)
internal/imgcache/counters_internal_test.go
+6 -3
View File
@@ -36,7 +36,8 @@ func TestNewCache_RegistersItsCountersOnItsRegistry(t *testing.T) {
registry := prometheus.NewRegistry()
_, err := NewCache(evictionTestDB(t), CacheConfig{StateDir: t.TempDir()}, registry)
_, err := NewCache(evictionTestDB(t),
CacheConfig{StateDir: t.TempDir(), MaxBytes: 1 << 30}, registry)
if err != nil {
t.Fatalf("NewCache() error = %v", err)
}
@@ -68,12 +69,14 @@ func TestNewCache_FailsWhenItsCountersCannotBeRegistered(t *testing.T) {
registry := prometheus.NewRegistry()
_, err := NewCache(evictionTestDB(t), CacheConfig{StateDir: t.TempDir()}, registry)
_, err := NewCache(evictionTestDB(t),
CacheConfig{StateDir: t.TempDir(), MaxBytes: 1 << 30}, registry)
if err != nil {
t.Fatalf("NewCache() of the first cache error = %v", err)
}
_, err = NewCache(evictionTestDB(t), CacheConfig{StateDir: t.TempDir()}, registry)
_, err = NewCache(evictionTestDB(t),
CacheConfig{StateDir: t.TempDir(), MaxBytes: 1 << 30}, registry)
t.Logf("NewCache() of the second cache error = %v", err)
var alreadyRegistered prometheus.AlreadyRegisteredError
internal/imgcache/eviction.go
+17 -10
View File
@@ -52,18 +52,25 @@ const sqliteTimestampLayout = "2006-01-02 15:04:05"
const tempFilePrefix = ".tmp-"
// UsageBytes returns the total number of bytes of cache content
// tracked in the database (source content blobs plus processed
// variants). It reads the total the database keeps up to date as rows
// are added and removed, so it neither scans the cache directories nor
// sums the tables.
// tracked in the database, as the package function UsageBytes does, or 0
// when the disk cache is off.
func (c *Cache) UsageBytes(ctx context.Context) (int64, error) {
if c.disabled {
if c.config.MaxBytes == 0 {
return 0, nil
}
return UsageBytes(ctx, c.db)
}
// UsageBytes returns the total number of bytes of cache content tracked
// in db (source content blobs plus processed variants). It reads the
// total the database keeps up to date as rows are added and removed, so
// it neither scans the cache directories nor sums the tables. pixad reads
// it before it builds the cache, to work out the default cache_max_bytes.
func UsageBytes(ctx context.Context, db *sql.DB) (int64, error) {
var total int64
err := c.db.QueryRowContext(ctx,
err := db.QueryRowContext(ctx,
`SELECT total_size_bytes FROM cache_usage WHERE id = 1`,
).Scan(&total)
if err != nil {
@@ -86,9 +93,9 @@ type evictionCandidate struct {
// EvictToLimit evicts least-recently-used cache entries until total
// tracked usage is at or below the configured MaxBytes limit. It is a
// no-op when the cache is disabled or no limit is configured.
// no-op when the cache is disabled.
func (c *Cache) EvictToLimit(ctx context.Context) error {
if c.disabled || c.config.MaxBytes <= 0 {
if c.config.MaxBytes == 0 {
return nil
}
@@ -326,7 +333,7 @@ func (c *Cache) evictSourceBlob(ctx context.Context, key imagerequest.SourceKey)
// storing request. The notification channel has capacity one and drops
// when a wakeup is already pending.
func (c *Cache) notifyWritePressure() {
if c.disabled || c.config.MaxBytes <= 0 {
if c.config.MaxBytes == 0 {
return
}
@@ -485,7 +492,7 @@ func (c *Cache) runCleanExpired(ctx context.Context) {
// Once ctx is cancelled, it stops at the next file or row and returns
// ctx's error.
func (c *Cache) reconcileAccounting(ctx context.Context) error {
if c.disabled {
if c.config.MaxBytes == 0 {
return nil
}
internal/imgcache/eviction_internal_test.go
+4 -8
View File
@@ -66,15 +66,11 @@ func newEvictionTestCache(t *testing.T, maxBytes int64) (*Cache, string) {
tmpDir := t.TempDir()
db := evictionTestDB(t)
// maxBytes zero mirrors the production mapping of
// cache_max_bytes: 0 (handlers sets DisableDiskCache); at the
// CacheConfig layer itself a zero MaxBytes means "no limit" for
// backwards compatibility with existing fixtures.
// maxBytes zero turns the disk cache off, as cache_max_bytes: 0 does.
cache, err := NewCache(db, CacheConfig{
StateDir: tmpDir,
NegativeTTL: 5 * time.Minute,
MaxBytes: maxBytes,
DisableDiskCache: maxBytes == 0,
StateDir: tmpDir,
NegativeTTL: 5 * time.Minute,
MaxBytes: maxBytes,
}, prometheus.NewRegistry())
if err != nil {
t.Fatalf("failed to create cache: %v", err)
internal/imgcache/max_concurrent_processing_internal_test.go
+1 -1
View File
@@ -66,7 +66,7 @@ func TestService_Get_WaitsForSlotBeforeReadingCachedSource(t *testing.T) {
svc, fixtures := SetupTestService(t)
svc.processor = imageprocessor.New(
imageprocessor.Params{MaxConcurrentProcessing: 1},
imageprocessor.Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1},
)
// A first request caches the photo as a source.
internal/imgcache/negative_cache_internal_test.go
+15 -10
View File
@@ -20,6 +20,7 @@ func TestNegativeCache_StoreAndCheck(t *testing.T) {
cache, err := NewCache(db, CacheConfig{
StateDir: dir,
NegativeTTL: 5 * time.Minute,
MaxBytes: 1 << 30,
}, prometheus.NewRegistry())
if err != nil {
t.Fatal(err)
@@ -69,6 +70,7 @@ func TestNegativeCache_Expired(t *testing.T) {
cache, err := NewCache(db, CacheConfig{
StateDir: dir,
NegativeTTL: 1 * time.Millisecond, // very short TTL
MaxBytes: 1 << 30,
}, prometheus.NewRegistry())
if err != nil {
t.Fatal(err)
@@ -177,11 +179,11 @@ func TestEvictionLoopRemovesExpiredNegativeCacheRows(t *testing.T) {
t.Parallel()
tests := []struct {
name string
disableDiskCache bool
name string
maxBytes int64
}{
{name: "disk cache enabled", disableDiskCache: false},
{name: "disk cache disabled", disableDiskCache: true},
{name: "disk cache enabled", maxBytes: 1 << 30},
{name: "disk cache disabled", maxBytes: 0},
}
for _, tt := range tests {
@@ -189,9 +191,9 @@ func TestEvictionLoopRemovesExpiredNegativeCacheRows(t *testing.T) {
t.Parallel()
cache, err := NewCache(evictionTestDB(t), CacheConfig{
StateDir: t.TempDir(),
NegativeTTL: time.Second,
DisableDiskCache: tt.disableDiskCache,
StateDir: t.TempDir(),
NegativeTTL: time.Second,
MaxBytes: tt.maxBytes,
}, prometheus.NewRegistry())
if err != nil {
t.Fatalf("failed to create cache: %v", err)
@@ -266,7 +268,8 @@ func negativeCacheRowCount(tb testing.TB, cache *Cache, sourcePath string) int {
func TestCleanExpiredDeletesInBatches(t *testing.T) {
t.Parallel()
cache, err := NewCache(evictionTestDB(t), CacheConfig{StateDir: t.TempDir()},
cache, err := NewCache(evictionTestDB(t),
CacheConfig{StateDir: t.TempDir(), MaxBytes: 1 << 30},
prometheus.NewRegistry())
if err != nil {
t.Fatalf("failed to create cache: %v", err)
@@ -321,7 +324,8 @@ func TestCleanExpiredDeletesInBatches(t *testing.T) {
func TestCleanExpiredRemovesMoreThanOneBatch(t *testing.T) {
t.Parallel()
cache, err := NewCache(evictionTestDB(t), CacheConfig{StateDir: t.TempDir()},
cache, err := NewCache(evictionTestDB(t),
CacheConfig{StateDir: t.TempDir(), MaxBytes: 1 << 30},
prometheus.NewRegistry())
if err != nil {
t.Fatalf("failed to create cache: %v", err)
@@ -352,7 +356,8 @@ func TestCleanExpiredRemovesMoreThanOneBatch(t *testing.T) {
func TestCleanExpiredStopsOnceContextEnds(t *testing.T) {
t.Parallel()
cache, err := NewCache(evictionTestDB(t), CacheConfig{StateDir: t.TempDir()},
cache, err := NewCache(evictionTestDB(t),
CacheConfig{StateDir: t.TempDir(), MaxBytes: 1 << 30},
prometheus.NewRegistry())
if err != nil {
t.Fatalf("failed to create cache: %v", err)
internal/imgcache/service.go
+14 -39
View File
@@ -43,18 +43,20 @@ type Service struct {
type ServiceConfig struct {
// Cache is the cache instance
Cache *Cache
// FetcherConfig configures the upstream fetcher built when Fetcher is
// not set. Its AllowHTTP and MaxResponseSize are used either way.
FetcherConfig *httpfetcher.Config
// Fetcher is an optional custom fetcher (for testing)
// Fetcher fetches source images from upstream hosts
Fetcher httpfetcher.Fetcher
// Processor converts and resizes the images
Processor *imageprocessor.ImageProcessor
// SigningKey is the HMAC signing key (empty disables signing)
SigningKey string
// Allowlist is the list of hosts that don't require signatures
Allowlist []string
// MaxConcurrentProcessing is the most images processed at once; zero
// uses the image processor's default, half the CPUs
MaxConcurrentProcessing int
// AllowHTTP makes the service fetch sources over http instead of
// https (testing only), so the fetcher must allow http too
AllowHTTP bool
// MaxResponseSize is the largest source accepted, in bytes; a cached
// source larger than this is fetched again
MaxResponseSize int64
// Logger for logging
Logger *slog.Logger
}
@@ -75,32 +77,11 @@ func NewService(cfg *ServiceConfig) (*Service, error) {
return nil, errSigningKeyRequired
}
// Resolve fetcher config for defaults
fetcherCfg := cfg.FetcherConfig
if fetcherCfg == nil {
fetcherCfg = httpfetcher.DefaultConfig()
}
// Use custom fetcher if provided, otherwise create HTTP fetcher
var fetcher httpfetcher.Fetcher
if cfg.Fetcher != nil {
fetcher = cfg.Fetcher
} else {
fetcher = httpfetcher.New(fetcherCfg)
}
signer := signature.New(cfg.SigningKey)
log := cfg.Logger
if log == nil {
log = slog.Default()
}
allowHTTP := false
if cfg.FetcherConfig != nil {
allowHTTP = cfg.FetcherConfig.AllowHTTP
}
// JPEG XL is the default output format, so pixad does not start
// without it.
err := imageprocessor.CheckJPEGXLSupport()
@@ -108,21 +89,15 @@ func NewService(cfg *ServiceConfig) (*Service, error) {
return nil, err
}
maxResponseSize := fetcherCfg.MaxResponseSize
processor := imageprocessor.New(imageprocessor.Params{
MaxInputBytes: maxResponseSize,
MaxConcurrentProcessing: cfg.MaxConcurrentProcessing,
})
return &Service{
cache: cfg.Cache,
fetcher: fetcher,
processor: processor,
signer: signer,
fetcher: cfg.Fetcher,
processor: cfg.Processor,
signer: signature.New(cfg.SigningKey),
allowlist: allowlist.New(cfg.Allowlist),
log: log,
allowHTTP: allowHTTP,
maxResponseSize: maxResponseSize,
allowHTTP: cfg.AllowHTTP,
maxResponseSize: cfg.MaxResponseSize,
}, nil
}
internal/imgcache/testutil_internal_test.go
+11 -4
View File
@@ -17,6 +17,7 @@ import (
"github.com/prometheus/client_golang/prometheus"
"sneak.berlin/go/pixa/internal/database"
"sneak.berlin/go/pixa/internal/httpfetcher"
"sneak.berlin/go/pixa/internal/imageprocessor"
)
// Shared test data literals, extracted as constants for goconst.
@@ -185,16 +186,22 @@ func SetupTestService(
cache, err := NewCache(db, CacheConfig{
StateDir: tmpDir,
NegativeTTL: 5 * time.Minute,
MaxBytes: 1 << 30,
}, prometheus.NewRegistry())
if err != nil {
t.Fatalf("failed to create cache: %v", err)
}
svc, err := NewService(&ServiceConfig{
Cache: cache,
Fetcher: httpfetcher.NewMock(mockFS),
SigningKey: cfg.signingKey,
Allowlist: cfg.allowlist,
Cache: cache,
Fetcher: httpfetcher.NewMock(mockFS),
Processor: imageprocessor.New(imageprocessor.Params{
MaxInputBytes: 50 << 20,
MaxConcurrentProcessing: 2,
}),
SigningKey: cfg.signingKey,
Allowlist: cfg.allowlist,
MaxResponseSize: 50 << 20,
})
if err != nil {
t.Fatalf("failed to create service: %v", err)