sneak/pixa
1
0
Fork 0
Code Issues 7 Pull Requests 3 Actions 12 Packages Projects Releases Wiki Activity

Build the image pipeline in pixad's fx graph (closes #313)
check / check (push) Canceled after 0s
Details

Browse Source 
The new package internal/providers builds the cache, the fetcher, the
processor, the image service, the session manager and the encrypted URL
generator from the config. pixad's fx graph provides them with
providers.Constructors(), and handlers.New receives them built. The
server tests build the same graph, with only the fetcher's dial function
replaced. database.New opens the database, so these parts can read it
while fx builds them; the cache's stop hook still runs before the
database's.

Each setting's default is declared in internal/config, and
internal/providers works out the default cache_max_bytes. The cache's
MaxBytes is the one value for the disk cache limit, and 0 turns the disk
cache off. The fetcher and the processor take explicit values.

Model: opus-5-5
This commit was merged in pull request #330.
This commit is contained in:
clawbot
2026-10-26 12:28:23 +01:00
parent ac2bae0489
commit 63690a9ec9
54 changed files with 1181 additions and 1102 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/handlers/cache_hit_internal_test.go
+10 -3
View File
@@ -21,6 +21,7 @@ import (
"sneak.berlin/go/pixa/internal/database"
"sneak.berlin/go/pixa/internal/encurl"
"sneak.berlin/go/pixa/internal/imageprocessor"
"sneak.berlin/go/pixa/internal/imagerequest"
"sneak.berlin/go/pixa/internal/imgcache"
)
@@ -121,6 +122,7 @@ func newCountingDatabaseServer(
cache, err := imgcache.NewCache(db, imgcache.CacheConfig{
StateDir: tb.TempDir(),
NegativeTTL: 5 * time.Minute,
MaxBytes: 1 << 30,
}, prometheus.NewRegistry())
if err != nil {
tb.Fatalf("imgcache.NewCache() error = %v", err)
@@ -131,9 +133,14 @@ func newCountingDatabaseServer(
Fetcher: newMockFetcher(fstest.MapFS{
countingHost + photoPath: &fstest.MapFile{Data: photo.Bytes()},
}),
SigningKey: testSigningKey,
Allowlist: []string{countingHost},
Logger: slog.New(slog.DiscardHandler),
Processor: imageprocessor.New(imageprocessor.Params{
MaxInputBytes: 50 << 20,
MaxConcurrentProcessing: 2,
}),
SigningKey: testSigningKey,
Allowlist: []string{countingHost},
MaxResponseSize: 50 << 20,
Logger: slog.New(slog.DiscardHandler),
})
if err != nil {
tb.Fatalf("imgcache.NewService() error = %v", err)
internal/handlers/cache_max_bytes_internal_test.go
-151
View File
@@ -1,151 +0,0 @@
package handlers
import (
"os"
"path/filepath"
"testing"
"github.com/prometheus/client_golang/prometheus"
"go.uber.org/fx/fxtest"
"sneak.berlin/go/pixa/internal/config"
"sneak.berlin/go/pixa/internal/database"
"sneak.berlin/go/pixa/internal/globals"
"sneak.berlin/go/pixa/internal/logger"
)
// TestNewCacheConfigFromCacheMaxBytes checks the cache configuration
// built from cache_max_bytes: omitted, the cache works out the default
// limit; 0 turns the disk cache off; a positive value is the limit,
// unchanged.
func TestNewCacheConfigFromCacheMaxBytes(t *testing.T) {
t.Parallel()
const oneGiB = 1 << 30
cases := []struct {
name string
cacheMaxBytes int64
cacheMaxBytesExplicit bool
wantMaxBytes int64
wantUseDefaultMaxBytes bool
wantDisableDiskCache bool
}{
{
name: "cache_max_bytes omitted",
cacheMaxBytes: 0,
cacheMaxBytesExplicit: false,
wantMaxBytes: 0,
wantUseDefaultMaxBytes: true,
wantDisableDiskCache: false,
},
{
name: "cache_max_bytes: 0",
cacheMaxBytes: 0,
cacheMaxBytesExplicit: true,
wantMaxBytes: 0,
wantUseDefaultMaxBytes: false,
wantDisableDiskCache: true,
},
{
name: "cache_max_bytes: 1 GiB",
cacheMaxBytes: oneGiB,
cacheMaxBytesExplicit: true,
wantMaxBytes: oneGiB,
wantUseDefaultMaxBytes: false,
wantDisableDiskCache: false,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
cfg := &config.Config{
CacheMaxBytes: tc.cacheMaxBytes,
CacheMaxBytesExplicit: tc.cacheMaxBytesExplicit,
}
got := newCacheConfig(cfg, nil)
t.Logf("MaxBytes = %d, UseDefaultMaxBytes = %v, DisableDiskCache = %v",
got.MaxBytes, got.UseDefaultMaxBytes, got.DisableDiskCache)
if got.MaxBytes != tc.wantMaxBytes {
t.Errorf("MaxBytes = %d, want %d", got.MaxBytes, tc.wantMaxBytes)
}
if got.UseDefaultMaxBytes != tc.wantUseDefaultMaxBytes {
t.Errorf("UseDefaultMaxBytes = %v, want %v",
got.UseDefaultMaxBytes, tc.wantUseDefaultMaxBytes)
}
if got.DisableDiskCache != tc.wantDisableDiskCache {
t.Errorf("DisableDiskCache = %v, want %v",
got.DisableDiskCache, tc.wantDisableDiskCache)
}
})
}
}
// TestDiskCacheOffOnlyForExplicitZeroCacheMaxBytes starts the handlers
// once with cache_max_bytes omitted and once with cache_max_bytes: 0,
// and checks by whether the cache directories were created that the
// disk cache is on in the first case and off in the second.
func TestDiskCacheOffOnlyForExplicitZeroCacheMaxBytes(t *testing.T) {
t.Parallel()
cases := []struct {
name string
cacheMaxBytesExplicit bool
wantDiskCache bool
}{
{name: "cache_max_bytes omitted", cacheMaxBytesExplicit: false, wantDiskCache: true},
{name: "cache_max_bytes: 0", cacheMaxBytesExplicit: true, wantDiskCache: false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
stateDir := t.TempDir()
cfg := &config.Config{
SigningKey: testSigningKey,
StateDir: stateDir,
DBURL: "file:" + filepath.Join(stateDir, "state.sqlite3"),
CacheMaxBytes: 0,
CacheMaxBytesExplicit: tc.cacheMaxBytesExplicit,
}
lc := fxtest.NewLifecycle(t)
log, err := logger.New(lc, logger.Params{Globals: &globals.Globals{}})
if err != nil {
t.Fatalf("logger.New() error = %v", err)
}
db, err := database.New(lc, database.Params{Logger: log, Config: cfg})
if err != nil {
t.Fatalf("database.New() error = %v", err)
}
_, err = New(lc, Params{
Logger: log, Database: db, Config: cfg,
MetricsRegistry: prometheus.NewRegistry(),
})
if err != nil {
t.Fatalf("New() error = %v", err)
}
lc.RequireStart()
t.Cleanup(lc.RequireStop)
_, err = os.Stat(filepath.Join(stateDir, "cache", "variants"))
gotDiskCache := err == nil
if gotDiskCache != tc.wantDiskCache {
t.Errorf("cache directories created = %v, want %v",
gotDiskCache, tc.wantDiskCache)
}
})
}
}
internal/handlers/fetcher_internal_test.go
-63
View File
@@ -1,63 +0,0 @@
package handlers
import (
"net/http"
"net/netip"
"path/filepath"
"testing"
"time"
"github.com/go-chi/chi/v5"
"github.com/prometheus/client_golang/prometheus"
"go.uber.org/fx"
"go.uber.org/fx/fxtest"
"sneak.berlin/go/pixa/internal/config"
"sneak.berlin/go/pixa/internal/database"
"sneak.berlin/go/pixa/internal/globals"
"sneak.berlin/go/pixa/internal/healthcheck"
"sneak.berlin/go/pixa/internal/logger"
)
// TestHandlersBuildTheirOwnFetcherWhenNoneIsProvided builds the handlers as
// pixad does, in an fx app that provides no fetcher, and requests an image
// from 192.0.2.10, which is on the allowlist and in blocked_networks. The URL
// check accepts that address; only the dialer that refuses internal
// addresses checks blocked_networks, so the answer is 403 only if the
// fetcher the handlers build from the config connects with that dialer. Any
// other dialer would try to connect until the upstream fetch timeout, which
// is short so that the test then fails quickly.
func TestHandlersBuildTheirOwnFetcherWhenNoneIsProvided(t *testing.T) {
t.Parallel()
const host = "192.0.2.10"
stateDir := t.TempDir()
cfg := &config.Config{
SigningKey: testSigningKey,
StateDir: stateDir,
DBURL: "file:" + filepath.Join(stateDir, "state.sqlite3"),
AllowlistHosts: []string{host},
BlockedNetworks: []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24")},
UpstreamFetchTimeout: 2 * time.Second,
// With no connection slots, the fetch would fail before dialing.
UpstreamConnections: config.DefaultUpstreamConnections,
}
var h *Handlers
app := fxtest.New(t,
fx.Supply(cfg),
fx.Provide(globals.New, logger.New, database.New, healthcheck.New, New,
func() prometheus.Registerer { return prometheus.NewRegistry() }),
fx.Populate(&h),
)
app.RequireStart()
t.Cleanup(app.RequireStop)
r := chi.NewRouter()
r.Get("/v1/image/*", h.HandleImage())
rec := sendGet(t, r, photoURL(host))
checkErrorBody(t, rec, http.StatusForbidden, "forbidden")
}
internal/handlers/first_bytes_internal_test.go
+12 -5
View File
@@ -15,6 +15,7 @@ import (
"github.com/prometheus/client_golang/prometheus"
"sneak.berlin/go/pixa/internal/encurl"
"sneak.berlin/go/pixa/internal/httpfetcher"
"sneak.berlin/go/pixa/internal/imageprocessor"
"sneak.berlin/go/pixa/internal/imagerequest"
"sneak.berlin/go/pixa/internal/imgcache"
)
@@ -36,17 +37,23 @@ func newRoutesFetchingThrough(
cache, err := imgcache.NewCache(setupTestDB(t), imgcache.CacheConfig{
StateDir: t.TempDir(),
NegativeTTL: 5 * time.Minute,
MaxBytes: 1 << 30,
}, prometheus.NewRegistry())
if err != nil {
t.Fatalf("imgcache.NewCache() error = %v", err)
}
svc, err := imgcache.NewService(&imgcache.ServiceConfig{
Cache: cache,
Fetcher: fetcher,
SigningKey: testSigningKey,
Allowlist: []string{allowlistedHost},
Logger: log,
Cache: cache,
Fetcher: fetcher,
Processor: imageprocessor.New(imageprocessor.Params{
MaxInputBytes: 50 << 20,
MaxConcurrentProcessing: 2,
}),
SigningKey: testSigningKey,
Allowlist: []string{allowlistedHost},
MaxResponseSize: 50 << 20,
Logger: log,
})
if err != nil {
t.Fatalf("imgcache.NewService() error = %v", err)
internal/handlers/handlers.go
+9 -138
View File
@@ -4,19 +4,15 @@ package handlers
import (
"context"
"encoding/json"
"errors"
"log/slog"
"net/http"
"time"
"github.com/prometheus/client_golang/prometheus"
"go.uber.org/fx"
"sneak.berlin/go/pixa/internal/allowlist"
"sneak.berlin/go/pixa/internal/config"
"sneak.berlin/go/pixa/internal/database"
"sneak.berlin/go/pixa/internal/encurl"
"sneak.berlin/go/pixa/internal/healthcheck"
"sneak.berlin/go/pixa/internal/httpfetcher"
"sneak.berlin/go/pixa/internal/imgcache"
"sneak.berlin/go/pixa/internal/logger"
"sneak.berlin/go/pixa/internal/session"
@@ -28,29 +24,19 @@ type Params struct {
Logger *logger.Logger
Healthcheck *healthcheck.Healthcheck
Database *database.Database
Config *config.Config
// MetricsRegistry is the registry the image cache registers its
// counters on. pixad provides Prometheus' default registry, the one
// /metrics serves.
MetricsRegistry prometheus.Registerer
// Fetcher, when provided, fetches upstream images in place of the
// fetcher the handlers build from the config. Only tests provide one;
// pixad does not.
Fetcher httpfetcher.Fetcher `optional:"true"`
ImageService *imgcache.Service
SessionManager *session.Manager
EncryptedURLGenerator *encurl.Generator
}
// Handlers provides HTTP request handlers.
type Handlers struct {
log *slog.Logger
hc *healthcheck.Healthcheck
db *database.Database
config *config.Config
fetcher httpfetcher.Fetcher
imgSvc *imgcache.Service
imgCache *imgcache.Cache
sessMgr *session.Manager
encGen *encurl.Generator
csrfProtect func(http.Handler) http.Handler
@@ -58,50 +44,25 @@ type Handlers struct {
// refererBlocklist matches the hosts of referer_blocklist; its IsAllowed
// reports whether a URL's host is on that list.
refererBlocklist *allowlist.HostAllowList
// metricsRegistry is the registry the image cache registers its
// counters on.
metricsRegistry prometheus.Registerer
}
// New creates a new Handlers instance.
func New(lc fx.Lifecycle, params Params) (*Handlers, error) {
func New(_ fx.Lifecycle, params Params) (*Handlers, error) {
csrfProtect, err := newCSRFProtect(params.Config.SigningKey, params.Config.Debug)
if err != nil {
return nil, err
}
s := &Handlers{
return &Handlers{
log: params.Logger.Get(),
hc: params.Healthcheck,
db: params.Database,
config: params.Config,
fetcher: params.Fetcher,
imgSvc: params.ImageService,
sessMgr: params.SessionManager,
encGen: params.EncryptedURLGenerator,
csrfProtect: csrfProtect,
refererBlocklist: allowlist.New(params.Config.RefererBlocklist),
metricsRegistry: params.MetricsRegistry,
}
lc.Append(fx.Hook{
//nolint:contextcheck // the cache's goroutines outlive OnStart; OnStop stops them
OnStart: func(_ context.Context) error {
return s.initImageService()
},
// The pending access times are written here, before the database's
// stop hook closes it.
OnStop: func(ctx context.Context) error {
if s.imgCache == nil {
return nil
}
return errors.Join(
s.imgCache.StopEviction(ctx),
s.imgCache.StopPendingAccessTimeWrites(ctx),
)
},
})
return s, nil
}, nil
}
// WaitForProcessing waits until no image is being processed, or until ctx
@@ -110,96 +71,6 @@ func (s *Handlers) WaitForProcessing(ctx context.Context) int {
return s.imgSvc.WaitForProcessing(ctx)
}
// newCacheConfig builds the image cache's configuration from cfg.
// cache_max_bytes: 0 disables the disk cache entirely; any other value
// is the eviction limit in bytes; when it is omitted, the cache works
// out the default limit itself.
func newCacheConfig(cfg *config.Config, log *slog.Logger) imgcache.CacheConfig {
return imgcache.CacheConfig{
StateDir: cfg.StateDir,
NegativeTTL: imgcache.DefaultNegativeTTL,
MaxBytes: cfg.CacheMaxBytes,
UseDefaultMaxBytes: !cfg.CacheMaxBytesExplicit,
DisableDiskCache: cfg.CacheMaxBytesExplicit && cfg.CacheMaxBytes == 0,
Logger: log,
}
}
// initImageService initializes the image cache and service.
func (s *Handlers) initImageService() error {
cache, err := imgcache.NewCache(
s.db.DB(), newCacheConfig(s.config, s.log), s.metricsRegistry)
if err != nil {
return err
}
s.imgCache = cache
// Background eviction: startup reconciliation, then periodic and
// write-pressure passes, and the removal of expired negative cache
// rows, which is all it does when the disk cache is disabled.
cache.StartEviction(imgcache.DefaultEvictionInterval)
// Writes the access times that requests hand over, so that no request
// waits for the database to record them
cache.StartPendingAccessTimeWrites()
// Create the fetcher config
fetcherCfg := httpfetcher.DefaultConfig()
fetcherCfg.AllowHTTP = s.config.AllowHTTP
fetcherCfg.Timeout = s.config.UpstreamFetchTimeout
fetcherCfg.MaxResponseSize = s.config.UpstreamMaxResponseSize
if s.config.UpstreamConnectionsPerHost > 0 {
fetcherCfg.MaxConnectionsPerHost = s.config.UpstreamConnectionsPerHost
}
fetcherCfg.MaxConnections = s.config.UpstreamConnections
fetcherCfg.BlockedNetworks = s.config.BlockedNetworks
// Create the service. With no fetcher provided, it builds its own from
// fetcherCfg.
svc, err := imgcache.NewService(&imgcache.ServiceConfig{
Cache: cache,
FetcherConfig: fetcherCfg,
Fetcher: s.fetcher,
SigningKey: s.config.SigningKey,
Allowlist: s.config.AllowlistHosts,
MaxConcurrentProcessing: s.config.MaxConcurrentProcessing,
Logger: s.log,
})
if err != nil {
return err
}
s.imgSvc = svc
s.log.Info("image service initialized")
// Initialize session manager (signing key is validated at config load
// time). Session cookies are always Secure/HttpOnly/SameSite=Strict.
// The random value that each logout replaces is saved in the database,
// so it outlasts a restart.
sessMgr, err := session.NewManagerWithDatabase(
context.Background(), s.config.SigningKey, s.db.DB())
if err != nil {
return err
}
s.sessMgr = sessMgr
// Initialize encrypted URL generator
encGen, err := encurl.NewGenerator(s.config.SigningKey)
if err != nil {
return err
}
s.encGen = encGen
s.log.Info("session manager and URL generator initialized")
return nil
}
func (s *Handlers) respondJSON(w http.ResponseWriter, data any, status int) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
internal/handlers/handlers_internal_test.go
+11 -4
View File
@@ -21,6 +21,7 @@ import (
"github.com/prometheus/client_golang/prometheus"
"sneak.berlin/go/pixa/internal/database"
"sneak.berlin/go/pixa/internal/httpfetcher"
"sneak.berlin/go/pixa/internal/imageprocessor"
"sneak.berlin/go/pixa/internal/imgcache"
)
@@ -48,16 +49,22 @@ func setupTestHandler(t *testing.T) *testFixtures {
cache, err := imgcache.NewCache(db, imgcache.CacheConfig{
StateDir: tmpDir,
NegativeTTL: 5 * time.Minute,
MaxBytes: 1 << 30,
}, prometheus.NewRegistry())
if err != nil {
t.Fatalf("failed to create cache: %v", err)
}
svc, err := imgcache.NewService(&imgcache.ServiceConfig{
Cache: cache,
Fetcher: newMockFetcher(mockFS),
SigningKey: "test-signing-key-must-be-32-chars",
Allowlist: []string{goodHost},
Cache: cache,
Fetcher: newMockFetcher(mockFS),
Processor: imageprocessor.New(imageprocessor.Params{
MaxInputBytes: 50 << 20,
MaxConcurrentProcessing: 2,
}),
SigningKey: "test-signing-key-must-be-32-chars",
Allowlist: []string{goodHost},
MaxResponseSize: 50 << 20,
})
if err != nil {
t.Fatalf("failed to create service: %v", err)
internal/handlers/image_cache_control_internal_test.go
+9 -2
View File
@@ -15,6 +15,7 @@ import (
"github.com/prometheus/client_golang/prometheus"
"sneak.berlin/go/pixa/internal/encurl"
"sneak.berlin/go/pixa/internal/imageprocessor"
"sneak.berlin/go/pixa/internal/imagerequest"
"sneak.berlin/go/pixa/internal/imgcache"
)
@@ -34,6 +35,7 @@ func newSignedHostServer(
cache, err := imgcache.NewCache(setupTestDB(t), imgcache.CacheConfig{
StateDir: t.TempDir(),
NegativeTTL: 5 * time.Minute,
MaxBytes: 1 << 30,
}, prometheus.NewRegistry())
if err != nil {
t.Fatalf("imgcache.NewCache() error = %v", err)
@@ -46,8 +48,13 @@ func newSignedHostServer(
Fetcher: newMockFetcher(fstest.MapFS{
signedHost + photoPath: &fstest.MapFile{Data: jpegData},
}),
SigningKey: testSigningKey,
Logger: log,
Processor: imageprocessor.New(imageprocessor.Params{
MaxInputBytes: 50 << 20,
MaxConcurrentProcessing: 2,
}),
SigningKey: testSigningKey,
MaxResponseSize: 50 << 20,
Logger: log,
})
if err != nil {
t.Fatalf("imgcache.NewService() error = %v", err)
internal/handlers/image_errors_internal_test.go
+17 -5
View File
@@ -15,6 +15,7 @@ import (
"github.com/go-chi/chi/v5"
"github.com/prometheus/client_golang/prometheus"
"sneak.berlin/go/pixa/internal/httpfetcher"
"sneak.berlin/go/pixa/internal/imageprocessor"
"sneak.berlin/go/pixa/internal/imagerequest"
"sneak.berlin/go/pixa/internal/imgcache"
"sneak.berlin/go/pixa/internal/signature"
@@ -32,16 +33,22 @@ func newImageRoute(t *testing.T, fetcher httpfetcher.Fetcher) http.Handler {
cache, err := imgcache.NewCache(setupTestDB(t), imgcache.CacheConfig{
StateDir: t.TempDir(),
NegativeTTL: 5 * time.Minute,
MaxBytes: 1 << 30,
}, prometheus.NewRegistry())
if err != nil {
t.Fatalf("failed to create cache: %v", err)
}
svc, err := imgcache.NewService(&imgcache.ServiceConfig{
Cache: cache,
Fetcher: fetcher,
SigningKey: testSigningKey,
Allowlist: []string{allowlistedHost},
Cache: cache,
Fetcher: fetcher,
Processor: imageprocessor.New(imageprocessor.Params{
MaxInputBytes: 50 << 20,
MaxConcurrentProcessing: 2,
}),
SigningKey: testSigningKey,
Allowlist: []string{allowlistedHost},
MaxResponseSize: 50 << 20,
})
if err != nil {
t.Fatalf("failed to create service: %v", err)
@@ -169,7 +176,12 @@ func TestHandleImage_ErrorAnswers(t *testing.T) {
photos := newPhotoFetcher(t,
signedHost, parentHost, siblingHost, subdomainHost, appendedHost)
// The real fetcher refuses localhost before any lookup or connection.
realFetcher := httpfetcher.New(httpfetcher.DefaultConfig())
realFetcher := httpfetcher.New(&httpfetcher.Config{
Timeout: 30 * time.Second,
MaxResponseSize: 50 << 20,
MaxConnectionsPerHost: 20,
MaxConnections: 64,
})
exp := time.Now().Add(time.Hour)
expired := time.Now().Add(-time.Hour)
internal/handlers/referer_blocklist_internal_test.go
+11 -4
View File
@@ -14,6 +14,7 @@ import (
"sneak.berlin/go/pixa/internal/allowlist"
"sneak.berlin/go/pixa/internal/encurl"
"sneak.berlin/go/pixa/internal/httpfetcher"
"sneak.berlin/go/pixa/internal/imageprocessor"
"sneak.berlin/go/pixa/internal/imgcache"
)
@@ -51,16 +52,22 @@ func newRefererRoutes(t *testing.T) (http.Handler, *Handlers, *countingFetcher)
cache, err := imgcache.NewCache(setupTestDB(t), imgcache.CacheConfig{
StateDir: t.TempDir(),
NegativeTTL: 5 * time.Minute,
MaxBytes: 1 << 30,
}, prometheus.NewRegistry())
if err != nil {
t.Fatalf("imgcache.NewCache() error = %v", err)
}
svc, err := imgcache.NewService(&imgcache.ServiceConfig{
Cache: cache,
Fetcher: fetcher,
SigningKey: testSigningKey,
Allowlist: []string{allowlistedHost},
Cache: cache,
Fetcher: fetcher,
Processor: imageprocessor.New(imageprocessor.Params{
MaxInputBytes: 50 << 20,
MaxConcurrentProcessing: 2,
}),
SigningKey: testSigningKey,
Allowlist: []string{allowlistedHost},
MaxResponseSize: 50 << 20,
})
if err != nil {
t.Fatalf("imgcache.NewService() error = %v", err)