Build the image pipeline in pixad's fx graph (closes #313)

internal/app now provides the cache, the fetcher, the processor, the
image service, the session manager and the encrypted URL generator, and
handlers.New receives them built. database.New opens the database, so
these parts can read it while fx builds them; the cache's stop hook
still runs before the database's.

Each setting's default is declared in internal/config, the default
cache_max_bytes included, which internal/app works out before it builds
the cache. The cache's MaxBytes is the one value for the disk cache
limit, and 0 turns the disk cache off. The fetcher and the processor
take explicit values. The server tests build the pipeline themselves,
as they cannot import internal/app.

Model: opus-5-5

internal/server/image_proxy_internal_test.go

@@ -19,19 +19,20 @@ import (
 	"testing"
 	"time"
 
-	"github.com/prometheus/client_golang/prometheus"
 	"go.uber.org/fx"
 	"go.uber.org/fx/fxtest"
 
 	"sneak.berlin/go/pixa/internal/config"
 	"sneak.berlin/go/pixa/internal/database"
+	"sneak.berlin/go/pixa/internal/encurl"
 	"sneak.berlin/go/pixa/internal/globals"
 	"sneak.berlin/go/pixa/internal/handlers"
 	"sneak.berlin/go/pixa/internal/healthcheck"
-	"sneak.berlin/go/pixa/internal/httpfetcher"
 	"sneak.berlin/go/pixa/internal/imagerequest"
+	"sneak.berlin/go/pixa/internal/imgcache"
 	"sneak.berlin/go/pixa/internal/logger"
 	"sneak.berlin/go/pixa/internal/middleware"
+	"sneak.berlin/go/pixa/internal/session"
 )
 
 // upstreamHost is the upstream host of the image URLs below. It is a
@@ -120,9 +121,10 @@ func TestImageProxyFlow(t *testing.T) {
 }
 
 // startImageProxy starts the components pixad's fx app builds, from a config
-// with a fresh state directory and upstreamHost on the allowlist, and with an
-// upstream fetcher that connects every upstream address to upstream. It
-// returns the server with its routes, the database and the state directory.
+// with a fresh state directory and upstreamHost on the allowlist, with the
+// image pipeline newImagePipeline builds and an upstream fetcher that
+// connects every upstream address to upstream. It returns the server with
+// its routes, the database and the state directory.
 func startImageProxy(
 	t *testing.T, upstream *httptest.Server,
 ) (*Server, *sql.DB, string) {
@@ -138,22 +140,21 @@ func startImageProxy(
 		AllowHTTP: true,
 		// A limit of its own, so the cache does not size itself from the
 		// host's free disk space.
-		CacheMaxBytes:           64 << 20,
-		CacheMaxBytesExplicit:   true,
-		UpstreamMaxResponseSize: config.DefaultUpstreamMaxResponseSize,
-		DownstreamTimeout:       config.DefaultDownstreamTimeout,
+		CacheMaxBytes:              64 << 20,
+		CacheMaxBytesExplicit:      true,
+		UpstreamFetchTimeout:       config.DefaultUpstreamFetchTimeout,
+		UpstreamMaxResponseSize:    config.DefaultUpstreamMaxResponseSize,
+		UpstreamConnectionsPerHost: config.DefaultUpstreamConnectionsPerHost,
+		UpstreamConnections:        config.DefaultUpstreamConnections,
+		MaxConcurrentProcessing:    2,
+		DownstreamTimeout:          config.DefaultDownstreamTimeout,
 	}
 
-	fetcherCfg := httpfetcher.DefaultConfig()
-	fetcherCfg.AllowHTTP = true
-	fetcherCfg.DialContext = func(
-		ctx context.Context, network, _ string,
-	) (net.Conn, error) {
+	dialUpstream := func(ctx context.Context, network, _ string) (net.Conn, error) {
 		var dialer net.Dialer
 
 		return dialer.DialContext(ctx, network, upstream.Listener.Addr().String())
 	}
-	fetcher := httpfetcher.New(fetcherCfg)
 
 	var (
 		h  *handlers.Handlers
@@ -170,8 +171,11 @@ func startImageProxy(
 			healthcheck.New,
 			handlers.New,
 			middleware.New,
-			func() httpfetcher.Fetcher { return fetcher },
-			func() prometheus.Registerer { return prometheus.NewRegistry() },
+			func(db *database.Database) (
+				*imgcache.Service, *session.Manager, *encurl.Generator, error,
+			) {
+				return newImagePipeline(cfg, db, dialUpstream)
+			},
 		),
 		fx.Populate(&h, &mw, &db),
 	)
@@ -188,8 +192,9 @@ func startImageProxy(
 
 // startImageProxyWithBlockedNetworks starts the components pixad's fx app
 // builds, from a config with a fresh state directory and blockedNetworks as
-// blocked_networks. Unlike startImageProxy it provides no upstream fetcher, so
-// the handlers build the one pixad uses, which checks every address it
+// blocked_networks, with the image pipeline newImagePipeline builds. Unlike
+// startImageProxy it gives the fetcher no dial function, so the fetcher
+// connects with the dialer pixad uses, which checks every address it
 // connects to against the built-in blocked ranges and blocked_networks. It
 // returns the server with its routes.
 func startImageProxyWithBlockedNetworks(
@@ -207,8 +212,9 @@ func startImageProxyWithBlockedNetworks(
 		// fails within this time at the latest.
 		UpstreamFetchTimeout: 2 * time.Second,
 		// With no connection slots, the fetch would fail before connecting.
-		UpstreamConnections: config.DefaultUpstreamConnections,
-		DownstreamTimeout:   config.DefaultDownstreamTimeout,
+		UpstreamConnectionsPerHost: config.DefaultUpstreamConnectionsPerHost,
+		UpstreamConnections:        config.DefaultUpstreamConnections,
+		DownstreamTimeout:          config.DefaultDownstreamTimeout,
 	}
 
 	var (
@@ -225,7 +231,11 @@ func startImageProxyWithBlockedNetworks(
 			healthcheck.New,
 			handlers.New,
 			middleware.New,
-			func() prometheus.Registerer { return prometheus.NewRegistry() },
+			func(db *database.Database) (
+				*imgcache.Service, *session.Manager, *encurl.Generator, error,
+			) {
+				return newImagePipeline(cfg, db, nil)
+			},
 		),
 		fx.Populate(&h, &mw),
 	)

internal/server/login_rate_limit_internal_test.go

@@ -1,7 +1,9 @@
 package server
 
 import (
+	"context"
 	"io"
+	"net"
 	"net/http"
 	"net/http/httptest"
 	"net/netip"
@@ -17,11 +19,16 @@ import (
 
 	"sneak.berlin/go/pixa/internal/config"
 	"sneak.berlin/go/pixa/internal/database"
+	"sneak.berlin/go/pixa/internal/encurl"
 	"sneak.berlin/go/pixa/internal/globals"
 	"sneak.berlin/go/pixa/internal/handlers"
 	"sneak.berlin/go/pixa/internal/healthcheck"
+	"sneak.berlin/go/pixa/internal/httpfetcher"
+	"sneak.berlin/go/pixa/internal/imageprocessor"
+	"sneak.berlin/go/pixa/internal/imgcache"
 	"sneak.berlin/go/pixa/internal/logger"
 	"sneak.berlin/go/pixa/internal/middleware"
+	"sneak.berlin/go/pixa/internal/session"
 )
 
 // testSigningKey is a throwaway signing key; submitting it logs in.
@@ -80,9 +87,14 @@ func newTestServer(t *testing.T) *Server {
 		t.Fatalf("healthcheck.New() error = %v", err)
 	}
 
+	svc, sessMgr, encGen, err := newImagePipeline(cfg, db, nil)
+	if err != nil {
+		t.Fatalf("newImagePipeline() error = %v", err)
+	}
+
 	h, err := handlers.New(lc, handlers.Params{
-		Logger: log, Healthcheck: hc, Database: db, Config: cfg,
-		MetricsRegistry: prometheus.NewRegistry(),
+		Logger: log, Healthcheck: hc, Config: cfg,
+		ImageService: svc, SessionManager: sessMgr, EncryptedURLGenerator: encGen,
 	})
 	if err != nil {
 		t.Fatalf("handlers.New() error = %v", err)
@@ -102,6 +114,66 @@ func newTestServer(t *testing.T) *Server {
 	return s
 }
 
+// newImagePipeline builds from cfg and db the parts handlers.New takes
+// that pixad's fx app builds with the providers of internal/app: the image
+// service, the session manager and the encrypted URL generator. These
+// tests cannot import internal/app, as it imports this package. The
+// upstream fetcher connects with dialContext when that is not nil, as
+// httpfetcher.Config says. The disk cache has the limit cache_max_bytes
+// sets, and is off when it is omitted, so that no test sizes it from the
+// host's free disk space. The cache's background goroutines do not run.
+func newImagePipeline(
+	cfg *config.Config,
+	db *database.Database,
+	dialContext func(ctx context.Context, network, addr string) (net.Conn, error),
+) (*imgcache.Service, *session.Manager, *encurl.Generator, error) {
+	cache, err := imgcache.NewCache(db.DB(), imgcache.CacheConfig{
+		StateDir:    cfg.StateDir,
+		NegativeTTL: imgcache.DefaultNegativeTTL,
+		MaxBytes:    cfg.CacheMaxBytes,
+	}, prometheus.NewRegistry())
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	svc, err := imgcache.NewService(&imgcache.ServiceConfig{
+		Cache: cache,
+		Fetcher: httpfetcher.New(&httpfetcher.Config{
+			Timeout:               cfg.UpstreamFetchTimeout,
+			MaxResponseSize:       cfg.UpstreamMaxResponseSize,
+			AllowHTTP:             cfg.AllowHTTP,
+			MaxConnectionsPerHost: cfg.UpstreamConnectionsPerHost,
+			MaxConnections:        cfg.UpstreamConnections,
+			BlockedNetworks:       cfg.BlockedNetworks,
+			DialContext:           dialContext,
+		}),
+		Processor: imageprocessor.New(imageprocessor.Params{
+			MaxInputBytes:           cfg.UpstreamMaxResponseSize,
+			MaxConcurrentProcessing: cfg.MaxConcurrentProcessing,
+		}),
+		SigningKey:      cfg.SigningKey,
+		Allowlist:       cfg.AllowlistHosts,
+		AllowHTTP:       cfg.AllowHTTP,
+		MaxResponseSize: cfg.UpstreamMaxResponseSize,
+	})
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	sessMgr, err := session.NewManagerWithDatabase(
+		context.Background(), cfg.SigningKey, db.DB())
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	encGen, err := encurl.NewGenerator(cfg.SigningKey)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	return svc, sessMgr, encGen, nil
+}
+
 // clientRequest builds a request for / arriving from remoteAddr, carrying
 // forwardedFor as its X-Forwarded-For header when that is not empty.
 func clientRequest(

internal/server/logout_internal_test.go

@@ -9,15 +9,16 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/prometheus/client_golang/prometheus"
 	"go.uber.org/fx"
 	"go.uber.org/fx/fxtest"
 
 	"sneak.berlin/go/pixa/internal/config"
 	"sneak.berlin/go/pixa/internal/database"
+	"sneak.berlin/go/pixa/internal/encurl"
 	"sneak.berlin/go/pixa/internal/globals"
 	"sneak.berlin/go/pixa/internal/handlers"
 	"sneak.berlin/go/pixa/internal/healthcheck"
+	"sneak.berlin/go/pixa/internal/imgcache"
 	"sneak.berlin/go/pixa/internal/logger"
 	"sneak.berlin/go/pixa/internal/middleware"
 	"sneak.berlin/go/pixa/internal/session"
@@ -28,8 +29,9 @@ import (
 var logoutFormPattern = regexp.MustCompile(
 	`(?s)action="/logout".*?name="gorilla\.csrf\.Token" value="([^"]+)"`)
 
-// startPixa starts the components pixad's fx app builds, from cfg, and
-// returns the server with its routes, as newTestServer does.
+// startPixa starts the components pixad's fx app builds, from cfg, with the
+// image pipeline newImagePipeline builds, and returns the server with its
+// routes, as newTestServer does.
 func startPixa(t *testing.T, cfg *config.Config) *Server {
 	t.Helper()
 
@@ -47,7 +49,11 @@ func startPixa(t *testing.T, cfg *config.Config) *Server {
 			healthcheck.New,
 			handlers.New,
 			middleware.New,
-			func() prometheus.Registerer { return prometheus.NewRegistry() },
+			func(db *database.Database) (
+				*imgcache.Service, *session.Manager, *encurl.Generator, error,
+			) {
+				return newImagePipeline(cfg, db, nil)
+			},
 		),
 		fx.Populate(&h, &mw),
 	)