Build the image pipeline in pixad's fx graph (closes #313)

internal/app now provides the cache, the fetcher, the processor, the
image service, the session manager and the encrypted URL generator, and
handlers.New receives them built. database.New opens the database, so
these parts can read it while fx builds them; the cache's stop hook
still runs before the database's.

Each setting's default is declared in internal/config, the default
cache_max_bytes included, which internal/app works out before it builds
the cache. The cache's MaxBytes is the one value for the disk cache
limit, and 0 turns the disk cache off. The fetcher and the processor
take explicit values. The server tests build the pipeline themselves,
as they cannot import internal/app.

Model: opus-5-5

internal/server/image_proxy_internal_test.go

@@ -19,19 +19,20 @@ import (
 	"testing"
 	"time"
 
-	"github.com/prometheus/client_golang/prometheus"
 	"go.uber.org/fx"
 	"go.uber.org/fx/fxtest"
 
 	"sneak.berlin/go/pixa/internal/config"
 	"sneak.berlin/go/pixa/internal/database"
+	"sneak.berlin/go/pixa/internal/encurl"
 	"sneak.berlin/go/pixa/internal/globals"
 	"sneak.berlin/go/pixa/internal/handlers"
 	"sneak.berlin/go/pixa/internal/healthcheck"
-	"sneak.berlin/go/pixa/internal/httpfetcher"
 	"sneak.berlin/go/pixa/internal/imagerequest"
+	"sneak.berlin/go/pixa/internal/imgcache"
 	"sneak.berlin/go/pixa/internal/logger"
 	"sneak.berlin/go/pixa/internal/middleware"
+	"sneak.berlin/go/pixa/internal/session"
 )
 
 // upstreamHost is the upstream host of the image URLs below. It is a
@@ -120,9 +121,10 @@ func TestImageProxyFlow(t *testing.T) {
 }
 
 // startImageProxy starts the components pixad's fx app builds, from a config
-// with a fresh state directory and upstreamHost on the allowlist, and with an
-// upstream fetcher that connects every upstream address to upstream. It
-// returns the server with its routes, the database and the state directory.
+// with a fresh state directory and upstreamHost on the allowlist, with the
+// image pipeline newImagePipeline builds and an upstream fetcher that
+// connects every upstream address to upstream. It returns the server with
+// its routes, the database and the state directory.
 func startImageProxy(
 	t *testing.T, upstream *httptest.Server,
 ) (*Server, *sql.DB, string) {
@@ -138,22 +140,21 @@ func startImageProxy(
 		AllowHTTP: true,
 		// A limit of its own, so the cache does not size itself from the
 		// host's free disk space.
-		CacheMaxBytes:           64 << 20,
-		CacheMaxBytesExplicit:   true,
-		UpstreamMaxResponseSize: config.DefaultUpstreamMaxResponseSize,
-		DownstreamTimeout:       config.DefaultDownstreamTimeout,
+		CacheMaxBytes:              64 << 20,
+		CacheMaxBytesExplicit:      true,
+		UpstreamFetchTimeout:       config.DefaultUpstreamFetchTimeout,
+		UpstreamMaxResponseSize:    config.DefaultUpstreamMaxResponseSize,
+		UpstreamConnectionsPerHost: config.DefaultUpstreamConnectionsPerHost,
+		UpstreamConnections:        config.DefaultUpstreamConnections,
+		MaxConcurrentProcessing:    2,
+		DownstreamTimeout:          config.DefaultDownstreamTimeout,
 	}
 
-	fetcherCfg := httpfetcher.DefaultConfig()
-	fetcherCfg.AllowHTTP = true
-	fetcherCfg.DialContext = func(
-		ctx context.Context, network, _ string,
-	) (net.Conn, error) {
+	dialUpstream := func(ctx context.Context, network, _ string) (net.Conn, error) {
 		var dialer net.Dialer
 
 		return dialer.DialContext(ctx, network, upstream.Listener.Addr().String())
 	}
-	fetcher := httpfetcher.New(fetcherCfg)
 
 	var (
 		h  *handlers.Handlers
@@ -170,8 +171,11 @@ func startImageProxy(
 			healthcheck.New,
 			handlers.New,
 			middleware.New,
-			func() httpfetcher.Fetcher { return fetcher },
-			func() prometheus.Registerer { return prometheus.NewRegistry() },
+			func(db *database.Database) (
+				*imgcache.Service, *session.Manager, *encurl.Generator, error,
+			) {
+				return newImagePipeline(cfg, db, dialUpstream)
+			},
 		),
 		fx.Populate(&h, &mw, &db),
 	)
@@ -188,8 +192,9 @@ func startImageProxy(
 
 // startImageProxyWithBlockedNetworks starts the components pixad's fx app
 // builds, from a config with a fresh state directory and blockedNetworks as
-// blocked_networks. Unlike startImageProxy it provides no upstream fetcher, so
-// the handlers build the one pixad uses, which checks every address it
+// blocked_networks, with the image pipeline newImagePipeline builds. Unlike
+// startImageProxy it gives the fetcher no dial function, so the fetcher
+// connects with the dialer pixad uses, which checks every address it
 // connects to against the built-in blocked ranges and blocked_networks. It
 // returns the server with its routes.
 func startImageProxyWithBlockedNetworks(
@@ -207,8 +212,9 @@ func startImageProxyWithBlockedNetworks(
 		// fails within this time at the latest.
 		UpstreamFetchTimeout: 2 * time.Second,
 		// With no connection slots, the fetch would fail before connecting.
-		UpstreamConnections: config.DefaultUpstreamConnections,
-		DownstreamTimeout:   config.DefaultDownstreamTimeout,
+		UpstreamConnectionsPerHost: config.DefaultUpstreamConnectionsPerHost,
+		UpstreamConnections:        config.DefaultUpstreamConnections,
+		DownstreamTimeout:          config.DefaultDownstreamTimeout,
 	}
 
 	var (
@@ -225,7 +231,11 @@ func startImageProxyWithBlockedNetworks(
 			healthcheck.New,
 			handlers.New,
 			middleware.New,
-			func() prometheus.Registerer { return prometheus.NewRegistry() },
+			func(db *database.Database) (
+				*imgcache.Service, *session.Manager, *encurl.Generator, error,
+			) {
+				return newImagePipeline(cfg, db, nil)
+			},
 		),
 		fx.Populate(&h, &mw),
 	)