From f9b157c424a9093cb4db04c612b68db309a40422 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 17:55:25 +0000 Subject: [PATCH] Test the image proxy flow end to end (closes #80) TestImageProxyFlow in internal/server starts the database, handlers and middleware from the constructors pixad uses, in an fx app with a fresh state directory, and replaces only the upstream origin with a local test server, reached through the real fetcher by its new DialContext. For a resize with a change to JPEG and for orig, the first request answers 200 with the right content type, decoded size and X-Pixa-Cache MISS; the second answers HIT with the same image while the upstream has had one request; the source and the converted image are then on disk with their rows in SQLite. TODO.md records it and drops the item from Future Steps. Model: opus-5-5 --- TODO.md | 16 +- internal/server/image_proxy_internal_test.go | 302 +++++++++++++++++++ 2 files changed, 317 insertions(+), 1 deletion(-) create mode 100644 internal/server/image_proxy_internal_test.go diff --git a/TODO.md b/TODO.md index 3c1d04b..b30fbd2 100644 --- a/TODO.md +++ b/TODO.md @@ -31,6 +31,21 @@ P2: security: referer blacklist # Completed Steps +- 2026-10-04 an integration test of the image proxy flow (closes #80): + `TestImageProxyFlow` in `internal/server` starts the database, handlers and + middleware from the constructors `pixad` uses, with a fresh state directory, + and replaces only the upstream origin with a local test server. For a resize + with a change to JPEG and for `orig`, the first request goes through the + router, the real fetcher, libvips, the disk cache and SQLite and answers 200 + with the right content type and size and `X-Pixa-Cache: MISS`; the second + answers `HIT` with the same image and the upstream has had one request; the + source and the converted image are then in `cache/sources` and + `cache/variants`, with their rows in `source_content`, `source_metadata` and + `variant_content`. Two optional fields make this possible, which `pixad` does + not set and the config file and environment cannot: + `httpfetcher.Config.DialContext` connects in place of the dialer that refuses + internal addresses, the URL and redirect checks still running, and + `handlers.Params.Fetcher` replaces the fetcher the handlers build. - 2026-10-04 logging in, logging out, the URL generator and `/v1/e/` have handler tests (closes #77): new tests in `internal/handlers`, with no network, check that `GET /` without a login session shows the login form; a @@ -549,5 +564,4 @@ P2: security: referer blacklist - optional Sentry error reporting - comprehensive request logging - Prometheus performance metrics - - integration tests for the image proxy flow - load tests to verify the 1k to 5k req/s target diff --git a/internal/server/image_proxy_internal_test.go b/internal/server/image_proxy_internal_test.go new file mode 100644 index 0000000..24b7961 --- /dev/null +++ b/internal/server/image_proxy_internal_test.go @@ -0,0 +1,302 @@ +package server + +import ( + "bytes" + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "image" + "image/color" + "image/jpeg" + "image/png" + "io" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync/atomic" + "testing" + + "go.uber.org/fx" + "go.uber.org/fx/fxtest" + + "sneak.berlin/go/pixa/internal/config" + "sneak.berlin/go/pixa/internal/database" + "sneak.berlin/go/pixa/internal/globals" + "sneak.berlin/go/pixa/internal/handlers" + "sneak.berlin/go/pixa/internal/healthcheck" + "sneak.berlin/go/pixa/internal/httpfetcher" + "sneak.berlin/go/pixa/internal/logger" + "sneak.berlin/go/pixa/internal/middleware" +) + +// upstreamHost is the upstream host of the image URLs below. It is a +// documentation address (RFC 5737), which the fetcher's URL check accepts as +// public; the fetcher's dial function connects it to the test upstream server. +const upstreamHost = "192.0.2.10" + +// TestImageProxyFlow requests images through pixa's router, handlers, +// upstream fetcher, image processor, disk cache and database, with only the +// upstream origin replaced by a local test server. The first request for a URL +// is fetched and converted; the second is served from the cache without +// another upstream request. The source and the converted image are then on +// disk, with their rows in the database. +func TestImageProxyFlow(t *testing.T) { + t.Parallel() + + source := encodeTestPNG(t, 64, 48) + + tests := []struct { + name string + sizeFormat string // the . part of the image URL + contentType string + decodeConfig func(io.Reader) (image.Config, error) + width, height int + }{ + {"resize and convert to JPEG", "32x24.jpeg", "image/jpeg", + jpeg.DecodeConfig, 32, 24}, + {"orig", "orig.orig", "image/png", png.DecodeConfig, 64, 48}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + var upstreamRequests atomic.Int32 + + upstream := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + upstreamRequests.Add(1) + w.Header().Set("Content-Type", "image/png") + _, _ = w.Write(source) + })) + t.Cleanup(upstream.Close) + + s, db, stateDir := startImageProxy(t, upstream) + target := "/v1/image/" + upstreamHost + "/photo.png/" + tt.sizeFormat + + first := getImage(t, s, target) + + if got := first.Header().Get("X-Pixa-Cache"); got != "MISS" { + t.Errorf("first X-Pixa-Cache = %q, want MISS", got) + } + + if got := first.Header().Get("Content-Type"); got != tt.contentType { + t.Errorf("Content-Type = %q, want %q", got, tt.contentType) + } + + decoded, err := tt.decodeConfig(bytes.NewReader(first.Body.Bytes())) + if err != nil { + t.Fatalf("decoding the image: %v", err) + } + + if decoded.Width != tt.width || decoded.Height != tt.height { + t.Errorf("image is %dx%d, want %dx%d", + decoded.Width, decoded.Height, tt.width, tt.height) + } + + second := getImage(t, s, target) + + if got := second.Header().Get("X-Pixa-Cache"); got != "HIT" { + t.Errorf("second X-Pixa-Cache = %q, want HIT", got) + } + + if !bytes.Equal(second.Body.Bytes(), first.Body.Bytes()) { + t.Error("the second response is not the image the first served") + } + + if got := upstreamRequests.Load(); got != 1 { + t.Errorf("upstream received %d requests, want 1", got) + } + + checkSourceCached(t, db, stateDir, source) + checkVariantCached(t, db, stateDir, first.Body.Bytes(), tt.contentType) + }) + } +} + +// startImageProxy starts the components pixad's fx app builds, from a config +// with a fresh state directory and upstreamHost on the allowlist, and with an +// upstream fetcher that connects every upstream address to upstream. It +// returns the server with its routes, the database and the state directory. +func startImageProxy( + t *testing.T, upstream *httptest.Server, +) (*Server, *sql.DB, string) { + t.Helper() + + stateDir := t.TempDir() + cfg := &config.Config{ + SigningKey: testSigningKey, + StateDir: stateDir, + DBURL: "file:" + filepath.Join(stateDir, "state.sqlite3"), + AllowlistHosts: []string{upstreamHost}, + // The test upstream server has no TLS. + AllowHTTP: true, + // A limit of its own, so the cache does not size itself from the + // host's free disk space. + CacheMaxBytes: 64 << 20, + CacheMaxBytesExplicit: true, + UpstreamMaxResponseSize: config.DefaultUpstreamMaxResponseSize, + DownstreamTimeout: config.DefaultDownstreamTimeout, + } + + fetcherCfg := httpfetcher.DefaultConfig() + fetcherCfg.AllowHTTP = true + fetcherCfg.DialContext = func( + ctx context.Context, network, _ string, + ) (net.Conn, error) { + var dialer net.Dialer + + return dialer.DialContext(ctx, network, upstream.Listener.Addr().String()) + } + fetcher := httpfetcher.New(fetcherCfg) + + var ( + h *handlers.Handlers + mw *middleware.Middleware + db *database.Database + ) + + app := fxtest.New(t, + fx.Supply(cfg), + fx.Provide( + globals.New, + logger.New, + database.New, + healthcheck.New, + handlers.New, + middleware.New, + func() httpfetcher.Fetcher { return fetcher }, + ), + fx.Populate(&h, &mw, &db), + ) + app.RequireStart() + t.Cleanup(app.RequireStop) + + // Requests go straight to the router, as in newTestServer; the server's + // own start hook, which listens on a port, is left out. + s := &Server{config: cfg, mw: mw, h: h} + s.SetupRoutes() + + return s, db.DB(), stateDir +} + +// getImage sends a GET for target to s and fails unless it answers 200. +func getImage(t *testing.T, s *Server, target string) *httptest.ResponseRecorder { + t.Helper() + + rec := httptest.NewRecorder() + s.ServeHTTP(rec, httptest.NewRequestWithContext( + t.Context(), http.MethodGet, target, nil)) + t.Logf("GET %s: %d, X-Pixa-Cache %s", + target, rec.Code, rec.Header().Get("X-Pixa-Cache")) + + if rec.Code != http.StatusOK { + t.Fatalf("GET %s status = %d, want %d; body %s", + target, rec.Code, http.StatusOK, rec.Body.String()) + } + + return rec +} + +// checkSourceCached checks that source is stored under its SHA-256 in +// cache/sources, recorded in source_content, and that the source URL's row in +// source_metadata points at it. +func checkSourceCached(t *testing.T, db *sql.DB, stateDir string, source []byte) { + t.Helper() + + sum := sha256.Sum256(source) + hash := hex.EncodeToString(sum[:]) + + checkFile(t, filepath.Join(stateDir, "cache", "sources", hash[0:2], hash[2:4], hash), + source) + + var rows int + + err := db.QueryRowContext(t.Context(), + "SELECT COUNT(*) FROM source_content WHERE content_hash = ?", hash, + ).Scan(&rows) + if err != nil || rows != 1 { + t.Errorf("source_content rows for the source = %d (error %v), want 1", + rows, err) + } + + var metadataHash string + + err = db.QueryRowContext(t.Context(), + `SELECT content_hash FROM source_metadata + WHERE source_host = ? AND source_path = ?`, + upstreamHost, "/photo.png", + ).Scan(&metadataHash) + if err != nil || metadataHash != hash { + t.Errorf("source_metadata content_hash = %q (error %v), want %q", + metadataHash, err, hash) + } +} + +// checkVariantCached checks that the converted image served is recorded in +// variant_content with contentType, and stored under its cache key in +// cache/variants. +func checkVariantCached( + t *testing.T, db *sql.DB, stateDir string, served []byte, contentType string, +) { + t.Helper() + + var cacheKey, storedType string + + err := db.QueryRowContext(t.Context(), + "SELECT cache_key, content_type FROM variant_content", + ).Scan(&cacheKey, &storedType) + if err != nil { + t.Fatalf("variant_content row: %v", err) + } + + if storedType != contentType { + t.Errorf("variant_content content_type = %q, want %q", + storedType, contentType) + } + + checkFile(t, filepath.Join(stateDir, "cache", "variants", + cacheKey[0:2], cacheKey[2:4], cacheKey), served) +} + +// checkFile checks that the file at path holds want. +func checkFile(t *testing.T, path string, want []byte) { + t.Helper() + + //nolint:gosec // G304: a path under the test's state directory + got, err := os.ReadFile(path) + if err != nil { + t.Errorf("reading %s: %v", path, err) + + return + } + + if !bytes.Equal(got, want) { + t.Errorf("%s holds %d bytes that are not the %d expected", + path, len(got), len(want)) + } +} + +// encodeTestPNG returns an opaque width x height PNG of one color. +func encodeTestPNG(t *testing.T, width, height int) []byte { + t.Helper() + + img := image.NewRGBA(image.Rect(0, 0, width, height)) + for y := range height { + for x := range width { + img.Set(x, y, color.RGBA{R: 200, G: 40, B: 40, A: 255}) + } + } + + var buf bytes.Buffer + + err := png.Encode(&buf, img) + if err != nil { + t.Fatalf("encoding the test PNG: %v", err) + } + + return buf.Bytes() +}