Detect a fetched image's format from its bytes once (closes #319)

magic.DetectFormat returns an imagerequest.Format. ValidateMagicBytes
returns the format it checked against the upstream Content-Type. The
cache reads a cached source's format from its file's first bytes, as it
reads a variant's. The proxy hands the format to Process. Process
refuses a format that is not a source format, and input libvips would
load with another loader, such as an SVG passing as AVIF.
Format.MIMEType declares each MIME string once, and the fetcher, the
cache's variant read and the Accept matching use it. magic.MIMEType,
magic.ImageFormat, MIMEToImageFormat, magic's SVG detection,
detectFormat, formatFromString, FormatToMIME and Result.InputFormat are
gone.

Model: opus-5-5

internal/imgcache/cache.go

@@ -175,7 +175,7 @@ func (c *Cache) Variant(
 		return nil, 0, "", err
 	}
 
-	contentType, err := outputContentType(file)
+	format, err := readFormat(file)
 	if err != nil {
 		_ = file.Close()
 
@@ -184,7 +184,7 @@ func (c *Cache) Variant(
 
 	c.touchVariant(cacheKey)
 
-	return file, size, contentType, nil
+	return file, size, format.MIMEType(), nil
 }
 
 // PutVariant stores a processed variant by its cache key and records
@@ -227,15 +227,17 @@ func (c *Cache) PutVariant(
 	return nil
 }
 
-// Source opens the cached source of src and returns a reader and its size
-// in bytes. A source whose row or file is missing is ErrNotFound, as is any
-// source of a disabled cache. A source it opens has its access time handed
-// to the background writer without waiting (see touchSourceContent).
+// Source opens the cached source of src and returns a reader, its size in
+// bytes and its format, read from the file's first bytes. A source whose row
+// or file is missing is ErrNotFound, as is any source of a disabled cache; a
+// file whose first bytes are not those of a source format is another error.
+// A source it opens has its access time handed to the background writer
+// without waiting (see touchSourceContent).
 func (c *Cache) Source(
 	ctx context.Context, src imagerequest.Source,
-) (io.ReadCloser, int64, error) {
+) (io.ReadCloser, int64, imagerequest.Format, error) {
 	if c.config.MaxBytes == 0 {
-		return nil, 0, ErrNotFound
+		return nil, 0, "", ErrNotFound
 	}
 
 	key := src.Key()
@@ -246,21 +248,28 @@ func (c *Cache) Source(
 		SELECT EXISTS(SELECT 1 FROM source_content WHERE source_key = ?)
 	`, string(key)).Scan(&rowExists)
 	if err != nil {
-		return nil, 0, fmt.Errorf("failed to lookup source: %w", err)
+		return nil, 0, "", fmt.Errorf("failed to lookup source: %w", err)
 	}
 
 	if !rowExists {
-		return nil, 0, ErrNotFound
+		return nil, 0, "", ErrNotFound
 	}
 
 	file, size, err := c.srcContent.Open(string(key))
 	if err != nil {
-		return nil, 0, err
+		return nil, 0, "", err
+	}
+
+	format, err := readFormat(file)
+	if err != nil {
+		_ = file.Close()
+
+		return nil, 0, "", err
 	}
 
 	c.touchSourceContent(key)
 
-	return file, size, nil
+	return file, size, format, nil
 }
 
 // PutSource stores fetched source content: it writes the file named by
@@ -440,13 +449,10 @@ func (c *Cache) cleanExpiredBatch(ctx context.Context) (int64, error) {
 	return deleted, nil
 }
 
-// errNotAnOutputFormat is returned for a variant file whose first bytes are
-// those of a format pixa reads but never writes, such as SVG.
-var errNotAnOutputFormat = errors.New("variant file is not in an output format")
-
-// outputContentType reads the first bytes of content, without moving its
-// offset, and returns the content type of the output format they show.
-func outputContentType(content io.ReaderAt) (string, error) {
+// readFormat reads the first bytes of a source or variant file, without
+// moving its offset, and returns the format they show. Sources and variants
+// are in the same six formats.
+func readFormat(content io.ReaderAt) (imagerequest.Format, error) {
 	firstBytes := make([]byte, magic.MinMagicBytes)
 
 	_, err := content.ReadAt(firstBytes, 0)
@@ -454,15 +460,5 @@ func outputContentType(content io.ReaderAt) (string, error) {
 		return "", fmt.Errorf("failed to read the first bytes: %w", err)
 	}
 
-	detected, err := magic.DetectFormat(firstBytes)
-	if err != nil {
-		return "", err
-	}
-
-	_, isOutputFormat := magic.MIMEToImageFormat(string(detected))
-	if !isOutputFormat {
-		return "", fmt.Errorf("%w: %s", errNotAnOutputFormat, detected)
-	}
-
-	return string(detected), nil
+	return magic.DetectFormat(firstBytes)
 }

internal/imgcache/cache_hit_internal_test.go

@@ -32,7 +32,7 @@ func TestVariant_ContentTypeFromFirstBytes(t *testing.T) {
 		imagerequest.FormatWebP, imagerequest.FormatAVIF, imagerequest.FormatJXL,
 	} {
 		result, err := processor.Process(t.Context(), bytes.NewReader(source),
-			&imagerequest.Transform{
+			imagerequest.FormatJPEG, &imagerequest.Transform{
 				Size:    imagerequest.Size{Width: 20, Height: 15},
 				Format:  format,
 				Quality: 85,
@@ -68,7 +68,7 @@ func TestVariant_ContentTypeFromFirstBytes(t *testing.T) {
 
 		t.Logf("%s: %d bytes served as %s", format, size, contentType)
 
-		if want := imageprocessor.FormatToMIME(format); contentType != want {
+		if want := format.MIMEType(); contentType != want {
 			t.Errorf("%s variant content type = %q, want %q", format, contentType, want)
 		}
 
@@ -96,7 +96,7 @@ func TestVariant_RefusesFileOfNoOutputFormat(t *testing.T) {
 	}{
 		{"text", []byte("this is text, not an image"), magic.ErrUnknownFormat},
 		{"SVG", []byte(`<svg xmlns="http://www.w3.org/2000/svg"></svg>`),
-			errNotAnOutputFormat},
+			magic.ErrUnknownFormat},
 		{"JPEG cut to 10 bytes", jpegImage[:10], io.EOF},
 		{"empty", nil, io.EOF},
 	}

internal/imgcache/eviction_internal_test.go

@@ -455,7 +455,7 @@ func assertDisabledCacheReadsAlwaysMiss(
 
 	ctx := t.Context()
 
-	reader, _, err := cache.Source(ctx, req.Source)
+	reader, _, _, err := cache.Source(ctx, req.Source)
 	if !errors.Is(err, ErrNotFound) {
 		t.Fatalf("Source on disabled cache error = %v, want %v", err, ErrNotFound)
 	}

internal/imgcache/pendingaccesstimes_internal_test.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"database/sql"
 	"errors"
+	"image/color"
 	"io"
 	"log/slog"
 	"strings"
@@ -204,8 +205,8 @@ func TestSource_HandsAccessTimeToTheBackgroundWriter(t *testing.T) {
 	t.Parallel()
 
 	cache, _ := newEvictionTestCache(t, 1<<30)
-	key := storeEvictionTestSource(t, cache, "example.com", "/photo.jpg",
-		[]byte("source bytes"))
+	content := generateTestJPEG(t, 10, 10, color.RGBA{0, 0, 255, 255})
+	key := storeEvictionTestSource(t, cache, "example.com", "/photo.jpg", content)
 	anHourAgo := time.Now().Add(-time.Hour)
 	setSourceLastAccessed(t, cache, key, anHourAgo)
 
@@ -215,7 +216,7 @@ func TestSource_HandsAccessTimeToTheBackgroundWriter(t *testing.T) {
 	`
 	anHourAgoInDatabase := anHourAgo.UTC().Format(sqliteTimestampFormat)
 
-	reader, _, err := cache.Source(t.Context(),
+	reader, _, _, err := cache.Source(t.Context(),
 		imagerequest.Source{Host: "example.com", Path: "/photo.jpg"})
 	if err != nil {
 		t.Fatalf("Source() error = %v; want the source of %q", err, key)
@@ -224,7 +225,7 @@ func TestSource_HandsAccessTimeToTheBackgroundWriter(t *testing.T) {
 	found, err := io.ReadAll(reader)
 	_ = reader.Close()
 
-	if err != nil || string(found) != "source bytes" {
+	if err != nil || !bytes.Equal(found, content) {
 		t.Fatalf("Source() reads %q, %v; want the source of %q", found, err, key)
 	}
 

internal/imgcache/source_content_internal_test.go

@@ -2,12 +2,17 @@ package imgcache
 
 import (
 	"bytes"
+	"errors"
+	"fmt"
+	"image/color"
 	"io"
 	"os"
 	"path/filepath"
 	"testing"
 
+	"sneak.berlin/go/pixa/internal/imageprocessor"
 	"sneak.berlin/go/pixa/internal/imagerequest"
+	"sneak.berlin/go/pixa/internal/magic"
 )
 
 // TestPutSource_StoresIdenticalBytesOfTwoURLsTwice stores the same bytes
@@ -64,7 +69,7 @@ func TestEvictSourceBlob_LeavesSourceWithIdenticalBytesServed(t *testing.T) {
 	t.Parallel()
 
 	cache, _ := newEvictionTestCache(t, 1<<30)
-	content := bytes.Repeat([]byte{0x5B}, 400)
+	content := generateTestJPEG(t, 20, 20, color.RGBA{0x5B, 0x5B, 0x5B, 255})
 
 	evicted := storeEvictionTestSource(t, cache, testHostCDN, "/evicted.jpg", content)
 	storeEvictionTestSource(t, cache, testHostCDN, "/kept.jpg", content)
@@ -85,7 +90,7 @@ func TestEvictSourceBlob_LeavesSourceWithIdenticalBytesServed(t *testing.T) {
 		t.Errorf("the evicted source still has %d source_content rows", n)
 	}
 
-	reader, _, err := cache.Source(t.Context(),
+	reader, _, _, err := cache.Source(t.Context(),
 		imagerequest.Source{Host: testHostCDN, Path: "/kept.jpg"})
 	t.Logf("Source() of the kept source: error %v", err)
 
@@ -135,3 +140,99 @@ func TestPutSource_FailedRowWriteLeavesNoFile(t *testing.T) {
 		t.Error("the source's file is on disk although its row was not written")
 	}
 }
+
+// TestSource_FormatFromFirstBytes stores an image in each source format
+// under one source URL, in turn. After each store, Source must return that
+// image whole, with its format read from the file.
+func TestSource_FormatFromFirstBytes(t *testing.T) {
+	t.Parallel()
+
+	cache, _ := newEvictionTestCache(t, 1<<30)
+	processor := imageprocessor.New(imageprocessor.Params{
+		MaxInputBytes:           50 << 20,
+		MaxConcurrentProcessing: 1,
+	})
+	jpegImage := generateTestJPEG(t, 40, 30, color.RGBA{200, 40, 40, 255})
+	src := imagerequest.Source{Host: testHostCDN, Path: testPathCat}
+
+	for _, format := range []imagerequest.Format{
+		imagerequest.FormatJPEG, imagerequest.FormatPNG, imagerequest.FormatGIF,
+		imagerequest.FormatWebP, imagerequest.FormatAVIF, imagerequest.FormatJXL,
+	} {
+		result, err := processor.Process(t.Context(), bytes.NewReader(jpegImage),
+			imagerequest.FormatJPEG, &imagerequest.Transform{Format: format})
+		if err != nil {
+			t.Fatalf("making a %s image: %v", format, err)
+		}
+
+		image, err := io.ReadAll(result.Content)
+		if err != nil {
+			t.Fatalf("reading the %s image: %v", format, err)
+		}
+
+		_, err = cache.PutSource(t.Context(), src, bytes.NewReader(image))
+		if err != nil {
+			t.Fatalf("PutSource(%s) error = %v", format, err)
+		}
+
+		reader, size, got, err := cache.Source(t.Context(), src)
+		if err != nil {
+			t.Fatalf("Source() of the %s image error = %v", format, err)
+		}
+
+		served, err := io.ReadAll(reader)
+		_ = reader.Close()
+
+		if err != nil {
+			t.Fatalf("reading the %s source: %v", format, err)
+		}
+
+		t.Logf("%s: %d bytes read as %s", format, size, got)
+
+		if got != format {
+			t.Errorf("Source() of the %s image format = %q, want %q",
+				format, got, format)
+		}
+
+		if size != int64(len(image)) || !bytes.Equal(served, image) {
+			t.Errorf("%s source read as %d bytes, size %d, want the %d bytes stored",
+				format, len(served), size, len(image))
+		}
+	}
+}
+
+// TestSource_RefusesFileOfNoSourceFormat checks that Source refuses a
+// cached source whose file's first bytes are not those of a source format,
+// with an error other than ErrNotFound.
+func TestSource_RefusesFileOfNoSourceFormat(t *testing.T) {
+	t.Parallel()
+
+	cache, _ := newEvictionTestCache(t, 1<<30)
+	jpegImage := generateTestJPEG(t, 10, 10, color.RGBA{0, 0, 255, 255})
+
+	tests := []struct {
+		name    string
+		content []byte
+		wantErr error
+	}{
+		{"text", []byte("this is text, not an image"), magic.ErrUnknownFormat},
+		{"SVG", []byte(`<svg xmlns="http://www.w3.org/2000/svg"></svg>`),
+			magic.ErrUnknownFormat},
+		{"JPEG cut to 10 bytes", jpegImage[:10], io.EOF},
+		{"empty file", nil, io.EOF},
+	}
+
+	for i, tt := range tests {
+		path := fmt.Sprintf("/refused%d.jpg", i)
+		storeEvictionTestSource(t, cache, testHostCDN, path, tt.content)
+
+		reader, _, format, err := cache.Source(t.Context(),
+			imagerequest.Source{Host: testHostCDN, Path: path})
+		t.Logf("%s: Source() error = %v", tt.name, err)
+
+		if reader != nil || format != "" || !errors.Is(err, tt.wantErr) {
+			t.Errorf("%s: Source() = reader %v, format %q, error %v; "+
+				"want no reader and %v", tt.name, reader, format, err, tt.wantErr)
+		}
+	}
+}