clawbot
17376d885b
check / check (push) Canceled after 0s
The new package internal/imageproxy holds Proxy, which was imgcache.Service, with its Config, ImageResponse, CacheStatus and errors. imageproxy.New makes and registers the counters of upstream fetches and transforms. internal/imgcache holds the cache only, with its hit and miss counters, and imgcache.CacheConfig is imgcache.Config. The cache reads and writes each kind it keeps with one pair of methods: Variant and PutVariant, Source and PutSource, Failure and PutFailure. Source replaces LookupSource followed by GetSourceContent. imgcache.go and module.go are gone. The proxy's tests moved with it. The cache's tests check that the reconciliation pass removes an empty or cut file and its row. The proxy's tests remove them by hand, then check that Get fetches or converts again. Model: opus-5-5
356 lines
9.5 KiB
Go
356 lines
9.5 KiB
Go
package imageproxy
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"database/sql"
|
|
"image"
|
|
"image/color"
|
|
"image/gif"
|
|
"image/jpeg"
|
|
"image/png"
|
|
"io/fs"
|
|
"path/filepath"
|
|
"testing"
|
|
"testing/fstest"
|
|
"time"
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
"sneak.berlin/go/pixa/internal/database"
|
|
"sneak.berlin/go/pixa/internal/httpfetcher"
|
|
"sneak.berlin/go/pixa/internal/imageprocessor"
|
|
"sneak.berlin/go/pixa/internal/imagerequest"
|
|
"sneak.berlin/go/pixa/internal/imgcache"
|
|
)
|
|
|
|
// Shared test data literals, extracted as constants for goconst.
|
|
const (
|
|
testContentTypeJPEG = "image/jpeg"
|
|
)
|
|
|
|
// sqliteTimestampFormat matches the format SQLite's CURRENT_TIMESTAMP
|
|
// produces, so injected timestamps compare correctly against ones the
|
|
// cache writes.
|
|
const sqliteTimestampFormat = "2006-01-02 15:04:05"
|
|
|
|
// TestFixtures contains paths to test files in the mock filesystem.
|
|
type TestFixtures struct {
|
|
// Valid image files
|
|
GoodHostJPEG string // valid JPEG on GoodHost
|
|
GoodHostPNG string // valid PNG on GoodHost
|
|
GoodHostGIF string // valid GIF on GoodHost
|
|
OtherHostJPEG string // valid JPEG on OtherHost
|
|
OtherHostPNG string // valid PNG on OtherHost
|
|
|
|
// Invalid/edge case files
|
|
InvalidFile string // file with wrong magic bytes
|
|
EmptyFile string // zero-byte file
|
|
TextFile string // text file masquerading as image
|
|
|
|
// Hostnames
|
|
GoodHost string // a host the mock fetcher serves
|
|
OtherHost string // a second host the mock fetcher serves
|
|
}
|
|
|
|
// DefaultFixtures returns the standard test fixture paths.
|
|
func DefaultFixtures() *TestFixtures {
|
|
return &TestFixtures{
|
|
GoodHostJPEG: "goodhost.example.com/images/photo.jpg",
|
|
GoodHostPNG: "goodhost.example.com/images/logo.png",
|
|
GoodHostGIF: "goodhost.example.com/images/animation.gif",
|
|
OtherHostJPEG: "otherhost.example.com/uploads/image.jpg",
|
|
OtherHostPNG: "otherhost.example.com/uploads/icon.png",
|
|
InvalidFile: "goodhost.example.com/images/fake.jpg",
|
|
EmptyFile: "goodhost.example.com/images/empty.jpg",
|
|
TextFile: "goodhost.example.com/images/text.png",
|
|
GoodHost: "goodhost.example.com",
|
|
OtherHost: "otherhost.example.com",
|
|
}
|
|
}
|
|
|
|
// NewTestFS creates a mock filesystem with test images.
|
|
func NewTestFS(t *testing.T) (fs.FS, *TestFixtures) {
|
|
t.Helper()
|
|
|
|
fixtures := DefaultFixtures()
|
|
|
|
// Generate test images
|
|
jpegData := generateTestJPEG(t, 100, 100, color.RGBA{255, 0, 0, 255})
|
|
pngData := generateTestPNG(t, 100, 100, color.RGBA{0, 255, 0, 255})
|
|
gifData := generateTestGIF(t, 100, 100, color.RGBA{0, 0, 255, 255})
|
|
|
|
// Create the mock filesystem
|
|
mockFS := fstest.MapFS{
|
|
// Good host files
|
|
fixtures.GoodHostJPEG: &fstest.MapFile{Data: jpegData},
|
|
fixtures.GoodHostPNG: &fstest.MapFile{Data: pngData},
|
|
fixtures.GoodHostGIF: &fstest.MapFile{Data: gifData},
|
|
|
|
// Other host files
|
|
fixtures.OtherHostJPEG: &fstest.MapFile{Data: jpegData},
|
|
fixtures.OtherHostPNG: &fstest.MapFile{Data: pngData},
|
|
|
|
// Invalid files
|
|
fixtures.InvalidFile: &fstest.MapFile{Data: []byte("not a real image file content")},
|
|
fixtures.EmptyFile: &fstest.MapFile{Data: []byte{}},
|
|
fixtures.TextFile: &fstest.MapFile{Data: []byte("This is just text, not a PNG")},
|
|
}
|
|
|
|
return mockFS, fixtures
|
|
}
|
|
|
|
// generateTestJPEG creates a minimal valid JPEG image.
|
|
func generateTestJPEG(t *testing.T, width, height int, c color.Color) []byte {
|
|
t.Helper()
|
|
|
|
img := image.NewRGBA(image.Rect(0, 0, width, height))
|
|
for y := range height {
|
|
for x := range width {
|
|
img.Set(x, y, c)
|
|
}
|
|
}
|
|
|
|
var buf bytes.Buffer
|
|
|
|
err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: 85})
|
|
if err != nil {
|
|
t.Fatalf("failed to encode test JPEG: %v", err)
|
|
}
|
|
|
|
return buf.Bytes()
|
|
}
|
|
|
|
// generateTestPNG creates a minimal valid PNG image.
|
|
func generateTestPNG(t *testing.T, width, height int, c color.Color) []byte {
|
|
t.Helper()
|
|
|
|
img := image.NewRGBA(image.Rect(0, 0, width, height))
|
|
for y := range height {
|
|
for x := range width {
|
|
img.Set(x, y, c)
|
|
}
|
|
}
|
|
|
|
var buf bytes.Buffer
|
|
|
|
err := png.Encode(&buf, img)
|
|
if err != nil {
|
|
t.Fatalf("failed to encode test PNG: %v", err)
|
|
}
|
|
|
|
return buf.Bytes()
|
|
}
|
|
|
|
// generateTestGIF creates a minimal valid GIF image.
|
|
func generateTestGIF(t *testing.T, width, height int, c color.Color) []byte {
|
|
t.Helper()
|
|
|
|
img := image.NewPaletted(
|
|
image.Rect(0, 0, width, height),
|
|
[]color.Color{c, color.White},
|
|
)
|
|
for y := range height {
|
|
for x := range width {
|
|
img.SetColorIndex(x, y, 0)
|
|
}
|
|
}
|
|
|
|
var buf bytes.Buffer
|
|
|
|
err := gif.Encode(&buf, img, nil)
|
|
if err != nil {
|
|
t.Fatalf("failed to encode test GIF: %v", err)
|
|
}
|
|
|
|
return buf.Bytes()
|
|
}
|
|
|
|
// testProxy is a Proxy with the database and the state directory of its
|
|
// cache, through which a test reads and changes the cache behind the proxy.
|
|
// registry is the metrics registry the proxy and its cache share.
|
|
type testProxy struct {
|
|
*Proxy
|
|
|
|
db *sql.DB
|
|
stateDir string
|
|
registry *prometheus.Registry
|
|
}
|
|
|
|
// setupTestProxy creates a Proxy with mock fetcher for testing.
|
|
func setupTestProxy(t *testing.T) (*testProxy, *TestFixtures) {
|
|
t.Helper()
|
|
|
|
mockFS, fixtures := NewTestFS(t)
|
|
|
|
// Create temp directory for cache
|
|
tmpDir := t.TempDir()
|
|
|
|
// Create in-memory database
|
|
db := setupTestDB(t)
|
|
|
|
registry := prometheus.NewRegistry()
|
|
|
|
cache, err := imgcache.NewCache(db, imgcache.Config{
|
|
StateDir: tmpDir,
|
|
NegativeTTL: 5 * time.Minute,
|
|
MaxBytes: 1 << 30,
|
|
}, registry)
|
|
if err != nil {
|
|
t.Fatalf("failed to create cache: %v", err)
|
|
}
|
|
|
|
proxy, err := New(&Config{
|
|
Cache: cache,
|
|
Fetcher: httpfetcher.NewMock(mockFS),
|
|
Processor: imageprocessor.New(imageprocessor.Params{
|
|
MaxInputBytes: 50 << 20,
|
|
MaxConcurrentProcessing: 2,
|
|
}),
|
|
MaxResponseSize: 50 << 20,
|
|
}, registry)
|
|
if err != nil {
|
|
t.Fatalf("failed to create proxy: %v", err)
|
|
}
|
|
|
|
return &testProxy{
|
|
Proxy: proxy,
|
|
db: db,
|
|
stateDir: tmpDir,
|
|
registry: registry,
|
|
}, fixtures
|
|
}
|
|
|
|
// counterValues holds the values of the counters of a proxy and its cache.
|
|
type counterValues struct {
|
|
hitCount int64
|
|
missCount int64
|
|
upstreamFetchCount int64
|
|
upstreamFetchBytes int64
|
|
transformCount int64
|
|
}
|
|
|
|
// readCounters reads the counters of svc and its cache from their metrics
|
|
// registry.
|
|
func readCounters(t *testing.T, svc *testProxy) counterValues {
|
|
t.Helper()
|
|
|
|
families, err := svc.registry.Gather()
|
|
if err != nil {
|
|
t.Fatalf("failed to gather the counters: %v", err)
|
|
}
|
|
|
|
values := make(map[string]int64, len(families))
|
|
|
|
for _, family := range families {
|
|
values[family.GetName()] = int64(family.GetMetric()[0].GetCounter().GetValue())
|
|
}
|
|
|
|
return counterValues{
|
|
hitCount: values["pixa_cache_hits_total"],
|
|
missCount: values["pixa_cache_misses_total"],
|
|
upstreamFetchCount: values["pixa_upstream_fetches_total"],
|
|
upstreamFetchBytes: values["pixa_upstream_fetch_bytes_total"],
|
|
transformCount: values["pixa_transforms_total"],
|
|
}
|
|
}
|
|
|
|
// sourceFilePath returns the path of the file of the source key in svc's
|
|
// cache, as README.md gives it: <state_dir>/cache/sources/<ab>/<cd>/<key>.
|
|
func sourceFilePath(svc *testProxy, key imagerequest.SourceKey) string {
|
|
name := string(key)
|
|
|
|
return filepath.Join(svc.stateDir, "cache", "sources", name[0:2], name[2:4], name)
|
|
}
|
|
|
|
// variantFilePath returns the path of the file of the variant key in svc's
|
|
// cache, as README.md gives it: <state_dir>/cache/variants/<ab>/<cd>/<key>.
|
|
func variantFilePath(svc *testProxy, key imagerequest.VariantKey) string {
|
|
name := string(key)
|
|
|
|
return filepath.Join(svc.stateDir, "cache", "variants", name[0:2], name[2:4], name)
|
|
}
|
|
|
|
// setVariantLastAccessed backdates the last access time of a tracked
|
|
// variant.
|
|
func setVariantLastAccessed(
|
|
t *testing.T, db *sql.DB, key imagerequest.VariantKey, when time.Time,
|
|
) {
|
|
t.Helper()
|
|
|
|
res, err := db.ExecContext(t.Context(),
|
|
`UPDATE variant_content SET last_accessed_at = ? WHERE cache_key = ?`,
|
|
when.UTC().Format(sqliteTimestampFormat), string(key),
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("failed to set variant last_accessed_at: %v", err)
|
|
}
|
|
|
|
affected, err := res.RowsAffected()
|
|
if err != nil {
|
|
t.Fatalf("failed to read affected rows: %v", err)
|
|
}
|
|
|
|
if affected != 1 {
|
|
t.Fatalf("variant %s has no accounting row (affected=%d); "+
|
|
"stores must track variants in the database", key, affected)
|
|
}
|
|
}
|
|
|
|
// setSourceLastAccessed backdates the last access time of a tracked
|
|
// source content blob.
|
|
func setSourceLastAccessed(
|
|
t *testing.T, db *sql.DB, key imagerequest.SourceKey, when time.Time,
|
|
) {
|
|
t.Helper()
|
|
|
|
res, err := db.ExecContext(t.Context(),
|
|
`UPDATE source_content SET last_accessed_at = ? WHERE source_key = ?`,
|
|
when.UTC().Format(sqliteTimestampFormat), string(key),
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("failed to set source last_accessed_at: %v", err)
|
|
}
|
|
|
|
affected, err := res.RowsAffected()
|
|
if err != nil {
|
|
t.Fatalf("failed to read affected rows: %v", err)
|
|
}
|
|
|
|
if affected != 1 {
|
|
t.Fatalf("source %s has no accounting row (affected=%d)", key, affected)
|
|
}
|
|
}
|
|
|
|
// countRows returns the number of rows the given query yields.
|
|
func countRows(t *testing.T, db *sql.DB, query string, args ...any) int {
|
|
t.Helper()
|
|
|
|
var n int
|
|
|
|
err := db.QueryRowContext(t.Context(), query, args...).Scan(&n)
|
|
if err != nil {
|
|
t.Fatalf("count query %q failed: %v", query, err)
|
|
}
|
|
|
|
return n
|
|
}
|
|
|
|
// setupTestDB creates an in-memory SQLite database for testing
|
|
// using the production schema.
|
|
func setupTestDB(t *testing.T) *sql.DB {
|
|
t.Helper()
|
|
|
|
db, err := sql.Open("sqlite", ":memory:")
|
|
if err != nil {
|
|
t.Fatalf("failed to open test db: %v", err)
|
|
}
|
|
|
|
// Use the real production schema via migrations
|
|
err = database.ApplyMigrations(context.Background(), db, nil)
|
|
if err != nil {
|
|
t.Fatalf("failed to apply migrations: %v", err)
|
|
}
|
|
|
|
return db
|
|
}
|