Compare commits
14 Commits
5690dc39f4
...
chore/remo
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c4368b1541 | ||
| 2fb36c5ccb | |||
|
|
40c4b53b01 | ||
| b800ef86d8 | |||
| 3d857da237 | |||
| 46a92c3514 | |||
| 39354e41c3 | |||
| d1eff1315b | |||
| 814c735295 | |||
| 7ce6f81d8d | |||
|
|
e651e672aa | ||
|
|
79ceed2ee4 | ||
|
|
e3b346e881 | ||
|
|
0ff3071337 |
4
.gitignore
vendored
4
.gitignore
vendored
@@ -18,3 +18,7 @@ vendor/
|
||||
# Data
|
||||
/data/
|
||||
*.sqlite3
|
||||
|
||||
# Local dev configs
|
||||
config.yaml
|
||||
config.dev.yml
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
# Development config for local Docker testing
|
||||
signing_key: "dev-signing-key-minimum-32-chars!"
|
||||
debug: true
|
||||
allow_http: true
|
||||
whitelist_hosts:
|
||||
- localhost
|
||||
- s3.sneak.cloud
|
||||
- static.sneak.cloud
|
||||
- sneak.berlin
|
||||
- github.com
|
||||
- user-images.githubusercontent.com
|
||||
10
config.yaml
10
config.yaml
@@ -1,10 +0,0 @@
|
||||
debug: true
|
||||
port: 8080
|
||||
state_dir: ./data
|
||||
signing_key: "test-signing-key-for-development-only"
|
||||
whitelist_hosts:
|
||||
- "*.example.com"
|
||||
- "images.unsplash.com"
|
||||
- "picsum.photos"
|
||||
- "s3.sneak.cloud"
|
||||
allow_http: false
|
||||
@@ -297,19 +297,21 @@ func (c *Cache) CleanExpired(ctx context.Context) error {
|
||||
func (c *Cache) Stats(ctx context.Context) (*CacheStats, error) {
|
||||
var stats CacheStats
|
||||
|
||||
// Fetch hit/miss counts from the stats table
|
||||
err := c.db.QueryRowContext(ctx, `
|
||||
SELECT hit_count, miss_count, upstream_fetch_count, upstream_fetch_bytes, transform_count
|
||||
SELECT hit_count, miss_count
|
||||
FROM cache_stats WHERE id = 1
|
||||
`).Scan(&stats.HitCount, &stats.MissCount, &stats.TotalItems, &stats.TotalSizeBytes, &stats.HitRate)
|
||||
`).Scan(&stats.HitCount, &stats.MissCount)
|
||||
|
||||
if err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, fmt.Errorf("failed to get cache stats: %w", err)
|
||||
}
|
||||
|
||||
// Get actual counts
|
||||
// Get actual item count and total size from content tables
|
||||
_ = c.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM request_cache`).Scan(&stats.TotalItems)
|
||||
_ = c.db.QueryRowContext(ctx, `SELECT COALESCE(SUM(size_bytes), 0) FROM output_content`).Scan(&stats.TotalSizeBytes)
|
||||
|
||||
// Compute hit rate as a ratio
|
||||
if stats.HitCount+stats.MissCount > 0 {
|
||||
stats.HitRate = float64(stats.HitCount) / float64(stats.HitCount+stats.MissCount)
|
||||
}
|
||||
|
||||
43
internal/imgcache/divzero_test.go
Normal file
43
internal/imgcache/divzero_test.go
Normal file
@@ -0,0 +1,43 @@
|
||||
package imgcache
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSizePercentSafeWithZeroFetchBytes(t *testing.T) {
|
||||
// Simulate the calculation from processAndStore
|
||||
fetchBytes := int64(0)
|
||||
outputSize := int64(100)
|
||||
|
||||
var sizePercent float64
|
||||
if fetchBytes > 0 {
|
||||
sizePercent = float64(outputSize) / float64(fetchBytes) * 100.0
|
||||
}
|
||||
|
||||
// sizePercent should be 0, not Inf or NaN
|
||||
if math.IsInf(sizePercent, 0) || math.IsNaN(sizePercent) {
|
||||
t.Errorf("sizePercent = %f, should not be Inf/NaN", sizePercent)
|
||||
}
|
||||
|
||||
// Should produce valid log output
|
||||
result := fmt.Sprintf("%.1f%%", sizePercent)
|
||||
if result != "0.0%" {
|
||||
t.Errorf("formatted size ratio = %q, want %q", result, "0.0%")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSizePercentNormalCase(t *testing.T) {
|
||||
fetchBytes := int64(1000)
|
||||
outputSize := int64(500)
|
||||
|
||||
var sizePercent float64
|
||||
if fetchBytes > 0 {
|
||||
sizePercent = float64(outputSize) / float64(fetchBytes) * 100.0
|
||||
}
|
||||
|
||||
if math.Abs(sizePercent-50.0) > 0.001 {
|
||||
t.Errorf("sizePercent = %f, want 50.0", sizePercent)
|
||||
}
|
||||
}
|
||||
@@ -79,11 +79,18 @@ type ImageRequest struct {
|
||||
Signature string
|
||||
// Expires is the signature expiration timestamp
|
||||
Expires time.Time
|
||||
// AllowHTTP indicates whether HTTP (non-TLS) is allowed for this request
|
||||
AllowHTTP bool
|
||||
}
|
||||
|
||||
// SourceURL returns the full upstream URL to fetch
|
||||
// SourceURL returns the full upstream URL to fetch.
|
||||
// Uses http:// scheme when AllowHTTP is true, otherwise https://.
|
||||
func (r *ImageRequest) SourceURL() string {
|
||||
url := "https://" + r.SourceHost + r.SourcePath
|
||||
scheme := "https"
|
||||
if r.AllowHTTP {
|
||||
scheme = "http"
|
||||
}
|
||||
url := scheme + "://" + r.SourceHost + r.SourcePath
|
||||
if r.SourceQuery != "" {
|
||||
url += "?" + r.SourceQuery
|
||||
}
|
||||
|
||||
@@ -2,27 +2,11 @@ package imgcache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/database"
|
||||
)
|
||||
|
||||
func setupTestDB(t *testing.T) *sql.DB {
|
||||
t.Helper()
|
||||
db, err := sql.Open("sqlite", ":memory:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := database.ApplyMigrations(db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { db.Close() })
|
||||
return db
|
||||
}
|
||||
|
||||
func TestNegativeCache_StoreAndCheck(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
dir := t.TempDir()
|
||||
|
||||
@@ -21,6 +21,7 @@ type Service struct {
|
||||
signer *Signer
|
||||
whitelist *HostWhitelist
|
||||
log *slog.Logger
|
||||
allowHTTP bool
|
||||
}
|
||||
|
||||
// ServiceConfig holds configuration for the image service.
|
||||
@@ -68,6 +69,11 @@ func NewService(cfg *ServiceConfig) (*Service, error) {
|
||||
log = slog.Default()
|
||||
}
|
||||
|
||||
allowHTTP := false
|
||||
if cfg.FetcherConfig != nil {
|
||||
allowHTTP = cfg.FetcherConfig.AllowHTTP
|
||||
}
|
||||
|
||||
return &Service{
|
||||
cache: cfg.Cache,
|
||||
fetcher: fetcher,
|
||||
@@ -75,6 +81,7 @@ func NewService(cfg *ServiceConfig) (*Service, error) {
|
||||
signer: signer,
|
||||
whitelist: NewHostWhitelist(cfg.Whitelist),
|
||||
log: log,
|
||||
allowHTTP: allowHTTP,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -83,6 +90,9 @@ var ErrNegativeCached = errors.New("request is in negative cache (recently faile
|
||||
|
||||
// Get retrieves a processed image, fetching and processing if necessary.
|
||||
func (s *Service) Get(ctx context.Context, req *ImageRequest) (*ImageResponse, error) {
|
||||
// Propagate AllowHTTP setting to the request
|
||||
req.AllowHTTP = s.allowHTTP
|
||||
|
||||
// Check negative cache first - skip fetching for recently-failed URLs
|
||||
negHit, err := s.cache.checkNegativeCache(ctx, req)
|
||||
if err != nil {
|
||||
@@ -169,8 +179,8 @@ func (s *Service) processFromSourceOrFetch(
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch from upstream if we don't have source data
|
||||
if sourceData == nil {
|
||||
// Fetch from upstream if we don't have source data or it's empty
|
||||
if len(sourceData) == 0 {
|
||||
resp, err := s.fetchAndProcess(ctx, req, cacheKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -280,7 +290,11 @@ func (s *Service) processAndStore(
|
||||
|
||||
// Log conversion details
|
||||
outputSize := int64(len(processedData))
|
||||
sizePercent := float64(outputSize) / float64(fetchBytes) * 100.0 //nolint:mnd // percentage calculation
|
||||
|
||||
var sizePercent float64
|
||||
if fetchBytes > 0 {
|
||||
sizePercent = float64(outputSize) / float64(fetchBytes) * 100.0 //nolint:mnd // percentage calculation
|
||||
}
|
||||
|
||||
s.log.Info("image converted",
|
||||
"host", req.SourceHost,
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
@@ -96,22 +97,24 @@ func (s *Signer) GenerateSignedURL(req *ImageRequest, ttl time.Duration) (path s
|
||||
sizeStr = fmt.Sprintf("%dx%d", req.Size.Width, req.Size.Height)
|
||||
}
|
||||
|
||||
// Build the path
|
||||
path = fmt.Sprintf("/v1/image/%s%s/%s.%s",
|
||||
req.SourceHost,
|
||||
req.SourcePath,
|
||||
sizeStr,
|
||||
req.Format,
|
||||
)
|
||||
|
||||
// Append query if present
|
||||
// Build the path.
|
||||
// When a source query is present, it is embedded as a path segment
|
||||
// (e.g. /host/path?query/size.fmt) so that ParseImagePath can extract
|
||||
// it from the last-slash split. The "?" inside a path segment is
|
||||
// percent-encoded by clients but chi delivers it decoded, which is
|
||||
// exactly what the URL parser expects.
|
||||
if req.SourceQuery != "" {
|
||||
// The query goes before the size segment in our URL scheme
|
||||
// So we need to rebuild the path
|
||||
path = fmt.Sprintf("/v1/image/%s%s?%s/%s.%s",
|
||||
path = fmt.Sprintf("/v1/image/%s%s%%3F%s/%s.%s",
|
||||
req.SourceHost,
|
||||
req.SourcePath,
|
||||
url.PathEscape(req.SourceQuery),
|
||||
sizeStr,
|
||||
req.Format,
|
||||
)
|
||||
} else {
|
||||
path = fmt.Sprintf("/v1/image/%s%s/%s.%s",
|
||||
req.SourceHost,
|
||||
req.SourcePath,
|
||||
req.SourceQuery,
|
||||
sizeStr,
|
||||
req.Format,
|
||||
)
|
||||
|
||||
55
internal/imgcache/signature_query_test.go
Normal file
55
internal/imgcache/signature_query_test.go
Normal file
@@ -0,0 +1,55 @@
|
||||
package imgcache
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestGenerateSignedURL_WithQueryString(t *testing.T) {
|
||||
signer := NewSigner("test-secret-key-for-testing!")
|
||||
|
||||
req := &ImageRequest{
|
||||
SourceHost: "cdn.example.com",
|
||||
SourcePath: "/photos/cat.jpg",
|
||||
SourceQuery: "token=abc&v=2",
|
||||
Size: Size{Width: 800, Height: 600},
|
||||
Format: FormatWebP,
|
||||
}
|
||||
|
||||
path, _, _ := signer.GenerateSignedURL(req, time.Hour)
|
||||
|
||||
// The path must NOT contain a bare "?" that would be interpreted as a query string delimiter.
|
||||
// The size segment must appear as the last path component.
|
||||
if strings.Contains(path, "?token=abc") {
|
||||
t.Errorf("GenerateSignedURL() produced bare query string in path: %q", path)
|
||||
}
|
||||
|
||||
// The size segment must be present in the path
|
||||
if !strings.Contains(path, "/800x600.webp") {
|
||||
t.Errorf("GenerateSignedURL() missing size segment in path: %q", path)
|
||||
}
|
||||
|
||||
// Path should end with the size.format, not with query params
|
||||
if !strings.HasSuffix(path, "/800x600.webp") {
|
||||
t.Errorf("GenerateSignedURL() path should end with size.format: %q", path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateSignedURL_WithoutQueryString(t *testing.T) {
|
||||
signer := NewSigner("test-secret-key-for-testing!")
|
||||
|
||||
req := &ImageRequest{
|
||||
SourceHost: "cdn.example.com",
|
||||
SourcePath: "/photos/cat.jpg",
|
||||
Size: Size{Width: 800, Height: 600},
|
||||
Format: FormatWebP,
|
||||
}
|
||||
|
||||
path, _, _ := signer.GenerateSignedURL(req, time.Hour)
|
||||
|
||||
expected := "/v1/image/cdn.example.com/photos/cat.jpg/800x600.webp"
|
||||
if path != expected {
|
||||
t.Errorf("GenerateSignedURL() path = %q, want %q", path, expected)
|
||||
}
|
||||
}
|
||||
44
internal/imgcache/sourceurl_test.go
Normal file
44
internal/imgcache/sourceurl_test.go
Normal file
@@ -0,0 +1,44 @@
|
||||
package imgcache
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestImageRequest_SourceURL_DefaultHTTPS(t *testing.T) {
|
||||
req := &ImageRequest{
|
||||
SourceHost: "cdn.example.com",
|
||||
SourcePath: "/photos/cat.jpg",
|
||||
SourceQuery: "v=2",
|
||||
}
|
||||
|
||||
got := req.SourceURL()
|
||||
want := "https://cdn.example.com/photos/cat.jpg?v=2"
|
||||
if got != want {
|
||||
t.Errorf("SourceURL() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageRequest_SourceURL_AllowHTTP(t *testing.T) {
|
||||
req := &ImageRequest{
|
||||
SourceHost: "localhost:8080",
|
||||
SourcePath: "/photos/cat.jpg",
|
||||
AllowHTTP: true,
|
||||
}
|
||||
|
||||
got := req.SourceURL()
|
||||
want := "http://localhost:8080/photos/cat.jpg"
|
||||
if got != want {
|
||||
t.Errorf("SourceURL() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageRequest_SourceURL_AllowHTTPFalse(t *testing.T) {
|
||||
req := &ImageRequest{
|
||||
SourceHost: "cdn.example.com",
|
||||
SourcePath: "/img.jpg",
|
||||
AllowHTTP: false,
|
||||
}
|
||||
|
||||
got := req.SourceURL()
|
||||
if got != "https://cdn.example.com/img.jpg" {
|
||||
t.Errorf("SourceURL() = %q, want https scheme", got)
|
||||
}
|
||||
}
|
||||
90
internal/imgcache/stats_test.go
Normal file
90
internal/imgcache/stats_test.go
Normal file
@@ -0,0 +1,90 @@
|
||||
package imgcache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"math"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/database"
|
||||
)
|
||||
|
||||
func setupStatsTestDB(t *testing.T) *sql.DB {
|
||||
t.Helper()
|
||||
db, err := sql.Open("sqlite", ":memory:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := database.ApplyMigrations(db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { db.Close() })
|
||||
return db
|
||||
}
|
||||
|
||||
func TestStats_HitRateIsRatio(t *testing.T) {
|
||||
db := setupStatsTestDB(t)
|
||||
dir := t.TempDir()
|
||||
|
||||
cache, err := NewCache(db, CacheConfig{
|
||||
StateDir: dir,
|
||||
CacheTTL: time.Hour,
|
||||
NegativeTTL: 5 * time.Minute,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Set some hit/miss counts and a transform_count
|
||||
_, err = db.ExecContext(ctx, `
|
||||
UPDATE cache_stats SET hit_count = 75, miss_count = 25, transform_count = 9999 WHERE id = 1
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
stats, err := cache.Stats(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if stats.HitCount != 75 {
|
||||
t.Errorf("HitCount = %d, want 75", stats.HitCount)
|
||||
}
|
||||
if stats.MissCount != 25 {
|
||||
t.Errorf("MissCount = %d, want 25", stats.MissCount)
|
||||
}
|
||||
|
||||
// HitRate should be 0.75, NOT 9999 (transform_count)
|
||||
expectedRate := 0.75
|
||||
if math.Abs(stats.HitRate-expectedRate) > 0.001 {
|
||||
t.Errorf("HitRate = %f, want %f (was it scanning transform_count?)", stats.HitRate, expectedRate)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStats_ZeroCounts(t *testing.T) {
|
||||
db := setupStatsTestDB(t)
|
||||
dir := t.TempDir()
|
||||
|
||||
cache, err := NewCache(db, CacheConfig{
|
||||
StateDir: dir,
|
||||
CacheTTL: time.Hour,
|
||||
NegativeTTL: 5 * time.Minute,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
stats, err := cache.Stats(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// With zero hits and misses, HitRate should be 0, not some garbage value
|
||||
if stats.HitRate != 0.0 {
|
||||
t.Errorf("HitRate = %f, want 0.0 for zero counts", stats.HitRate)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user