sneak/pixa
1
0
Fork 0
Code Issues 7 Pull Requests 3 Actions 9 Packages Projects Releases Wiki Activity

Count hits, misses, fetches and transforms as Prometheus counters (closes #311)
check / check (push) Canceled after 0s
Details

Browse Source 
The five counts the cache_stats table held are now Prometheus counters in
memory, in internal/imgcache/counters.go, on the default registry that
/metrics serves. The Increment methods add to them, so counting makes no
database call.

cache_stats, Cache.Stats, Service.Stats, CacheStats and the pending counts
are gone, with their tests. The background goroutine now writes only access
times, and its names say so: StartPendingAccessTimeWrites,
StopPendingAccessTimeWrites and pendingaccesstimes.go.

The counters count for the whole process, so the tests that check how much
they rose do not call t.Parallel. Two stop tests whose write under way was a
hit count now use an access time.

Model: opus-5-5
This commit is contained in:
sneak
2026-10-26 08:50:41 +00:00
parent 6b19e4c96c
commit c3a6799763
22 changed files with 672 additions and 1194 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/imgcache/stats_internal_test.go
+39 -268
View File
@@ -3,248 +3,22 @@ package imgcache
import (
"bytes"
"context"
"database/sql"
"image/color"
"io"
"io/fs"
"log/slog"
"math"
"strings"
"testing"
"time"
"sneak.berlin/go/pixa/internal/database"
"sneak.berlin/go/pixa/internal/httpfetcher"
"sneak.berlin/go/pixa/internal/imagerequest"
)
func setupStatsTestDB(t *testing.T) *sql.DB {
t.Helper()
db, err := sql.Open("sqlite", ":memory:")
if err != nil {
t.Fatal(err)
}
err = database.ApplyMigrations(context.Background(), db, nil)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = db.Close() })
return db
}
func TestStats_HitRateIsRatio(t *testing.T) {
t.Parallel()
db := setupStatsTestDB(t)
dir := t.TempDir()
cache, err := NewCache(db, CacheConfig{
StateDir: dir,
NegativeTTL: 5 * time.Minute,
})
if err != nil {
t.Fatal(err)
}
ctx := context.Background()
// Set some hit/miss counts and a transform_count
_, err = db.ExecContext(ctx, `
UPDATE cache_stats
SET hit_count = 75, miss_count = 25, transform_count = 9999
WHERE id = 1
`)
if err != nil {
t.Fatal(err)
}
stats, err := cache.Stats(ctx)
if err != nil {
t.Fatal(err)
}
if stats.HitCount != 75 {
t.Errorf("HitCount = %d, want 75", stats.HitCount)
}
if stats.MissCount != 25 {
t.Errorf("MissCount = %d, want 25", stats.MissCount)
}
// HitRate should be 0.75, NOT 9999 (transform_count)
expectedRate := 0.75
if math.Abs(stats.HitRate-expectedRate) > 0.001 {
t.Errorf("HitRate = %f, want %f (was it scanning transform_count?)",
stats.HitRate, expectedRate)
}
}
func TestStats_ZeroCounts(t *testing.T) {
t.Parallel()
db := setupStatsTestDB(t)
dir := t.TempDir()
cache, err := NewCache(db, CacheConfig{
StateDir: dir,
NegativeTTL: 5 * time.Minute,
})
if err != nil {
t.Fatal(err)
}
stats, err := cache.Stats(context.Background())
if err != nil {
t.Fatal(err)
}
// With zero hits and misses, HitRate should be 0, not some garbage value
if stats.HitRate != 0.0 {
t.Errorf("HitRate = %f, want 0.0 for zero counts", stats.HitRate)
}
}
// TestStats_LogsFailedCountQueries verifies that a failed item count query
// and a failed size query are each logged at warn and Stats still succeeds.
func TestStats_LogsFailedCountQueries(t *testing.T) {
t.Parallel()
db := setupStatsTestDB(t)
var logBuf bytes.Buffer
cache, err := NewCache(db, CacheConfig{
StateDir: t.TempDir(),
NegativeTTL: 5 * time.Minute,
Logger: slog.New(slog.NewJSONHandler(&logBuf, nil)),
})
if err != nil {
t.Fatal(err)
}
_, err = db.ExecContext(t.Context(),
`DROP TABLE source_content; DROP TABLE variant_content`)
if err != nil {
t.Fatal(err)
}
_, err = cache.Stats(t.Context())
if err != nil {
t.Fatalf("Stats() error = %v, want nil", err)
}
for _, msg := range []string{
"failed to count cache items for stats",
"failed to sum cache size for stats",
} {
want := `"level":"WARN","msg":"` + msg + `"`
if !strings.Contains(logBuf.String(), want) {
t.Errorf("log missing %s; got %q", want, logBuf.String())
}
}
}
// TestStats_TotalsCountSourcesAndVariants verifies that TotalItems and
// TotalSizeBytes cover the stored source images and processed variants.
func TestStats_TotalsCountSourcesAndVariants(t *testing.T) {
t.Parallel()
cache, _ := newEvictionTestCache(t, 1<<30)
storeEvictionTestSource(t, cache, testHostCDN, testPathCat,
bytes.Repeat([]byte{0xAA}, 1000))
storeEvictionTestVariant(t, cache, testVariantKeyOne,
bytes.Repeat([]byte{0xAB}, 500))
storeEvictionTestVariant(t, cache, testVariantKeyTwo,
bytes.Repeat([]byte{0xAC}, 250))
stats, err := cache.Stats(t.Context())
if err != nil {
t.Fatalf("Stats() error = %v", err)
}
if stats.TotalItems != 3 {
t.Errorf("TotalItems = %d, want 3 (1 source, 2 variants)", stats.TotalItems)
}
if stats.TotalSizeBytes != 1750 {
t.Errorf("TotalSizeBytes = %d, want 1750 (1000+500+250)",
stats.TotalSizeBytes)
}
}
// TestStats_DisabledCacheReportsNoItems verifies that a disabled disk cache
// reports no items and no size, even when its database still holds the
// rows of an earlier run with the disk cache enabled.
func TestStats_DisabledCacheReportsNoItems(t *testing.T) {
t.Parallel()
enabled, _ := newEvictionTestCache(t, 1<<30)
storeEvictionTestSource(t, enabled, testHostCDN, testPathCat,
bytes.Repeat([]byte{0xAA}, 1000))
storeEvictionTestVariant(t, enabled, testVariantKeyOne,
bytes.Repeat([]byte{0xAB}, 500))
disabled, err := NewCache(enabled.db, CacheConfig{
StateDir: t.TempDir(),
NegativeTTL: 5 * time.Minute,
DisableDiskCache: true,
})
if err != nil {
t.Fatal(err)
}
stats, err := disabled.Stats(t.Context())
if err != nil {
t.Fatalf("Stats() error = %v", err)
}
if stats.TotalItems != 0 || stats.TotalSizeBytes != 0 {
t.Errorf("TotalItems = %d, TotalSizeBytes = %d, want 0 and 0",
stats.TotalItems, stats.TotalSizeBytes)
}
}
// cacheStatsCounters holds the counters of the cache_stats row, in column
// order.
type cacheStatsCounters struct {
hitCount int64
missCount int64
upstreamFetchCount int64
upstreamFetchBytes int64
transformCount int64
}
// readCacheStatsCounters reads the counters of the cache_stats row.
func readCacheStatsCounters(t *testing.T, cache *Cache) cacheStatsCounters {
t.Helper()
var got cacheStatsCounters
err := cache.db.QueryRowContext(t.Context(), `
SELECT hit_count, miss_count, upstream_fetch_count,
upstream_fetch_bytes, transform_count
FROM cache_stats WHERE id = 1
`).Scan(&got.hitCount, &got.missCount, &got.upstreamFetchCount,
&got.upstreamFetchBytes, &got.transformCount)
if err != nil {
t.Fatalf("failed to read cache_stats: %v", err)
}
return got
}
// TestService_Get_CountsStats walks Get through a miss that fetches the
// source, a hit, a miss that reuses the cached source, and two misses whose
// source cannot be used, checking every cache_stats counter after each.
// source cannot be used, checking how much each request added to every
// counter in counters.go.
//
//nolint:paralleltest // it reads counters that other tests add to
func TestService_Get_CountsStats(t *testing.T) {
t.Parallel()
svc, fixtures := SetupTestService(t)
// NewTestFS builds the same files the test service's fetcher serves.
@@ -262,27 +36,30 @@ func TestService_Get_CountsStats(t *testing.T) {
photoBytes, fakeBytes := int64(len(photo)), int64(len(fake))
// want is hits, misses, upstream fetches, upstream bytes, transforms.
// want is what the request adds to hits, misses, upstream fetches,
// upstream bytes and transforms.
steps := []struct {
name string
path string
size int
wantErr bool
want cacheStatsCounters
want counterValues
}{
{"miss that fetches the source", testPathPhoto, 50, false,
cacheStatsCounters{0, 1, 1, photoBytes, 1}},
counterValues{0, 1, 1, photoBytes, 1}},
{"hit", testPathPhoto, 50, false,
cacheStatsCounters{1, 1, 1, photoBytes, 1}},
counterValues{1, 0, 0, 0, 0}},
{"miss that reuses the cached source", testPathPhoto, 25, false,
cacheStatsCounters{1, 2, 1, photoBytes, 2}},
counterValues{0, 1, 0, 0, 1}},
{"miss whose source fails the magic byte check", "/images/fake.jpg", 50, true,
cacheStatsCounters{1, 3, 2, photoBytes + fakeBytes, 2}},
counterValues{0, 1, 1, fakeBytes, 0}},
{"miss whose source is not found", "/images/nonexistent.jpg", 50, true,
cacheStatsCounters{1, 4, 2, photoBytes + fakeBytes, 2}},
counterValues{0, 1, 0, 0, 0}},
}
for _, step := range steps {
before := readCounters()
resp, err := svc.Get(t.Context(), &imagerequest.Request{
Source: imagerequest.Source{
Host: fixtures.GoodHost,
@@ -303,11 +80,10 @@ func TestService_Get_CountsStats(t *testing.T) {
_ = resp.Content.Close()
}
waitForPendingCountsWritten(t, svc.cache)
got := readCacheStatsCounters(t, svc.cache)
got := readCounters().minus(before)
if got != step.want {
t.Fatalf("after the %s: counters = %+v, want %+v", step.name, got, step.want)
t.Fatalf("the %s added %+v to the counters, want %+v",
step.name, got, step.want)
}
}
}
@@ -343,37 +119,36 @@ func (u *fakeUpstream) Read(p []byte) (int, error) {
return 0, u.err
}
// TestService_Get_CountsInterruptedMisses checks every cache_stats counter
// after a miss whose request context ends during or after the upstream
// fetch, and after a miss whose upstream body is over the size limit.
// TestService_Get_CountsInterruptedMisses checks how much a miss whose
// request context ends during or after the upstream fetch, and a miss whose
// upstream body is over the size limit, add to every counter in counters.go.
//
//nolint:paralleltest // it reads counters that other tests add to
func TestService_Get_CountsInterruptedMisses(t *testing.T) {
t.Parallel()
photo := generateTestJPEG(t, 100, 100, color.RGBA{255, 0, 0, 255})
half := len(photo) / 2
// want is hits, misses, upstream fetches, upstream bytes, transforms.
// want is what the request adds to hits, misses, upstream fetches,
// upstream bytes and transforms.
tests := []struct {
name string
served int // bytes of the photo the upstream body serves
cancel bool // whether the body then ends the request context
readErr error // what the body then returns
wantErr bool
want cacheStatsCounters
want counterValues
}{
{"request context ends during the fetch", half, true, context.Canceled, true,
cacheStatsCounters{0, 1, 1, int64(half), 0}},
counterValues{0, 1, 1, int64(half), 0}},
{"request context ends after the fetch", len(photo), true, io.EOF, false,
cacheStatsCounters{0, 1, 1, int64(len(photo)), 1}},
counterValues{0, 1, 1, int64(len(photo)), 1}},
{"upstream body over the size limit", half, false,
httpfetcher.ErrResponseTooLarge, true,
cacheStatsCounters{0, 1, 1, int64(half), 0}},
counterValues{0, 1, 1, int64(half), 0}},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
svc, fixtures := SetupTestService(t)
ctx, cancel := context.WithCancel(t.Context())
@@ -389,6 +164,8 @@ func TestService_Get_CountsInterruptedMisses(t *testing.T) {
svc.fetcher = upstream
before := readCounters()
resp, err := svc.Get(ctx, &imagerequest.Request{
Source: imagerequest.Source{
Host: fixtures.GoodHost,
@@ -411,22 +188,20 @@ func TestService_Get_CountsInterruptedMisses(t *testing.T) {
_ = resp.Content.Close()
}
waitForPendingCountsWritten(t, svc.cache)
got := readCacheStatsCounters(t, svc.cache)
got := readCounters().minus(before)
if got != tc.want {
t.Errorf("counters = %+v, want %+v", got, tc.want)
t.Errorf("the miss added %+v to the counters, want %+v", got, tc.want)
}
})
}
}
// TestService_Get_CountsHitAfterRequestEnds checks every cache_stats counter
// after a hit served with a request context that has already ended: only
// the hit count moves.
// TestService_Get_CountsHitAfterRequestEnds checks every counter in
// counters.go after a hit served with a request context that has already
// ended: only the hit count moves.
//
//nolint:paralleltest // it reads counters that other tests add to
func TestService_Get_CountsHitAfterRequestEnds(t *testing.T) {
t.Parallel()
svc, fixtures := SetupTestService(t)
req := &imagerequest.Request{
@@ -450,9 +225,7 @@ func TestService_Get_CountsHitAfterRequestEnds(t *testing.T) {
_ = resp.Content.Close()
waitForPendingCountsWritten(t, svc.cache)
want := readCacheStatsCounters(t, svc.cache)
want := readCounters()
want.hitCount++
ctx, cancel := context.WithCancel(t.Context())
@@ -469,9 +242,7 @@ func TestService_Get_CountsHitAfterRequestEnds(t *testing.T) {
t.Fatalf("CacheStatus = %v, want %v", resp.CacheStatus, CacheHit)
}
waitForPendingCountsWritten(t, svc.cache)
got := readCacheStatsCounters(t, svc.cache)
got := readCounters()
if got != want {
t.Errorf("counters = %+v, want %+v", got, want)
}