Export cache, eviction, upstream and transform metrics (closes #85) #353
No files matched your search
@@ -1167,10 +1167,33 @@ Each item below is one setting, named by its config key:
|
||||
- the Go runtime and process metrics
|
||||
- `pixa_cache_hits_total`: image requests served from the cache
|
||||
- `pixa_cache_misses_total`: image requests not served from the cache
|
||||
- `pixa_source_cache_hits_total`: variants made or checked whose source was
|
||||
in the cache
|
||||
- `pixa_source_cache_misses_total`: variants made or checked whose source
|
||||
was not in the cache
|
||||
- `pixa_cache_usage_bytes`: the bytes of sources and variants in the cache,
|
||||
as eviction last read them; eviction runs after each store and every five
|
||||
minutes
|
||||
- `pixa_cache_max_bytes`: the disk cache limit, `cache_max_bytes`
|
||||
- `pixa_cache_evictions_total`: sources and variants removed by eviction
|
||||
- `pixa_cache_evicted_bytes_total`: the bytes those evictions freed
|
||||
- `pixa_upstream_fetches_total`: fetches of a source image that read any
|
||||
bytes
|
||||
- `pixa_upstream_fetch_bytes_total`: the bytes those fetches read
|
||||
- `pixa_upstream_fetch_duration_seconds`: a histogram of how long those
|
||||
fetches took, up to the last byte of the response
|
||||
- `pixa_upstream_fetch_errors_total`: fetches of a source image that failed,
|
||||
before or while reading the response
|
||||
- `pixa_transforms_total`: images converted
|
||||
- `pixa_transform_duration_seconds`: a histogram of how long those
|
||||
conversions took, including the wait for a processing slot
|
||||
- `pixa_transforms_in_progress`: images being converted or waiting for a
|
||||
processing slot
|
||||
|
||||
pixa measures the `pixa_` metrics whether or not `/metrics` is served. They
|
||||
have no labels, so their number never grows with the hosts and paths pixa
|
||||
serves. Requests that share work (see Limits) count one source cache hit or
|
||||
miss between them.
|
||||
|
||||
A request's `handler` label is the pattern of the route it matches, such as
|
||||
`/v1/image/*`, never its path. Requests that match no route share the label
|
||||
|
||||
@@ -27,12 +27,5 @@ P2: security: per-origin rate limiting
|
||||
- P2: HTTP response handling
|
||||
- Last-Modified headers
|
||||
- P2: operational
|
||||
- Prometheus metrics still missing:
|
||||
- cache hits and misses, split into source images and variants
|
||||
- cache size on disk and its limit
|
||||
- evictions and the bytes they free
|
||||
- upstream fetch duration and errors
|
||||
- transcode duration
|
||||
- images in processing
|
||||
- measure the 1k to 5k req/s target with `script/loadtest` on a machine not
|
||||
shared with other work
|
||||
@@ -158,8 +158,9 @@ access_control_allow_origin: "*"
|
||||
# Sentry DSN for error reporting (default: empty, which turns it off)
|
||||
sentry_dsn: ""
|
||||
|
||||
# Username and password for /metrics, set together (default: unset). Metrics
|
||||
# are measured and /metrics is served only when both are set.
|
||||
# Username and password for /metrics, set together (default: unset). pixa
|
||||
# serves /metrics, and measures HTTP requests, only when both are set. It
|
||||
# measures the pixa_ metrics either way.
|
||||
# metrics:
|
||||
# username: "admin"
|
||||
# password: "secret"
|
||||
|
||||
@@ -37,11 +37,15 @@ type Proxy struct {
|
||||
// same time share one fetch and one transcode.
|
||||
variantsInProgress singleflight.Group
|
||||
|
||||
// The counts of upstream fetches and transforms, made and registered by
|
||||
// New. They are kept in memory, so adding to one waits for nothing.
|
||||
upstreamFetches prometheus.Counter
|
||||
upstreamFetchBytes prometheus.Counter
|
||||
transforms prometheus.Counter
|
||||
// The metrics of upstream fetches and transforms, made and registered by
|
||||
// New. They are kept in memory, so updating one waits for nothing.
|
||||
upstreamFetches prometheus.Counter
|
||||
upstreamFetchBytes prometheus.Counter
|
||||
upstreamFetchDurations prometheus.Histogram
|
||||
upstreamFetchErrors prometheus.Counter
|
||||
transforms prometheus.Counter
|
||||
transformDurations prometheus.Histogram
|
||||
transformsInProgress prometheus.Gauge
|
||||
}
|
||||
|
||||
// Config holds the proxy's configuration.
|
||||
@@ -62,9 +66,9 @@ type Config struct {
|
||||
// errCacheRequired is returned when the proxy is built without a cache.
|
||||
var errCacheRequired = errors.New("cache is required")
|
||||
|
||||
// New creates a new image proxy and registers its counters on
|
||||
// New creates a new image proxy and registers its metrics on
|
||||
// metricsRegistry. pixad passes Prometheus' default registry, the one
|
||||
// /metrics serves. It fails when metricsRegistry already holds counters of
|
||||
// /metrics serves. It fails when metricsRegistry already holds metrics of
|
||||
// the same names.
|
||||
func New(
|
||||
cfg *Config, metricsRegistry prometheus.Registerer,
|
||||
@@ -99,18 +103,39 @@ func New(
|
||||
Name: "pixa_upstream_fetch_bytes_total",
|
||||
Help: "Bytes read from upstream by fetches of source images.",
|
||||
}),
|
||||
upstreamFetchDurations: prometheus.NewHistogram(prometheus.HistogramOpts{
|
||||
Name: "pixa_upstream_fetch_duration_seconds",
|
||||
Help: "Duration of the fetches pixa_upstream_fetches_total counts, " +
|
||||
"until the response's last byte.",
|
||||
}),
|
||||
upstreamFetchErrors: prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Name: "pixa_upstream_fetch_errors_total",
|
||||
Help: "Fetches of a source image from upstream that failed, " +
|
||||
"before or while reading the response.",
|
||||
}),
|
||||
transforms: prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Name: "pixa_transforms_total",
|
||||
Help: "Images the image processor transformed.",
|
||||
}),
|
||||
transformDurations: prometheus.NewHistogram(prometheus.HistogramOpts{
|
||||
Name: "pixa_transform_duration_seconds",
|
||||
Help: "Duration of the transforms pixa_transforms_total counts, " +
|
||||
"waiting for a processing slot included.",
|
||||
}),
|
||||
transformsInProgress: prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Name: "pixa_transforms_in_progress",
|
||||
Help: "Images being transformed or waiting for a processing slot.",
|
||||
}),
|
||||
}
|
||||
|
||||
for _, counter := range []prometheus.Counter{
|
||||
p.upstreamFetches, p.upstreamFetchBytes, p.transforms,
|
||||
for _, metric := range []prometheus.Collector{
|
||||
p.upstreamFetches, p.upstreamFetchBytes, p.upstreamFetchDurations,
|
||||
p.upstreamFetchErrors, p.transforms, p.transformDurations,
|
||||
p.transformsInProgress,
|
||||
} {
|
||||
err = metricsRegistry.Register(counter)
|
||||
err = metricsRegistry.Register(metric)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to register the proxy's counters: %w", err)
|
||||
return nil, fmt.Errorf("failed to register the proxy's metrics: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -341,7 +366,10 @@ func (p *Proxy) loadCachedSource(
|
||||
}
|
||||
|
||||
// processFromSourceOrFetch makes the variant req asks for, from the cached
|
||||
// source when there is one and from a fetched source otherwise.
|
||||
// source when there is one and from a fetched source otherwise. It counts a
|
||||
// source cache miss when it fetches the source because the source has no row
|
||||
// in the cache or its cached file cannot be used, and a source cache hit
|
||||
// otherwise.
|
||||
//
|
||||
// A cached source last checked with upstream CacheTTL or more ago is checked
|
||||
// first: upstream is asked to send it only if it changed since. When it
|
||||
@@ -362,6 +390,8 @@ func (p *Proxy) processFromSourceOrFetch(
|
||||
p.log.Warn("failed to look up cached source, fetching", "error", err)
|
||||
}
|
||||
|
||||
p.cache.IncrementSourceStats(false)
|
||||
|
||||
return p.fetchAndProcess(ctx, req, cacheKey)
|
||||
}
|
||||
|
||||
@@ -375,16 +405,21 @@ func (p *Proxy) processFromSourceOrFetch(
|
||||
case err != nil:
|
||||
p.recordFailedCheck(ctx, req, err)
|
||||
default:
|
||||
p.cache.IncrementSourceStats(true)
|
||||
|
||||
return p.processFetched(ctx, req, cacheKey, fetched)
|
||||
}
|
||||
}
|
||||
|
||||
variant := p.renewVariant(ctx, cacheKey, cached)
|
||||
if variant != nil {
|
||||
p.cache.IncrementSourceStats(true)
|
||||
|
||||
return variant, nil
|
||||
}
|
||||
|
||||
source, sourceSize, sourceFormat := p.loadCachedSource(ctx, req.Source)
|
||||
p.cache.IncrementSourceStats(source != nil)
|
||||
|
||||
// Fetch from upstream if we don't have source data or it's empty
|
||||
if source == nil {
|
||||
@@ -581,36 +616,32 @@ type fetchedSource struct {
|
||||
// fetchSource fetches req's source from upstream and stores it in the cache.
|
||||
// A non-empty ifNoneMatch or ifModifiedSince asks upstream to send the source
|
||||
// only if it changed since; errSourceNotModified says it did not. It counts
|
||||
// the fetch with the bytes read from upstream, including when reading the
|
||||
// response or a later step fails. It fetches nothing while the source URL is
|
||||
// in the negative cache. The caller closes the returned response.
|
||||
// the fetch with the bytes read from upstream and its duration, including
|
||||
// when reading the response or a later step fails, and counts a fetch that
|
||||
// fails or whose response cannot be read as an upstream fetch error. It
|
||||
// fetches nothing while the source URL is in the negative cache. The caller
|
||||
// closes the returned response.
|
||||
func (p *Proxy) fetchSource(
|
||||
ctx context.Context,
|
||||
req *imagerequest.Request,
|
||||
ifNoneMatch, ifModifiedSince string,
|
||||
) (*fetchedSource, error) {
|
||||
// Check negative cache - skip fetching for recently-failed URLs
|
||||
negHit, err := p.cache.Failure(ctx, req.Source)
|
||||
err := p.refuseNegativeCached(ctx, req)
|
||||
if err != nil {
|
||||
p.log.Warn("negative cache check failed", "error", err)
|
||||
}
|
||||
|
||||
if negHit {
|
||||
p.log.Debug("negative cache hit",
|
||||
"host", req.Source.Host,
|
||||
"path", req.Source.Path,
|
||||
)
|
||||
|
||||
return nil, fmt.Errorf("%w: %w", httpfetcher.ErrUpstreamError, ErrNegativeCached)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Fetch from upstream
|
||||
p.log.Debug("fetching from upstream", "host", req.Source.Host,
|
||||
"path", req.Source.Path, "query", req.Source.Query)
|
||||
|
||||
fetchStart := time.Now()
|
||||
|
||||
fetchResult, err := p.fetcher.Fetch(ctx, req.Source,
|
||||
ifNoneMatch, ifModifiedSince)
|
||||
if err != nil {
|
||||
p.upstreamFetchErrors.Inc()
|
||||
|
||||
// Store negative cache for certain errors
|
||||
if isNegativeCacheable(err) {
|
||||
p.storeNegative(ctx, req)
|
||||
@@ -634,9 +665,11 @@ func (p *Proxy) fetchSource(
|
||||
fetchBytes := int64(len(sourceData))
|
||||
|
||||
// Counted also when the request context has ended meanwhile
|
||||
p.incrementUpstreamFetch(fetchBytes)
|
||||
p.incrementUpstreamFetch(fetchBytes, time.Since(fetchStart))
|
||||
|
||||
if err != nil {
|
||||
p.upstreamFetchErrors.Inc()
|
||||
|
||||
_ = fetchResult.Content.Close()
|
||||
|
||||
return nil, fmt.Errorf("failed to read upstream response: %w", err)
|
||||
@@ -677,6 +710,29 @@ func (p *Proxy) fetchSource(
|
||||
}, nil
|
||||
}
|
||||
|
||||
// refuseNegativeCached returns an error wrapping ErrNegativeCached while
|
||||
// req's source URL is in the negative cache, and nil otherwise. A failure to
|
||||
// read the negative cache is only logged, and returns nil.
|
||||
func (p *Proxy) refuseNegativeCached(
|
||||
ctx context.Context, req *imagerequest.Request,
|
||||
) error {
|
||||
negHit, err := p.cache.Failure(ctx, req.Source)
|
||||
if err != nil {
|
||||
p.log.Warn("negative cache check failed", "error", err)
|
||||
}
|
||||
|
||||
if !negHit {
|
||||
return nil
|
||||
}
|
||||
|
||||
p.log.Debug("negative cache hit",
|
||||
"host", req.Source.Host,
|
||||
"path", req.Source.Path,
|
||||
)
|
||||
|
||||
return fmt.Errorf("%w: %w", httpfetcher.ErrUpstreamError, ErrNegativeCached)
|
||||
}
|
||||
|
||||
// fetchedSourceInfo returns what the cache keeps about a source upstream
|
||||
// just sent: fetched and checked now, with the ETag and Last-Modified
|
||||
// headers of fetchResult.
|
||||
@@ -722,15 +778,16 @@ func (p *Proxy) logUpstreamFetch(
|
||||
)
|
||||
}
|
||||
|
||||
// incrementUpstreamFetch counts one upstream fetch that read fetchBytes bytes.
|
||||
// A fetch that read no bytes is not counted.
|
||||
func (p *Proxy) incrementUpstreamFetch(fetchBytes int64) {
|
||||
// incrementUpstreamFetch counts one upstream fetch that read fetchBytes bytes
|
||||
// and took duration. A fetch that read no bytes is not counted.
|
||||
func (p *Proxy) incrementUpstreamFetch(fetchBytes int64, duration time.Duration) {
|
||||
if fetchBytes <= 0 {
|
||||
return
|
||||
}
|
||||
|
||||
p.upstreamFetches.Inc()
|
||||
p.upstreamFetchBytes.Add(float64(fetchBytes))
|
||||
p.upstreamFetchDurations.Observe(duration.Seconds())
|
||||
}
|
||||
|
||||
// storeNegative stores a negative cache entry for req's source, whose fetch
|
||||
@@ -772,8 +829,7 @@ func (p *Proxy) processAndStore(
|
||||
|
||||
// ctx has the deadline of the request doing the processing, which can
|
||||
// end while a request waiting for its image has time left
|
||||
processResult, err := p.processor.Process(ctx, source, sourceFormat,
|
||||
&req.Transform)
|
||||
processResult, err := p.transform(ctx, source, sourceFormat, &req.Transform)
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
return nil, fmt.Errorf("%w: %w", ErrProcessingTimedOut, err)
|
||||
}
|
||||
@@ -786,6 +842,7 @@ func (p *Proxy) processAndStore(
|
||||
|
||||
// Counted also when the request context has ended meanwhile
|
||||
p.transforms.Inc()
|
||||
p.transformDurations.Observe(processDuration.Seconds())
|
||||
|
||||
// Read processed content
|
||||
processedData, err := io.ReadAll(processResult.Content)
|
||||
@@ -838,6 +895,21 @@ func (p *Proxy) processAndStore(
|
||||
}, nil
|
||||
}
|
||||
|
||||
// transform has the image processor transform the image read from source
|
||||
// as requested says, and counts it in pixa_transforms_in_progress until the
|
||||
// processor returns or panics.
|
||||
func (p *Proxy) transform(
|
||||
ctx context.Context,
|
||||
source io.Reader,
|
||||
sourceFormat imagerequest.Format,
|
||||
requested *imagerequest.Transform,
|
||||
) (*imageprocessor.Result, error) {
|
||||
p.transformsInProgress.Inc()
|
||||
defer p.transformsInProgress.Dec()
|
||||
|
||||
return p.processor.Process(ctx, source, sourceFormat, requested)
|
||||
}
|
||||
|
||||
// isNegativeCacheable returns true if the error should be cached.
|
||||
func isNegativeCacheable(err error) bool {
|
||||
return errors.Is(err, httpfetcher.ErrUpstreamError)
|
||||
|
||||
+12
-17
@@ -111,10 +111,16 @@ type Cache struct {
|
||||
pendingAccessTimesDone chan struct{}
|
||||
pendingAccessTimesCancel context.CancelFunc
|
||||
|
||||
// The counts of cache hits and misses, made and registered by
|
||||
// registerCounters (see counters.go)
|
||||
cacheHits prometheus.Counter
|
||||
cacheMisses prometheus.Counter
|
||||
// The cache's metrics, made and registered by registerMetrics (see
|
||||
// metrics.go)
|
||||
cacheHits prometheus.Counter
|
||||
cacheMisses prometheus.Counter
|
||||
sourceCacheHits prometheus.Counter
|
||||
sourceCacheMisses prometheus.Counter
|
||||
usageBytes prometheus.Gauge
|
||||
maxBytes prometheus.Gauge
|
||||
evictions prometheus.Counter
|
||||
evictedBytes prometheus.Counter
|
||||
|
||||
// reconciliationPageSize is the most rows one read of a content table
|
||||
// returns in the reconciliation pass. NewCache sets it to
|
||||
@@ -128,7 +134,7 @@ type Cache struct {
|
||||
reconciliationReadTestHook func(rows int)
|
||||
}
|
||||
|
||||
// NewCache creates a new cache instance and registers its counters on
|
||||
// NewCache creates a new cache instance and registers its metrics on
|
||||
// metricsRegistry. pixad passes Prometheus' default registry, the one
|
||||
// /metrics serves. It makes no database call.
|
||||
func NewCache(
|
||||
@@ -157,7 +163,7 @@ func NewCache(
|
||||
reconciliationPageSize: defaultReconciliationPageSize,
|
||||
}
|
||||
|
||||
err := c.registerCounters(metricsRegistry)
|
||||
err := c.registerMetrics(metricsRegistry)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -656,17 +662,6 @@ func (c *Cache) CleanExpired(ctx context.Context) error {
|
||||
}
|
||||
}
|
||||
|
||||
// IncrementStats counts a cache hit or miss. It only adds to the cache's
|
||||
// Prometheus counters in memory (see counters.go), so it never waits for the
|
||||
// database or fails.
|
||||
func (c *Cache) IncrementStats(hit bool) {
|
||||
if hit {
|
||||
c.cacheHits.Inc()
|
||||
} else {
|
||||
c.cacheMisses.Inc()
|
||||
}
|
||||
}
|
||||
|
||||
// touchVariant hands the access time of a variant served from the cache to
|
||||
// the goroutine StartPendingAccessTimeWrites starts, which writes it in its
|
||||
// next batch. It never waits: while maxPendingAccessTimes access times of
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
package imgcache
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
// registerCounters makes the cache's counters and registers them on
|
||||
// metricsRegistry. The counters are kept in memory, so adding to one waits
|
||||
// for nothing and makes no database call. It fails when metricsRegistry
|
||||
// already holds counters of the same names.
|
||||
func (c *Cache) registerCounters(metricsRegistry prometheus.Registerer) error {
|
||||
c.cacheHits = prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Name: "pixa_cache_hits_total",
|
||||
Help: "Image requests served from the cache.",
|
||||
})
|
||||
c.cacheMisses = prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Name: "pixa_cache_misses_total",
|
||||
Help: "Image requests not served from the cache.",
|
||||
})
|
||||
|
||||
for _, counter := range []prometheus.Counter{c.cacheHits, c.cacheMisses} {
|
||||
err := metricsRegistry.Register(counter)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to register the cache's counters: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -92,8 +92,9 @@ type evictionCandidate struct {
|
||||
}
|
||||
|
||||
// EvictToLimit evicts least-recently-used cache entries until total
|
||||
// tracked usage is at or below the configured MaxBytes limit. It is a
|
||||
// no-op when the cache is disabled.
|
||||
// tracked usage is at or below the configured MaxBytes limit. It sets the
|
||||
// pixa_cache_usage_bytes gauge to each usage it reads. It is a no-op when
|
||||
// the cache is disabled.
|
||||
func (c *Cache) EvictToLimit(ctx context.Context) error {
|
||||
if c.config.MaxBytes == 0 {
|
||||
return nil
|
||||
@@ -105,6 +106,8 @@ func (c *Cache) EvictToLimit(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
|
||||
c.usageBytes.Set(float64(usage))
|
||||
|
||||
if usage <= c.config.MaxBytes {
|
||||
return nil
|
||||
}
|
||||
@@ -345,37 +348,95 @@ func (c *Cache) sourceCandidates(ctx context.Context) ([]evictionCandidate, erro
|
||||
return candidates, nil
|
||||
}
|
||||
|
||||
// evictVariant removes one variant: accounting row first, then the
|
||||
// file, so the database never references a deleted file.
|
||||
// evictVariant removes one variant (see removeVariant) and counts the
|
||||
// eviction with the size its row recorded.
|
||||
func (c *Cache) evictVariant(
|
||||
ctx context.Context, cacheKey imagerequest.VariantKey,
|
||||
) error {
|
||||
_, err := c.db.ExecContext(ctx,
|
||||
`DELETE FROM variant_content WHERE cache_key = ?`, string(cacheKey))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete variant accounting row: %w", err)
|
||||
}
|
||||
|
||||
err = c.variants.Delete(string(cacheKey))
|
||||
sizeBytes, err := c.removeVariant(ctx, cacheKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
c.countEviction(sizeBytes)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// evictSourceBlob removes one source: its row first, then its file, as
|
||||
// evictVariant does for a variant. It holds no lock: a store of the same
|
||||
// source in between can leave the source's row with no file, which
|
||||
// Source treats as a miss.
|
||||
func (c *Cache) evictSourceBlob(ctx context.Context, key imagerequest.SourceKey) error {
|
||||
_, err := c.db.ExecContext(ctx,
|
||||
`DELETE FROM source_content WHERE source_key = ?`, string(key))
|
||||
// removeVariant removes one variant: accounting row first, then the
|
||||
// file, so the database never references a deleted file. It returns the
|
||||
// size the row recorded, or 0 when the variant had no row.
|
||||
func (c *Cache) removeVariant(
|
||||
ctx context.Context, cacheKey imagerequest.VariantKey,
|
||||
) (int64, error) {
|
||||
sizeBytes, err := c.deleteRowReturningSize(ctx,
|
||||
`DELETE FROM variant_content WHERE cache_key = ? RETURNING size_bytes`,
|
||||
string(cacheKey))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete source content row: %w", err)
|
||||
return 0, fmt.Errorf("failed to delete variant accounting row: %w", err)
|
||||
}
|
||||
|
||||
return c.srcContent.Delete(string(key))
|
||||
err = c.variants.Delete(string(cacheKey))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return sizeBytes, nil
|
||||
}
|
||||
|
||||
// evictSourceBlob removes one source (see removeSourceBlob) and counts the
|
||||
// eviction with the size its row recorded.
|
||||
func (c *Cache) evictSourceBlob(ctx context.Context, key imagerequest.SourceKey) error {
|
||||
sizeBytes, err := c.removeSourceBlob(ctx, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
c.countEviction(sizeBytes)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// removeSourceBlob removes one source: its row first, then its file, as
|
||||
// removeVariant does for a variant, and returns the size the row recorded,
|
||||
// or 0 when the source had no row. It holds no lock: a store of the same
|
||||
// source in between can leave the source's row with no file, which Source
|
||||
// treats as a miss.
|
||||
func (c *Cache) removeSourceBlob(
|
||||
ctx context.Context, key imagerequest.SourceKey,
|
||||
) (int64, error) {
|
||||
sizeBytes, err := c.deleteRowReturningSize(ctx,
|
||||
`DELETE FROM source_content WHERE source_key = ? RETURNING size_bytes`,
|
||||
string(key))
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to delete source content row: %w", err)
|
||||
}
|
||||
|
||||
err = c.srcContent.Delete(string(key))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return sizeBytes, nil
|
||||
}
|
||||
|
||||
// deleteRowReturningSize runs deleteQuery, which deletes the row of key and
|
||||
// returns its size_bytes, and returns that size, or 0 when there was no row.
|
||||
func (c *Cache) deleteRowReturningSize(
|
||||
ctx context.Context, deleteQuery string, key string,
|
||||
) (int64, error) {
|
||||
var sizeBytes int64
|
||||
|
||||
err := c.db.QueryRowContext(ctx, deleteQuery, key).Scan(&sizeBytes)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return sizeBytes, nil
|
||||
}
|
||||
|
||||
// notifyWritePressure wakes the background evictor after a store, so
|
||||
@@ -673,7 +734,9 @@ func (c *Cache) reconcileVariantFile(
|
||||
c.log.Warn("removing damaged cached variant file", "path", path,
|
||||
"file_bytes", info.Size(), "recorded_bytes", recordedSize)
|
||||
|
||||
return c.evictVariant(ctx, cacheKey)
|
||||
_, err = c.removeVariant(ctx, cacheKey)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// adoptVariantFile inserts an accounting row for a variant file that
|
||||
@@ -835,7 +898,9 @@ func (c *Cache) reconcileSourceFile(
|
||||
c.log.Warn("removing damaged cached source file", "path", path,
|
||||
"file_bytes", info.Size(), "recorded_bytes", recordedSize)
|
||||
|
||||
return c.evictSourceBlob(ctx, key)
|
||||
_, err = c.removeSourceBlob(ctx, key)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// removeUntrackedSourceFile deletes a source blob file that has no
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
package imgcache
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
// registerMetrics makes the cache's metrics and registers them on
|
||||
// metricsRegistry. The metrics are kept in memory, so updating one waits
|
||||
// for nothing and makes no database call. It fails when metricsRegistry
|
||||
// already holds metrics of the same names.
|
||||
func (c *Cache) registerMetrics(metricsRegistry prometheus.Registerer) error {
|
||||
c.cacheHits = prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Name: "pixa_cache_hits_total",
|
||||
Help: "Image requests served from the cache.",
|
||||
})
|
||||
c.cacheMisses = prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Name: "pixa_cache_misses_total",
|
||||
Help: "Image requests not served from the cache.",
|
||||
})
|
||||
c.sourceCacheHits = prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Name: "pixa_source_cache_hits_total",
|
||||
Help: "Variants made or checked whose source was in the cache.",
|
||||
})
|
||||
c.sourceCacheMisses = prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Name: "pixa_source_cache_misses_total",
|
||||
Help: "Variants made or checked whose source was not in the cache.",
|
||||
})
|
||||
c.usageBytes = prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Name: "pixa_cache_usage_bytes",
|
||||
Help: "Bytes of sources and variants in the cache, as eviction last read them.",
|
||||
})
|
||||
c.maxBytes = prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Name: "pixa_cache_max_bytes",
|
||||
Help: "The disk cache limit in bytes, cache_max_bytes.",
|
||||
})
|
||||
c.evictions = prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Name: "pixa_cache_evictions_total",
|
||||
Help: "Sources and variants removed by eviction.",
|
||||
})
|
||||
c.evictedBytes = prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Name: "pixa_cache_evicted_bytes_total",
|
||||
Help: "Bytes of the sources and variants removed by eviction.",
|
||||
})
|
||||
|
||||
c.maxBytes.Set(float64(c.config.MaxBytes))
|
||||
|
||||
for _, metric := range []prometheus.Collector{
|
||||
c.cacheHits, c.cacheMisses, c.sourceCacheHits, c.sourceCacheMisses,
|
||||
c.usageBytes, c.maxBytes, c.evictions, c.evictedBytes,
|
||||
} {
|
||||
err := metricsRegistry.Register(metric)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to register the cache's metrics: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// IncrementStats counts a cache hit or miss of an image request. It only
|
||||
// adds to the cache's counters in memory, so it never waits for the
|
||||
// database or fails.
|
||||
func (c *Cache) IncrementStats(hit bool) {
|
||||
if hit {
|
||||
c.cacheHits.Inc()
|
||||
} else {
|
||||
c.cacheMisses.Inc()
|
||||
}
|
||||
}
|
||||
|
||||
// IncrementSourceStats counts whether a variant being made or checked found
|
||||
// its source in the cache. Like IncrementStats, it never waits or fails.
|
||||
func (c *Cache) IncrementSourceStats(hit bool) {
|
||||
if hit {
|
||||
c.sourceCacheHits.Inc()
|
||||
} else {
|
||||
c.sourceCacheMisses.Inc()
|
||||
}
|
||||
}
|
||||
|
||||
// countEviction counts one source or variant of sizeBytes bytes removed by
|
||||
// eviction.
|
||||
func (c *Cache) countEviction(sizeBytes int64) {
|
||||
c.evictions.Inc()
|
||||
c.evictedBytes.Add(float64(sizeBytes))
|
||||
}
|
||||
@@ -35,7 +35,7 @@ import (
|
||||
|
||||
// Constructors returns the constructors of the parts this package builds,
|
||||
// for fx.Provide. The fx graph must also provide the prometheus.Registerer
|
||||
// the cache and the image service register their counters on.
|
||||
// the cache and the image service register their metrics on.
|
||||
func Constructors() []any {
|
||||
return []any{
|
||||
newKeys,
|
||||
|
||||
Reference in new issue
Block a user