Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8feed69143 |
No files matched your search
@@ -2,8 +2,9 @@
|
||||
|
||||
pixa is a GPL-3.0-licensed Go web server by [@sneak](https://sneak.berlin) that
|
||||
proxies images from upstream sources, optionally resizing or transforming them,
|
||||
and serves the results. Both source and transformed images are cached on disk.
|
||||
pixa checks a cached image with upstream again after 24 hours (see Cache TTL).
|
||||
and serves the results. Both source and transformed images are cached to disk so
|
||||
that subsequent requests are served without origin fetches or additional
|
||||
processing.
|
||||
|
||||
## Getting Started
|
||||
|
||||
@@ -206,7 +207,7 @@ These terms are the names the code uses:
|
||||
- the upstream answered with an error status
|
||||
- the body's first bytes were not those of the format its `Content-Type`
|
||||
names
|
||||
- a check of a cached image failed in any way (see Cache TTL)
|
||||
- a check of a cached source failed in any way (see Cache TTL)
|
||||
- **Image cache**: the part of pixa that keeps the sources, the variants and the
|
||||
negative cache (see Storage).
|
||||
- **Cache TTL**: how long pixa serves a cached source and its variants before it
|
||||
@@ -234,9 +235,8 @@ The image cache keeps these tables in the database:
|
||||
unchanged
|
||||
- the `ETag` and `Last-Modified` headers upstream sent with it
|
||||
- `variant_content`: a row for each variant, by variant key (column
|
||||
`cache_key`), with:
|
||||
- its size and when it was last used
|
||||
- when its source was last checked with upstream
|
||||
`cache_key`), with its size, when it was last used, and when its source was
|
||||
last checked with upstream
|
||||
- `cache_usage`: the total size of the cached sources and variants, which
|
||||
eviction reads
|
||||
- `negative_cache`: a row for each source URL in the negative cache, with when
|
||||
@@ -307,12 +307,6 @@ source or confirmed it unchanged. So a source whose checks keep failing is
|
||||
served for at most 30 days. Eviction to stay under the disk cache limit can
|
||||
remove images sooner.
|
||||
|
||||
That eviction can remove a source and keep its variants. A request for such a
|
||||
variant after 24 hours fetches the whole image again, since pixa no longer has
|
||||
the source's `ETag` or `Last-Modified`. pixa then converts the variant again.
|
||||
When that fetch fails, pixa serves the cached variant and puts the source URL in
|
||||
the negative cache, as for any failed check.
|
||||
|
||||
A variant file's modification time is when its source was last checked with
|
||||
upstream. So a cache hit still reads no database row to know whether the variant
|
||||
is older than 24 hours. A request that checks its source with upstream counts as
|
||||
|
||||
@@ -31,8 +31,8 @@ P2: security: per-origin rate limiting
|
||||
# Completed Steps
|
||||
|
||||
- 2026-11-11 a cached source is checked with upstream once it is 24 hours old
|
||||
(closes #69). Before, it and its variants were served until eviction, so a
|
||||
changed image at the same URL was never picked up.
|
||||
(closes #69, #83). Before, it and its variants were served until eviction, so
|
||||
a changed image at the same URL was never picked up.
|
||||
- `source_content` keeps the source's `fetched_at`, `checked_at`, and the
|
||||
`ETag` and `Last-Modified` upstream sent.
|
||||
- The check sends them as `If-None-Match` and `If-Modified-Since`. A 304
|
||||
@@ -41,11 +41,9 @@ P2: security: per-origin rate limiting
|
||||
still makes no database call. A variant made before the source's
|
||||
`fetched_at` is converted again.
|
||||
- A failed check serves the cached image and puts the source URL in the
|
||||
negative cache, so upstream is asked at most once per 5 minutes. A variant
|
||||
whose source was evicted is checked with a full fetch. When that fetch
|
||||
fails, the cached variant is served the same way.
|
||||
negative cache, so upstream is asked at most once per 5 minutes.
|
||||
- The eviction goroutine evicts sources and variants whose `checked_at` is
|
||||
30 days old (`EvictionAfterLastCheck`).
|
||||
30 days old (`CacheMaxAge`).
|
||||
- 2026-10-27 the reconciliation pass skips a directory it cannot list and a file
|
||||
whose info it cannot read, and goes on (closes #343). It stopped before its
|
||||
later checks at a directory it could not list, or at a source or variant file
|
||||
|
||||
@@ -26,22 +26,6 @@ const (
|
||||
// errConnectionRefused stands for an upstream that cannot be reached.
|
||||
var errConnectionRefused = errors.New("connection refused")
|
||||
|
||||
// upstreamFailure is one way a fetch from upstream fails.
|
||||
type upstreamFailure struct {
|
||||
name string
|
||||
err error
|
||||
}
|
||||
|
||||
// upstreamFailures returns the ways a check of a cached image with upstream
|
||||
// can fail: an error status, a timeout and no connection.
|
||||
func upstreamFailures() []upstreamFailure {
|
||||
return []upstreamFailure{
|
||||
{"error status", fmt.Errorf("%w: status 503", httpfetcher.ErrUpstreamError)},
|
||||
{"timeout", httpfetcher.ErrUpstreamTimeout},
|
||||
{"no connection", errConnectionRefused},
|
||||
}
|
||||
}
|
||||
|
||||
// conditionalUpstream serves one JPEG image with an ETag or a Last-Modified
|
||||
// header, or both. It answers 304 Not Modified to a fetch whose If-None-Match
|
||||
// is its ETag, or, when it has no ETag, whose If-Modified-Since is its
|
||||
@@ -308,8 +292,17 @@ func TestProxy_Get_ChangedSourceAfterTTLReplacesSourceAndVariants(t *testing.T)
|
||||
func TestProxy_Get_UpstreamFailureAfterTTLServesCachedImage(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, failure := range upstreamFailures() {
|
||||
t.Run(failure.name, func(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
err error
|
||||
}{
|
||||
{"error status", fmt.Errorf("%w: status 503", httpfetcher.ErrUpstreamError)},
|
||||
{"timeout", httpfetcher.ErrUpstreamTimeout},
|
||||
{"no connection", errConnectionRefused},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
svc, fixtures, upstream := setupConditionalUpstreamProxy(t)
|
||||
@@ -317,7 +310,7 @@ func TestProxy_Get_UpstreamFailureAfterTTLServesCachedImage(t *testing.T) {
|
||||
|
||||
first, _ := getImage(t, svc, variant)
|
||||
setBackSourceAndVariant(t, svc, variant, time.Now().AddDate(0, 0, -2))
|
||||
upstream.fail(failure.err)
|
||||
upstream.fail(tt.err)
|
||||
|
||||
second, _ := getImage(t, svc, variant)
|
||||
t.Logf("upstream fetches = %d", upstream.fetches())
|
||||
@@ -363,86 +356,6 @@ func TestProxy_Get_FailingUpstreamIsAskedOnceWhileImageIsServedOften(t *testing.
|
||||
}
|
||||
}
|
||||
|
||||
// TestProxy_Get_UpstreamFailureAfterTTLServesVariantOfEvictedSource caches a
|
||||
// variant, sets its file time two days back, removes its source's row and
|
||||
// file, as eviction does, and makes upstream fail. The cached variant must
|
||||
// still be served.
|
||||
func TestProxy_Get_UpstreamFailureAfterTTLServesVariantOfEvictedSource(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, failure := range upstreamFailures() {
|
||||
t.Run(failure.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
svc, fixtures, upstream := setupConditionalUpstreamProxy(t)
|
||||
variant := photoVariant(fixtures, 85, imagerequest.FitCover)
|
||||
|
||||
first, _ := getImage(t, svc, variant)
|
||||
setBackSourceAndVariant(t, svc, variant, time.Now().AddDate(0, 0, -2))
|
||||
evictSource(t, svc, variant.Source)
|
||||
upstream.fail(failure.err)
|
||||
|
||||
second, _ := getImage(t, svc, variant)
|
||||
t.Logf("upstream fetches = %d", upstream.fetches())
|
||||
|
||||
if !bytes.Equal(first, second) {
|
||||
t.Error("the variant served while upstream fails differs from the cached one")
|
||||
}
|
||||
|
||||
if n := upstream.fetches(); n != 2 {
|
||||
t.Errorf("upstream fetches = %d, want 2", n)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestProxy_Get_FailingUpstreamIsAskedOnceForVariantOfEvictedSource caches a
|
||||
// variant, sets its file time two days back, removes its source's row and
|
||||
// file, as eviction does, and makes every fetch time out. Serving the variant
|
||||
// ten times must ask upstream once: the failed check puts the source URL in
|
||||
// the negative cache.
|
||||
func TestProxy_Get_FailingUpstreamIsAskedOnceForVariantOfEvictedSource(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
svc, fixtures, upstream := setupConditionalUpstreamProxy(t)
|
||||
variant := photoVariant(fixtures, 85, imagerequest.FitCover)
|
||||
|
||||
first, _ := getImage(t, svc, variant)
|
||||
setBackSourceAndVariant(t, svc, variant, time.Now().AddDate(0, 0, -2))
|
||||
evictSource(t, svc, variant.Source)
|
||||
upstream.fail(httpfetcher.ErrUpstreamTimeout)
|
||||
|
||||
for range 10 {
|
||||
image, _ := getImage(t, svc, variant)
|
||||
if !bytes.Equal(image, first) {
|
||||
t.Fatal("the variant served while upstream fails differs from the cached one")
|
||||
}
|
||||
}
|
||||
|
||||
t.Logf("upstream fetches = %d", upstream.fetches())
|
||||
|
||||
if n := upstream.fetches(); n != 2 {
|
||||
t.Errorf("upstream fetches = %d, want 2: the first fetch and one check", n)
|
||||
}
|
||||
}
|
||||
|
||||
// evictSource removes the row and then the file of src's cached source, as
|
||||
// the cache's eviction does, and leaves its variants cached.
|
||||
func evictSource(t *testing.T, svc *testProxy, src imagerequest.Source) {
|
||||
t.Helper()
|
||||
|
||||
_, err := svc.db.ExecContext(t.Context(),
|
||||
`DELETE FROM source_content WHERE source_key = ?`, string(src.Key()))
|
||||
if err != nil {
|
||||
t.Fatalf("deleting the source's row: %v", err)
|
||||
}
|
||||
|
||||
err = os.Remove(sourceFilePath(svc, src.Key()))
|
||||
if err != nil {
|
||||
t.Fatalf("removing the source's file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// setBackSourceAndVariant sets back to when the times the cache keeps for
|
||||
// req's source and for the variant req asks for: the source's fetch and check
|
||||
// times, both last access times, the variant's check time, and the variant
|
||||
|
||||
@@ -348,9 +348,7 @@ func (p *Proxy) loadCachedSource(
|
||||
// changed, the new bytes replace the cached source and the variant is made
|
||||
// from them. When it did not change, or the check fails, the cached source
|
||||
// is used: the variant's cached file is served again if it was made from the
|
||||
// cached source's bytes, and otherwise the variant is made from them. When
|
||||
// the source is not cached, a failed fetch still serves the variant's cached
|
||||
// file (see fetchAndProcess).
|
||||
// cached source's bytes, and otherwise the variant is made from them.
|
||||
func (p *Proxy) processFromSourceOrFetch(
|
||||
ctx context.Context,
|
||||
req *imagerequest.Request,
|
||||
@@ -373,7 +371,15 @@ func (p *Proxy) processFromSourceOrFetch(
|
||||
cached.CheckedAt = time.Now().UTC()
|
||||
p.renewSource(ctx, req.Source, cached.CheckedAt)
|
||||
case err != nil:
|
||||
p.recordFailedCheck(ctx, req, err)
|
||||
// Any failure, a timeout included, puts the source URL in the
|
||||
// negative cache, so upstream is asked at most once per
|
||||
// NegativeTTL while it fails, however often the image is served
|
||||
if !errors.Is(err, ErrNegativeCached) {
|
||||
p.log.Warn("checking source with upstream failed, using cached source",
|
||||
"request_id", middleware.GetReqID(ctx),
|
||||
"host", req.Source.Host, "path", req.Source.Path, "error", err)
|
||||
p.storeNegative(ctx, req)
|
||||
}
|
||||
default:
|
||||
return p.processFetched(ctx, req, cacheKey, fetched)
|
||||
}
|
||||
@@ -400,24 +406,6 @@ func (p *Proxy) processFromSourceOrFetch(
|
||||
source, sourceFormat, sourceSize)
|
||||
}
|
||||
|
||||
// recordFailedCheck logs a failed check of req's source with upstream, after
|
||||
// which the image is still served from the cache. It puts the source URL in
|
||||
// the negative cache, whatever the failure, a timeout included, so upstream
|
||||
// is asked at most once per NegativeTTL while it fails, however often the
|
||||
// image is served. A failure that was a negative cache hit records nothing.
|
||||
func (p *Proxy) recordFailedCheck(
|
||||
ctx context.Context, req *imagerequest.Request, err error,
|
||||
) {
|
||||
if errors.Is(err, ErrNegativeCached) {
|
||||
return
|
||||
}
|
||||
|
||||
p.log.Warn("checking source with upstream failed, serving from the cache",
|
||||
"request_id", middleware.GetReqID(ctx),
|
||||
"host", req.Source.Host, "path", req.Source.Path, "error", err)
|
||||
p.storeNegative(ctx, req)
|
||||
}
|
||||
|
||||
// renewSource records that upstream answered at checkedAt that src's cached
|
||||
// source had not changed. A failure to record it is only logged: the source
|
||||
// is then checked again on its next use.
|
||||
@@ -449,39 +437,12 @@ func (p *Proxy) renewVariant(
|
||||
return nil
|
||||
}
|
||||
|
||||
return p.readCachedVariant(cacheKey, reader, contentType)
|
||||
}
|
||||
|
||||
// cachedVariantIgnoringTTL returns the cached variant of cacheKey whatever
|
||||
// its age. It returns nil when the variant is not cached or cannot be read.
|
||||
func (p *Proxy) cachedVariantIgnoringTTL(
|
||||
cacheKey imagerequest.VariantKey,
|
||||
) *processedVariant {
|
||||
reader, _, contentType, err := p.cache.VariantIgnoringTTL(cacheKey)
|
||||
if err != nil {
|
||||
if !errors.Is(err, imgcache.ErrNotFound) {
|
||||
p.log.Warn("cached variant not served", "cache_key", cacheKey,
|
||||
"error", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
return p.readCachedVariant(cacheKey, reader, contentType)
|
||||
}
|
||||
|
||||
// readCachedVariant reads the cached variant of cacheKey that reader opened,
|
||||
// whose content type is contentType, and closes reader. It returns nil when
|
||||
// the variant cannot be read.
|
||||
func (p *Proxy) readCachedVariant(
|
||||
cacheKey imagerequest.VariantKey, reader io.ReadCloser, contentType string,
|
||||
) *processedVariant {
|
||||
defer func() { _ = reader.Close() }()
|
||||
|
||||
data, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
p.log.Warn("failed to read cached variant", "cache_key", cacheKey,
|
||||
"error", err)
|
||||
p.log.Warn("failed to read cached variant, making it again",
|
||||
"cache_key", cacheKey, "error", err)
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -490,10 +451,7 @@ func (p *Proxy) readCachedVariant(
|
||||
}
|
||||
|
||||
// fetchAndProcess fetches the source from upstream, stores it, and makes
|
||||
// and stores the variant from it. When the fetch fails while the variant is
|
||||
// still cached, as it can be after eviction removed its source, the cached
|
||||
// variant is served whatever its age, and the failure is recorded as a failed
|
||||
// check (see recordFailedCheck).
|
||||
// and stores the variant from it.
|
||||
func (p *Proxy) fetchAndProcess(
|
||||
ctx context.Context,
|
||||
req *imagerequest.Request,
|
||||
@@ -501,14 +459,7 @@ func (p *Proxy) fetchAndProcess(
|
||||
) (*processedVariant, error) {
|
||||
fetched, err := p.fetchSource(ctx, req, "", "")
|
||||
if err != nil {
|
||||
variant := p.cachedVariantIgnoringTTL(cacheKey)
|
||||
if variant == nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
p.recordFailedCheck(ctx, req, err)
|
||||
|
||||
return variant, nil
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return p.processFetched(ctx, req, cacheKey, fetched)
|
||||
|
||||
+15
-48
@@ -3,12 +3,8 @@
|
||||
// - the variants made from them
|
||||
// - the negative cache: the source URLs whose fetch failed recently
|
||||
//
|
||||
// Each image is a file with a row in the database. A source and its
|
||||
// variants are served until the cache TTL has passed since upstream last sent
|
||||
// the source or confirmed it unchanged. Then the source is due to be checked
|
||||
// with upstream again (see Stale). Eviction removes:
|
||||
// - the least recently used images, to keep the cache under its size limit
|
||||
// - the images not checked with upstream for EvictionAfterLastCheck
|
||||
// Each image is a file with a row in the database. Eviction removes the
|
||||
// least recently used images to keep the cache under its size limit.
|
||||
package imgcache
|
||||
|
||||
import (
|
||||
@@ -49,11 +45,10 @@ const DefaultNegativeTTL = 5 * time.Minute
|
||||
// variants without asking upstream again.
|
||||
const DefaultCacheTTL = 24 * time.Hour
|
||||
|
||||
// EvictionAfterLastCheck is how long after upstream last sent a cached
|
||||
// source, or answered that it had not changed, the source and its variants
|
||||
// are evicted (see EvictNotCheckedRecently). Until then a source whose checks
|
||||
// fail is served.
|
||||
const EvictionAfterLastCheck = 30 * 24 * time.Hour
|
||||
// CacheMaxAge is how long after upstream last sent a cached source, or
|
||||
// answered that it had not changed, the source and its variants are evicted
|
||||
// (see EvictPastMaxAge). Until then a source whose checks fail is served.
|
||||
const CacheMaxAge = 30 * 24 * time.Hour
|
||||
|
||||
// ErrStale is returned by Variant for a variant whose source was last
|
||||
// checked with upstream CacheTTL or more ago.
|
||||
@@ -213,34 +208,25 @@ func (c *Cache) Variant(
|
||||
return nil, 0, "", ErrStale
|
||||
}
|
||||
|
||||
return c.finishOpeningVariant(cacheKey, file, info)
|
||||
}
|
||||
|
||||
// VariantIgnoringTTL opens a cached variant whatever its age, and returns it
|
||||
// as Variant does. The image proxy serves it when a fetch of the variant's
|
||||
// source fails while the source is not cached.
|
||||
func (c *Cache) VariantIgnoringTTL(
|
||||
cacheKey imagerequest.VariantKey,
|
||||
) (io.ReadCloser, int64, string, error) {
|
||||
if c.config.MaxBytes == 0 {
|
||||
return nil, 0, "", ErrNotFound
|
||||
}
|
||||
|
||||
file, info, err := c.openVariant(cacheKey)
|
||||
format, err := readFormat(file)
|
||||
if err != nil {
|
||||
_ = file.Close()
|
||||
|
||||
return nil, 0, "", err
|
||||
}
|
||||
|
||||
return c.finishOpeningVariant(cacheKey, file, info)
|
||||
c.touchVariant(cacheKey)
|
||||
|
||||
return file, info.Size(), format.MIMEType(), nil
|
||||
}
|
||||
|
||||
// RenewVariant opens a cached variant made from the source bytes that source
|
||||
// describes, whatever its age, and returns it as Variant does. It moves the
|
||||
// variant file's modification time, and the checked_at of its row, forward
|
||||
// to source.CheckedAt, so Variant serves the file until CacheTTL after that
|
||||
// check and eviction keeps it until EvictionAfterLastCheck after it. A
|
||||
// variant made before source.FetchedAt was made from older bytes, and is
|
||||
// ErrNotFound, as is a variant not in the cache.
|
||||
// check and eviction keeps it until CacheMaxAge after it. A variant made
|
||||
// before source.FetchedAt was made from older bytes, and is ErrNotFound, as
|
||||
// is a variant not in the cache.
|
||||
func (c *Cache) RenewVariant(
|
||||
ctx context.Context, cacheKey imagerequest.VariantKey, source SourceInfo,
|
||||
) (io.ReadCloser, int64, string, error) {
|
||||
@@ -660,25 +646,6 @@ func (c *Cache) openVariant(
|
||||
return file, info, nil
|
||||
}
|
||||
|
||||
// finishOpeningVariant reads the content type of a variant file openVariant
|
||||
// opened from its first bytes, hands the variant's access time to the
|
||||
// background writer (see touchVariant), and returns the file as Variant does.
|
||||
// It closes the file when its first bytes are not those of an output format.
|
||||
func (c *Cache) finishOpeningVariant(
|
||||
cacheKey imagerequest.VariantKey, file *os.File, info fs.FileInfo,
|
||||
) (io.ReadCloser, int64, string, error) {
|
||||
format, err := readFormat(file)
|
||||
if err != nil {
|
||||
_ = file.Close()
|
||||
|
||||
return nil, 0, "", err
|
||||
}
|
||||
|
||||
c.touchVariant(cacheKey)
|
||||
|
||||
return file, info.Size(), format.MIMEType(), nil
|
||||
}
|
||||
|
||||
// readFormat reads the first bytes of a source or variant file, without
|
||||
// moving its offset, and returns the format they show. Sources and variants
|
||||
// are in the same six formats.
|
||||
|
||||
@@ -131,20 +131,20 @@ func (c *Cache) EvictToLimit(ctx context.Context) error {
|
||||
}
|
||||
}
|
||||
|
||||
// EvictNotCheckedRecently evicts each source and variant whose checked_at is
|
||||
// EvictionAfterLastCheck or more ago, evictionBatchSize of each at a time,
|
||||
// until none is left. It is a no-op when the cache is disabled.
|
||||
func (c *Cache) EvictNotCheckedRecently(ctx context.Context) error {
|
||||
// EvictPastMaxAge evicts each source and variant whose checked_at is
|
||||
// CacheMaxAge or more ago, evictionBatchSize of each at a time, until none
|
||||
// is left. It is a no-op when the cache is disabled.
|
||||
func (c *Cache) EvictPastMaxAge(ctx context.Context) error {
|
||||
if c.config.MaxBytes == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
cutoff := time.Now().UTC().Add(-EvictionAfterLastCheck)
|
||||
cutoff := time.Now().UTC().Add(-CacheMaxAge)
|
||||
|
||||
for {
|
||||
sources, err := queryStringColumn[imagerequest.SourceKey](ctx, c.db, `
|
||||
SELECT source_key FROM source_content WHERE checked_at < ? LIMIT ?
|
||||
`, "sources not checked recently", "source key", cutoff, evictionBatchSize)
|
||||
`, "sources past the maximum age", "source key", cutoff, evictionBatchSize)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -158,7 +158,7 @@ func (c *Cache) EvictNotCheckedRecently(ctx context.Context) error {
|
||||
|
||||
variants, err := queryStringColumn[imagerequest.VariantKey](ctx, c.db, `
|
||||
SELECT cache_key FROM variant_content WHERE checked_at < ? LIMIT ?
|
||||
`, "variants not checked recently", "variant key", cutoff, evictionBatchSize)
|
||||
`, "variants past the maximum age", "variant key", cutoff, evictionBatchSize)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -171,7 +171,7 @@ func (c *Cache) EvictNotCheckedRecently(ctx context.Context) error {
|
||||
}
|
||||
|
||||
if len(sources) > 0 || len(variants) > 0 {
|
||||
c.log.Info("evicted cache content not checked with upstream recently",
|
||||
c.log.Info("evicted cache content past the maximum age",
|
||||
"sources", len(sources), "variants", len(variants))
|
||||
}
|
||||
|
||||
@@ -394,9 +394,9 @@ func (c *Cache) notifyWritePressure() {
|
||||
}
|
||||
|
||||
// StartEviction launches the background eviction goroutine, which
|
||||
// removes the expired negative cache rows, and the sources and variants not
|
||||
// checked for EvictionAfterLastCheck, at startup and again on every periodic
|
||||
// tick thereafter, reconciles the database accounting with the
|
||||
// removes the expired negative cache rows, and the sources and variants
|
||||
// past CacheMaxAge, at startup and again on every periodic tick
|
||||
// thereafter, reconciles the database accounting with the
|
||||
// cache directories at startup and again on the first periodic tick
|
||||
// that comes at least ten times as long after a reconciliation pass
|
||||
// ended as that pass took, and evicts to the configured limit on the given
|
||||
@@ -443,7 +443,7 @@ func (c *Cache) evictionLoop(ctx context.Context, interval time.Duration) {
|
||||
|
||||
nextReconciliation := c.runReconciliationPass(ctx)
|
||||
c.runCleanExpired(ctx)
|
||||
c.runEvictNotCheckedRecently(ctx)
|
||||
c.runEvictPastMaxAge(ctx)
|
||||
c.runEvictionPass(ctx)
|
||||
|
||||
ticker := time.NewTicker(interval)
|
||||
@@ -469,7 +469,7 @@ func (c *Cache) evictionLoop(ctx context.Context, interval time.Duration) {
|
||||
}
|
||||
|
||||
c.runCleanExpired(ctx)
|
||||
c.runEvictNotCheckedRecently(ctx)
|
||||
c.runEvictPastMaxAge(ctx)
|
||||
case <-c.evictionPressure:
|
||||
}
|
||||
|
||||
@@ -527,18 +527,17 @@ func (c *Cache) runCleanExpired(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// runEvictNotCheckedRecently runs EvictNotCheckedRecently, logging a failure
|
||||
// instead of propagating it (the loop must keep running). It does nothing
|
||||
// once ctx is cancelled.
|
||||
func (c *Cache) runEvictNotCheckedRecently(ctx context.Context) {
|
||||
// runEvictPastMaxAge runs EvictPastMaxAge, logging a failure instead of
|
||||
// propagating it (the loop must keep running). It does nothing once ctx is
|
||||
// cancelled.
|
||||
func (c *Cache) runEvictPastMaxAge(ctx context.Context) {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
|
||||
err := c.EvictNotCheckedRecently(ctx)
|
||||
err := c.EvictPastMaxAge(ctx)
|
||||
if err != nil {
|
||||
c.log.Warn("evicting cache content not checked recently failed",
|
||||
"error", err)
|
||||
c.log.Warn("eviction past the maximum age failed", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+6
-6
@@ -10,11 +10,11 @@ import (
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
)
|
||||
|
||||
// TestEvictNotCheckedRecently stores a source and a variant last checked with
|
||||
// TestEvictPastMaxAge stores a source and a variant last checked with
|
||||
// upstream 31 days ago and a source and a variant checked 29 days ago.
|
||||
// EvictNotCheckedRecently must remove the rows and files of the first two and
|
||||
// keep the other two.
|
||||
func TestEvictNotCheckedRecently(t *testing.T) {
|
||||
// EvictPastMaxAge must remove the rows and files of the first two and keep
|
||||
// the other two.
|
||||
func TestEvictPastMaxAge(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cache, _ := newEvictionTestCache(t, 1<<30)
|
||||
@@ -47,9 +47,9 @@ func TestEvictNotCheckedRecently(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
err := cache.EvictNotCheckedRecently(t.Context())
|
||||
err := cache.EvictPastMaxAge(t.Context())
|
||||
if err != nil {
|
||||
t.Fatalf("EvictNotCheckedRecently() error = %v", err)
|
||||
t.Fatalf("EvictPastMaxAge() error = %v", err)
|
||||
}
|
||||
|
||||
sourceRows, err := queryStringColumn[imagerequest.SourceKey](t.Context(),
|
||||
Reference in new issue
Block a user