Compare commits

..
1 Commits
Author SHA1 Message Date
clawbot 880845f38c Check cached sources with upstream after the cache TTL (closes #69)
check / check (push) Canceled after 0s
A cached image was served until eviction, so a change at the same URL
was never picked up.

After 24 hours a request checks the source with upstream, sending its
stored ETag and Last-Modified. A 304 renews it with no download or
conversion. New bytes replace it, and older variants are converted
again. A failed check serves the cached image and puts the URL in the
negative cache. Images not checked for 30 days are evicted.

Variant rows keep the source's ETag and Last-Modified, so a variant
whose source was evicted is checked the same way.

A variant file's modification time is its source's check time when it
was made or renewed, so a hit reads no database row.

Model: opus-5-5
2026-11-11 10:38:51 +00:00
9 changed files with 197 additions and 90 deletions

No files matched your search

+17 -10
View File
@@ -236,7 +236,9 @@ The image cache keeps these tables in the database:
- `variant_content`: a row for each variant, by variant key (column
`cache_key`), with:
- its size and when it was last used
- when its source was last checked with upstream
- its source's check time when the variant was made or last renewed
- the `ETag` and `Last-Modified` headers upstream sent with the source it
was made from
- `cache_usage`: the total size of the cached sources and variants, which
eviction reads
- `negative_cache`: a row for each source URL in the negative cache, with when
@@ -307,16 +309,21 @@ source or confirmed it unchanged. So a source whose checks keep failing is
served for at most 30 days. Eviction to stay under the disk cache limit can
remove images sooner.
That eviction can remove a source and keep its variants. A request for such a
variant after 24 hours fetches the whole image again, since pixa no longer has
the source's `ETag` or `Last-Modified`. pixa then converts the variant again.
When that fetch fails, pixa serves the cached variant and puts the source URL in
the negative cache, as for any failed check.
That eviction can remove a source and keep its variants. Each variant's row
keeps the `ETag` and `Last-Modified` of the source it was made from. So pixa
checks such a variant with upstream in the same way after 24 hours:
A variant file's modification time is when its source was last checked with
upstream. So a cache hit still reads no database row to know whether the variant
is older than 24 hours. A request that checks its source with upstream counts as
a miss.
- When upstream answers 304 Not Modified, pixa serves the cached variant for
another 24 hours. Nothing is downloaded or converted again.
- When upstream sends the image, pixa caches it as the source and converts the
variant again.
- When the check fails, pixa serves the cached variant and puts the source URL
in the negative cache.
A variant file's modification time is its source's check time when the variant
was made or last renewed. So a cache hit reads no database row to know whether
the variant is older than 24 hours. A request that checks its source with
upstream counts as a miss.
### Routes
+7 -6
View File
@@ -37,13 +37,14 @@ P2: security: per-origin rate limiting
`ETag` and `Last-Modified` upstream sent.
- The check sends them as `If-None-Match` and `If-Modified-Since`. A 304
renews the source; new bytes replace it.
- A variant file's modification time is its source's `checked_at`, so a hit
still makes no database call. A variant made before the source's
`fetched_at` is converted again.
- A variant file's modification time is its source's `checked_at` when the
variant was made or last renewed, so a hit makes no database call. A
variant made before the source's `fetched_at` is converted again.
- `variant_content` keeps the `ETag` and `Last-Modified` of the source a
variant was made from. A variant whose source was evicted is checked with
them, and a 304 renews the variant.
- A failed check serves the cached image and puts the source URL in the
negative cache, so upstream is asked at most once per 5 minutes. A variant
whose source was evicted is checked with a full fetch. When that fetch
fails, the cached variant is served the same way.
negative cache, so upstream is asked at most once per 5 minutes.
- The eviction goroutine evicts sources and variants whose `checked_at` is
30 days old (`EvictionAfterLastCheck`).
- 2026-10-27 the reconciliation pass skips a directory it cannot list and a file
+6 -1
View File
@@ -31,11 +31,16 @@ CREATE INDEX IF NOT EXISTS idx_source_content_checked
-- later, in a batch. checked_at is the checked_at of the source the
-- variant was made from, when it was made or last renewed; the variant
-- file's modification time holds the same time for cache hits.
-- upstream_etag and upstream_last_modified are those of the source bytes
-- the variant was made from, so the variant can still be checked with
-- upstream after its source is evicted.
CREATE TABLE IF NOT EXISTS variant_content (
cache_key TEXT PRIMARY KEY,
size_bytes INTEGER NOT NULL,
last_accessed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
checked_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
checked_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
upstream_etag TEXT NOT NULL DEFAULT '',
upstream_last_modified TEXT NOT NULL DEFAULT ''
);
CREATE INDEX IF NOT EXISTS idx_variant_content_last_accessed
ON variant_content(last_accessed_at);
+18 -6
View File
@@ -184,22 +184,30 @@ func TestProxy_Get_FreshVariantAsksUpstreamNothing(t *testing.T) {
// TestProxy_Get_UnchangedSourceAfterTTLIsNotDownloadedAgain caches a variant,
// sets its source's check time and the variant's file time two days back,
// and requests it again. Upstream must be asked with the validator it sent,
// answer 304, and send no bytes; the variant must be served again without
// being made again, and then be served from the cache until the TTL passes.
// and requests it again, with the source still cached and with the source's
// row and file removed, as eviction does. Upstream must be asked with the
// validator it sent, answer 304, and send no bytes; the variant must be served
// again without being made again, and then be served from the cache until the
// TTL passes.
func TestProxy_Get_UnchangedSourceAfterTTLIsNotDownloadedAgain(t *testing.T) {
t.Parallel()
tests := []struct {
name string
sourceEvicted bool
etag string
lastModified string
wantIfNoneMatch string
wantIfModifiedSince string
}{
{"ETag", testETag, "", testETag, ""},
{"Last-Modified only", "", testLastModified, "", testLastModified},
{"both", testETag, testLastModified, testETag, testLastModified},
{"ETag", false, testETag, "", testETag, ""},
{"Last-Modified only", false, "", testLastModified, "", testLastModified},
{"both", false, testETag, testLastModified, testETag, testLastModified},
{"ETag, source evicted", true, testETag, "", testETag, ""},
{"Last-Modified only, source evicted", true, "", testLastModified, "",
testLastModified},
{"both, source evicted", true, testETag, testLastModified, testETag,
testLastModified},
}
for _, tt := range tests {
@@ -214,6 +222,10 @@ func TestProxy_Get_UnchangedSourceAfterTTLIsNotDownloadedAgain(t *testing.T) {
first, _ := getImage(t, svc, variant)
setBackSourceAndVariant(t, svc, variant, time.Now().AddDate(0, 0, -2))
if tt.sourceEvicted {
evictSource(t, svc, variant.Source)
}
second, status := getImage(t, svc, variant)
ifNoneMatch, ifModifiedSince := upstream.lastConditionalHeaders()
+53 -27
View File
@@ -349,8 +349,8 @@ func (p *Proxy) loadCachedSource(
// from them. When it did not change, or the check fails, the cached source
// is used: the variant's cached file is served again if it was made from the
// cached source's bytes, and otherwise the variant is made from them. When
// the source is not cached, a failed fetch still serves the variant's cached
// file (see fetchAndProcess).
// the source is not cached but the variant is, the variant is checked with
// upstream the same way (see fetchAndProcess).
func (p *Proxy) processFromSourceOrFetch(
ctx context.Context,
req *imagerequest.Request,
@@ -396,8 +396,8 @@ func (p *Proxy) processFromSourceOrFetch(
// Process using cached source; nothing was fetched from upstream. The
// image processor reads the source only once it has a processing slot,
// so a request waiting for one holds none of it in memory.
return p.processAndStore(ctx, req, cacheKey, cached.CheckedAt,
source, sourceFormat, sourceSize)
return p.processAndStore(ctx, req, cacheKey, cached, source, sourceFormat,
sourceSize)
}
// recordFailedCheck logs a failed check of req's source with upstream, after
@@ -453,10 +453,13 @@ func (p *Proxy) renewVariant(
}
// cachedVariantIgnoringTTL returns the cached variant of cacheKey whatever
// its age. It returns nil when the variant is not cached or cannot be read.
// its age, with what its row keeps about the source bytes it was made from
// (see imgcache.Cache.LookupVariant). It returns nil when the variant is not
// cached or cannot be read. The SourceInfo is empty when the variant is not
// returned, or its row is missing or cannot be read.
func (p *Proxy) cachedVariantIgnoringTTL(
cacheKey imagerequest.VariantKey,
) *processedVariant {
ctx context.Context, cacheKey imagerequest.VariantKey,
) (*processedVariant, imgcache.SourceInfo) {
reader, _, contentType, err := p.cache.VariantIgnoringTTL(cacheKey)
if err != nil {
if !errors.Is(err, imgcache.ErrNotFound) {
@@ -464,10 +467,21 @@ func (p *Proxy) cachedVariantIgnoringTTL(
"error", err)
}
return nil
return nil, imgcache.SourceInfo{}
}
return p.readCachedVariant(cacheKey, reader, contentType)
variant := p.readCachedVariant(cacheKey, reader, contentType)
if variant == nil {
return nil, imgcache.SourceInfo{}
}
madeFrom, err := p.cache.LookupVariant(ctx, cacheKey)
if err != nil && !errors.Is(err, imgcache.ErrNotFound) {
p.log.Warn("failed to look up cached variant", "cache_key", cacheKey,
"error", err)
}
return variant, madeFrom
}
// readCachedVariant reads the cached variant of cacheKey that reader opened,
@@ -490,28 +504,40 @@ func (p *Proxy) readCachedVariant(
}
// fetchAndProcess fetches the source from upstream, stores it, and makes
// and stores the variant from it. When the fetch fails while the variant is
// still cached, as it can be after eviction removed its source, the cached
// variant is served whatever its age, and the failure is recorded as a failed
// check (see recordFailedCheck).
// and stores the variant from it.
//
// The variant can still be cached after eviction removed its source. The
// fetch then sends the ETag and Last-Modified of the source bytes the variant
// was made from, so upstream sends the source only if it changed since. When
// it did not change, the cached variant is renewed and served, with no
// download and no conversion. When the fetch fails, the cached variant is
// served whatever its age, and the failure is recorded as a failed check (see
// recordFailedCheck).
func (p *Proxy) fetchAndProcess(
ctx context.Context,
req *imagerequest.Request,
cacheKey imagerequest.VariantKey,
) (*processedVariant, error) {
fetched, err := p.fetchSource(ctx, req, "", "")
if err != nil {
variant := p.cachedVariantIgnoringTTL(cacheKey)
if variant == nil {
return nil, err
variant, madeFrom := p.cachedVariantIgnoringTTL(ctx, cacheKey)
fetched, err := p.fetchSource(ctx, req, madeFrom.ETag, madeFrom.LastModified)
switch {
case err == nil:
return p.processFetched(ctx, req, cacheKey, fetched)
case variant == nil:
return nil, err
case errors.Is(err, errSourceNotModified):
err = p.cache.SetVariantCheckedAt(ctx, cacheKey, time.Now().UTC())
if err != nil {
p.log.Warn("failed to renew cached variant", "cache_key", cacheKey,
"error", err)
}
default:
p.recordFailedCheck(ctx, req, err)
return variant, nil
}
return p.processFetched(ctx, req, cacheKey, fetched)
return variant, nil
}
// processFetched makes and stores the variant from the source fetchSource
@@ -528,7 +554,7 @@ func (p *Proxy) processFetched(
// processing slot.
defer func() { _ = fetched.response.Close() }()
return p.processAndStore(ctx, req, cacheKey, fetched.info.CheckedAt,
return p.processAndStore(ctx, req, cacheKey, fetched.info,
bytes.NewReader(fetched.data), fetched.format, int64(len(fetched.data)))
}
@@ -722,13 +748,13 @@ type processedVariant struct {
}
// processAndStore processes the image read from source, which is in
// sourceFormat, and stores the result. sourceCheckedAt is when the source
// was last checked with upstream (see imgcache.Cache.PutVariant).
// sourceFormat, and stores the result. sourceInfo describes the source's
// bytes, and is stored with the variant (see imgcache.Cache.PutVariant).
func (p *Proxy) processAndStore(
ctx context.Context,
req *imagerequest.Request,
cacheKey imagerequest.VariantKey,
sourceCheckedAt time.Time,
sourceInfo imgcache.SourceInfo,
source io.Reader,
sourceFormat imagerequest.Format,
fetchBytes int64,
@@ -789,7 +815,7 @@ func (p *Proxy) processAndStore(
)
// Store variant to cache
err = p.cache.PutVariant(ctx, cacheKey, sourceCheckedAt,
err = p.cache.PutVariant(ctx, cacheKey, sourceInfo,
bytes.NewReader(processedData))
if err != nil {
p.log.Warn("failed to store variant", "error", err)
+90 -36
View File
@@ -55,8 +55,8 @@ const DefaultCacheTTL = 24 * time.Hour
// fail is served.
const EvictionAfterLastCheck = 30 * 24 * time.Hour
// ErrStale is returned by Variant for a variant whose source was last
// checked with upstream CacheTTL or more ago.
// ErrStale is returned by Variant for a variant whose file's modification
// time is CacheTTL or more ago (see Variant).
var ErrStale = errors.New("cached variant is older than the cache TTL")
// Config holds cache configuration.
@@ -188,13 +188,13 @@ func NewCache(
// Variant opens a cached variant and returns a reader, its size and its
// content type, read from the variant file's first bytes. It opens that one
// file and makes no database call: the file's modification time is when its
// source was last checked with upstream (see PutVariant), and a variant
// whose source was checked CacheTTL or more ago is ErrStale. A variant it
// opens has its access time handed to the background writer without waiting
// (see touchVariant). A variant not in the cache, or any variant of a
// disabled cache, is ErrNotFound; a variant file whose first bytes are not
// those of an output format is another error.
// file and makes no database call: the file's modification time is its
// source's check time when the variant was made or last renewed (see
// PutVariant), and a variant whose time is CacheTTL or more ago is ErrStale.
// A variant it opens has its access time handed to the background writer
// without waiting (see touchVariant). A variant not in the cache, or any
// variant of a disabled cache, is ErrNotFound; a variant file whose first
// bytes are not those of an output format is another error.
func (c *Cache) Variant(
cacheKey imagerequest.VariantKey,
) (io.ReadCloser, int64, string, error) {
@@ -217,8 +217,8 @@ func (c *Cache) Variant(
}
// VariantIgnoringTTL opens a cached variant whatever its age, and returns it
// as Variant does. The image proxy serves it when a fetch of the variant's
// source fails while the source is not cached.
// as Variant does. The image proxy reads it for a variant whose source is not
// cached, before it checks the source with upstream (see LookupVariant).
func (c *Cache) VariantIgnoringTTL(
cacheKey imagerequest.VariantKey,
) (io.ReadCloser, int64, string, error) {
@@ -236,10 +236,8 @@ func (c *Cache) VariantIgnoringTTL(
// RenewVariant opens a cached variant made from the source bytes that source
// describes, whatever its age, and returns it as Variant does. It moves the
// variant file's modification time, and the checked_at of its row, forward
// to source.CheckedAt, so Variant serves the file until CacheTTL after that
// check and eviction keeps it until EvictionAfterLastCheck after it. A
// variant made before source.FetchedAt was made from older bytes, and is
// variant's check time forward to source.CheckedAt (see SetVariantCheckedAt).
// A variant made before source.FetchedAt was made from older bytes, and is
// ErrNotFound, as is a variant not in the cache.
func (c *Cache) RenewVariant(
ctx context.Context, cacheKey imagerequest.VariantKey, source SourceInfo,
@@ -267,20 +265,12 @@ func (c *Cache) RenewVariant(
}
if info.ModTime().Before(source.CheckedAt) {
err = c.variants.SetModTime(string(cacheKey), source.CheckedAt)
err = c.SetVariantCheckedAt(ctx, cacheKey, source.CheckedAt)
if err != nil {
_ = file.Close()
return nil, 0, "", err
}
_, err = c.db.ExecContext(ctx,
`UPDATE variant_content SET checked_at = ? WHERE cache_key = ?`,
source.CheckedAt.UTC(), string(cacheKey))
if err != nil {
c.log.Warn("failed to renew variant's checked_at",
"cache_key", cacheKey, "error", err)
}
}
c.touchVariant(cacheKey)
@@ -288,6 +278,36 @@ func (c *Cache) RenewVariant(
return file, info.Size(), format.MIMEType(), nil
}
// SetVariantCheckedAt records checkedAt as when upstream last sent the source
// bytes the cached variant of cacheKey was made from, or answered that they
// had not changed. It sets the variant file's modification time to checkedAt,
// so Variant serves the file until CacheTTL after it. It sets the checked_at
// of the variant's row too, so eviction keeps the variant until
// EvictionAfterLastCheck after it. A failure to write the row is only logged.
// On a disabled cache it is a no-op.
func (c *Cache) SetVariantCheckedAt(
ctx context.Context, cacheKey imagerequest.VariantKey, checkedAt time.Time,
) error {
if c.config.MaxBytes == 0 {
return nil
}
err := c.variants.SetModTime(string(cacheKey), checkedAt)
if err != nil {
return err
}
_, err = c.db.ExecContext(ctx,
`UPDATE variant_content SET checked_at = ? WHERE cache_key = ?`,
checkedAt.UTC(), string(cacheKey))
if err != nil {
c.log.Warn("failed to renew variant's checked_at",
"cache_key", cacheKey, "error", err)
}
return nil
}
// Stale reports whether a source last checked with upstream at checkedAt,
// and its variants, are due to be checked again.
func (c *Cache) Stale(checkedAt time.Time) bool {
@@ -295,16 +315,16 @@ func (c *Cache) Stale(checkedAt time.Time) bool {
}
// PutVariant stores a processed variant by its cache key and records it in
// the size accounting. sourceCheckedAt is when the source the variant was
// made from was last checked with upstream; it becomes the modification time
// of the variant's file, which Variant and RenewVariant read. On a disabled
// cache it is a no-op. The accounting insert is best-effort (the startup
// reconciliation pass adopts any variant file that misses its accounting
// row).
// the size accounting. source describes the source bytes the variant was made
// from. Its CheckedAt becomes the modification time of the variant's file,
// which Variant and RenewVariant read. The variant's row keeps its CheckedAt,
// ETag and LastModified (see LookupVariant). On a disabled cache it is a
// no-op. The accounting insert is best-effort (the startup reconciliation
// pass adopts any variant file that misses its accounting row).
func (c *Cache) PutVariant(
ctx context.Context,
cacheKey imagerequest.VariantKey,
sourceCheckedAt time.Time,
source SourceInfo,
content io.Reader,
) error {
if c.config.MaxBytes == 0 {
@@ -323,7 +343,7 @@ func (c *Cache) PutVariant(
// A file left with the time of its writing could be served as made from
// bytes fetched after the ones it was made from
err = c.variants.SetModTime(string(cacheKey), sourceCheckedAt)
err = c.variants.SetModTime(string(cacheKey), source.CheckedAt)
if err != nil {
deleteErr := c.variants.Delete(string(cacheKey))
if deleteErr != nil {
@@ -337,13 +357,17 @@ func (c *Cache) PutVariant(
size := int64(len(data))
_, err = c.db.ExecContext(ctx, `
INSERT INTO variant_content (cache_key, size_bytes, checked_at)
VALUES (?, ?, ?)
INSERT INTO variant_content (cache_key, size_bytes, checked_at,
upstream_etag, upstream_last_modified)
VALUES (?, ?, ?, ?, ?)
ON CONFLICT(cache_key) DO UPDATE SET
size_bytes = excluded.size_bytes,
last_accessed_at = CURRENT_TIMESTAMP,
checked_at = excluded.checked_at
`, string(cacheKey), size, sourceCheckedAt.UTC())
checked_at = excluded.checked_at,
upstream_etag = excluded.upstream_etag,
upstream_last_modified = excluded.upstream_last_modified
`, string(cacheKey), size, source.CheckedAt.UTC(), source.ETag,
source.LastModified)
if err != nil {
c.log.Warn("failed to record variant in size accounting",
"cache_key", cacheKey, "error", err)
@@ -441,6 +465,36 @@ func (c *Cache) LookupSource(
return info, nil
}
// LookupVariant returns what the row of the cached variant of cacheKey keeps
// about the source bytes the variant was made from: the CheckedAt, ETag and
// LastModified they had when the variant was made or last renewed. The row
// keeps no FetchedAt, so it is zero. With the ETag and LastModified the image
// proxy checks the variant with upstream after its source is evicted. A
// variant with no row is ErrNotFound, as is any variant of a disabled cache.
func (c *Cache) LookupVariant(
ctx context.Context, cacheKey imagerequest.VariantKey,
) (SourceInfo, error) {
if c.config.MaxBytes == 0 {
return SourceInfo{}, ErrNotFound
}
var info SourceInfo
err := c.db.QueryRowContext(ctx, `
SELECT checked_at, upstream_etag, upstream_last_modified
FROM variant_content WHERE cache_key = ?
`, string(cacheKey)).Scan(&info.CheckedAt, &info.ETag, &info.LastModified)
if errors.Is(err, sql.ErrNoRows) {
return SourceInfo{}, ErrNotFound
}
if err != nil {
return SourceInfo{}, fmt.Errorf("failed to look up variant: %w", err)
}
return info, nil
}
// RenewSource records that upstream answered at checkedAt that the cached
// source of src had not changed.
func (c *Cache) RenewSource(
+2 -1
View File
@@ -49,7 +49,8 @@ func TestVariant_ContentTypeFromFirstBytes(t *testing.T) {
key := imagerequest.VariantKey("aabbccdd10" + string(format))
err = cache.PutVariant(t.Context(), key, time.Now(), bytes.NewReader(image))
err = cache.PutVariant(t.Context(), key, SourceInfo{CheckedAt: time.Now()},
bytes.NewReader(image))
if err != nil {
t.Fatalf("PutVariant(%s) error = %v", format, err)
}
@@ -40,7 +40,7 @@ func TestEvictNotCheckedRecently(t *testing.T) {
for key, checkedAt := range map[imagerequest.VariantKey]time.Time{
oldVariant: old, recentVariant: recent,
} {
err := cache.PutVariant(t.Context(), key, checkedAt,
err := cache.PutVariant(t.Context(), key, SourceInfo{CheckedAt: checkedAt},
bytes.NewReader([]byte("variant")))
if err != nil {
t.Fatalf("PutVariant(%s) error = %v", key, err)
+3 -2
View File
@@ -117,7 +117,8 @@ func storeEvictionTestVariant(
) {
t.Helper()
err := cache.PutVariant(t.Context(), key, time.Now(), bytes.NewReader(content))
err := cache.PutVariant(t.Context(), key, SourceInfo{CheckedAt: time.Now()},
bytes.NewReader(content))
if err != nil {
t.Fatalf("PutVariant(%s) failed: %v", key, err)
}
@@ -432,7 +433,7 @@ func assertDisabledCacheWritesAreNoOps(
ctx := t.Context()
err := cache.PutVariant(
ctx, imagerequest.CacheKey(req), time.Now(), bytes.NewReader([]byte("data")),
ctx, imagerequest.CacheKey(req), SourceInfo{}, bytes.NewReader([]byte("data")),
)
if err != nil {
t.Fatalf("PutVariant on disabled cache must be a no-op, got error: %v", err)