Compare commits

..
1 Commits
Author SHA1 Message Date
clawbot 049a051596 Check cached sources with upstream after the cache TTL (closes #69)
check / check (push) Canceled after 0s
A cached source and its variants were served until eviction, so a
changed image at the same URL was never picked up.

After 24 hours a request checks the source with upstream, sending its
stored ETag and Last-Modified. A 304 renews the source with no download
or conversion. New bytes replace the source, and older variants are
converted again. A failed check serves the cached image and puts the
URL in the negative cache. This includes the full fetch made for a
variant whose source was evicted. Images not checked for 30 days are
evicted.

A variant file's modification time is its source's check time, so a
hit still makes no database call.

Model: opus-5-5
2026-11-11 09:53:04 +00:00
9 changed files with 90 additions and 197 deletions

No files matched your search

+10 -17
View File
@@ -236,9 +236,7 @@ The image cache keeps these tables in the database:
- `variant_content`: a row for each variant, by variant key (column
`cache_key`), with:
- its size and when it was last used
- its source's check time when the variant was made or last renewed
- the `ETag` and `Last-Modified` headers upstream sent with the source it
was made from
- when its source was last checked with upstream
- `cache_usage`: the total size of the cached sources and variants, which
eviction reads
- `negative_cache`: a row for each source URL in the negative cache, with when
@@ -309,21 +307,16 @@ source or confirmed it unchanged. So a source whose checks keep failing is
served for at most 30 days. Eviction to stay under the disk cache limit can
remove images sooner.
That eviction can remove a source and keep its variants. Each variant's row
keeps the `ETag` and `Last-Modified` of the source it was made from. So pixa
checks such a variant with upstream in the same way after 24 hours:
That eviction can remove a source and keep its variants. A request for such a
variant after 24 hours fetches the whole image again, since pixa no longer has
the source's `ETag` or `Last-Modified`. pixa then converts the variant again.
When that fetch fails, pixa serves the cached variant and puts the source URL in
the negative cache, as for any failed check.
- When upstream answers 304 Not Modified, pixa serves the cached variant for
another 24 hours. Nothing is downloaded or converted again.
- When upstream sends the image, pixa caches it as the source and converts the
variant again.
- When the check fails, pixa serves the cached variant and puts the source URL
in the negative cache.
A variant file's modification time is its source's check time when the variant
was made or last renewed. So a cache hit reads no database row to know whether
the variant is older than 24 hours. A request that checks its source with
upstream counts as a miss.
A variant file's modification time is when its source was last checked with
upstream. So a cache hit still reads no database row to know whether the variant
is older than 24 hours. A request that checks its source with upstream counts as
a miss.
### Routes
+6 -7
View File
@@ -37,14 +37,13 @@ P2: security: per-origin rate limiting
`ETag` and `Last-Modified` upstream sent.
- The check sends them as `If-None-Match` and `If-Modified-Since`. A 304
renews the source; new bytes replace it.
- A variant file's modification time is its source's `checked_at` when the
variant was made or last renewed, so a hit makes no database call. A
variant made before the source's `fetched_at` is converted again.
- `variant_content` keeps the `ETag` and `Last-Modified` of the source a
variant was made from. A variant whose source was evicted is checked with
them, and a 304 renews the variant.
- A variant file's modification time is its source's `checked_at`, so a hit
still makes no database call. A variant made before the source's
`fetched_at` is converted again.
- A failed check serves the cached image and puts the source URL in the
negative cache, so upstream is asked at most once per 5 minutes.
negative cache, so upstream is asked at most once per 5 minutes. A variant
whose source was evicted is checked with a full fetch. When that fetch
fails, the cached variant is served the same way.
- The eviction goroutine evicts sources and variants whose `checked_at` is
30 days old (`EvictionAfterLastCheck`).
- 2026-10-27 the reconciliation pass skips a directory it cannot list and a file
+1 -6
View File
@@ -31,16 +31,11 @@ CREATE INDEX IF NOT EXISTS idx_source_content_checked
-- later, in a batch. checked_at is the checked_at of the source the
-- variant was made from, when it was made or last renewed; the variant
-- file's modification time holds the same time for cache hits.
-- upstream_etag and upstream_last_modified are those of the source bytes
-- the variant was made from, so the variant can still be checked with
-- upstream after its source is evicted.
CREATE TABLE IF NOT EXISTS variant_content (
cache_key TEXT PRIMARY KEY,
size_bytes INTEGER NOT NULL,
last_accessed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
checked_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
upstream_etag TEXT NOT NULL DEFAULT '',
upstream_last_modified TEXT NOT NULL DEFAULT ''
checked_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_variant_content_last_accessed
ON variant_content(last_accessed_at);
+6 -18
View File
@@ -184,30 +184,22 @@ func TestProxy_Get_FreshVariantAsksUpstreamNothing(t *testing.T) {
// TestProxy_Get_UnchangedSourceAfterTTLIsNotDownloadedAgain caches a variant,
// sets its source's check time and the variant's file time two days back,
// and requests it again, with the source still cached and with the source's
// row and file removed, as eviction does. Upstream must be asked with the
// validator it sent, answer 304, and send no bytes; the variant must be served
// again without being made again, and then be served from the cache until the
// TTL passes.
// and requests it again. Upstream must be asked with the validator it sent,
// answer 304, and send no bytes; the variant must be served again without
// being made again, and then be served from the cache until the TTL passes.
func TestProxy_Get_UnchangedSourceAfterTTLIsNotDownloadedAgain(t *testing.T) {
t.Parallel()
tests := []struct {
name string
sourceEvicted bool
etag string
lastModified string
wantIfNoneMatch string
wantIfModifiedSince string
}{
{"ETag", false, testETag, "", testETag, ""},
{"Last-Modified only", false, "", testLastModified, "", testLastModified},
{"both", false, testETag, testLastModified, testETag, testLastModified},
{"ETag, source evicted", true, testETag, "", testETag, ""},
{"Last-Modified only, source evicted", true, "", testLastModified, "",
testLastModified},
{"both, source evicted", true, testETag, testLastModified, testETag,
testLastModified},
{"ETag", testETag, "", testETag, ""},
{"Last-Modified only", "", testLastModified, "", testLastModified},
{"both", testETag, testLastModified, testETag, testLastModified},
}
for _, tt := range tests {
@@ -222,10 +214,6 @@ func TestProxy_Get_UnchangedSourceAfterTTLIsNotDownloadedAgain(t *testing.T) {
first, _ := getImage(t, svc, variant)
setBackSourceAndVariant(t, svc, variant, time.Now().AddDate(0, 0, -2))
if tt.sourceEvicted {
evictSource(t, svc, variant.Source)
}
second, status := getImage(t, svc, variant)
ifNoneMatch, ifModifiedSince := upstream.lastConditionalHeaders()
+27 -53
View File
@@ -349,8 +349,8 @@ func (p *Proxy) loadCachedSource(
// from them. When it did not change, or the check fails, the cached source
// is used: the variant's cached file is served again if it was made from the
// cached source's bytes, and otherwise the variant is made from them. When
// the source is not cached but the variant is, the variant is checked with
// upstream the same way (see fetchAndProcess).
// the source is not cached, a failed fetch still serves the variant's cached
// file (see fetchAndProcess).
func (p *Proxy) processFromSourceOrFetch(
ctx context.Context,
req *imagerequest.Request,
@@ -396,8 +396,8 @@ func (p *Proxy) processFromSourceOrFetch(
// Process using cached source; nothing was fetched from upstream. The
// image processor reads the source only once it has a processing slot,
// so a request waiting for one holds none of it in memory.
return p.processAndStore(ctx, req, cacheKey, cached, source, sourceFormat,
sourceSize)
return p.processAndStore(ctx, req, cacheKey, cached.CheckedAt,
source, sourceFormat, sourceSize)
}
// recordFailedCheck logs a failed check of req's source with upstream, after
@@ -453,13 +453,10 @@ func (p *Proxy) renewVariant(
}
// cachedVariantIgnoringTTL returns the cached variant of cacheKey whatever
// its age, with what its row keeps about the source bytes it was made from
// (see imgcache.Cache.LookupVariant). It returns nil when the variant is not
// cached or cannot be read. The SourceInfo is empty when the variant is not
// returned, or its row is missing or cannot be read.
// its age. It returns nil when the variant is not cached or cannot be read.
func (p *Proxy) cachedVariantIgnoringTTL(
ctx context.Context, cacheKey imagerequest.VariantKey,
) (*processedVariant, imgcache.SourceInfo) {
cacheKey imagerequest.VariantKey,
) *processedVariant {
reader, _, contentType, err := p.cache.VariantIgnoringTTL(cacheKey)
if err != nil {
if !errors.Is(err, imgcache.ErrNotFound) {
@@ -467,21 +464,10 @@ func (p *Proxy) cachedVariantIgnoringTTL(
"error", err)
}
return nil, imgcache.SourceInfo{}
return nil
}
variant := p.readCachedVariant(cacheKey, reader, contentType)
if variant == nil {
return nil, imgcache.SourceInfo{}
}
madeFrom, err := p.cache.LookupVariant(ctx, cacheKey)
if err != nil && !errors.Is(err, imgcache.ErrNotFound) {
p.log.Warn("failed to look up cached variant", "cache_key", cacheKey,
"error", err)
}
return variant, madeFrom
return p.readCachedVariant(cacheKey, reader, contentType)
}
// readCachedVariant reads the cached variant of cacheKey that reader opened,
@@ -504,40 +490,28 @@ func (p *Proxy) readCachedVariant(
}
// fetchAndProcess fetches the source from upstream, stores it, and makes
// and stores the variant from it.
//
// The variant can still be cached after eviction removed its source. The
// fetch then sends the ETag and Last-Modified of the source bytes the variant
// was made from, so upstream sends the source only if it changed since. When
// it did not change, the cached variant is renewed and served, with no
// download and no conversion. When the fetch fails, the cached variant is
// served whatever its age, and the failure is recorded as a failed check (see
// recordFailedCheck).
// and stores the variant from it. When the fetch fails while the variant is
// still cached, as it can be after eviction removed its source, the cached
// variant is served whatever its age, and the failure is recorded as a failed
// check (see recordFailedCheck).
func (p *Proxy) fetchAndProcess(
ctx context.Context,
req *imagerequest.Request,
cacheKey imagerequest.VariantKey,
) (*processedVariant, error) {
variant, madeFrom := p.cachedVariantIgnoringTTL(ctx, cacheKey)
fetched, err := p.fetchSource(ctx, req, madeFrom.ETag, madeFrom.LastModified)
switch {
case err == nil:
return p.processFetched(ctx, req, cacheKey, fetched)
case variant == nil:
return nil, err
case errors.Is(err, errSourceNotModified):
err = p.cache.SetVariantCheckedAt(ctx, cacheKey, time.Now().UTC())
if err != nil {
p.log.Warn("failed to renew cached variant", "cache_key", cacheKey,
"error", err)
fetched, err := p.fetchSource(ctx, req, "", "")
if err != nil {
variant := p.cachedVariantIgnoringTTL(cacheKey)
if variant == nil {
return nil, err
}
default:
p.recordFailedCheck(ctx, req, err)
return variant, nil
}
return variant, nil
return p.processFetched(ctx, req, cacheKey, fetched)
}
// processFetched makes and stores the variant from the source fetchSource
@@ -554,7 +528,7 @@ func (p *Proxy) processFetched(
// processing slot.
defer func() { _ = fetched.response.Close() }()
return p.processAndStore(ctx, req, cacheKey, fetched.info,
return p.processAndStore(ctx, req, cacheKey, fetched.info.CheckedAt,
bytes.NewReader(fetched.data), fetched.format, int64(len(fetched.data)))
}
@@ -748,13 +722,13 @@ type processedVariant struct {
}
// processAndStore processes the image read from source, which is in
// sourceFormat, and stores the result. sourceInfo describes the source's
// bytes, and is stored with the variant (see imgcache.Cache.PutVariant).
// sourceFormat, and stores the result. sourceCheckedAt is when the source
// was last checked with upstream (see imgcache.Cache.PutVariant).
func (p *Proxy) processAndStore(
ctx context.Context,
req *imagerequest.Request,
cacheKey imagerequest.VariantKey,
sourceInfo imgcache.SourceInfo,
sourceCheckedAt time.Time,
source io.Reader,
sourceFormat imagerequest.Format,
fetchBytes int64,
@@ -815,7 +789,7 @@ func (p *Proxy) processAndStore(
)
// Store variant to cache
err = p.cache.PutVariant(ctx, cacheKey, sourceInfo,
err = p.cache.PutVariant(ctx, cacheKey, sourceCheckedAt,
bytes.NewReader(processedData))
if err != nil {
p.log.Warn("failed to store variant", "error", err)
+36 -90
View File
@@ -55,8 +55,8 @@ const DefaultCacheTTL = 24 * time.Hour
// fail is served.
const EvictionAfterLastCheck = 30 * 24 * time.Hour
// ErrStale is returned by Variant for a variant whose file's modification
// time is CacheTTL or more ago (see Variant).
// ErrStale is returned by Variant for a variant whose source was last
// checked with upstream CacheTTL or more ago.
var ErrStale = errors.New("cached variant is older than the cache TTL")
// Config holds cache configuration.
@@ -188,13 +188,13 @@ func NewCache(
// Variant opens a cached variant and returns a reader, its size and its
// content type, read from the variant file's first bytes. It opens that one
// file and makes no database call: the file's modification time is its
// source's check time when the variant was made or last renewed (see
// PutVariant), and a variant whose time is CacheTTL or more ago is ErrStale.
// A variant it opens has its access time handed to the background writer
// without waiting (see touchVariant). A variant not in the cache, or any
// variant of a disabled cache, is ErrNotFound; a variant file whose first
// bytes are not those of an output format is another error.
// file and makes no database call: the file's modification time is when its
// source was last checked with upstream (see PutVariant), and a variant
// whose source was checked CacheTTL or more ago is ErrStale. A variant it
// opens has its access time handed to the background writer without waiting
// (see touchVariant). A variant not in the cache, or any variant of a
// disabled cache, is ErrNotFound; a variant file whose first bytes are not
// those of an output format is another error.
func (c *Cache) Variant(
cacheKey imagerequest.VariantKey,
) (io.ReadCloser, int64, string, error) {
@@ -217,8 +217,8 @@ func (c *Cache) Variant(
}
// VariantIgnoringTTL opens a cached variant whatever its age, and returns it
// as Variant does. The image proxy reads it for a variant whose source is not
// cached, before it checks the source with upstream (see LookupVariant).
// as Variant does. The image proxy serves it when a fetch of the variant's
// source fails while the source is not cached.
func (c *Cache) VariantIgnoringTTL(
cacheKey imagerequest.VariantKey,
) (io.ReadCloser, int64, string, error) {
@@ -236,8 +236,10 @@ func (c *Cache) VariantIgnoringTTL(
// RenewVariant opens a cached variant made from the source bytes that source
// describes, whatever its age, and returns it as Variant does. It moves the
// variant's check time forward to source.CheckedAt (see SetVariantCheckedAt).
// A variant made before source.FetchedAt was made from older bytes, and is
// variant file's modification time, and the checked_at of its row, forward
// to source.CheckedAt, so Variant serves the file until CacheTTL after that
// check and eviction keeps it until EvictionAfterLastCheck after it. A
// variant made before source.FetchedAt was made from older bytes, and is
// ErrNotFound, as is a variant not in the cache.
func (c *Cache) RenewVariant(
ctx context.Context, cacheKey imagerequest.VariantKey, source SourceInfo,
@@ -265,12 +267,20 @@ func (c *Cache) RenewVariant(
}
if info.ModTime().Before(source.CheckedAt) {
err = c.SetVariantCheckedAt(ctx, cacheKey, source.CheckedAt)
err = c.variants.SetModTime(string(cacheKey), source.CheckedAt)
if err != nil {
_ = file.Close()
return nil, 0, "", err
}
_, err = c.db.ExecContext(ctx,
`UPDATE variant_content SET checked_at = ? WHERE cache_key = ?`,
source.CheckedAt.UTC(), string(cacheKey))
if err != nil {
c.log.Warn("failed to renew variant's checked_at",
"cache_key", cacheKey, "error", err)
}
}
c.touchVariant(cacheKey)
@@ -278,36 +288,6 @@ func (c *Cache) RenewVariant(
return file, info.Size(), format.MIMEType(), nil
}
// SetVariantCheckedAt records checkedAt as when upstream last sent the source
// bytes the cached variant of cacheKey was made from, or answered that they
// had not changed. It sets the variant file's modification time to checkedAt,
// so Variant serves the file until CacheTTL after it. It sets the checked_at
// of the variant's row too, so eviction keeps the variant until
// EvictionAfterLastCheck after it. A failure to write the row is only logged.
// On a disabled cache it is a no-op.
func (c *Cache) SetVariantCheckedAt(
ctx context.Context, cacheKey imagerequest.VariantKey, checkedAt time.Time,
) error {
if c.config.MaxBytes == 0 {
return nil
}
err := c.variants.SetModTime(string(cacheKey), checkedAt)
if err != nil {
return err
}
_, err = c.db.ExecContext(ctx,
`UPDATE variant_content SET checked_at = ? WHERE cache_key = ?`,
checkedAt.UTC(), string(cacheKey))
if err != nil {
c.log.Warn("failed to renew variant's checked_at",
"cache_key", cacheKey, "error", err)
}
return nil
}
// Stale reports whether a source last checked with upstream at checkedAt,
// and its variants, are due to be checked again.
func (c *Cache) Stale(checkedAt time.Time) bool {
@@ -315,16 +295,16 @@ func (c *Cache) Stale(checkedAt time.Time) bool {
}
// PutVariant stores a processed variant by its cache key and records it in
// the size accounting. source describes the source bytes the variant was made
// from. Its CheckedAt becomes the modification time of the variant's file,
// which Variant and RenewVariant read. The variant's row keeps its CheckedAt,
// ETag and LastModified (see LookupVariant). On a disabled cache it is a
// no-op. The accounting insert is best-effort (the startup reconciliation
// pass adopts any variant file that misses its accounting row).
// the size accounting. sourceCheckedAt is when the source the variant was
// made from was last checked with upstream; it becomes the modification time
// of the variant's file, which Variant and RenewVariant read. On a disabled
// cache it is a no-op. The accounting insert is best-effort (the startup
// reconciliation pass adopts any variant file that misses its accounting
// row).
func (c *Cache) PutVariant(
ctx context.Context,
cacheKey imagerequest.VariantKey,
source SourceInfo,
sourceCheckedAt time.Time,
content io.Reader,
) error {
if c.config.MaxBytes == 0 {
@@ -343,7 +323,7 @@ func (c *Cache) PutVariant(
// A file left with the time of its writing could be served as made from
// bytes fetched after the ones it was made from
err = c.variants.SetModTime(string(cacheKey), source.CheckedAt)
err = c.variants.SetModTime(string(cacheKey), sourceCheckedAt)
if err != nil {
deleteErr := c.variants.Delete(string(cacheKey))
if deleteErr != nil {
@@ -357,17 +337,13 @@ func (c *Cache) PutVariant(
size := int64(len(data))
_, err = c.db.ExecContext(ctx, `
INSERT INTO variant_content (cache_key, size_bytes, checked_at,
upstream_etag, upstream_last_modified)
VALUES (?, ?, ?, ?, ?)
INSERT INTO variant_content (cache_key, size_bytes, checked_at)
VALUES (?, ?, ?)
ON CONFLICT(cache_key) DO UPDATE SET
size_bytes = excluded.size_bytes,
last_accessed_at = CURRENT_TIMESTAMP,
checked_at = excluded.checked_at,
upstream_etag = excluded.upstream_etag,
upstream_last_modified = excluded.upstream_last_modified
`, string(cacheKey), size, source.CheckedAt.UTC(), source.ETag,
source.LastModified)
checked_at = excluded.checked_at
`, string(cacheKey), size, sourceCheckedAt.UTC())
if err != nil {
c.log.Warn("failed to record variant in size accounting",
"cache_key", cacheKey, "error", err)
@@ -465,36 +441,6 @@ func (c *Cache) LookupSource(
return info, nil
}
// LookupVariant returns what the row of the cached variant of cacheKey keeps
// about the source bytes the variant was made from: the CheckedAt, ETag and
// LastModified they had when the variant was made or last renewed. The row
// keeps no FetchedAt, so it is zero. With the ETag and LastModified the image
// proxy checks the variant with upstream after its source is evicted. A
// variant with no row is ErrNotFound, as is any variant of a disabled cache.
func (c *Cache) LookupVariant(
ctx context.Context, cacheKey imagerequest.VariantKey,
) (SourceInfo, error) {
if c.config.MaxBytes == 0 {
return SourceInfo{}, ErrNotFound
}
var info SourceInfo
err := c.db.QueryRowContext(ctx, `
SELECT checked_at, upstream_etag, upstream_last_modified
FROM variant_content WHERE cache_key = ?
`, string(cacheKey)).Scan(&info.CheckedAt, &info.ETag, &info.LastModified)
if errors.Is(err, sql.ErrNoRows) {
return SourceInfo{}, ErrNotFound
}
if err != nil {
return SourceInfo{}, fmt.Errorf("failed to look up variant: %w", err)
}
return info, nil
}
// RenewSource records that upstream answered at checkedAt that the cached
// source of src had not changed.
func (c *Cache) RenewSource(
+1 -2
View File
@@ -49,8 +49,7 @@ func TestVariant_ContentTypeFromFirstBytes(t *testing.T) {
key := imagerequest.VariantKey("aabbccdd10" + string(format))
err = cache.PutVariant(t.Context(), key, SourceInfo{CheckedAt: time.Now()},
bytes.NewReader(image))
err = cache.PutVariant(t.Context(), key, time.Now(), bytes.NewReader(image))
if err != nil {
t.Fatalf("PutVariant(%s) error = %v", format, err)
}
@@ -40,7 +40,7 @@ func TestEvictNotCheckedRecently(t *testing.T) {
for key, checkedAt := range map[imagerequest.VariantKey]time.Time{
oldVariant: old, recentVariant: recent,
} {
err := cache.PutVariant(t.Context(), key, SourceInfo{CheckedAt: checkedAt},
err := cache.PutVariant(t.Context(), key, checkedAt,
bytes.NewReader([]byte("variant")))
if err != nil {
t.Fatalf("PutVariant(%s) error = %v", key, err)
+2 -3
View File
@@ -117,8 +117,7 @@ func storeEvictionTestVariant(
) {
t.Helper()
err := cache.PutVariant(t.Context(), key, SourceInfo{CheckedAt: time.Now()},
bytes.NewReader(content))
err := cache.PutVariant(t.Context(), key, time.Now(), bytes.NewReader(content))
if err != nil {
t.Fatalf("PutVariant(%s) failed: %v", key, err)
}
@@ -433,7 +432,7 @@ func assertDisabledCacheWritesAreNoOps(
ctx := t.Context()
err := cache.PutVariant(
ctx, imagerequest.CacheKey(req), SourceInfo{}, bytes.NewReader([]byte("data")),
ctx, imagerequest.CacheKey(req), time.Now(), bytes.NewReader([]byte("data")),
)
if err != nil {
t.Fatalf("PutVariant on disabled cache must be a no-op, got error: %v", err)