Check cached sources with upstream after the cache TTL (closes #69, closes #83)
check / check (push) Canceled after 0s

A cached source and its variants were served until eviction, so a
changed image at the same URL was never picked up.

Once a source was last checked 24 hours ago, the next request for one
of its images asks upstream with If-None-Match and If-Modified-Since,
from the ETag and Last-Modified it stored. A 304 renews the source and
its variant without a download or a conversion. New bytes replace the
source, and each variant made before them is converted again. A failed
check serves the cached image and puts the URL in the negative cache.
Sources and variants not checked successfully for 30 days are evicted.

A variant file's modification time is its source's check time, so a
hit still makes no database call.

Existing tests change only in call signatures. The test that asserted
two-day-old entries are served without asking upstream is replaced by
the new ones.

Model: opus-5-5
This commit is contained in:
clawbot committed 2026-11-11 00:24:06 +00:00
1 parent 9f1e9f5253
commit 8feed69143
28 files changed
+1262 -226

No files matched your search

+75 -6
View File
@@ -131,6 +131,56 @@ func (c *Cache) EvictToLimit(ctx context.Context) error {
}
}
// EvictPastMaxAge evicts each source and variant whose checked_at is
// CacheMaxAge or more ago, evictionBatchSize of each at a time, until none
// is left. It is a no-op when the cache is disabled.
func (c *Cache) EvictPastMaxAge(ctx context.Context) error {
if c.config.MaxBytes == 0 {
return nil
}
cutoff := time.Now().UTC().Add(-CacheMaxAge)
for {
sources, err := queryStringColumn[imagerequest.SourceKey](ctx, c.db, `
SELECT source_key FROM source_content WHERE checked_at < ? LIMIT ?
`, "sources past the maximum age", "source key", cutoff, evictionBatchSize)
if err != nil {
return err
}
for _, key := range sources {
err = c.evictSourceBlob(ctx, key)
if err != nil {
return err
}
}
variants, err := queryStringColumn[imagerequest.VariantKey](ctx, c.db, `
SELECT cache_key FROM variant_content WHERE checked_at < ? LIMIT ?
`, "variants past the maximum age", "variant key", cutoff, evictionBatchSize)
if err != nil {
return err
}
for _, key := range variants {
err = c.evictVariant(ctx, key)
if err != nil {
return err
}
}
if len(sources) > 0 || len(variants) > 0 {
c.log.Info("evicted cache content past the maximum age",
"sources", len(sources), "variants", len(variants))
}
if len(sources) < evictionBatchSize && len(variants) < evictionBatchSize {
return nil
}
}
}
// evictBatch fetches one batch of LRU candidates across variants and
// source blobs and evicts them oldest-first until excessBytes are
// freed or the batch is exhausted. It returns the bytes freed. A
@@ -344,8 +394,9 @@ func (c *Cache) notifyWritePressure() {
}
// StartEviction launches the background eviction goroutine, which
// removes the expired negative cache rows at startup and again on every
// periodic tick thereafter, reconciles the database accounting with the
// removes the expired negative cache rows, and the sources and variants
// past CacheMaxAge, at startup and again on every periodic tick
// thereafter, reconciles the database accounting with the
// cache directories at startup and again on the first periodic tick
// that comes at least ten times as long after a reconciliation pass
// ended as that pass took, and evicts to the configured limit on the given
@@ -392,6 +443,7 @@ func (c *Cache) evictionLoop(ctx context.Context, interval time.Duration) {
nextReconciliation := c.runReconciliationPass(ctx)
c.runCleanExpired(ctx)
c.runEvictPastMaxAge(ctx)
c.runEvictionPass(ctx)
ticker := time.NewTicker(interval)
@@ -417,6 +469,7 @@ func (c *Cache) evictionLoop(ctx context.Context, interval time.Duration) {
}
c.runCleanExpired(ctx)
c.runEvictPastMaxAge(ctx)
case <-c.evictionPressure:
}
@@ -474,6 +527,20 @@ func (c *Cache) runCleanExpired(ctx context.Context) {
}
}
// runEvictPastMaxAge runs EvictPastMaxAge, logging a failure instead of
// propagating it (the loop must keep running). It does nothing once ctx is
// cancelled.
func (c *Cache) runEvictPastMaxAge(ctx context.Context) {
if ctx.Err() != nil {
return
}
err := c.EvictPastMaxAge(ctx)
if err != nil {
c.log.Warn("eviction past the maximum age failed", "error", err)
}
}
// reconcileAccounting synchronizes the database size accounting with
// the actual contents of the cache directories. It runs at startup and
// then on the first periodic eviction tick that comes at least ten times
@@ -609,16 +676,18 @@ func (c *Cache) reconcileVariantFile(
}
// adoptVariantFile inserts an accounting row for a variant file that
// has none, using the file's size and modification time.
// has none, using the file's size and modification time, which is also
// the variant's checked_at.
func (c *Cache) adoptVariantFile(
ctx context.Context, info fs.FileInfo, cacheKey imagerequest.VariantKey,
) error {
modTime := info.ModTime().UTC().Format(sqliteTimestampLayout)
_, err := c.db.ExecContext(ctx, `
INSERT INTO variant_content (cache_key, size_bytes, last_accessed_at)
VALUES (?, ?, ?)
`, string(cacheKey), info.Size(), modTime)
INSERT INTO variant_content (cache_key, size_bytes, last_accessed_at,
checked_at)
VALUES (?, ?, ?, ?)
`, string(cacheKey), info.Size(), modTime, info.ModTime().UTC())
if err != nil {
return fmt.Errorf("failed to adopt variant file into accounting: %w", err)
}