Check cached sources with upstream after the cache TTL (closes #69)
check / check (push) Canceled after 0s

A cached image was served until eviction, so a change at the same URL
was never picked up.

After 24 hours a request checks the source with upstream, sending its
stored ETag and Last-Modified. A 304 renews it with no download or
conversion. New bytes replace it, and older variants are converted
again. A failed check serves the cached image and puts the URL in the
negative cache. Images not checked for 30 days are evicted.

Variant rows keep the source's ETag and Last-Modified, so a variant
whose source was evicted is checked the same way.

A variant file's modification time is its source's check time when it
was made or renewed, so a hit reads no database row.

Model: opus-5-5
This commit is contained in:
clawbot committed 2026-11-11 10:38:51 +00:00
1 parent 9f1e9f5253
commit 880845f38c
28 files changed
+1552 -231

No files matched your search

+76 -6
View File
@@ -131,6 +131,56 @@ func (c *Cache) EvictToLimit(ctx context.Context) error {
}
}
// EvictNotCheckedRecently evicts each source and variant whose checked_at is
// EvictionAfterLastCheck or more ago, evictionBatchSize of each at a time,
// until none is left. It is a no-op when the cache is disabled.
func (c *Cache) EvictNotCheckedRecently(ctx context.Context) error {
if c.config.MaxBytes == 0 {
return nil
}
cutoff := time.Now().UTC().Add(-EvictionAfterLastCheck)
for {
sources, err := queryStringColumn[imagerequest.SourceKey](ctx, c.db, `
SELECT source_key FROM source_content WHERE checked_at < ? LIMIT ?
`, "sources not checked recently", "source key", cutoff, evictionBatchSize)
if err != nil {
return err
}
for _, key := range sources {
err = c.evictSourceBlob(ctx, key)
if err != nil {
return err
}
}
variants, err := queryStringColumn[imagerequest.VariantKey](ctx, c.db, `
SELECT cache_key FROM variant_content WHERE checked_at < ? LIMIT ?
`, "variants not checked recently", "variant key", cutoff, evictionBatchSize)
if err != nil {
return err
}
for _, key := range variants {
err = c.evictVariant(ctx, key)
if err != nil {
return err
}
}
if len(sources) > 0 || len(variants) > 0 {
c.log.Info("evicted cache content not checked with upstream recently",
"sources", len(sources), "variants", len(variants))
}
if len(sources) < evictionBatchSize && len(variants) < evictionBatchSize {
return nil
}
}
}
// evictBatch fetches one batch of LRU candidates across variants and
// source blobs and evicts them oldest-first until excessBytes are
// freed or the batch is exhausted. It returns the bytes freed. A
@@ -344,8 +394,9 @@ func (c *Cache) notifyWritePressure() {
}
// StartEviction launches the background eviction goroutine, which
// removes the expired negative cache rows at startup and again on every
// periodic tick thereafter, reconciles the database accounting with the
// removes the expired negative cache rows, and the sources and variants not
// checked for EvictionAfterLastCheck, at startup and again on every periodic
// tick thereafter, reconciles the database accounting with the
// cache directories at startup and again on the first periodic tick
// that comes at least ten times as long after a reconciliation pass
// ended as that pass took, and evicts to the configured limit on the given
@@ -392,6 +443,7 @@ func (c *Cache) evictionLoop(ctx context.Context, interval time.Duration) {
nextReconciliation := c.runReconciliationPass(ctx)
c.runCleanExpired(ctx)
c.runEvictNotCheckedRecently(ctx)
c.runEvictionPass(ctx)
ticker := time.NewTicker(interval)
@@ -417,6 +469,7 @@ func (c *Cache) evictionLoop(ctx context.Context, interval time.Duration) {
}
c.runCleanExpired(ctx)
c.runEvictNotCheckedRecently(ctx)
case <-c.evictionPressure:
}
@@ -474,6 +527,21 @@ func (c *Cache) runCleanExpired(ctx context.Context) {
}
}
// runEvictNotCheckedRecently runs EvictNotCheckedRecently, logging a failure
// instead of propagating it (the loop must keep running). It does nothing
// once ctx is cancelled.
func (c *Cache) runEvictNotCheckedRecently(ctx context.Context) {
if ctx.Err() != nil {
return
}
err := c.EvictNotCheckedRecently(ctx)
if err != nil {
c.log.Warn("evicting cache content not checked recently failed",
"error", err)
}
}
// reconcileAccounting synchronizes the database size accounting with
// the actual contents of the cache directories. It runs at startup and
// then on the first periodic eviction tick that comes at least ten times
@@ -609,16 +677,18 @@ func (c *Cache) reconcileVariantFile(
}
// adoptVariantFile inserts an accounting row for a variant file that
// has none, using the file's size and modification time.
// has none, using the file's size and modification time, which is also
// the variant's checked_at.
func (c *Cache) adoptVariantFile(
ctx context.Context, info fs.FileInfo, cacheKey imagerequest.VariantKey,
) error {
modTime := info.ModTime().UTC().Format(sqliteTimestampLayout)
_, err := c.db.ExecContext(ctx, `
INSERT INTO variant_content (cache_key, size_bytes, last_accessed_at)
VALUES (?, ?, ?)
`, string(cacheKey), info.Size(), modTime)
INSERT INTO variant_content (cache_key, size_bytes, last_accessed_at,
checked_at)
VALUES (?, ?, ?, ?)
`, string(cacheKey), info.Size(), modTime, info.ModTime().UTC())
if err != nil {
return fmt.Errorf("failed to adopt variant file into accounting: %w", err)
}