Check cached sources with upstream after the cache TTL (closes #69)
check / check (push) Canceled after 0s
check / check (push) Canceled after 0s
A cached image was served until eviction, so a change at the same URL was never picked up. After 24 hours a request checks the source with upstream, sending its stored ETag and Last-Modified. A 304 renews it with no download or conversion. New bytes replace it, and older variants are converted again. A failed check serves the cached image and puts the URL in the negative cache. Images not checked for 30 days are evicted. Variant rows keep the source's ETag and Last-Modified, so a variant whose source was evicted is checked the same way. A variant file's modification time is its source's check time when it was made or renewed, so a hit reads no database row. Model: opus-5-5
This commit is contained in:
1 parent
b3b8408d18
commit
3f487ee0ce
28 files changed
+1556
-233
No files matched your search
@@ -131,6 +131,56 @@ func (c *Cache) EvictToLimit(ctx context.Context) error {
|
||||
}
|
||||
}
|
||||
|
||||
// EvictNotCheckedRecently evicts each source and variant whose checked_at is
|
||||
// EvictionAfterLastCheck or more ago, evictionBatchSize of each at a time,
|
||||
// until none is left. It is a no-op when the cache is disabled.
|
||||
func (c *Cache) EvictNotCheckedRecently(ctx context.Context) error {
|
||||
if c.config.MaxBytes == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
cutoff := time.Now().UTC().Add(-EvictionAfterLastCheck)
|
||||
|
||||
for {
|
||||
sources, err := queryStringColumn[imagerequest.SourceKey](ctx, c.db, `
|
||||
SELECT source_key FROM source_content WHERE checked_at < ? LIMIT ?
|
||||
`, "sources not checked recently", "source key", cutoff, evictionBatchSize)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, key := range sources {
|
||||
err = c.evictSourceBlob(ctx, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
variants, err := queryStringColumn[imagerequest.VariantKey](ctx, c.db, `
|
||||
SELECT cache_key FROM variant_content WHERE checked_at < ? LIMIT ?
|
||||
`, "variants not checked recently", "variant key", cutoff, evictionBatchSize)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, key := range variants {
|
||||
err = c.evictVariant(ctx, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if len(sources) > 0 || len(variants) > 0 {
|
||||
c.log.Info("evicted cache content not checked with upstream recently",
|
||||
"sources", len(sources), "variants", len(variants))
|
||||
}
|
||||
|
||||
if len(sources) < evictionBatchSize && len(variants) < evictionBatchSize {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// evictBatch fetches one batch of LRU candidates across variants and
|
||||
// source blobs and evicts them oldest-first until excessBytes are
|
||||
// freed or the batch is exhausted. It returns the bytes freed. A
|
||||
@@ -344,8 +394,9 @@ func (c *Cache) notifyWritePressure() {
|
||||
}
|
||||
|
||||
// StartEviction launches the background eviction goroutine, which
|
||||
// removes the expired negative cache rows at startup and again on every
|
||||
// periodic tick thereafter, reconciles the database accounting with the
|
||||
// removes the expired negative cache rows, and the sources and variants not
|
||||
// checked for EvictionAfterLastCheck, at startup and again on every periodic
|
||||
// tick thereafter, reconciles the database accounting with the
|
||||
// cache directories at startup and again on the first periodic tick
|
||||
// that comes at least ten times as long after a reconciliation pass
|
||||
// ended as that pass took, and evicts to the configured limit on the given
|
||||
@@ -392,6 +443,7 @@ func (c *Cache) evictionLoop(ctx context.Context, interval time.Duration) {
|
||||
|
||||
nextReconciliation := c.runReconciliationPass(ctx)
|
||||
c.runCleanExpired(ctx)
|
||||
c.runEvictNotCheckedRecently(ctx)
|
||||
c.runEvictionPass(ctx)
|
||||
|
||||
ticker := time.NewTicker(interval)
|
||||
@@ -417,6 +469,7 @@ func (c *Cache) evictionLoop(ctx context.Context, interval time.Duration) {
|
||||
}
|
||||
|
||||
c.runCleanExpired(ctx)
|
||||
c.runEvictNotCheckedRecently(ctx)
|
||||
case <-c.evictionPressure:
|
||||
}
|
||||
|
||||
@@ -474,6 +527,21 @@ func (c *Cache) runCleanExpired(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// runEvictNotCheckedRecently runs EvictNotCheckedRecently, logging a failure
|
||||
// instead of propagating it (the loop must keep running). It does nothing
|
||||
// once ctx is cancelled.
|
||||
func (c *Cache) runEvictNotCheckedRecently(ctx context.Context) {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
|
||||
err := c.EvictNotCheckedRecently(ctx)
|
||||
if err != nil {
|
||||
c.log.Warn("evicting cache content not checked recently failed",
|
||||
"error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// reconcileAccounting synchronizes the database size accounting with
|
||||
// the actual contents of the cache directories. It runs at startup and
|
||||
// then on the first periodic eviction tick that comes at least ten times
|
||||
@@ -609,16 +677,18 @@ func (c *Cache) reconcileVariantFile(
|
||||
}
|
||||
|
||||
// adoptVariantFile inserts an accounting row for a variant file that
|
||||
// has none, using the file's size and modification time.
|
||||
// has none, using the file's size and modification time, which is also
|
||||
// the variant's checked_at.
|
||||
func (c *Cache) adoptVariantFile(
|
||||
ctx context.Context, info fs.FileInfo, cacheKey imagerequest.VariantKey,
|
||||
) error {
|
||||
modTime := info.ModTime().UTC().Format(sqliteTimestampLayout)
|
||||
|
||||
_, err := c.db.ExecContext(ctx, `
|
||||
INSERT INTO variant_content (cache_key, size_bytes, last_accessed_at)
|
||||
VALUES (?, ?, ?)
|
||||
`, string(cacheKey), info.Size(), modTime)
|
||||
INSERT INTO variant_content (cache_key, size_bytes, last_accessed_at,
|
||||
checked_at)
|
||||
VALUES (?, ?, ?, ?)
|
||||
`, string(cacheKey), info.Size(), modTime, info.ModTime().UTC())
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to adopt variant file into accounting: %w", err)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user