Check cached sources with upstream after the cache TTL (closes #69)

A cached image was served until eviction, so a change at the same URL was never picked up. After 24 hours a request checks the source with upstream, sending its stored ETag and Last-Modified. A 304 renews it with no download or conversion. New bytes replace it, and older variants are converted again. A failed check serves the cached image and puts the URL in the negative cache. Images not checked for 30 days are evicted. Variant rows keep the source's ETag and Last-Modified, so a variant whose source was evicted is checked the same way. A variant file's modification time is its source's check time when it was made or renewed, so a hit reads no database row. Model: opus-5-5
2026-11-11 10:38:51 +00:00
1 parent 9f1e9f5253
commit 880845f38c
@@ -3,8 +3,12 @@
 //   - the variants made from them
 //   - the negative cache: the source URLs whose fetch failed recently
 //
-// Each image is a file with a row in the database. Eviction removes the
-// least recently used images to keep the cache under its size limit.
+// Each image is a file with a row in the database. A source and its
+// variants are served until the cache TTL has passed since upstream last sent
+// the source or confirmed it unchanged. Then the source is due to be checked
+// with upstream again (see Stale). Eviction removes:
+//   - the least recently used images, to keep the cache under its size limit
+//   - the images not checked with upstream for EvictionAfterLastCheck
 package imgcache

 import (
@@ -13,7 +17,9 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"log/slog"
+	"os"
 	"path/filepath"
 	"time"

@@ -38,11 +44,31 @@ const cleanExpiredBatchSize = 1000
 // cache.
 const DefaultNegativeTTL = 5 * time.Minute

+// DefaultCacheTTL is how long after upstream last sent a cached source, or
+// answered that it had not changed, pixad serves the source and its
+// variants without asking upstream again.
+const DefaultCacheTTL = 24 * time.Hour
+
+// EvictionAfterLastCheck is how long after upstream last sent a cached
+// source, or answered that it had not changed, the source and its variants
+// are evicted (see EvictNotCheckedRecently). Until then a source whose checks
+// fail is served.
+const EvictionAfterLastCheck = 30 * 24 * time.Hour
+
+// ErrStale is returned by Variant for a variant whose file's modification
+// time is CacheTTL or more ago (see Variant).
+var ErrStale = errors.New("cached variant is older than the cache TTL")
+
 // Config holds cache configuration.
 type Config struct {
 	StateDir    string
 	NegativeTTL time.Duration

+	// CacheTTL is how long a cached source and its variants are served
+	// before upstream is asked again whether the source changed. Zero means
+	// DefaultCacheTTL.
+	CacheTTL time.Duration
+
 	// MaxBytes is the disk cache size limit in bytes that eviction
 	// enforces. Zero turns the disk cache off entirely: no cache
 	// directories are created, lookups always miss, stores are no-ops,
@@ -113,6 +139,10 @@ func NewCache(
 		log = slog.Default()
 	}

+	if config.CacheTTL == 0 {
+		config.CacheTTL = DefaultCacheTTL
+	}
+
 	c := &Cache{
 		db:               db,
 		config:           config,
@@ -158,11 +188,13 @@ func NewCache(

 // Variant opens a cached variant and returns a reader, its size and its
 // content type, read from the variant file's first bytes. It opens that one
-// file and makes no database call. A variant it opens has its access time
-// handed to the background writer without waiting (see touchVariant). A
-// variant not in the cache, or any variant of a disabled cache, is
-// ErrNotFound; a variant file whose first bytes are not those of an output
-// format is another error.
+// file and makes no database call: the file's modification time is its
+// source's check time when the variant was made or last renewed (see
+// PutVariant), and a variant whose time is CacheTTL or more ago is ErrStale.
+// A variant it opens has its access time handed to the background writer
+// without waiting (see touchVariant). A variant not in the cache, or any
+// variant of a disabled cache, is ErrNotFound; a variant file whose first
+// bytes are not those of an output format is another error.
 func (c *Cache) Variant(
 	cacheKey imagerequest.VariantKey,
 ) (io.ReadCloser, int64, string, error) {
@@ -170,11 +202,61 @@ func (c *Cache) Variant(
 		return nil, 0, "", ErrNotFound
 	}

-	file, size, err := c.variants.Open(string(cacheKey))
+	file, info, err := c.openVariant(cacheKey)
 	if err != nil {
 		return nil, 0, "", err
 	}

+	if c.Stale(info.ModTime()) {
+		_ = file.Close()
+
+		return nil, 0, "", ErrStale
+	}
+
+	return c.finishOpeningVariant(cacheKey, file, info)
+}
+
+// VariantIgnoringTTL opens a cached variant whatever its age, and returns it
+// as Variant does. The image proxy reads it for a variant whose source is not
+// cached, before it checks the source with upstream (see LookupVariant).
+func (c *Cache) VariantIgnoringTTL(
+	cacheKey imagerequest.VariantKey,
+) (io.ReadCloser, int64, string, error) {
+	if c.config.MaxBytes == 0 {
+		return nil, 0, "", ErrNotFound
+	}
+
+	file, info, err := c.openVariant(cacheKey)
+	if err != nil {
+		return nil, 0, "", err
+	}
+
+	return c.finishOpeningVariant(cacheKey, file, info)
+}
+
+// RenewVariant opens a cached variant made from the source bytes that source
+// describes, whatever its age, and returns it as Variant does. It moves the
+// variant's check time forward to source.CheckedAt (see SetVariantCheckedAt).
+// A variant made before source.FetchedAt was made from older bytes, and is
+// ErrNotFound, as is a variant not in the cache.
+func (c *Cache) RenewVariant(
+	ctx context.Context, cacheKey imagerequest.VariantKey, source SourceInfo,
+) (io.ReadCloser, int64, string, error) {
+	if c.config.MaxBytes == 0 {
+		return nil, 0, "", ErrNotFound
+	}
+
+	file, info, err := c.openVariant(cacheKey)
+	if err != nil {
+		return nil, 0, "", err
+	}
+
+	if info.ModTime().Before(source.FetchedAt) {
+		_ = file.Close()
+
+		return nil, 0, "", ErrNotFound
+	}
+
 	format, err := readFormat(file)
 	if err != nil {
 		_ = file.Close()
@@ -182,17 +264,68 @@ func (c *Cache) Variant(
 		return nil, 0, "", err
 	}

+	if info.ModTime().Before(source.CheckedAt) {
+		err = c.SetVariantCheckedAt(ctx, cacheKey, source.CheckedAt)
+		if err != nil {
+			_ = file.Close()
+
+			return nil, 0, "", err
+		}
+	}
+
 	c.touchVariant(cacheKey)

-	return file, size, format.MIMEType(), nil
+	return file, info.Size(), format.MIMEType(), nil
 }

-// PutVariant stores a processed variant by its cache key and records
-// it in the size accounting. On a disabled cache it is a no-op. The
-// accounting insert is best-effort (the startup reconciliation pass
-// adopts any variant file that misses its accounting row).
+// SetVariantCheckedAt records checkedAt as when upstream last sent the source
+// bytes the cached variant of cacheKey was made from, or answered that they
+// had not changed. It sets the variant file's modification time to checkedAt,
+// so Variant serves the file until CacheTTL after it. It sets the checked_at
+// of the variant's row too, so eviction keeps the variant until
+// EvictionAfterLastCheck after it. A failure to write the row is only logged.
+// On a disabled cache it is a no-op.
+func (c *Cache) SetVariantCheckedAt(
+	ctx context.Context, cacheKey imagerequest.VariantKey, checkedAt time.Time,
+) error {
+	if c.config.MaxBytes == 0 {
+		return nil
+	}
+
+	err := c.variants.SetModTime(string(cacheKey), checkedAt)
+	if err != nil {
+		return err
+	}
+
+	_, err = c.db.ExecContext(ctx,
+		`UPDATE variant_content SET checked_at = ? WHERE cache_key = ?`,
+		checkedAt.UTC(), string(cacheKey))
+	if err != nil {
+		c.log.Warn("failed to renew variant's checked_at",
+			"cache_key", cacheKey, "error", err)
+	}
+
+	return nil
+}
+
+// Stale reports whether a source last checked with upstream at checkedAt,
+// and its variants, are due to be checked again.
+func (c *Cache) Stale(checkedAt time.Time) bool {
+	return time.Since(checkedAt) >= c.config.CacheTTL
+}
+
+// PutVariant stores a processed variant by its cache key and records it in
+// the size accounting. source describes the source bytes the variant was made
+// from. Its CheckedAt becomes the modification time of the variant's file,
+// which Variant and RenewVariant read. The variant's row keeps its CheckedAt,
+// ETag and LastModified (see LookupVariant). On a disabled cache it is a
+// no-op. The accounting insert is best-effort (the startup reconciliation
+// pass adopts any variant file that misses its accounting row).
 func (c *Cache) PutVariant(
-	ctx context.Context, cacheKey imagerequest.VariantKey, content io.Reader,
+	ctx context.Context,
+	cacheKey imagerequest.VariantKey,
+	source SourceInfo,
+	content io.Reader,
 ) error {
 	if c.config.MaxBytes == 0 {
 		return nil
@@ -208,15 +341,33 @@ func (c *Cache) PutVariant(
 		return err
 	}

+	// A file left with the time of its writing could be served as made from
+	// bytes fetched after the ones it was made from
+	err = c.variants.SetModTime(string(cacheKey), source.CheckedAt)
+	if err != nil {
+		deleteErr := c.variants.Delete(string(cacheKey))
+		if deleteErr != nil {
+			c.log.Warn("failed to remove variant file whose time was not set",
+				"cache_key", cacheKey, "error", deleteErr)
+		}
+
+		return err
+	}
+
 	size := int64(len(data))

 	_, err = c.db.ExecContext(ctx, `
-		INSERT INTO variant_content (cache_key, size_bytes)
-		VALUES (?, ?)
+		INSERT INTO variant_content (cache_key, size_bytes, checked_at,
+			upstream_etag, upstream_last_modified)
+		VALUES (?, ?, ?, ?, ?)
 		ON CONFLICT(cache_key) DO UPDATE SET
 			size_bytes = excluded.size_bytes,
-			last_accessed_at = CURRENT_TIMESTAMP
-	`, string(cacheKey), size)
+			last_accessed_at = CURRENT_TIMESTAMP,
+			checked_at = excluded.checked_at,
+			upstream_etag = excluded.upstream_etag,
+			upstream_last_modified = excluded.upstream_last_modified
+	`, string(cacheKey), size, source.CheckedAt.UTC(), source.ETag,
+		source.LastModified)
 	if err != nil {
 		c.log.Warn("failed to record variant in size accounting",
 			"cache_key", cacheKey, "error", err)
@@ -272,16 +423,104 @@ func (c *Cache) Source(
 	return file, size, format, nil
 }

-// PutSource stores fetched source content: it writes the file named by
-// the source's key, then the source's row. When the row cannot be written
-// the file is removed and the store fails, as a file with no row is never
-// served or evicted; if that removal fails too, the next reconciliation
-// pass removes the file. On a disabled cache it is a no-op returning an
-// empty key.
+// SourceInfo is what the cache keeps in a cached source's row besides its
+// size and access time.
+type SourceInfo struct {
+	// FetchedAt is when the cached bytes were fetched.
+	FetchedAt time.Time
+	// CheckedAt is when upstream last sent the bytes or answered that they
+	// had not changed.
+	CheckedAt time.Time
+	// ETag is the ETag header upstream sent with the bytes, or "".
+	ETag string
+	// LastModified is the Last-Modified header upstream sent with the
+	// bytes, or "".
+	LastModified string
+}
+
+// LookupSource returns what the cache keeps about the cached source of src.
+// A source with no row is ErrNotFound, as is any source of a disabled cache.
+func (c *Cache) LookupSource(
+	ctx context.Context, src imagerequest.Source,
+) (SourceInfo, error) {
+	if c.config.MaxBytes == 0 {
+		return SourceInfo{}, ErrNotFound
+	}
+
+	var info SourceInfo
+
+	err := c.db.QueryRowContext(ctx, `
+		SELECT fetched_at, checked_at, upstream_etag, upstream_last_modified
+		FROM source_content WHERE source_key = ?
+	`, string(src.Key())).Scan(
+		&info.FetchedAt, &info.CheckedAt, &info.ETag, &info.LastModified)
+	if errors.Is(err, sql.ErrNoRows) {
+		return SourceInfo{}, ErrNotFound
+	}
+
+	if err != nil {
+		return SourceInfo{}, fmt.Errorf("failed to look up source: %w", err)
+	}
+
+	return info, nil
+}
+
+// LookupVariant returns what the row of the cached variant of cacheKey keeps
+// about the source bytes the variant was made from: the CheckedAt, ETag and
+// LastModified they had when the variant was made or last renewed. The row
+// keeps no FetchedAt, so it is zero. With the ETag and LastModified the image
+// proxy checks the variant with upstream after its source is evicted. A
+// variant with no row is ErrNotFound, as is any variant of a disabled cache.
+func (c *Cache) LookupVariant(
+	ctx context.Context, cacheKey imagerequest.VariantKey,
+) (SourceInfo, error) {
+	if c.config.MaxBytes == 0 {
+		return SourceInfo{}, ErrNotFound
+	}
+
+	var info SourceInfo
+
+	err := c.db.QueryRowContext(ctx, `
+		SELECT checked_at, upstream_etag, upstream_last_modified
+		FROM variant_content WHERE cache_key = ?
+	`, string(cacheKey)).Scan(&info.CheckedAt, &info.ETag, &info.LastModified)
+	if errors.Is(err, sql.ErrNoRows) {
+		return SourceInfo{}, ErrNotFound
+	}
+
+	if err != nil {
+		return SourceInfo{}, fmt.Errorf("failed to look up variant: %w", err)
+	}
+
+	return info, nil
+}
+
+// RenewSource records that upstream answered at checkedAt that the cached
+// source of src had not changed.
+func (c *Cache) RenewSource(
+	ctx context.Context, src imagerequest.Source, checkedAt time.Time,
+) error {
+	_, err := c.db.ExecContext(ctx,
+		`UPDATE source_content SET checked_at = ? WHERE source_key = ?`,
+		checkedAt.UTC(), string(src.Key()))
+	if err != nil {
+		return fmt.Errorf("failed to renew source: %w", err)
+	}
+
+	return nil
+}
+
+// PutSource stores fetched source content with what info says about it: it
+// writes the file named by the source's key, then the source's row. When the
+// row cannot be written the file is removed and the store fails, as a file
+// with no row is never served or evicted; if that removal fails too, the
+// next reconciliation pass removes the file. On a disabled cache it is a
+// no-op returning an empty key.
 func (c *Cache) PutSource(
 	ctx context.Context,
 	src imagerequest.Source,
 	content io.Reader,
+	info SourceInfo,
 ) (imagerequest.SourceKey, error) {
 	if c.config.MaxBytes == 0 {
 		return "", nil
@@ -302,12 +541,18 @@ func (c *Cache) PutSource(
 	size := int64(len(data))

 	_, err = c.db.ExecContext(ctx, `
-		INSERT INTO source_content (source_key, size_bytes)
-		VALUES (?, ?)
+		INSERT INTO source_content (source_key, size_bytes, fetched_at,
+			checked_at, upstream_etag, upstream_last_modified)
+		VALUES (?, ?, ?, ?, ?, ?)
 		ON CONFLICT(source_key) DO UPDATE SET
 			size_bytes = excluded.size_bytes,
-			last_accessed_at = CURRENT_TIMESTAMP
-	`, string(key), size)
+			last_accessed_at = CURRENT_TIMESTAMP,
+			fetched_at = excluded.fetched_at,
+			checked_at = excluded.checked_at,
+			upstream_etag = excluded.upstream_etag,
+			upstream_last_modified = excluded.upstream_last_modified
+	`, string(key), size, info.FetchedAt.UTC(), info.CheckedAt.UTC(), info.ETag,
+		info.LastModified)
 	if err != nil {
 		deleteErr := c.srcContent.Delete(string(key))
 		if deleteErr != nil {
@@ -449,6 +694,45 @@ func (c *Cache) cleanExpiredBatch(ctx context.Context) (int64, error) {
 	return deleted, nil
 }

+// openVariant opens the file of a cached variant and returns it with its
+// information, which gives its size and its modification time.
+func (c *Cache) openVariant(
+	cacheKey imagerequest.VariantKey,
+) (*os.File, fs.FileInfo, error) {
+	file, _, err := c.variants.Open(string(cacheKey))
+	if err != nil {
+		return nil, nil, err
+	}
+
+	info, err := file.Stat()
+	if err != nil {
+		_ = file.Close()
+
+		return nil, nil, fmt.Errorf("failed to stat variant: %w", err)
+	}
+
+	return file, info, nil
+}
+
+// finishOpeningVariant reads the content type of a variant file openVariant
+// opened from its first bytes, hands the variant's access time to the
+// background writer (see touchVariant), and returns the file as Variant does.
+// It closes the file when its first bytes are not those of an output format.
+func (c *Cache) finishOpeningVariant(
+	cacheKey imagerequest.VariantKey, file *os.File, info fs.FileInfo,
+) (io.ReadCloser, int64, string, error) {
+	format, err := readFormat(file)
+	if err != nil {
+		_ = file.Close()
+
+		return nil, 0, "", err
+	}
+
+	c.touchVariant(cacheKey)
+
+	return file, info.Size(), format.MIMEType(), nil
+}
+
 // readFormat reads the first bytes of a source or variant file, without
 // moving its offset, and returns the format they show. Sources and variants
 // are in the same six formats.