Check cached sources with upstream after the cache TTL (closes #69, closes #83)
check / check (push) Canceled after 0s
check / check (push) Canceled after 0s
A cached source and its variants were served until eviction, so a changed image at the same URL was never picked up. Once a source was last checked 24 hours ago, the next request for one of its images asks upstream with If-None-Match and If-Modified-Since, from the ETag and Last-Modified it stored. A 304 renews the source and its variant without a download or a conversion. New bytes replace the source, and each variant made before them is converted again. A failed check serves the cached image and puts the URL in the negative cache. Sources and variants not checked successfully for 30 days are evicted. A variant file's modification time is its source's check time, so a hit still makes no database call. Existing tests change only in call signatures. The test that asserted two-day-old entries are served without asking upstream is replaced by the new ones. Model: opus-5-5
This commit is contained in:
1 parent
9f1e9f5253
commit
8feed69143
28 files changed
+1262
-226
No files matched your search
+223
-26
@@ -13,7 +13,9 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
@@ -38,11 +40,30 @@ const cleanExpiredBatchSize = 1000
|
||||
// cache.
|
||||
const DefaultNegativeTTL = 5 * time.Minute
|
||||
|
||||
// DefaultCacheTTL is how long after upstream last sent a cached source, or
|
||||
// answered that it had not changed, pixad serves the source and its
|
||||
// variants without asking upstream again.
|
||||
const DefaultCacheTTL = 24 * time.Hour
|
||||
|
||||
// CacheMaxAge is how long after upstream last sent a cached source, or
|
||||
// answered that it had not changed, the source and its variants are evicted
|
||||
// (see EvictPastMaxAge). Until then a source whose checks fail is served.
|
||||
const CacheMaxAge = 30 * 24 * time.Hour
|
||||
|
||||
// ErrStale is returned by Variant for a variant whose source was last
|
||||
// checked with upstream CacheTTL or more ago.
|
||||
var ErrStale = errors.New("cached variant is older than the cache TTL")
|
||||
|
||||
// Config holds cache configuration.
|
||||
type Config struct {
|
||||
StateDir string
|
||||
NegativeTTL time.Duration
|
||||
|
||||
// CacheTTL is how long a cached source and its variants are served
|
||||
// before upstream is asked again whether the source changed. Zero means
|
||||
// DefaultCacheTTL.
|
||||
CacheTTL time.Duration
|
||||
|
||||
// MaxBytes is the disk cache size limit in bytes that eviction
|
||||
// enforces. Zero turns the disk cache off entirely: no cache
|
||||
// directories are created, lookups always miss, stores are no-ops,
|
||||
@@ -113,6 +134,10 @@ func NewCache(
|
||||
log = slog.Default()
|
||||
}
|
||||
|
||||
if config.CacheTTL == 0 {
|
||||
config.CacheTTL = DefaultCacheTTL
|
||||
}
|
||||
|
||||
c := &Cache{
|
||||
db: db,
|
||||
config: config,
|
||||
@@ -158,11 +183,13 @@ func NewCache(
|
||||
|
||||
// Variant opens a cached variant and returns a reader, its size and its
|
||||
// content type, read from the variant file's first bytes. It opens that one
|
||||
// file and makes no database call. A variant it opens has its access time
|
||||
// handed to the background writer without waiting (see touchVariant). A
|
||||
// variant not in the cache, or any variant of a disabled cache, is
|
||||
// ErrNotFound; a variant file whose first bytes are not those of an output
|
||||
// format is another error.
|
||||
// file and makes no database call: the file's modification time is when its
|
||||
// source was last checked with upstream (see PutVariant), and a variant
|
||||
// whose source was checked CacheTTL or more ago is ErrStale. A variant it
|
||||
// opens has its access time handed to the background writer without waiting
|
||||
// (see touchVariant). A variant not in the cache, or any variant of a
|
||||
// disabled cache, is ErrNotFound; a variant file whose first bytes are not
|
||||
// those of an output format is another error.
|
||||
func (c *Cache) Variant(
|
||||
cacheKey imagerequest.VariantKey,
|
||||
) (io.ReadCloser, int64, string, error) {
|
||||
@@ -170,11 +197,17 @@ func (c *Cache) Variant(
|
||||
return nil, 0, "", ErrNotFound
|
||||
}
|
||||
|
||||
file, size, err := c.variants.Open(string(cacheKey))
|
||||
file, info, err := c.openVariant(cacheKey)
|
||||
if err != nil {
|
||||
return nil, 0, "", err
|
||||
}
|
||||
|
||||
if c.Stale(info.ModTime()) {
|
||||
_ = file.Close()
|
||||
|
||||
return nil, 0, "", ErrStale
|
||||
}
|
||||
|
||||
format, err := readFormat(file)
|
||||
if err != nil {
|
||||
_ = file.Close()
|
||||
@@ -184,15 +217,81 @@ func (c *Cache) Variant(
|
||||
|
||||
c.touchVariant(cacheKey)
|
||||
|
||||
return file, size, format.MIMEType(), nil
|
||||
return file, info.Size(), format.MIMEType(), nil
|
||||
}
|
||||
|
||||
// PutVariant stores a processed variant by its cache key and records
|
||||
// it in the size accounting. On a disabled cache it is a no-op. The
|
||||
// accounting insert is best-effort (the startup reconciliation pass
|
||||
// adopts any variant file that misses its accounting row).
|
||||
// RenewVariant opens a cached variant made from the source bytes that source
|
||||
// describes, whatever its age, and returns it as Variant does. It moves the
|
||||
// variant file's modification time, and the checked_at of its row, forward
|
||||
// to source.CheckedAt, so Variant serves the file until CacheTTL after that
|
||||
// check and eviction keeps it until CacheMaxAge after it. A variant made
|
||||
// before source.FetchedAt was made from older bytes, and is ErrNotFound, as
|
||||
// is a variant not in the cache.
|
||||
func (c *Cache) RenewVariant(
|
||||
ctx context.Context, cacheKey imagerequest.VariantKey, source SourceInfo,
|
||||
) (io.ReadCloser, int64, string, error) {
|
||||
if c.config.MaxBytes == 0 {
|
||||
return nil, 0, "", ErrNotFound
|
||||
}
|
||||
|
||||
file, info, err := c.openVariant(cacheKey)
|
||||
if err != nil {
|
||||
return nil, 0, "", err
|
||||
}
|
||||
|
||||
if info.ModTime().Before(source.FetchedAt) {
|
||||
_ = file.Close()
|
||||
|
||||
return nil, 0, "", ErrNotFound
|
||||
}
|
||||
|
||||
format, err := readFormat(file)
|
||||
if err != nil {
|
||||
_ = file.Close()
|
||||
|
||||
return nil, 0, "", err
|
||||
}
|
||||
|
||||
if info.ModTime().Before(source.CheckedAt) {
|
||||
err = c.variants.SetModTime(string(cacheKey), source.CheckedAt)
|
||||
if err != nil {
|
||||
_ = file.Close()
|
||||
|
||||
return nil, 0, "", err
|
||||
}
|
||||
|
||||
_, err = c.db.ExecContext(ctx,
|
||||
`UPDATE variant_content SET checked_at = ? WHERE cache_key = ?`,
|
||||
source.CheckedAt.UTC(), string(cacheKey))
|
||||
if err != nil {
|
||||
c.log.Warn("failed to renew variant's checked_at",
|
||||
"cache_key", cacheKey, "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
c.touchVariant(cacheKey)
|
||||
|
||||
return file, info.Size(), format.MIMEType(), nil
|
||||
}
|
||||
|
||||
// Stale reports whether a source last checked with upstream at checkedAt,
|
||||
// and its variants, are due to be checked again.
|
||||
func (c *Cache) Stale(checkedAt time.Time) bool {
|
||||
return time.Since(checkedAt) >= c.config.CacheTTL
|
||||
}
|
||||
|
||||
// PutVariant stores a processed variant by its cache key and records it in
|
||||
// the size accounting. sourceCheckedAt is when the source the variant was
|
||||
// made from was last checked with upstream; it becomes the modification time
|
||||
// of the variant's file, which Variant and RenewVariant read. On a disabled
|
||||
// cache it is a no-op. The accounting insert is best-effort (the startup
|
||||
// reconciliation pass adopts any variant file that misses its accounting
|
||||
// row).
|
||||
func (c *Cache) PutVariant(
|
||||
ctx context.Context, cacheKey imagerequest.VariantKey, content io.Reader,
|
||||
ctx context.Context,
|
||||
cacheKey imagerequest.VariantKey,
|
||||
sourceCheckedAt time.Time,
|
||||
content io.Reader,
|
||||
) error {
|
||||
if c.config.MaxBytes == 0 {
|
||||
return nil
|
||||
@@ -208,15 +307,29 @@ func (c *Cache) PutVariant(
|
||||
return err
|
||||
}
|
||||
|
||||
// A file left with the time of its writing could be served as made from
|
||||
// bytes fetched after the ones it was made from
|
||||
err = c.variants.SetModTime(string(cacheKey), sourceCheckedAt)
|
||||
if err != nil {
|
||||
deleteErr := c.variants.Delete(string(cacheKey))
|
||||
if deleteErr != nil {
|
||||
c.log.Warn("failed to remove variant file whose time was not set",
|
||||
"cache_key", cacheKey, "error", deleteErr)
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
size := int64(len(data))
|
||||
|
||||
_, err = c.db.ExecContext(ctx, `
|
||||
INSERT INTO variant_content (cache_key, size_bytes)
|
||||
VALUES (?, ?)
|
||||
INSERT INTO variant_content (cache_key, size_bytes, checked_at)
|
||||
VALUES (?, ?, ?)
|
||||
ON CONFLICT(cache_key) DO UPDATE SET
|
||||
size_bytes = excluded.size_bytes,
|
||||
last_accessed_at = CURRENT_TIMESTAMP
|
||||
`, string(cacheKey), size)
|
||||
last_accessed_at = CURRENT_TIMESTAMP,
|
||||
checked_at = excluded.checked_at
|
||||
`, string(cacheKey), size, sourceCheckedAt.UTC())
|
||||
if err != nil {
|
||||
c.log.Warn("failed to record variant in size accounting",
|
||||
"cache_key", cacheKey, "error", err)
|
||||
@@ -272,16 +385,74 @@ func (c *Cache) Source(
|
||||
return file, size, format, nil
|
||||
}
|
||||
|
||||
// PutSource stores fetched source content: it writes the file named by
|
||||
// the source's key, then the source's row. When the row cannot be written
|
||||
// the file is removed and the store fails, as a file with no row is never
|
||||
// served or evicted; if that removal fails too, the next reconciliation
|
||||
// pass removes the file. On a disabled cache it is a no-op returning an
|
||||
// empty key.
|
||||
// SourceInfo is what the cache keeps in a cached source's row besides its
|
||||
// size and access time.
|
||||
type SourceInfo struct {
|
||||
// FetchedAt is when the cached bytes were fetched.
|
||||
FetchedAt time.Time
|
||||
// CheckedAt is when upstream last sent the bytes or answered that they
|
||||
// had not changed.
|
||||
CheckedAt time.Time
|
||||
// ETag is the ETag header upstream sent with the bytes, or "".
|
||||
ETag string
|
||||
// LastModified is the Last-Modified header upstream sent with the
|
||||
// bytes, or "".
|
||||
LastModified string
|
||||
}
|
||||
|
||||
// LookupSource returns what the cache keeps about the cached source of src.
|
||||
// A source with no row is ErrNotFound, as is any source of a disabled cache.
|
||||
func (c *Cache) LookupSource(
|
||||
ctx context.Context, src imagerequest.Source,
|
||||
) (SourceInfo, error) {
|
||||
if c.config.MaxBytes == 0 {
|
||||
return SourceInfo{}, ErrNotFound
|
||||
}
|
||||
|
||||
var info SourceInfo
|
||||
|
||||
err := c.db.QueryRowContext(ctx, `
|
||||
SELECT fetched_at, checked_at, upstream_etag, upstream_last_modified
|
||||
FROM source_content WHERE source_key = ?
|
||||
`, string(src.Key())).Scan(
|
||||
&info.FetchedAt, &info.CheckedAt, &info.ETag, &info.LastModified)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return SourceInfo{}, ErrNotFound
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return SourceInfo{}, fmt.Errorf("failed to look up source: %w", err)
|
||||
}
|
||||
|
||||
return info, nil
|
||||
}
|
||||
|
||||
// RenewSource records that upstream answered at checkedAt that the cached
|
||||
// source of src had not changed.
|
||||
func (c *Cache) RenewSource(
|
||||
ctx context.Context, src imagerequest.Source, checkedAt time.Time,
|
||||
) error {
|
||||
_, err := c.db.ExecContext(ctx,
|
||||
`UPDATE source_content SET checked_at = ? WHERE source_key = ?`,
|
||||
checkedAt.UTC(), string(src.Key()))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to renew source: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// PutSource stores fetched source content with what info says about it: it
|
||||
// writes the file named by the source's key, then the source's row. When the
|
||||
// row cannot be written the file is removed and the store fails, as a file
|
||||
// with no row is never served or evicted; if that removal fails too, the
|
||||
// next reconciliation pass removes the file. On a disabled cache it is a
|
||||
// no-op returning an empty key.
|
||||
func (c *Cache) PutSource(
|
||||
ctx context.Context,
|
||||
src imagerequest.Source,
|
||||
content io.Reader,
|
||||
info SourceInfo,
|
||||
) (imagerequest.SourceKey, error) {
|
||||
if c.config.MaxBytes == 0 {
|
||||
return "", nil
|
||||
@@ -302,12 +473,18 @@ func (c *Cache) PutSource(
|
||||
size := int64(len(data))
|
||||
|
||||
_, err = c.db.ExecContext(ctx, `
|
||||
INSERT INTO source_content (source_key, size_bytes)
|
||||
VALUES (?, ?)
|
||||
INSERT INTO source_content (source_key, size_bytes, fetched_at,
|
||||
checked_at, upstream_etag, upstream_last_modified)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(source_key) DO UPDATE SET
|
||||
size_bytes = excluded.size_bytes,
|
||||
last_accessed_at = CURRENT_TIMESTAMP
|
||||
`, string(key), size)
|
||||
last_accessed_at = CURRENT_TIMESTAMP,
|
||||
fetched_at = excluded.fetched_at,
|
||||
checked_at = excluded.checked_at,
|
||||
upstream_etag = excluded.upstream_etag,
|
||||
upstream_last_modified = excluded.upstream_last_modified
|
||||
`, string(key), size, info.FetchedAt.UTC(), info.CheckedAt.UTC(), info.ETag,
|
||||
info.LastModified)
|
||||
if err != nil {
|
||||
deleteErr := c.srcContent.Delete(string(key))
|
||||
if deleteErr != nil {
|
||||
@@ -449,6 +626,26 @@ func (c *Cache) cleanExpiredBatch(ctx context.Context) (int64, error) {
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
// openVariant opens the file of a cached variant and returns it with its
|
||||
// information, which gives its size and its modification time.
|
||||
func (c *Cache) openVariant(
|
||||
cacheKey imagerequest.VariantKey,
|
||||
) (*os.File, fs.FileInfo, error) {
|
||||
file, _, err := c.variants.Open(string(cacheKey))
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
_ = file.Close()
|
||||
|
||||
return nil, nil, fmt.Errorf("failed to stat variant: %w", err)
|
||||
}
|
||||
|
||||
return file, info, nil
|
||||
}
|
||||
|
||||
// readFormat reads the first bytes of a source or variant file, without
|
||||
// moving its offset, and returns the format they show. Sources and variants
|
||||
// are in the same six formats.
|
||||
|
||||
@@ -49,7 +49,7 @@ func TestVariant_ContentTypeFromFirstBytes(t *testing.T) {
|
||||
|
||||
key := imagerequest.VariantKey("aabbccdd10" + string(format))
|
||||
|
||||
err = cache.PutVariant(t.Context(), key, bytes.NewReader(image))
|
||||
err = cache.PutVariant(t.Context(), key, time.Now(), bytes.NewReader(image))
|
||||
if err != nil {
|
||||
t.Fatalf("PutVariant(%s) error = %v", format, err)
|
||||
}
|
||||
|
||||
@@ -131,6 +131,56 @@ func (c *Cache) EvictToLimit(ctx context.Context) error {
|
||||
}
|
||||
}
|
||||
|
||||
// EvictPastMaxAge evicts each source and variant whose checked_at is
|
||||
// CacheMaxAge or more ago, evictionBatchSize of each at a time, until none
|
||||
// is left. It is a no-op when the cache is disabled.
|
||||
func (c *Cache) EvictPastMaxAge(ctx context.Context) error {
|
||||
if c.config.MaxBytes == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
cutoff := time.Now().UTC().Add(-CacheMaxAge)
|
||||
|
||||
for {
|
||||
sources, err := queryStringColumn[imagerequest.SourceKey](ctx, c.db, `
|
||||
SELECT source_key FROM source_content WHERE checked_at < ? LIMIT ?
|
||||
`, "sources past the maximum age", "source key", cutoff, evictionBatchSize)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, key := range sources {
|
||||
err = c.evictSourceBlob(ctx, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
variants, err := queryStringColumn[imagerequest.VariantKey](ctx, c.db, `
|
||||
SELECT cache_key FROM variant_content WHERE checked_at < ? LIMIT ?
|
||||
`, "variants past the maximum age", "variant key", cutoff, evictionBatchSize)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, key := range variants {
|
||||
err = c.evictVariant(ctx, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if len(sources) > 0 || len(variants) > 0 {
|
||||
c.log.Info("evicted cache content past the maximum age",
|
||||
"sources", len(sources), "variants", len(variants))
|
||||
}
|
||||
|
||||
if len(sources) < evictionBatchSize && len(variants) < evictionBatchSize {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// evictBatch fetches one batch of LRU candidates across variants and
|
||||
// source blobs and evicts them oldest-first until excessBytes are
|
||||
// freed or the batch is exhausted. It returns the bytes freed. A
|
||||
@@ -344,8 +394,9 @@ func (c *Cache) notifyWritePressure() {
|
||||
}
|
||||
|
||||
// StartEviction launches the background eviction goroutine, which
|
||||
// removes the expired negative cache rows at startup and again on every
|
||||
// periodic tick thereafter, reconciles the database accounting with the
|
||||
// removes the expired negative cache rows, and the sources and variants
|
||||
// past CacheMaxAge, at startup and again on every periodic tick
|
||||
// thereafter, reconciles the database accounting with the
|
||||
// cache directories at startup and again on the first periodic tick
|
||||
// that comes at least ten times as long after a reconciliation pass
|
||||
// ended as that pass took, and evicts to the configured limit on the given
|
||||
@@ -392,6 +443,7 @@ func (c *Cache) evictionLoop(ctx context.Context, interval time.Duration) {
|
||||
|
||||
nextReconciliation := c.runReconciliationPass(ctx)
|
||||
c.runCleanExpired(ctx)
|
||||
c.runEvictPastMaxAge(ctx)
|
||||
c.runEvictionPass(ctx)
|
||||
|
||||
ticker := time.NewTicker(interval)
|
||||
@@ -417,6 +469,7 @@ func (c *Cache) evictionLoop(ctx context.Context, interval time.Duration) {
|
||||
}
|
||||
|
||||
c.runCleanExpired(ctx)
|
||||
c.runEvictPastMaxAge(ctx)
|
||||
case <-c.evictionPressure:
|
||||
}
|
||||
|
||||
@@ -474,6 +527,20 @@ func (c *Cache) runCleanExpired(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// runEvictPastMaxAge runs EvictPastMaxAge, logging a failure instead of
|
||||
// propagating it (the loop must keep running). It does nothing once ctx is
|
||||
// cancelled.
|
||||
func (c *Cache) runEvictPastMaxAge(ctx context.Context) {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
|
||||
err := c.EvictPastMaxAge(ctx)
|
||||
if err != nil {
|
||||
c.log.Warn("eviction past the maximum age failed", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// reconcileAccounting synchronizes the database size accounting with
|
||||
// the actual contents of the cache directories. It runs at startup and
|
||||
// then on the first periodic eviction tick that comes at least ten times
|
||||
@@ -609,16 +676,18 @@ func (c *Cache) reconcileVariantFile(
|
||||
}
|
||||
|
||||
// adoptVariantFile inserts an accounting row for a variant file that
|
||||
// has none, using the file's size and modification time.
|
||||
// has none, using the file's size and modification time, which is also
|
||||
// the variant's checked_at.
|
||||
func (c *Cache) adoptVariantFile(
|
||||
ctx context.Context, info fs.FileInfo, cacheKey imagerequest.VariantKey,
|
||||
) error {
|
||||
modTime := info.ModTime().UTC().Format(sqliteTimestampLayout)
|
||||
|
||||
_, err := c.db.ExecContext(ctx, `
|
||||
INSERT INTO variant_content (cache_key, size_bytes, last_accessed_at)
|
||||
VALUES (?, ?, ?)
|
||||
`, string(cacheKey), info.Size(), modTime)
|
||||
INSERT INTO variant_content (cache_key, size_bytes, last_accessed_at,
|
||||
checked_at)
|
||||
VALUES (?, ?, ?, ?)
|
||||
`, string(cacheKey), info.Size(), modTime, info.ModTime().UTC())
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to adopt variant file into accounting: %w", err)
|
||||
}
|
||||
|
||||
@@ -101,7 +101,7 @@ func storeEvictionTestSource(
|
||||
}
|
||||
|
||||
key, err := cache.PutSource(
|
||||
context.Background(), req.Source, bytes.NewReader(content),
|
||||
context.Background(), req.Source, bytes.NewReader(content), SourceInfo{},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("PutSource(%s%s) failed: %v", host, path, err)
|
||||
@@ -117,7 +117,7 @@ func storeEvictionTestVariant(
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
err := cache.PutVariant(t.Context(), key, bytes.NewReader(content))
|
||||
err := cache.PutVariant(t.Context(), key, time.Now(), bytes.NewReader(content))
|
||||
if err != nil {
|
||||
t.Fatalf("PutVariant(%s) failed: %v", key, err)
|
||||
}
|
||||
@@ -432,13 +432,14 @@ func assertDisabledCacheWritesAreNoOps(
|
||||
ctx := t.Context()
|
||||
|
||||
err := cache.PutVariant(
|
||||
ctx, imagerequest.CacheKey(req), bytes.NewReader([]byte("data")),
|
||||
ctx, imagerequest.CacheKey(req), time.Now(), bytes.NewReader([]byte("data")),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("PutVariant on disabled cache must be a no-op, got error: %v", err)
|
||||
}
|
||||
|
||||
key, err := cache.PutSource(ctx, req.Source, bytes.NewReader([]byte("data")))
|
||||
key, err := cache.PutSource(ctx, req.Source, bytes.NewReader([]byte("data")),
|
||||
SourceInfo{})
|
||||
if err != nil {
|
||||
t.Fatalf("PutSource on disabled cache must be a no-op, got error: %v", err)
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Storage constants.
|
||||
@@ -91,6 +92,17 @@ func (s *FileStore) Size(key string) (int64, error) {
|
||||
return stat.Size(), nil
|
||||
}
|
||||
|
||||
// SetModTime sets the modification time of the file for key to modTime,
|
||||
// leaving its access time as it is.
|
||||
func (s *FileStore) SetModTime(key string, modTime time.Time) error {
|
||||
err := os.Chtimes(s.keyToPath(key), time.Time{}, modTime)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to set modification time: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete removes the file for key. A missing file is not an error.
|
||||
func (s *FileStore) Delete(key string) error {
|
||||
err := os.Remove(s.keyToPath(key))
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
package imgcache
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
)
|
||||
|
||||
// TestEvictPastMaxAge stores a source and a variant last checked with
|
||||
// upstream 31 days ago and a source and a variant checked 29 days ago.
|
||||
// EvictPastMaxAge must remove the rows and files of the first two and keep
|
||||
// the other two.
|
||||
func TestEvictPastMaxAge(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cache, _ := newEvictionTestCache(t, 1<<30)
|
||||
old := time.Now().Add(-31 * 24 * time.Hour)
|
||||
recent := time.Now().Add(-29 * 24 * time.Hour)
|
||||
|
||||
oldSource := imagerequest.Source{Host: testHostCDN, Path: "/old.jpg"}
|
||||
recentSource := imagerequest.Source{Host: testHostCDN, Path: "/recent.jpg"}
|
||||
|
||||
for src, checkedAt := range map[imagerequest.Source]time.Time{
|
||||
oldSource: old, recentSource: recent,
|
||||
} {
|
||||
_, err := cache.PutSource(t.Context(), src, bytes.NewReader([]byte("source")),
|
||||
SourceInfo{FetchedAt: checkedAt, CheckedAt: checkedAt})
|
||||
if err != nil {
|
||||
t.Fatalf("PutSource(%s) error = %v", src.Path, err)
|
||||
}
|
||||
}
|
||||
|
||||
oldVariant := imagerequest.VariantKey("aabbccdd0001")
|
||||
recentVariant := imagerequest.VariantKey("aabbccdd0002")
|
||||
|
||||
for key, checkedAt := range map[imagerequest.VariantKey]time.Time{
|
||||
oldVariant: old, recentVariant: recent,
|
||||
} {
|
||||
err := cache.PutVariant(t.Context(), key, checkedAt,
|
||||
bytes.NewReader([]byte("variant")))
|
||||
if err != nil {
|
||||
t.Fatalf("PutVariant(%s) error = %v", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
err := cache.EvictPastMaxAge(t.Context())
|
||||
if err != nil {
|
||||
t.Fatalf("EvictPastMaxAge() error = %v", err)
|
||||
}
|
||||
|
||||
sourceRows, err := queryStringColumn[imagerequest.SourceKey](t.Context(),
|
||||
cache.db, `SELECT source_key FROM source_content`, "sources", "source")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
variantRows, err := queryStringColumn[imagerequest.VariantKey](t.Context(),
|
||||
cache.db, `SELECT cache_key FROM variant_content`, "variants", "variant")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Logf("source rows left = %v, variant rows left = %v", sourceRows, variantRows)
|
||||
|
||||
if !slices.Equal(sourceRows, []imagerequest.SourceKey{recentSource.Key()}) {
|
||||
t.Errorf("source rows left = %v, want only %s", sourceRows, recentSource.Key())
|
||||
}
|
||||
|
||||
if !slices.Equal(variantRows, []imagerequest.VariantKey{recentVariant}) {
|
||||
t.Errorf("variant rows left = %v, want only %s", variantRows, recentVariant)
|
||||
}
|
||||
|
||||
for path, wantKept := range map[string]bool{
|
||||
cache.srcContent.keyToPath(string(oldSource.Key())): false,
|
||||
cache.srcContent.keyToPath(string(recentSource.Key())): true,
|
||||
cache.variants.keyToPath(string(oldVariant)): false,
|
||||
cache.variants.keyToPath(string(recentVariant)): true,
|
||||
} {
|
||||
_, statErr := os.Stat(path)
|
||||
if kept := statErr == nil; kept != wantKept {
|
||||
t.Errorf("file %s kept = %t, want %t (%v)", path, kept, wantKept, statErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -128,7 +128,7 @@ func TestPutSource_FailedRowWriteLeavesNoFile(t *testing.T) {
|
||||
Source: imagerequest.Source{Host: testHostCDN, Path: testPathCat},
|
||||
}
|
||||
key, err := cache.PutSource(t.Context(), req.Source,
|
||||
bytes.NewReader([]byte("source bytes")))
|
||||
bytes.NewReader([]byte("source bytes")), SourceInfo{})
|
||||
t.Logf("PutSource() = %q, error = %v", key, err)
|
||||
|
||||
if err == nil {
|
||||
@@ -170,7 +170,7 @@ func TestSource_FormatFromFirstBytes(t *testing.T) {
|
||||
t.Fatalf("reading the %s image: %v", format, err)
|
||||
}
|
||||
|
||||
_, err = cache.PutSource(t.Context(), src, bytes.NewReader(image))
|
||||
_, err = cache.PutSource(t.Context(), src, bytes.NewReader(image), SourceInfo{})
|
||||
if err != nil {
|
||||
t.Fatalf("PutSource(%s) error = %v", format, err)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user