Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
049a051596 |
No files matched your search
@@ -236,9 +236,7 @@ The image cache keeps these tables in the database:
|
|||||||
- `variant_content`: a row for each variant, by variant key (column
|
- `variant_content`: a row for each variant, by variant key (column
|
||||||
`cache_key`), with:
|
`cache_key`), with:
|
||||||
- its size and when it was last used
|
- its size and when it was last used
|
||||||
- its source's check time when the variant was made or last renewed
|
- when its source was last checked with upstream
|
||||||
- the `ETag` and `Last-Modified` headers upstream sent with the source it
|
|
||||||
was made from
|
|
||||||
- `cache_usage`: the total size of the cached sources and variants, which
|
- `cache_usage`: the total size of the cached sources and variants, which
|
||||||
eviction reads
|
eviction reads
|
||||||
- `negative_cache`: a row for each source URL in the negative cache, with when
|
- `negative_cache`: a row for each source URL in the negative cache, with when
|
||||||
@@ -309,21 +307,16 @@ source or confirmed it unchanged. So a source whose checks keep failing is
|
|||||||
served for at most 30 days. Eviction to stay under the disk cache limit can
|
served for at most 30 days. Eviction to stay under the disk cache limit can
|
||||||
remove images sooner.
|
remove images sooner.
|
||||||
|
|
||||||
That eviction can remove a source and keep its variants. Each variant's row
|
That eviction can remove a source and keep its variants. A request for such a
|
||||||
keeps the `ETag` and `Last-Modified` of the source it was made from. So pixa
|
variant after 24 hours fetches the whole image again, since pixa no longer has
|
||||||
checks such a variant with upstream in the same way after 24 hours:
|
the source's `ETag` or `Last-Modified`. pixa then converts the variant again.
|
||||||
|
When that fetch fails, pixa serves the cached variant and puts the source URL in
|
||||||
|
the negative cache, as for any failed check.
|
||||||
|
|
||||||
- When upstream answers 304 Not Modified, pixa serves the cached variant for
|
A variant file's modification time is when its source was last checked with
|
||||||
another 24 hours. Nothing is downloaded or converted again.
|
upstream. So a cache hit still reads no database row to know whether the variant
|
||||||
- When upstream sends the image, pixa caches it as the source and converts the
|
is older than 24 hours. A request that checks its source with upstream counts as
|
||||||
variant again.
|
a miss.
|
||||||
- When the check fails, pixa serves the cached variant and puts the source URL
|
|
||||||
in the negative cache.
|
|
||||||
|
|
||||||
A variant file's modification time is its source's check time when the variant
|
|
||||||
was made or last renewed. So a cache hit reads no database row to know whether
|
|
||||||
the variant is older than 24 hours. A request that checks its source with
|
|
||||||
upstream counts as a miss.
|
|
||||||
|
|
||||||
### Routes
|
### Routes
|
||||||
|
|
||||||
|
|||||||
@@ -37,14 +37,13 @@ P2: security: per-origin rate limiting
|
|||||||
`ETag` and `Last-Modified` upstream sent.
|
`ETag` and `Last-Modified` upstream sent.
|
||||||
- The check sends them as `If-None-Match` and `If-Modified-Since`. A 304
|
- The check sends them as `If-None-Match` and `If-Modified-Since`. A 304
|
||||||
renews the source; new bytes replace it.
|
renews the source; new bytes replace it.
|
||||||
- A variant file's modification time is its source's `checked_at` when the
|
- A variant file's modification time is its source's `checked_at`, so a hit
|
||||||
variant was made or last renewed, so a hit makes no database call. A
|
still makes no database call. A variant made before the source's
|
||||||
variant made before the source's `fetched_at` is converted again.
|
`fetched_at` is converted again.
|
||||||
- `variant_content` keeps the `ETag` and `Last-Modified` of the source a
|
|
||||||
variant was made from. A variant whose source was evicted is checked with
|
|
||||||
them, and a 304 renews the variant.
|
|
||||||
- A failed check serves the cached image and puts the source URL in the
|
- A failed check serves the cached image and puts the source URL in the
|
||||||
negative cache, so upstream is asked at most once per 5 minutes.
|
negative cache, so upstream is asked at most once per 5 minutes. A variant
|
||||||
|
whose source was evicted is checked with a full fetch. When that fetch
|
||||||
|
fails, the cached variant is served the same way.
|
||||||
- The eviction goroutine evicts sources and variants whose `checked_at` is
|
- The eviction goroutine evicts sources and variants whose `checked_at` is
|
||||||
30 days old (`EvictionAfterLastCheck`).
|
30 days old (`EvictionAfterLastCheck`).
|
||||||
- 2026-10-27 the reconciliation pass skips a directory it cannot list and a file
|
- 2026-10-27 the reconciliation pass skips a directory it cannot list and a file
|
||||||
|
|||||||
@@ -31,16 +31,11 @@ CREATE INDEX IF NOT EXISTS idx_source_content_checked
|
|||||||
-- later, in a batch. checked_at is the checked_at of the source the
|
-- later, in a batch. checked_at is the checked_at of the source the
|
||||||
-- variant was made from, when it was made or last renewed; the variant
|
-- variant was made from, when it was made or last renewed; the variant
|
||||||
-- file's modification time holds the same time for cache hits.
|
-- file's modification time holds the same time for cache hits.
|
||||||
-- upstream_etag and upstream_last_modified are those of the source bytes
|
|
||||||
-- the variant was made from, so the variant can still be checked with
|
|
||||||
-- upstream after its source is evicted.
|
|
||||||
CREATE TABLE IF NOT EXISTS variant_content (
|
CREATE TABLE IF NOT EXISTS variant_content (
|
||||||
cache_key TEXT PRIMARY KEY,
|
cache_key TEXT PRIMARY KEY,
|
||||||
size_bytes INTEGER NOT NULL,
|
size_bytes INTEGER NOT NULL,
|
||||||
last_accessed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
last_accessed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
checked_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
checked_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||||
upstream_etag TEXT NOT NULL DEFAULT '',
|
|
||||||
upstream_last_modified TEXT NOT NULL DEFAULT ''
|
|
||||||
);
|
);
|
||||||
CREATE INDEX IF NOT EXISTS idx_variant_content_last_accessed
|
CREATE INDEX IF NOT EXISTS idx_variant_content_last_accessed
|
||||||
ON variant_content(last_accessed_at);
|
ON variant_content(last_accessed_at);
|
||||||
|
|||||||
@@ -184,30 +184,22 @@ func TestProxy_Get_FreshVariantAsksUpstreamNothing(t *testing.T) {
|
|||||||
|
|
||||||
// TestProxy_Get_UnchangedSourceAfterTTLIsNotDownloadedAgain caches a variant,
|
// TestProxy_Get_UnchangedSourceAfterTTLIsNotDownloadedAgain caches a variant,
|
||||||
// sets its source's check time and the variant's file time two days back,
|
// sets its source's check time and the variant's file time two days back,
|
||||||
// and requests it again, with the source still cached and with the source's
|
// and requests it again. Upstream must be asked with the validator it sent,
|
||||||
// row and file removed, as eviction does. Upstream must be asked with the
|
// answer 304, and send no bytes; the variant must be served again without
|
||||||
// validator it sent, answer 304, and send no bytes; the variant must be served
|
// being made again, and then be served from the cache until the TTL passes.
|
||||||
// again without being made again, and then be served from the cache until the
|
|
||||||
// TTL passes.
|
|
||||||
func TestProxy_Get_UnchangedSourceAfterTTLIsNotDownloadedAgain(t *testing.T) {
|
func TestProxy_Get_UnchangedSourceAfterTTLIsNotDownloadedAgain(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
sourceEvicted bool
|
|
||||||
etag string
|
etag string
|
||||||
lastModified string
|
lastModified string
|
||||||
wantIfNoneMatch string
|
wantIfNoneMatch string
|
||||||
wantIfModifiedSince string
|
wantIfModifiedSince string
|
||||||
}{
|
}{
|
||||||
{"ETag", false, testETag, "", testETag, ""},
|
{"ETag", testETag, "", testETag, ""},
|
||||||
{"Last-Modified only", false, "", testLastModified, "", testLastModified},
|
{"Last-Modified only", "", testLastModified, "", testLastModified},
|
||||||
{"both", false, testETag, testLastModified, testETag, testLastModified},
|
{"both", testETag, testLastModified, testETag, testLastModified},
|
||||||
{"ETag, source evicted", true, testETag, "", testETag, ""},
|
|
||||||
{"Last-Modified only, source evicted", true, "", testLastModified, "",
|
|
||||||
testLastModified},
|
|
||||||
{"both, source evicted", true, testETag, testLastModified, testETag,
|
|
||||||
testLastModified},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
@@ -222,10 +214,6 @@ func TestProxy_Get_UnchangedSourceAfterTTLIsNotDownloadedAgain(t *testing.T) {
|
|||||||
first, _ := getImage(t, svc, variant)
|
first, _ := getImage(t, svc, variant)
|
||||||
setBackSourceAndVariant(t, svc, variant, time.Now().AddDate(0, 0, -2))
|
setBackSourceAndVariant(t, svc, variant, time.Now().AddDate(0, 0, -2))
|
||||||
|
|
||||||
if tt.sourceEvicted {
|
|
||||||
evictSource(t, svc, variant.Source)
|
|
||||||
}
|
|
||||||
|
|
||||||
second, status := getImage(t, svc, variant)
|
second, status := getImage(t, svc, variant)
|
||||||
|
|
||||||
ifNoneMatch, ifModifiedSince := upstream.lastConditionalHeaders()
|
ifNoneMatch, ifModifiedSince := upstream.lastConditionalHeaders()
|
||||||
|
|||||||
@@ -349,8 +349,8 @@ func (p *Proxy) loadCachedSource(
|
|||||||
// from them. When it did not change, or the check fails, the cached source
|
// from them. When it did not change, or the check fails, the cached source
|
||||||
// is used: the variant's cached file is served again if it was made from the
|
// is used: the variant's cached file is served again if it was made from the
|
||||||
// cached source's bytes, and otherwise the variant is made from them. When
|
// cached source's bytes, and otherwise the variant is made from them. When
|
||||||
// the source is not cached but the variant is, the variant is checked with
|
// the source is not cached, a failed fetch still serves the variant's cached
|
||||||
// upstream the same way (see fetchAndProcess).
|
// file (see fetchAndProcess).
|
||||||
func (p *Proxy) processFromSourceOrFetch(
|
func (p *Proxy) processFromSourceOrFetch(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
req *imagerequest.Request,
|
req *imagerequest.Request,
|
||||||
@@ -396,8 +396,8 @@ func (p *Proxy) processFromSourceOrFetch(
|
|||||||
// Process using cached source; nothing was fetched from upstream. The
|
// Process using cached source; nothing was fetched from upstream. The
|
||||||
// image processor reads the source only once it has a processing slot,
|
// image processor reads the source only once it has a processing slot,
|
||||||
// so a request waiting for one holds none of it in memory.
|
// so a request waiting for one holds none of it in memory.
|
||||||
return p.processAndStore(ctx, req, cacheKey, cached, source, sourceFormat,
|
return p.processAndStore(ctx, req, cacheKey, cached.CheckedAt,
|
||||||
sourceSize)
|
source, sourceFormat, sourceSize)
|
||||||
}
|
}
|
||||||
|
|
||||||
// recordFailedCheck logs a failed check of req's source with upstream, after
|
// recordFailedCheck logs a failed check of req's source with upstream, after
|
||||||
@@ -453,13 +453,10 @@ func (p *Proxy) renewVariant(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// cachedVariantIgnoringTTL returns the cached variant of cacheKey whatever
|
// cachedVariantIgnoringTTL returns the cached variant of cacheKey whatever
|
||||||
// its age, with what its row keeps about the source bytes it was made from
|
// its age. It returns nil when the variant is not cached or cannot be read.
|
||||||
// (see imgcache.Cache.LookupVariant). It returns nil when the variant is not
|
|
||||||
// cached or cannot be read. The SourceInfo is empty when the variant is not
|
|
||||||
// returned, or its row is missing or cannot be read.
|
|
||||||
func (p *Proxy) cachedVariantIgnoringTTL(
|
func (p *Proxy) cachedVariantIgnoringTTL(
|
||||||
ctx context.Context, cacheKey imagerequest.VariantKey,
|
cacheKey imagerequest.VariantKey,
|
||||||
) (*processedVariant, imgcache.SourceInfo) {
|
) *processedVariant {
|
||||||
reader, _, contentType, err := p.cache.VariantIgnoringTTL(cacheKey)
|
reader, _, contentType, err := p.cache.VariantIgnoringTTL(cacheKey)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if !errors.Is(err, imgcache.ErrNotFound) {
|
if !errors.Is(err, imgcache.ErrNotFound) {
|
||||||
@@ -467,21 +464,10 @@ func (p *Proxy) cachedVariantIgnoringTTL(
|
|||||||
"error", err)
|
"error", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil, imgcache.SourceInfo{}
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
variant := p.readCachedVariant(cacheKey, reader, contentType)
|
return p.readCachedVariant(cacheKey, reader, contentType)
|
||||||
if variant == nil {
|
|
||||||
return nil, imgcache.SourceInfo{}
|
|
||||||
}
|
|
||||||
|
|
||||||
madeFrom, err := p.cache.LookupVariant(ctx, cacheKey)
|
|
||||||
if err != nil && !errors.Is(err, imgcache.ErrNotFound) {
|
|
||||||
p.log.Warn("failed to look up cached variant", "cache_key", cacheKey,
|
|
||||||
"error", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return variant, madeFrom
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// readCachedVariant reads the cached variant of cacheKey that reader opened,
|
// readCachedVariant reads the cached variant of cacheKey that reader opened,
|
||||||
@@ -504,40 +490,28 @@ func (p *Proxy) readCachedVariant(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// fetchAndProcess fetches the source from upstream, stores it, and makes
|
// fetchAndProcess fetches the source from upstream, stores it, and makes
|
||||||
// and stores the variant from it.
|
// and stores the variant from it. When the fetch fails while the variant is
|
||||||
//
|
// still cached, as it can be after eviction removed its source, the cached
|
||||||
// The variant can still be cached after eviction removed its source. The
|
// variant is served whatever its age, and the failure is recorded as a failed
|
||||||
// fetch then sends the ETag and Last-Modified of the source bytes the variant
|
// check (see recordFailedCheck).
|
||||||
// was made from, so upstream sends the source only if it changed since. When
|
|
||||||
// it did not change, the cached variant is renewed and served, with no
|
|
||||||
// download and no conversion. When the fetch fails, the cached variant is
|
|
||||||
// served whatever its age, and the failure is recorded as a failed check (see
|
|
||||||
// recordFailedCheck).
|
|
||||||
func (p *Proxy) fetchAndProcess(
|
func (p *Proxy) fetchAndProcess(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
req *imagerequest.Request,
|
req *imagerequest.Request,
|
||||||
cacheKey imagerequest.VariantKey,
|
cacheKey imagerequest.VariantKey,
|
||||||
) (*processedVariant, error) {
|
) (*processedVariant, error) {
|
||||||
variant, madeFrom := p.cachedVariantIgnoringTTL(ctx, cacheKey)
|
fetched, err := p.fetchSource(ctx, req, "", "")
|
||||||
|
if err != nil {
|
||||||
fetched, err := p.fetchSource(ctx, req, madeFrom.ETag, madeFrom.LastModified)
|
variant := p.cachedVariantIgnoringTTL(cacheKey)
|
||||||
|
if variant == nil {
|
||||||
switch {
|
return nil, err
|
||||||
case err == nil:
|
|
||||||
return p.processFetched(ctx, req, cacheKey, fetched)
|
|
||||||
case variant == nil:
|
|
||||||
return nil, err
|
|
||||||
case errors.Is(err, errSourceNotModified):
|
|
||||||
err = p.cache.SetVariantCheckedAt(ctx, cacheKey, time.Now().UTC())
|
|
||||||
if err != nil {
|
|
||||||
p.log.Warn("failed to renew cached variant", "cache_key", cacheKey,
|
|
||||||
"error", err)
|
|
||||||
}
|
}
|
||||||
default:
|
|
||||||
p.recordFailedCheck(ctx, req, err)
|
p.recordFailedCheck(ctx, req, err)
|
||||||
|
|
||||||
|
return variant, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return variant, nil
|
return p.processFetched(ctx, req, cacheKey, fetched)
|
||||||
}
|
}
|
||||||
|
|
||||||
// processFetched makes and stores the variant from the source fetchSource
|
// processFetched makes and stores the variant from the source fetchSource
|
||||||
@@ -554,7 +528,7 @@ func (p *Proxy) processFetched(
|
|||||||
// processing slot.
|
// processing slot.
|
||||||
defer func() { _ = fetched.response.Close() }()
|
defer func() { _ = fetched.response.Close() }()
|
||||||
|
|
||||||
return p.processAndStore(ctx, req, cacheKey, fetched.info,
|
return p.processAndStore(ctx, req, cacheKey, fetched.info.CheckedAt,
|
||||||
bytes.NewReader(fetched.data), fetched.format, int64(len(fetched.data)))
|
bytes.NewReader(fetched.data), fetched.format, int64(len(fetched.data)))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -748,13 +722,13 @@ type processedVariant struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// processAndStore processes the image read from source, which is in
|
// processAndStore processes the image read from source, which is in
|
||||||
// sourceFormat, and stores the result. sourceInfo describes the source's
|
// sourceFormat, and stores the result. sourceCheckedAt is when the source
|
||||||
// bytes, and is stored with the variant (see imgcache.Cache.PutVariant).
|
// was last checked with upstream (see imgcache.Cache.PutVariant).
|
||||||
func (p *Proxy) processAndStore(
|
func (p *Proxy) processAndStore(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
req *imagerequest.Request,
|
req *imagerequest.Request,
|
||||||
cacheKey imagerequest.VariantKey,
|
cacheKey imagerequest.VariantKey,
|
||||||
sourceInfo imgcache.SourceInfo,
|
sourceCheckedAt time.Time,
|
||||||
source io.Reader,
|
source io.Reader,
|
||||||
sourceFormat imagerequest.Format,
|
sourceFormat imagerequest.Format,
|
||||||
fetchBytes int64,
|
fetchBytes int64,
|
||||||
@@ -815,7 +789,7 @@ func (p *Proxy) processAndStore(
|
|||||||
)
|
)
|
||||||
|
|
||||||
// Store variant to cache
|
// Store variant to cache
|
||||||
err = p.cache.PutVariant(ctx, cacheKey, sourceInfo,
|
err = p.cache.PutVariant(ctx, cacheKey, sourceCheckedAt,
|
||||||
bytes.NewReader(processedData))
|
bytes.NewReader(processedData))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
p.log.Warn("failed to store variant", "error", err)
|
p.log.Warn("failed to store variant", "error", err)
|
||||||
|
|||||||
+36
-90
@@ -55,8 +55,8 @@ const DefaultCacheTTL = 24 * time.Hour
|
|||||||
// fail is served.
|
// fail is served.
|
||||||
const EvictionAfterLastCheck = 30 * 24 * time.Hour
|
const EvictionAfterLastCheck = 30 * 24 * time.Hour
|
||||||
|
|
||||||
// ErrStale is returned by Variant for a variant whose file's modification
|
// ErrStale is returned by Variant for a variant whose source was last
|
||||||
// time is CacheTTL or more ago (see Variant).
|
// checked with upstream CacheTTL or more ago.
|
||||||
var ErrStale = errors.New("cached variant is older than the cache TTL")
|
var ErrStale = errors.New("cached variant is older than the cache TTL")
|
||||||
|
|
||||||
// Config holds cache configuration.
|
// Config holds cache configuration.
|
||||||
@@ -188,13 +188,13 @@ func NewCache(
|
|||||||
|
|
||||||
// Variant opens a cached variant and returns a reader, its size and its
|
// Variant opens a cached variant and returns a reader, its size and its
|
||||||
// content type, read from the variant file's first bytes. It opens that one
|
// content type, read from the variant file's first bytes. It opens that one
|
||||||
// file and makes no database call: the file's modification time is its
|
// file and makes no database call: the file's modification time is when its
|
||||||
// source's check time when the variant was made or last renewed (see
|
// source was last checked with upstream (see PutVariant), and a variant
|
||||||
// PutVariant), and a variant whose time is CacheTTL or more ago is ErrStale.
|
// whose source was checked CacheTTL or more ago is ErrStale. A variant it
|
||||||
// A variant it opens has its access time handed to the background writer
|
// opens has its access time handed to the background writer without waiting
|
||||||
// without waiting (see touchVariant). A variant not in the cache, or any
|
// (see touchVariant). A variant not in the cache, or any variant of a
|
||||||
// variant of a disabled cache, is ErrNotFound; a variant file whose first
|
// disabled cache, is ErrNotFound; a variant file whose first bytes are not
|
||||||
// bytes are not those of an output format is another error.
|
// those of an output format is another error.
|
||||||
func (c *Cache) Variant(
|
func (c *Cache) Variant(
|
||||||
cacheKey imagerequest.VariantKey,
|
cacheKey imagerequest.VariantKey,
|
||||||
) (io.ReadCloser, int64, string, error) {
|
) (io.ReadCloser, int64, string, error) {
|
||||||
@@ -217,8 +217,8 @@ func (c *Cache) Variant(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// VariantIgnoringTTL opens a cached variant whatever its age, and returns it
|
// VariantIgnoringTTL opens a cached variant whatever its age, and returns it
|
||||||
// as Variant does. The image proxy reads it for a variant whose source is not
|
// as Variant does. The image proxy serves it when a fetch of the variant's
|
||||||
// cached, before it checks the source with upstream (see LookupVariant).
|
// source fails while the source is not cached.
|
||||||
func (c *Cache) VariantIgnoringTTL(
|
func (c *Cache) VariantIgnoringTTL(
|
||||||
cacheKey imagerequest.VariantKey,
|
cacheKey imagerequest.VariantKey,
|
||||||
) (io.ReadCloser, int64, string, error) {
|
) (io.ReadCloser, int64, string, error) {
|
||||||
@@ -236,8 +236,10 @@ func (c *Cache) VariantIgnoringTTL(
|
|||||||
|
|
||||||
// RenewVariant opens a cached variant made from the source bytes that source
|
// RenewVariant opens a cached variant made from the source bytes that source
|
||||||
// describes, whatever its age, and returns it as Variant does. It moves the
|
// describes, whatever its age, and returns it as Variant does. It moves the
|
||||||
// variant's check time forward to source.CheckedAt (see SetVariantCheckedAt).
|
// variant file's modification time, and the checked_at of its row, forward
|
||||||
// A variant made before source.FetchedAt was made from older bytes, and is
|
// to source.CheckedAt, so Variant serves the file until CacheTTL after that
|
||||||
|
// check and eviction keeps it until EvictionAfterLastCheck after it. A
|
||||||
|
// variant made before source.FetchedAt was made from older bytes, and is
|
||||||
// ErrNotFound, as is a variant not in the cache.
|
// ErrNotFound, as is a variant not in the cache.
|
||||||
func (c *Cache) RenewVariant(
|
func (c *Cache) RenewVariant(
|
||||||
ctx context.Context, cacheKey imagerequest.VariantKey, source SourceInfo,
|
ctx context.Context, cacheKey imagerequest.VariantKey, source SourceInfo,
|
||||||
@@ -265,12 +267,20 @@ func (c *Cache) RenewVariant(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if info.ModTime().Before(source.CheckedAt) {
|
if info.ModTime().Before(source.CheckedAt) {
|
||||||
err = c.SetVariantCheckedAt(ctx, cacheKey, source.CheckedAt)
|
err = c.variants.SetModTime(string(cacheKey), source.CheckedAt)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = file.Close()
|
_ = file.Close()
|
||||||
|
|
||||||
return nil, 0, "", err
|
return nil, 0, "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
_, err = c.db.ExecContext(ctx,
|
||||||
|
`UPDATE variant_content SET checked_at = ? WHERE cache_key = ?`,
|
||||||
|
source.CheckedAt.UTC(), string(cacheKey))
|
||||||
|
if err != nil {
|
||||||
|
c.log.Warn("failed to renew variant's checked_at",
|
||||||
|
"cache_key", cacheKey, "error", err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
c.touchVariant(cacheKey)
|
c.touchVariant(cacheKey)
|
||||||
@@ -278,36 +288,6 @@ func (c *Cache) RenewVariant(
|
|||||||
return file, info.Size(), format.MIMEType(), nil
|
return file, info.Size(), format.MIMEType(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetVariantCheckedAt records checkedAt as when upstream last sent the source
|
|
||||||
// bytes the cached variant of cacheKey was made from, or answered that they
|
|
||||||
// had not changed. It sets the variant file's modification time to checkedAt,
|
|
||||||
// so Variant serves the file until CacheTTL after it. It sets the checked_at
|
|
||||||
// of the variant's row too, so eviction keeps the variant until
|
|
||||||
// EvictionAfterLastCheck after it. A failure to write the row is only logged.
|
|
||||||
// On a disabled cache it is a no-op.
|
|
||||||
func (c *Cache) SetVariantCheckedAt(
|
|
||||||
ctx context.Context, cacheKey imagerequest.VariantKey, checkedAt time.Time,
|
|
||||||
) error {
|
|
||||||
if c.config.MaxBytes == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
err := c.variants.SetModTime(string(cacheKey), checkedAt)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = c.db.ExecContext(ctx,
|
|
||||||
`UPDATE variant_content SET checked_at = ? WHERE cache_key = ?`,
|
|
||||||
checkedAt.UTC(), string(cacheKey))
|
|
||||||
if err != nil {
|
|
||||||
c.log.Warn("failed to renew variant's checked_at",
|
|
||||||
"cache_key", cacheKey, "error", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stale reports whether a source last checked with upstream at checkedAt,
|
// Stale reports whether a source last checked with upstream at checkedAt,
|
||||||
// and its variants, are due to be checked again.
|
// and its variants, are due to be checked again.
|
||||||
func (c *Cache) Stale(checkedAt time.Time) bool {
|
func (c *Cache) Stale(checkedAt time.Time) bool {
|
||||||
@@ -315,16 +295,16 @@ func (c *Cache) Stale(checkedAt time.Time) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// PutVariant stores a processed variant by its cache key and records it in
|
// PutVariant stores a processed variant by its cache key and records it in
|
||||||
// the size accounting. source describes the source bytes the variant was made
|
// the size accounting. sourceCheckedAt is when the source the variant was
|
||||||
// from. Its CheckedAt becomes the modification time of the variant's file,
|
// made from was last checked with upstream; it becomes the modification time
|
||||||
// which Variant and RenewVariant read. The variant's row keeps its CheckedAt,
|
// of the variant's file, which Variant and RenewVariant read. On a disabled
|
||||||
// ETag and LastModified (see LookupVariant). On a disabled cache it is a
|
// cache it is a no-op. The accounting insert is best-effort (the startup
|
||||||
// no-op. The accounting insert is best-effort (the startup reconciliation
|
// reconciliation pass adopts any variant file that misses its accounting
|
||||||
// pass adopts any variant file that misses its accounting row).
|
// row).
|
||||||
func (c *Cache) PutVariant(
|
func (c *Cache) PutVariant(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
cacheKey imagerequest.VariantKey,
|
cacheKey imagerequest.VariantKey,
|
||||||
source SourceInfo,
|
sourceCheckedAt time.Time,
|
||||||
content io.Reader,
|
content io.Reader,
|
||||||
) error {
|
) error {
|
||||||
if c.config.MaxBytes == 0 {
|
if c.config.MaxBytes == 0 {
|
||||||
@@ -343,7 +323,7 @@ func (c *Cache) PutVariant(
|
|||||||
|
|
||||||
// A file left with the time of its writing could be served as made from
|
// A file left with the time of its writing could be served as made from
|
||||||
// bytes fetched after the ones it was made from
|
// bytes fetched after the ones it was made from
|
||||||
err = c.variants.SetModTime(string(cacheKey), source.CheckedAt)
|
err = c.variants.SetModTime(string(cacheKey), sourceCheckedAt)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
deleteErr := c.variants.Delete(string(cacheKey))
|
deleteErr := c.variants.Delete(string(cacheKey))
|
||||||
if deleteErr != nil {
|
if deleteErr != nil {
|
||||||
@@ -357,17 +337,13 @@ func (c *Cache) PutVariant(
|
|||||||
size := int64(len(data))
|
size := int64(len(data))
|
||||||
|
|
||||||
_, err = c.db.ExecContext(ctx, `
|
_, err = c.db.ExecContext(ctx, `
|
||||||
INSERT INTO variant_content (cache_key, size_bytes, checked_at,
|
INSERT INTO variant_content (cache_key, size_bytes, checked_at)
|
||||||
upstream_etag, upstream_last_modified)
|
VALUES (?, ?, ?)
|
||||||
VALUES (?, ?, ?, ?, ?)
|
|
||||||
ON CONFLICT(cache_key) DO UPDATE SET
|
ON CONFLICT(cache_key) DO UPDATE SET
|
||||||
size_bytes = excluded.size_bytes,
|
size_bytes = excluded.size_bytes,
|
||||||
last_accessed_at = CURRENT_TIMESTAMP,
|
last_accessed_at = CURRENT_TIMESTAMP,
|
||||||
checked_at = excluded.checked_at,
|
checked_at = excluded.checked_at
|
||||||
upstream_etag = excluded.upstream_etag,
|
`, string(cacheKey), size, sourceCheckedAt.UTC())
|
||||||
upstream_last_modified = excluded.upstream_last_modified
|
|
||||||
`, string(cacheKey), size, source.CheckedAt.UTC(), source.ETag,
|
|
||||||
source.LastModified)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
c.log.Warn("failed to record variant in size accounting",
|
c.log.Warn("failed to record variant in size accounting",
|
||||||
"cache_key", cacheKey, "error", err)
|
"cache_key", cacheKey, "error", err)
|
||||||
@@ -465,36 +441,6 @@ func (c *Cache) LookupSource(
|
|||||||
return info, nil
|
return info, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// LookupVariant returns what the row of the cached variant of cacheKey keeps
|
|
||||||
// about the source bytes the variant was made from: the CheckedAt, ETag and
|
|
||||||
// LastModified they had when the variant was made or last renewed. The row
|
|
||||||
// keeps no FetchedAt, so it is zero. With the ETag and LastModified the image
|
|
||||||
// proxy checks the variant with upstream after its source is evicted. A
|
|
||||||
// variant with no row is ErrNotFound, as is any variant of a disabled cache.
|
|
||||||
func (c *Cache) LookupVariant(
|
|
||||||
ctx context.Context, cacheKey imagerequest.VariantKey,
|
|
||||||
) (SourceInfo, error) {
|
|
||||||
if c.config.MaxBytes == 0 {
|
|
||||||
return SourceInfo{}, ErrNotFound
|
|
||||||
}
|
|
||||||
|
|
||||||
var info SourceInfo
|
|
||||||
|
|
||||||
err := c.db.QueryRowContext(ctx, `
|
|
||||||
SELECT checked_at, upstream_etag, upstream_last_modified
|
|
||||||
FROM variant_content WHERE cache_key = ?
|
|
||||||
`, string(cacheKey)).Scan(&info.CheckedAt, &info.ETag, &info.LastModified)
|
|
||||||
if errors.Is(err, sql.ErrNoRows) {
|
|
||||||
return SourceInfo{}, ErrNotFound
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return SourceInfo{}, fmt.Errorf("failed to look up variant: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return info, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// RenewSource records that upstream answered at checkedAt that the cached
|
// RenewSource records that upstream answered at checkedAt that the cached
|
||||||
// source of src had not changed.
|
// source of src had not changed.
|
||||||
func (c *Cache) RenewSource(
|
func (c *Cache) RenewSource(
|
||||||
|
|||||||
@@ -49,8 +49,7 @@ func TestVariant_ContentTypeFromFirstBytes(t *testing.T) {
|
|||||||
|
|
||||||
key := imagerequest.VariantKey("aabbccdd10" + string(format))
|
key := imagerequest.VariantKey("aabbccdd10" + string(format))
|
||||||
|
|
||||||
err = cache.PutVariant(t.Context(), key, SourceInfo{CheckedAt: time.Now()},
|
err = cache.PutVariant(t.Context(), key, time.Now(), bytes.NewReader(image))
|
||||||
bytes.NewReader(image))
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("PutVariant(%s) error = %v", format, err)
|
t.Fatalf("PutVariant(%s) error = %v", format, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -40,7 +40,7 @@ func TestEvictNotCheckedRecently(t *testing.T) {
|
|||||||
for key, checkedAt := range map[imagerequest.VariantKey]time.Time{
|
for key, checkedAt := range map[imagerequest.VariantKey]time.Time{
|
||||||
oldVariant: old, recentVariant: recent,
|
oldVariant: old, recentVariant: recent,
|
||||||
} {
|
} {
|
||||||
err := cache.PutVariant(t.Context(), key, SourceInfo{CheckedAt: checkedAt},
|
err := cache.PutVariant(t.Context(), key, checkedAt,
|
||||||
bytes.NewReader([]byte("variant")))
|
bytes.NewReader([]byte("variant")))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("PutVariant(%s) error = %v", key, err)
|
t.Fatalf("PutVariant(%s) error = %v", key, err)
|
||||||
|
|||||||
@@ -117,8 +117,7 @@ func storeEvictionTestVariant(
|
|||||||
) {
|
) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
err := cache.PutVariant(t.Context(), key, SourceInfo{CheckedAt: time.Now()},
|
err := cache.PutVariant(t.Context(), key, time.Now(), bytes.NewReader(content))
|
||||||
bytes.NewReader(content))
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("PutVariant(%s) failed: %v", key, err)
|
t.Fatalf("PutVariant(%s) failed: %v", key, err)
|
||||||
}
|
}
|
||||||
@@ -433,7 +432,7 @@ func assertDisabledCacheWritesAreNoOps(
|
|||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
|
|
||||||
err := cache.PutVariant(
|
err := cache.PutVariant(
|
||||||
ctx, imagerequest.CacheKey(req), SourceInfo{}, bytes.NewReader([]byte("data")),
|
ctx, imagerequest.CacheKey(req), time.Now(), bytes.NewReader([]byte("data")),
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("PutVariant on disabled cache must be a no-op, got error: %v", err)
|
t.Fatalf("PutVariant on disabled cache must be a no-op, got error: %v", err)
|
||||||
|
|||||||
Reference in new issue
Block a user