Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
445ff628fd |
No files matched your search
@@ -233,8 +233,8 @@ These terms are the names the code uses:
|
|||||||
- **Variant**: the image pixa serves for an image request, made from its source
|
- **Variant**: the image pixa serves for an image request, made from its source
|
||||||
with its transform.
|
with its transform.
|
||||||
- **Variant key**: the SHA-256 hash of a variant's source, size, format, quality
|
- **Variant key**: the SHA-256 hash of a variant's source, size, format, quality
|
||||||
and fit. It names the variant's file and its row in the database, and gives
|
and fit. It names the variant's file and its row in the database. The
|
||||||
its `ETag`.
|
variant's `ETag` is made from it and the file's modification time.
|
||||||
- **Signed URL**: a `/v1/image/` URL whose `sig` parameter signs its image
|
- **Signed URL**: a `/v1/image/` URL whose `sig` parameter signs its image
|
||||||
request and its expiry `exp` with `signing_key`.
|
request and its expiry `exp` with `signing_key`.
|
||||||
- **Encrypted URL**: a `/v1/e/` URL whose token holds its image request and any
|
- **Encrypted URL**: a `/v1/e/` URL whose token holds its image request and any
|
||||||
@@ -249,8 +249,8 @@ These terms are the names the code uses:
|
|||||||
- a check of a cached image failed in any way (see Cache TTL).
|
- a check of a cached image failed in any way (see Cache TTL).
|
||||||
- **Image cache**: the part of pixa that keeps the sources, the variants and the
|
- **Image cache**: the part of pixa that keeps the sources, the variants and the
|
||||||
negative cache (see Storage).
|
negative cache (see Storage).
|
||||||
- **Cache TTL**: how long pixa serves a cached source and its variants before it
|
- **Cache TTL**: how long pixa uses a cached source and serves its variants
|
||||||
checks the source with upstream again: 24 hours (see Cache TTL).
|
before it checks the source with upstream again: 24 hours (see Cache TTL).
|
||||||
- **Disk cache limit** (`cache_max_bytes`): the size in bytes that the cached
|
- **Disk cache limit** (`cache_max_bytes`): the size in bytes that the cached
|
||||||
sources and variants may reach together.
|
sources and variants may reach together.
|
||||||
- **Image proxy**: the part of pixa that serves a variant from the image cache
|
- **Image proxy**: the part of pixa that serves a variant from the image cache
|
||||||
@@ -522,7 +522,9 @@ the image after pixa would refuse the URL. A URL with no expiry gets one year.
|
|||||||
`immutable` only stops a client revalidating while its copy is fresh. The
|
`immutable` only stops a client revalidating while its copy is fresh. The
|
||||||
response also carries `X-Pixa-Cache`: `HIT` for an image served from the cache,
|
response also carries `X-Pixa-Cache`: `HIT` for an image served from the cache,
|
||||||
and `MISS` for one made for this request or older than 24 hours (see Cache TTL).
|
and `MISS` for one made for this request or older than 24 hours (see Cache TTL).
|
||||||
The `ETag` is the first 16 characters of the variant key, in quotes.
|
The `ETag` is made from the first 16 characters of the variant key and the
|
||||||
|
variant file's modification time (see Cache TTL). So a variant made again from a
|
||||||
|
changed source gets a new `ETag`.
|
||||||
|
|
||||||
### Signed URLs
|
### Signed URLs
|
||||||
|
|
||||||
@@ -894,16 +896,17 @@ also removes the expired rows of the negative cache every five minutes, at most
|
|||||||
|
|
||||||
### Cache TTL
|
### Cache TTL
|
||||||
|
|
||||||
pixa serves a cached source and its variants for 24 hours after upstream last
|
pixa uses a cached source for 24 hours after upstream last sent it or confirmed
|
||||||
sent the source or confirmed it unchanged. After that, the next request for an
|
it unchanged. During that time it serves the source's variants from the cache.
|
||||||
image of that source checks the source with upstream first:
|
After that, the next request for an image of that source checks the source with
|
||||||
|
upstream first:
|
||||||
|
|
||||||
- pixa asks upstream to send the source only if it changed. It sends the
|
- pixa asks upstream to send the source only if it changed. It sends the
|
||||||
source's `ETag` as `If-None-Match` and its `Last-Modified` as
|
source's `ETag` as `If-None-Match` and its `Last-Modified` as
|
||||||
`If-Modified-Since`, when upstream gave them. With neither, upstream sends the
|
`If-Modified-Since`, when upstream gave them. With neither, upstream sends the
|
||||||
whole image again.
|
whole image again.
|
||||||
- When upstream answers 304 Not Modified, pixa serves the cached source and its
|
- When upstream answers 304 Not Modified, pixa uses the cached source and serves
|
||||||
variants for another 24 hours. Nothing is downloaded or converted again.
|
its variants for another 24 hours. Nothing is downloaded or converted again.
|
||||||
- When upstream sends the image, it replaces the cached source. Each variant
|
- When upstream sends the image, it replaces the cached source. Each variant
|
||||||
made from the old image is converted again from the new one when it is next
|
made from the old image is converted again from the new one when it is next
|
||||||
requested.
|
requested.
|
||||||
@@ -913,8 +916,8 @@ image of that source checks the source with upstream first:
|
|||||||
often it serves the image.
|
often it serves the image.
|
||||||
|
|
||||||
pixa evicts a source and its variants 30 days after upstream last sent the
|
pixa evicts a source and its variants 30 days after upstream last sent the
|
||||||
source or confirmed it unchanged. So a source whose checks keep failing is
|
source or confirmed it unchanged. So pixa uses a source whose checks keep
|
||||||
served for at most 30 days. Eviction to stay under the disk cache limit can
|
failing for at most 30 days. Eviction to stay under the disk cache limit can
|
||||||
remove images sooner.
|
remove images sooner.
|
||||||
|
|
||||||
That eviction can remove a source and keep its variants. Each variant's row
|
That eviction can remove a source and keep its variants. Each variant's row
|
||||||
|
|||||||
@@ -31,7 +31,7 @@ P2: security: per-origin rate limiting
|
|||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
- 2026-11-11 a cached source is checked with upstream once it is 24 hours old
|
- 2026-11-11 a cached source is checked with upstream once it is 24 hours old
|
||||||
(closes #69). Before, it and its variants were served until eviction, so a
|
(closes #69). Before, it was used and its variants served until eviction, so a
|
||||||
changed image at the same URL was never picked up.
|
changed image at the same URL was never picked up.
|
||||||
- `source_content` keeps the source's `fetched_at`, `checked_at`, and the
|
- `source_content` keeps the source's `fetched_at`, `checked_at`, and the
|
||||||
`ETag` and `Last-Modified` upstream sent.
|
`ETag` and `Last-Modified` upstream sent.
|
||||||
@@ -40,6 +40,9 @@ P2: security: per-origin rate limiting
|
|||||||
- A variant file's modification time is its source's `checked_at` when the
|
- A variant file's modification time is its source's `checked_at` when the
|
||||||
variant was made or last renewed, so a hit makes no database call. A
|
variant was made or last renewed, so a hit makes no database call. A
|
||||||
variant made before the source's `fetched_at` is converted again.
|
variant made before the source's `fetched_at` is converted again.
|
||||||
|
- A variant's `ETag` is made from its variant key and its file's
|
||||||
|
modification time, so a variant made again from new bytes gets a new
|
||||||
|
`ETag`.
|
||||||
- `variant_content` keeps the `ETag` and `Last-Modified` of the source a
|
- `variant_content` keeps the `ETag` and `Last-Modified` of the source a
|
||||||
variant was made from. A variant whose source was evicted is checked with
|
variant was made from. A variant whose source was evicted is checked with
|
||||||
them, and a 304 renews the variant.
|
them, and a 304 renews the variant.
|
||||||
|
|||||||
@@ -257,6 +257,90 @@ func TestHandleImage_ConditionalRequest_IfNoneMatch_DifferentETag(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestHandleImage_ConditionalRequest_EarlierETagGetsChangedImage serves an
|
||||||
|
// image, changes it upstream, and requests it again with the ETag first
|
||||||
|
// served, once the cache TTL has passed. The changed image must be served,
|
||||||
|
// not 304 Not Modified.
|
||||||
|
func TestHandleImage_ConditionalRequest_EarlierETagGetsChangedImage(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const host = "goodhost.example.com"
|
||||||
|
|
||||||
|
upstream := fstest.MapFS{
|
||||||
|
host + photoPath: &fstest.MapFile{
|
||||||
|
Data: generateTestJPEG(t, 100, 100, color.RGBA{255, 0, 0, 255}),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// A cache TTL of one nanosecond has passed by the second request
|
||||||
|
cache, err := imgcache.NewCache(setupTestDB(t), imgcache.Config{
|
||||||
|
StateDir: t.TempDir(),
|
||||||
|
NegativeTTL: 5 * time.Minute,
|
||||||
|
CacheTTL: time.Nanosecond,
|
||||||
|
MaxBytes: 1 << 30,
|
||||||
|
}, prometheus.NewRegistry())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("imgcache.NewCache() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
svc, err := imageproxy.New(&imageproxy.Config{
|
||||||
|
Cache: cache,
|
||||||
|
Fetcher: newMockFetcher(upstream),
|
||||||
|
Processor: imageprocessor.New(imageprocessor.Params{
|
||||||
|
MaxInputBytes: 50 << 20,
|
||||||
|
MaxConcurrentProcessing: 2,
|
||||||
|
}),
|
||||||
|
MaxResponseSize: 50 << 20,
|
||||||
|
Logger: slog.New(slog.DiscardHandler),
|
||||||
|
}, prometheus.NewRegistry())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("imageproxy.New() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
h := &Handlers{
|
||||||
|
imgSvc: svc,
|
||||||
|
log: slog.New(slog.DiscardHandler),
|
||||||
|
signer: signature.New(testSigningKey),
|
||||||
|
allowlist: allowlist.New([]string{host}),
|
||||||
|
}
|
||||||
|
|
||||||
|
r := chi.NewRouter()
|
||||||
|
r.Get("/v1/image/*", h.HandleImage())
|
||||||
|
|
||||||
|
target := "/v1/image/" + host + photoPath + "/50x50.jpeg"
|
||||||
|
|
||||||
|
first := httptest.NewRecorder()
|
||||||
|
r.ServeHTTP(first,
|
||||||
|
httptest.NewRequestWithContext(t.Context(), http.MethodGet, target, nil))
|
||||||
|
|
||||||
|
etag := first.Header().Get("ETag")
|
||||||
|
if first.Code != http.StatusOK || etag == "" {
|
||||||
|
t.Fatalf("first request: status = %d, ETag = %q, want %d and an ETag",
|
||||||
|
first.Code, etag, http.StatusOK)
|
||||||
|
}
|
||||||
|
|
||||||
|
upstream[host+photoPath] = &fstest.MapFile{
|
||||||
|
Data: generateTestJPEG(t, 100, 100, color.RGBA{0, 0, 255, 255}),
|
||||||
|
}
|
||||||
|
|
||||||
|
req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, target, nil)
|
||||||
|
req.Header.Set("If-None-Match", etag)
|
||||||
|
|
||||||
|
second := httptest.NewRecorder()
|
||||||
|
r.ServeHTTP(second, req)
|
||||||
|
t.Logf("first ETag %s; with it after the change: status %d, ETag %s, %d bytes",
|
||||||
|
etag, second.Code, second.Header().Get("ETag"), second.Body.Len())
|
||||||
|
|
||||||
|
if second.Code != http.StatusOK {
|
||||||
|
t.Fatalf("status = %d, want %d with the changed image",
|
||||||
|
second.Code, http.StatusOK)
|
||||||
|
}
|
||||||
|
|
||||||
|
if bytes.Equal(second.Body.Bytes(), first.Body.Bytes()) {
|
||||||
|
t.Error("the image served is the one made before the change")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestHandleImage_ETagHeader(t *testing.T) {
|
func TestHandleImage_ETagHeader(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -168,16 +168,16 @@ func (p *Proxy) Get(
|
|||||||
) (*ImageResponse, error) {
|
) (*ImageResponse, error) {
|
||||||
cacheKey := imagerequest.CacheKey(req)
|
cacheKey := imagerequest.CacheKey(req)
|
||||||
|
|
||||||
reader, size, contentType, err := p.cache.Variant(cacheKey)
|
cached, err := p.cache.Variant(cacheKey)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
p.cache.IncrementStats(true)
|
p.cache.IncrementStats(true)
|
||||||
|
|
||||||
return &ImageResponse{
|
return &ImageResponse{
|
||||||
Content: reader,
|
Content: cached.Content,
|
||||||
ContentLength: size,
|
ContentLength: cached.Size,
|
||||||
ContentType: contentType,
|
ContentType: cached.ContentType,
|
||||||
CacheStatus: CacheHit,
|
CacheStatus: CacheHit,
|
||||||
ETag: formatETag(cacheKey),
|
ETag: formatETag(cacheKey, cached.CheckedAt),
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -288,7 +288,7 @@ func (p *Proxy) processOrWait(
|
|||||||
ContentLength: int64(len(variant.data)),
|
ContentLength: int64(len(variant.data)),
|
||||||
ContentType: variant.contentType,
|
ContentType: variant.contentType,
|
||||||
FetchedBytes: variant.fetchedBytes,
|
FetchedBytes: variant.fetchedBytes,
|
||||||
ETag: formatETag(cacheKey),
|
ETag: formatETag(cacheKey, variant.checkedAt),
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -439,7 +439,7 @@ func (p *Proxy) renewVariant(
|
|||||||
ctx context.Context, cacheKey imagerequest.VariantKey,
|
ctx context.Context, cacheKey imagerequest.VariantKey,
|
||||||
source imgcache.SourceInfo,
|
source imgcache.SourceInfo,
|
||||||
) *processedVariant {
|
) *processedVariant {
|
||||||
reader, _, contentType, err := p.cache.RenewVariant(ctx, cacheKey, source)
|
cached, err := p.cache.RenewVariant(ctx, cacheKey, source)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if !errors.Is(err, imgcache.ErrNotFound) {
|
if !errors.Is(err, imgcache.ErrNotFound) {
|
||||||
p.log.Warn("cached variant not renewed, making it again",
|
p.log.Warn("cached variant not renewed, making it again",
|
||||||
@@ -449,7 +449,7 @@ func (p *Proxy) renewVariant(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return p.readCachedVariant(cacheKey, reader, contentType)
|
return p.readCachedVariant(cacheKey, cached)
|
||||||
}
|
}
|
||||||
|
|
||||||
// cachedVariantIgnoringTTL returns the cached variant of cacheKey whatever
|
// cachedVariantIgnoringTTL returns the cached variant of cacheKey whatever
|
||||||
@@ -460,7 +460,7 @@ func (p *Proxy) renewVariant(
|
|||||||
func (p *Proxy) cachedVariantIgnoringTTL(
|
func (p *Proxy) cachedVariantIgnoringTTL(
|
||||||
ctx context.Context, cacheKey imagerequest.VariantKey,
|
ctx context.Context, cacheKey imagerequest.VariantKey,
|
||||||
) (*processedVariant, imgcache.SourceInfo) {
|
) (*processedVariant, imgcache.SourceInfo) {
|
||||||
reader, _, contentType, err := p.cache.VariantIgnoringTTL(cacheKey)
|
cached, err := p.cache.VariantIgnoringTTL(cacheKey)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if !errors.Is(err, imgcache.ErrNotFound) {
|
if !errors.Is(err, imgcache.ErrNotFound) {
|
||||||
p.log.Warn("cached variant not served", "cache_key", cacheKey,
|
p.log.Warn("cached variant not served", "cache_key", cacheKey,
|
||||||
@@ -470,7 +470,7 @@ func (p *Proxy) cachedVariantIgnoringTTL(
|
|||||||
return nil, imgcache.SourceInfo{}
|
return nil, imgcache.SourceInfo{}
|
||||||
}
|
}
|
||||||
|
|
||||||
variant := p.readCachedVariant(cacheKey, reader, contentType)
|
variant := p.readCachedVariant(cacheKey, cached)
|
||||||
if variant == nil {
|
if variant == nil {
|
||||||
return nil, imgcache.SourceInfo{}
|
return nil, imgcache.SourceInfo{}
|
||||||
}
|
}
|
||||||
@@ -484,15 +484,14 @@ func (p *Proxy) cachedVariantIgnoringTTL(
|
|||||||
return variant, madeFrom
|
return variant, madeFrom
|
||||||
}
|
}
|
||||||
|
|
||||||
// readCachedVariant reads the cached variant of cacheKey that reader opened,
|
// readCachedVariant reads the cached variant of cacheKey from its opened file,
|
||||||
// whose content type is contentType, and closes reader. It returns nil when
|
// and closes the file. It returns nil when the variant cannot be read.
|
||||||
// the variant cannot be read.
|
|
||||||
func (p *Proxy) readCachedVariant(
|
func (p *Proxy) readCachedVariant(
|
||||||
cacheKey imagerequest.VariantKey, reader io.ReadCloser, contentType string,
|
cacheKey imagerequest.VariantKey, file imgcache.VariantFile,
|
||||||
) *processedVariant {
|
) *processedVariant {
|
||||||
defer func() { _ = reader.Close() }()
|
defer func() { _ = file.Content.Close() }()
|
||||||
|
|
||||||
data, err := io.ReadAll(reader)
|
data, err := io.ReadAll(file.Content)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
p.log.Warn("failed to read cached variant", "cache_key", cacheKey,
|
p.log.Warn("failed to read cached variant", "cache_key", cacheKey,
|
||||||
"error", err)
|
"error", err)
|
||||||
@@ -500,7 +499,11 @@ func (p *Proxy) readCachedVariant(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return &processedVariant{data: data, contentType: contentType}
|
return &processedVariant{
|
||||||
|
data: data,
|
||||||
|
contentType: file.ContentType,
|
||||||
|
checkedAt: file.CheckedAt,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// fetchAndProcess fetches the source from upstream, stores it, and makes
|
// fetchAndProcess fetches the source from upstream, stores it, and makes
|
||||||
@@ -528,7 +531,9 @@ func (p *Proxy) fetchAndProcess(
|
|||||||
case variant == nil:
|
case variant == nil:
|
||||||
return nil, err
|
return nil, err
|
||||||
case errors.Is(err, errSourceNotModified):
|
case errors.Is(err, errSourceNotModified):
|
||||||
err = p.cache.SetVariantCheckedAt(ctx, cacheKey, time.Now().UTC())
|
variant.checkedAt = time.Now().UTC()
|
||||||
|
|
||||||
|
err = p.cache.SetVariantCheckedAt(ctx, cacheKey, variant.checkedAt)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
p.log.Warn("failed to renew cached variant", "cache_key", cacheKey,
|
p.log.Warn("failed to renew cached variant", "cache_key", cacheKey,
|
||||||
"error", err)
|
"error", err)
|
||||||
@@ -745,6 +750,9 @@ type processedVariant struct {
|
|||||||
data []byte
|
data []byte
|
||||||
contentType string
|
contentType string
|
||||||
fetchedBytes int64
|
fetchedBytes int64
|
||||||
|
// checkedAt is its source's check time when the variant was made or last
|
||||||
|
// renewed, which its cached file has as its modification time
|
||||||
|
checkedAt time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
// processAndStore processes the image read from source, which is in
|
// processAndStore processes the image read from source, which is in
|
||||||
@@ -826,6 +834,7 @@ func (p *Proxy) processAndStore(
|
|||||||
data: processedData,
|
data: processedData,
|
||||||
contentType: processResult.ContentType,
|
contentType: processResult.ContentType,
|
||||||
fetchedBytes: fetchBytes,
|
fetchedBytes: fetchBytes,
|
||||||
|
checkedAt: sourceInfo.CheckedAt,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -837,13 +846,16 @@ func isNegativeCacheable(err error) bool {
|
|||||||
// etagHashLength is the number of hash characters to use for ETags.
|
// etagHashLength is the number of hash characters to use for ETags.
|
||||||
const etagHashLength = 16
|
const etagHashLength = 16
|
||||||
|
|
||||||
// formatETag formats a VariantKey as a quoted ETag value.
|
// formatETag returns the quoted ETag of the variant of key: the first
|
||||||
func formatETag(key imagerequest.VariantKey) string {
|
// etagHashLength characters of key, then checkedAt in Unix nanoseconds.
|
||||||
|
// checkedAt is its source's check time when the variant was made or last
|
||||||
|
// renewed, so a variant made again from new source bytes gets another ETag.
|
||||||
|
func formatETag(key imagerequest.VariantKey, checkedAt time.Time) string {
|
||||||
hash := string(key)
|
hash := string(key)
|
||||||
// Use first 16 characters of hash for a shorter but still unique ETag
|
// Use first 16 characters of hash for a shorter but still unique ETag
|
||||||
if len(hash) > etagHashLength {
|
if len(hash) > etagHashLength {
|
||||||
hash = hash[:etagHashLength]
|
hash = hash[:etagHashLength]
|
||||||
}
|
}
|
||||||
|
|
||||||
return `"` + hash + `"`
|
return fmt.Sprintf(`"%s-%d"`, hash, checkedAt.UnixNano())
|
||||||
}
|
}
|
||||||
+69
-62
@@ -3,8 +3,8 @@
|
|||||||
// - the variants made from them
|
// - the variants made from them
|
||||||
// - the negative cache: the source URLs whose fetch failed recently
|
// - the negative cache: the source URLs whose fetch failed recently
|
||||||
//
|
//
|
||||||
// Each image is a file with a row in the database. A source and its
|
// Each image is a file with a row in the database. A source is used, and its
|
||||||
// variants are served until the cache TTL has passed since upstream last sent
|
// variants are served, until the cache TTL has passed since upstream last sent
|
||||||
// the source or confirmed it unchanged. Then the source is due to be checked
|
// the source or confirmed it unchanged. Then the source is due to be checked
|
||||||
// with upstream again (see Stale). Eviction removes:
|
// with upstream again (see Stale). Eviction removes:
|
||||||
// - the least recently used images, to keep the cache under its size limit
|
// - the least recently used images, to keep the cache under its size limit
|
||||||
@@ -45,14 +45,14 @@ const cleanExpiredBatchSize = 1000
|
|||||||
const DefaultNegativeTTL = 5 * time.Minute
|
const DefaultNegativeTTL = 5 * time.Minute
|
||||||
|
|
||||||
// DefaultCacheTTL is how long after upstream last sent a cached source, or
|
// DefaultCacheTTL is how long after upstream last sent a cached source, or
|
||||||
// answered that it had not changed, pixad serves the source and its
|
// answered that it had not changed, pixad uses the source and serves its
|
||||||
// variants without asking upstream again.
|
// variants without asking upstream again.
|
||||||
const DefaultCacheTTL = 24 * time.Hour
|
const DefaultCacheTTL = 24 * time.Hour
|
||||||
|
|
||||||
// EvictionAfterLastCheck is how long after upstream last sent a cached
|
// EvictionAfterLastCheck is how long after upstream last sent a cached
|
||||||
// source, or answered that it had not changed, the source and its variants
|
// source, or answered that it had not changed, the source and its variants
|
||||||
// are evicted (see EvictNotCheckedRecently). Until then a source whose checks
|
// are evicted (see EvictNotCheckedRecently). Until then a source whose checks
|
||||||
// fail is served.
|
// fail is used.
|
||||||
const EvictionAfterLastCheck = 30 * 24 * time.Hour
|
const EvictionAfterLastCheck = 30 * 24 * time.Hour
|
||||||
|
|
||||||
// ErrStale is returned by Variant for a variant whose file's modification
|
// ErrStale is returned by Variant for a variant whose file's modification
|
||||||
@@ -64,7 +64,7 @@ type Config struct {
|
|||||||
StateDir string
|
StateDir string
|
||||||
NegativeTTL time.Duration
|
NegativeTTL time.Duration
|
||||||
|
|
||||||
// CacheTTL is how long a cached source and its variants are served
|
// CacheTTL is how long a cached source is used, and its variants served,
|
||||||
// before upstream is asked again whether the source changed. Zero means
|
// before upstream is asked again whether the source changed. Zero means
|
||||||
// DefaultCacheTTL.
|
// DefaultCacheTTL.
|
||||||
CacheTTL time.Duration
|
CacheTTL time.Duration
|
||||||
@@ -186,31 +186,43 @@ func NewCache(
|
|||||||
return c, nil
|
return c, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Variant opens a cached variant and returns a reader, its size and its
|
// VariantFile is the open file of a cached variant, as Variant,
|
||||||
// content type, read from the variant file's first bytes. It opens that one
|
// VariantIgnoringTTL and RenewVariant return it.
|
||||||
// file and makes no database call: the file's modification time is its
|
type VariantFile struct {
|
||||||
// source's check time when the variant was made or last renewed (see
|
// Content reads the file. The caller closes it.
|
||||||
// PutVariant), and a variant whose time is CacheTTL or more ago is ErrStale.
|
Content io.ReadCloser
|
||||||
// A variant it opens has its access time handed to the background writer
|
// Size is the file's size in bytes.
|
||||||
// without waiting (see touchVariant). A variant not in the cache, or any
|
Size int64
|
||||||
// variant of a disabled cache, is ErrNotFound; a variant file whose first
|
// ContentType is the MIME type of the variant's format, read from the
|
||||||
|
// file's first bytes.
|
||||||
|
ContentType string
|
||||||
|
// CheckedAt is the file's modification time: its source's check time
|
||||||
|
// when the variant was made or last renewed.
|
||||||
|
CheckedAt time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// Variant opens a cached variant and returns its file (see VariantFile). It
|
||||||
|
// opens that one file and makes no database call: the file's modification
|
||||||
|
// time is its source's check time when the variant was made or last renewed
|
||||||
|
// (see PutVariant), and a variant whose time is CacheTTL or more ago is
|
||||||
|
// ErrStale. A variant it opens has its access time handed to the background
|
||||||
|
// writer without waiting (see touchVariant). A variant not in the cache, or
|
||||||
|
// any variant of a disabled cache, is ErrNotFound; a variant file whose first
|
||||||
// bytes are not those of an output format is another error.
|
// bytes are not those of an output format is another error.
|
||||||
func (c *Cache) Variant(
|
func (c *Cache) Variant(cacheKey imagerequest.VariantKey) (VariantFile, error) {
|
||||||
cacheKey imagerequest.VariantKey,
|
|
||||||
) (io.ReadCloser, int64, string, error) {
|
|
||||||
if c.config.MaxBytes == 0 {
|
if c.config.MaxBytes == 0 {
|
||||||
return nil, 0, "", ErrNotFound
|
return VariantFile{}, ErrNotFound
|
||||||
}
|
}
|
||||||
|
|
||||||
file, info, err := c.openVariant(cacheKey)
|
file, info, err := c.variants.Open(string(cacheKey))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, 0, "", err
|
return VariantFile{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if c.Stale(info.ModTime()) {
|
if c.Stale(info.ModTime()) {
|
||||||
_ = file.Close()
|
_ = file.Close()
|
||||||
|
|
||||||
return nil, 0, "", ErrStale
|
return VariantFile{}, ErrStale
|
||||||
}
|
}
|
||||||
|
|
||||||
return c.finishOpeningVariant(cacheKey, file, info)
|
return c.finishOpeningVariant(cacheKey, file, info)
|
||||||
@@ -221,14 +233,14 @@ func (c *Cache) Variant(
|
|||||||
// cached, before it checks the source with upstream (see LookupVariant).
|
// cached, before it checks the source with upstream (see LookupVariant).
|
||||||
func (c *Cache) VariantIgnoringTTL(
|
func (c *Cache) VariantIgnoringTTL(
|
||||||
cacheKey imagerequest.VariantKey,
|
cacheKey imagerequest.VariantKey,
|
||||||
) (io.ReadCloser, int64, string, error) {
|
) (VariantFile, error) {
|
||||||
if c.config.MaxBytes == 0 {
|
if c.config.MaxBytes == 0 {
|
||||||
return nil, 0, "", ErrNotFound
|
return VariantFile{}, ErrNotFound
|
||||||
}
|
}
|
||||||
|
|
||||||
file, info, err := c.openVariant(cacheKey)
|
file, info, err := c.variants.Open(string(cacheKey))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, 0, "", err
|
return VariantFile{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return c.finishOpeningVariant(cacheKey, file, info)
|
return c.finishOpeningVariant(cacheKey, file, info)
|
||||||
@@ -241,41 +253,50 @@ func (c *Cache) VariantIgnoringTTL(
|
|||||||
// ErrNotFound, as is a variant not in the cache.
|
// ErrNotFound, as is a variant not in the cache.
|
||||||
func (c *Cache) RenewVariant(
|
func (c *Cache) RenewVariant(
|
||||||
ctx context.Context, cacheKey imagerequest.VariantKey, source SourceInfo,
|
ctx context.Context, cacheKey imagerequest.VariantKey, source SourceInfo,
|
||||||
) (io.ReadCloser, int64, string, error) {
|
) (VariantFile, error) {
|
||||||
if c.config.MaxBytes == 0 {
|
if c.config.MaxBytes == 0 {
|
||||||
return nil, 0, "", ErrNotFound
|
return VariantFile{}, ErrNotFound
|
||||||
}
|
}
|
||||||
|
|
||||||
file, info, err := c.openVariant(cacheKey)
|
file, info, err := c.variants.Open(string(cacheKey))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, 0, "", err
|
return VariantFile{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if info.ModTime().Before(source.FetchedAt) {
|
if info.ModTime().Before(source.FetchedAt) {
|
||||||
_ = file.Close()
|
_ = file.Close()
|
||||||
|
|
||||||
return nil, 0, "", ErrNotFound
|
return VariantFile{}, ErrNotFound
|
||||||
}
|
}
|
||||||
|
|
||||||
format, err := readFormat(file)
|
format, err := readFormat(file)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = file.Close()
|
_ = file.Close()
|
||||||
|
|
||||||
return nil, 0, "", err
|
return VariantFile{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if info.ModTime().Before(source.CheckedAt) {
|
checkedAt := info.ModTime()
|
||||||
|
|
||||||
|
if checkedAt.Before(source.CheckedAt) {
|
||||||
err = c.SetVariantCheckedAt(ctx, cacheKey, source.CheckedAt)
|
err = c.SetVariantCheckedAt(ctx, cacheKey, source.CheckedAt)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = file.Close()
|
_ = file.Close()
|
||||||
|
|
||||||
return nil, 0, "", err
|
return VariantFile{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
checkedAt = source.CheckedAt
|
||||||
}
|
}
|
||||||
|
|
||||||
c.touchVariant(cacheKey)
|
c.touchVariant(cacheKey)
|
||||||
|
|
||||||
return file, info.Size(), format.MIMEType(), nil
|
return VariantFile{
|
||||||
|
Content: file,
|
||||||
|
Size: info.Size(),
|
||||||
|
ContentType: format.MIMEType(),
|
||||||
|
CheckedAt: checkedAt,
|
||||||
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetVariantCheckedAt records checkedAt as when upstream last sent the source
|
// SetVariantCheckedAt records checkedAt as when upstream last sent the source
|
||||||
@@ -406,7 +427,7 @@ func (c *Cache) Source(
|
|||||||
return nil, 0, "", ErrNotFound
|
return nil, 0, "", ErrNotFound
|
||||||
}
|
}
|
||||||
|
|
||||||
file, size, err := c.srcContent.Open(string(key))
|
file, info, err := c.srcContent.Open(string(key))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, 0, "", err
|
return nil, 0, "", err
|
||||||
}
|
}
|
||||||
@@ -420,7 +441,7 @@ func (c *Cache) Source(
|
|||||||
|
|
||||||
c.touchSourceContent(key)
|
c.touchSourceContent(key)
|
||||||
|
|
||||||
return file, size, format, nil
|
return file, info.Size(), format, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// SourceInfo is what the cache keeps in a cached source's row besides its
|
// SourceInfo is what the cache keeps in a cached source's row besides its
|
||||||
@@ -694,43 +715,29 @@ func (c *Cache) cleanExpiredBatch(ctx context.Context) (int64, error) {
|
|||||||
return deleted, nil
|
return deleted, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// openVariant opens the file of a cached variant and returns it with its
|
// finishOpeningVariant returns a variant file that FileStore.Open opened, with
|
||||||
// information, which gives its size and its modification time.
|
// its information info, as Variant does. It reads the content type from the
|
||||||
func (c *Cache) openVariant(
|
// file's first bytes and hands the variant's access time to the background
|
||||||
cacheKey imagerequest.VariantKey,
|
// writer (see touchVariant). It closes the file when its first bytes are not
|
||||||
) (*os.File, fs.FileInfo, error) {
|
// those of an output format.
|
||||||
file, _, err := c.variants.Open(string(cacheKey))
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
info, err := file.Stat()
|
|
||||||
if err != nil {
|
|
||||||
_ = file.Close()
|
|
||||||
|
|
||||||
return nil, nil, fmt.Errorf("failed to stat variant: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return file, info, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// finishOpeningVariant reads the content type of a variant file openVariant
|
|
||||||
// opened from its first bytes, hands the variant's access time to the
|
|
||||||
// background writer (see touchVariant), and returns the file as Variant does.
|
|
||||||
// It closes the file when its first bytes are not those of an output format.
|
|
||||||
func (c *Cache) finishOpeningVariant(
|
func (c *Cache) finishOpeningVariant(
|
||||||
cacheKey imagerequest.VariantKey, file *os.File, info fs.FileInfo,
|
cacheKey imagerequest.VariantKey, file *os.File, info fs.FileInfo,
|
||||||
) (io.ReadCloser, int64, string, error) {
|
) (VariantFile, error) {
|
||||||
format, err := readFormat(file)
|
format, err := readFormat(file)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = file.Close()
|
_ = file.Close()
|
||||||
|
|
||||||
return nil, 0, "", err
|
return VariantFile{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
c.touchVariant(cacheKey)
|
c.touchVariant(cacheKey)
|
||||||
|
|
||||||
return file, info.Size(), format.MIMEType(), nil
|
return VariantFile{
|
||||||
|
Content: file,
|
||||||
|
Size: info.Size(),
|
||||||
|
ContentType: format.MIMEType(),
|
||||||
|
CheckedAt: info.ModTime(),
|
||||||
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// readFormat reads the first bytes of a source or variant file, without
|
// readFormat reads the first bytes of a source or variant file, without
|
||||||
|
|||||||
@@ -55,11 +55,13 @@ func TestVariant_ContentTypeFromFirstBytes(t *testing.T) {
|
|||||||
t.Fatalf("PutVariant(%s) error = %v", format, err)
|
t.Fatalf("PutVariant(%s) error = %v", format, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
reader, size, contentType, err := cache.Variant(key)
|
variant, err := cache.Variant(key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Variant(%s) error = %v", format, err)
|
t.Fatalf("Variant(%s) error = %v", format, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
reader, size, contentType := variant.Content, variant.Size, variant.ContentType
|
||||||
|
|
||||||
served, err := io.ReadAll(reader)
|
served, err := io.ReadAll(reader)
|
||||||
_ = reader.Close()
|
_ = reader.Close()
|
||||||
|
|
||||||
@@ -110,16 +112,19 @@ func TestVariant_RefusesFileOfNoOutputFormat(t *testing.T) {
|
|||||||
t.Fatalf("%s: storing the variant file: %v", tt.name, err)
|
t.Fatalf("%s: storing the variant file: %v", tt.name, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
reader, _, contentType, err := cache.Variant(key)
|
variant, err := cache.Variant(key)
|
||||||
t.Logf("%s: Variant() error = %v", tt.name, err)
|
t.Logf("%s: Variant() error = %v", tt.name, err)
|
||||||
|
|
||||||
|
reader, contentType := variant.Content, variant.ContentType
|
||||||
if reader != nil || contentType != "" || !errors.Is(err, tt.wantErr) {
|
if reader != nil || contentType != "" || !errors.Is(err, tt.wantErr) {
|
||||||
t.Errorf("%s: Variant() = reader %v, content type %q, error %v; "+
|
t.Errorf("%s: Variant() = reader %v, content type %q, error %v; "+
|
||||||
"want no reader and %v", tt.name, reader, contentType, err, tt.wantErr)
|
"want no reader and %v", tt.name, reader, contentType, err, tt.wantErr)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
reader, _, _, err := cache.Variant(testVariantKeyFour)
|
variant, err := cache.Variant(testVariantKeyFour)
|
||||||
|
|
||||||
|
reader := variant.Content
|
||||||
if reader != nil || !errors.Is(err, ErrNotFound) {
|
if reader != nil || !errors.Is(err, ErrNotFound) {
|
||||||
t.Errorf("Variant() of a variant with no file = reader %v, error %v; "+
|
t.Errorf("Variant() of a variant with no file = reader %v, error %v; "+
|
||||||
"want no reader and %v", reader, err, ErrNotFound)
|
"want no reader and %v", reader, err, ErrNotFound)
|
||||||
@@ -127,7 +132,9 @@ func TestVariant_RefusesFileOfNoOutputFormat(t *testing.T) {
|
|||||||
|
|
||||||
disabled, _ := newEvictionTestCache(t, 0)
|
disabled, _ := newEvictionTestCache(t, 0)
|
||||||
|
|
||||||
reader, _, _, err = disabled.Variant(testVariantKeyOne)
|
variant, err = disabled.Variant(testVariantKeyOne)
|
||||||
|
|
||||||
|
reader = variant.Content
|
||||||
if reader != nil || !errors.Is(err, ErrNotFound) {
|
if reader != nil || !errors.Is(err, ErrNotFound) {
|
||||||
t.Errorf("Variant() on a disabled cache = reader %v, error %v; "+
|
t.Errorf("Variant() on a disabled cache = reader %v, error %v; "+
|
||||||
"want no reader and %v", reader, err, ErrNotFound)
|
"want no reader and %v", reader, err, ErrNotFound)
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package imgcache
|
|||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"time"
|
"time"
|
||||||
@@ -55,26 +56,26 @@ func (s *FileStore) Write(key string, data []byte) error {
|
|||||||
return writeFileSynced(path, data)
|
return writeFileSynced(path, data)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Open opens the file for key and returns it with its size in bytes. A
|
// Open opens the file for key and returns it with its information, which
|
||||||
// missing file is ErrNotFound.
|
// gives its size and its modification time. A missing file is ErrNotFound.
|
||||||
func (s *FileStore) Open(key string) (*os.File, int64, error) {
|
func (s *FileStore) Open(key string) (*os.File, fs.FileInfo, error) {
|
||||||
f, err := os.Open(s.keyToPath(key))
|
f, err := os.Open(s.keyToPath(key))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(err) {
|
||||||
return nil, 0, ErrNotFound
|
return nil, nil, ErrNotFound
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil, 0, fmt.Errorf("failed to open content: %w", err)
|
return nil, nil, fmt.Errorf("failed to open content: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
stat, err := f.Stat()
|
info, err := f.Stat()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = f.Close()
|
_ = f.Close()
|
||||||
|
|
||||||
return nil, 0, fmt.Errorf("failed to stat content: %w", err)
|
return nil, nil, fmt.Errorf("failed to stat content: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return f, stat.Size(), nil
|
return f, info, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Size returns the size in bytes of the file for key. A missing file is
|
// Size returns the size in bytes of the file for key. A missing file is
|
||||||
|
|||||||
@@ -38,15 +38,15 @@ func TestFileStore_WriteAndOpen(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Open and verify size and content
|
// Open and verify size and content
|
||||||
r, size, err := storage.Open(key)
|
r, info, err := storage.Open(key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Open() error = %v", err)
|
t.Fatalf("Open() error = %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
defer func() { _ = r.Close() }()
|
defer func() { _ = r.Close() }()
|
||||||
|
|
||||||
if size != int64(len(content)) {
|
if info.Size() != int64(len(content)) {
|
||||||
t.Errorf("Open() size = %d, want %d", size, len(content))
|
t.Errorf("Open() size = %d, want %d", info.Size(), len(content))
|
||||||
}
|
}
|
||||||
|
|
||||||
loaded, err := io.ReadAll(r)
|
loaded, err := io.ReadAll(r)
|
||||||
|
|||||||
Reference in new issue
Block a user