Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3f487ee0ce |
No files matched your search
@@ -233,8 +233,8 @@ These terms are the names the code uses:
|
|||||||
- **Variant**: the image pixa serves for an image request, made from its source
|
- **Variant**: the image pixa serves for an image request, made from its source
|
||||||
with its transform.
|
with its transform.
|
||||||
- **Variant key**: the SHA-256 hash of a variant's source, size, format, quality
|
- **Variant key**: the SHA-256 hash of a variant's source, size, format, quality
|
||||||
and fit. It names the variant's file and its row in the database. The
|
and fit. It names the variant's file and its row in the database, and gives
|
||||||
variant's `ETag` is made from it and the file's modification time.
|
its `ETag`.
|
||||||
- **Signed URL**: a `/v1/image/` URL whose `sig` parameter signs its image
|
- **Signed URL**: a `/v1/image/` URL whose `sig` parameter signs its image
|
||||||
request and its expiry `exp` with `signing_key`.
|
request and its expiry `exp` with `signing_key`.
|
||||||
- **Encrypted URL**: a `/v1/e/` URL whose token holds its image request and any
|
- **Encrypted URL**: a `/v1/e/` URL whose token holds its image request and any
|
||||||
@@ -249,8 +249,8 @@ These terms are the names the code uses:
|
|||||||
- a check of a cached image failed in any way (see Cache TTL).
|
- a check of a cached image failed in any way (see Cache TTL).
|
||||||
- **Image cache**: the part of pixa that keeps the sources, the variants and the
|
- **Image cache**: the part of pixa that keeps the sources, the variants and the
|
||||||
negative cache (see Storage).
|
negative cache (see Storage).
|
||||||
- **Cache TTL**: how long pixa uses a cached source and serves its variants
|
- **Cache TTL**: how long pixa serves a cached source and its variants before it
|
||||||
before it checks the source with upstream again: 24 hours (see Cache TTL).
|
checks the source with upstream again: 24 hours (see Cache TTL).
|
||||||
- **Disk cache limit** (`cache_max_bytes`): the size in bytes that the cached
|
- **Disk cache limit** (`cache_max_bytes`): the size in bytes that the cached
|
||||||
sources and variants may reach together.
|
sources and variants may reach together.
|
||||||
- **Image proxy**: the part of pixa that serves a variant from the image cache
|
- **Image proxy**: the part of pixa that serves a variant from the image cache
|
||||||
@@ -522,9 +522,7 @@ the image after pixa would refuse the URL. A URL with no expiry gets one year.
|
|||||||
`immutable` only stops a client revalidating while its copy is fresh. The
|
`immutable` only stops a client revalidating while its copy is fresh. The
|
||||||
response also carries `X-Pixa-Cache`: `HIT` for an image served from the cache,
|
response also carries `X-Pixa-Cache`: `HIT` for an image served from the cache,
|
||||||
and `MISS` for one made for this request or older than 24 hours (see Cache TTL).
|
and `MISS` for one made for this request or older than 24 hours (see Cache TTL).
|
||||||
The `ETag` is made from the first 16 characters of the variant key and the
|
The `ETag` is the first 16 characters of the variant key, in quotes.
|
||||||
variant file's modification time (see Cache TTL). So a variant made again from a
|
|
||||||
changed source gets a new `ETag`.
|
|
||||||
|
|
||||||
### Signed URLs
|
### Signed URLs
|
||||||
|
|
||||||
@@ -896,17 +894,16 @@ also removes the expired rows of the negative cache every five minutes, at most
|
|||||||
|
|
||||||
### Cache TTL
|
### Cache TTL
|
||||||
|
|
||||||
pixa uses a cached source for 24 hours after upstream last sent it or confirmed
|
pixa serves a cached source and its variants for 24 hours after upstream last
|
||||||
it unchanged. During that time it serves the source's variants from the cache.
|
sent the source or confirmed it unchanged. After that, the next request for an
|
||||||
After that, the next request for an image of that source checks the source with
|
image of that source checks the source with upstream first:
|
||||||
upstream first:
|
|
||||||
|
|
||||||
- pixa asks upstream to send the source only if it changed. It sends the
|
- pixa asks upstream to send the source only if it changed. It sends the
|
||||||
source's `ETag` as `If-None-Match` and its `Last-Modified` as
|
source's `ETag` as `If-None-Match` and its `Last-Modified` as
|
||||||
`If-Modified-Since`, when upstream gave them. With neither, upstream sends the
|
`If-Modified-Since`, when upstream gave them. With neither, upstream sends the
|
||||||
whole image again.
|
whole image again.
|
||||||
- When upstream answers 304 Not Modified, pixa uses the cached source and serves
|
- When upstream answers 304 Not Modified, pixa serves the cached source and its
|
||||||
its variants for another 24 hours. Nothing is downloaded or converted again.
|
variants for another 24 hours. Nothing is downloaded or converted again.
|
||||||
- When upstream sends the image, it replaces the cached source. Each variant
|
- When upstream sends the image, it replaces the cached source. Each variant
|
||||||
made from the old image is converted again from the new one when it is next
|
made from the old image is converted again from the new one when it is next
|
||||||
requested.
|
requested.
|
||||||
@@ -916,8 +913,8 @@ upstream first:
|
|||||||
often it serves the image.
|
often it serves the image.
|
||||||
|
|
||||||
pixa evicts a source and its variants 30 days after upstream last sent the
|
pixa evicts a source and its variants 30 days after upstream last sent the
|
||||||
source or confirmed it unchanged. So pixa uses a source whose checks keep
|
source or confirmed it unchanged. So a source whose checks keep failing is
|
||||||
failing for at most 30 days. Eviction to stay under the disk cache limit can
|
served for at most 30 days. Eviction to stay under the disk cache limit can
|
||||||
remove images sooner.
|
remove images sooner.
|
||||||
|
|
||||||
That eviction can remove a source and keep its variants. Each variant's row
|
That eviction can remove a source and keep its variants. Each variant's row
|
||||||
|
|||||||
@@ -31,7 +31,7 @@ P2: security: per-origin rate limiting
|
|||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
- 2026-11-11 a cached source is checked with upstream once it is 24 hours old
|
- 2026-11-11 a cached source is checked with upstream once it is 24 hours old
|
||||||
(closes #69). Before, it was used and its variants served until eviction, so a
|
(closes #69). Before, it and its variants were served until eviction, so a
|
||||||
changed image at the same URL was never picked up.
|
changed image at the same URL was never picked up.
|
||||||
- `source_content` keeps the source's `fetched_at`, `checked_at`, and the
|
- `source_content` keeps the source's `fetched_at`, `checked_at`, and the
|
||||||
`ETag` and `Last-Modified` upstream sent.
|
`ETag` and `Last-Modified` upstream sent.
|
||||||
@@ -40,9 +40,6 @@ P2: security: per-origin rate limiting
|
|||||||
- A variant file's modification time is its source's `checked_at` when the
|
- A variant file's modification time is its source's `checked_at` when the
|
||||||
variant was made or last renewed, so a hit makes no database call. A
|
variant was made or last renewed, so a hit makes no database call. A
|
||||||
variant made before the source's `fetched_at` is converted again.
|
variant made before the source's `fetched_at` is converted again.
|
||||||
- A variant's `ETag` is made from its variant key and its file's
|
|
||||||
modification time, so a variant made again from new bytes gets a new
|
|
||||||
`ETag`.
|
|
||||||
- `variant_content` keeps the `ETag` and `Last-Modified` of the source a
|
- `variant_content` keeps the `ETag` and `Last-Modified` of the source a
|
||||||
variant was made from. A variant whose source was evicted is checked with
|
variant was made from. A variant whose source was evicted is checked with
|
||||||
them, and a 304 renews the variant.
|
them, and a 304 renews the variant.
|
||||||
|
|||||||
@@ -257,90 +257,6 @@ func TestHandleImage_ConditionalRequest_IfNoneMatch_DifferentETag(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestHandleImage_ConditionalRequest_EarlierETagGetsChangedImage serves an
|
|
||||||
// image, changes it upstream, and requests it again with the ETag first
|
|
||||||
// served, once the cache TTL has passed. The changed image must be served,
|
|
||||||
// not 304 Not Modified.
|
|
||||||
func TestHandleImage_ConditionalRequest_EarlierETagGetsChangedImage(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const host = "goodhost.example.com"
|
|
||||||
|
|
||||||
upstream := fstest.MapFS{
|
|
||||||
host + photoPath: &fstest.MapFile{
|
|
||||||
Data: generateTestJPEG(t, 100, 100, color.RGBA{255, 0, 0, 255}),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
// A cache TTL of one nanosecond has passed by the second request
|
|
||||||
cache, err := imgcache.NewCache(setupTestDB(t), imgcache.Config{
|
|
||||||
StateDir: t.TempDir(),
|
|
||||||
NegativeTTL: 5 * time.Minute,
|
|
||||||
CacheTTL: time.Nanosecond,
|
|
||||||
MaxBytes: 1 << 30,
|
|
||||||
}, prometheus.NewRegistry())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("imgcache.NewCache() error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
svc, err := imageproxy.New(&imageproxy.Config{
|
|
||||||
Cache: cache,
|
|
||||||
Fetcher: newMockFetcher(upstream),
|
|
||||||
Processor: imageprocessor.New(imageprocessor.Params{
|
|
||||||
MaxInputBytes: 50 << 20,
|
|
||||||
MaxConcurrentProcessing: 2,
|
|
||||||
}),
|
|
||||||
MaxResponseSize: 50 << 20,
|
|
||||||
Logger: slog.New(slog.DiscardHandler),
|
|
||||||
}, prometheus.NewRegistry())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("imageproxy.New() error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
h := &Handlers{
|
|
||||||
imgSvc: svc,
|
|
||||||
log: slog.New(slog.DiscardHandler),
|
|
||||||
signer: signature.New(testSigningKey),
|
|
||||||
allowlist: allowlist.New([]string{host}),
|
|
||||||
}
|
|
||||||
|
|
||||||
r := chi.NewRouter()
|
|
||||||
r.Get("/v1/image/*", h.HandleImage())
|
|
||||||
|
|
||||||
target := "/v1/image/" + host + photoPath + "/50x50.jpeg"
|
|
||||||
|
|
||||||
first := httptest.NewRecorder()
|
|
||||||
r.ServeHTTP(first,
|
|
||||||
httptest.NewRequestWithContext(t.Context(), http.MethodGet, target, nil))
|
|
||||||
|
|
||||||
etag := first.Header().Get("ETag")
|
|
||||||
if first.Code != http.StatusOK || etag == "" {
|
|
||||||
t.Fatalf("first request: status = %d, ETag = %q, want %d and an ETag",
|
|
||||||
first.Code, etag, http.StatusOK)
|
|
||||||
}
|
|
||||||
|
|
||||||
upstream[host+photoPath] = &fstest.MapFile{
|
|
||||||
Data: generateTestJPEG(t, 100, 100, color.RGBA{0, 0, 255, 255}),
|
|
||||||
}
|
|
||||||
|
|
||||||
req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, target, nil)
|
|
||||||
req.Header.Set("If-None-Match", etag)
|
|
||||||
|
|
||||||
second := httptest.NewRecorder()
|
|
||||||
r.ServeHTTP(second, req)
|
|
||||||
t.Logf("first ETag %s; with it after the change: status %d, ETag %s, %d bytes",
|
|
||||||
etag, second.Code, second.Header().Get("ETag"), second.Body.Len())
|
|
||||||
|
|
||||||
if second.Code != http.StatusOK {
|
|
||||||
t.Fatalf("status = %d, want %d with the changed image",
|
|
||||||
second.Code, http.StatusOK)
|
|
||||||
}
|
|
||||||
|
|
||||||
if bytes.Equal(second.Body.Bytes(), first.Body.Bytes()) {
|
|
||||||
t.Error("the image served is the one made before the change")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestHandleImage_ETagHeader(t *testing.T) {
|
func TestHandleImage_ETagHeader(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -168,16 +168,16 @@ func (p *Proxy) Get(
|
|||||||
) (*ImageResponse, error) {
|
) (*ImageResponse, error) {
|
||||||
cacheKey := imagerequest.CacheKey(req)
|
cacheKey := imagerequest.CacheKey(req)
|
||||||
|
|
||||||
cached, err := p.cache.Variant(cacheKey)
|
reader, size, contentType, err := p.cache.Variant(cacheKey)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
p.cache.IncrementStats(true)
|
p.cache.IncrementStats(true)
|
||||||
|
|
||||||
return &ImageResponse{
|
return &ImageResponse{
|
||||||
Content: cached.Content,
|
Content: reader,
|
||||||
ContentLength: cached.Size,
|
ContentLength: size,
|
||||||
ContentType: cached.ContentType,
|
ContentType: contentType,
|
||||||
CacheStatus: CacheHit,
|
CacheStatus: CacheHit,
|
||||||
ETag: formatETag(cacheKey, cached.CheckedAt),
|
ETag: formatETag(cacheKey),
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -288,7 +288,7 @@ func (p *Proxy) processOrWait(
|
|||||||
ContentLength: int64(len(variant.data)),
|
ContentLength: int64(len(variant.data)),
|
||||||
ContentType: variant.contentType,
|
ContentType: variant.contentType,
|
||||||
FetchedBytes: variant.fetchedBytes,
|
FetchedBytes: variant.fetchedBytes,
|
||||||
ETag: formatETag(cacheKey, variant.checkedAt),
|
ETag: formatETag(cacheKey),
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -439,7 +439,7 @@ func (p *Proxy) renewVariant(
|
|||||||
ctx context.Context, cacheKey imagerequest.VariantKey,
|
ctx context.Context, cacheKey imagerequest.VariantKey,
|
||||||
source imgcache.SourceInfo,
|
source imgcache.SourceInfo,
|
||||||
) *processedVariant {
|
) *processedVariant {
|
||||||
cached, err := p.cache.RenewVariant(ctx, cacheKey, source)
|
reader, _, contentType, err := p.cache.RenewVariant(ctx, cacheKey, source)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if !errors.Is(err, imgcache.ErrNotFound) {
|
if !errors.Is(err, imgcache.ErrNotFound) {
|
||||||
p.log.Warn("cached variant not renewed, making it again",
|
p.log.Warn("cached variant not renewed, making it again",
|
||||||
@@ -449,7 +449,7 @@ func (p *Proxy) renewVariant(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return p.readCachedVariant(cacheKey, cached)
|
return p.readCachedVariant(cacheKey, reader, contentType)
|
||||||
}
|
}
|
||||||
|
|
||||||
// cachedVariantIgnoringTTL returns the cached variant of cacheKey whatever
|
// cachedVariantIgnoringTTL returns the cached variant of cacheKey whatever
|
||||||
@@ -460,7 +460,7 @@ func (p *Proxy) renewVariant(
|
|||||||
func (p *Proxy) cachedVariantIgnoringTTL(
|
func (p *Proxy) cachedVariantIgnoringTTL(
|
||||||
ctx context.Context, cacheKey imagerequest.VariantKey,
|
ctx context.Context, cacheKey imagerequest.VariantKey,
|
||||||
) (*processedVariant, imgcache.SourceInfo) {
|
) (*processedVariant, imgcache.SourceInfo) {
|
||||||
cached, err := p.cache.VariantIgnoringTTL(cacheKey)
|
reader, _, contentType, err := p.cache.VariantIgnoringTTL(cacheKey)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if !errors.Is(err, imgcache.ErrNotFound) {
|
if !errors.Is(err, imgcache.ErrNotFound) {
|
||||||
p.log.Warn("cached variant not served", "cache_key", cacheKey,
|
p.log.Warn("cached variant not served", "cache_key", cacheKey,
|
||||||
@@ -470,7 +470,7 @@ func (p *Proxy) cachedVariantIgnoringTTL(
|
|||||||
return nil, imgcache.SourceInfo{}
|
return nil, imgcache.SourceInfo{}
|
||||||
}
|
}
|
||||||
|
|
||||||
variant := p.readCachedVariant(cacheKey, cached)
|
variant := p.readCachedVariant(cacheKey, reader, contentType)
|
||||||
if variant == nil {
|
if variant == nil {
|
||||||
return nil, imgcache.SourceInfo{}
|
return nil, imgcache.SourceInfo{}
|
||||||
}
|
}
|
||||||
@@ -484,14 +484,15 @@ func (p *Proxy) cachedVariantIgnoringTTL(
|
|||||||
return variant, madeFrom
|
return variant, madeFrom
|
||||||
}
|
}
|
||||||
|
|
||||||
// readCachedVariant reads the cached variant of cacheKey from its opened file,
|
// readCachedVariant reads the cached variant of cacheKey that reader opened,
|
||||||
// and closes the file. It returns nil when the variant cannot be read.
|
// whose content type is contentType, and closes reader. It returns nil when
|
||||||
|
// the variant cannot be read.
|
||||||
func (p *Proxy) readCachedVariant(
|
func (p *Proxy) readCachedVariant(
|
||||||
cacheKey imagerequest.VariantKey, file imgcache.VariantFile,
|
cacheKey imagerequest.VariantKey, reader io.ReadCloser, contentType string,
|
||||||
) *processedVariant {
|
) *processedVariant {
|
||||||
defer func() { _ = file.Content.Close() }()
|
defer func() { _ = reader.Close() }()
|
||||||
|
|
||||||
data, err := io.ReadAll(file.Content)
|
data, err := io.ReadAll(reader)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
p.log.Warn("failed to read cached variant", "cache_key", cacheKey,
|
p.log.Warn("failed to read cached variant", "cache_key", cacheKey,
|
||||||
"error", err)
|
"error", err)
|
||||||
@@ -499,11 +500,7 @@ func (p *Proxy) readCachedVariant(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return &processedVariant{
|
return &processedVariant{data: data, contentType: contentType}
|
||||||
data: data,
|
|
||||||
contentType: file.ContentType,
|
|
||||||
checkedAt: file.CheckedAt,
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// fetchAndProcess fetches the source from upstream, stores it, and makes
|
// fetchAndProcess fetches the source from upstream, stores it, and makes
|
||||||
@@ -531,9 +528,7 @@ func (p *Proxy) fetchAndProcess(
|
|||||||
case variant == nil:
|
case variant == nil:
|
||||||
return nil, err
|
return nil, err
|
||||||
case errors.Is(err, errSourceNotModified):
|
case errors.Is(err, errSourceNotModified):
|
||||||
variant.checkedAt = time.Now().UTC()
|
err = p.cache.SetVariantCheckedAt(ctx, cacheKey, time.Now().UTC())
|
||||||
|
|
||||||
err = p.cache.SetVariantCheckedAt(ctx, cacheKey, variant.checkedAt)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
p.log.Warn("failed to renew cached variant", "cache_key", cacheKey,
|
p.log.Warn("failed to renew cached variant", "cache_key", cacheKey,
|
||||||
"error", err)
|
"error", err)
|
||||||
@@ -750,9 +745,6 @@ type processedVariant struct {
|
|||||||
data []byte
|
data []byte
|
||||||
contentType string
|
contentType string
|
||||||
fetchedBytes int64
|
fetchedBytes int64
|
||||||
// checkedAt is its source's check time when the variant was made or last
|
|
||||||
// renewed, which its cached file has as its modification time
|
|
||||||
checkedAt time.Time
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// processAndStore processes the image read from source, which is in
|
// processAndStore processes the image read from source, which is in
|
||||||
@@ -834,7 +826,6 @@ func (p *Proxy) processAndStore(
|
|||||||
data: processedData,
|
data: processedData,
|
||||||
contentType: processResult.ContentType,
|
contentType: processResult.ContentType,
|
||||||
fetchedBytes: fetchBytes,
|
fetchedBytes: fetchBytes,
|
||||||
checkedAt: sourceInfo.CheckedAt,
|
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -846,16 +837,13 @@ func isNegativeCacheable(err error) bool {
|
|||||||
// etagHashLength is the number of hash characters to use for ETags.
|
// etagHashLength is the number of hash characters to use for ETags.
|
||||||
const etagHashLength = 16
|
const etagHashLength = 16
|
||||||
|
|
||||||
// formatETag returns the quoted ETag of the variant of key: the first
|
// formatETag formats a VariantKey as a quoted ETag value.
|
||||||
// etagHashLength characters of key, then checkedAt in Unix nanoseconds.
|
func formatETag(key imagerequest.VariantKey) string {
|
||||||
// checkedAt is its source's check time when the variant was made or last
|
|
||||||
// renewed, so a variant made again from new source bytes gets another ETag.
|
|
||||||
func formatETag(key imagerequest.VariantKey, checkedAt time.Time) string {
|
|
||||||
hash := string(key)
|
hash := string(key)
|
||||||
// Use first 16 characters of hash for a shorter but still unique ETag
|
// Use first 16 characters of hash for a shorter but still unique ETag
|
||||||
if len(hash) > etagHashLength {
|
if len(hash) > etagHashLength {
|
||||||
hash = hash[:etagHashLength]
|
hash = hash[:etagHashLength]
|
||||||
}
|
}
|
||||||
|
|
||||||
return fmt.Sprintf(`"%s-%d"`, hash, checkedAt.UnixNano())
|
return `"` + hash + `"`
|
||||||
}
|
}
|
||||||
+62
-69
@@ -3,8 +3,8 @@
|
|||||||
// - the variants made from them
|
// - the variants made from them
|
||||||
// - the negative cache: the source URLs whose fetch failed recently
|
// - the negative cache: the source URLs whose fetch failed recently
|
||||||
//
|
//
|
||||||
// Each image is a file with a row in the database. A source is used, and its
|
// Each image is a file with a row in the database. A source and its
|
||||||
// variants are served, until the cache TTL has passed since upstream last sent
|
// variants are served until the cache TTL has passed since upstream last sent
|
||||||
// the source or confirmed it unchanged. Then the source is due to be checked
|
// the source or confirmed it unchanged. Then the source is due to be checked
|
||||||
// with upstream again (see Stale). Eviction removes:
|
// with upstream again (see Stale). Eviction removes:
|
||||||
// - the least recently used images, to keep the cache under its size limit
|
// - the least recently used images, to keep the cache under its size limit
|
||||||
@@ -45,14 +45,14 @@ const cleanExpiredBatchSize = 1000
|
|||||||
const DefaultNegativeTTL = 5 * time.Minute
|
const DefaultNegativeTTL = 5 * time.Minute
|
||||||
|
|
||||||
// DefaultCacheTTL is how long after upstream last sent a cached source, or
|
// DefaultCacheTTL is how long after upstream last sent a cached source, or
|
||||||
// answered that it had not changed, pixad uses the source and serves its
|
// answered that it had not changed, pixad serves the source and its
|
||||||
// variants without asking upstream again.
|
// variants without asking upstream again.
|
||||||
const DefaultCacheTTL = 24 * time.Hour
|
const DefaultCacheTTL = 24 * time.Hour
|
||||||
|
|
||||||
// EvictionAfterLastCheck is how long after upstream last sent a cached
|
// EvictionAfterLastCheck is how long after upstream last sent a cached
|
||||||
// source, or answered that it had not changed, the source and its variants
|
// source, or answered that it had not changed, the source and its variants
|
||||||
// are evicted (see EvictNotCheckedRecently). Until then a source whose checks
|
// are evicted (see EvictNotCheckedRecently). Until then a source whose checks
|
||||||
// fail is used.
|
// fail is served.
|
||||||
const EvictionAfterLastCheck = 30 * 24 * time.Hour
|
const EvictionAfterLastCheck = 30 * 24 * time.Hour
|
||||||
|
|
||||||
// ErrStale is returned by Variant for a variant whose file's modification
|
// ErrStale is returned by Variant for a variant whose file's modification
|
||||||
@@ -64,7 +64,7 @@ type Config struct {
|
|||||||
StateDir string
|
StateDir string
|
||||||
NegativeTTL time.Duration
|
NegativeTTL time.Duration
|
||||||
|
|
||||||
// CacheTTL is how long a cached source is used, and its variants served,
|
// CacheTTL is how long a cached source and its variants are served
|
||||||
// before upstream is asked again whether the source changed. Zero means
|
// before upstream is asked again whether the source changed. Zero means
|
||||||
// DefaultCacheTTL.
|
// DefaultCacheTTL.
|
||||||
CacheTTL time.Duration
|
CacheTTL time.Duration
|
||||||
@@ -186,43 +186,31 @@ func NewCache(
|
|||||||
return c, nil
|
return c, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// VariantFile is the open file of a cached variant, as Variant,
|
// Variant opens a cached variant and returns a reader, its size and its
|
||||||
// VariantIgnoringTTL and RenewVariant return it.
|
// content type, read from the variant file's first bytes. It opens that one
|
||||||
type VariantFile struct {
|
// file and makes no database call: the file's modification time is its
|
||||||
// Content reads the file. The caller closes it.
|
// source's check time when the variant was made or last renewed (see
|
||||||
Content io.ReadCloser
|
// PutVariant), and a variant whose time is CacheTTL or more ago is ErrStale.
|
||||||
// Size is the file's size in bytes.
|
// A variant it opens has its access time handed to the background writer
|
||||||
Size int64
|
// without waiting (see touchVariant). A variant not in the cache, or any
|
||||||
// ContentType is the MIME type of the variant's format, read from the
|
// variant of a disabled cache, is ErrNotFound; a variant file whose first
|
||||||
// file's first bytes.
|
|
||||||
ContentType string
|
|
||||||
// CheckedAt is the file's modification time: its source's check time
|
|
||||||
// when the variant was made or last renewed.
|
|
||||||
CheckedAt time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// Variant opens a cached variant and returns its file (see VariantFile). It
|
|
||||||
// opens that one file and makes no database call: the file's modification
|
|
||||||
// time is its source's check time when the variant was made or last renewed
|
|
||||||
// (see PutVariant), and a variant whose time is CacheTTL or more ago is
|
|
||||||
// ErrStale. A variant it opens has its access time handed to the background
|
|
||||||
// writer without waiting (see touchVariant). A variant not in the cache, or
|
|
||||||
// any variant of a disabled cache, is ErrNotFound; a variant file whose first
|
|
||||||
// bytes are not those of an output format is another error.
|
// bytes are not those of an output format is another error.
|
||||||
func (c *Cache) Variant(cacheKey imagerequest.VariantKey) (VariantFile, error) {
|
func (c *Cache) Variant(
|
||||||
|
cacheKey imagerequest.VariantKey,
|
||||||
|
) (io.ReadCloser, int64, string, error) {
|
||||||
if c.config.MaxBytes == 0 {
|
if c.config.MaxBytes == 0 {
|
||||||
return VariantFile{}, ErrNotFound
|
return nil, 0, "", ErrNotFound
|
||||||
}
|
}
|
||||||
|
|
||||||
file, info, err := c.variants.Open(string(cacheKey))
|
file, info, err := c.openVariant(cacheKey)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return VariantFile{}, err
|
return nil, 0, "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
if c.Stale(info.ModTime()) {
|
if c.Stale(info.ModTime()) {
|
||||||
_ = file.Close()
|
_ = file.Close()
|
||||||
|
|
||||||
return VariantFile{}, ErrStale
|
return nil, 0, "", ErrStale
|
||||||
}
|
}
|
||||||
|
|
||||||
return c.finishOpeningVariant(cacheKey, file, info)
|
return c.finishOpeningVariant(cacheKey, file, info)
|
||||||
@@ -233,14 +221,14 @@ func (c *Cache) Variant(cacheKey imagerequest.VariantKey) (VariantFile, error) {
|
|||||||
// cached, before it checks the source with upstream (see LookupVariant).
|
// cached, before it checks the source with upstream (see LookupVariant).
|
||||||
func (c *Cache) VariantIgnoringTTL(
|
func (c *Cache) VariantIgnoringTTL(
|
||||||
cacheKey imagerequest.VariantKey,
|
cacheKey imagerequest.VariantKey,
|
||||||
) (VariantFile, error) {
|
) (io.ReadCloser, int64, string, error) {
|
||||||
if c.config.MaxBytes == 0 {
|
if c.config.MaxBytes == 0 {
|
||||||
return VariantFile{}, ErrNotFound
|
return nil, 0, "", ErrNotFound
|
||||||
}
|
}
|
||||||
|
|
||||||
file, info, err := c.variants.Open(string(cacheKey))
|
file, info, err := c.openVariant(cacheKey)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return VariantFile{}, err
|
return nil, 0, "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
return c.finishOpeningVariant(cacheKey, file, info)
|
return c.finishOpeningVariant(cacheKey, file, info)
|
||||||
@@ -253,50 +241,41 @@ func (c *Cache) VariantIgnoringTTL(
|
|||||||
// ErrNotFound, as is a variant not in the cache.
|
// ErrNotFound, as is a variant not in the cache.
|
||||||
func (c *Cache) RenewVariant(
|
func (c *Cache) RenewVariant(
|
||||||
ctx context.Context, cacheKey imagerequest.VariantKey, source SourceInfo,
|
ctx context.Context, cacheKey imagerequest.VariantKey, source SourceInfo,
|
||||||
) (VariantFile, error) {
|
) (io.ReadCloser, int64, string, error) {
|
||||||
if c.config.MaxBytes == 0 {
|
if c.config.MaxBytes == 0 {
|
||||||
return VariantFile{}, ErrNotFound
|
return nil, 0, "", ErrNotFound
|
||||||
}
|
}
|
||||||
|
|
||||||
file, info, err := c.variants.Open(string(cacheKey))
|
file, info, err := c.openVariant(cacheKey)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return VariantFile{}, err
|
return nil, 0, "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
if info.ModTime().Before(source.FetchedAt) {
|
if info.ModTime().Before(source.FetchedAt) {
|
||||||
_ = file.Close()
|
_ = file.Close()
|
||||||
|
|
||||||
return VariantFile{}, ErrNotFound
|
return nil, 0, "", ErrNotFound
|
||||||
}
|
}
|
||||||
|
|
||||||
format, err := readFormat(file)
|
format, err := readFormat(file)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = file.Close()
|
_ = file.Close()
|
||||||
|
|
||||||
return VariantFile{}, err
|
return nil, 0, "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
checkedAt := info.ModTime()
|
if info.ModTime().Before(source.CheckedAt) {
|
||||||
|
|
||||||
if checkedAt.Before(source.CheckedAt) {
|
|
||||||
err = c.SetVariantCheckedAt(ctx, cacheKey, source.CheckedAt)
|
err = c.SetVariantCheckedAt(ctx, cacheKey, source.CheckedAt)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = file.Close()
|
_ = file.Close()
|
||||||
|
|
||||||
return VariantFile{}, err
|
return nil, 0, "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
checkedAt = source.CheckedAt
|
|
||||||
}
|
}
|
||||||
|
|
||||||
c.touchVariant(cacheKey)
|
c.touchVariant(cacheKey)
|
||||||
|
|
||||||
return VariantFile{
|
return file, info.Size(), format.MIMEType(), nil
|
||||||
Content: file,
|
|
||||||
Size: info.Size(),
|
|
||||||
ContentType: format.MIMEType(),
|
|
||||||
CheckedAt: checkedAt,
|
|
||||||
}, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetVariantCheckedAt records checkedAt as when upstream last sent the source
|
// SetVariantCheckedAt records checkedAt as when upstream last sent the source
|
||||||
@@ -427,7 +406,7 @@ func (c *Cache) Source(
|
|||||||
return nil, 0, "", ErrNotFound
|
return nil, 0, "", ErrNotFound
|
||||||
}
|
}
|
||||||
|
|
||||||
file, info, err := c.srcContent.Open(string(key))
|
file, size, err := c.srcContent.Open(string(key))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, 0, "", err
|
return nil, 0, "", err
|
||||||
}
|
}
|
||||||
@@ -441,7 +420,7 @@ func (c *Cache) Source(
|
|||||||
|
|
||||||
c.touchSourceContent(key)
|
c.touchSourceContent(key)
|
||||||
|
|
||||||
return file, info.Size(), format, nil
|
return file, size, format, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// SourceInfo is what the cache keeps in a cached source's row besides its
|
// SourceInfo is what the cache keeps in a cached source's row besides its
|
||||||
@@ -715,29 +694,43 @@ func (c *Cache) cleanExpiredBatch(ctx context.Context) (int64, error) {
|
|||||||
return deleted, nil
|
return deleted, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// finishOpeningVariant returns a variant file that FileStore.Open opened, with
|
// openVariant opens the file of a cached variant and returns it with its
|
||||||
// its information info, as Variant does. It reads the content type from the
|
// information, which gives its size and its modification time.
|
||||||
// file's first bytes and hands the variant's access time to the background
|
func (c *Cache) openVariant(
|
||||||
// writer (see touchVariant). It closes the file when its first bytes are not
|
cacheKey imagerequest.VariantKey,
|
||||||
// those of an output format.
|
) (*os.File, fs.FileInfo, error) {
|
||||||
|
file, _, err := c.variants.Open(string(cacheKey))
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
info, err := file.Stat()
|
||||||
|
if err != nil {
|
||||||
|
_ = file.Close()
|
||||||
|
|
||||||
|
return nil, nil, fmt.Errorf("failed to stat variant: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return file, info, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// finishOpeningVariant reads the content type of a variant file openVariant
|
||||||
|
// opened from its first bytes, hands the variant's access time to the
|
||||||
|
// background writer (see touchVariant), and returns the file as Variant does.
|
||||||
|
// It closes the file when its first bytes are not those of an output format.
|
||||||
func (c *Cache) finishOpeningVariant(
|
func (c *Cache) finishOpeningVariant(
|
||||||
cacheKey imagerequest.VariantKey, file *os.File, info fs.FileInfo,
|
cacheKey imagerequest.VariantKey, file *os.File, info fs.FileInfo,
|
||||||
) (VariantFile, error) {
|
) (io.ReadCloser, int64, string, error) {
|
||||||
format, err := readFormat(file)
|
format, err := readFormat(file)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = file.Close()
|
_ = file.Close()
|
||||||
|
|
||||||
return VariantFile{}, err
|
return nil, 0, "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
c.touchVariant(cacheKey)
|
c.touchVariant(cacheKey)
|
||||||
|
|
||||||
return VariantFile{
|
return file, info.Size(), format.MIMEType(), nil
|
||||||
Content: file,
|
|
||||||
Size: info.Size(),
|
|
||||||
ContentType: format.MIMEType(),
|
|
||||||
CheckedAt: info.ModTime(),
|
|
||||||
}, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// readFormat reads the first bytes of a source or variant file, without
|
// readFormat reads the first bytes of a source or variant file, without
|
||||||
|
|||||||
@@ -55,13 +55,11 @@ func TestVariant_ContentTypeFromFirstBytes(t *testing.T) {
|
|||||||
t.Fatalf("PutVariant(%s) error = %v", format, err)
|
t.Fatalf("PutVariant(%s) error = %v", format, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
variant, err := cache.Variant(key)
|
reader, size, contentType, err := cache.Variant(key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Variant(%s) error = %v", format, err)
|
t.Fatalf("Variant(%s) error = %v", format, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
reader, size, contentType := variant.Content, variant.Size, variant.ContentType
|
|
||||||
|
|
||||||
served, err := io.ReadAll(reader)
|
served, err := io.ReadAll(reader)
|
||||||
_ = reader.Close()
|
_ = reader.Close()
|
||||||
|
|
||||||
@@ -112,19 +110,16 @@ func TestVariant_RefusesFileOfNoOutputFormat(t *testing.T) {
|
|||||||
t.Fatalf("%s: storing the variant file: %v", tt.name, err)
|
t.Fatalf("%s: storing the variant file: %v", tt.name, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
variant, err := cache.Variant(key)
|
reader, _, contentType, err := cache.Variant(key)
|
||||||
t.Logf("%s: Variant() error = %v", tt.name, err)
|
t.Logf("%s: Variant() error = %v", tt.name, err)
|
||||||
|
|
||||||
reader, contentType := variant.Content, variant.ContentType
|
|
||||||
if reader != nil || contentType != "" || !errors.Is(err, tt.wantErr) {
|
if reader != nil || contentType != "" || !errors.Is(err, tt.wantErr) {
|
||||||
t.Errorf("%s: Variant() = reader %v, content type %q, error %v; "+
|
t.Errorf("%s: Variant() = reader %v, content type %q, error %v; "+
|
||||||
"want no reader and %v", tt.name, reader, contentType, err, tt.wantErr)
|
"want no reader and %v", tt.name, reader, contentType, err, tt.wantErr)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
variant, err := cache.Variant(testVariantKeyFour)
|
reader, _, _, err := cache.Variant(testVariantKeyFour)
|
||||||
|
|
||||||
reader := variant.Content
|
|
||||||
if reader != nil || !errors.Is(err, ErrNotFound) {
|
if reader != nil || !errors.Is(err, ErrNotFound) {
|
||||||
t.Errorf("Variant() of a variant with no file = reader %v, error %v; "+
|
t.Errorf("Variant() of a variant with no file = reader %v, error %v; "+
|
||||||
"want no reader and %v", reader, err, ErrNotFound)
|
"want no reader and %v", reader, err, ErrNotFound)
|
||||||
@@ -132,9 +127,7 @@ func TestVariant_RefusesFileOfNoOutputFormat(t *testing.T) {
|
|||||||
|
|
||||||
disabled, _ := newEvictionTestCache(t, 0)
|
disabled, _ := newEvictionTestCache(t, 0)
|
||||||
|
|
||||||
variant, err = disabled.Variant(testVariantKeyOne)
|
reader, _, _, err = disabled.Variant(testVariantKeyOne)
|
||||||
|
|
||||||
reader = variant.Content
|
|
||||||
if reader != nil || !errors.Is(err, ErrNotFound) {
|
if reader != nil || !errors.Is(err, ErrNotFound) {
|
||||||
t.Errorf("Variant() on a disabled cache = reader %v, error %v; "+
|
t.Errorf("Variant() on a disabled cache = reader %v, error %v; "+
|
||||||
"want no reader and %v", reader, err, ErrNotFound)
|
"want no reader and %v", reader, err, ErrNotFound)
|
||||||
|
|||||||
@@ -3,7 +3,6 @@ package imgcache
|
|||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io/fs"
|
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"time"
|
"time"
|
||||||
@@ -56,26 +55,26 @@ func (s *FileStore) Write(key string, data []byte) error {
|
|||||||
return writeFileSynced(path, data)
|
return writeFileSynced(path, data)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Open opens the file for key and returns it with its information, which
|
// Open opens the file for key and returns it with its size in bytes. A
|
||||||
// gives its size and its modification time. A missing file is ErrNotFound.
|
// missing file is ErrNotFound.
|
||||||
func (s *FileStore) Open(key string) (*os.File, fs.FileInfo, error) {
|
func (s *FileStore) Open(key string) (*os.File, int64, error) {
|
||||||
f, err := os.Open(s.keyToPath(key))
|
f, err := os.Open(s.keyToPath(key))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(err) {
|
||||||
return nil, nil, ErrNotFound
|
return nil, 0, ErrNotFound
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil, nil, fmt.Errorf("failed to open content: %w", err)
|
return nil, 0, fmt.Errorf("failed to open content: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
info, err := f.Stat()
|
stat, err := f.Stat()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = f.Close()
|
_ = f.Close()
|
||||||
|
|
||||||
return nil, nil, fmt.Errorf("failed to stat content: %w", err)
|
return nil, 0, fmt.Errorf("failed to stat content: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return f, info, nil
|
return f, stat.Size(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Size returns the size in bytes of the file for key. A missing file is
|
// Size returns the size in bytes of the file for key. A missing file is
|
||||||
|
|||||||
@@ -38,15 +38,15 @@ func TestFileStore_WriteAndOpen(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Open and verify size and content
|
// Open and verify size and content
|
||||||
r, info, err := storage.Open(key)
|
r, size, err := storage.Open(key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Open() error = %v", err)
|
t.Fatalf("Open() error = %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
defer func() { _ = r.Close() }()
|
defer func() { _ = r.Close() }()
|
||||||
|
|
||||||
if info.Size() != int64(len(content)) {
|
if size != int64(len(content)) {
|
||||||
t.Errorf("Open() size = %d, want %d", info.Size(), len(content))
|
t.Errorf("Open() size = %d, want %d", size, len(content))
|
||||||
}
|
}
|
||||||
|
|
||||||
loaded, err := io.ReadAll(r)
|
loaded, err := io.ReadAll(r)
|
||||||
|
|||||||
Reference in new issue
Block a user