Detect a fetched image's format from its bytes once (closes #319)
check / check (push) Waiting to run
check / check (push) Waiting to run
magic.DetectFormat returns an imagerequest.Format. ValidateMagicBytes returns the format it checked against the upstream Content-Type. The cache reads a cached source's format from its file's first bytes, as it reads a variant's. The proxy hands the format to Process. Process refuses a format that is not a source format, and input libvips would load with another loader, such as an SVG passing as AVIF. Format.MIMEType declares each MIME string once, and the fetcher, the cache's variant read and the Accept matching use it. magic.MIMEType, magic.ImageFormat, MIMEToImageFormat, magic's SVG detection, detectFormat, formatFromString, FormatToMIME and Result.InputFormat are gone. Model: opus-5-5
This commit is contained in:
@@ -303,21 +303,22 @@ func withoutCancelKeepingDeadline(
|
||||
return context.WithDeadline(context.WithoutCancel(ctx), deadline)
|
||||
}
|
||||
|
||||
// loadCachedSource opens src's cached source, without reading it, and
|
||||
// returns it with its size; nil if it is not in the cache, cannot be read,
|
||||
// is empty or exceeds maxResponseSize.
|
||||
// loadCachedSource opens src's cached source, reading only the first bytes,
|
||||
// which give its format, and returns it with its size and its format. It
|
||||
// returns nil if the source is not in the cache, cannot be read, is in no
|
||||
// source format or exceeds maxResponseSize.
|
||||
func (p *Proxy) loadCachedSource(
|
||||
ctx context.Context, src imagerequest.Source,
|
||||
) (io.ReadCloser, int64) {
|
||||
reader, size, err := p.cache.Source(ctx, src)
|
||||
) (io.ReadCloser, int64, imagerequest.Format) {
|
||||
reader, size, format, err := p.cache.Source(ctx, src)
|
||||
if errors.Is(err, imgcache.ErrNotFound) {
|
||||
return nil, 0
|
||||
return nil, 0, ""
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
p.log.Warn("failed to load cached source, fetching", "error", err)
|
||||
|
||||
return nil, 0
|
||||
return nil, 0, ""
|
||||
}
|
||||
|
||||
p.log.Debug("using cached source", "source_key", src.Key())
|
||||
@@ -330,16 +331,10 @@ func (p *Proxy) loadCachedSource(
|
||||
"max_bytes", p.maxResponseSize,
|
||||
)
|
||||
|
||||
return nil, 0
|
||||
return nil, 0, ""
|
||||
}
|
||||
|
||||
if size == 0 {
|
||||
_ = reader.Close()
|
||||
|
||||
return nil, 0
|
||||
}
|
||||
|
||||
return reader, size
|
||||
return reader, size, format
|
||||
}
|
||||
|
||||
// processFromSourceOrFetch processes an image, using cached source content
|
||||
@@ -349,7 +344,7 @@ func (p *Proxy) processFromSourceOrFetch(
|
||||
req *imagerequest.Request,
|
||||
cacheKey imagerequest.VariantKey,
|
||||
) (*processedVariant, error) {
|
||||
source, sourceSize := p.loadCachedSource(ctx, req.Source)
|
||||
source, sourceSize, sourceFormat := p.loadCachedSource(ctx, req.Source)
|
||||
|
||||
// Fetch from upstream if we don't have source data or it's empty
|
||||
if source == nil {
|
||||
@@ -361,7 +356,7 @@ func (p *Proxy) processFromSourceOrFetch(
|
||||
// Process using cached source; nothing was fetched from upstream. The
|
||||
// image processor reads the source only once it has a processing slot,
|
||||
// so a request waiting for one holds none of it in memory.
|
||||
return p.processAndStore(ctx, req, cacheKey, source, sourceSize)
|
||||
return p.processAndStore(ctx, req, cacheKey, source, sourceFormat, sourceSize)
|
||||
}
|
||||
|
||||
// fetchAndProcess fetches from upstream, processes, and caches the result.
|
||||
@@ -442,10 +437,11 @@ func (p *Proxy) fetchAndProcess(
|
||||
"cipher", fetchResult.TLSCipherSuite,
|
||||
)
|
||||
|
||||
// Validate magic bytes match content type. A body that does not is the
|
||||
// upstream's error, so it gets a negative cache entry, as an upstream
|
||||
// error status does.
|
||||
err = magic.ValidateMagicBytes(sourceData, fetchResult.ContentType)
|
||||
// The source's format, detected from its first bytes, which must match
|
||||
// its content type. A body that does not is the upstream's error, so it
|
||||
// gets a negative cache entry, as an upstream error status does.
|
||||
sourceFormat, err := magic.ValidateMagicBytes(sourceData,
|
||||
fetchResult.ContentType)
|
||||
if err != nil {
|
||||
p.log.Warn("source refused", "request_id", middleware.GetReqID(ctx),
|
||||
"host", req.Source.Host, "path", req.Source.Path,
|
||||
@@ -463,7 +459,7 @@ func (p *Proxy) fetchAndProcess(
|
||||
}
|
||||
|
||||
return p.processAndStore(
|
||||
ctx, req, cacheKey, bytes.NewReader(sourceData), fetchBytes,
|
||||
ctx, req, cacheKey, bytes.NewReader(sourceData), sourceFormat, fetchBytes,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -497,13 +493,14 @@ type processedVariant struct {
|
||||
fetchedBytes int64
|
||||
}
|
||||
|
||||
// processAndStore processes the image read from source and stores the
|
||||
// result.
|
||||
// processAndStore processes the image read from source, which is in
|
||||
// sourceFormat, and stores the result.
|
||||
func (p *Proxy) processAndStore(
|
||||
ctx context.Context,
|
||||
req *imagerequest.Request,
|
||||
cacheKey imagerequest.VariantKey,
|
||||
source io.Reader,
|
||||
sourceFormat imagerequest.Format,
|
||||
fetchBytes int64,
|
||||
) (*processedVariant, error) {
|
||||
// Process the image
|
||||
@@ -511,7 +508,8 @@ func (p *Proxy) processAndStore(
|
||||
|
||||
// ctx has the deadline of the request doing the processing, which can
|
||||
// end while a request waiting for its image has time left
|
||||
processResult, err := p.processor.Process(ctx, source, &req.Transform)
|
||||
processResult, err := p.processor.Process(ctx, source, sourceFormat,
|
||||
&req.Transform)
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
return nil, fmt.Errorf("%w: %w", ErrProcessingTimedOut, err)
|
||||
}
|
||||
@@ -546,7 +544,7 @@ func (p *Proxy) processAndStore(
|
||||
"request_id", middleware.GetReqID(ctx),
|
||||
"host", req.Source.Host,
|
||||
"path", req.Source.Path,
|
||||
"src_format", processResult.InputFormat,
|
||||
"src_format", sourceFormat,
|
||||
"dst_format", req.Transform.Format,
|
||||
"src_bytes", fetchBytes,
|
||||
"dst_bytes", outputSize,
|
||||
|
||||
Reference in New Issue
Block a user