clawbot
bfe3861314
check / check (push) Waiting to run
Cached images were served until eviction, so upstream changes were missed. After 24 hours a request checks the source with upstream, sending its stored ETag and Last-Modified. A 304 renews it with no download or conversion. New bytes replace it; older variants are converted again. A failed check serves the cached image and puts the URL in the negative cache. Images not checked for 30 days are evicted. Variant rows keep the source's ETag and Last-Modified for checking a variant of an evicted source. A variant file's modification time is its source's check time when it was made or renewed. Hits read it without a database call. The ETag includes it, so new bytes get a new ETag. Model: opus-5-5
99 lines
2.5 KiB
Go
99 lines
2.5 KiB
Go
package httpfetcher
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io/fs"
|
|
"net/http"
|
|
"strings"
|
|
|
|
"sneak.berlin/go/pixa/internal/imagerequest"
|
|
)
|
|
|
|
// MockFetcher implements Fetcher using an embedded filesystem.
|
|
// Files are organized as: hostname/path/to/file.ext
|
|
// The source with host example.com and path /images/photo.jpg maps to
|
|
// example.com/images/photo.jpg; the source query is ignored.
|
|
type MockFetcher struct {
|
|
fs fs.FS
|
|
}
|
|
|
|
// NewMock creates a new mock fetcher backed by the given filesystem.
|
|
func NewMock(fsys fs.FS) *MockFetcher {
|
|
return &MockFetcher{fs: fsys}
|
|
}
|
|
|
|
// Fetch retrieves the source's content from the mock filesystem. It ignores
|
|
// ifNoneMatch and ifModifiedSince, and always sends the content.
|
|
func (m *MockFetcher) Fetch(
|
|
ctx context.Context, source imagerequest.Source, _, _ string,
|
|
) (*FetchResult, error) {
|
|
// Check context cancellation
|
|
select {
|
|
case <-ctx.Done():
|
|
return nil, ctx.Err()
|
|
default:
|
|
}
|
|
|
|
path := source.Host + source.Path
|
|
|
|
// Open the file
|
|
f, err := m.fs.Open(path)
|
|
if err != nil {
|
|
if errors.Is(err, fs.ErrNotExist) {
|
|
return nil, fmt.Errorf("%w: status 404", ErrUpstreamError)
|
|
}
|
|
|
|
return nil, fmt.Errorf("failed to open mock file: %w", err)
|
|
}
|
|
|
|
// Get file info for content length
|
|
stat, err := f.Stat()
|
|
if err != nil {
|
|
_ = f.Close()
|
|
|
|
return nil, fmt.Errorf("failed to stat mock file: %w", err)
|
|
}
|
|
|
|
// Detect content type from extension
|
|
contentType := detectContentTypeFromPath(path)
|
|
|
|
return &FetchResult{
|
|
Content: f,
|
|
ContentLength: stat.Size(),
|
|
ContentType: contentType,
|
|
Headers: make(http.Header),
|
|
}, nil
|
|
}
|
|
|
|
// The content types the mock fetcher gives files in no format pixa reads.
|
|
const (
|
|
contentTypeSVG = "image/svg+xml"
|
|
contentTypeOctetStream = "application/octet-stream"
|
|
)
|
|
|
|
// detectContentTypeFromPath returns the MIME type based on file extension.
|
|
func detectContentTypeFromPath(path string) string {
|
|
path = strings.ToLower(path)
|
|
|
|
switch {
|
|
case strings.HasSuffix(path, ".jpg"), strings.HasSuffix(path, ".jpeg"):
|
|
return imagerequest.FormatJPEG.MIMEType()
|
|
case strings.HasSuffix(path, ".png"):
|
|
return imagerequest.FormatPNG.MIMEType()
|
|
case strings.HasSuffix(path, ".gif"):
|
|
return imagerequest.FormatGIF.MIMEType()
|
|
case strings.HasSuffix(path, ".webp"):
|
|
return imagerequest.FormatWebP.MIMEType()
|
|
case strings.HasSuffix(path, ".avif"):
|
|
return imagerequest.FormatAVIF.MIMEType()
|
|
case strings.HasSuffix(path, ".jxl"):
|
|
return imagerequest.FormatJXL.MIMEType()
|
|
case strings.HasSuffix(path, ".svg"):
|
|
return contentTypeSVG
|
|
default:
|
|
return contentTypeOctetStream
|
|
}
|
|
}
|