- Add ETag generation based on output content hash (first 16 chars) - Add ContentLength to ImageResponse from cache - Add LoadWithSize method to ContentStorage - Add GetOutputWithSize method to Cache - Handle HEAD requests returning headers only - Handle If-None-Match conditional requests returning 304 - Register HEAD route for image proxy endpoint
321 lines
8.4 KiB
Go
321 lines
8.4 KiB
Go
package imgcache
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
)
|
|
|
|
// Storage constants.
|
|
const (
|
|
// StorageDirPerm is the permission mode for storage directories.
|
|
StorageDirPerm = 0750
|
|
// MinHashLength is the minimum hash length for path splitting.
|
|
MinHashLength = 4
|
|
)
|
|
|
|
// Storage errors.
|
|
var (
|
|
ErrNotFound = errors.New("content not found")
|
|
)
|
|
|
|
// ContentStorage handles content-addressable file storage.
|
|
// Files are stored at: <basedir>/<ab>/<cd>/<abcdef...sha256>
|
|
type ContentStorage struct {
|
|
baseDir string
|
|
}
|
|
|
|
// NewContentStorage creates a new content storage at the given base directory.
|
|
func NewContentStorage(baseDir string) (*ContentStorage, error) {
|
|
if err := os.MkdirAll(baseDir, StorageDirPerm); err != nil {
|
|
return nil, fmt.Errorf("failed to create storage directory: %w", err)
|
|
}
|
|
|
|
return &ContentStorage{baseDir: baseDir}, nil
|
|
}
|
|
|
|
// Store writes content to storage and returns its SHA256 hash.
|
|
// The content is read fully into memory to compute the hash before writing.
|
|
func (s *ContentStorage) Store(r io.Reader) (hash string, size int64, err error) {
|
|
// Read all content to compute hash
|
|
data, err := io.ReadAll(r)
|
|
if err != nil {
|
|
return "", 0, fmt.Errorf("failed to read content: %w", err)
|
|
}
|
|
|
|
// Compute hash
|
|
h := sha256.Sum256(data)
|
|
hash = hex.EncodeToString(h[:])
|
|
size = int64(len(data))
|
|
|
|
// Build path: <basedir>/<ab>/<cd>/<hash>
|
|
path := s.hashToPath(hash)
|
|
|
|
// Check if already exists
|
|
if _, err := os.Stat(path); err == nil {
|
|
return hash, size, nil
|
|
}
|
|
|
|
// Create directory structure
|
|
dir := filepath.Dir(path)
|
|
if err := os.MkdirAll(dir, StorageDirPerm); err != nil {
|
|
return "", 0, fmt.Errorf("failed to create directory: %w", err)
|
|
}
|
|
|
|
// Write to temp file first, then rename for atomicity
|
|
tmpFile, err := os.CreateTemp(dir, ".tmp-*")
|
|
if err != nil {
|
|
return "", 0, fmt.Errorf("failed to create temp file: %w", err)
|
|
}
|
|
tmpPath := tmpFile.Name()
|
|
|
|
defer func() {
|
|
if err != nil {
|
|
_ = os.Remove(tmpPath)
|
|
}
|
|
}()
|
|
|
|
if _, err := tmpFile.Write(data); err != nil {
|
|
_ = tmpFile.Close()
|
|
|
|
return "", 0, fmt.Errorf("failed to write content: %w", err)
|
|
}
|
|
|
|
if err := tmpFile.Close(); err != nil {
|
|
return "", 0, fmt.Errorf("failed to close temp file: %w", err)
|
|
}
|
|
|
|
// Atomic rename
|
|
if err := os.Rename(tmpPath, path); err != nil {
|
|
return "", 0, fmt.Errorf("failed to rename temp file: %w", err)
|
|
}
|
|
|
|
return hash, size, nil
|
|
}
|
|
|
|
// Load returns a reader for the content with the given hash.
|
|
func (s *ContentStorage) Load(hash string) (io.ReadCloser, error) {
|
|
path := s.hashToPath(hash)
|
|
|
|
f, err := os.Open(path) //nolint:gosec // content-addressable path from hash
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return nil, ErrNotFound
|
|
}
|
|
|
|
return nil, fmt.Errorf("failed to open content: %w", err)
|
|
}
|
|
|
|
return f, nil
|
|
}
|
|
|
|
// LoadWithSize returns a reader and file size for the content with the given hash.
|
|
func (s *ContentStorage) LoadWithSize(hash string) (io.ReadCloser, int64, error) {
|
|
path := s.hashToPath(hash)
|
|
|
|
f, err := os.Open(path) //nolint:gosec // content-addressable path from hash
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return nil, 0, ErrNotFound
|
|
}
|
|
|
|
return nil, 0, fmt.Errorf("failed to open content: %w", err)
|
|
}
|
|
|
|
stat, err := f.Stat()
|
|
if err != nil {
|
|
_ = f.Close()
|
|
|
|
return nil, 0, fmt.Errorf("failed to stat content: %w", err)
|
|
}
|
|
|
|
return f, stat.Size(), nil
|
|
}
|
|
|
|
// Delete removes content with the given hash.
|
|
func (s *ContentStorage) Delete(hash string) error {
|
|
path := s.hashToPath(hash)
|
|
|
|
err := os.Remove(path)
|
|
if err != nil && !os.IsNotExist(err) {
|
|
return fmt.Errorf("failed to delete content: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Exists checks if content with the given hash exists.
|
|
func (s *ContentStorage) Exists(hash string) bool {
|
|
path := s.hashToPath(hash)
|
|
_, err := os.Stat(path)
|
|
|
|
return err == nil
|
|
}
|
|
|
|
// Path returns the file path for a given hash.
|
|
func (s *ContentStorage) Path(hash string) string {
|
|
return s.hashToPath(hash)
|
|
}
|
|
|
|
// hashToPath converts a hash to a file path: <basedir>/<ab>/<cd>/<hash>
|
|
func (s *ContentStorage) hashToPath(hash string) string {
|
|
if len(hash) < MinHashLength {
|
|
return filepath.Join(s.baseDir, hash)
|
|
}
|
|
|
|
return filepath.Join(s.baseDir, hash[0:2], hash[2:4], hash)
|
|
}
|
|
|
|
// MetadataStorage handles JSON metadata file storage.
|
|
// Files are stored at: <basedir>/<hostname>/<path_hash>.json
|
|
type MetadataStorage struct {
|
|
baseDir string
|
|
}
|
|
|
|
// NewMetadataStorage creates a new metadata storage at the given base directory.
|
|
func NewMetadataStorage(baseDir string) (*MetadataStorage, error) {
|
|
if err := os.MkdirAll(baseDir, StorageDirPerm); err != nil {
|
|
return nil, fmt.Errorf("failed to create metadata directory: %w", err)
|
|
}
|
|
|
|
return &MetadataStorage{baseDir: baseDir}, nil
|
|
}
|
|
|
|
// SourceMetadata represents cached metadata about a source URL.
|
|
type SourceMetadata struct {
|
|
Host string `json:"host"`
|
|
Path string `json:"path"`
|
|
Query string `json:"query,omitempty"`
|
|
ContentHash string `json:"content_hash,omitempty"`
|
|
StatusCode int `json:"status_code"`
|
|
ContentType string `json:"content_type,omitempty"`
|
|
ResponseHeaders map[string][]string `json:"response_headers,omitempty"`
|
|
FetchedAt int64 `json:"fetched_at"`
|
|
ExpiresAt int64 `json:"expires_at,omitempty"`
|
|
ETag string `json:"etag,omitempty"`
|
|
LastModified string `json:"last_modified,omitempty"`
|
|
}
|
|
|
|
// Store writes metadata to storage.
|
|
func (s *MetadataStorage) Store(host, pathHash string, meta *SourceMetadata) error {
|
|
path := s.metaPath(host, pathHash)
|
|
|
|
// Create directory structure
|
|
dir := filepath.Dir(path)
|
|
if err := os.MkdirAll(dir, StorageDirPerm); err != nil {
|
|
return fmt.Errorf("failed to create directory: %w", err)
|
|
}
|
|
|
|
// Marshal to JSON
|
|
data, err := json.MarshalIndent(meta, "", " ")
|
|
if err != nil {
|
|
return fmt.Errorf("failed to marshal metadata: %w", err)
|
|
}
|
|
|
|
// Write to temp file first, then rename for atomicity
|
|
tmpFile, err := os.CreateTemp(dir, ".tmp-*.json")
|
|
if err != nil {
|
|
return fmt.Errorf("failed to create temp file: %w", err)
|
|
}
|
|
tmpPath := tmpFile.Name()
|
|
|
|
defer func() {
|
|
if err != nil {
|
|
_ = os.Remove(tmpPath)
|
|
}
|
|
}()
|
|
|
|
if _, err := tmpFile.Write(data); err != nil {
|
|
_ = tmpFile.Close()
|
|
|
|
return fmt.Errorf("failed to write metadata: %w", err)
|
|
}
|
|
|
|
if err := tmpFile.Close(); err != nil {
|
|
return fmt.Errorf("failed to close temp file: %w", err)
|
|
}
|
|
|
|
// Atomic rename
|
|
if err := os.Rename(tmpPath, path); err != nil {
|
|
return fmt.Errorf("failed to rename temp file: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Load reads metadata from storage.
|
|
func (s *MetadataStorage) Load(host, pathHash string) (*SourceMetadata, error) {
|
|
path := s.metaPath(host, pathHash)
|
|
|
|
data, err := os.ReadFile(path) //nolint:gosec // path derived from host+hash
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return nil, ErrNotFound
|
|
}
|
|
|
|
return nil, fmt.Errorf("failed to read metadata: %w", err)
|
|
}
|
|
|
|
var meta SourceMetadata
|
|
if err := json.Unmarshal(data, &meta); err != nil {
|
|
return nil, fmt.Errorf("failed to unmarshal metadata: %w", err)
|
|
}
|
|
|
|
return &meta, nil
|
|
}
|
|
|
|
// Delete removes metadata for the given host and path hash.
|
|
func (s *MetadataStorage) Delete(host, pathHash string) error {
|
|
path := s.metaPath(host, pathHash)
|
|
|
|
err := os.Remove(path)
|
|
if err != nil && !os.IsNotExist(err) {
|
|
return fmt.Errorf("failed to delete metadata: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Exists checks if metadata exists for the given host and path hash.
|
|
func (s *MetadataStorage) Exists(host, pathHash string) bool {
|
|
path := s.metaPath(host, pathHash)
|
|
_, err := os.Stat(path)
|
|
|
|
return err == nil
|
|
}
|
|
|
|
// metaPath returns the file path for metadata: <basedir>/<host>/<path_hash>.json
|
|
func (s *MetadataStorage) metaPath(host, pathHash string) string {
|
|
return filepath.Join(s.baseDir, host, pathHash+".json")
|
|
}
|
|
|
|
// HashPath computes the SHA256 hash of a path string.
|
|
func HashPath(path string) string {
|
|
h := sha256.Sum256([]byte(path))
|
|
|
|
return hex.EncodeToString(h[:])
|
|
}
|
|
|
|
// CacheKey generates a unique cache key for a request.
|
|
// Format: sha256(host:path:query:width:height:format:quality:fit_mode)
|
|
func CacheKey(req *ImageRequest) string {
|
|
data := fmt.Sprintf("%s:%s:%s:%d:%d:%s:%d:%s",
|
|
req.SourceHost,
|
|
req.SourcePath,
|
|
req.SourceQuery,
|
|
req.Size.Width,
|
|
req.Size.Height,
|
|
req.Format,
|
|
req.Quality,
|
|
req.FitMode,
|
|
)
|
|
h := sha256.Sum256([]byte(data))
|
|
|
|
return hex.EncodeToString(h[:])
|
|
}
|