Files
pixa/internal/imgcache/storage.go
sneak 15d9439e3d Add fetch/conversion metrics and improve logging
FetchResult now includes:
- StatusCode: HTTP status from upstream
- FetchDurationMs: time to fetch from upstream
- RemoteAddr: upstream server address

SourceMetadata now stores:
- ContentLength: size from upstream
- FetchDurationMs: fetch timing
- RemoteAddr: for debugging

Image conversion log now includes:
- host: source hostname (was missing)
- path: source path (renamed from file)
- convert_ms: image processing time
- quality: requested quality setting
- fit: requested fit mode
2026-01-08 12:34:26 -08:00

324 lines
8.6 KiB
Go

package imgcache
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
)
// Storage constants.
const (
// StorageDirPerm is the permission mode for storage directories.
StorageDirPerm = 0750
// MinHashLength is the minimum hash length for path splitting.
MinHashLength = 4
)
// Storage errors.
var (
ErrNotFound = errors.New("content not found")
)
// ContentStorage handles content-addressable file storage.
// Files are stored at: <basedir>/<ab>/<cd>/<abcdef...sha256>
type ContentStorage struct {
baseDir string
}
// NewContentStorage creates a new content storage at the given base directory.
func NewContentStorage(baseDir string) (*ContentStorage, error) {
if err := os.MkdirAll(baseDir, StorageDirPerm); err != nil {
return nil, fmt.Errorf("failed to create storage directory: %w", err)
}
return &ContentStorage{baseDir: baseDir}, nil
}
// Store writes content to storage and returns its SHA256 hash.
// The content is read fully into memory to compute the hash before writing.
func (s *ContentStorage) Store(r io.Reader) (hash string, size int64, err error) {
// Read all content to compute hash
data, err := io.ReadAll(r)
if err != nil {
return "", 0, fmt.Errorf("failed to read content: %w", err)
}
// Compute hash
h := sha256.Sum256(data)
hash = hex.EncodeToString(h[:])
size = int64(len(data))
// Build path: <basedir>/<ab>/<cd>/<hash>
path := s.hashToPath(hash)
// Check if already exists
if _, err := os.Stat(path); err == nil {
return hash, size, nil
}
// Create directory structure
dir := filepath.Dir(path)
if err := os.MkdirAll(dir, StorageDirPerm); err != nil {
return "", 0, fmt.Errorf("failed to create directory: %w", err)
}
// Write to temp file first, then rename for atomicity
tmpFile, err := os.CreateTemp(dir, ".tmp-*")
if err != nil {
return "", 0, fmt.Errorf("failed to create temp file: %w", err)
}
tmpPath := tmpFile.Name()
defer func() {
if err != nil {
_ = os.Remove(tmpPath)
}
}()
if _, err := tmpFile.Write(data); err != nil {
_ = tmpFile.Close()
return "", 0, fmt.Errorf("failed to write content: %w", err)
}
if err := tmpFile.Close(); err != nil {
return "", 0, fmt.Errorf("failed to close temp file: %w", err)
}
// Atomic rename
if err := os.Rename(tmpPath, path); err != nil {
return "", 0, fmt.Errorf("failed to rename temp file: %w", err)
}
return hash, size, nil
}
// Load returns a reader for the content with the given hash.
func (s *ContentStorage) Load(hash string) (io.ReadCloser, error) {
path := s.hashToPath(hash)
f, err := os.Open(path) //nolint:gosec // content-addressable path from hash
if err != nil {
if os.IsNotExist(err) {
return nil, ErrNotFound
}
return nil, fmt.Errorf("failed to open content: %w", err)
}
return f, nil
}
// LoadWithSize returns a reader and file size for the content with the given hash.
func (s *ContentStorage) LoadWithSize(hash string) (io.ReadCloser, int64, error) {
path := s.hashToPath(hash)
f, err := os.Open(path) //nolint:gosec // content-addressable path from hash
if err != nil {
if os.IsNotExist(err) {
return nil, 0, ErrNotFound
}
return nil, 0, fmt.Errorf("failed to open content: %w", err)
}
stat, err := f.Stat()
if err != nil {
_ = f.Close()
return nil, 0, fmt.Errorf("failed to stat content: %w", err)
}
return f, stat.Size(), nil
}
// Delete removes content with the given hash.
func (s *ContentStorage) Delete(hash string) error {
path := s.hashToPath(hash)
err := os.Remove(path)
if err != nil && !os.IsNotExist(err) {
return fmt.Errorf("failed to delete content: %w", err)
}
return nil
}
// Exists checks if content with the given hash exists.
func (s *ContentStorage) Exists(hash string) bool {
path := s.hashToPath(hash)
_, err := os.Stat(path)
return err == nil
}
// Path returns the file path for a given hash.
func (s *ContentStorage) Path(hash string) string {
return s.hashToPath(hash)
}
// hashToPath converts a hash to a file path: <basedir>/<ab>/<cd>/<hash>
func (s *ContentStorage) hashToPath(hash string) string {
if len(hash) < MinHashLength {
return filepath.Join(s.baseDir, hash)
}
return filepath.Join(s.baseDir, hash[0:2], hash[2:4], hash)
}
// MetadataStorage handles JSON metadata file storage.
// Files are stored at: <basedir>/<hostname>/<path_hash>.json
type MetadataStorage struct {
baseDir string
}
// NewMetadataStorage creates a new metadata storage at the given base directory.
func NewMetadataStorage(baseDir string) (*MetadataStorage, error) {
if err := os.MkdirAll(baseDir, StorageDirPerm); err != nil {
return nil, fmt.Errorf("failed to create metadata directory: %w", err)
}
return &MetadataStorage{baseDir: baseDir}, nil
}
// SourceMetadata represents cached metadata about a source URL.
type SourceMetadata struct {
Host string `json:"host"`
Path string `json:"path"`
Query string `json:"query,omitempty"`
ContentHash string `json:"content_hash,omitempty"`
StatusCode int `json:"status_code"`
ContentType string `json:"content_type,omitempty"`
ContentLength int64 `json:"content_length,omitempty"`
ResponseHeaders map[string][]string `json:"response_headers,omitempty"`
FetchedAt int64 `json:"fetched_at"`
FetchDurationMs int64 `json:"fetch_duration_ms,omitempty"`
ExpiresAt int64 `json:"expires_at,omitempty"`
ETag string `json:"etag,omitempty"`
LastModified string `json:"last_modified,omitempty"`
RemoteAddr string `json:"remote_addr,omitempty"`
}
// Store writes metadata to storage.
func (s *MetadataStorage) Store(host, pathHash string, meta *SourceMetadata) error {
path := s.metaPath(host, pathHash)
// Create directory structure
dir := filepath.Dir(path)
if err := os.MkdirAll(dir, StorageDirPerm); err != nil {
return fmt.Errorf("failed to create directory: %w", err)
}
// Marshal to JSON
data, err := json.MarshalIndent(meta, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal metadata: %w", err)
}
// Write to temp file first, then rename for atomicity
tmpFile, err := os.CreateTemp(dir, ".tmp-*.json")
if err != nil {
return fmt.Errorf("failed to create temp file: %w", err)
}
tmpPath := tmpFile.Name()
defer func() {
if err != nil {
_ = os.Remove(tmpPath)
}
}()
if _, err := tmpFile.Write(data); err != nil {
_ = tmpFile.Close()
return fmt.Errorf("failed to write metadata: %w", err)
}
if err := tmpFile.Close(); err != nil {
return fmt.Errorf("failed to close temp file: %w", err)
}
// Atomic rename
if err := os.Rename(tmpPath, path); err != nil {
return fmt.Errorf("failed to rename temp file: %w", err)
}
return nil
}
// Load reads metadata from storage.
func (s *MetadataStorage) Load(host, pathHash string) (*SourceMetadata, error) {
path := s.metaPath(host, pathHash)
data, err := os.ReadFile(path) //nolint:gosec // path derived from host+hash
if err != nil {
if os.IsNotExist(err) {
return nil, ErrNotFound
}
return nil, fmt.Errorf("failed to read metadata: %w", err)
}
var meta SourceMetadata
if err := json.Unmarshal(data, &meta); err != nil {
return nil, fmt.Errorf("failed to unmarshal metadata: %w", err)
}
return &meta, nil
}
// Delete removes metadata for the given host and path hash.
func (s *MetadataStorage) Delete(host, pathHash string) error {
path := s.metaPath(host, pathHash)
err := os.Remove(path)
if err != nil && !os.IsNotExist(err) {
return fmt.Errorf("failed to delete metadata: %w", err)
}
return nil
}
// Exists checks if metadata exists for the given host and path hash.
func (s *MetadataStorage) Exists(host, pathHash string) bool {
path := s.metaPath(host, pathHash)
_, err := os.Stat(path)
return err == nil
}
// metaPath returns the file path for metadata: <basedir>/<host>/<path_hash>.json
func (s *MetadataStorage) metaPath(host, pathHash string) string {
return filepath.Join(s.baseDir, host, pathHash+".json")
}
// HashPath computes the SHA256 hash of a path string.
func HashPath(path string) string {
h := sha256.Sum256([]byte(path))
return hex.EncodeToString(h[:])
}
// CacheKey generates a unique cache key for a request.
// Format: sha256(host:path:query:width:height:format:quality:fit_mode)
func CacheKey(req *ImageRequest) string {
data := fmt.Sprintf("%s:%s:%s:%d:%d:%s:%d:%s",
req.SourceHost,
req.SourcePath,
req.SourceQuery,
req.Size.Width,
req.Size.Height,
req.Format,
req.Quality,
req.FitMode,
)
h := sha256.Sum256([]byte(data))
return hex.EncodeToString(h[:])
}