Add fetch/conversion metrics and improve logging
FetchResult now includes: - StatusCode: HTTP status from upstream - FetchDurationMs: time to fetch from upstream - RemoteAddr: upstream server address SourceMetadata now stores: - ContentLength: size from upstream - FetchDurationMs: fetch timing - RemoteAddr: for debugging Image conversion log now includes: - host: source hostname (was missing) - path: source path (renamed from file) - convert_ms: image processing time - quality: requested quality setting - fit: requested fit mode
This commit is contained in:
@@ -238,10 +238,13 @@ func (c *Cache) StoreSource(
|
|||||||
Path: req.SourcePath,
|
Path: req.SourcePath,
|
||||||
Query: req.SourceQuery,
|
Query: req.SourceQuery,
|
||||||
ContentHash: contentHash,
|
ContentHash: contentHash,
|
||||||
StatusCode: httpStatusOK,
|
StatusCode: result.StatusCode,
|
||||||
ContentType: result.ContentType,
|
ContentType: result.ContentType,
|
||||||
|
ContentLength: result.ContentLength,
|
||||||
ResponseHeaders: result.Headers,
|
ResponseHeaders: result.Headers,
|
||||||
FetchedAt: time.Now().Unix(),
|
FetchedAt: time.Now().Unix(),
|
||||||
|
FetchDurationMs: result.FetchDurationMs,
|
||||||
|
RemoteAddr: result.RemoteAddr,
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := c.srcMetadata.Store(req.SourceHost, pathHash, meta); err != nil {
|
if err := c.srcMetadata.Store(req.SourceHost, pathHash, meta); err != nil {
|
||||||
|
|||||||
@@ -164,7 +164,12 @@ func (f *HTTPFetcher) Fetch(ctx context.Context, url string) (*FetchResult, erro
|
|||||||
req.Header.Set("User-Agent", f.config.UserAgent)
|
req.Header.Set("User-Agent", f.config.UserAgent)
|
||||||
req.Header.Set("Accept", strings.Join(f.config.AllowedContentTypes, ", "))
|
req.Header.Set("Accept", strings.Join(f.config.AllowedContentTypes, ", "))
|
||||||
|
|
||||||
|
startTime := time.Now()
|
||||||
|
|
||||||
resp, err := f.client.Do(req)
|
resp, err := f.client.Do(req)
|
||||||
|
|
||||||
|
fetchDuration := time.Since(startTime)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, context.DeadlineExceeded) {
|
if errors.Is(err, context.DeadlineExceeded) {
|
||||||
return nil, ErrUpstreamTimeout
|
return nil, ErrUpstreamTimeout
|
||||||
@@ -173,6 +178,12 @@ func (f *HTTPFetcher) Fetch(ctx context.Context, url string) (*FetchResult, erro
|
|||||||
return nil, fmt.Errorf("upstream request failed: %w", err)
|
return nil, fmt.Errorf("upstream request failed: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Get remote address if available
|
||||||
|
var remoteAddr string
|
||||||
|
if resp.Request != nil && resp.Request.URL != nil {
|
||||||
|
remoteAddr = resp.Request.Host
|
||||||
|
}
|
||||||
|
|
||||||
// Check status code
|
// Check status code
|
||||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||||
_ = resp.Body.Close()
|
_ = resp.Body.Close()
|
||||||
@@ -202,6 +213,9 @@ func (f *HTTPFetcher) Fetch(ctx context.Context, url string) (*FetchResult, erro
|
|||||||
ContentLength: resp.ContentLength,
|
ContentLength: resp.ContentLength,
|
||||||
ContentType: contentType,
|
ContentType: contentType,
|
||||||
Headers: resp.Header,
|
Headers: resp.Header,
|
||||||
|
StatusCode: resp.StatusCode,
|
||||||
|
FetchDurationMs: fetchDuration.Milliseconds(),
|
||||||
|
RemoteAddr: remoteAddr,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -178,6 +178,12 @@ type FetchResult struct {
|
|||||||
ContentType string
|
ContentType string
|
||||||
// Headers contains all response headers from upstream
|
// Headers contains all response headers from upstream
|
||||||
Headers map[string][]string
|
Headers map[string][]string
|
||||||
|
// StatusCode is the HTTP status code from upstream
|
||||||
|
StatusCode int
|
||||||
|
// FetchDurationMs is how long the fetch took in milliseconds
|
||||||
|
FetchDurationMs int64
|
||||||
|
// RemoteAddr is the IP:port of the upstream server
|
||||||
|
RemoteAddr string
|
||||||
}
|
}
|
||||||
|
|
||||||
// Processor handles image transformation (resize, format conversion)
|
// Processor handles image transformation (resize, format conversion)
|
||||||
|
|||||||
@@ -157,18 +157,23 @@ func (s *Service) fetchAndProcess(ctx context.Context, req *ImageRequest) (*Imag
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Process the image
|
// Process the image
|
||||||
|
processStart := time.Now()
|
||||||
|
|
||||||
processResult, err := s.processor.Process(ctx, bytes.NewReader(sourceData), req)
|
processResult, err := s.processor.Process(ctx, bytes.NewReader(sourceData), req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("image processing failed: %w", err)
|
return nil, fmt.Errorf("image processing failed: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
processDuration := time.Since(processStart)
|
||||||
|
|
||||||
// Log conversion details
|
// Log conversion details
|
||||||
inputSize := int64(len(sourceData))
|
inputSize := int64(len(sourceData))
|
||||||
outputSize := processResult.ContentLength
|
outputSize := processResult.ContentLength
|
||||||
sizePercent := float64(outputSize) / float64(inputSize) * 100.0 //nolint:mnd // percentage calculation
|
sizePercent := float64(outputSize) / float64(inputSize) * 100.0 //nolint:mnd // percentage calculation
|
||||||
|
|
||||||
s.log.Info("image converted",
|
s.log.Info("image converted",
|
||||||
"file", req.SourcePath,
|
"host", req.SourceHost,
|
||||||
|
"path", req.SourcePath,
|
||||||
"input_format", processResult.InputFormat,
|
"input_format", processResult.InputFormat,
|
||||||
"output_format", req.Format,
|
"output_format", req.Format,
|
||||||
"input_bytes", inputSize,
|
"input_bytes", inputSize,
|
||||||
@@ -176,6 +181,9 @@ func (s *Service) fetchAndProcess(ctx context.Context, req *ImageRequest) (*Imag
|
|||||||
"input_dimensions", fmt.Sprintf("%dx%d", processResult.InputWidth, processResult.InputHeight),
|
"input_dimensions", fmt.Sprintf("%dx%d", processResult.InputWidth, processResult.InputHeight),
|
||||||
"output_dimensions", fmt.Sprintf("%dx%d", processResult.Width, processResult.Height),
|
"output_dimensions", fmt.Sprintf("%dx%d", processResult.Width, processResult.Height),
|
||||||
"size_ratio", fmt.Sprintf("%.1f%%", sizePercent),
|
"size_ratio", fmt.Sprintf("%.1f%%", sizePercent),
|
||||||
|
"convert_ms", processDuration.Milliseconds(),
|
||||||
|
"quality", req.Quality,
|
||||||
|
"fit", req.FitMode,
|
||||||
)
|
)
|
||||||
|
|
||||||
// Store output content to cache
|
// Store output content to cache
|
||||||
|
|||||||
@@ -194,11 +194,14 @@ type SourceMetadata struct {
|
|||||||
ContentHash string `json:"content_hash,omitempty"`
|
ContentHash string `json:"content_hash,omitempty"`
|
||||||
StatusCode int `json:"status_code"`
|
StatusCode int `json:"status_code"`
|
||||||
ContentType string `json:"content_type,omitempty"`
|
ContentType string `json:"content_type,omitempty"`
|
||||||
|
ContentLength int64 `json:"content_length,omitempty"`
|
||||||
ResponseHeaders map[string][]string `json:"response_headers,omitempty"`
|
ResponseHeaders map[string][]string `json:"response_headers,omitempty"`
|
||||||
FetchedAt int64 `json:"fetched_at"`
|
FetchedAt int64 `json:"fetched_at"`
|
||||||
|
FetchDurationMs int64 `json:"fetch_duration_ms,omitempty"`
|
||||||
ExpiresAt int64 `json:"expires_at,omitempty"`
|
ExpiresAt int64 `json:"expires_at,omitempty"`
|
||||||
ETag string `json:"etag,omitempty"`
|
ETag string `json:"etag,omitempty"`
|
||||||
LastModified string `json:"last_modified,omitempty"`
|
LastModified string `json:"last_modified,omitempty"`
|
||||||
|
RemoteAddr string `json:"remote_addr,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Store writes metadata to storage.
|
// Store writes metadata to storage.
|
||||||
|
|||||||
Reference in New Issue
Block a user