Store a source the way a variant is stored, one row and one file per source URL (closes #308)
check / check (push) Canceled after 0s
check / check (push) Canceled after 0s
A source is now one source_content row and one file named by its SourceKey, the SHA-256 of its host, path and query, made as the variant key is. A store writes the file, then upserts the row, and removes the file when the row cannot be written; an eviction deletes the row, then the file, with no lock. source_metadata, the JSON files under cache/metadata/, MetadataStorage, the content lock, the eviction transaction and its test hook are gone, as is the reconciliation pass that dropped a source row whose file is missing. Two source URLs serving the same bytes store them twice. Model: opus-5-5
This commit is contained in:
+36
-229
@@ -1,8 +1,6 @@
|
||||
package imgcache
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -30,14 +28,8 @@ var (
|
||||
ErrNotFound = errors.New("content not found")
|
||||
)
|
||||
|
||||
// ContentHash is a SHA256 hash of file content (hex-encoded).
|
||||
type ContentHash string
|
||||
|
||||
// PathHash is a SHA256 hash of a URL path (hex-encoded).
|
||||
type PathHash string
|
||||
|
||||
// ContentStorage handles content-addressable file storage.
|
||||
// Files are stored at: <basedir>/<ab>/<cd>/<abcdef...sha256>
|
||||
// ContentStorage handles source image file storage, one file per source
|
||||
// URL. Files are stored at: <basedir>/<ab>/<cd>/<source key>
|
||||
type ContentStorage struct {
|
||||
baseDir string
|
||||
}
|
||||
@@ -52,38 +44,19 @@ func NewContentStorage(baseDir string) (*ContentStorage, error) {
|
||||
return &ContentStorage{baseDir: baseDir}, nil
|
||||
}
|
||||
|
||||
// Store writes content to storage and returns its SHA256 hash.
|
||||
// The content is read fully into memory to compute the hash before writing.
|
||||
func (s *ContentStorage) Store(r io.Reader) (ContentHash, int64, error) {
|
||||
// Read all content to compute hash
|
||||
data, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
return "", 0, fmt.Errorf("failed to read content: %w", err)
|
||||
}
|
||||
|
||||
// Compute hash
|
||||
h := sha256.Sum256(data)
|
||||
hash := ContentHash(hex.EncodeToString(h[:]))
|
||||
size := int64(len(data))
|
||||
|
||||
err = s.writeIfAbsent(hash, data)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
|
||||
return hash, size, nil
|
||||
}
|
||||
|
||||
// StoreHashed writes pre-hashed content to storage at the path derived
|
||||
// from hash, without recomputing it. Callers that already know the
|
||||
// hash before writing (e.g. because they must hold a hash-keyed lock
|
||||
// across the whole store operation) use this instead of Store. Like
|
||||
// Store, it is idempotent: a file of the content's size already at that
|
||||
// path is left untouched, and a file of another size is replaced.
|
||||
func (s *ContentStorage) StoreHashed(
|
||||
hash ContentHash, data []byte,
|
||||
// Store writes data with writeFileSynced to the path derived from key,
|
||||
// replacing any file already there, and returns its size.
|
||||
func (s *ContentStorage) Store(
|
||||
key imagerequest.SourceKey, data []byte,
|
||||
) (int64, error) {
|
||||
err := s.writeIfAbsent(hash, data)
|
||||
path := s.keyToPath(key)
|
||||
|
||||
err := os.MkdirAll(filepath.Dir(path), StorageDirPerm)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to create directory: %w", err)
|
||||
}
|
||||
|
||||
err = writeFileSynced(path, data)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
@@ -91,11 +64,11 @@ func (s *ContentStorage) StoreHashed(
|
||||
return int64(len(data)), nil
|
||||
}
|
||||
|
||||
// Load returns a reader for the content with the given hash.
|
||||
func (s *ContentStorage) Load(hash ContentHash) (io.ReadCloser, error) {
|
||||
path := s.hashToPath(hash)
|
||||
// Load returns a reader for the content with the given key.
|
||||
func (s *ContentStorage) Load(key imagerequest.SourceKey) (io.ReadCloser, error) {
|
||||
path := s.keyToPath(key)
|
||||
|
||||
f, err := os.Open(path) //nolint:gosec // path derived from content hash
|
||||
f, err := os.Open(path) //nolint:gosec // path derived from source key
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, ErrNotFound
|
||||
@@ -107,11 +80,13 @@ func (s *ContentStorage) Load(hash ContentHash) (io.ReadCloser, error) {
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// LoadWithSize returns a reader and file size for the content with the given hash.
|
||||
func (s *ContentStorage) LoadWithSize(hash ContentHash) (io.ReadCloser, int64, error) {
|
||||
path := s.hashToPath(hash)
|
||||
// LoadWithSize returns a reader and file size for the content with the given key.
|
||||
func (s *ContentStorage) LoadWithSize(
|
||||
key imagerequest.SourceKey,
|
||||
) (io.ReadCloser, int64, error) {
|
||||
path := s.keyToPath(key)
|
||||
|
||||
f, err := os.Open(path) //nolint:gosec // path derived from content hash
|
||||
f, err := os.Open(path) //nolint:gosec // path derived from source key
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, 0, ErrNotFound
|
||||
@@ -130,9 +105,9 @@ func (s *ContentStorage) LoadWithSize(hash ContentHash) (io.ReadCloser, int64, e
|
||||
return f, stat.Size(), nil
|
||||
}
|
||||
|
||||
// Delete removes content with the given hash.
|
||||
func (s *ContentStorage) Delete(hash ContentHash) error {
|
||||
path := s.hashToPath(hash)
|
||||
// Delete removes content with the given key.
|
||||
func (s *ContentStorage) Delete(key imagerequest.SourceKey) error {
|
||||
path := s.keyToPath(key)
|
||||
|
||||
err := os.Remove(path)
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
@@ -142,34 +117,14 @@ func (s *ContentStorage) Delete(hash ContentHash) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Exists checks if content with the given hash exists.
|
||||
func (s *ContentStorage) Exists(hash ContentHash) bool {
|
||||
path := s.hashToPath(hash)
|
||||
// Exists checks if content with the given key exists.
|
||||
func (s *ContentStorage) Exists(key imagerequest.SourceKey) bool {
|
||||
path := s.keyToPath(key)
|
||||
_, err := os.Stat(path)
|
||||
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// writeIfAbsent writes data with writeFileSynced to the path derived
|
||||
// from hash, unless a file of the data's size is already there.
|
||||
func (s *ContentStorage) writeIfAbsent(hash ContentHash, data []byte) error {
|
||||
// Build path: <basedir>/<ab>/<cd>/<hash>
|
||||
path := s.hashToPath(hash)
|
||||
|
||||
// A file of another size there is damaged, and is replaced
|
||||
info, statErr := os.Stat(path)
|
||||
if statErr == nil && info.Size() == int64(len(data)) {
|
||||
return nil
|
||||
}
|
||||
|
||||
err := os.MkdirAll(filepath.Dir(path), StorageDirPerm)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create directory: %w", err)
|
||||
}
|
||||
|
||||
return writeFileSynced(path, data)
|
||||
}
|
||||
|
||||
// writeFileSynced writes data to path through a temp file in the same
|
||||
// directory: the temp file is synced to disk, renamed to path, and the
|
||||
// directory synced after, so that neither a concurrent reader nor a
|
||||
@@ -244,167 +199,19 @@ func syncDirectory(dir string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// hashToPath converts a hash to a file path: <basedir>/<ab>/<cd>/<hash>
|
||||
func (s *ContentStorage) hashToPath(hash ContentHash) string {
|
||||
h := string(hash)
|
||||
if len(h) < MinHashLength {
|
||||
return filepath.Clean(filepath.Join(s.baseDir, h))
|
||||
// keyToPath converts a key to a file path: <basedir>/<ab>/<cd>/<key>
|
||||
func (s *ContentStorage) keyToPath(key imagerequest.SourceKey) string {
|
||||
k := string(key)
|
||||
if len(k) < MinHashLength {
|
||||
return filepath.Clean(filepath.Join(s.baseDir, k))
|
||||
}
|
||||
|
||||
return filepath.Clean(filepath.Join(s.baseDir, h[0:2], h[2:4], h))
|
||||
}
|
||||
|
||||
// MetadataStorage handles JSON metadata file storage.
|
||||
// Files are stored at: <basedir>/<hostname>/<path_hash>.json
|
||||
type MetadataStorage struct {
|
||||
baseDir string
|
||||
}
|
||||
|
||||
// NewMetadataStorage creates a new metadata storage at the given base directory.
|
||||
func NewMetadataStorage(baseDir string) (*MetadataStorage, error) {
|
||||
err := os.MkdirAll(baseDir, StorageDirPerm)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create metadata directory: %w", err)
|
||||
}
|
||||
|
||||
return &MetadataStorage{baseDir: baseDir}, nil
|
||||
}
|
||||
|
||||
// SourceMetadata represents cached metadata about a source URL.
|
||||
//
|
||||
//nolint:tagliatelle // stored metadata format uses snake_case
|
||||
type SourceMetadata struct {
|
||||
Host string `json:"host"`
|
||||
Path string `json:"path"`
|
||||
Query string `json:"query,omitempty"`
|
||||
ContentHash string `json:"content_hash,omitempty"`
|
||||
StatusCode int `json:"status_code"`
|
||||
ContentType string `json:"content_type,omitempty"`
|
||||
ContentLength int64 `json:"content_length,omitempty"`
|
||||
ResponseHeaders map[string][]string `json:"response_headers,omitempty"`
|
||||
FetchedAt int64 `json:"fetched_at"`
|
||||
FetchDurationMs int64 `json:"fetch_duration_ms,omitempty"`
|
||||
ExpiresAt int64 `json:"expires_at,omitempty"`
|
||||
ETag string `json:"etag,omitempty"`
|
||||
LastModified string `json:"last_modified,omitempty"`
|
||||
RemoteAddr string `json:"remote_addr,omitempty"`
|
||||
}
|
||||
|
||||
// Store writes metadata to storage.
|
||||
func (s *MetadataStorage) Store(
|
||||
host string, pathHash PathHash, meta *SourceMetadata,
|
||||
) error {
|
||||
path := s.metaPath(host, pathHash)
|
||||
|
||||
// Create directory structure
|
||||
dir := filepath.Dir(path)
|
||||
|
||||
err := os.MkdirAll(dir, StorageDirPerm)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create directory: %w", err)
|
||||
}
|
||||
|
||||
// Marshal to JSON
|
||||
data, err := json.MarshalIndent(meta, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal metadata: %w", err)
|
||||
}
|
||||
|
||||
// Write to temp file first, then rename for atomicity
|
||||
tmpFile, err := os.CreateTemp(dir, ".tmp-*.json")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create temp file: %w", err)
|
||||
}
|
||||
|
||||
tmpPath := tmpFile.Name()
|
||||
|
||||
_, err = tmpFile.Write(data)
|
||||
if err != nil {
|
||||
_ = tmpFile.Close()
|
||||
_ = os.Remove(tmpPath)
|
||||
|
||||
return fmt.Errorf("failed to write metadata: %w", err)
|
||||
}
|
||||
|
||||
err = tmpFile.Close()
|
||||
if err != nil {
|
||||
_ = os.Remove(tmpPath)
|
||||
|
||||
return fmt.Errorf("failed to close temp file: %w", err)
|
||||
}
|
||||
|
||||
// Atomic rename
|
||||
err = os.Rename(filepath.Clean(tmpPath), filepath.Clean(path))
|
||||
if err != nil {
|
||||
_ = os.Remove(tmpPath)
|
||||
|
||||
return fmt.Errorf("failed to rename temp file: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Load reads metadata from storage.
|
||||
func (s *MetadataStorage) Load(
|
||||
host string, pathHash PathHash,
|
||||
) (*SourceMetadata, error) {
|
||||
path := s.metaPath(host, pathHash)
|
||||
|
||||
data, err := os.ReadFile(path) //nolint:gosec // path derived from host+hash
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("failed to read metadata: %w", err)
|
||||
}
|
||||
|
||||
var meta SourceMetadata
|
||||
|
||||
err = json.Unmarshal(data, &meta)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal metadata: %w", err)
|
||||
}
|
||||
|
||||
return &meta, nil
|
||||
}
|
||||
|
||||
// Delete removes metadata for the given host and path hash.
|
||||
func (s *MetadataStorage) Delete(host string, pathHash PathHash) error {
|
||||
path := s.metaPath(host, pathHash)
|
||||
|
||||
err := os.Remove(path)
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return fmt.Errorf("failed to delete metadata: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Exists checks if metadata exists for the given host and path hash.
|
||||
func (s *MetadataStorage) Exists(host string, pathHash PathHash) bool {
|
||||
path := s.metaPath(host, pathHash)
|
||||
_, err := os.Stat(path)
|
||||
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// metaPath returns the file path for metadata: <basedir>/<host>/<path_hash>.json
|
||||
func (s *MetadataStorage) metaPath(host string, pathHash PathHash) string {
|
||||
return filepath.Clean(filepath.Join(s.baseDir, host, string(pathHash)+".json"))
|
||||
}
|
||||
|
||||
// HashPath computes the SHA256 hash of a path string.
|
||||
func HashPath(path string) PathHash {
|
||||
h := sha256.Sum256([]byte(path))
|
||||
|
||||
return PathHash(hex.EncodeToString(h[:]))
|
||||
return filepath.Clean(filepath.Join(s.baseDir, k[0:2], k[2:4], k))
|
||||
}
|
||||
|
||||
// VariantStorage handles key-based file storage for processed image variants.
|
||||
// Files are stored at: <basedir>/<ab>/<cd>/<cache_key>
|
||||
// Metadata is stored at: <basedir>/<ab>/<cd>/<cache_key>.meta
|
||||
// Unlike ContentStorage, the key is provided by the caller (not computed from content).
|
||||
type VariantStorage struct {
|
||||
baseDir string
|
||||
log *slog.Logger
|
||||
|
||||
Reference in New Issue
Block a user