Store a source the way a variant is stored, one row and one file per source URL (closes #308)
check / check (push) Canceled after 0s
check / check (push) Canceled after 0s
A source is now one source_content row and one file named by its SourceKey, the SHA-256 of its host, path and query, made as the variant key is. A store writes the file, then upserts the row, and removes the file when the row cannot be written; an eviction deletes the row, then the file, with no lock. source_metadata, the JSON files under cache/metadata/, MetadataStorage, the content lock, the eviction transaction and its test hook are gone, as is the reconciliation pass that dropped a source row whose file is missing, so TestReconciliationReadsAPageAtATime now counts 15 rows read, not 20. Two source URLs serving the same bytes store them twice. The tests of the removed sharing and lock go with them; the two eviction cancellation tests no longer use the hook. Model: opus-5-5
This commit is contained in:
+33
-226
@@ -1,8 +1,6 @@
|
||||
package imgcache
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -30,14 +28,8 @@ var (
|
||||
ErrNotFound = errors.New("content not found")
|
||||
)
|
||||
|
||||
// ContentHash is a SHA256 hash of file content (hex-encoded).
|
||||
type ContentHash string
|
||||
|
||||
// PathHash is a SHA256 hash of a URL path (hex-encoded).
|
||||
type PathHash string
|
||||
|
||||
// ContentStorage handles content-addressable file storage.
|
||||
// Files are stored at: <basedir>/<ab>/<cd>/<abcdef...sha256>
|
||||
// ContentStorage handles source image file storage, one file per source
|
||||
// URL. Files are stored at: <basedir>/<ab>/<cd>/<source key>
|
||||
type ContentStorage struct {
|
||||
baseDir string
|
||||
}
|
||||
@@ -52,38 +44,19 @@ func NewContentStorage(baseDir string) (*ContentStorage, error) {
|
||||
return &ContentStorage{baseDir: baseDir}, nil
|
||||
}
|
||||
|
||||
// Store writes content to storage and returns its SHA256 hash.
|
||||
// The content is read fully into memory to compute the hash before writing.
|
||||
func (s *ContentStorage) Store(r io.Reader) (ContentHash, int64, error) {
|
||||
// Read all content to compute hash
|
||||
data, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
return "", 0, fmt.Errorf("failed to read content: %w", err)
|
||||
}
|
||||
|
||||
// Compute hash
|
||||
h := sha256.Sum256(data)
|
||||
hash := ContentHash(hex.EncodeToString(h[:]))
|
||||
size := int64(len(data))
|
||||
|
||||
err = s.writeIfAbsent(hash, data)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
|
||||
return hash, size, nil
|
||||
}
|
||||
|
||||
// StoreHashed writes pre-hashed content to storage at the path derived
|
||||
// from hash, without recomputing it. Callers that already know the
|
||||
// hash before writing (e.g. because they must hold a hash-keyed lock
|
||||
// across the whole store operation) use this instead of Store. Like
|
||||
// Store, it is idempotent: a file of the content's size already at that
|
||||
// path is left untouched, and a file of another size is replaced.
|
||||
func (s *ContentStorage) StoreHashed(
|
||||
hash ContentHash, data []byte,
|
||||
// Store writes data with writeFileSynced to the path derived from key,
|
||||
// replacing any file already there, and returns its size.
|
||||
func (s *ContentStorage) Store(
|
||||
key imagerequest.SourceKey, data []byte,
|
||||
) (int64, error) {
|
||||
err := s.writeIfAbsent(hash, data)
|
||||
path := s.hashToPath(key)
|
||||
|
||||
err := os.MkdirAll(filepath.Dir(path), StorageDirPerm)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to create directory: %w", err)
|
||||
}
|
||||
|
||||
err = writeFileSynced(path, data)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
@@ -91,11 +64,11 @@ func (s *ContentStorage) StoreHashed(
|
||||
return int64(len(data)), nil
|
||||
}
|
||||
|
||||
// Load returns a reader for the content with the given hash.
|
||||
func (s *ContentStorage) Load(hash ContentHash) (io.ReadCloser, error) {
|
||||
path := s.hashToPath(hash)
|
||||
// Load returns a reader for the content with the given key.
|
||||
func (s *ContentStorage) Load(key imagerequest.SourceKey) (io.ReadCloser, error) {
|
||||
path := s.hashToPath(key)
|
||||
|
||||
f, err := os.Open(path) //nolint:gosec // path derived from content hash
|
||||
f, err := os.Open(path) //nolint:gosec // path derived from source key
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, ErrNotFound
|
||||
@@ -107,11 +80,13 @@ func (s *ContentStorage) Load(hash ContentHash) (io.ReadCloser, error) {
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// LoadWithSize returns a reader and file size for the content with the given hash.
|
||||
func (s *ContentStorage) LoadWithSize(hash ContentHash) (io.ReadCloser, int64, error) {
|
||||
path := s.hashToPath(hash)
|
||||
// LoadWithSize returns a reader and file size for the content with the given key.
|
||||
func (s *ContentStorage) LoadWithSize(
|
||||
key imagerequest.SourceKey,
|
||||
) (io.ReadCloser, int64, error) {
|
||||
path := s.hashToPath(key)
|
||||
|
||||
f, err := os.Open(path) //nolint:gosec // path derived from content hash
|
||||
f, err := os.Open(path) //nolint:gosec // path derived from source key
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, 0, ErrNotFound
|
||||
@@ -130,9 +105,9 @@ func (s *ContentStorage) LoadWithSize(hash ContentHash) (io.ReadCloser, int64, e
|
||||
return f, stat.Size(), nil
|
||||
}
|
||||
|
||||
// Delete removes content with the given hash.
|
||||
func (s *ContentStorage) Delete(hash ContentHash) error {
|
||||
path := s.hashToPath(hash)
|
||||
// Delete removes content with the given key.
|
||||
func (s *ContentStorage) Delete(key imagerequest.SourceKey) error {
|
||||
path := s.hashToPath(key)
|
||||
|
||||
err := os.Remove(path)
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
@@ -142,34 +117,14 @@ func (s *ContentStorage) Delete(hash ContentHash) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Exists checks if content with the given hash exists.
|
||||
func (s *ContentStorage) Exists(hash ContentHash) bool {
|
||||
path := s.hashToPath(hash)
|
||||
// Exists checks if content with the given key exists.
|
||||
func (s *ContentStorage) Exists(key imagerequest.SourceKey) bool {
|
||||
path := s.hashToPath(key)
|
||||
_, err := os.Stat(path)
|
||||
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// writeIfAbsent writes data with writeFileSynced to the path derived
|
||||
// from hash, unless a file of the data's size is already there.
|
||||
func (s *ContentStorage) writeIfAbsent(hash ContentHash, data []byte) error {
|
||||
// Build path: <basedir>/<ab>/<cd>/<hash>
|
||||
path := s.hashToPath(hash)
|
||||
|
||||
// A file of another size there is damaged, and is replaced
|
||||
info, statErr := os.Stat(path)
|
||||
if statErr == nil && info.Size() == int64(len(data)) {
|
||||
return nil
|
||||
}
|
||||
|
||||
err := os.MkdirAll(filepath.Dir(path), StorageDirPerm)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create directory: %w", err)
|
||||
}
|
||||
|
||||
return writeFileSynced(path, data)
|
||||
}
|
||||
|
||||
// writeFileSynced writes data to path through a temp file in the same
|
||||
// directory: the temp file is synced to disk, renamed to path, and the
|
||||
// directory synced after, so that neither a concurrent reader nor a
|
||||
@@ -244,9 +199,9 @@ func syncDirectory(dir string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// hashToPath converts a hash to a file path: <basedir>/<ab>/<cd>/<hash>
|
||||
func (s *ContentStorage) hashToPath(hash ContentHash) string {
|
||||
h := string(hash)
|
||||
// hashToPath converts a key to a file path: <basedir>/<ab>/<cd>/<key>
|
||||
func (s *ContentStorage) hashToPath(key imagerequest.SourceKey) string {
|
||||
h := string(key)
|
||||
if len(h) < MinHashLength {
|
||||
return filepath.Clean(filepath.Join(s.baseDir, h))
|
||||
}
|
||||
@@ -254,157 +209,9 @@ func (s *ContentStorage) hashToPath(hash ContentHash) string {
|
||||
return filepath.Clean(filepath.Join(s.baseDir, h[0:2], h[2:4], h))
|
||||
}
|
||||
|
||||
// MetadataStorage handles JSON metadata file storage.
|
||||
// Files are stored at: <basedir>/<hostname>/<path_hash>.json
|
||||
type MetadataStorage struct {
|
||||
baseDir string
|
||||
}
|
||||
|
||||
// NewMetadataStorage creates a new metadata storage at the given base directory.
|
||||
func NewMetadataStorage(baseDir string) (*MetadataStorage, error) {
|
||||
err := os.MkdirAll(baseDir, StorageDirPerm)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create metadata directory: %w", err)
|
||||
}
|
||||
|
||||
return &MetadataStorage{baseDir: baseDir}, nil
|
||||
}
|
||||
|
||||
// SourceMetadata represents cached metadata about a source URL.
|
||||
//
|
||||
//nolint:tagliatelle // stored metadata format uses snake_case
|
||||
type SourceMetadata struct {
|
||||
Host string `json:"host"`
|
||||
Path string `json:"path"`
|
||||
Query string `json:"query,omitempty"`
|
||||
ContentHash string `json:"content_hash,omitempty"`
|
||||
StatusCode int `json:"status_code"`
|
||||
ContentType string `json:"content_type,omitempty"`
|
||||
ContentLength int64 `json:"content_length,omitempty"`
|
||||
ResponseHeaders map[string][]string `json:"response_headers,omitempty"`
|
||||
FetchedAt int64 `json:"fetched_at"`
|
||||
FetchDurationMs int64 `json:"fetch_duration_ms,omitempty"`
|
||||
ExpiresAt int64 `json:"expires_at,omitempty"`
|
||||
ETag string `json:"etag,omitempty"`
|
||||
LastModified string `json:"last_modified,omitempty"`
|
||||
RemoteAddr string `json:"remote_addr,omitempty"`
|
||||
}
|
||||
|
||||
// Store writes metadata to storage.
|
||||
func (s *MetadataStorage) Store(
|
||||
host string, pathHash PathHash, meta *SourceMetadata,
|
||||
) error {
|
||||
path := s.metaPath(host, pathHash)
|
||||
|
||||
// Create directory structure
|
||||
dir := filepath.Dir(path)
|
||||
|
||||
err := os.MkdirAll(dir, StorageDirPerm)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create directory: %w", err)
|
||||
}
|
||||
|
||||
// Marshal to JSON
|
||||
data, err := json.MarshalIndent(meta, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal metadata: %w", err)
|
||||
}
|
||||
|
||||
// Write to temp file first, then rename for atomicity
|
||||
tmpFile, err := os.CreateTemp(dir, ".tmp-*.json")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create temp file: %w", err)
|
||||
}
|
||||
|
||||
tmpPath := tmpFile.Name()
|
||||
|
||||
_, err = tmpFile.Write(data)
|
||||
if err != nil {
|
||||
_ = tmpFile.Close()
|
||||
_ = os.Remove(tmpPath)
|
||||
|
||||
return fmt.Errorf("failed to write metadata: %w", err)
|
||||
}
|
||||
|
||||
err = tmpFile.Close()
|
||||
if err != nil {
|
||||
_ = os.Remove(tmpPath)
|
||||
|
||||
return fmt.Errorf("failed to close temp file: %w", err)
|
||||
}
|
||||
|
||||
// Atomic rename
|
||||
err = os.Rename(filepath.Clean(tmpPath), filepath.Clean(path))
|
||||
if err != nil {
|
||||
_ = os.Remove(tmpPath)
|
||||
|
||||
return fmt.Errorf("failed to rename temp file: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Load reads metadata from storage.
|
||||
func (s *MetadataStorage) Load(
|
||||
host string, pathHash PathHash,
|
||||
) (*SourceMetadata, error) {
|
||||
path := s.metaPath(host, pathHash)
|
||||
|
||||
data, err := os.ReadFile(path) //nolint:gosec // path derived from host+hash
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("failed to read metadata: %w", err)
|
||||
}
|
||||
|
||||
var meta SourceMetadata
|
||||
|
||||
err = json.Unmarshal(data, &meta)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal metadata: %w", err)
|
||||
}
|
||||
|
||||
return &meta, nil
|
||||
}
|
||||
|
||||
// Delete removes metadata for the given host and path hash.
|
||||
func (s *MetadataStorage) Delete(host string, pathHash PathHash) error {
|
||||
path := s.metaPath(host, pathHash)
|
||||
|
||||
err := os.Remove(path)
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return fmt.Errorf("failed to delete metadata: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Exists checks if metadata exists for the given host and path hash.
|
||||
func (s *MetadataStorage) Exists(host string, pathHash PathHash) bool {
|
||||
path := s.metaPath(host, pathHash)
|
||||
_, err := os.Stat(path)
|
||||
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// metaPath returns the file path for metadata: <basedir>/<host>/<path_hash>.json
|
||||
func (s *MetadataStorage) metaPath(host string, pathHash PathHash) string {
|
||||
return filepath.Clean(filepath.Join(s.baseDir, host, string(pathHash)+".json"))
|
||||
}
|
||||
|
||||
// HashPath computes the SHA256 hash of a path string.
|
||||
func HashPath(path string) PathHash {
|
||||
h := sha256.Sum256([]byte(path))
|
||||
|
||||
return PathHash(hex.EncodeToString(h[:]))
|
||||
}
|
||||
|
||||
// VariantStorage handles key-based file storage for processed image variants.
|
||||
// Files are stored at: <basedir>/<ab>/<cd>/<cache_key>
|
||||
// Metadata is stored at: <basedir>/<ab>/<cd>/<cache_key>.meta
|
||||
// Unlike ContentStorage, the key is provided by the caller (not computed from content).
|
||||
type VariantStorage struct {
|
||||
baseDir string
|
||||
log *slog.Logger
|
||||
|
||||
Reference in New Issue
Block a user