Store a source the way a variant is stored, one row and one file per source URL (closes #308)
check / check (push) Canceled after 0s
check / check (push) Canceled after 0s
A source is now one source_content row and one file named by its SourceKey, the SHA-256 of its host, path and query, made as the variant key is. A store writes the file, then upserts the row, and removes the file when the row cannot be written; an eviction deletes the row, then the file, with no lock. source_metadata, the JSON files under cache/metadata/, MetadataStorage, the content lock, the eviction transaction and its test hook are gone, as is the reconciliation pass that dropped a source row whose file is missing, so TestReconciliationReadsAPageAtATime now counts 15 rows read, not 20. Two source URLs serving the same bytes store them twice. The tests of the removed sharing and lock go with them; the two eviction cancellation tests no longer use the hook. Model: opus-5-5
This commit is contained in:
@@ -1,10 +1,14 @@
|
||||
-- Migration 001: Initial schema
|
||||
-- Creates all tables for the pixa caching image proxy
|
||||
|
||||
-- Source content blobs
|
||||
-- Files stored at: cache/sources/<ab>/<cd>/<sha256>
|
||||
-- Source images, one row per source URL
|
||||
-- Files stored at: cache/sources/<ab>/<cd>/<source_key>, where source_key
|
||||
-- is the SHA-256 of the source's host, path and query
|
||||
CREATE TABLE IF NOT EXISTS source_content (
|
||||
content_hash TEXT PRIMARY KEY,
|
||||
source_key TEXT PRIMARY KEY,
|
||||
source_host TEXT NOT NULL,
|
||||
source_path TEXT NOT NULL,
|
||||
source_query TEXT NOT NULL DEFAULT '',
|
||||
content_type TEXT NOT NULL,
|
||||
size_bytes INTEGER NOT NULL,
|
||||
fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
@@ -13,30 +17,6 @@ CREATE TABLE IF NOT EXISTS source_content (
|
||||
CREATE INDEX IF NOT EXISTS idx_source_content_last_accessed
|
||||
ON source_content(last_accessed_at);
|
||||
|
||||
-- Source URL metadata - maps URLs to content hashes
|
||||
-- JSON stored at: cache/metadata/<hostname>/<path_hash>.json
|
||||
CREATE TABLE IF NOT EXISTS source_metadata (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
source_host TEXT NOT NULL,
|
||||
source_path TEXT NOT NULL,
|
||||
source_query TEXT NOT NULL DEFAULT '',
|
||||
path_hash TEXT NOT NULL,
|
||||
content_hash TEXT,
|
||||
status_code INTEGER NOT NULL,
|
||||
content_type TEXT,
|
||||
response_headers TEXT,
|
||||
fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
expires_at DATETIME,
|
||||
etag TEXT,
|
||||
last_modified TEXT,
|
||||
UNIQUE(source_host, source_path, source_query),
|
||||
FOREIGN KEY (content_hash) REFERENCES source_content(content_hash)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_source_meta_host ON source_metadata(source_host);
|
||||
CREATE INDEX IF NOT EXISTS idx_source_meta_path_hash ON source_metadata(path_hash);
|
||||
CREATE INDEX IF NOT EXISTS idx_source_meta_expires ON source_metadata(expires_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_source_meta_content_hash ON source_metadata(content_hash);
|
||||
|
||||
-- Processed variant blobs
|
||||
-- Files stored at: cache/variants/<ab>/<cd>/<cache_key> (plus a .meta
|
||||
-- sidecar with the content type). Tracked here (like source content
|
||||
|
||||
Reference in New Issue
Block a user