Eviction no longer reads a whole table while requests wait on the database (closes #227)
check / check (push) Waiting to run
check / check (push) Waiting to run
UsageBytes now reads the new cache_usage row, which triggers on source_content and variant_content keep up to date in the statement that adds, removes or resizes a row. The reconciliation pass reads both tables 1000 rows per query, sums them, and corrects the total when it differs, unless a row changed while it summed. Source rows now get last_accessed_at when added, so choosing source images to evict reads that column's index instead of sorting the whole table. Stats still sums the tables, now in pages: an existing test drops both tables and expects that sum to fail. Model: opus-5-5
This commit is contained in:
@@ -3,14 +3,12 @@
|
||||
|
||||
-- Source content blobs
|
||||
-- Files stored at: cache/sources/<ab>/<cd>/<sha256>
|
||||
-- last_accessed_at is NULL until the first LRU touch; eviction falls
|
||||
-- back to fetched_at for rows that have never been touched.
|
||||
CREATE TABLE IF NOT EXISTS source_content (
|
||||
content_hash TEXT PRIMARY KEY,
|
||||
content_type TEXT NOT NULL,
|
||||
size_bytes INTEGER NOT NULL,
|
||||
fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
last_accessed_at DATETIME
|
||||
last_accessed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_source_content_last_accessed
|
||||
ON source_content(last_accessed_at);
|
||||
@@ -42,9 +40,8 @@ CREATE INDEX IF NOT EXISTS idx_source_meta_content_hash ON source_metadata(conte
|
||||
-- Processed variant blobs
|
||||
-- Files stored at: cache/variants/<ab>/<cd>/<cache_key> (plus a .meta
|
||||
-- sidecar with the content type). Tracked here (like source content
|
||||
-- blobs above) so total cache usage can be computed with a SUM query,
|
||||
-- never a directory scan, and so LRU eviction has a timestamp to order
|
||||
-- on.
|
||||
-- blobs above) so total cache usage is known without a directory scan,
|
||||
-- and so LRU eviction has a timestamp to order on.
|
||||
CREATE TABLE IF NOT EXISTS variant_content (
|
||||
cache_key TEXT PRIMARY KEY,
|
||||
size_bytes INTEGER NOT NULL,
|
||||
@@ -55,6 +52,74 @@ CREATE TABLE IF NOT EXISTS variant_content (
|
||||
CREATE INDEX IF NOT EXISTS idx_variant_content_last_accessed
|
||||
ON variant_content(last_accessed_at);
|
||||
|
||||
-- Total cache usage: the sum of size_bytes over source_content and
|
||||
-- variant_content, kept by the triggers below in the same statement
|
||||
-- that adds, removes or resizes a row, so eviction reads this one row
|
||||
-- instead of summing both tables. Each trigger also adds one to
|
||||
-- change_count; the reconciliation pass, which sums both tables a page
|
||||
-- at a time, corrects total_size_bytes only if change_count did not
|
||||
-- move while it summed.
|
||||
CREATE TABLE IF NOT EXISTS cache_usage (
|
||||
id INTEGER PRIMARY KEY CHECK (id = 1),
|
||||
total_size_bytes INTEGER NOT NULL DEFAULT 0,
|
||||
change_count INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
INSERT OR IGNORE INTO cache_usage (id) VALUES (1);
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS source_content_usage_insert
|
||||
AFTER INSERT ON source_content
|
||||
BEGIN
|
||||
UPDATE cache_usage
|
||||
SET total_size_bytes = total_size_bytes + NEW.size_bytes,
|
||||
change_count = change_count + 1
|
||||
WHERE id = 1;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS source_content_usage_delete
|
||||
AFTER DELETE ON source_content
|
||||
BEGIN
|
||||
UPDATE cache_usage
|
||||
SET total_size_bytes = total_size_bytes - OLD.size_bytes,
|
||||
change_count = change_count + 1
|
||||
WHERE id = 1;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS source_content_usage_update
|
||||
AFTER UPDATE OF size_bytes ON source_content
|
||||
BEGIN
|
||||
UPDATE cache_usage
|
||||
SET total_size_bytes = total_size_bytes - OLD.size_bytes + NEW.size_bytes,
|
||||
change_count = change_count + 1
|
||||
WHERE id = 1;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS variant_content_usage_insert
|
||||
AFTER INSERT ON variant_content
|
||||
BEGIN
|
||||
UPDATE cache_usage
|
||||
SET total_size_bytes = total_size_bytes + NEW.size_bytes,
|
||||
change_count = change_count + 1
|
||||
WHERE id = 1;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS variant_content_usage_delete
|
||||
AFTER DELETE ON variant_content
|
||||
BEGIN
|
||||
UPDATE cache_usage
|
||||
SET total_size_bytes = total_size_bytes - OLD.size_bytes,
|
||||
change_count = change_count + 1
|
||||
WHERE id = 1;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS variant_content_usage_update
|
||||
AFTER UPDATE OF size_bytes ON variant_content
|
||||
BEGIN
|
||||
UPDATE cache_usage
|
||||
SET total_size_bytes = total_size_bytes - OLD.size_bytes + NEW.size_bytes,
|
||||
change_count = change_count + 1
|
||||
WHERE id = 1;
|
||||
END;
|
||||
|
||||
-- Output/transformed content blobs
|
||||
-- Not written: transformed images are stored in cache/variants and
|
||||
-- tracked in variant_content above.
|
||||
|
||||
Reference in New Issue
Block a user