clawbot
3f487ee0ce
check / check (push) Canceled after 0s
A cached image was served until eviction, so a change at the same URL was never picked up. After 24 hours a request checks the source with upstream, sending its stored ETag and Last-Modified. A 304 renews it with no download or conversion. New bytes replace it, and older variants are converted again. A failed check serves the cached image and puts the URL in the negative cache. Images not checked for 30 days are evicted. Variant rows keep the source's ETag and Last-Modified, so a variant whose source was evicted is checked the same way. A variant file's modification time is its source's check time when it was made or renewed, so a hit reads no database row. Model: opus-5-5
138 lines
5.4 KiB
SQL
138 lines
5.4 KiB
SQL
-- Migration 001: Initial schema
|
|
-- Creates all tables for the pixa caching image proxy
|
|
|
|
-- Source images, one row per source URL
|
|
-- Files stored at: cache/sources/<ab>/<cd>/<source_key>, where source_key
|
|
-- is the SHA-256 of the source's host, path and query.
|
|
-- fetched_at is when the cached bytes were fetched. checked_at is when
|
|
-- upstream last sent them or answered that they had not changed.
|
|
-- upstream_etag and upstream_last_modified are the ETag and Last-Modified
|
|
-- headers upstream sent with the bytes, or '' when it sent none.
|
|
CREATE TABLE IF NOT EXISTS source_content (
|
|
source_key TEXT PRIMARY KEY,
|
|
size_bytes INTEGER NOT NULL,
|
|
last_accessed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
fetched_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
checked_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
upstream_etag TEXT NOT NULL DEFAULT '',
|
|
upstream_last_modified TEXT NOT NULL DEFAULT ''
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_source_content_last_accessed
|
|
ON source_content(last_accessed_at);
|
|
CREATE INDEX IF NOT EXISTS idx_source_content_checked
|
|
ON source_content(checked_at);
|
|
|
|
-- Processed variant blobs
|
|
-- Files stored at: cache/variants/<ab>/<cd>/<cache_key>. Tracked here
|
|
-- (like source content blobs above) so total cache usage is known
|
|
-- without a directory scan, and so LRU eviction has a timestamp to order
|
|
-- on. A cache hit does not read this table: it takes the content type
|
|
-- from the file's first bytes, and its last_accessed_at is written
|
|
-- later, in a batch. checked_at is the checked_at of the source the
|
|
-- variant was made from, when it was made or last renewed; the variant
|
|
-- file's modification time holds the same time for cache hits.
|
|
-- upstream_etag and upstream_last_modified are those of the source bytes
|
|
-- the variant was made from, so the variant can still be checked with
|
|
-- upstream after its source is evicted.
|
|
CREATE TABLE IF NOT EXISTS variant_content (
|
|
cache_key TEXT PRIMARY KEY,
|
|
size_bytes INTEGER NOT NULL,
|
|
last_accessed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
checked_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
upstream_etag TEXT NOT NULL DEFAULT '',
|
|
upstream_last_modified TEXT NOT NULL DEFAULT ''
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_variant_content_last_accessed
|
|
ON variant_content(last_accessed_at);
|
|
CREATE INDEX IF NOT EXISTS idx_variant_content_checked
|
|
ON variant_content(checked_at);
|
|
|
|
-- Total cache usage: the sum of size_bytes over source_content and
|
|
-- variant_content, kept by the triggers below in the same statement
|
|
-- that adds, removes or resizes a row, so eviction reads this one row
|
|
-- instead of summing both tables. Each trigger also adds one to
|
|
-- change_count; the reconciliation pass, which sums both tables a page
|
|
-- at a time, corrects total_size_bytes only if change_count did not
|
|
-- move while it summed.
|
|
CREATE TABLE IF NOT EXISTS cache_usage (
|
|
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
total_size_bytes INTEGER NOT NULL DEFAULT 0,
|
|
change_count INTEGER NOT NULL DEFAULT 0
|
|
);
|
|
INSERT OR IGNORE INTO cache_usage (id) VALUES (1);
|
|
|
|
CREATE TRIGGER IF NOT EXISTS source_content_usage_insert
|
|
AFTER INSERT ON source_content
|
|
BEGIN
|
|
UPDATE cache_usage
|
|
SET total_size_bytes = total_size_bytes + NEW.size_bytes,
|
|
change_count = change_count + 1
|
|
WHERE id = 1;
|
|
END;
|
|
|
|
CREATE TRIGGER IF NOT EXISTS source_content_usage_delete
|
|
AFTER DELETE ON source_content
|
|
BEGIN
|
|
UPDATE cache_usage
|
|
SET total_size_bytes = total_size_bytes - OLD.size_bytes,
|
|
change_count = change_count + 1
|
|
WHERE id = 1;
|
|
END;
|
|
|
|
CREATE TRIGGER IF NOT EXISTS source_content_usage_update
|
|
AFTER UPDATE OF size_bytes ON source_content
|
|
BEGIN
|
|
UPDATE cache_usage
|
|
SET total_size_bytes = total_size_bytes - OLD.size_bytes + NEW.size_bytes,
|
|
change_count = change_count + 1
|
|
WHERE id = 1;
|
|
END;
|
|
|
|
CREATE TRIGGER IF NOT EXISTS variant_content_usage_insert
|
|
AFTER INSERT ON variant_content
|
|
BEGIN
|
|
UPDATE cache_usage
|
|
SET total_size_bytes = total_size_bytes + NEW.size_bytes,
|
|
change_count = change_count + 1
|
|
WHERE id = 1;
|
|
END;
|
|
|
|
CREATE TRIGGER IF NOT EXISTS variant_content_usage_delete
|
|
AFTER DELETE ON variant_content
|
|
BEGIN
|
|
UPDATE cache_usage
|
|
SET total_size_bytes = total_size_bytes - OLD.size_bytes,
|
|
change_count = change_count + 1
|
|
WHERE id = 1;
|
|
END;
|
|
|
|
CREATE TRIGGER IF NOT EXISTS variant_content_usage_update
|
|
AFTER UPDATE OF size_bytes ON variant_content
|
|
BEGIN
|
|
UPDATE cache_usage
|
|
SET total_size_bytes = total_size_bytes - OLD.size_bytes + NEW.size_bytes,
|
|
change_count = change_count + 1
|
|
WHERE id = 1;
|
|
END;
|
|
|
|
-- Negative cache: source URLs whose fetch failed with an upstream error
|
|
-- status, or whose first bytes did not match their Content-Type. pixa does
|
|
-- not fetch such a URL again until its row expires.
|
|
CREATE TABLE IF NOT EXISTS negative_cache (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
source_host TEXT NOT NULL,
|
|
source_path TEXT NOT NULL,
|
|
source_query TEXT NOT NULL DEFAULT '',
|
|
expires_at DATETIME NOT NULL,
|
|
UNIQUE(source_host, source_path, source_query)
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_negative_cache_expires ON negative_cache(expires_at);
|
|
|
|
-- A random value that a login session carries from its login. pixa makes it
|
|
-- when the table has no row, and each logout replaces it; a login session
|
|
-- carrying another value is refused, so logging out ends every login session.
|
|
CREATE TABLE IF NOT EXISTS logout (
|
|
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
logout_value TEXT NOT NULL
|
|
);
|