Store a source the way a variant is stored, one row and one file per source URL (closes #308)
check / check (push) Waiting to run

A source is now one source_content row and one file named by its
SourceKey, the SHA-256 of its host, path and query, made as the variant
key is. A store writes the file, then upserts the row, and removes the
file when the row cannot be written; an eviction deletes the row, then
the file, with no lock. source_metadata, the JSON files under
cache/metadata/, MetadataStorage, the content lock, the eviction
transaction and its test hook are gone, as is the reconciliation pass
that dropped a source row whose file is missing. Two source URLs
serving the same bytes store them twice.

Model: opus-5-5
This commit is contained in:
2026-10-21 14:20:28 +00:00
parent 00d8ba26c5
commit 5d78e62ae9
24 changed files with 622 additions and 1468 deletions
+7 -27
View File
@@ -1,10 +1,14 @@
-- Migration 001: Initial schema
-- Creates all tables for the pixa caching image proxy
-- Source content blobs
-- Files stored at: cache/sources/<ab>/<cd>/<sha256>
-- Source images, one row per source URL
-- Files stored at: cache/sources/<ab>/<cd>/<source_key>, where source_key
-- is the SHA-256 of the source's host, path and query
CREATE TABLE IF NOT EXISTS source_content (
content_hash TEXT PRIMARY KEY,
source_key TEXT PRIMARY KEY,
source_host TEXT NOT NULL,
source_path TEXT NOT NULL,
source_query TEXT NOT NULL DEFAULT '',
content_type TEXT NOT NULL,
size_bytes INTEGER NOT NULL,
fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP,
@@ -13,30 +17,6 @@ CREATE TABLE IF NOT EXISTS source_content (
CREATE INDEX IF NOT EXISTS idx_source_content_last_accessed
ON source_content(last_accessed_at);
-- Source URL metadata - maps URLs to content hashes
-- JSON stored at: cache/metadata/<hostname>/<path_hash>.json
CREATE TABLE IF NOT EXISTS source_metadata (
id INTEGER PRIMARY KEY AUTOINCREMENT,
source_host TEXT NOT NULL,
source_path TEXT NOT NULL,
source_query TEXT NOT NULL DEFAULT '',
path_hash TEXT NOT NULL,
content_hash TEXT,
status_code INTEGER NOT NULL,
content_type TEXT,
response_headers TEXT,
fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP,
expires_at DATETIME,
etag TEXT,
last_modified TEXT,
UNIQUE(source_host, source_path, source_query),
FOREIGN KEY (content_hash) REFERENCES source_content(content_hash)
);
CREATE INDEX IF NOT EXISTS idx_source_meta_host ON source_metadata(source_host);
CREATE INDEX IF NOT EXISTS idx_source_meta_path_hash ON source_metadata(path_hash);
CREATE INDEX IF NOT EXISTS idx_source_meta_expires ON source_metadata(expires_at);
CREATE INDEX IF NOT EXISTS idx_source_meta_content_hash ON source_metadata(content_hash);
-- Processed variant blobs
-- Files stored at: cache/variants/<ab>/<cd>/<cache_key> (plus a .meta
-- sidecar with the content type). Tracked here (like source content