Store a source the way a variant is stored, one row and one file per source URL (closes #308)
check / check (push) Waiting to run

A source is now one source_content row and one file named by its
SourceKey, the SHA-256 of its host, path and query, made as the variant
key is. A store writes the file, then upserts the row, and removes the
file when the row cannot be written; an eviction deletes the row, then
the file, with no lock. source_metadata, the JSON files under
cache/metadata/, MetadataStorage, the content lock, the eviction
transaction and its test hook are gone, as is the reconciliation pass
that dropped a source row whose file is missing. Two source URLs
serving the same bytes store them twice.

Model: opus-5-5
This commit is contained in:
2026-10-21 14:20:28 +00:00
parent 00d8ba26c5
commit 5d78e62ae9
24 changed files with 622 additions and 1468 deletions
+7 -20
View File
@@ -24,26 +24,14 @@ func setupTestDB(t *testing.T) *sql.DB {
// Create schema
schema := `
CREATE TABLE source_content (
content_hash TEXT PRIMARY KEY,
content_type TEXT NOT NULL,
size_bytes INTEGER NOT NULL,
fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE source_metadata (
id INTEGER PRIMARY KEY AUTOINCREMENT,
source_key TEXT PRIMARY KEY,
source_host TEXT NOT NULL,
source_path TEXT NOT NULL,
source_query TEXT NOT NULL DEFAULT '',
path_hash TEXT NOT NULL,
content_hash TEXT,
status_code INTEGER NOT NULL,
content_type TEXT,
response_headers TEXT,
content_type TEXT NOT NULL,
size_bytes INTEGER NOT NULL,
fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP,
expires_at DATETIME,
etag TEXT,
last_modified TEXT,
UNIQUE(source_host, source_path, source_query)
last_accessed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE output_content (
content_hash TEXT PRIMARY KEY,
@@ -171,14 +159,14 @@ func TestCache_StoreAndLookup(t *testing.T) {
Headers: map[string][]string{testHeaderContentType: {testContentTypeJPEG}},
}
contentHash, err := cache.StoreSource(
sourceKey, err := cache.StoreSource(
ctx, req, bytes.NewReader(sourceContent), fetchResult)
if err != nil {
t.Fatalf("StoreSource() error = %v", err)
}
if contentHash == "" {
t.Error("StoreSource() returned empty hash")
if sourceKey == "" {
t.Error("StoreSource() returned empty key")
}
// Store variant
@@ -547,7 +535,6 @@ func TestCache_StorageDirectoriesCreated(t *testing.T) {
dirs := []string{
"cache/sources",
"cache/variants",
"cache/metadata",
}
for _, dir := range dirs {