Store a source the way a variant is stored, one row and one file per source URL (closes #308)
check / check (push) Canceled after 0s
check / check (push) Canceled after 0s
A source is now one source_content row and one file named by its SourceKey, the SHA-256 of its host, path and query, made as the variant key is. A store writes the file, then upserts the row, and removes the file when the row cannot be written; an eviction deletes the row, then the file, with no lock. source_metadata, the JSON files under cache/metadata/, MetadataStorage, the content lock, the eviction transaction and its test hook are gone, as is the reconciliation pass that dropped a source row whose file is missing. Two source URLs serving the same bytes store them twice. Model: opus-5-5
This commit is contained in:
@@ -24,26 +24,14 @@ func setupTestDB(t *testing.T) *sql.DB {
|
||||
// Create schema
|
||||
schema := `
|
||||
CREATE TABLE source_content (
|
||||
content_hash TEXT PRIMARY KEY,
|
||||
content_type TEXT NOT NULL,
|
||||
size_bytes INTEGER NOT NULL,
|
||||
fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE TABLE source_metadata (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
source_key TEXT PRIMARY KEY,
|
||||
source_host TEXT NOT NULL,
|
||||
source_path TEXT NOT NULL,
|
||||
source_query TEXT NOT NULL DEFAULT '',
|
||||
path_hash TEXT NOT NULL,
|
||||
content_hash TEXT,
|
||||
status_code INTEGER NOT NULL,
|
||||
content_type TEXT,
|
||||
response_headers TEXT,
|
||||
content_type TEXT NOT NULL,
|
||||
size_bytes INTEGER NOT NULL,
|
||||
fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
expires_at DATETIME,
|
||||
etag TEXT,
|
||||
last_modified TEXT,
|
||||
UNIQUE(source_host, source_path, source_query)
|
||||
last_accessed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE TABLE output_content (
|
||||
content_hash TEXT PRIMARY KEY,
|
||||
@@ -171,14 +159,14 @@ func TestCache_StoreAndLookup(t *testing.T) {
|
||||
Headers: map[string][]string{testHeaderContentType: {testContentTypeJPEG}},
|
||||
}
|
||||
|
||||
contentHash, err := cache.StoreSource(
|
||||
sourceKey, err := cache.StoreSource(
|
||||
ctx, req, bytes.NewReader(sourceContent), fetchResult)
|
||||
if err != nil {
|
||||
t.Fatalf("StoreSource() error = %v", err)
|
||||
}
|
||||
|
||||
if contentHash == "" {
|
||||
t.Error("StoreSource() returned empty hash")
|
||||
if sourceKey == "" {
|
||||
t.Error("StoreSource() returned empty key")
|
||||
}
|
||||
|
||||
// Store variant
|
||||
@@ -547,7 +535,6 @@ func TestCache_StorageDirectoriesCreated(t *testing.T) {
|
||||
dirs := []string{
|
||||
"cache/sources",
|
||||
"cache/variants",
|
||||
"cache/metadata",
|
||||
}
|
||||
|
||||
for _, dir := range dirs {
|
||||
|
||||
Reference in New Issue
Block a user