Store a source the way a variant is stored, one row and one file per source URL (closes #308)
check / check (push) Canceled after 0s
check / check (push) Canceled after 0s
A source is now one source_content row and one file named by its SourceKey, the SHA-256 of its host, path and query, made as the variant key is. A store writes the file, then upserts the row, and removes the file when the row cannot be written; an eviction deletes the row, then the file, with no lock. source_metadata, the JSON files under cache/metadata/, MetadataStorage, the content lock, the eviction transaction and its test hook are gone, as is the reconciliation pass that dropped a source row whose file is missing, so TestReconciliationReadsAPageAtATime now counts 15 rows read, not 20. Two source URLs serving the same bytes store them twice. The tests of the removed sharing and lock go with them; the two eviction cancellation tests no longer use the hook. Model: opus-5-5
This commit is contained in:
@@ -24,26 +24,14 @@ func setupTestDB(t *testing.T) *sql.DB {
|
||||
// Create schema
|
||||
schema := `
|
||||
CREATE TABLE source_content (
|
||||
content_hash TEXT PRIMARY KEY,
|
||||
content_type TEXT NOT NULL,
|
||||
size_bytes INTEGER NOT NULL,
|
||||
fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE TABLE source_metadata (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
source_key TEXT PRIMARY KEY,
|
||||
source_host TEXT NOT NULL,
|
||||
source_path TEXT NOT NULL,
|
||||
source_query TEXT NOT NULL DEFAULT '',
|
||||
path_hash TEXT NOT NULL,
|
||||
content_hash TEXT,
|
||||
status_code INTEGER NOT NULL,
|
||||
content_type TEXT,
|
||||
response_headers TEXT,
|
||||
content_type TEXT NOT NULL,
|
||||
size_bytes INTEGER NOT NULL,
|
||||
fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
expires_at DATETIME,
|
||||
etag TEXT,
|
||||
last_modified TEXT,
|
||||
UNIQUE(source_host, source_path, source_query)
|
||||
last_accessed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE TABLE output_content (
|
||||
content_hash TEXT PRIMARY KEY,
|
||||
@@ -171,14 +159,14 @@ func TestCache_StoreAndLookup(t *testing.T) {
|
||||
Headers: map[string][]string{testHeaderContentType: {testContentTypeJPEG}},
|
||||
}
|
||||
|
||||
contentHash, err := cache.StoreSource(
|
||||
sourceKey, err := cache.StoreSource(
|
||||
ctx, req, bytes.NewReader(sourceContent), fetchResult)
|
||||
if err != nil {
|
||||
t.Fatalf("StoreSource() error = %v", err)
|
||||
}
|
||||
|
||||
if contentHash == "" {
|
||||
t.Error("StoreSource() returned empty hash")
|
||||
if sourceKey == "" {
|
||||
t.Error("StoreSource() returned empty key")
|
||||
}
|
||||
|
||||
// Store variant
|
||||
@@ -547,7 +535,6 @@ func TestCache_StorageDirectoriesCreated(t *testing.T) {
|
||||
dirs := []string{
|
||||
"cache/sources",
|
||||
"cache/variants",
|
||||
"cache/metadata",
|
||||
}
|
||||
|
||||
for _, dir := range dirs {
|
||||
|
||||
Reference in New Issue
Block a user