Store a source the way a variant is stored, one row and one file per source URL (closes #308)
check / check (push) Canceled after 0s

A source is now one source_content row and one file named by its
SourceKey, the SHA-256 of its host, path and query, made as the variant
key is. A store writes the file, then upserts the row, and removes the
file when the row cannot be written; an eviction deletes the row, then
the file, with no lock. source_metadata, the JSON files under
cache/metadata/, MetadataStorage, the content lock, the eviction
transaction and its test hook are gone, as is the reconciliation pass
that dropped a source row whose file is missing, so
TestReconciliationReadsAPageAtATime now counts 15 rows read, not 20.
Two source URLs serving the same bytes store them twice. The tests of
the removed sharing and lock go with them; the two eviction
cancellation tests no longer use the hook.

Model: opus-5-5
This commit is contained in:
2026-10-21 13:50:43 +00:00
parent 00d8ba26c5
commit e06192b7a5
24 changed files with 599 additions and 1561 deletions
+17 -41
View File
@@ -550,21 +550,17 @@ func TestService_Get_Caching(t *testing.T) {
// TestService_Get_ServesSourceAndVariantCachedTwoDaysAgo caches a source and
// a variant of it and sets back two days every time pixa stored for them.
// For the source: fetched_at in source_metadata, fetched_at and
// last_accessed_at in source_content, the fetched_at in its metadata file,
// and the access and modification times of its file and of its metadata
// file. For the variant: created_at and last_accessed_at in variant_content,
// the created_at in its .meta file, and the access and modification times of
// its file and of its .meta file. It then checks that the variant is still
// served from the cache and that a new variant is made from the cached source
// without fetching it again.
// For the source: fetched_at and last_accessed_at in source_content, and the
// access and modification times of its file. For the variant: created_at and
// last_accessed_at in variant_content, the created_at in its .meta file, and
// the access and modification times of its file and of its .meta file. It
// then checks that the variant is still served from the cache and that a new
// variant is made from the cached source without fetching it again.
//
// Left at the present: the files' status change and birth times, which a
// program cannot set, and the times of the directories holding the files,
// which other entries share. The source's stored response headers are empty,
// as the mock fetcher sends none. expires_at, etag and last_modified in
// source_metadata and in the metadata file are never written, nor are the
// request_cache and output_content tables.
// which other entries share. The request_cache and output_content tables are
// never written.
func TestService_Get_ServesSourceAndVariantCachedTwoDaysAgo(t *testing.T) {
t.Parallel()
@@ -615,39 +611,19 @@ func backdateSource(
) {
t.Helper()
pathHash := HashPath(req.Source.Path + "?" + req.Source.Query)
meta, err := cache.srcMetadata.Load(req.Source.Host, pathHash)
if err != nil {
t.Fatalf("reading the source's metadata file: %v", err)
}
meta.FetchedAt = when.Unix()
err = cache.srcMetadata.Store(req.Source.Host, pathHash, meta)
if err != nil {
t.Fatalf("writing the source's metadata file: %v", err)
}
key := req.Source.Key()
dbTime := when.UTC().Format(sqliteTimestampFormat)
updateOneRow(t, cache, `
UPDATE source_metadata SET fetched_at = ?
WHERE source_host = ? AND source_path = ? AND source_query = ?
`, dbTime, req.Source.Host, req.Source.Path, req.Source.Query)
updateOneRow(t, cache,
`UPDATE source_content SET fetched_at = ? WHERE content_hash = ?`,
dbTime, meta.ContentHash)
setSourceLastAccessed(t, cache, ContentHash(meta.ContentHash), when)
`UPDATE source_content SET fetched_at = ? WHERE source_key = ?`,
dbTime, string(key))
setSourceLastAccessed(t, cache, key, when)
for _, path := range []string{
cache.srcContent.hashToPath(ContentHash(meta.ContentHash)),
cache.srcMetadata.metaPath(req.Source.Host, pathHash),
} {
err = os.Chtimes(path, when, when)
if err != nil {
t.Fatalf("setting back the times of %s: %v", path, err)
}
path := cache.srcContent.hashToPath(key)
err := os.Chtimes(path, when, when)
if err != nil {
t.Fatalf("setting back the times of %s: %v", path, err)
}
}