Store a source the way a variant is stored, one row and one file per source URL (closes #308)
check / check (push) Canceled after 0s
check / check (push) Canceled after 0s
A source is now one source_content row and one file named by its SourceKey, the SHA-256 of its host, path and query, made as the variant key is. A store writes the file, then upserts the row, and removes the file when the row cannot be written; an eviction deletes the row, then the file, with no lock. source_metadata, the JSON files under cache/metadata/, MetadataStorage, the content lock, the eviction transaction and its test hook are gone, as is the reconciliation pass that dropped a source row whose file is missing. Two source URLs serving the same bytes store them twice. Model: opus-5-5
This commit was merged in pull request #324.
This commit is contained in:
@@ -550,21 +550,17 @@ func TestService_Get_Caching(t *testing.T) {
|
||||
|
||||
// TestService_Get_ServesSourceAndVariantCachedTwoDaysAgo caches a source and
|
||||
// a variant of it and sets back two days every time pixa stored for them.
|
||||
// For the source: fetched_at in source_metadata, fetched_at and
|
||||
// last_accessed_at in source_content, the fetched_at in its metadata file,
|
||||
// and the access and modification times of its file and of its metadata
|
||||
// file. For the variant: created_at and last_accessed_at in variant_content,
|
||||
// the created_at in its .meta file, and the access and modification times of
|
||||
// its file and of its .meta file. It then checks that the variant is still
|
||||
// served from the cache and that a new variant is made from the cached source
|
||||
// without fetching it again.
|
||||
// For the source: fetched_at and last_accessed_at in source_content, and the
|
||||
// access and modification times of its file. For the variant: created_at and
|
||||
// last_accessed_at in variant_content, the created_at in its .meta file, and
|
||||
// the access and modification times of its file and of its .meta file. It
|
||||
// then checks that the variant is still served from the cache and that a new
|
||||
// variant is made from the cached source without fetching it again.
|
||||
//
|
||||
// Left at the present: the files' status change and birth times, which a
|
||||
// program cannot set, and the times of the directories holding the files,
|
||||
// which other entries share. The source's stored response headers are empty,
|
||||
// as the mock fetcher sends none. expires_at, etag and last_modified in
|
||||
// source_metadata and in the metadata file are never written, nor are the
|
||||
// request_cache and output_content tables.
|
||||
// which other entries share. The request_cache and output_content tables are
|
||||
// never written.
|
||||
func TestService_Get_ServesSourceAndVariantCachedTwoDaysAgo(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -615,39 +611,19 @@ func backdateSource(
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
pathHash := HashPath(req.Source.Path + "?" + req.Source.Query)
|
||||
|
||||
meta, err := cache.srcMetadata.Load(req.Source.Host, pathHash)
|
||||
if err != nil {
|
||||
t.Fatalf("reading the source's metadata file: %v", err)
|
||||
}
|
||||
|
||||
meta.FetchedAt = when.Unix()
|
||||
|
||||
err = cache.srcMetadata.Store(req.Source.Host, pathHash, meta)
|
||||
if err != nil {
|
||||
t.Fatalf("writing the source's metadata file: %v", err)
|
||||
}
|
||||
|
||||
key := req.Source.Key()
|
||||
dbTime := when.UTC().Format(sqliteTimestampFormat)
|
||||
|
||||
updateOneRow(t, cache, `
|
||||
UPDATE source_metadata SET fetched_at = ?
|
||||
WHERE source_host = ? AND source_path = ? AND source_query = ?
|
||||
`, dbTime, req.Source.Host, req.Source.Path, req.Source.Query)
|
||||
updateOneRow(t, cache,
|
||||
`UPDATE source_content SET fetched_at = ? WHERE content_hash = ?`,
|
||||
dbTime, meta.ContentHash)
|
||||
setSourceLastAccessed(t, cache, ContentHash(meta.ContentHash), when)
|
||||
`UPDATE source_content SET fetched_at = ? WHERE source_key = ?`,
|
||||
dbTime, string(key))
|
||||
setSourceLastAccessed(t, cache, key, when)
|
||||
|
||||
for _, path := range []string{
|
||||
cache.srcContent.hashToPath(ContentHash(meta.ContentHash)),
|
||||
cache.srcMetadata.metaPath(req.Source.Host, pathHash),
|
||||
} {
|
||||
err = os.Chtimes(path, when, when)
|
||||
if err != nil {
|
||||
t.Fatalf("setting back the times of %s: %v", path, err)
|
||||
}
|
||||
path := cache.srcContent.keyToPath(key)
|
||||
|
||||
err := os.Chtimes(path, when, when)
|
||||
if err != nil {
|
||||
t.Fatalf("setting back the times of %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user