Store a source the way a variant is stored, one row and one file per source URL (closes #308)
check / check (push) Canceled after 0s
check / check (push) Canceled after 0s
A source is now one source_content row and one file named by its SourceKey, the SHA-256 of its host, path and query, made as the variant key is. A store writes the file, then upserts the row, and removes the file when the row cannot be written; an eviction deletes the row, then the file, with no lock. source_metadata, the JSON files under cache/metadata/, MetadataStorage, the content lock, the eviction transaction and its test hook are gone, as is the reconciliation pass that dropped a source row whose file is missing. Two source URLs serving the same bytes store them twice. Model: opus-5-5
This commit is contained in:
@@ -1,7 +1,7 @@
|
||||
// Package imagerequest holds the image request type, its parts, its checks,
|
||||
// the parsing of the /v1/image/ path and the variant key. It imports no other
|
||||
// package of this repo, so every package that handles an image request can
|
||||
// import it.
|
||||
// the parsing of the /v1/image/ path, the variant key and the source key. It
|
||||
// imports no other package of this repo, so every package that handles an
|
||||
// image request can import it.
|
||||
package imagerequest
|
||||
|
||||
import (
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
package imagerequest
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// SourceKey is a SHA256 hash identifying a cached source image (hex-encoded).
|
||||
type SourceKey string
|
||||
|
||||
// Key returns the key the source is cached under, made as CacheKey makes a
|
||||
// variant key.
|
||||
// Format: sha256(host:path:query)
|
||||
// The path and query go through escapePercentAndColon, so that a ":" in
|
||||
// either cannot be taken for the one between them.
|
||||
func (s Source) Key() SourceKey {
|
||||
data := fmt.Sprintf("%s:%s:%s",
|
||||
s.Host,
|
||||
escapePercentAndColon(s.Path),
|
||||
escapePercentAndColon(s.Query),
|
||||
)
|
||||
h := sha256.Sum256([]byte(data))
|
||||
|
||||
return SourceKey(hex.EncodeToString(h[:]))
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package imagerequest
|
||||
|
||||
import "testing"
|
||||
|
||||
// TestSourceKey_DiffersForEachSource checks that each source gets a key of
|
||||
// 64 hex characters, the same each time, and that no two of these sources
|
||||
// share one, also when they differ only by a ":" or "%" in the path or query.
|
||||
func TestSourceKey_DiffersForEachSource(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
sources := []Source{
|
||||
{Host: testHostCDN, Path: testPathCat},
|
||||
{Host: testHostCDN, Path: testPathCat, Query: "v=2"},
|
||||
{Host: "other.example.com", Path: testPathCat},
|
||||
{Host: testHostCDN, Path: "/c:d"},
|
||||
{Host: testHostCDN, Path: "/c", Query: "d:"},
|
||||
{Host: testHostCDN, Path: "/c%3Ad"},
|
||||
}
|
||||
|
||||
seen := make(map[SourceKey]Source)
|
||||
|
||||
for _, source := range sources {
|
||||
key := source.Key()
|
||||
t.Logf("Key() of %+v = %s", source, key)
|
||||
|
||||
if len(key) != 64 {
|
||||
t.Errorf("Key() of %+v length = %d, want 64", source, len(key))
|
||||
}
|
||||
|
||||
if again := source.Key(); again != key {
|
||||
t.Errorf("Key() of %+v not deterministic: %s vs %s", source, key, again)
|
||||
}
|
||||
|
||||
if earlier, found := seen[key]; found {
|
||||
t.Errorf("Key() gave %+v and %+v the same key", earlier, source)
|
||||
}
|
||||
|
||||
seen[key] = source
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user