Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bb0ebb56ee | ||
|
|
39c9093d49 | ||
|
|
d3c8b5f422 |
@@ -504,12 +504,6 @@ Key settings in more detail:
|
|||||||
waits for an upstream connection and for a processing slot (up to 10 seconds
|
waits for an upstream connection and for a processing slot (up to 10 seconds
|
||||||
each), so keep it longer than `upstream_fetch_timeout` plus 20 seconds
|
each), so keep it longer than `upstream_fetch_timeout` plus 20 seconds
|
||||||
- `signing_key` — HMAC secret for URL signatures
|
- `signing_key` — HMAC secret for URL signatures
|
||||||
- `db_url` — the SQLite database to open; omitted, it is
|
|
||||||
`file:<state_dir>/state.sqlite3?_pragma=journal_mode(WAL)`, which keeps the
|
|
||||||
database in WAL mode. pixa adds `_pragma=busy_timeout(5000)` to any `db_url`,
|
|
||||||
so a write that finds another in progress waits up to five seconds for it
|
|
||||||
instead of failing. WAL mode comes only from the URL: keep
|
|
||||||
`_pragma=journal_mode(WAL)` in one you set
|
|
||||||
- `cache_max_bytes` — disk cache size limit in bytes; `0` disables the
|
- `cache_max_bytes` — disk cache size limit in bytes; `0` disables the
|
||||||
disk cache entirely; omitted defaults to 75% of the sum of the free space on
|
disk cache entirely; omitted defaults to 75% of the sum of the free space on
|
||||||
the filesystem containing `<state_dir>/cache/` and the bytes of source and
|
the filesystem containing `<state_dir>/cache/` and the bytes of source and
|
||||||
|
|||||||
@@ -31,12 +31,6 @@ P2: security: per-IP rate limiting on the image routes
|
|||||||
|
|
||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
- 2026-10-04 SQLite writes no longer fail with "database is locked" (closes
|
|
||||||
#198): pixa adds `_pragma=busy_timeout(5000)` to every `db_url`, so a write
|
|
||||||
that finds another in progress on another connection waits up to five seconds
|
|
||||||
for it, and the default `db_url` turns on WAL mode with
|
|
||||||
`_pragma=journal_mode(WAL)`. The old default's `_journal_mode=WAL` is not a
|
|
||||||
parameter the driver reads, so the database was never in WAL mode.
|
|
||||||
- 2026-10-04 referer blocklist (closes #90): `referer_blocklist`
|
- 2026-10-04 referer blocklist (closes #90): `referer_blocklist`
|
||||||
(`PIXA_REFERER_BLOCKLIST`) lists hosts, written and matched as for
|
(`PIXA_REFERER_BLOCKLIST`) lists hosts, written and matched as for
|
||||||
`allowlist_hosts` with the same matcher; an entry that is not a bare host
|
`allowlist_hosts` with the same matcher; an entry that is not a bare host
|
||||||
|
|||||||
+3
-4
@@ -34,10 +34,9 @@ maintenance_mode: false
|
|||||||
state_dir: ./data
|
state_dir: ./data
|
||||||
|
|
||||||
# SQLite database URL (default:
|
# SQLite database URL (default:
|
||||||
# file:<state_dir>/state.sqlite3?_pragma=journal_mode(WAL)). pixa adds
|
# file:<state_dir>/state.sqlite3?_journal_mode=WAL). An empty value aborts
|
||||||
# _pragma=busy_timeout(5000) to it. An empty value aborts startup; leave the
|
# startup; leave the key out to use the default.
|
||||||
# key out to use the default.
|
# db_url: "file:./data/state.sqlite3?_journal_mode=WAL"
|
||||||
# db_url: "file:./data/state.sqlite3?_pragma=journal_mode(WAL)"
|
|
||||||
|
|
||||||
# Image proxy settings
|
# Image proxy settings
|
||||||
# HMAC signing key for URL signatures (required, at least 32 characters)
|
# HMAC signing key for URL signatures (required, at least 32 characters)
|
||||||
|
|||||||
@@ -325,8 +325,7 @@ func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) {
|
|||||||
settingName(keyDBURL), errValueEmpty)
|
settingName(keyDBURL), errValueEmpty)
|
||||||
}
|
}
|
||||||
|
|
||||||
// The driver sets the journal mode only through a _pragma parameter.
|
c.DBURL = fmt.Sprintf("file:%s/state.sqlite3?_journal_mode=WAL", c.StateDir)
|
||||||
c.DBURL = fmt.Sprintf("file:%s/state.sqlite3?_pragma=journal_mode(WAL)", c.StateDir)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if loader.err != nil {
|
if loader.err != nil {
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
package config
|
package config
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"database/sql"
|
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -10,8 +9,6 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.eeqj.de/sneak/smartconfig"
|
"git.eeqj.de/sneak/smartconfig"
|
||||||
|
|
||||||
_ "modernc.org/sqlite" // SQLite driver registration
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// validTestSigningKey is a 32-character signing key that satisfies the
|
// validTestSigningKey is a 32-character signing key that satisfies the
|
||||||
@@ -97,43 +94,12 @@ func TestOmittedValuesUseDefaults(t *testing.T) {
|
|||||||
t.Errorf("AllowlistHosts = %v, want empty", c.AllowlistHosts)
|
t.Errorf("AllowlistHosts = %v, want empty", c.AllowlistHosts)
|
||||||
}
|
}
|
||||||
|
|
||||||
wantDBURL := "file:" + DefaultStateDir +
|
wantDBURL := "file:" + DefaultStateDir + "/state.sqlite3?_journal_mode=WAL"
|
||||||
"/state.sqlite3?_pragma=journal_mode(WAL)"
|
|
||||||
if c.DBURL != wantDBURL {
|
if c.DBURL != wantDBURL {
|
||||||
t.Errorf("DBURL = %q, want derived default %q", c.DBURL, wantDBURL)
|
t.Errorf("DBURL = %q, want derived default %q", c.DBURL, wantDBURL)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestDefaultDBURLOpensTheDatabaseInWALMode opens the db_url derived from
|
|
||||||
// state_dir with the SQLite driver pixad uses and checks that the database
|
|
||||||
// is in WAL mode: the driver ignores any parameter it does not know.
|
|
||||||
func TestDefaultDBURLOpensTheDatabaseInWALMode(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
c, err := configFromYAML(t, signingKeyLine+"state_dir: "+t.TempDir()+"\n")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("config with only state_dir set should be valid, got: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
db, err := sql.Open("sqlite", c.DBURL)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to open %q: %v", c.DBURL, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Cleanup(func() { _ = db.Close() })
|
|
||||||
|
|
||||||
var journalMode string
|
|
||||||
|
|
||||||
err = db.QueryRowContext(t.Context(), "PRAGMA journal_mode").Scan(&journalMode)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to read the journal mode of %q: %v", c.DBURL, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if journalMode != "wal" {
|
|
||||||
t.Errorf("journal mode of %q = %q, want wal", c.DBURL, journalMode)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestExplicitValidValuesAreUsed(t *testing.T) {
|
func TestExplicitValidValuesAreUsed(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -318,20 +284,6 @@ func invalidHostAndCredentialCases() []abortCase {
|
|||||||
keyAllowlistHosts, "example.com/images",
|
keyAllowlistHosts, "example.com/images",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
|
||||||
name: "allowlist host with wildcard",
|
|
||||||
yaml: signingKeyLine + "allowlist_hosts:\n - \"*.example.com\"\n",
|
|
||||||
wantErrSubstrings: []string{
|
|
||||||
keyAllowlistHosts, "*.example.com",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "allowlist host with port",
|
|
||||||
yaml: signingKeyLine + "allowlist_hosts:\n - example.com:8443\n",
|
|
||||||
wantErrSubstrings: []string{
|
|
||||||
keyAllowlistHosts, "example.com:8443",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
name: "allowlist host with whitespace",
|
name: "allowlist host with whitespace",
|
||||||
yaml: signingKeyLine + "allowlist_hosts:\n - \"exa mple.com\"\n",
|
yaml: signingKeyLine + "allowlist_hosts:\n - \"exa mple.com\"\n",
|
||||||
|
|||||||
@@ -24,25 +24,6 @@ func TestRefererBlocklistParsed(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestRefererBlocklistAcceptsIPAddresses checks that IPv4 and IPv6 addresses,
|
|
||||||
// the IPv6 one written without brackets, are accepted as entries.
|
|
||||||
func TestRefererBlocklistAcceptsIPAddresses(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
c, err := configFromYAML(t, signingKeyLine+`referer_blocklist:
|
|
||||||
- 192.0.2.7
|
|
||||||
- "2001:db8::7"
|
|
||||||
`)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("IP address entries should load, got error: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
want := []string{"192.0.2.7", "2001:db8::7"}
|
|
||||||
if !slices.Equal(c.RefererBlocklist, want) {
|
|
||||||
t.Errorf("RefererBlocklist = %v, want %v", c.RefererBlocklist, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRefererBlocklistOmittedIsEmpty checks that an omitted key blocks no
|
// TestRefererBlocklistOmittedIsEmpty checks that an omitted key blocks no
|
||||||
// referer.
|
// referer.
|
||||||
func TestRefererBlocklistOmittedIsEmpty(t *testing.T) {
|
func TestRefererBlocklistOmittedIsEmpty(t *testing.T) {
|
||||||
@@ -79,27 +60,6 @@ func TestRefererBlocklistInvalidAbortsStartup(t *testing.T) {
|
|||||||
keyRefererBlocklist, "leech.example/page",
|
keyRefererBlocklist, "leech.example/page",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
|
||||||
name: "wildcard entry",
|
|
||||||
yaml: signingKeyLine + "referer_blocklist:\n - \"*.leech.example\"\n",
|
|
||||||
wantErrSubstrings: []string{
|
|
||||||
keyRefererBlocklist, "*.leech.example",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "entry with a port",
|
|
||||||
yaml: signingKeyLine + "referer_blocklist:\n - leech.example:8080\n",
|
|
||||||
wantErrSubstrings: []string{
|
|
||||||
keyRefererBlocklist, "leech.example:8080",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "two leading dots",
|
|
||||||
yaml: signingKeyLine + "referer_blocklist:\n - ..leech.example\n",
|
|
||||||
wantErrSubstrings: []string{
|
|
||||||
keyRefererBlocklist, "..leech.example",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
name: "dot only",
|
name: "dot only",
|
||||||
yaml: signingKeyLine + "referer_blocklist:\n - \".\"\n",
|
yaml: signingKeyLine + "referer_blocklist:\n - \".\"\n",
|
||||||
|
|||||||
@@ -1,153 +0,0 @@
|
|||||||
package database
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"database/sql"
|
|
||||||
"fmt"
|
|
||||||
"log/slog"
|
|
||||||
"path/filepath"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"sneak.berlin/go/pixa/internal/config"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestConcurrentWritesAllSucceed opens a database the way pixad does and
|
|
||||||
// writes to it from several goroutines at once, so the writes run on
|
|
||||||
// separate connections, as one request's writes and the background eviction
|
|
||||||
// pass do. Every write must succeed, none failing with "database is locked",
|
|
||||||
// whether or not db_url already has parameters, and the parameters it has
|
|
||||||
// must still apply.
|
|
||||||
func TestConcurrentWritesAllSucceed(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
query string
|
|
||||||
wantJournalMode string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "db_url without parameters",
|
|
||||||
query: "",
|
|
||||||
wantJournalMode: "delete",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "db_url with the WAL parameter",
|
|
||||||
query: "?_pragma=journal_mode(WAL)",
|
|
||||||
wantJournalMode: "wal",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dbURL := "file:" + filepath.Join(t.TempDir(), "state.sqlite3") + tt.query
|
|
||||||
|
|
||||||
d := &Database{
|
|
||||||
log: slog.New(slog.DiscardHandler),
|
|
||||||
config: &config.Config{DBURL: dbURL},
|
|
||||||
}
|
|
||||||
|
|
||||||
err := d.connect(t.Context())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to connect to %q: %v", dbURL, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Cleanup(func() { _ = d.db.Close() })
|
|
||||||
|
|
||||||
writeConcurrently(t, d.db)
|
|
||||||
|
|
||||||
var journalMode string
|
|
||||||
|
|
||||||
err = d.db.QueryRowContext(t.Context(), "PRAGMA journal_mode").
|
|
||||||
Scan(&journalMode)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to read the journal mode: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if journalMode != tt.wantJournalMode {
|
|
||||||
t.Errorf("journal mode = %q, want %q", journalMode, tt.wantJournalMode)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeConcurrently runs writeLikeOneRequest from several goroutines at once
|
|
||||||
// and checks that every write was made.
|
|
||||||
func writeConcurrently(t *testing.T, db *sql.DB) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
const (
|
|
||||||
writers = 4
|
|
||||||
requestsEach = 20
|
|
||||||
totalRequests = writers * requestsEach
|
|
||||||
)
|
|
||||||
|
|
||||||
ctx := t.Context()
|
|
||||||
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
|
|
||||||
for writer := range writers {
|
|
||||||
wg.Go(func() {
|
|
||||||
for request := range requestsEach {
|
|
||||||
key := fmt.Sprintf("%d-%d", writer, request)
|
|
||||||
|
|
||||||
err := writeLikeOneRequest(ctx, db, key)
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("writer %d: %v", writer, err)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
wg.Wait()
|
|
||||||
|
|
||||||
var hits, sources int
|
|
||||||
|
|
||||||
err := db.QueryRowContext(ctx, `
|
|
||||||
SELECT hit_count, (SELECT COUNT(*) FROM source_content)
|
|
||||||
FROM cache_stats WHERE id = 1
|
|
||||||
`).Scan(&hits, &sources)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to count the writes: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if hits != totalRequests || sources != totalRequests {
|
|
||||||
t.Errorf("hit_count = %d and %d source_content rows, want %d of each",
|
|
||||||
hits, sources, totalRequests)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeLikeOneRequest makes the writes one request and the eviction pass
|
|
||||||
// make: it counts a cache hit, stores a source, records a transformed image
|
|
||||||
// and deletes that record again.
|
|
||||||
func writeLikeOneRequest(ctx context.Context, db *sql.DB, key string) error {
|
|
||||||
_, err := db.ExecContext(ctx,
|
|
||||||
`UPDATE cache_stats SET hit_count = hit_count + 1 WHERE id = 1`)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("counting a cache hit: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = db.ExecContext(ctx, `INSERT INTO source_content
|
|
||||||
(content_hash, content_type, size_bytes) VALUES (?, 'image/png', 1)`, key)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("storing a source: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = db.ExecContext(ctx, `INSERT INTO variant_content
|
|
||||||
(cache_key, size_bytes, content_type) VALUES (?, 1, 'image/png')`, key)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("recording a transformed image: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = db.ExecContext(ctx,
|
|
||||||
`DELETE FROM variant_content WHERE cache_key = ?`, key)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("evicting a transformed image: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -243,17 +243,7 @@ func (s *Database) DB() *sql.DB {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *Database) connect(ctx context.Context) error {
|
func (s *Database) connect(ctx context.Context) error {
|
||||||
// Requests and the eviction pass write on separate connections. With
|
dbURL := s.config.DBURL
|
||||||
// a busy timeout, a write that finds another one in progress waits up
|
|
||||||
// to five seconds for it instead of failing at once with "database is
|
|
||||||
// locked". The driver runs each _pragma parameter on every connection
|
|
||||||
// it opens.
|
|
||||||
separator := "?"
|
|
||||||
if strings.Contains(s.config.DBURL, "?") {
|
|
||||||
separator = "&"
|
|
||||||
}
|
|
||||||
|
|
||||||
dbURL := s.config.DBURL + separator + "_pragma=busy_timeout(5000)"
|
|
||||||
|
|
||||||
s.log.Info("connecting to database", "url", dbURL)
|
s.log.Info("connecting to database", "url", dbURL)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user