3 Commits
Author SHA1 Message Date
clawbot bb0ebb56ee Read referer_blocklist through strictLoader and fix test lint
check / check (push) Failing after 2s
newFromSmartConfig went over the line limit, so referer_blocklist is now
read through strictLoader, like the other typed settings, instead of its
own parse step. The tests stop repeating string literals that goconst
counts: the environment test uses other hosts, two case names change, and
the handler test names each image route's URL instead of its path.

Model: opus-5-5
2026-10-04 18:19:59 +00:00
clawbot 39c9093d49 Refuse image requests whose Referer is on referer_blocklist (closes #90)
check / check (push) Failing after 2s
A new setting, referer_blocklist (PIXA_REFERER_BLOCKLIST), lists hosts
written and matched as allowlist_hosts are, with the same matcher and the
same entry check; a bad entry aborts startup naming the setting and the
entry. Both image routes check the Referer first and answer 403 with the
JSON error, so a blocked request fetches nothing and is refused whether or
not the image is cached. No Referer, or one that does not parse as a URL
with a host, is served; README.md and config.example.yml say this makes the
list easy to get around. The CIDR-list entry reader is renamed listEntries
now that host lists use it too.

Model: opus-5-5
2026-10-04 18:09:16 +00:00
clawbot d3c8b5f422 Add failing tests for referer_blocklist
check / check (push) Failing after 3s
Both image routes must refuse a request whose Referer names a host on
referer_blocklist with 403 and the JSON error, without fetching from the
upstream host and whether or not the image is cached, and must serve a
request with no Referer, one that does not parse, or one naming another
host. Hosts match as allowlist_hosts matches them. The config tests check
the list is read from the file and from PIXA_REFERER_BLOCKLIST, and that an
entry that is not a host aborts startup naming the setting and the entry.
These do not compile until the setting exists.

Model: opus-5-5
2026-10-04 18:06:04 +00:00
8 changed files with 6 additions and 271 deletions
-6
View File
@@ -504,12 +504,6 @@ Key settings in more detail:
waits for an upstream connection and for a processing slot (up to 10 seconds waits for an upstream connection and for a processing slot (up to 10 seconds
each), so keep it longer than `upstream_fetch_timeout` plus 20 seconds each), so keep it longer than `upstream_fetch_timeout` plus 20 seconds
- `signing_key` — HMAC secret for URL signatures - `signing_key` — HMAC secret for URL signatures
- `db_url` — the SQLite database to open; omitted, it is
`file:<state_dir>/state.sqlite3?_pragma=journal_mode(WAL)`, which keeps the
database in WAL mode. pixa adds `_pragma=busy_timeout(5000)` to any `db_url`,
so a write that finds another in progress waits up to five seconds for it
instead of failing. WAL mode comes only from the URL: keep
`_pragma=journal_mode(WAL)` in one you set
- `cache_max_bytes` — disk cache size limit in bytes; `0` disables the - `cache_max_bytes` — disk cache size limit in bytes; `0` disables the
disk cache entirely; omitted defaults to 75% of the sum of the free space on disk cache entirely; omitted defaults to 75% of the sum of the free space on
the filesystem containing `<state_dir>/cache/` and the bytes of source and the filesystem containing `<state_dir>/cache/` and the bytes of source and
-6
View File
@@ -31,12 +31,6 @@ P2: security: per-IP rate limiting on the image routes
# Completed Steps # Completed Steps
- 2026-10-04 SQLite writes no longer fail with "database is locked" (closes
#198): pixa adds `_pragma=busy_timeout(5000)` to every `db_url`, so a write
that finds another in progress on another connection waits up to five seconds
for it, and the default `db_url` turns on WAL mode with
`_pragma=journal_mode(WAL)`. The old default's `_journal_mode=WAL` is not a
parameter the driver reads, so the database was never in WAL mode.
- 2026-10-04 referer blocklist (closes #90): `referer_blocklist` - 2026-10-04 referer blocklist (closes #90): `referer_blocklist`
(`PIXA_REFERER_BLOCKLIST`) lists hosts, written and matched as for (`PIXA_REFERER_BLOCKLIST`) lists hosts, written and matched as for
`allowlist_hosts` with the same matcher; an entry that is not a bare host `allowlist_hosts` with the same matcher; an entry that is not a bare host
+3 -4
View File
@@ -34,10 +34,9 @@ maintenance_mode: false
state_dir: ./data state_dir: ./data
# SQLite database URL (default: # SQLite database URL (default:
# file:<state_dir>/state.sqlite3?_pragma=journal_mode(WAL)). pixa adds # file:<state_dir>/state.sqlite3?_journal_mode=WAL). An empty value aborts
# _pragma=busy_timeout(5000) to it. An empty value aborts startup; leave the # startup; leave the key out to use the default.
# key out to use the default. # db_url: "file:./data/state.sqlite3?_journal_mode=WAL"
# db_url: "file:./data/state.sqlite3?_pragma=journal_mode(WAL)"
# Image proxy settings # Image proxy settings
# HMAC signing key for URL signatures (required, at least 32 characters) # HMAC signing key for URL signatures (required, at least 32 characters)
+1 -2
View File
@@ -325,8 +325,7 @@ func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) {
settingName(keyDBURL), errValueEmpty) settingName(keyDBURL), errValueEmpty)
} }
// The driver sets the journal mode only through a _pragma parameter. c.DBURL = fmt.Sprintf("file:%s/state.sqlite3?_journal_mode=WAL", c.StateDir)
c.DBURL = fmt.Sprintf("file:%s/state.sqlite3?_pragma=journal_mode(WAL)", c.StateDir)
} }
if loader.err != nil { if loader.err != nil {
@@ -1,7 +1,6 @@
package config package config
import ( import (
"database/sql"
"log/slog" "log/slog"
"os" "os"
"path/filepath" "path/filepath"
@@ -10,8 +9,6 @@ import (
"time" "time"
"git.eeqj.de/sneak/smartconfig" "git.eeqj.de/sneak/smartconfig"
_ "modernc.org/sqlite" // SQLite driver registration
) )
// validTestSigningKey is a 32-character signing key that satisfies the // validTestSigningKey is a 32-character signing key that satisfies the
@@ -97,43 +94,12 @@ func TestOmittedValuesUseDefaults(t *testing.T) {
t.Errorf("AllowlistHosts = %v, want empty", c.AllowlistHosts) t.Errorf("AllowlistHosts = %v, want empty", c.AllowlistHosts)
} }
wantDBURL := "file:" + DefaultStateDir + wantDBURL := "file:" + DefaultStateDir + "/state.sqlite3?_journal_mode=WAL"
"/state.sqlite3?_pragma=journal_mode(WAL)"
if c.DBURL != wantDBURL { if c.DBURL != wantDBURL {
t.Errorf("DBURL = %q, want derived default %q", c.DBURL, wantDBURL) t.Errorf("DBURL = %q, want derived default %q", c.DBURL, wantDBURL)
} }
} }
// TestDefaultDBURLOpensTheDatabaseInWALMode opens the db_url derived from
// state_dir with the SQLite driver pixad uses and checks that the database
// is in WAL mode: the driver ignores any parameter it does not know.
func TestDefaultDBURLOpensTheDatabaseInWALMode(t *testing.T) {
t.Parallel()
c, err := configFromYAML(t, signingKeyLine+"state_dir: "+t.TempDir()+"\n")
if err != nil {
t.Fatalf("config with only state_dir set should be valid, got: %v", err)
}
db, err := sql.Open("sqlite", c.DBURL)
if err != nil {
t.Fatalf("failed to open %q: %v", c.DBURL, err)
}
t.Cleanup(func() { _ = db.Close() })
var journalMode string
err = db.QueryRowContext(t.Context(), "PRAGMA journal_mode").Scan(&journalMode)
if err != nil {
t.Fatalf("failed to read the journal mode of %q: %v", c.DBURL, err)
}
if journalMode != "wal" {
t.Errorf("journal mode of %q = %q, want wal", c.DBURL, journalMode)
}
}
func TestExplicitValidValuesAreUsed(t *testing.T) { func TestExplicitValidValuesAreUsed(t *testing.T) {
t.Parallel() t.Parallel()
@@ -318,20 +284,6 @@ func invalidHostAndCredentialCases() []abortCase {
keyAllowlistHosts, "example.com/images", keyAllowlistHosts, "example.com/images",
}, },
}, },
{
name: "allowlist host with wildcard",
yaml: signingKeyLine + "allowlist_hosts:\n - \"*.example.com\"\n",
wantErrSubstrings: []string{
keyAllowlistHosts, "*.example.com",
},
},
{
name: "allowlist host with port",
yaml: signingKeyLine + "allowlist_hosts:\n - example.com:8443\n",
wantErrSubstrings: []string{
keyAllowlistHosts, "example.com:8443",
},
},
{ {
name: "allowlist host with whitespace", name: "allowlist host with whitespace",
yaml: signingKeyLine + "allowlist_hosts:\n - \"exa mple.com\"\n", yaml: signingKeyLine + "allowlist_hosts:\n - \"exa mple.com\"\n",
@@ -24,25 +24,6 @@ func TestRefererBlocklistParsed(t *testing.T) {
} }
} }
// TestRefererBlocklistAcceptsIPAddresses checks that IPv4 and IPv6 addresses,
// the IPv6 one written without brackets, are accepted as entries.
func TestRefererBlocklistAcceptsIPAddresses(t *testing.T) {
t.Parallel()
c, err := configFromYAML(t, signingKeyLine+`referer_blocklist:
- 192.0.2.7
- "2001:db8::7"
`)
if err != nil {
t.Fatalf("IP address entries should load, got error: %v", err)
}
want := []string{"192.0.2.7", "2001:db8::7"}
if !slices.Equal(c.RefererBlocklist, want) {
t.Errorf("RefererBlocklist = %v, want %v", c.RefererBlocklist, want)
}
}
// TestRefererBlocklistOmittedIsEmpty checks that an omitted key blocks no // TestRefererBlocklistOmittedIsEmpty checks that an omitted key blocks no
// referer. // referer.
func TestRefererBlocklistOmittedIsEmpty(t *testing.T) { func TestRefererBlocklistOmittedIsEmpty(t *testing.T) {
@@ -79,27 +60,6 @@ func TestRefererBlocklistInvalidAbortsStartup(t *testing.T) {
keyRefererBlocklist, "leech.example/page", keyRefererBlocklist, "leech.example/page",
}, },
}, },
{
name: "wildcard entry",
yaml: signingKeyLine + "referer_blocklist:\n - \"*.leech.example\"\n",
wantErrSubstrings: []string{
keyRefererBlocklist, "*.leech.example",
},
},
{
name: "entry with a port",
yaml: signingKeyLine + "referer_blocklist:\n - leech.example:8080\n",
wantErrSubstrings: []string{
keyRefererBlocklist, "leech.example:8080",
},
},
{
name: "two leading dots",
yaml: signingKeyLine + "referer_blocklist:\n - ..leech.example\n",
wantErrSubstrings: []string{
keyRefererBlocklist, "..leech.example",
},
},
{ {
name: "dot only", name: "dot only",
yaml: signingKeyLine + "referer_blocklist:\n - \".\"\n", yaml: signingKeyLine + "referer_blocklist:\n - \".\"\n",
@@ -1,153 +0,0 @@
package database
import (
"context"
"database/sql"
"fmt"
"log/slog"
"path/filepath"
"sync"
"testing"
"sneak.berlin/go/pixa/internal/config"
)
// TestConcurrentWritesAllSucceed opens a database the way pixad does and
// writes to it from several goroutines at once, so the writes run on
// separate connections, as one request's writes and the background eviction
// pass do. Every write must succeed, none failing with "database is locked",
// whether or not db_url already has parameters, and the parameters it has
// must still apply.
func TestConcurrentWritesAllSucceed(t *testing.T) {
t.Parallel()
tests := []struct {
name string
query string
wantJournalMode string
}{
{
name: "db_url without parameters",
query: "",
wantJournalMode: "delete",
},
{
name: "db_url with the WAL parameter",
query: "?_pragma=journal_mode(WAL)",
wantJournalMode: "wal",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
dbURL := "file:" + filepath.Join(t.TempDir(), "state.sqlite3") + tt.query
d := &Database{
log: slog.New(slog.DiscardHandler),
config: &config.Config{DBURL: dbURL},
}
err := d.connect(t.Context())
if err != nil {
t.Fatalf("failed to connect to %q: %v", dbURL, err)
}
t.Cleanup(func() { _ = d.db.Close() })
writeConcurrently(t, d.db)
var journalMode string
err = d.db.QueryRowContext(t.Context(), "PRAGMA journal_mode").
Scan(&journalMode)
if err != nil {
t.Fatalf("failed to read the journal mode: %v", err)
}
if journalMode != tt.wantJournalMode {
t.Errorf("journal mode = %q, want %q", journalMode, tt.wantJournalMode)
}
})
}
}
// writeConcurrently runs writeLikeOneRequest from several goroutines at once
// and checks that every write was made.
func writeConcurrently(t *testing.T, db *sql.DB) {
t.Helper()
const (
writers = 4
requestsEach = 20
totalRequests = writers * requestsEach
)
ctx := t.Context()
var wg sync.WaitGroup
for writer := range writers {
wg.Go(func() {
for request := range requestsEach {
key := fmt.Sprintf("%d-%d", writer, request)
err := writeLikeOneRequest(ctx, db, key)
if err != nil {
t.Errorf("writer %d: %v", writer, err)
return
}
}
})
}
wg.Wait()
var hits, sources int
err := db.QueryRowContext(ctx, `
SELECT hit_count, (SELECT COUNT(*) FROM source_content)
FROM cache_stats WHERE id = 1
`).Scan(&hits, &sources)
if err != nil {
t.Fatalf("failed to count the writes: %v", err)
}
if hits != totalRequests || sources != totalRequests {
t.Errorf("hit_count = %d and %d source_content rows, want %d of each",
hits, sources, totalRequests)
}
}
// writeLikeOneRequest makes the writes one request and the eviction pass
// make: it counts a cache hit, stores a source, records a transformed image
// and deletes that record again.
func writeLikeOneRequest(ctx context.Context, db *sql.DB, key string) error {
_, err := db.ExecContext(ctx,
`UPDATE cache_stats SET hit_count = hit_count + 1 WHERE id = 1`)
if err != nil {
return fmt.Errorf("counting a cache hit: %w", err)
}
_, err = db.ExecContext(ctx, `INSERT INTO source_content
(content_hash, content_type, size_bytes) VALUES (?, 'image/png', 1)`, key)
if err != nil {
return fmt.Errorf("storing a source: %w", err)
}
_, err = db.ExecContext(ctx, `INSERT INTO variant_content
(cache_key, size_bytes, content_type) VALUES (?, 1, 'image/png')`, key)
if err != nil {
return fmt.Errorf("recording a transformed image: %w", err)
}
_, err = db.ExecContext(ctx,
`DELETE FROM variant_content WHERE cache_key = ?`, key)
if err != nil {
return fmt.Errorf("evicting a transformed image: %w", err)
}
return nil
}
+1 -11
View File
@@ -243,17 +243,7 @@ func (s *Database) DB() *sql.DB {
} }
func (s *Database) connect(ctx context.Context) error { func (s *Database) connect(ctx context.Context) error {
// Requests and the eviction pass write on separate connections. With dbURL := s.config.DBURL
// a busy timeout, a write that finds another one in progress waits up
// to five seconds for it instead of failing at once with "database is
// locked". The driver runs each _pragma parameter on every connection
// it opens.
separator := "?"
if strings.Contains(s.config.DBURL, "?") {
separator = "&"
}
dbURL := s.config.DBURL + separator + "_pragma=busy_timeout(5000)"
s.log.Info("connecting to database", "url", dbURL) s.log.Info("connecting to database", "url", dbURL)