Blocklists and an AS percentage file fetched by URL (closes #29)

SWWAF_BLOCKLIST_URLS names lists of addresses and netblocks, fetched every SWWAF_BLOCKLIST_REFRESH (24h, never under 1h); an IPv4-mapped line stands for its IPv4 address or netblock. reputation.json keeps each list's last try, failed or not, even one cut off by a stop, which a restart waits on as a running instance does, and its last good copy, whole, used while a fetch fails. SWWAF_BLOCKLIST_ACTION denies, limits or only logs a listed client; the log line names the lists, each raises reputation_hit, and a failed fetch raises source_failure. SWWAF_ASN_LIMIT_PERCENT_URL is fetched the same way and counts as SWWAF_ASN_LIMIT_PERCENT does, the lower winning. Judgement call: a failed fetch is retried after the refresh, not sooner. Not done: ban notes do not name the lists yet. Model: opus-5-5
2026-10-07 16:51:48 +00:00
parent 82e20e0cb5
commit 271aaafccc
19 changed files with 2588 additions and 322 deletions
@@ -0,0 +1,499 @@
+// Package reputation fetches the lists the settings name by URL: the
+// blocklists of SWWAF_BLOCKLIST_URLS, and the file of AS:percent lines
+// SWWAF_ASN_LIMIT_PERCENT_URL names. It keeps the last good copy of each,
+// whole, comment lines included, which is used while a fetch fails, and
+// when each was last tried, which the state package writes to
+// reputation.json and reads from it, so that a restart keeps them too.
+package reputation
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/http"
+	"net/netip"
+	"slices"
+	"strings"
+	"sync"
+	"time"
+
+	"sneak.berlin/go/smallwebwaf/internal/alerts"
+	"sneak.berlin/go/smallwebwaf/internal/config"
+)
+
+const (
+	// maxListBytes is the most of a list that is read. A longer one is a
+	// failure, so that a wrong URL cannot fill the memory.
+	maxListBytes = 16 << 20
+	// fetchTimeout bounds one fetch of a list.
+	fetchTimeout = time.Minute
+	// mappedBits is the length of ::ffff:0.0.0.0/96, the netblock of every
+	// IPv4-mapped address.
+	mappedBits = 96
+)
+
+var (
+	errStatus        = errors.New("the server answered")
+	errTooLong       = errors.New("the list is longer than 16 MiB")
+	errNotNetblock   = errors.New("is not an address or a netblock, such as 192.0.2.0/24")
+	errNotASNPercent = errors.New(
+		"is not an AS number, : and a percentage, such as AS64496:50")
+)
+
+// List is a list as reputation.json holds it: the URL it is fetched from,
+// when it was last tried, the fetch failed or not, and its last good copy:
+// when that was fetched, and its lines, as fetched, comment lines
+// included, both left out while no fetch of it has succeeded.
+type List struct {
+	URL     string    `json:"url"`
+	Tried   time.Time `json:"tried"`
+	Fetched time.Time `json:"fetched,omitzero"`
+	Lines   []string  `json:"lines,omitzero"`
+}
+
+// Params are what New needs.
+type Params struct {
+	// BlocklistURLs are the blocklists (SWWAF_BLOCKLIST_URLS), and
+	// ASNLimitPercentURL the file of AS:percent lines
+	// (SWWAF_ASN_LIMIT_PERCENT_URL), "" while it is unset.
+	BlocklistURLs      []string
+	ASNLimitPercentURL string
+	// Refresh is how long after a list was last fetched or tried it is
+	// fetched again (SWWAF_BLOCKLIST_REFRESH).
+	Refresh time.Duration
+	// Now tells the time, normally time.Now in UTC.
+	Now func() time.Time
+	// ProcessLog receives each fetch of a list, and why one failed.
+	ProcessLog *slog.Logger
+	// Alerts receive a source_failure alert for each fetch that fails.
+	Alerts *alerts.Queue
+}
+
+// Lists are the lists Params names, each with its last good copy. They
+// are safe for concurrent use.
+type Lists struct {
+	params     Params
+	httpClient *http.Client
+
+	mu sync.Mutex
+	// lists are by URL, one for each URL Params names.
+	lists map[string]*list
+}
+
+// list is one list: what reputation.json keeps of it, its last try, zero
+// before the first, and its last good copy, what that copy says, and how
+// many fetches of it failed.
+type list struct {
+	kept     List
+	entries  entries
+	failures int
+}
+
+// entries are what the lines of a copy say: for a blocklist, the netblocks
+// it names, with the lengths among them, and for the file of AS:percent
+// lines, the percentage it gives each AS number.
+type entries struct {
+	netblocks map[netip.Prefix]bool
+	lengths   []int
+	percents  map[string]int64
+}
+
+// New returns the lists, without a copy of any yet.
+func New(params Params) *Lists {
+	l := &Lists{params: params, httpClient: &http.Client{}, lists: map[string]*list{}}
+
+	for _, listURL := range l.URLs() {
+		l.lists[listURL] = &list{kept: List{URL: listURL}}
+	}
+
+	return l
+}
+
+// URLs returns the URL of every list: the blocklists' in the order
+// SWWAF_BLOCKLIST_URLS names them, then SWWAF_ASN_LIMIT_PERCENT_URL.
+func (l *Lists) URLs() []string {
+	urls := slices.Clone(l.params.BlocklistURLs)
+	if l.params.ASNLimitPercentURL != "" {
+		urls = append(urls, l.params.ASNLimitPercentURL)
+	}
+
+	return urls
+}
+
+// ListedBy returns the URLs of the blocklists whose copy lists addr, in
+// the order SWWAF_BLOCKLIST_URLS names them.
+func (l *Lists) ListedBy(addr netip.Addr) []string {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+
+	var listedBy []string
+
+	for _, listURL := range l.params.BlocklistURLs {
+		if l.lists[listURL].entries.contain(addr) {
+			listedBy = append(listedBy, listURL)
+		}
+	}
+
+	return listedBy
+}
+
+// ASNLimitPercent returns the percentage the copy of the file of
+// AS:percent lines gives asn, and whether it lists asn.
+func (l *Lists) ASNLimitPercent(asn string) (int64, bool) {
+	if l.params.ASNLimitPercentURL == "" {
+		return 0, false
+	}
+
+	l.mu.Lock()
+	defer l.mu.Unlock()
+
+	percent, listed := l.lists[l.params.ASNLimitPercentURL].entries.percents[asn]
+
+	return percent, listed
+}
+
+// Fetched returns when the copy in use of the list at listURL was
+// fetched, or zero while there is none.
+func (l *Lists) Fetched(listURL string) time.Time {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+
+	return l.lists[listURL].kept.Fetched
+}
+
+// Failures returns how many fetches of the list at listURL failed.
+func (l *Lists) Failures(listURL string) int {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+
+	return l.lists[listURL].failures
+}
+
+// Run fetches each list once Refresh has passed since it was last fetched
+// or tried, the later of the two, until ctx is done. A list never tried is
+// fetched at once, and so is one whose last try or copy, read from
+// reputation.json, is that old.
+func (l *Lists) Run(ctx context.Context) {
+	if len(l.lists) == 0 {
+		return
+	}
+
+	for ctx.Err() == nil {
+		next := l.fetchDue(ctx)
+		timer := time.NewTimer(next.Sub(l.params.Now()))
+
+		select {
+		case <-ctx.Done():
+		case <-timer.C:
+		}
+
+		timer.Stop()
+	}
+}
+
+// Snapshot returns each list that has been tried, with its copy, if it
+// has one, sorted by URL, as reputation.json lists them.
+func (l *Lists) Snapshot() []List {
+	l.mu.Lock()
+
+	tried := make([]List, 0, len(l.lists))
+
+	for _, held := range l.lists {
+		if !held.kept.Tried.IsZero() {
+			tried = append(tried, held.kept)
+		}
+	}
+
+	l.mu.Unlock()
+
+	slices.SortFunc(tried, func(a, b List) int {
+		return strings.Compare(a.URL, b.URL)
+	})
+
+	return tried
+}
+
+// Load puts lists, read from reputation.json, in place of the last tries
+// and copies held. A list Params does not name is dropped. A copy with a
+// line that parse refuses is an error, and then nothing changes.
+func (l *Lists) Load(lists []List) error {
+	found := make(map[string]entries, len(lists))
+
+	for _, kept := range lists {
+		if _, named := l.lists[kept.URL]; !named {
+			continue
+		}
+
+		read, err := l.parse(kept.URL, kept.Lines)
+		if err != nil {
+			return fmt.Errorf("the copy of %s: %w", kept.URL, err)
+		}
+
+		found[kept.URL] = read
+	}
+
+	l.mu.Lock()
+	defer l.mu.Unlock()
+
+	for listURL, held := range l.lists {
+		held.kept, held.entries = List{URL: listURL}, entries{}
+	}
+
+	for _, kept := range lists {
+		read, named := found[kept.URL]
+		if named {
+			l.lists[kept.URL].kept, l.lists[kept.URL].entries = kept, read
+		}
+	}
+
+	return nil
+}
+
+// fetchDue fetches each list that is due, one after another, and returns
+// when the next is due. Once ctx has ended, it starts none, since a fetch
+// cut off is noted as a try.
+func (l *Lists) fetchDue(ctx context.Context) time.Time {
+	var next time.Time
+
+	for _, listURL := range l.URLs() {
+		due := l.due(listURL)
+		if ctx.Err() == nil && !l.params.Now().Before(due) {
+			l.fetch(ctx, listURL)
+			due = l.due(listURL)
+		}
+
+		if next.IsZero() || due.Before(next) {
+			next = due
+		}
+	}
+
+	return next
+}
+
+// due returns when the list at listURL is to be fetched: Refresh after it
+// was last fetched or tried, the later of the two.
+func (l *Lists) due(listURL string) time.Time {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+
+	held := l.lists[listURL]
+
+	last := held.kept.Fetched
+	if held.kept.Tried.After(last) {
+		last = held.kept.Tried
+	}
+
+	return last.Add(l.params.Refresh)
+}
+
+// fetch fetches the list at listURL, and notes the try. A good copy takes
+// the place of the one held. A failure leaves that in use, and is counted,
+// logged and raised as a source_failure alert. A fetch cut off as ctx
+// ends, as smallwebwaf stops, is no failure, but is still noted as a try,
+// so that a restart waits for it: the server may have had its request.
+func (l *Lists) fetch(ctx context.Context, listURL string) {
+	lines, err := l.get(ctx, listURL)
+
+	var found entries
+	if err == nil {
+		found, err = l.parse(listURL, lines)
+	}
+
+	cutOff := err != nil && ctx.Err() != nil
+	now := l.params.Now()
+
+	l.mu.Lock()
+
+	held := l.lists[listURL]
+	held.kept.Tried = now
+
+	if err == nil {
+		held.kept.Fetched, held.kept.Lines = now, lines
+		held.entries = found
+	} else if !cutOff {
+		held.failures++
+	}
+
+	l.mu.Unlock()
+
+	if cutOff {
+		return
+	}
+
+	if err != nil {
+		const failed = "fetching a list failed"
+
+		// Raised before it is logged, so that the alert is there once the
+		// log line is.
+		l.params.Alerts.Raise(alerts.Alert{
+			Event:  alerts.EventSourceFailure,
+			Reason: failed,
+			Detail: map[string]any{"source": listURL, "error": err.Error()},
+		})
+		l.params.ProcessLog.Warn(failed, "url", listURL, "error", err.Error())
+
+		return
+	}
+
+	l.params.ProcessLog.Info("fetched a list", "url", listURL, "lines", len(lines))
+}
+
+// get fetches the list at listURL, and returns its lines. An answer other
+// than 200, or a list longer than maxListBytes, is a failure.
+func (l *Lists) get(ctx context.Context, listURL string) ([]string, error) {
+	ctx, cancel := context.WithTimeout(ctx, fetchTimeout)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, listURL, http.NoBody)
+	if err != nil {
+		return nil, fmt.Errorf("make the request: %w", err)
+	}
+
+	res, err := l.httpClient.Do(req)
+	if err != nil {
+		// Do's error names the URL, which the log line and the alert name
+		// already: only what went wrong is kept.
+		return nil, fmt.Errorf("fetch the list: %w", errors.Unwrap(err))
+	}
+
+	defer func() {
+		_ = res.Body.Close()
+	}()
+
+	if res.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("%w %s", errStatus, res.Status)
+	}
+
+	body, err := io.ReadAll(io.LimitReader(res.Body, maxListBytes+1))
+	if err != nil {
+		return nil, fmt.Errorf("read the list: %w", err)
+	}
+
+	if len(body) > maxListBytes {
+		return nil, errTooLong
+	}
+
+	lines := []string{}
+	for line := range strings.Lines(string(body)) {
+		lines = append(lines, strings.TrimSuffix(line, "\n"))
+	}
+
+	return lines, nil
+}
+
+// parse reads the lines of the list at listURL: those of a blocklist, or
+// of the file of AS:percent lines. Anything after a ; or a # on a line is
+// left out, and so is a line left blank. Any other line that does not read
+// is an error naming it by its number.
+func (l *Lists) parse(listURL string, lines []string) (entries, error) {
+	if listURL == l.params.ASNLimitPercentURL {
+		return parsePercents(lines)
+	}
+
+	return parseNetblocks(lines)
+}
+
+// parseNetblocks reads a blocklist's lines, each an address or a netblock
+// as the settings take them.
+func parseNetblocks(lines []string) (entries, error) {
+	found := entries{netblocks: map[netip.Prefix]bool{}}
+
+	for i, line := range lines {
+		text := withoutComment(line)
+		if text == "" {
+			continue
+		}
+
+		netblock, ok := parseNetblock(text)
+		if !ok {
+			return entries{}, fmt.Errorf("line %d %w", i+1, errNotNetblock)
+		}
+
+		found.netblocks[netblock] = true
+
+		if !slices.Contains(found.lengths, netblock.Bits()) {
+			found.lengths = append(found.lengths, netblock.Bits())
+		}
+	}
+
+	return found, nil
+}
+
+// parseNetblock reads text, a line of a blocklist, and reports whether it
+// is an address or a netblock as the settings take them. A client's IPv4
+// address is checked as IPv4, never IPv4-mapped, so an IPv4-mapped line,
+// such as ::ffff:192.0.2.0/120, is read as the IPv4 address or netblock it
+// stands for, 192.0.2.0/24, and a mapped netblock shorter than /96, which
+// stands for none, is refused.
+func parseNetblock(text string) (netip.Prefix, bool) {
+	netblock, err := config.ParseNetblock(text)
+	if err != nil {
+		return netip.Prefix{}, false
+	}
+
+	// The address as written: ParseNetblock's has the bits past the
+	// netblock's length cleared, the ::ffff among them below /96.
+	written, _, _ := strings.Cut(text, "/")
+	if addr, _ := netip.ParseAddr(written); !addr.Is4In6() {
+		return netblock, true
+	}
+
+	if netblock.Bits() < mappedBits {
+		return netip.Prefix{}, false
+	}
+
+	return netip.PrefixFrom(netblock.Addr().Unmap(), netblock.Bits()-mappedBits), true
+}
+
+// parsePercents reads the lines of the file of AS:percent lines, each an
+// AS number, : and a percentage, as SWWAF_ASN_LIMIT_PERCENT takes them. An
+// AS number listed more than once gets the lowest of its percentages.
+func parsePercents(lines []string) (entries, error) {
+	found := entries{percents: map[string]int64{}}
+
+	for i, line := range lines {
+		text := withoutComment(line)
+		if text == "" {
+			continue
+		}
+
+		asnText, percentText, _ := strings.Cut(text, ":")
+		asn, asnErr := config.ParseASN(asnText)
+
+		percent, percentErr := config.ParsePercent(percentText)
+		if asnErr != nil || percentErr != nil {
+			return entries{}, fmt.Errorf("line %d %w", i+1, errNotASNPercent)
+		}
+
+		earlier, listed := found.percents[asn]
+		if !listed || percent < earlier {
+			found.percents[asn] = percent
+		}
+	}
+
+	return found, nil
+}
+
+// withoutComment returns line without anything after a ; or a #, and
+// without the spaces around what is left.
+func withoutComment(line string) string {
+	text, _, _ := strings.Cut(line, ";")
+	text, _, _ = strings.Cut(text, "#")
+
+	return strings.TrimSpace(text)
+}
+
+// contain reports whether the netblocks of a blocklist's copy hold addr:
+// whether addr, cut to one of their lengths, is one of them.
+func (e entries) contain(addr netip.Addr) bool {
+	for _, length := range e.lengths {
+		netblock, err := addr.Prefix(length)
+		if err == nil && e.netblocks[netblock] {
+			return true
+		}
+	}
+
+	return false
+}