smallwebwaf/internal/reputation/reputation.go

// Package reputation fetches the lists the settings name by URL: the
// blocklists of SWWAF_BLOCKLIST_URLS, and the file of AS:percent lines
// SWWAF_ASN_LIMIT_PERCENT_URL names. It keeps the last good copy of each,
// whole, comment lines included, which is used while a fetch fails, and
// when each was last tried. It also asks the DNSBL zones of
// SWWAF_DNSBL_ZONES about clients, and keeps their verdicts, and checks
// clients with AbuseIPDB, and keeps their scores and the checks spent
// today. The state package writes all of these to reputation.json and
// reads them from it, so that a restart keeps them too.
package reputation

import (
	"context"
	"errors"
	"fmt"
	"io"
	"log/slog"
	"net/http"
	"net/netip"
	"slices"
	"strings"
	"sync"
	"time"

	"sneak.berlin/go/smallwebwaf/internal/alerts"
	"sneak.berlin/go/smallwebwaf/internal/config"
)

const (
	// maxListBytes is the most of a list that is read. A longer one is a
	// failure, so that a wrong URL cannot fill the memory.
	maxListBytes = 16 << 20
	// fetchTimeout bounds one fetch of a list.
	fetchTimeout = time.Minute
	// mappedBits is the length of ::ffff:0.0.0.0/96, the netblock of every
	// IPv4-mapped address.
	mappedBits = 96
)

var (
	errStatus        = errors.New("the server answered")
	errTooLong       = errors.New("the list is longer than 16 MiB")
	errNotNetblock   = errors.New("is not an address or a netblock, such as 192.0.2.0/24")
	errNotASNPercent = errors.New(
		"is not an AS number, : and a percentage, such as AS64496:50")
)

// List is a list as reputation.json holds it: the URL it is fetched from,
// when it was last tried, the fetch failed or not, and its last good copy:
// when that was fetched, and its lines, as fetched, comment lines
// included, both left out while no fetch of it has succeeded.
type List struct {
	URL     string    `json:"url"`
	Tried   time.Time `json:"tried"`
	Fetched time.Time `json:"fetched,omitzero"`
	Lines   []string  `json:"lines,omitzero"`
}

// Params are what New needs.
type Params struct {
	// BlocklistURLs are the blocklists (SWWAF_BLOCKLIST_URLS), and
	// ASNLimitPercentURL the file of AS:percent lines
	// (SWWAF_ASN_LIMIT_PERCENT_URL), "" while it is unset.
	BlocklistURLs      []string
	ASNLimitPercentURL string
	// Refresh is how long after a list was last fetched or tried it is
	// fetched again (SWWAF_BLOCKLIST_REFRESH).
	Refresh time.Duration
	// Now tells the time, normally time.Now in UTC.
	Now func() time.Time
	// ProcessLog receives each fetch of a list, and why one failed.
	ProcessLog *slog.Logger
	// Alerts receive a source_failure alert for each fetch that fails.
	Alerts *alerts.Queue
}

// Lists are the lists Params names, each with its last good copy. They
// are safe for concurrent use.
type Lists struct {
	params     Params
	httpClient *http.Client

	mu sync.Mutex
	// lists are by URL, one for each URL Params names.
	lists map[string]*list
}

// list is one list: what reputation.json keeps of it, its last try, zero
// before the first, and its last good copy, what that copy says, and how
// many fetches of it failed.
type list struct {
	kept     List
	entries  entries
	failures int
}

// entries are what the lines of a copy say: for a blocklist, the netblocks
// it names, with the lengths among them, and for the file of AS:percent
// lines, the percentage it gives each AS number.
type entries struct {
	netblocks map[netip.Prefix]bool
	lengths   []int
	percents  map[string]int64
}

// New returns the lists, without a copy of any yet.
func New(params Params) *Lists {
	l := &Lists{params: params, httpClient: &http.Client{}, lists: map[string]*list{}}

	for _, listURL := range l.URLs() {
		l.lists[listURL] = &list{kept: List{URL: listURL}}
	}

	return l
}

// URLs returns the URL of every list: the blocklists' in the order
// SWWAF_BLOCKLIST_URLS names them, then SWWAF_ASN_LIMIT_PERCENT_URL.
func (l *Lists) URLs() []string {
	urls := slices.Clone(l.params.BlocklistURLs)
	if l.params.ASNLimitPercentURL != "" {
		urls = append(urls, l.params.ASNLimitPercentURL)
	}

	return urls
}

// ListedBy returns the URLs of the blocklists whose copy lists addr, in
// the order SWWAF_BLOCKLIST_URLS names them.
func (l *Lists) ListedBy(addr netip.Addr) []string {
	l.mu.Lock()
	defer l.mu.Unlock()

	var listedBy []string

	for _, listURL := range l.params.BlocklistURLs {
		if l.lists[listURL].entries.contain(addr) {
			listedBy = append(listedBy, listURL)
		}
	}

	return listedBy
}

// ASNLimitPercent returns the percentage the copy of the file of
// AS:percent lines gives asn, and whether it lists asn.
func (l *Lists) ASNLimitPercent(asn string) (int64, bool) {
	if l.params.ASNLimitPercentURL == "" {
		return 0, false
	}

	l.mu.Lock()
	defer l.mu.Unlock()

	percent, listed := l.lists[l.params.ASNLimitPercentURL].entries.percents[asn]

	return percent, listed
}

// Fetched returns when the copy in use of the list at listURL was
// fetched, or zero while there is none.
func (l *Lists) Fetched(listURL string) time.Time {
	l.mu.Lock()
	defer l.mu.Unlock()

	return l.lists[listURL].kept.Fetched
}

// Failures returns how many fetches of the list at listURL failed.
func (l *Lists) Failures(listURL string) int {
	l.mu.Lock()
	defer l.mu.Unlock()

	return l.lists[listURL].failures
}

// Run fetches each list once Refresh has passed since it was last fetched
// or tried, the later of the two, until ctx is done. A list never tried is
// fetched at once, and so is one whose last try or copy, read from
// reputation.json, is that old.
func (l *Lists) Run(ctx context.Context) {
	if len(l.lists) == 0 {
		return
	}

	for ctx.Err() == nil {
		next := l.fetchDue(ctx)
		timer := time.NewTimer(next.Sub(l.params.Now()))

		select {
		case <-ctx.Done():
		case <-timer.C:
		}

		timer.Stop()
	}
}

// Snapshot returns each list that has been tried, with its copy, if it
// has one, sorted by URL, as reputation.json lists them.
func (l *Lists) Snapshot() []List {
	l.mu.Lock()

	tried := make([]List, 0, len(l.lists))

	for _, held := range l.lists {
		if !held.kept.Tried.IsZero() {
			tried = append(tried, held.kept)
		}
	}

	l.mu.Unlock()

	slices.SortFunc(tried, func(a, b List) int {
		return strings.Compare(a.URL, b.URL)
	})

	return tried
}

// Load puts lists, read from reputation.json, in place of the last tries
// and copies held. A list Params does not name is dropped. A copy with a
// line that parse refuses is an error, and then nothing changes.
func (l *Lists) Load(lists []List) error {
	found := make(map[string]entries, len(lists))

	for _, kept := range lists {
		if _, named := l.lists[kept.URL]; !named {
			continue
		}

		read, err := l.parse(kept.URL, kept.Lines)
		if err != nil {
			return fmt.Errorf("the copy of %s: %w", kept.URL, err)
		}

		found[kept.URL] = read
	}

	l.mu.Lock()
	defer l.mu.Unlock()

	for listURL, held := range l.lists {
		held.kept, held.entries = List{URL: listURL}, entries{}
	}

	for _, kept := range lists {
		read, named := found[kept.URL]
		if named {
			l.lists[kept.URL].kept, l.lists[kept.URL].entries = kept, read
		}
	}

	return nil
}

// fetchDue fetches each list that is due, one after another, and returns
// when the next is due. Once ctx has ended, it starts none, since a fetch
// cut off is noted as a try.
func (l *Lists) fetchDue(ctx context.Context) time.Time {
	var next time.Time

	for _, listURL := range l.URLs() {
		due := l.due(listURL)
		if ctx.Err() == nil && !l.params.Now().Before(due) {
			l.fetch(ctx, listURL)
			due = l.due(listURL)
		}

		if next.IsZero() || due.Before(next) {
			next = due
		}
	}

	return next
}

// due returns when the list at listURL is to be fetched: Refresh after it
// was last fetched or tried, the later of the two.
func (l *Lists) due(listURL string) time.Time {
	l.mu.Lock()
	defer l.mu.Unlock()

	held := l.lists[listURL]

	last := held.kept.Fetched
	if held.kept.Tried.After(last) {
		last = held.kept.Tried
	}

	return last.Add(l.params.Refresh)
}

// fetch fetches the list at listURL, and notes the try. A good copy takes
// the place of the one held. A failure leaves that in use, and is counted,
// logged and raised as a source_failure alert. A fetch cut off as ctx
// ends, as smallwebwaf stops, is no failure, but is still noted as a try,
// so that a restart waits for it: the server may have had its request.
func (l *Lists) fetch(ctx context.Context, listURL string) {
	lines, err := l.get(ctx, listURL)

	var found entries
	if err == nil {
		found, err = l.parse(listURL, lines)
	}

	cutOff := err != nil && ctx.Err() != nil
	now := l.params.Now()

	l.mu.Lock()

	held := l.lists[listURL]
	held.kept.Tried = now

	if err == nil {
		held.kept.Fetched, held.kept.Lines = now, lines
		held.entries = found
	} else if !cutOff {
		held.failures++
	}

	l.mu.Unlock()

	if cutOff {
		return
	}

	if err != nil {
		const failed = "fetching a list failed"

		// Raised before it is logged, so that the alert is there once the
		// log line is.
		raiseFailure(l.params.Alerts, failed, listURL, err)
		l.params.ProcessLog.Warn(failed, "url", listURL, "error", err.Error())

		return
	}

	l.params.ProcessLog.Info("fetched a list", "url", listURL, "lines", len(lines))
}

// raiseFailure raises a source_failure alert into queue, with reason, and
// in its detail the source that failed, a list's URL, a zone with its key
// masked or abuseipdb, and err.
func raiseFailure(queue *alerts.Queue, reason, source string, err error) {
	queue.Raise(alerts.Alert{
		Event:  alerts.EventSourceFailure,
		Reason: reason,
		Detail: map[string]any{"source": source, "error": err.Error()},
	})
}

// get fetches the list at listURL, and returns its lines. An answer other
// than 200, or a list longer than maxListBytes, is a failure.
func (l *Lists) get(ctx context.Context, listURL string) ([]string, error) {
	ctx, cancel := context.WithTimeout(ctx, fetchTimeout)
	defer cancel()

	req, err := http.NewRequestWithContext(ctx, http.MethodGet, listURL, http.NoBody)
	if err != nil {
		return nil, fmt.Errorf("make the request: %w", err)
	}

	res, err := l.httpClient.Do(req)
	if err != nil {
		// Do's error names the URL, which the log line and the alert name
		// already: only what went wrong is kept.
		return nil, fmt.Errorf("fetch the list: %w", errors.Unwrap(err))
	}

	defer func() {
		_ = res.Body.Close()
	}()

	if res.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("%w %s", errStatus, res.Status)
	}

	body, err := io.ReadAll(io.LimitReader(res.Body, maxListBytes+1))
	if err != nil {
		return nil, fmt.Errorf("read the list: %w", err)
	}

	if len(body) > maxListBytes {
		return nil, errTooLong
	}

	lines := []string{}
	for line := range strings.Lines(string(body)) {
		lines = append(lines, strings.TrimSuffix(line, "\n"))
	}

	return lines, nil
}

// parse reads the lines of the list at listURL: those of a blocklist, or
// of the file of AS:percent lines. Anything after a ; or a # on a line is
// left out, and so is a line left blank. Any other line that does not read
// is an error naming it by its number.
func (l *Lists) parse(listURL string, lines []string) (entries, error) {
	if listURL == l.params.ASNLimitPercentURL {
		return parsePercents(lines)
	}

	return parseNetblocks(lines)
}

// parseNetblocks reads a blocklist's lines, each an address or a netblock
// as the settings take them.
func parseNetblocks(lines []string) (entries, error) {
	found := entries{netblocks: map[netip.Prefix]bool{}}

	for i, line := range lines {
		text := withoutComment(line)
		if text == "" {
			continue
		}

		netblock, ok := parseNetblock(text)
		if !ok {
			return entries{}, fmt.Errorf("line %d %w", i+1, errNotNetblock)
		}

		found.netblocks[netblock] = true

		if !slices.Contains(found.lengths, netblock.Bits()) {
			found.lengths = append(found.lengths, netblock.Bits())
		}
	}

	return found, nil
}

// parseNetblock reads text, a line of a blocklist, and reports whether it
// is an address or a netblock as the settings take them. A client's IPv4
// address is checked as IPv4, never IPv4-mapped, so an IPv4-mapped line,
// such as ::ffff:192.0.2.0/120, is read as the IPv4 address or netblock it
// stands for, 192.0.2.0/24, and a mapped netblock shorter than /96, which
// stands for none, is refused.
func parseNetblock(text string) (netip.Prefix, bool) {
	netblock, err := config.ParseNetblock(text)
	if err != nil {
		return netip.Prefix{}, false
	}

	// The address as written: ParseNetblock's has the bits past the
	// netblock's length cleared, the ::ffff among them below /96.
	written, _, _ := strings.Cut(text, "/")
	if addr, _ := netip.ParseAddr(written); !addr.Is4In6() {
		return netblock, true
	}

	if netblock.Bits() < mappedBits {
		return netip.Prefix{}, false
	}

	return netip.PrefixFrom(netblock.Addr().Unmap(), netblock.Bits()-mappedBits), true
}

// parsePercents reads the lines of the file of AS:percent lines, each an
// AS number, : and a percentage, as SWWAF_ASN_LIMIT_PERCENT takes them. An
// AS number listed more than once gets the lowest of its percentages.
func parsePercents(lines []string) (entries, error) {
	found := entries{percents: map[string]int64{}}

	for i, line := range lines {
		text := withoutComment(line)
		if text == "" {
			continue
		}

		asnText, percentText, _ := strings.Cut(text, ":")
		asn, asnErr := config.ParseASN(asnText)

		percent, percentErr := config.ParsePercent(percentText)
		if asnErr != nil || percentErr != nil {
			return entries{}, fmt.Errorf("line %d %w", i+1, errNotASNPercent)
		}

		earlier, listed := found.percents[asn]
		if !listed || percent < earlier {
			found.percents[asn] = percent
		}
	}

	return found, nil
}

// withoutComment returns line without anything after a ; or a #, and
// without the spaces around what is left.
func withoutComment(line string) string {
	text, _, _ := strings.Cut(line, ";")
	text, _, _ = strings.Cut(text, "#")

	return strings.TrimSpace(text)
}

// contain reports whether the netblocks of a blocklist's copy hold addr:
// whether addr, cut to one of their lengths, is one of them.
func (e entries) contain(addr netip.Addr) bool {
	for _, length := range e.lengths {
		netblock, err := addr.Prefix(length)
		if err == nil && e.netblocks[netblock] {
			return true
		}
	}

	return false
}