Send every log line to a syslog server as well (closes #28)

With SWWAF_LOG_REMOTE_URL set (syslog+udp, syslog+tcp or syslog+tls),
every line on stdout is also sent as the message of an RFC 5424 record,
octet-counted over TCP and TLS, from a bounded buffer that drops its
oldest line when full, so a slow or unreachable server holds up nothing.
Failed connections are retried with backoff; lines sent, dropped and
waiting are metrics. At a stop the lines still waiting get at most two
seconds. Standard library only: log/syslog writes only the older format.

Deviation: SWWAF_LOG_REMOTE_APP_NAME defaults to the host's name until
SWWAF_INSTANCE_NAME exists.

Model: opus-5-5

internal/config/config.go

@@ -4,6 +4,7 @@
 package config
 
 import (
+	"crypto/x509"
 	"errors"
 	"fmt"
 	"log/slog"
@@ -12,12 +13,15 @@ import (
 	"net/http"
 	"net/netip"
 	"net/url"
+	"os"
 	"path/filepath"
 	"slices"
 	"strconv"
 	"strings"
 	"time"
 	"unicode/utf8"
+
+	"sneak.berlin/go/smallwebwaf/internal/remotelog"
 )
 
 // Config is smallwebwaf's settings. A timeout, size or rate limit of zero
@@ -115,6 +119,20 @@ type Config struct {
 	// metrics (SWWAF_METRICS_TOP_N).
 	MetricsToken string
 	MetricsTopN  int
+	// LogRemoteURL is where every line on stdout is also sent
+	// (SWWAF_LOG_REMOTE_URL), nil while it is unset and nothing is sent.
+	// LogRemoteTLSCAs are the certificates a syslog+tls endpoint's
+	// certificate must chain to (SWWAF_LOG_REMOTE_TLS_CA_FILE), nil while
+	// it is unset and the host's own are used. LogRemoteBuffer is the most
+	// lines held while they wait to be sent (SWWAF_LOG_REMOTE_BUFFER).
+	// LogRemoteFacility is the number of the syslog facility
+	// (SWWAF_LOG_REMOTE_FACILITY), and LogRemoteAppName the APP-NAME
+	// (SWWAF_LOG_REMOTE_APP_NAME), of the records the lines are sent in.
+	LogRemoteURL      *url.URL
+	LogRemoteTLSCAs   *x509.CertPool
+	LogRemoteBuffer   int
+	LogRemoteFacility int
+	LogRemoteAppName  string
 
 	// settings are the values read, as given or by default, for the
 	// log line at start.
@@ -166,8 +184,15 @@ var (
 		"is not the length of an IPv4 netblock, from 0 to 32, such as 24")
 	errNotAbsolutePath = errors.New(
 		"is not an absolute path, such as /var/lib/smallwebwaf")
-	errShortToken = errors.New("is shorter than 32 characters")
-	errNotMode    = errors.New("is not enforce or observe")
+	errShortToken      = errors.New("is shorter than 32 characters")
+	errNotMode         = errors.New("is not enforce or observe")
+	errNotLogRemoteURL = errors.New(
+		"is not syslog+udp, syslog+tcp or syslog+tls with a host and a port, " +
+			"and nothing more, such as syslog+tls://logs.example:6514")
+	errNoCertificate = errors.New("holds no PEM certificate")
+	errNotFacility   = errors.New("is not a syslog facility such as local0 or daemon")
+	errNotAppName    = errors.New(
+		"is not 1 to 48 printable ASCII characters without a space, such as gitea")
 )
 
 // FromEnvironment reads the settings with lookupEnv, normally
@@ -209,8 +234,16 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) {
 		StateCounterInterval: env.durationNotOff("SWWAF_STATE_COUNTER_INTERVAL", "15m"),
 		MetricsToken:         env.token("SWWAF_METRICS_TOKEN"),
 		MetricsTopN:          env.numberNotOff("SWWAF_METRICS_TOP_N", "50"),
+		LogRemoteURL:         env.logRemoteURL("SWWAF_LOG_REMOTE_URL"),
+		LogRemoteTLSCAs:      env.certificates("SWWAF_LOG_REMOTE_TLS_CA_FILE"),
+		LogRemoteBuffer:      env.numberNotOff("SWWAF_LOG_REMOTE_BUFFER", "10000"),
+		LogRemoteFacility:    env.facility("SWWAF_LOG_REMOTE_FACILITY", "local0"),
 	}
 
+	hostname, _ := os.Hostname() // "" when the host has no name to give
+	cfg.LogRemoteAppName = env.appName("SWWAF_LOG_REMOTE_APP_NAME", hostname,
+		cfg.LogRemoteURL != nil)
+
 	for _, country := range cfg.ExclusivelyAllowedCountries {
 		if slices.Contains(cfg.DeniedCountries, country) {
 			env.check("SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES",
@@ -405,6 +438,68 @@ func (e *environment) token(name string) string {
 	return value
 }
 
+// logRemoteURL reads the setting that is where every log line is also
+// sent. Unset or empty, it is nil, and nothing is sent.
+func (e *environment) logRemoteURL(name string) *url.URL {
+	value := e.value(name, "")
+	if value == "" {
+		return nil
+	}
+
+	remote, err := parseLogRemoteURL(value)
+	e.check(name, err)
+
+	return remote
+}
+
+// certificates reads a setting that is the path of a file of PEM
+// certificates. Unset or empty, it is nil.
+func (e *environment) certificates(name string) *x509.CertPool {
+	path := e.value(name, "")
+	if path == "" {
+		return nil
+	}
+
+	pem, err := os.ReadFile(path) //nolint:gosec // a file the admin names
+	if err != nil {
+		e.check(name, fmt.Errorf("cannot be read: %w", err))
+
+		return nil
+	}
+
+	pool := x509.NewCertPool()
+	if !pool.AppendCertsFromPEM(pem) {
+		e.check(name, fmt.Errorf("%q %w", path, errNoCertificate))
+
+		return nil
+	}
+
+	return pool
+}
+
+// facility reads a setting that is a syslog facility, and returns its
+// number.
+func (e *environment) facility(name, defaultValue string) int {
+	number, err := parseFacility(e.value(name, defaultValue))
+	e.check(name, err)
+
+	return number
+}
+
+// appName reads the setting that is the APP-NAME of the records the log
+// lines are sent in. Its value is checked when it is set, and while lines
+// are sent, when they would be sent with its default.
+func (e *environment) appName(name, defaultValue string, sending bool) string {
+	_, set := e.lookupEnv(name)
+
+	value := e.value(name, defaultValue)
+	if (set || sending) && !isAppName(value) {
+		e.check(name, fmt.Errorf("%q %w", value, errNotAppName))
+	}
+
+	return value
+}
+
 // parseDuration reads a duration in Go's syntax, such as 90s or 15m, a
 // whole number of days such as 7d, or off.
 func parseDuration(value string) (time.Duration, error) {
@@ -709,3 +804,64 @@ func parseUpstreamURL(value string) (*url.URL, error) {
 
 	return upstream, nil
 }
+
+// parseLogRemoteURL reads where every log line is also sent:
+// syslog+udp, syslog+tcp or syslog+tls, a host and a port from 1 to
+// 65535, and nothing else.
+func parseLogRemoteURL(value string) (*url.URL, error) {
+	remote, err := url.Parse(value)
+	if err != nil {
+		return nil, fmt.Errorf("%q %w", value, errNotLogRemoteURL)
+	}
+
+	schemes := []string{remotelog.SchemeUDP, remotelog.SchemeTCP, remotelog.SchemeTLS}
+	port, err := strconv.ParseUint(remote.Port(), 10, 16)
+
+	onlySchemeHostAndPort := slices.Contains(schemes, remote.Scheme) &&
+		remote.Hostname() != "" && err == nil && port != 0 &&
+		remote.User == nil && remote.Opaque == "" &&
+		(remote.Path == "" || remote.Path == "/") &&
+		remote.RawQuery == "" && remote.Fragment == ""
+	if !onlySchemeHostAndPort {
+		return nil, fmt.Errorf("%q %w", value, errNotLogRemoteURL)
+	}
+
+	return remote, nil
+}
+
+// parseFacility reads the name of a syslog facility, and returns its
+// number, as RFC 5424 numbers them.
+func parseFacility(value string) (int, error) {
+	//nolint:mnd // the facilities' numbers in RFC 5424
+	number, known := map[string]int{
+		"kern": 0, "user": 1, "mail": 2, "daemon": 3, "auth": 4, "syslog": 5,
+		"lpr": 6, "news": 7, "uucp": 8, "cron": 9, "authpriv": 10, "ftp": 11,
+		"local0": 16, "local1": 17, "local2": 18, "local3": 19,
+		"local4": 20, "local5": 21, "local6": 22, "local7": 23,
+	}[value]
+	if !known {
+		return 0, fmt.Errorf("%q %w", value, errNotFacility)
+	}
+
+	return number, nil
+}
+
+// appNameMaxLength is the most characters RFC 5424 allows in an
+// APP-NAME.
+const appNameMaxLength = 48
+
+// isAppName reports whether value can be an APP-NAME: 1 to
+// appNameMaxLength printable ASCII characters, none of them a space.
+func isAppName(value string) bool {
+	if value == "" || len(value) > appNameMaxLength {
+		return false
+	}
+
+	for _, char := range []byte(value) {
+		if char < '!' || char > '~' {
+			return false
+		}
+	}
+
+	return true
+}

internal/config/config_test.go

@@ -2,10 +2,13 @@ package config_test
 
 import (
 	"bytes"
+	"crypto/x509"
 	"encoding/json"
 	"log/slog"
 	"maps"
 	"net/netip"
+	"os"
+	"path/filepath"
 	"slices"
 	"strings"
 	"testing"
@@ -47,8 +50,27 @@ const (
 	stateCounterInterval    = "SWWAF_STATE_COUNTER_INTERVAL"
 	metricsToken            = "SWWAF_METRICS_TOKEN" //nolint:gosec // the setting's name
 	metricsTopN             = "SWWAF_METRICS_TOP_N"
+	logRemoteURL            = "SWWAF_LOG_REMOTE_URL"
+	logRemoteTLSCAFile      = "SWWAF_LOG_REMOTE_TLS_CA_FILE"
+	logRemoteBuffer         = "SWWAF_LOG_REMOTE_BUFFER"
+	logRemoteFacility       = "SWWAF_LOG_REMOTE_FACILITY"
+	logRemoteAppName        = "SWWAF_LOG_REMOTE_APP_NAME"
 )
 
+// testCA is a CA certificate, of which only that it reads matters here.
+const testCA = `-----BEGIN CERTIFICATE-----
+MIIBkzCCATmgAwIBAgIUeySaE27dnr6A2HijrMB13gTLUKIwCgYIKoZIzj0EAwIw
+HjEcMBoGA1UEAwwTc21hbGx3ZWJ3YWYgdGVzdCBDQTAgFw0yNjEwMDYxNDI2MTNa
+GA8yMTI2MDkxMjE0MjYxM1owHjEcMBoGA1UEAwwTc21hbGx3ZWJ3YWYgdGVzdCBD
+QTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABKDEhcWKKhet2KgSdME+iEPxyEyn
+2sd9IdElbt8DM2SfCdB2JsXo0C07UNZaywMPMfn/n8LNI/PKwu+N2uX7gfSjUzBR
+MB0GA1UdDgQWBBRZo3BPLv0KbV4drw6JI1JIUKmoRzAfBgNVHSMEGDAWgBRZo3BP
+Lv0KbV4drw6JI1JIUKmoRzAPBgNVHRMBAf8EBTADAQH/MAoGCCqGSM49BAMCA0gA
+MEUCICC5k+76UpWoSwVbZA+atu5WcALEOJGqwOUWua3zemhcAiEA8Hfdxgwp0z2v
+rlG9y/jrJb6ORy3kTLWo2EA0BA67vuI=
+-----END CERTIFICATE-----
+`
+
 // token is a token of 32 characters, the shortest allowed.
 const token = "0123456789abcdef0123456789abcdef"
 
@@ -198,6 +220,131 @@ func TestValuesAsSet(t *testing.T) {
 	wantCountries(t, allowedCountries, cfg.ExclusivelyAllowedCountries, "DE")
 }
 
+func TestRemoteLogSettingsDefaults(t *testing.T) {
+	t.Parallel()
+
+	cfg := fromEnvironment(t, environment{})
+	hostname, _ := os.Hostname()
+
+	if cfg.LogRemoteURL != nil || cfg.LogRemoteTLSCAs != nil ||
+		cfg.LogRemoteBuffer != 10000 || cfg.LogRemoteFacility != 16 ||
+		cfg.LogRemoteAppName != hostname {
+		t.Errorf("remote log settings %v, %v, %d, %d and %q, want no URL, no "+
+			"certificates, 10000, 16 and %q", cfg.LogRemoteURL, cfg.LogRemoteTLSCAs,
+			cfg.LogRemoteBuffer, cfg.LogRemoteFacility, cfg.LogRemoteAppName, hostname)
+	}
+}
+
+func TestRemoteLogSettingsAsSet(t *testing.T) {
+	t.Parallel()
+
+	caFile := filepath.Join(t.TempDir(), "ca.pem")
+
+	err := os.WriteFile(caFile, []byte(testCA), 0o600)
+	if err != nil {
+		t.Fatalf("write %s: %v", caFile, err)
+	}
+
+	cfg := fromEnvironment(t, environment{
+		logRemoteURL:       "syslog+tls://logs.example:6514",
+		logRemoteTLSCAFile: caFile,
+		logRemoteBuffer:    "500",
+		logRemoteFacility:  "daemon",
+		logRemoteAppName:   "fsn1app1/gitea",
+	})
+
+	roots := x509.NewCertPool()
+	roots.AppendCertsFromPEM([]byte(testCA))
+
+	if cfg.LogRemoteURL.String() != "syslog+tls://logs.example:6514" ||
+		!roots.Equal(cfg.LogRemoteTLSCAs) || cfg.LogRemoteBuffer != 500 ||
+		cfg.LogRemoteFacility != 3 || cfg.LogRemoteAppName != "fsn1app1/gitea" {
+		t.Errorf("remote log settings %v, %v, %d, %d and %q", cfg.LogRemoteURL,
+			cfg.LogRemoteTLSCAs, cfg.LogRemoteBuffer, cfg.LogRemoteFacility,
+			cfg.LogRemoteAppName)
+	}
+}
+
+func TestRemoteLogURLForms(t *testing.T) {
+	t.Parallel()
+
+	for _, value := range []string{
+		"syslog+udp://192.0.2.1:514",
+		"syslog+tcp://[2001:db8::1]:514",
+		"syslog+tls://logs.example:6514/",
+	} {
+		cfg := fromEnvironment(t, environment{logRemoteURL: value})
+		if cfg.LogRemoteURL.String() != value {
+			t.Errorf("%s read as %v", value, cfg.LogRemoteURL)
+		}
+	}
+
+	cfg := fromEnvironment(t, environment{logRemoteURL: ""})
+	if cfg.LogRemoteURL != nil {
+		t.Errorf("set but empty, %s read as %v", logRemoteURL, cfg.LogRemoteURL)
+	}
+}
+
+func TestRemoteLogFacilitiesByNumber(t *testing.T) {
+	t.Parallel()
+
+	for name, number := range map[string]int{
+		"kern": 0, "user": 1, "auth": 4, "authpriv": 10, "ftp": 11,
+		"local0": 16, "local5": 21, "local7": 23,
+	} {
+		cfg := fromEnvironment(t, environment{logRemoteFacility: name})
+		if cfg.LogRemoteFacility != number {
+			t.Errorf("%s read as %d, want %d", name, cfg.LogRemoteFacility, number)
+		}
+	}
+}
+
+func TestInvalidRemoteLogSettingStopsTheStart(t *testing.T) {
+	t.Parallel()
+
+	for _, tc := range []struct{ name, value string }{
+		{logRemoteURL, "logs.example:514"},
+		{logRemoteURL, "syslog://logs.example:514"},
+		{logRemoteURL, "http://logs.example:514"},
+		{logRemoteURL, "syslog+udp://logs.example"},
+		{logRemoteURL, "syslog+tcp://:514"},
+		{logRemoteURL, "syslog+tcp://logs.example:0"},
+		{logRemoteURL, "syslog+tls://logs.example:65536"},
+		{logRemoteURL, "syslog+tls://user@logs.example:6514"},
+		{logRemoteURL, "syslog+tcp://logs.example:514/app"},
+		{logRemoteURL, "syslog+tcp://logs.example:514?tls=1"},
+		{logRemoteTLSCAFile, "/nonexistent/ca.pem"},
+		{logRemoteBuffer, off}, {logRemoteBuffer, "0"}, {logRemoteBuffer, "10K"},
+		{logRemoteFacility, "local8"}, {logRemoteFacility, "LOCAL0"},
+		{logRemoteFacility, "16"}, {logRemoteFacility, ""},
+		{logRemoteAppName, ""}, {logRemoteAppName, "my app"},
+		{logRemoteAppName, "gitéa"}, {logRemoteAppName, strings.Repeat("a", 49)},
+	} {
+		_, err := config.FromEnvironment(environment{tc.name: tc.value}.lookupEnv)
+		if err == nil || !strings.HasPrefix(err.Error(), tc.name+": ") {
+			t.Errorf("%s=%q: error %v, want one naming it", tc.name, tc.value, err)
+		}
+	}
+}
+
+func TestRemoteLogCAFileWithoutCertificateStopsTheStart(t *testing.T) {
+	t.Parallel()
+
+	caFile := filepath.Join(t.TempDir(), "ca.pem")
+
+	err := os.WriteFile(caFile, []byte("not a certificate\n"), 0o600)
+	if err != nil {
+		t.Fatalf("write %s: %v", caFile, err)
+	}
+
+	_, err = config.FromEnvironment(environment{logRemoteTLSCAFile: caFile}.lookupEnv)
+
+	want := logRemoteTLSCAFile + `: "` + caFile + `" holds no PEM certificate`
+	if err == nil || err.Error() != want {
+		t.Errorf("error %v, want %s", err, want)
+	}
+}
+
 func TestCodeOnBothCountryListsStopsTheStart(t *testing.T) {
 	t.Parallel()
 
@@ -428,6 +575,8 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
 		t.Fatalf("decode %s: %v", out.Bytes(), err)
 	}
 
+	hostname, _ := os.Hostname()
+
 	want := map[string]string{
 		listenAddr:              ":8080",
 		upstreamURL:             "http://127.0.0.1:8081",
@@ -460,6 +609,11 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
 		stateCounterInterval:    "15m",
 		metricsToken:            "",
 		metricsTopN:             "50",
+		logRemoteURL:            "",
+		logRemoteTLSCAFile:      "",
+		logRemoteBuffer:         "10000",
+		logRemoteFacility:       "local0",
+		logRemoteAppName:        hostname,
 	}
 	if !maps.Equal(line.Settings, want) {
 		t.Errorf("logged settings\n%v\nwant\n%v", line.Settings, want)