Blocklists and an AS percentage file fetched by URL (closes #29)

SWWAF_BLOCKLIST_URLS names lists of addresses and netblocks, fetched every
SWWAF_BLOCKLIST_REFRESH (24h, never under 1h); an IPv4-mapped line stands
for its IPv4 address or netblock. reputation.json keeps each list's last
try, failed or not, even one cut off by a stop, which a restart waits on
as a running instance does, and its last good copy, whole, used while a
fetch fails. SWWAF_BLOCKLIST_ACTION denies, limits or only logs a listed
client; the log line names the lists, each raises reputation_hit, and a
failed fetch raises source_failure. SWWAF_ASN_LIMIT_PERCENT_URL is fetched
the same way and counts as SWWAF_ASN_LIMIT_PERCENT does, the lower winning.

Judgement call: a failed fetch is retried after the refresh, not sooner.
Not done: ban notes do not name the lists yet.

Model: opus-5-5

README.md

@@ -1625,17 +1625,19 @@ defaults judge it by what it does to your service.
 names, `SWWAF_BLOCKLIST_REFRESH` after it last fetched it or tried to, 24 hours
 by default and never less than one, one list after another. Since
 `reputation.json` keeps when each list was last tried, the fetch failed or not,
-at start `smallwebwaf` fetches at once only a list it has never tried, and one
-it last tried that long ago; any other waits its turn, so that restarts do not
-fetch a list more often. A fetch fails when the server answers other than `200`,
-when it does not finish within a minute, when the list is longer than 16 MiB, or
-when a line of it is not an address or a netblock, or for
+even one cut off as `smallwebwaf` stopped, whose request the server may have
+had, at start `smallwebwaf` fetches at once only a list it has never tried, and
+one it last tried that long ago; any other waits its turn, so that restarts do
+not fetch a list more often. A fetch fails when the server answers other than
+`200`, when it does not finish within a minute, when the list is longer than 16
+MiB, or when a line of it is not an address or a netblock, or for
 `SWWAF_ASN_LIMIT_PERCENT_URL`, not an AS number, `:` and a percentage. The copy
 fetched before then stays in use, and the failure is counted, logged and raised
-as a `source_failure` alert. The last good copy of each list is kept whole,
-comment lines included, in `reputation.json` (see "State files" above), so that
-a restart keeps it in use too. Each list is named by its URL, in the request
-log, the alerts and the metrics, so keep a secret out of it.
+as a `source_failure` alert; a fetch cut off as `smallwebwaf` stops is not a
+failure. The last good copy of each list is kept whole, comment lines included,
+in `reputation.json` (see "State files" above), so that a restart keeps it in
+use too. Each list is named by its URL, in the request log, the alerts and the
+metrics, so keep a secret out of it.
 
 A client in `SWWAF_ALLOW_NETS` is not checked. Any other is checked by its own
 address after the country lists, and `SWWAF_BLOCKLIST_ACTION` says what is done

internal/reputation/reputation.go

@@ -252,13 +252,14 @@ func (l *Lists) Load(lists []List) error {
 }
 
 // fetchDue fetches each list that is due, one after another, and returns
-// when the next is due.
+// when the next is due. Once ctx has ended, it starts none, since a fetch
+// cut off is noted as a try.
 func (l *Lists) fetchDue(ctx context.Context) time.Time {
 	var next time.Time
 
 	for _, listURL := range l.URLs() {
 		due := l.due(listURL)
-		if !l.params.Now().Before(due) {
+		if ctx.Err() == nil && !l.params.Now().Before(due) {
 			l.fetch(ctx, listURL)
 			due = l.due(listURL)
 		}
@@ -287,10 +288,11 @@ func (l *Lists) due(listURL string) time.Time {
 	return last.Add(l.params.Refresh)
 }
 
-// fetch fetches the list at listURL. A good copy takes the place of the
-// one held. A failure leaves that in use, and is counted, logged and
-// raised as a source_failure alert; a fetch cut off as ctx ends, as
-// smallwebwaf stops, is none.
+// fetch fetches the list at listURL, and notes the try. A good copy takes
+// the place of the one held. A failure leaves that in use, and is counted,
+// logged and raised as a source_failure alert. A fetch cut off as ctx
+// ends, as smallwebwaf stops, is no failure, but is still noted as a try,
+// so that a restart waits for it: the server may have had its request.
 func (l *Lists) fetch(ctx context.Context, listURL string) {
 	lines, err := l.get(ctx, listURL)
 
@@ -299,10 +301,7 @@ func (l *Lists) fetch(ctx context.Context, listURL string) {
 		found, err = l.parse(listURL, lines)
 	}
 
-	if ctx.Err() != nil {
-		return
-	}
-
+	cutOff := err != nil && ctx.Err() != nil
 	now := l.params.Now()
 
 	l.mu.Lock()
@@ -313,12 +312,16 @@ func (l *Lists) fetch(ctx context.Context, listURL string) {
 	if err == nil {
 		held.kept.Fetched, held.kept.Lines = now, lines
 		held.entries = found
-	} else {
+	} else if !cutOff {
 		held.failures++
 	}
 
 	l.mu.Unlock()
 
+	if cutOff {
+		return
+	}
+
 	if err != nil {
 		const failed = "fetching a list failed"
 

internal/reputation/reputation_test.go

@@ -326,13 +326,14 @@ func TestRestartWaitsRefreshAfterTheLastTryEvenOneThatFailed(t *testing.T) {
 	})
 }
 
-func TestFetchCutOffAsItStopsIsNoFailure(t *testing.T) {
+func TestFetchCutOffAsItStopsIsNoFailureButARestartWaitsForIt(t *testing.T) {
 	t.Parallel()
 
 	synctest.Test(t, func(t *testing.T) {
+		// Stopped 30 seconds into the fetch of drop.txt, before tor.txt's.
 		servers := &standIn{lists: map[string]string{}, hanging: true}
 		queue := newQueue()
-		p := params(dropURL)
+		p := params(dropURL, torURL)
 		p.Alerts = queue
 
 		lists := reputation.New(p)
@@ -346,14 +347,43 @@ func TestFetchCutOffAsItStopsIsNoFailure(t *testing.T) {
 			close(stopped)
 		}()
 
-		synctest.Wait()
+		time.Sleep(30 * time.Second)
 		stop()
 		<-stopped
 
+		wantFetches(t, servers, 1)
+
 		if lists.Failures(dropURL) != 0 || len(waiting(queue)) != 0 {
 			t.Errorf("%d failures and alerts %+v, want none", lists.Failures(dropURL),
 				waiting(queue))
 		}
+
+		// Restarted an hour later with what reputation.json keeps, it fetches
+		// tor.txt, never tried, at once, and drop.txt a refresh after its
+		// cut-off try.
+		time.Sleep(time.Hour)
+
+		restarted := &standIn{lists: map[string]string{
+			dropURL: "203.0.113.7\n", torURL: "198.51.100.7\n",
+		}}
+		again := reputation.New(params(dropURL, torURL))
+		again.SetTransport(restarted)
+
+		err := again.Load(lists.Snapshot())
+		if err != nil {
+			t.Fatalf("load: %v", err)
+		}
+
+		run(t, again)
+		wantFetches(t, restarted, 1)
+		wantListedBy(t, again, "198.51.100.7", torURL)
+
+		time.Sleep(refresh - time.Hour - time.Nanosecond)
+		wantFetches(t, restarted, 1)
+
+		time.Sleep(time.Nanosecond)
+		wantFetches(t, restarted, 2)
+		wantListedBy(t, again, "203.0.113.7", dropURL)
 	})
 }