Every time in UTC, and every day as YYYY-MM-DD (closes #313)

AbuseIPDB's daily count keeps its day as a civil.Date, taken from the UTC
clock and written to reputation.json as "YYYY-MM-DD"; it was a time.Time
at midnight. Every time read from a state file is converted to UTC, so a
time an admin writes with an offset is written back in UTC.
TestMainRunsWAF runs sww waf with TZ 14 hours ahead of UTC and checks
that the process line, the request line and clients.json are in UTC.
SPEC.md states the rule near its start; AGENTS.md makes it an iron rule.

Model: opus-5-5

AGENTS.md

@@ -37,3 +37,6 @@ These are the rules for agents working on this repository.
         - `192.168.0.0/16`.
     - `SPEC.md` states that default.
     - Forwarded headers count only when the peer is a trusted proxy.
+- Every time is in UTC, whatever `TZ` or the host's time zone says.
+    - Every day is a date written `YYYY-MM-DD`, never a time.
+    - `SPEC.md` states this near its start.

SPEC.md

@@ -12,6 +12,10 @@ it is built, except what the last section lists. `README.md` says how to run it.
 under `docs/` give the settings, the formats and the lists. This document links
 to them.
 
+The whole application stores, logs and handles every time in UTC, whatever `TZ`
+or the host's time zone says. It keeps every day as a date written `YYYY-MM-DD`,
+never as a time.
+
 ## Purpose
 
 - It protects self-hosted apps on the open internet from scrapers, scanners and
@@ -1775,6 +1779,7 @@ in. `docs/state-files.md` lists the files, what each holds, and their sizes.
     - a top-level `version` number;
     - entries sorted by client address;
     - times in RFC 3339 UTC;
+    - days as `YYYY-MM-DD`;
     - durations and sizes as plain numbers, with the unit in the field name.
 - `bans.json`, `reputation.json` and `alerts.json` are indented, so that a
   person can open `bans.json` in an editor, find an address, and remove or add

go.mod

@@ -3,6 +3,7 @@ module sneak.berlin/go/smallwebwaf
 go 1.26.0
 
 require (
+	cloud.google.com/go v0.123.0
 	github.com/corazawaf/coraza-coreruleset/v4 v4.25.0
 	github.com/corazawaf/coraza/v3 v3.8.2-0.20261022102553-d028be399c68
 	github.com/fsnotify/fsnotify v1.10.1

go.sum

@@ -1,3 +1,5 @@
+cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE=
+cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=

internal/lookup/file.go

@@ -26,7 +26,7 @@ type FileParams struct {
 	// Path is the lookup database, the IPinfo Lite file in its .mmdb form
 	// (SWW_LOOKUP_DB_PATH).
 	Path string
-	// Now tells the time, normally time.Now.
+	// Now tells the time, normally time.Now in UTC.
 	Now func() time.Time
 	// ProcessLog receives each reading of the file, and why a replacement
 	// of it cannot be read.

internal/lookup/lookup.go

@@ -74,7 +74,7 @@ type Params struct {
 	// Answered, unless nil, is given each answer GeoJS gives, once it is
 	// kept.
 	Answered func(Answer)
-	// Now tells the time, normally time.Now.
+	// Now tells the time, normally time.Now in UTC.
 	Now func() time.Time
 	// ProcessLog receives GeoJS's failures.
 	ProcessLog *slog.Logger

internal/reputation/abuseipdb.go

@@ -14,6 +14,7 @@ import (
 	"sync"
 	"time"
 
+	"cloud.google.com/go/civil"
 	"github.com/hashicorp/golang-lru/v2/simplelru"
 	"sneak.berlin/go/smallwebwaf/internal/alerts"
 )
@@ -27,8 +28,6 @@ const (
 	AbuseIPDBSource = "abuseipdb"
 	// maxAnswerBytes is the most of an answer of AbuseIPDB that is read.
 	maxAnswerBytes = 64 << 10
-	// day is the length of the day the checks are counted in, in UTC.
-	day = 24 * time.Hour
 )
 
 var (
@@ -50,9 +49,9 @@ type Score struct {
 // AbuseIPDB: the day, in UTC, of the checks Spent counts, zero before the
 // first, and the scores still in use.
 type Checks struct {
-	Day    time.Time `json:"day,omitzero"`
-	Spent  int       `json:"spent"`
-	Scores []Score   `json:"scores"`
+	Day    civil.Date `json:"day,omitzero"`
+	Spent  int        `json:"spent"`
+	Scores []Score    `json:"scores"`
 }
 
 // AbuseIPDBParams are what NewAbuseIPDB needs.
@@ -93,7 +92,7 @@ type AbuseIPDB struct {
 	// checking are the clients whose check is under way.
 	checking map[netip.Prefix]bool
 	// day is the day, in UTC, of the checks spent counts.
-	day   time.Time
+	day   civil.Date
 	spent int
 	// checks and failures count the checks made and those that failed,
 	// and retryAt is when a client may be checked again after the last
@@ -143,7 +142,7 @@ func (a *AbuseIPDB) Hit(
 		return kept.Score, kept.Score >= a.params.MinScore
 	}
 
-	if today := now.Truncate(day); !a.day.Equal(today) {
+	if today := civil.DateOf(now.UTC()); a.day != today {
 		a.day, a.spent = today, 0
 	}
 
@@ -190,7 +189,7 @@ func (a *AbuseIPDB) BudgetLeft() int {
 	a.mu.Lock()
 	defer a.mu.Unlock()
 
-	if !a.day.Equal(a.params.Now().Truncate(day)) {
+	if a.day != civil.DateOf(a.params.Now().UTC()) {
 		return a.params.DailyBudget
 	}
 

internal/reputation/abuseipdb_test.go

@@ -202,6 +202,43 @@ func TestDailyBudgetKeptAcrossARestartAndWholeAgainAsTheDayEnds(t *testing.T) {
 	})
 }
 
+func TestChecksAreCountedInUTCDaysWhateverTheClocksZone(t *testing.T) {
+	t.Parallel()
+
+	synctest.Test(t, func(t *testing.T) {
+		// A clock 14 hours ahead of UTC, whose day starts at 10:00 UTC.
+		ahead := time.FixedZone("UTC+14", 14*60*60)
+		p := abuseIPDBParams()
+		p.DailyBudget = 1
+		p.Now = func() time.Time { return time.Now().In(ahead) }
+		abuseIPDB := &abuseIPDBStandIn{scores: map[string]int64{suspect: 100}}
+		checker := newAbuseIPDB(abuseIPDB, p)
+
+		// At 09:00 UTC the one check is spent.
+		time.Sleep(9 * time.Hour)
+		hitFrom(t, checker, suspect, true)
+		synctest.Wait()
+		wantBudgetLeft(t, checker, 0)
+
+		// At 11:00 UTC the clock's day has changed, but the UTC day has not.
+		time.Sleep(2 * time.Hour)
+		wantBudgetLeft(t, checker, 0)
+
+		encoded, err := json.Marshal(checker.Snapshot())
+		if err != nil {
+			t.Fatalf("encode: %v", err)
+		}
+
+		if !strings.Contains(string(encoded), `"day":"2000-01-01"`) {
+			t.Errorf("reputation.json would hold %s, want the day 2000-01-01", encoded)
+		}
+
+		// At 00:00 UTC the budget is whole again.
+		time.Sleep(13 * time.Hour)
+		wantBudgetLeft(t, checker, 1)
+	})
+}
+
 func TestFailedCheckGivesNoScoreAndNoClientIsCheckedForAMinute(t *testing.T) {
 	t.Parallel()
 

internal/state/state.go

@@ -171,9 +171,11 @@ type alertsFile struct {
 // check refuses the first entry without a field it needs, which would
 // otherwise be read as something the entry does not say. data is the
 // file, for a field that may be null or "" but not left out, which the
-// struct cannot tell apart.
+// struct cannot tell apart. Its toUTC then converts every time in it to
+// UTC, since an admin may write a time with an offset.
 type stateFile interface {
 	check(data []byte) error
+	toUTC()
 }
 
 // Load checks that files can be written in Dir, and reads the state files
@@ -918,6 +920,80 @@ func missing(i int, field string) error {
 	return fmt.Errorf("entry %d %w %q", i+1, errMissing, field)
 }
 
+// toUTC converts every time of each ban to UTC.
+func (f *bansFile) toUTC() {
+	for i := range f.Bans {
+		ban := &f.Bans[i]
+		convertToUTC(&ban.Start, ban.Expires, ban.Lifted, &ban.Notes.Request.Time)
+	}
+}
+
+// toUTC converts every time of each client to UTC.
+func (f *clientsFile) toUTC() {
+	for i := range f.Clients {
+		client := &f.Clients[i]
+		convertToUTC(&client.Minute.Start, &client.Hour.Start, &client.Day.Start,
+			&client.MinuteBytes.Start, &client.HourBytes.Start,
+			&client.DayBytes.Start, &client.MinuteRefusals.Start,
+			&client.History.FirstSeen, &client.History.LastSeen,
+			&client.History.LookedUp)
+	}
+}
+
+// toUTC converts both times of each answer to UTC.
+func (f *lookupsFile) toUTC() {
+	for i := range f.Lookups {
+		convertToUTC(&f.Lookups[i].Answered, &f.Lookups[i].Used)
+	}
+}
+
+// toUTC converts every time of each list, verdict and AbuseIPDB score to
+// UTC.
+func (f *reputationFile) toUTC() {
+	for i := range f.Lists {
+		convertToUTC(&f.Lists[i].Tried, &f.Lists[i].Fetched)
+	}
+
+	for i := range f.Verdicts {
+		convertToUTC(&f.Verdicts[i].Fetched)
+	}
+
+	for i := range f.AbuseIPDB.Scores {
+		convertToUTC(&f.AbuseIPDB.Scores[i].Fetched)
+	}
+}
+
+// toUTC converts every time of each cooldown, of the hour, of each alert
+// waiting and of each anomaly counter to UTC.
+func (f *alertsFile) toUTC() {
+	for i := range f.Cooldowns {
+		convertToUTC(&f.Cooldowns[i].Sent)
+	}
+
+	convertToUTC(&f.Hour.Start)
+
+	for _, waiting := range f.Waiting {
+		for i := range waiting {
+			convertToUTC(&waiting[i].Time)
+		}
+	}
+
+	for i := range f.AnomalyCounters {
+		counter := &f.AnomalyCounters[i]
+		convertToUTC(&counter.Minute.Start, &counter.Hour.Start,
+			&counter.MinuteBytes.Start, &counter.HourBytes.Start)
+	}
+}
+
+// convertToUTC converts each of times that is not nil to UTC.
+func convertToUTC(times ...*time.Time) {
+	for _, at := range times {
+		if at != nil {
+			*at = at.UTC()
+		}
+	}
+}
+
 // encodeOnePerLine encodes a state file whose entries, under key, are one
 // to a line, so that grep shows everything about one client.
 func encodeOnePerLine[E any](key string, entries []E) ([]byte, error) {
@@ -955,7 +1031,8 @@ func checkWritable(dir string) error {
 }
 
 // parse reads data, what the state file at path holds, into file, a
-// pointer to that file's struct, and checks its entries.
+// pointer to that file's struct, checks its entries, and converts every
+// time in it to UTC.
 func parse(path string, data []byte, file stateFile) error {
 	// The version is read first, so that a file of another version is
 	// refused for that, and not for an entry this version cannot read.
@@ -985,6 +1062,8 @@ func parse(path string, data []byte, file stateFile) error {
 		return fmt.Errorf("%s%s: %w", path, position(data, err), err)
 	}
 
+	file.toUTC()
+
 	return nil
 }
 

internal/state/state_test.go

@@ -14,6 +14,7 @@ import (
 	"os"
 	"path/filepath"
 	"reflect"
+	"regexp"
 	"slices"
 	"strconv"
 	"strings"
@@ -21,6 +22,7 @@ import (
 	"testing/synctest"
 	"time"
 
+	"cloud.google.com/go/civil"
 	"sneak.berlin/go/smallwebwaf/internal/alerts"
 	"sneak.berlin/go/smallwebwaf/internal/anomaly"
 	"sneak.berlin/go/smallwebwaf/internal/bans"
@@ -258,7 +260,7 @@ const filledReputationJSON = `{
         }
     ],
     "abuseipdb": {
-        "day": "2026-10-06T00:00:00Z",
+        "day": "2026-10-06",
         "spent": 3,
         "scores": [
             {
@@ -1306,7 +1308,7 @@ func TestEditOfEachFileTakenIn(t *testing.T) {
 		`"tried": "2026-10-06T00:00:00Z", "fetched": "2026-10-06T00:00:00Z", `+
 		`"lines": ["198.51.100.7"]}], "verdicts": [{"zone": "`+dnsblZone+`", `+
 		`"client": "198.51.100.7", "listed": true, "fetched": "2026-10-06T00:00:00Z"}], `+
-		`"abuseipdb": {"day": "2026-10-06T00:00:00Z", "spent": 9, "scores": [`+
+		`"abuseipdb": {"day": "2026-10-06", "spent": 9, "scores": [`+
 		`{"client": "198.51.100.7/32", "score": 80, "fetched": "2026-10-06T00:00:00Z"}]}}`)
 	wantTakenIn(t, lines, dir, reputationJSON)
 
@@ -1321,7 +1323,7 @@ func TestEditOfEachFileTakenIn(t *testing.T) {
 	}})
 
 	checks := reputation.Checks{
-		Day: midnight(), Spent: 9,
+		Day: civil.DateOf(midnight()), Spent: 9,
 		Scores: []reputation.Score{{Client: client, Score: 80, Fetched: midnight()}},
 	}
 	if got := params.AbuseIPDB.Snapshot(); !reflect.DeepEqual(got, checks) {
@@ -1350,6 +1352,62 @@ func TestEditOfEachFileTakenIn(t *testing.T) {
 	}
 }
 
+func TestTimesEditedWithAnOffsetWrittenBackInUTC(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	params := newParams(dir)
+	fill(params)
+	// A lifted ban, so that bans.json holds every time a ban has.
+	params.Ledger.Lift(netip.MustParseAddr("192.0.2.1"), midnight())
+
+	files := load(t, params)
+
+	err := files.WriteAll()
+	if err != nil {
+		t.Fatalf("write: %v", err)
+	}
+
+	// An admin saves each file with every time in it two hours ahead of
+	// UTC, midnight as "2026-10-06T02:00:00+02:00".
+	stamp := regexp.MustCompile(`"\d{4}-\d\d-\d\dT[^"]*Z"`)
+	ahead := time.FixedZone("UTC+2", 2*60*60)
+	written := map[string]string{}
+
+	for _, name := range []string{
+		bansJSON, clientsJSON, lookupsJSON, reputationJSON, alertsJSON,
+	} {
+		written[name] = readFile(t, filepath.Join(dir, name))
+		edited := stamp.ReplaceAllStringFunc(written[name], func(quoted string) string {
+			at, err := time.Parse(time.RFC3339Nano, strings.Trim(quoted, `"`))
+			if err != nil {
+				t.Fatalf("%s holds %s: %v", name, quoted, err)
+			}
+
+			return `"` + at.In(ahead).Format(time.RFC3339Nano) + `"`
+		})
+
+		if !strings.Contains(edited, "+02:00") {
+			t.Fatalf("%s holds no time:\n%s", name, edited)
+		}
+
+		edit(t, dir, name, edited)
+	}
+
+	// Each write takes its edit in first, and writes the same times back in
+	// UTC.
+	err = files.WriteAll()
+	if err != nil {
+		t.Fatalf("write: %v", err)
+	}
+
+	for name, want := range written {
+		if got := readFile(t, filepath.Join(dir, name)); got != want {
+			t.Errorf("%s written back as\n%s\nwant\n%s", name, got, want)
+		}
+	}
+}
+
 func TestOwnWritesAreNotTakenIn(t *testing.T) {
 	t.Parallel()
 
@@ -1837,10 +1895,12 @@ func fill(params state.Params) {
 		},
 		{Zone: dnsblZone, Client: client.Addr(), Listed: true, Fetched: now.Add(-time.Hour)},
 	})
-	params.AbuseIPDB.Load(reputation.Checks{Day: now, Spent: 3, Scores: []reputation.Score{
-		{Client: netip.MustParsePrefix("2001:db8::/64"), Fetched: now.Add(-2 * time.Hour)},
-		{Client: client, Score: 100, Fetched: now.Add(-time.Hour)},
-	}})
+	params.AbuseIPDB.Load(reputation.Checks{
+		Day: civil.DateOf(now), Spent: 3, Scores: []reputation.Score{
+			{Client: netip.MustParsePrefix("2001:db8::/64"), Fetched: now.Add(-2 * time.Hour)},
+			{Client: client, Score: 100, Fetched: now.Add(-time.Hour)},
+		},
+	})
 
 	// An alert waiting, a repeat of it the cooldown holds back, another
 	// alert waiting, and one past the two an hour, for the hour's summary.

internal/sww/waf_test.go

@@ -9,10 +9,14 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"syscall"
 	"testing"
 	"time"
+	// The time zone database, so that the process TestMainRunsWAF starts
+	// finds the zone TZ names on any host.
+	_ "time/tzdata"
 
 	"sneak.berlin/go/smallwebwaf/internal/sww"
 	"sneak.berlin/go/smallwebwaf/internal/waf"
@@ -37,6 +41,14 @@ func TestMain(m *testing.M) {
 func TestMainRunsWAF(t *testing.T) {
 	t.Parallel()
 
+	// Kiritimati is 14 hours ahead of UTC; every time sww gives stays in UTC.
+	const zone = "Pacific/Kiritimati"
+
+	_, err := time.LoadLocation(zone)
+	if err != nil {
+		t.Fatalf("load %s: %v", zone, err)
+	}
+
 	binary, err := os.Executable()
 	if err != nil {
 		t.Fatalf("find the test binary: %v", err)
@@ -45,14 +57,16 @@ func TestMainRunsWAF(t *testing.T) {
 	ctx, cancel := context.WithTimeout(t.Context(), 3*waitLimit)
 	defer cancel()
 
+	dir := t.TempDir()
 	out := &output{}
 	cmd := exec.CommandContext(ctx, binary) //nolint:gosec // this test binary
 	cmd.Stdout = out
 
 	cmd.Env = append(os.Environ(),
+		"TZ="+zone,
 		commandForMain+"="+wafCommand,
 		listenAddr+"="+localhost+":0",
-		stateDir+"="+t.TempDir(),
+		stateDir+"="+dir,
 		rulesDir+"="+t.TempDir(),
 		lookupSource+"=off")
 
@@ -66,9 +80,11 @@ func TestMainRunsWAF(t *testing.T) {
 		_ = cmd.Wait()
 	})
 
-	addr, _ := out.line(t, "msg", "starting")["address"].(string)
+	starting := out.line(t, "msg", "starting")
+	addr, _ := starting["address"].(string)
 	ask(t, addr, "/", http.StatusOK, waf.DispositionAllow)
 
+	// The files are written as sww stops.
 	err = cmd.Process.Signal(syscall.SIGTERM)
 	if err != nil {
 		t.Fatalf("signal: %v", err)
@@ -76,7 +92,39 @@ func TestMainRunsWAF(t *testing.T) {
 
 	err = cmd.Wait()
 	if err != nil {
-		t.Errorf("exit: %v; output:\n%s", err, out.text())
+		t.Fatalf("exit: %v; output:\n%s", err, out.text())
+	}
+
+	wantUTC(t, "the process line", starting["time"])
+	wantUTC(t, "the request line", out.line(t, "type", "request")["time"])
+
+	path := filepath.Join(dir, "clients.json")
+
+	clients, err := os.ReadFile(path) //nolint:gosec // a file in the test's directory
+	if err != nil {
+		t.Fatalf("read clients.json: %v", err)
+	}
+
+	stamp := regexp.MustCompile(`"\d{4}-\d\d-\d\dT[^"]*"`)
+
+	times := stamp.FindAllString(string(clients), -1)
+	if len(times) == 0 {
+		t.Errorf("clients.json holds no time:\n%s", clients)
+	}
+
+	for _, stored := range times {
+		wantUTC(t, "clients.json", strings.Trim(stored, `"`))
+	}
+}
+
+// wantUTC checks that value, a time read from the log line or file that
+// where names, is in UTC.
+func wantUTC(t *testing.T, where string, value any) {
+	t.Helper()
+
+	text, _ := value.(string)
+	if !strings.HasSuffix(text, "Z") {
+		t.Errorf("%s gives the time %v, want one in UTC", where, value)
 	}
 }