From 879d3a95838db01896306a786660d5e9098bd05f Mon Sep 17 00:00:00 2001 From: sneak Date: Wed, 11 Nov 2026 00:15:46 +0000 Subject: [PATCH] Every time in UTC, and every day as YYYY-MM-DD (closes #313) AbuseIPDB's daily count keeps its day as a civil.Date, taken from the UTC clock and written to reputation.json as "YYYY-MM-DD"; it was a time.Time at midnight. A test runs sww discriminator with TZ 14 hours ahead of UTC and checks the process line, the request line and clients.json are in UTC. SPEC.md states the rule near its start; AGENTS.md makes it an iron rule. Model: opus-5-5 --- AGENTS.md | 3 + SPEC.md | 5 ++ go.mod | 1 + go.sum | 2 + internal/lookup/file.go | 2 +- internal/lookup/lookup.go | 2 +- internal/reputation/abuseipdb.go | 15 ++--- internal/reputation/abuseipdb_test.go | 37 +++++++++++ internal/state/state_test.go | 17 +++-- internal/sww/discriminator_test.go | 93 +++++++++++++++++++++++++++ 10 files changed, 160 insertions(+), 17 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 3bf4d1e..2217ba1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -37,3 +37,6 @@ These are the rules for agents working on this repository. - `192.168.0.0/16`. - `SPEC.md` states that default. - Forwarded headers count only when the peer is a trusted proxy. +- Every time is in UTC, whatever `TZ` or the host's time zone says. + - Every day is a date written `YYYY-MM-DD`, never a time. + - `SPEC.md` states this near its start. diff --git a/SPEC.md b/SPEC.md index 5e6e6e1..b38a217 100644 --- a/SPEC.md +++ b/SPEC.md @@ -12,6 +12,10 @@ it is built, except what the last section lists. `README.md` says how to run it. under `docs/` give the settings, the formats and the lists. This document links to them. +The whole application stores, logs and handles every time in UTC, whatever `TZ` +or the host's time zone says. It keeps every day as a date written `YYYY-MM-DD`, +never as a time. + ## Purpose - It protects self-hosted apps on the open internet from scrapers, scanners and @@ -1781,6 +1785,7 @@ sizes. - a top-level `version` number; - entries sorted by client address; - times in RFC 3339 UTC; + - days as `YYYY-MM-DD`; - durations and sizes as plain numbers, with the unit in the field name. - `bans.json`, `reputation.json` and `alerts.json` are indented, so that a person can open `bans.json` in an editor, find an address, and remove or add diff --git a/go.mod b/go.mod index 6f7510b..20f2146 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module sneak.berlin/go/smallwebwaf go 1.26.0 require ( + cloud.google.com/go v0.123.0 github.com/corazawaf/coraza-coreruleset/v4 v4.25.0 github.com/corazawaf/coraza/v3 v3.8.2-0.20261022102553-d028be399c68 github.com/fsnotify/fsnotify v1.10.1 diff --git a/go.sum b/go.sum index 9abff90..e76b666 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= +cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= diff --git a/internal/lookup/file.go b/internal/lookup/file.go index 5cf7c72..f5dd4e3 100644 --- a/internal/lookup/file.go +++ b/internal/lookup/file.go @@ -26,7 +26,7 @@ type FileParams struct { // Path is the lookup database, the IPinfo Lite file in its .mmdb form // (SWW_LOOKUP_DB_PATH). Path string - // Now tells the time, normally time.Now. + // Now tells the time, normally time.Now in UTC. Now func() time.Time // ProcessLog receives each reading of the file, and why a replacement // of it cannot be read. diff --git a/internal/lookup/lookup.go b/internal/lookup/lookup.go index fdcfd1f..621525a 100644 --- a/internal/lookup/lookup.go +++ b/internal/lookup/lookup.go @@ -74,7 +74,7 @@ type Params struct { // Answered, unless nil, is given each answer GeoJS gives, once it is // kept. Answered func(Answer) - // Now tells the time, normally time.Now. + // Now tells the time, normally time.Now in UTC. Now func() time.Time // ProcessLog receives GeoJS's failures. ProcessLog *slog.Logger diff --git a/internal/reputation/abuseipdb.go b/internal/reputation/abuseipdb.go index b7de34b..6ee8c59 100644 --- a/internal/reputation/abuseipdb.go +++ b/internal/reputation/abuseipdb.go @@ -14,6 +14,7 @@ import ( "sync" "time" + "cloud.google.com/go/civil" "github.com/hashicorp/golang-lru/v2/simplelru" "sneak.berlin/go/smallwebwaf/internal/alerts" ) @@ -27,8 +28,6 @@ const ( AbuseIPDBSource = "abuseipdb" // maxAnswerBytes is the most of an answer of AbuseIPDB that is read. maxAnswerBytes = 64 << 10 - // day is the length of the day the checks are counted in, in UTC. - day = 24 * time.Hour ) var ( @@ -50,9 +49,9 @@ type Score struct { // AbuseIPDB: the day, in UTC, of the checks Spent counts, zero before the // first, and the scores still in use. type Checks struct { - Day time.Time `json:"day,omitzero"` - Spent int `json:"spent"` - Scores []Score `json:"scores"` + Day civil.Date `json:"day,omitzero"` + Spent int `json:"spent"` + Scores []Score `json:"scores"` } // AbuseIPDBParams are what NewAbuseIPDB needs. @@ -93,7 +92,7 @@ type AbuseIPDB struct { // checking are the clients whose check is under way. checking map[netip.Prefix]bool // day is the day, in UTC, of the checks spent counts. - day time.Time + day civil.Date spent int // checks and failures count the checks made and those that failed, // and retryAt is when a client may be checked again after the last @@ -143,7 +142,7 @@ func (a *AbuseIPDB) Hit( return kept.Score, kept.Score >= a.params.MinScore } - if today := now.Truncate(day); !a.day.Equal(today) { + if today := civil.DateOf(now.UTC()); a.day != today { a.day, a.spent = today, 0 } @@ -190,7 +189,7 @@ func (a *AbuseIPDB) BudgetLeft() int { a.mu.Lock() defer a.mu.Unlock() - if !a.day.Equal(a.params.Now().Truncate(day)) { + if a.day != civil.DateOf(a.params.Now().UTC()) { return a.params.DailyBudget } diff --git a/internal/reputation/abuseipdb_test.go b/internal/reputation/abuseipdb_test.go index 70460d2..bddce1d 100644 --- a/internal/reputation/abuseipdb_test.go +++ b/internal/reputation/abuseipdb_test.go @@ -202,6 +202,43 @@ func TestDailyBudgetKeptAcrossARestartAndWholeAgainAsTheDayEnds(t *testing.T) { }) } +func TestChecksAreCountedInUTCDaysWhateverTheClocksZone(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + // A clock 14 hours ahead of UTC, whose day starts at 10:00 UTC. + ahead := time.FixedZone("UTC+14", 14*60*60) + p := abuseIPDBParams() + p.DailyBudget = 1 + p.Now = func() time.Time { return time.Now().In(ahead) } + abuseIPDB := &abuseIPDBStandIn{scores: map[string]int64{suspect: 100}} + checker := newAbuseIPDB(abuseIPDB, p) + + // At 09:00 UTC the one check is spent. + time.Sleep(9 * time.Hour) + hitFrom(t, checker, suspect, true) + synctest.Wait() + wantBudgetLeft(t, checker, 0) + + // At 11:00 UTC the clock's day has changed, but the UTC day has not. + time.Sleep(2 * time.Hour) + wantBudgetLeft(t, checker, 0) + + encoded, err := json.Marshal(checker.Snapshot()) + if err != nil { + t.Fatalf("encode: %v", err) + } + + if !strings.Contains(string(encoded), `"day":"2000-01-01"`) { + t.Errorf("reputation.json would hold %s, want the day 2000-01-01", encoded) + } + + // At 00:00 UTC the budget is whole again. + time.Sleep(13 * time.Hour) + wantBudgetLeft(t, checker, 1) + }) +} + func TestFailedCheckGivesNoScoreAndNoClientIsCheckedForAMinute(t *testing.T) { t.Parallel() diff --git a/internal/state/state_test.go b/internal/state/state_test.go index 2fb9020..d0c9941 100644 --- a/internal/state/state_test.go +++ b/internal/state/state_test.go @@ -21,6 +21,7 @@ import ( "testing/synctest" "time" + "cloud.google.com/go/civil" "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/anomaly" "sneak.berlin/go/smallwebwaf/internal/bans" @@ -258,7 +259,7 @@ const filledReputationJSON = `{ } ], "abuseipdb": { - "day": "2026-10-06T00:00:00Z", + "day": "2026-10-06", "spent": 3, "scores": [ { @@ -1306,7 +1307,7 @@ func TestEditOfEachFileTakenIn(t *testing.T) { `"tried": "2026-10-06T00:00:00Z", "fetched": "2026-10-06T00:00:00Z", `+ `"lines": ["198.51.100.7"]}], "verdicts": [{"zone": "`+dnsblZone+`", `+ `"client": "198.51.100.7", "listed": true, "fetched": "2026-10-06T00:00:00Z"}], `+ - `"abuseipdb": {"day": "2026-10-06T00:00:00Z", "spent": 9, "scores": [`+ + `"abuseipdb": {"day": "2026-10-06", "spent": 9, "scores": [`+ `{"client": "198.51.100.7/32", "score": 80, "fetched": "2026-10-06T00:00:00Z"}]}}`) wantTakenIn(t, lines, dir, reputationJSON) @@ -1321,7 +1322,7 @@ func TestEditOfEachFileTakenIn(t *testing.T) { }}) checks := reputation.Checks{ - Day: midnight(), Spent: 9, + Day: civil.DateOf(midnight()), Spent: 9, Scores: []reputation.Score{{Client: client, Score: 80, Fetched: midnight()}}, } if got := params.AbuseIPDB.Snapshot(); !reflect.DeepEqual(got, checks) { @@ -1837,10 +1838,12 @@ func fill(params state.Params) { }, {Zone: dnsblZone, Client: client.Addr(), Listed: true, Fetched: now.Add(-time.Hour)}, }) - params.AbuseIPDB.Load(reputation.Checks{Day: now, Spent: 3, Scores: []reputation.Score{ - {Client: netip.MustParsePrefix("2001:db8::/64"), Fetched: now.Add(-2 * time.Hour)}, - {Client: client, Score: 100, Fetched: now.Add(-time.Hour)}, - }}) + params.AbuseIPDB.Load(reputation.Checks{ + Day: civil.DateOf(now), Spent: 3, Scores: []reputation.Score{ + {Client: netip.MustParsePrefix("2001:db8::/64"), Fetched: now.Add(-2 * time.Hour)}, + {Client: client, Score: 100, Fetched: now.Add(-time.Hour)}, + }, + }) // An alert waiting, a repeat of it the cooldown holds back, another // alert waiting, and one past the two an hour, for the hour's summary. diff --git a/internal/sww/discriminator_test.go b/internal/sww/discriminator_test.go index 077b7bf..b292eee 100644 --- a/internal/sww/discriminator_test.go +++ b/internal/sww/discriminator_test.go @@ -9,10 +9,14 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "strings" "syscall" "testing" "time" + // The time zone database, so that the process TestTimesAreInUTCWhateverTZSays + // starts finds the zone TZ names on any host. + _ "time/tzdata" "sneak.berlin/go/smallwebwaf/internal/discriminator" "sneak.berlin/go/smallwebwaf/internal/sww" @@ -80,6 +84,95 @@ func TestMainRunsDiscriminator(t *testing.T) { } } +func TestTimesAreInUTCWhateverTZSays(t *testing.T) { + t.Parallel() + + // Kiritimati is 14 hours ahead of UTC. + const zone = "Pacific/Kiritimati" + + _, err := time.LoadLocation(zone) + if err != nil { + t.Fatalf("load %s: %v", zone, err) + } + + binary, err := os.Executable() + if err != nil { + t.Fatalf("find the test binary: %v", err) + } + + ctx, cancel := context.WithTimeout(t.Context(), 3*waitLimit) + defer cancel() + + dir := t.TempDir() + out := &output{} + cmd := exec.CommandContext(ctx, binary) //nolint:gosec // this test binary + cmd.Stdout = out + + cmd.Env = append(os.Environ(), + "TZ="+zone, + commandForMain+"="+discriminatorCommand, + listenAddr+"="+localhost+":0", + stateDir+"="+dir, + rulesDir+"="+t.TempDir(), + lookupSource+"=off") + + err = cmd.Start() + if err != nil { + t.Fatalf("start: %v", err) + } + + t.Cleanup(func() { + _ = cmd.Process.Kill() + _ = cmd.Wait() + }) + + starting := out.line(t, "msg", "starting") + addr, _ := starting["address"].(string) + ask(t, addr, "/", http.StatusOK, discriminator.DispositionAllow) + + // The files are written as sww stops. + err = cmd.Process.Signal(syscall.SIGTERM) + if err != nil { + t.Fatalf("signal: %v", err) + } + + err = cmd.Wait() + if err != nil { + t.Fatalf("exit: %v; output:\n%s", err, out.text()) + } + + wantUTC(t, "the process line", starting["time"]) + wantUTC(t, "the request line", out.line(t, "type", "request")["time"]) + + path := filepath.Join(dir, "clients.json") + + clients, err := os.ReadFile(path) //nolint:gosec // a file in the test's directory + if err != nil { + t.Fatalf("read clients.json: %v", err) + } + + stamp := regexp.MustCompile(`"\d{4}-\d\d-\d\dT[^"]*"`) + + times := stamp.FindAllString(string(clients), -1) + if len(times) == 0 { + t.Errorf("clients.json holds no time:\n%s", clients) + } + + for _, stored := range times { + wantUTC(t, "clients.json", strings.Trim(stored, `"`)) + } +} + +// wantUTC checks that value, a time where, is in UTC. +func wantUTC(t *testing.T, where string, value any) { + t.Helper() + + text, _ := value.(string) + if !strings.HasSuffix(text, "Z") { + t.Errorf("%s gives the time %v, want one in UTC", where, value) + } +} + func TestMainWithoutACommandItKnowsPrintsTheCommands(t *testing.T) { t.Parallel()