From 2bdecf2f34d37cf89dc3c88dabec8fc9460cef10 Mon Sep 17 00:00:00 2001 From: sneak Date: Wed, 11 Nov 2026 00:15:46 +0000 Subject: [PATCH] Every time in UTC, and every day as YYYY-MM-DD (closes #313) AbuseIPDB's daily count keeps its day as a civil.Date, taken from the UTC clock and written to reputation.json as "YYYY-MM-DD"; it was a time.Time at midnight. Every time read from a state file is converted to UTC, so a time an admin writes with an offset is written back in UTC. TestMainRunsDiscriminator runs sww discriminator with TZ 14 hours ahead of UTC and checks that the process line, the request line and clients.json are in UTC. SPEC.md states the rule near its start; AGENTS.md makes it an iron rule. Model: opus-5-5 --- AGENTS.md | 3 + SPEC.md | 5 ++ go.mod | 1 + go.sum | 2 + internal/lookup/file.go | 2 +- internal/lookup/lookup.go | 2 +- internal/reputation/abuseipdb.go | 15 +++-- internal/reputation/abuseipdb_test.go | 37 ++++++++++++ internal/state/state.go | 83 ++++++++++++++++++++++++++- internal/state/state_test.go | 74 +++++++++++++++++++++--- internal/sww/discriminator_test.go | 54 ++++++++++++++++- 11 files changed, 256 insertions(+), 22 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 3bf4d1e..2217ba1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -37,3 +37,6 @@ These are the rules for agents working on this repository. - `192.168.0.0/16`. - `SPEC.md` states that default. - Forwarded headers count only when the peer is a trusted proxy. +- Every time is in UTC, whatever `TZ` or the host's time zone says. + - Every day is a date written `YYYY-MM-DD`, never a time. + - `SPEC.md` states this near its start. diff --git a/SPEC.md b/SPEC.md index 5e6e6e1..b38a217 100644 --- a/SPEC.md +++ b/SPEC.md @@ -12,6 +12,10 @@ it is built, except what the last section lists. `README.md` says how to run it. under `docs/` give the settings, the formats and the lists. This document links to them. +The whole application stores, logs and handles every time in UTC, whatever `TZ` +or the host's time zone says. It keeps every day as a date written `YYYY-MM-DD`, +never as a time. + ## Purpose - It protects self-hosted apps on the open internet from scrapers, scanners and @@ -1781,6 +1785,7 @@ sizes. - a top-level `version` number; - entries sorted by client address; - times in RFC 3339 UTC; + - days as `YYYY-MM-DD`; - durations and sizes as plain numbers, with the unit in the field name. - `bans.json`, `reputation.json` and `alerts.json` are indented, so that a person can open `bans.json` in an editor, find an address, and remove or add diff --git a/go.mod b/go.mod index 6f7510b..20f2146 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module sneak.berlin/go/smallwebwaf go 1.26.0 require ( + cloud.google.com/go v0.123.0 github.com/corazawaf/coraza-coreruleset/v4 v4.25.0 github.com/corazawaf/coraza/v3 v3.8.2-0.20261022102553-d028be399c68 github.com/fsnotify/fsnotify v1.10.1 diff --git a/go.sum b/go.sum index 9abff90..e76b666 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= +cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= diff --git a/internal/lookup/file.go b/internal/lookup/file.go index 5cf7c72..f5dd4e3 100644 --- a/internal/lookup/file.go +++ b/internal/lookup/file.go @@ -26,7 +26,7 @@ type FileParams struct { // Path is the lookup database, the IPinfo Lite file in its .mmdb form // (SWW_LOOKUP_DB_PATH). Path string - // Now tells the time, normally time.Now. + // Now tells the time, normally time.Now in UTC. Now func() time.Time // ProcessLog receives each reading of the file, and why a replacement // of it cannot be read. diff --git a/internal/lookup/lookup.go b/internal/lookup/lookup.go index fdcfd1f..621525a 100644 --- a/internal/lookup/lookup.go +++ b/internal/lookup/lookup.go @@ -74,7 +74,7 @@ type Params struct { // Answered, unless nil, is given each answer GeoJS gives, once it is // kept. Answered func(Answer) - // Now tells the time, normally time.Now. + // Now tells the time, normally time.Now in UTC. Now func() time.Time // ProcessLog receives GeoJS's failures. ProcessLog *slog.Logger diff --git a/internal/reputation/abuseipdb.go b/internal/reputation/abuseipdb.go index b7de34b..6ee8c59 100644 --- a/internal/reputation/abuseipdb.go +++ b/internal/reputation/abuseipdb.go @@ -14,6 +14,7 @@ import ( "sync" "time" + "cloud.google.com/go/civil" "github.com/hashicorp/golang-lru/v2/simplelru" "sneak.berlin/go/smallwebwaf/internal/alerts" ) @@ -27,8 +28,6 @@ const ( AbuseIPDBSource = "abuseipdb" // maxAnswerBytes is the most of an answer of AbuseIPDB that is read. maxAnswerBytes = 64 << 10 - // day is the length of the day the checks are counted in, in UTC. - day = 24 * time.Hour ) var ( @@ -50,9 +49,9 @@ type Score struct { // AbuseIPDB: the day, in UTC, of the checks Spent counts, zero before the // first, and the scores still in use. type Checks struct { - Day time.Time `json:"day,omitzero"` - Spent int `json:"spent"` - Scores []Score `json:"scores"` + Day civil.Date `json:"day,omitzero"` + Spent int `json:"spent"` + Scores []Score `json:"scores"` } // AbuseIPDBParams are what NewAbuseIPDB needs. @@ -93,7 +92,7 @@ type AbuseIPDB struct { // checking are the clients whose check is under way. checking map[netip.Prefix]bool // day is the day, in UTC, of the checks spent counts. - day time.Time + day civil.Date spent int // checks and failures count the checks made and those that failed, // and retryAt is when a client may be checked again after the last @@ -143,7 +142,7 @@ func (a *AbuseIPDB) Hit( return kept.Score, kept.Score >= a.params.MinScore } - if today := now.Truncate(day); !a.day.Equal(today) { + if today := civil.DateOf(now.UTC()); a.day != today { a.day, a.spent = today, 0 } @@ -190,7 +189,7 @@ func (a *AbuseIPDB) BudgetLeft() int { a.mu.Lock() defer a.mu.Unlock() - if !a.day.Equal(a.params.Now().Truncate(day)) { + if a.day != civil.DateOf(a.params.Now().UTC()) { return a.params.DailyBudget } diff --git a/internal/reputation/abuseipdb_test.go b/internal/reputation/abuseipdb_test.go index 70460d2..bddce1d 100644 --- a/internal/reputation/abuseipdb_test.go +++ b/internal/reputation/abuseipdb_test.go @@ -202,6 +202,43 @@ func TestDailyBudgetKeptAcrossARestartAndWholeAgainAsTheDayEnds(t *testing.T) { }) } +func TestChecksAreCountedInUTCDaysWhateverTheClocksZone(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + // A clock 14 hours ahead of UTC, whose day starts at 10:00 UTC. + ahead := time.FixedZone("UTC+14", 14*60*60) + p := abuseIPDBParams() + p.DailyBudget = 1 + p.Now = func() time.Time { return time.Now().In(ahead) } + abuseIPDB := &abuseIPDBStandIn{scores: map[string]int64{suspect: 100}} + checker := newAbuseIPDB(abuseIPDB, p) + + // At 09:00 UTC the one check is spent. + time.Sleep(9 * time.Hour) + hitFrom(t, checker, suspect, true) + synctest.Wait() + wantBudgetLeft(t, checker, 0) + + // At 11:00 UTC the clock's day has changed, but the UTC day has not. + time.Sleep(2 * time.Hour) + wantBudgetLeft(t, checker, 0) + + encoded, err := json.Marshal(checker.Snapshot()) + if err != nil { + t.Fatalf("encode: %v", err) + } + + if !strings.Contains(string(encoded), `"day":"2000-01-01"`) { + t.Errorf("reputation.json would hold %s, want the day 2000-01-01", encoded) + } + + // At 00:00 UTC the budget is whole again. + time.Sleep(13 * time.Hour) + wantBudgetLeft(t, checker, 1) + }) +} + func TestFailedCheckGivesNoScoreAndNoClientIsCheckedForAMinute(t *testing.T) { t.Parallel() diff --git a/internal/state/state.go b/internal/state/state.go index 6058859..ed9474a 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -171,9 +171,11 @@ type alertsFile struct { // check refuses the first entry without a field it needs, which would // otherwise be read as something the entry does not say. data is the // file, for a field that may be null or "" but not left out, which the -// struct cannot tell apart. +// struct cannot tell apart. Its toUTC then converts every time in it to +// UTC, since an admin may write a time with an offset. type stateFile interface { check(data []byte) error + toUTC() } // Load checks that files can be written in Dir, and reads the state files @@ -918,6 +920,80 @@ func missing(i int, field string) error { return fmt.Errorf("entry %d %w %q", i+1, errMissing, field) } +// toUTC converts every time of each ban to UTC. +func (f *bansFile) toUTC() { + for i := range f.Bans { + ban := &f.Bans[i] + convertToUTC(&ban.Start, ban.Expires, ban.Lifted, &ban.Notes.Request.Time) + } +} + +// toUTC converts every time of each client to UTC. +func (f *clientsFile) toUTC() { + for i := range f.Clients { + client := &f.Clients[i] + convertToUTC(&client.Minute.Start, &client.Hour.Start, &client.Day.Start, + &client.MinuteBytes.Start, &client.HourBytes.Start, + &client.DayBytes.Start, &client.MinuteRefusals.Start, + &client.History.FirstSeen, &client.History.LastSeen, + &client.History.LookedUp) + } +} + +// toUTC converts both times of each answer to UTC. +func (f *lookupsFile) toUTC() { + for i := range f.Lookups { + convertToUTC(&f.Lookups[i].Answered, &f.Lookups[i].Used) + } +} + +// toUTC converts every time of each list, verdict and AbuseIPDB score to +// UTC. +func (f *reputationFile) toUTC() { + for i := range f.Lists { + convertToUTC(&f.Lists[i].Tried, &f.Lists[i].Fetched) + } + + for i := range f.Verdicts { + convertToUTC(&f.Verdicts[i].Fetched) + } + + for i := range f.AbuseIPDB.Scores { + convertToUTC(&f.AbuseIPDB.Scores[i].Fetched) + } +} + +// toUTC converts every time of each cooldown, of the hour, of each alert +// waiting and of each anomaly counter to UTC. +func (f *alertsFile) toUTC() { + for i := range f.Cooldowns { + convertToUTC(&f.Cooldowns[i].Sent) + } + + convertToUTC(&f.Hour.Start) + + for _, waiting := range f.Waiting { + for i := range waiting { + convertToUTC(&waiting[i].Time) + } + } + + for i := range f.AnomalyCounters { + counter := &f.AnomalyCounters[i] + convertToUTC(&counter.Minute.Start, &counter.Hour.Start, + &counter.MinuteBytes.Start, &counter.HourBytes.Start) + } +} + +// convertToUTC converts each of times that is not nil to UTC. +func convertToUTC(times ...*time.Time) { + for _, at := range times { + if at != nil { + *at = at.UTC() + } + } +} + // encodeOnePerLine encodes a state file whose entries, under key, are one // to a line, so that grep shows everything about one client. func encodeOnePerLine[E any](key string, entries []E) ([]byte, error) { @@ -955,7 +1031,8 @@ func checkWritable(dir string) error { } // parse reads data, what the state file at path holds, into file, a -// pointer to that file's struct, and checks its entries. +// pointer to that file's struct, checks its entries, and converts every +// time in it to UTC. func parse(path string, data []byte, file stateFile) error { // The version is read first, so that a file of another version is // refused for that, and not for an entry this version cannot read. @@ -985,6 +1062,8 @@ func parse(path string, data []byte, file stateFile) error { return fmt.Errorf("%s%s: %w", path, position(data, err), err) } + file.toUTC() + return nil } diff --git a/internal/state/state_test.go b/internal/state/state_test.go index 2fb9020..036e517 100644 --- a/internal/state/state_test.go +++ b/internal/state/state_test.go @@ -14,6 +14,7 @@ import ( "os" "path/filepath" "reflect" + "regexp" "slices" "strconv" "strings" @@ -21,6 +22,7 @@ import ( "testing/synctest" "time" + "cloud.google.com/go/civil" "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/anomaly" "sneak.berlin/go/smallwebwaf/internal/bans" @@ -258,7 +260,7 @@ const filledReputationJSON = `{ } ], "abuseipdb": { - "day": "2026-10-06T00:00:00Z", + "day": "2026-10-06", "spent": 3, "scores": [ { @@ -1306,7 +1308,7 @@ func TestEditOfEachFileTakenIn(t *testing.T) { `"tried": "2026-10-06T00:00:00Z", "fetched": "2026-10-06T00:00:00Z", `+ `"lines": ["198.51.100.7"]}], "verdicts": [{"zone": "`+dnsblZone+`", `+ `"client": "198.51.100.7", "listed": true, "fetched": "2026-10-06T00:00:00Z"}], `+ - `"abuseipdb": {"day": "2026-10-06T00:00:00Z", "spent": 9, "scores": [`+ + `"abuseipdb": {"day": "2026-10-06", "spent": 9, "scores": [`+ `{"client": "198.51.100.7/32", "score": 80, "fetched": "2026-10-06T00:00:00Z"}]}}`) wantTakenIn(t, lines, dir, reputationJSON) @@ -1321,7 +1323,7 @@ func TestEditOfEachFileTakenIn(t *testing.T) { }}) checks := reputation.Checks{ - Day: midnight(), Spent: 9, + Day: civil.DateOf(midnight()), Spent: 9, Scores: []reputation.Score{{Client: client, Score: 80, Fetched: midnight()}}, } if got := params.AbuseIPDB.Snapshot(); !reflect.DeepEqual(got, checks) { @@ -1350,6 +1352,62 @@ func TestEditOfEachFileTakenIn(t *testing.T) { } } +func TestTimesEditedWithAnOffsetWrittenBackInUTC(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + params := newParams(dir) + fill(params) + // A lifted ban, so that bans.json holds every time a ban has. + params.Ledger.Lift(netip.MustParseAddr("192.0.2.1"), midnight()) + + files := load(t, params) + + err := files.WriteAll() + if err != nil { + t.Fatalf("write: %v", err) + } + + // An admin saves each file with every time in it two hours ahead of + // UTC, midnight as "2026-10-06T02:00:00+02:00". + stamp := regexp.MustCompile(`"\d{4}-\d\d-\d\dT[^"]*Z"`) + ahead := time.FixedZone("UTC+2", 2*60*60) + written := map[string]string{} + + for _, name := range []string{ + bansJSON, clientsJSON, lookupsJSON, reputationJSON, alertsJSON, + } { + written[name] = readFile(t, filepath.Join(dir, name)) + edited := stamp.ReplaceAllStringFunc(written[name], func(quoted string) string { + at, err := time.Parse(time.RFC3339Nano, strings.Trim(quoted, `"`)) + if err != nil { + t.Fatalf("%s holds %s: %v", name, quoted, err) + } + + return `"` + at.In(ahead).Format(time.RFC3339Nano) + `"` + }) + + if !strings.Contains(edited, "+02:00") { + t.Fatalf("%s holds no time:\n%s", name, edited) + } + + edit(t, dir, name, edited) + } + + // Each write takes its edit in first, and writes the same times back in + // UTC. + err = files.WriteAll() + if err != nil { + t.Fatalf("write: %v", err) + } + + for name, want := range written { + if got := readFile(t, filepath.Join(dir, name)); got != want { + t.Errorf("%s written back as\n%s\nwant\n%s", name, got, want) + } + } +} + func TestOwnWritesAreNotTakenIn(t *testing.T) { t.Parallel() @@ -1837,10 +1895,12 @@ func fill(params state.Params) { }, {Zone: dnsblZone, Client: client.Addr(), Listed: true, Fetched: now.Add(-time.Hour)}, }) - params.AbuseIPDB.Load(reputation.Checks{Day: now, Spent: 3, Scores: []reputation.Score{ - {Client: netip.MustParsePrefix("2001:db8::/64"), Fetched: now.Add(-2 * time.Hour)}, - {Client: client, Score: 100, Fetched: now.Add(-time.Hour)}, - }}) + params.AbuseIPDB.Load(reputation.Checks{ + Day: civil.DateOf(now), Spent: 3, Scores: []reputation.Score{ + {Client: netip.MustParsePrefix("2001:db8::/64"), Fetched: now.Add(-2 * time.Hour)}, + {Client: client, Score: 100, Fetched: now.Add(-time.Hour)}, + }, + }) // An alert waiting, a repeat of it the cooldown holds back, another // alert waiting, and one past the two an hour, for the hour's summary. diff --git a/internal/sww/discriminator_test.go b/internal/sww/discriminator_test.go index 077b7bf..baa43c2 100644 --- a/internal/sww/discriminator_test.go +++ b/internal/sww/discriminator_test.go @@ -9,10 +9,14 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "strings" "syscall" "testing" "time" + // The time zone database, so that the process TestMainRunsDiscriminator + // starts finds the zone TZ names on any host. + _ "time/tzdata" "sneak.berlin/go/smallwebwaf/internal/discriminator" "sneak.berlin/go/smallwebwaf/internal/sww" @@ -37,6 +41,14 @@ func TestMain(m *testing.M) { func TestMainRunsDiscriminator(t *testing.T) { t.Parallel() + // Kiritimati is 14 hours ahead of UTC; every time sww gives stays in UTC. + const zone = "Pacific/Kiritimati" + + _, err := time.LoadLocation(zone) + if err != nil { + t.Fatalf("load %s: %v", zone, err) + } + binary, err := os.Executable() if err != nil { t.Fatalf("find the test binary: %v", err) @@ -45,14 +57,16 @@ func TestMainRunsDiscriminator(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 3*waitLimit) defer cancel() + dir := t.TempDir() out := &output{} cmd := exec.CommandContext(ctx, binary) //nolint:gosec // this test binary cmd.Stdout = out cmd.Env = append(os.Environ(), + "TZ="+zone, commandForMain+"="+discriminatorCommand, listenAddr+"="+localhost+":0", - stateDir+"="+t.TempDir(), + stateDir+"="+dir, rulesDir+"="+t.TempDir(), lookupSource+"=off") @@ -66,9 +80,11 @@ func TestMainRunsDiscriminator(t *testing.T) { _ = cmd.Wait() }) - addr, _ := out.line(t, "msg", "starting")["address"].(string) + starting := out.line(t, "msg", "starting") + addr, _ := starting["address"].(string) ask(t, addr, "/", http.StatusOK, discriminator.DispositionAllow) + // The files are written as sww stops. err = cmd.Process.Signal(syscall.SIGTERM) if err != nil { t.Fatalf("signal: %v", err) @@ -76,7 +92,39 @@ func TestMainRunsDiscriminator(t *testing.T) { err = cmd.Wait() if err != nil { - t.Errorf("exit: %v; output:\n%s", err, out.text()) + t.Fatalf("exit: %v; output:\n%s", err, out.text()) + } + + wantUTC(t, "the process line", starting["time"]) + wantUTC(t, "the request line", out.line(t, "type", "request")["time"]) + + path := filepath.Join(dir, "clients.json") + + clients, err := os.ReadFile(path) //nolint:gosec // a file in the test's directory + if err != nil { + t.Fatalf("read clients.json: %v", err) + } + + stamp := regexp.MustCompile(`"\d{4}-\d\d-\d\dT[^"]*"`) + + times := stamp.FindAllString(string(clients), -1) + if len(times) == 0 { + t.Errorf("clients.json holds no time:\n%s", clients) + } + + for _, stored := range times { + wantUTC(t, "clients.json", strings.Trim(stored, `"`)) + } +} + +// wantUTC checks that value, a time read from the log line or file that +// where names, is in UTC. +func wantUTC(t *testing.T, where string, value any) { + t.Helper() + + text, _ := value.(string) + if !strings.HasSuffix(text, "Z") { + t.Errorf("%s gives the time %v, want one in UTC", where, value) } }