diff --git a/AGENTS.md b/AGENTS.md index 3bf4d1e..2217ba1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -37,3 +37,6 @@ These are the rules for agents working on this repository. - `192.168.0.0/16`. - `SPEC.md` states that default. - Forwarded headers count only when the peer is a trusted proxy. +- Every time is in UTC, whatever `TZ` or the host's time zone says. + - Every day is a date written `YYYY-MM-DD`, never a time. + - `SPEC.md` states this near its start. diff --git a/SPEC.md b/SPEC.md index 5e6e6e1..b38a217 100644 --- a/SPEC.md +++ b/SPEC.md @@ -12,6 +12,10 @@ it is built, except what the last section lists. `README.md` says how to run it. under `docs/` give the settings, the formats and the lists. This document links to them. +The whole application stores, logs and handles every time in UTC, whatever `TZ` +or the host's time zone says. It keeps every day as a date written `YYYY-MM-DD`, +never as a time. + ## Purpose - It protects self-hosted apps on the open internet from scrapers, scanners and @@ -1781,6 +1785,7 @@ sizes. - a top-level `version` number; - entries sorted by client address; - times in RFC 3339 UTC; + - days as `YYYY-MM-DD`; - durations and sizes as plain numbers, with the unit in the field name. - `bans.json`, `reputation.json` and `alerts.json` are indented, so that a person can open `bans.json` in an editor, find an address, and remove or add diff --git a/go.mod b/go.mod index 6f7510b..20f2146 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module sneak.berlin/go/smallwebwaf go 1.26.0 require ( + cloud.google.com/go v0.123.0 github.com/corazawaf/coraza-coreruleset/v4 v4.25.0 github.com/corazawaf/coraza/v3 v3.8.2-0.20261022102553-d028be399c68 github.com/fsnotify/fsnotify v1.10.1 diff --git a/go.sum b/go.sum index 9abff90..e76b666 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= +cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= diff --git a/internal/lookup/file.go b/internal/lookup/file.go index 5cf7c72..f5dd4e3 100644 --- a/internal/lookup/file.go +++ b/internal/lookup/file.go @@ -26,7 +26,7 @@ type FileParams struct { // Path is the lookup database, the IPinfo Lite file in its .mmdb form // (SWW_LOOKUP_DB_PATH). Path string - // Now tells the time, normally time.Now. + // Now tells the time, normally time.Now in UTC. Now func() time.Time // ProcessLog receives each reading of the file, and why a replacement // of it cannot be read. diff --git a/internal/lookup/lookup.go b/internal/lookup/lookup.go index fdcfd1f..621525a 100644 --- a/internal/lookup/lookup.go +++ b/internal/lookup/lookup.go @@ -74,7 +74,7 @@ type Params struct { // Answered, unless nil, is given each answer GeoJS gives, once it is // kept. Answered func(Answer) - // Now tells the time, normally time.Now. + // Now tells the time, normally time.Now in UTC. Now func() time.Time // ProcessLog receives GeoJS's failures. ProcessLog *slog.Logger diff --git a/internal/reputation/abuseipdb.go b/internal/reputation/abuseipdb.go index b7de34b..6ee8c59 100644 --- a/internal/reputation/abuseipdb.go +++ b/internal/reputation/abuseipdb.go @@ -14,6 +14,7 @@ import ( "sync" "time" + "cloud.google.com/go/civil" "github.com/hashicorp/golang-lru/v2/simplelru" "sneak.berlin/go/smallwebwaf/internal/alerts" ) @@ -27,8 +28,6 @@ const ( AbuseIPDBSource = "abuseipdb" // maxAnswerBytes is the most of an answer of AbuseIPDB that is read. maxAnswerBytes = 64 << 10 - // day is the length of the day the checks are counted in, in UTC. - day = 24 * time.Hour ) var ( @@ -50,9 +49,9 @@ type Score struct { // AbuseIPDB: the day, in UTC, of the checks Spent counts, zero before the // first, and the scores still in use. type Checks struct { - Day time.Time `json:"day,omitzero"` - Spent int `json:"spent"` - Scores []Score `json:"scores"` + Day civil.Date `json:"day,omitzero"` + Spent int `json:"spent"` + Scores []Score `json:"scores"` } // AbuseIPDBParams are what NewAbuseIPDB needs. @@ -93,7 +92,7 @@ type AbuseIPDB struct { // checking are the clients whose check is under way. checking map[netip.Prefix]bool // day is the day, in UTC, of the checks spent counts. - day time.Time + day civil.Date spent int // checks and failures count the checks made and those that failed, // and retryAt is when a client may be checked again after the last @@ -143,7 +142,7 @@ func (a *AbuseIPDB) Hit( return kept.Score, kept.Score >= a.params.MinScore } - if today := now.Truncate(day); !a.day.Equal(today) { + if today := civil.DateOf(now.UTC()); a.day != today { a.day, a.spent = today, 0 } @@ -190,7 +189,7 @@ func (a *AbuseIPDB) BudgetLeft() int { a.mu.Lock() defer a.mu.Unlock() - if !a.day.Equal(a.params.Now().Truncate(day)) { + if a.day != civil.DateOf(a.params.Now().UTC()) { return a.params.DailyBudget } diff --git a/internal/reputation/abuseipdb_test.go b/internal/reputation/abuseipdb_test.go index 70460d2..bddce1d 100644 --- a/internal/reputation/abuseipdb_test.go +++ b/internal/reputation/abuseipdb_test.go @@ -202,6 +202,43 @@ func TestDailyBudgetKeptAcrossARestartAndWholeAgainAsTheDayEnds(t *testing.T) { }) } +func TestChecksAreCountedInUTCDaysWhateverTheClocksZone(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + // A clock 14 hours ahead of UTC, whose day starts at 10:00 UTC. + ahead := time.FixedZone("UTC+14", 14*60*60) + p := abuseIPDBParams() + p.DailyBudget = 1 + p.Now = func() time.Time { return time.Now().In(ahead) } + abuseIPDB := &abuseIPDBStandIn{scores: map[string]int64{suspect: 100}} + checker := newAbuseIPDB(abuseIPDB, p) + + // At 09:00 UTC the one check is spent. + time.Sleep(9 * time.Hour) + hitFrom(t, checker, suspect, true) + synctest.Wait() + wantBudgetLeft(t, checker, 0) + + // At 11:00 UTC the clock's day has changed, but the UTC day has not. + time.Sleep(2 * time.Hour) + wantBudgetLeft(t, checker, 0) + + encoded, err := json.Marshal(checker.Snapshot()) + if err != nil { + t.Fatalf("encode: %v", err) + } + + if !strings.Contains(string(encoded), `"day":"2000-01-01"`) { + t.Errorf("reputation.json would hold %s, want the day 2000-01-01", encoded) + } + + // At 00:00 UTC the budget is whole again. + time.Sleep(13 * time.Hour) + wantBudgetLeft(t, checker, 1) + }) +} + func TestFailedCheckGivesNoScoreAndNoClientIsCheckedForAMinute(t *testing.T) { t.Parallel() diff --git a/internal/state/state_test.go b/internal/state/state_test.go index 2fb9020..d0c9941 100644 --- a/internal/state/state_test.go +++ b/internal/state/state_test.go @@ -21,6 +21,7 @@ import ( "testing/synctest" "time" + "cloud.google.com/go/civil" "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/anomaly" "sneak.berlin/go/smallwebwaf/internal/bans" @@ -258,7 +259,7 @@ const filledReputationJSON = `{ } ], "abuseipdb": { - "day": "2026-10-06T00:00:00Z", + "day": "2026-10-06", "spent": 3, "scores": [ { @@ -1306,7 +1307,7 @@ func TestEditOfEachFileTakenIn(t *testing.T) { `"tried": "2026-10-06T00:00:00Z", "fetched": "2026-10-06T00:00:00Z", `+ `"lines": ["198.51.100.7"]}], "verdicts": [{"zone": "`+dnsblZone+`", `+ `"client": "198.51.100.7", "listed": true, "fetched": "2026-10-06T00:00:00Z"}], `+ - `"abuseipdb": {"day": "2026-10-06T00:00:00Z", "spent": 9, "scores": [`+ + `"abuseipdb": {"day": "2026-10-06", "spent": 9, "scores": [`+ `{"client": "198.51.100.7/32", "score": 80, "fetched": "2026-10-06T00:00:00Z"}]}}`) wantTakenIn(t, lines, dir, reputationJSON) @@ -1321,7 +1322,7 @@ func TestEditOfEachFileTakenIn(t *testing.T) { }}) checks := reputation.Checks{ - Day: midnight(), Spent: 9, + Day: civil.DateOf(midnight()), Spent: 9, Scores: []reputation.Score{{Client: client, Score: 80, Fetched: midnight()}}, } if got := params.AbuseIPDB.Snapshot(); !reflect.DeepEqual(got, checks) { @@ -1837,10 +1838,12 @@ func fill(params state.Params) { }, {Zone: dnsblZone, Client: client.Addr(), Listed: true, Fetched: now.Add(-time.Hour)}, }) - params.AbuseIPDB.Load(reputation.Checks{Day: now, Spent: 3, Scores: []reputation.Score{ - {Client: netip.MustParsePrefix("2001:db8::/64"), Fetched: now.Add(-2 * time.Hour)}, - {Client: client, Score: 100, Fetched: now.Add(-time.Hour)}, - }}) + params.AbuseIPDB.Load(reputation.Checks{ + Day: civil.DateOf(now), Spent: 3, Scores: []reputation.Score{ + {Client: netip.MustParsePrefix("2001:db8::/64"), Fetched: now.Add(-2 * time.Hour)}, + {Client: client, Score: 100, Fetched: now.Add(-time.Hour)}, + }, + }) // An alert waiting, a repeat of it the cooldown holds back, another // alert waiting, and one past the two an hour, for the hour's summary. diff --git a/internal/sww/discriminator_test.go b/internal/sww/discriminator_test.go index 077b7bf..b292eee 100644 --- a/internal/sww/discriminator_test.go +++ b/internal/sww/discriminator_test.go @@ -9,10 +9,14 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "strings" "syscall" "testing" "time" + // The time zone database, so that the process TestTimesAreInUTCWhateverTZSays + // starts finds the zone TZ names on any host. + _ "time/tzdata" "sneak.berlin/go/smallwebwaf/internal/discriminator" "sneak.berlin/go/smallwebwaf/internal/sww" @@ -80,6 +84,95 @@ func TestMainRunsDiscriminator(t *testing.T) { } } +func TestTimesAreInUTCWhateverTZSays(t *testing.T) { + t.Parallel() + + // Kiritimati is 14 hours ahead of UTC. + const zone = "Pacific/Kiritimati" + + _, err := time.LoadLocation(zone) + if err != nil { + t.Fatalf("load %s: %v", zone, err) + } + + binary, err := os.Executable() + if err != nil { + t.Fatalf("find the test binary: %v", err) + } + + ctx, cancel := context.WithTimeout(t.Context(), 3*waitLimit) + defer cancel() + + dir := t.TempDir() + out := &output{} + cmd := exec.CommandContext(ctx, binary) //nolint:gosec // this test binary + cmd.Stdout = out + + cmd.Env = append(os.Environ(), + "TZ="+zone, + commandForMain+"="+discriminatorCommand, + listenAddr+"="+localhost+":0", + stateDir+"="+dir, + rulesDir+"="+t.TempDir(), + lookupSource+"=off") + + err = cmd.Start() + if err != nil { + t.Fatalf("start: %v", err) + } + + t.Cleanup(func() { + _ = cmd.Process.Kill() + _ = cmd.Wait() + }) + + starting := out.line(t, "msg", "starting") + addr, _ := starting["address"].(string) + ask(t, addr, "/", http.StatusOK, discriminator.DispositionAllow) + + // The files are written as sww stops. + err = cmd.Process.Signal(syscall.SIGTERM) + if err != nil { + t.Fatalf("signal: %v", err) + } + + err = cmd.Wait() + if err != nil { + t.Fatalf("exit: %v; output:\n%s", err, out.text()) + } + + wantUTC(t, "the process line", starting["time"]) + wantUTC(t, "the request line", out.line(t, "type", "request")["time"]) + + path := filepath.Join(dir, "clients.json") + + clients, err := os.ReadFile(path) //nolint:gosec // a file in the test's directory + if err != nil { + t.Fatalf("read clients.json: %v", err) + } + + stamp := regexp.MustCompile(`"\d{4}-\d\d-\d\dT[^"]*"`) + + times := stamp.FindAllString(string(clients), -1) + if len(times) == 0 { + t.Errorf("clients.json holds no time:\n%s", clients) + } + + for _, stored := range times { + wantUTC(t, "clients.json", strings.Trim(stored, `"`)) + } +} + +// wantUTC checks that value, a time where, is in UTC. +func wantUTC(t *testing.T, where string, value any) { + t.Helper() + + text, _ := value.(string) + if !strings.HasSuffix(text, "Z") { + t.Errorf("%s gives the time %v, want one in UTC", where, value) + } +} + func TestMainWithoutACommandItKnowsPrintsTheCommands(t *testing.T) { t.Parallel()