diff --git a/AGENTS.md b/AGENTS.md index 3bf4d1e..2217ba1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -37,3 +37,6 @@ These are the rules for agents working on this repository. - `192.168.0.0/16`. - `SPEC.md` states that default. - Forwarded headers count only when the peer is a trusted proxy. +- Every time is in UTC, whatever `TZ` or the host's time zone says. + - Every day is a date written `YYYY-MM-DD`, never a time. + - `SPEC.md` states this near its start. diff --git a/SPEC.md b/SPEC.md index 5e6e6e1..b38a217 100644 --- a/SPEC.md +++ b/SPEC.md @@ -12,6 +12,10 @@ it is built, except what the last section lists. `README.md` says how to run it. under `docs/` give the settings, the formats and the lists. This document links to them. +The whole application stores, logs and handles every time in UTC, whatever `TZ` +or the host's time zone says. It keeps every day as a date written `YYYY-MM-DD`, +never as a time. + ## Purpose - It protects self-hosted apps on the open internet from scrapers, scanners and @@ -1781,6 +1785,7 @@ sizes. - a top-level `version` number; - entries sorted by client address; - times in RFC 3339 UTC; + - days as `YYYY-MM-DD`; - durations and sizes as plain numbers, with the unit in the field name. - `bans.json`, `reputation.json` and `alerts.json` are indented, so that a person can open `bans.json` in an editor, find an address, and remove or add diff --git a/go.mod b/go.mod index 6f7510b..20f2146 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module sneak.berlin/go/smallwebwaf go 1.26.0 require ( + cloud.google.com/go v0.123.0 github.com/corazawaf/coraza-coreruleset/v4 v4.25.0 github.com/corazawaf/coraza/v3 v3.8.2-0.20261022102553-d028be399c68 github.com/fsnotify/fsnotify v1.10.1 diff --git a/go.sum b/go.sum index 9abff90..e76b666 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= +cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= diff --git a/internal/lookup/file.go b/internal/lookup/file.go index 5cf7c72..f5dd4e3 100644 --- a/internal/lookup/file.go +++ b/internal/lookup/file.go @@ -26,7 +26,7 @@ type FileParams struct { // Path is the lookup database, the IPinfo Lite file in its .mmdb form // (SWW_LOOKUP_DB_PATH). Path string - // Now tells the time, normally time.Now. + // Now tells the time, normally time.Now in UTC. Now func() time.Time // ProcessLog receives each reading of the file, and why a replacement // of it cannot be read. diff --git a/internal/lookup/lookup.go b/internal/lookup/lookup.go index fdcfd1f..621525a 100644 --- a/internal/lookup/lookup.go +++ b/internal/lookup/lookup.go @@ -74,7 +74,7 @@ type Params struct { // Answered, unless nil, is given each answer GeoJS gives, once it is // kept. Answered func(Answer) - // Now tells the time, normally time.Now. + // Now tells the time, normally time.Now in UTC. Now func() time.Time // ProcessLog receives GeoJS's failures. ProcessLog *slog.Logger diff --git a/internal/reputation/abuseipdb.go b/internal/reputation/abuseipdb.go index b7de34b..6ee8c59 100644 --- a/internal/reputation/abuseipdb.go +++ b/internal/reputation/abuseipdb.go @@ -14,6 +14,7 @@ import ( "sync" "time" + "cloud.google.com/go/civil" "github.com/hashicorp/golang-lru/v2/simplelru" "sneak.berlin/go/smallwebwaf/internal/alerts" ) @@ -27,8 +28,6 @@ const ( AbuseIPDBSource = "abuseipdb" // maxAnswerBytes is the most of an answer of AbuseIPDB that is read. maxAnswerBytes = 64 << 10 - // day is the length of the day the checks are counted in, in UTC. - day = 24 * time.Hour ) var ( @@ -50,9 +49,9 @@ type Score struct { // AbuseIPDB: the day, in UTC, of the checks Spent counts, zero before the // first, and the scores still in use. type Checks struct { - Day time.Time `json:"day,omitzero"` - Spent int `json:"spent"` - Scores []Score `json:"scores"` + Day civil.Date `json:"day,omitzero"` + Spent int `json:"spent"` + Scores []Score `json:"scores"` } // AbuseIPDBParams are what NewAbuseIPDB needs. @@ -93,7 +92,7 @@ type AbuseIPDB struct { // checking are the clients whose check is under way. checking map[netip.Prefix]bool // day is the day, in UTC, of the checks spent counts. - day time.Time + day civil.Date spent int // checks and failures count the checks made and those that failed, // and retryAt is when a client may be checked again after the last @@ -143,7 +142,7 @@ func (a *AbuseIPDB) Hit( return kept.Score, kept.Score >= a.params.MinScore } - if today := now.Truncate(day); !a.day.Equal(today) { + if today := civil.DateOf(now.UTC()); a.day != today { a.day, a.spent = today, 0 } @@ -190,7 +189,7 @@ func (a *AbuseIPDB) BudgetLeft() int { a.mu.Lock() defer a.mu.Unlock() - if !a.day.Equal(a.params.Now().Truncate(day)) { + if a.day != civil.DateOf(a.params.Now().UTC()) { return a.params.DailyBudget } diff --git a/internal/reputation/abuseipdb_test.go b/internal/reputation/abuseipdb_test.go index 70460d2..bddce1d 100644 --- a/internal/reputation/abuseipdb_test.go +++ b/internal/reputation/abuseipdb_test.go @@ -202,6 +202,43 @@ func TestDailyBudgetKeptAcrossARestartAndWholeAgainAsTheDayEnds(t *testing.T) { }) } +func TestChecksAreCountedInUTCDaysWhateverTheClocksZone(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + // A clock 14 hours ahead of UTC, whose day starts at 10:00 UTC. + ahead := time.FixedZone("UTC+14", 14*60*60) + p := abuseIPDBParams() + p.DailyBudget = 1 + p.Now = func() time.Time { return time.Now().In(ahead) } + abuseIPDB := &abuseIPDBStandIn{scores: map[string]int64{suspect: 100}} + checker := newAbuseIPDB(abuseIPDB, p) + + // At 09:00 UTC the one check is spent. + time.Sleep(9 * time.Hour) + hitFrom(t, checker, suspect, true) + synctest.Wait() + wantBudgetLeft(t, checker, 0) + + // At 11:00 UTC the clock's day has changed, but the UTC day has not. + time.Sleep(2 * time.Hour) + wantBudgetLeft(t, checker, 0) + + encoded, err := json.Marshal(checker.Snapshot()) + if err != nil { + t.Fatalf("encode: %v", err) + } + + if !strings.Contains(string(encoded), `"day":"2000-01-01"`) { + t.Errorf("reputation.json would hold %s, want the day 2000-01-01", encoded) + } + + // At 00:00 UTC the budget is whole again. + time.Sleep(13 * time.Hour) + wantBudgetLeft(t, checker, 1) + }) +} + func TestFailedCheckGivesNoScoreAndNoClientIsCheckedForAMinute(t *testing.T) { t.Parallel() diff --git a/internal/state/state.go b/internal/state/state.go index 6058859..ed9474a 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -171,9 +171,11 @@ type alertsFile struct { // check refuses the first entry without a field it needs, which would // otherwise be read as something the entry does not say. data is the // file, for a field that may be null or "" but not left out, which the -// struct cannot tell apart. +// struct cannot tell apart. Its toUTC then converts every time in it to +// UTC, since an admin may write a time with an offset. type stateFile interface { check(data []byte) error + toUTC() } // Load checks that files can be written in Dir, and reads the state files @@ -918,6 +920,80 @@ func missing(i int, field string) error { return fmt.Errorf("entry %d %w %q", i+1, errMissing, field) } +// toUTC converts every time of each ban to UTC. +func (f *bansFile) toUTC() { + for i := range f.Bans { + ban := &f.Bans[i] + convertToUTC(&ban.Start, ban.Expires, ban.Lifted, &ban.Notes.Request.Time) + } +} + +// toUTC converts every time of each client to UTC. +func (f *clientsFile) toUTC() { + for i := range f.Clients { + client := &f.Clients[i] + convertToUTC(&client.Minute.Start, &client.Hour.Start, &client.Day.Start, + &client.MinuteBytes.Start, &client.HourBytes.Start, + &client.DayBytes.Start, &client.MinuteRefusals.Start, + &client.History.FirstSeen, &client.History.LastSeen, + &client.History.LookedUp) + } +} + +// toUTC converts both times of each answer to UTC. +func (f *lookupsFile) toUTC() { + for i := range f.Lookups { + convertToUTC(&f.Lookups[i].Answered, &f.Lookups[i].Used) + } +} + +// toUTC converts every time of each list, verdict and AbuseIPDB score to +// UTC. +func (f *reputationFile) toUTC() { + for i := range f.Lists { + convertToUTC(&f.Lists[i].Tried, &f.Lists[i].Fetched) + } + + for i := range f.Verdicts { + convertToUTC(&f.Verdicts[i].Fetched) + } + + for i := range f.AbuseIPDB.Scores { + convertToUTC(&f.AbuseIPDB.Scores[i].Fetched) + } +} + +// toUTC converts every time of each cooldown, of the hour, of each alert +// waiting and of each anomaly counter to UTC. +func (f *alertsFile) toUTC() { + for i := range f.Cooldowns { + convertToUTC(&f.Cooldowns[i].Sent) + } + + convertToUTC(&f.Hour.Start) + + for _, waiting := range f.Waiting { + for i := range waiting { + convertToUTC(&waiting[i].Time) + } + } + + for i := range f.AnomalyCounters { + counter := &f.AnomalyCounters[i] + convertToUTC(&counter.Minute.Start, &counter.Hour.Start, + &counter.MinuteBytes.Start, &counter.HourBytes.Start) + } +} + +// convertToUTC converts each of times that is not nil to UTC. +func convertToUTC(times ...*time.Time) { + for _, at := range times { + if at != nil { + *at = at.UTC() + } + } +} + // encodeOnePerLine encodes a state file whose entries, under key, are one // to a line, so that grep shows everything about one client. func encodeOnePerLine[E any](key string, entries []E) ([]byte, error) { @@ -955,7 +1031,8 @@ func checkWritable(dir string) error { } // parse reads data, what the state file at path holds, into file, a -// pointer to that file's struct, and checks its entries. +// pointer to that file's struct, checks its entries, and converts every +// time in it to UTC. func parse(path string, data []byte, file stateFile) error { // The version is read first, so that a file of another version is // refused for that, and not for an entry this version cannot read. @@ -985,6 +1062,8 @@ func parse(path string, data []byte, file stateFile) error { return fmt.Errorf("%s%s: %w", path, position(data, err), err) } + file.toUTC() + return nil } diff --git a/internal/state/state_test.go b/internal/state/state_test.go index 2fb9020..036e517 100644 --- a/internal/state/state_test.go +++ b/internal/state/state_test.go @@ -14,6 +14,7 @@ import ( "os" "path/filepath" "reflect" + "regexp" "slices" "strconv" "strings" @@ -21,6 +22,7 @@ import ( "testing/synctest" "time" + "cloud.google.com/go/civil" "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/anomaly" "sneak.berlin/go/smallwebwaf/internal/bans" @@ -258,7 +260,7 @@ const filledReputationJSON = `{ } ], "abuseipdb": { - "day": "2026-10-06T00:00:00Z", + "day": "2026-10-06", "spent": 3, "scores": [ { @@ -1306,7 +1308,7 @@ func TestEditOfEachFileTakenIn(t *testing.T) { `"tried": "2026-10-06T00:00:00Z", "fetched": "2026-10-06T00:00:00Z", `+ `"lines": ["198.51.100.7"]}], "verdicts": [{"zone": "`+dnsblZone+`", `+ `"client": "198.51.100.7", "listed": true, "fetched": "2026-10-06T00:00:00Z"}], `+ - `"abuseipdb": {"day": "2026-10-06T00:00:00Z", "spent": 9, "scores": [`+ + `"abuseipdb": {"day": "2026-10-06", "spent": 9, "scores": [`+ `{"client": "198.51.100.7/32", "score": 80, "fetched": "2026-10-06T00:00:00Z"}]}}`) wantTakenIn(t, lines, dir, reputationJSON) @@ -1321,7 +1323,7 @@ func TestEditOfEachFileTakenIn(t *testing.T) { }}) checks := reputation.Checks{ - Day: midnight(), Spent: 9, + Day: civil.DateOf(midnight()), Spent: 9, Scores: []reputation.Score{{Client: client, Score: 80, Fetched: midnight()}}, } if got := params.AbuseIPDB.Snapshot(); !reflect.DeepEqual(got, checks) { @@ -1350,6 +1352,62 @@ func TestEditOfEachFileTakenIn(t *testing.T) { } } +func TestTimesEditedWithAnOffsetWrittenBackInUTC(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + params := newParams(dir) + fill(params) + // A lifted ban, so that bans.json holds every time a ban has. + params.Ledger.Lift(netip.MustParseAddr("192.0.2.1"), midnight()) + + files := load(t, params) + + err := files.WriteAll() + if err != nil { + t.Fatalf("write: %v", err) + } + + // An admin saves each file with every time in it two hours ahead of + // UTC, midnight as "2026-10-06T02:00:00+02:00". + stamp := regexp.MustCompile(`"\d{4}-\d\d-\d\dT[^"]*Z"`) + ahead := time.FixedZone("UTC+2", 2*60*60) + written := map[string]string{} + + for _, name := range []string{ + bansJSON, clientsJSON, lookupsJSON, reputationJSON, alertsJSON, + } { + written[name] = readFile(t, filepath.Join(dir, name)) + edited := stamp.ReplaceAllStringFunc(written[name], func(quoted string) string { + at, err := time.Parse(time.RFC3339Nano, strings.Trim(quoted, `"`)) + if err != nil { + t.Fatalf("%s holds %s: %v", name, quoted, err) + } + + return `"` + at.In(ahead).Format(time.RFC3339Nano) + `"` + }) + + if !strings.Contains(edited, "+02:00") { + t.Fatalf("%s holds no time:\n%s", name, edited) + } + + edit(t, dir, name, edited) + } + + // Each write takes its edit in first, and writes the same times back in + // UTC. + err = files.WriteAll() + if err != nil { + t.Fatalf("write: %v", err) + } + + for name, want := range written { + if got := readFile(t, filepath.Join(dir, name)); got != want { + t.Errorf("%s written back as\n%s\nwant\n%s", name, got, want) + } + } +} + func TestOwnWritesAreNotTakenIn(t *testing.T) { t.Parallel() @@ -1837,10 +1895,12 @@ func fill(params state.Params) { }, {Zone: dnsblZone, Client: client.Addr(), Listed: true, Fetched: now.Add(-time.Hour)}, }) - params.AbuseIPDB.Load(reputation.Checks{Day: now, Spent: 3, Scores: []reputation.Score{ - {Client: netip.MustParsePrefix("2001:db8::/64"), Fetched: now.Add(-2 * time.Hour)}, - {Client: client, Score: 100, Fetched: now.Add(-time.Hour)}, - }}) + params.AbuseIPDB.Load(reputation.Checks{ + Day: civil.DateOf(now), Spent: 3, Scores: []reputation.Score{ + {Client: netip.MustParsePrefix("2001:db8::/64"), Fetched: now.Add(-2 * time.Hour)}, + {Client: client, Score: 100, Fetched: now.Add(-time.Hour)}, + }, + }) // An alert waiting, a repeat of it the cooldown holds back, another // alert waiting, and one past the two an hour, for the hour's summary. diff --git a/internal/sww/discriminator_test.go b/internal/sww/discriminator_test.go index 077b7bf..baa43c2 100644 --- a/internal/sww/discriminator_test.go +++ b/internal/sww/discriminator_test.go @@ -9,10 +9,14 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "strings" "syscall" "testing" "time" + // The time zone database, so that the process TestMainRunsDiscriminator + // starts finds the zone TZ names on any host. + _ "time/tzdata" "sneak.berlin/go/smallwebwaf/internal/discriminator" "sneak.berlin/go/smallwebwaf/internal/sww" @@ -37,6 +41,14 @@ func TestMain(m *testing.M) { func TestMainRunsDiscriminator(t *testing.T) { t.Parallel() + // Kiritimati is 14 hours ahead of UTC; every time sww gives stays in UTC. + const zone = "Pacific/Kiritimati" + + _, err := time.LoadLocation(zone) + if err != nil { + t.Fatalf("load %s: %v", zone, err) + } + binary, err := os.Executable() if err != nil { t.Fatalf("find the test binary: %v", err) @@ -45,14 +57,16 @@ func TestMainRunsDiscriminator(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 3*waitLimit) defer cancel() + dir := t.TempDir() out := &output{} cmd := exec.CommandContext(ctx, binary) //nolint:gosec // this test binary cmd.Stdout = out cmd.Env = append(os.Environ(), + "TZ="+zone, commandForMain+"="+discriminatorCommand, listenAddr+"="+localhost+":0", - stateDir+"="+t.TempDir(), + stateDir+"="+dir, rulesDir+"="+t.TempDir(), lookupSource+"=off") @@ -66,9 +80,11 @@ func TestMainRunsDiscriminator(t *testing.T) { _ = cmd.Wait() }) - addr, _ := out.line(t, "msg", "starting")["address"].(string) + starting := out.line(t, "msg", "starting") + addr, _ := starting["address"].(string) ask(t, addr, "/", http.StatusOK, discriminator.DispositionAllow) + // The files are written as sww stops. err = cmd.Process.Signal(syscall.SIGTERM) if err != nil { t.Fatalf("signal: %v", err) @@ -76,7 +92,39 @@ func TestMainRunsDiscriminator(t *testing.T) { err = cmd.Wait() if err != nil { - t.Errorf("exit: %v; output:\n%s", err, out.text()) + t.Fatalf("exit: %v; output:\n%s", err, out.text()) + } + + wantUTC(t, "the process line", starting["time"]) + wantUTC(t, "the request line", out.line(t, "type", "request")["time"]) + + path := filepath.Join(dir, "clients.json") + + clients, err := os.ReadFile(path) //nolint:gosec // a file in the test's directory + if err != nil { + t.Fatalf("read clients.json: %v", err) + } + + stamp := regexp.MustCompile(`"\d{4}-\d\d-\d\dT[^"]*"`) + + times := stamp.FindAllString(string(clients), -1) + if len(times) == 0 { + t.Errorf("clients.json holds no time:\n%s", clients) + } + + for _, stored := range times { + wantUTC(t, "clients.json", strings.Trim(stored, `"`)) + } +} + +// wantUTC checks that value, a time read from the log line or file that +// where names, is in UTC. +func wantUTC(t *testing.T, where string, value any) { + t.Helper() + + text, _ := value.(string) + if !strings.HasSuffix(text, "Z") { + t.Errorf("%s gives the time %v, want one in UTC", where, value) } }