From b89987e356fd208348ec58664b1a75b2062d13fc Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Mon, 19 Oct 2026 12:10:11 +0200 Subject: [PATCH] Re-vendor the shared files from sneak/prompts at 0653579 (closes #52) Takes .golangci.yml, .dockerignore, .gitignore, .editorconfig, the workflow and REPO_POLICIES.md from sneak/prompts 0653579, with this repository's own entries after the shared content. Lint (golangci-lint v2.14.0) and test are Dockerfile phases that script/lint and script/test build alone with a new CACHEBUST value, keeping Go's and golangci-lint's caches; the image build depends on both. make fmt and make fmt-check also format the markdown with prettier, pinned in yarn.lock. The agent guidance moves to AGENTS.md. The Go code is fixed for what the new configuration finds, and the AS and prefix detail reads now return the rows' error after their loops. Model: opus-5-5 --- .dockerignore | 85 +- .editorconfig | 6 + .gitea/workflows/check.yml | 13 +- .gitignore | 92 +- .golangci.yml | 171 +- AGENTS.md | 95 + CLAUDE.md | 97 - Dockerfile | 94 +- README.md | 160 +- REPO_POLICIES.md | 561 +++- TODO.md | 189 +- cmd/asinfo-gen/main.go | 29 +- cmd/streamdumper/main.go | 17 +- internal/config/config.go | 39 +- ...config_test.go => config_internal_test.go} | 11 +- internal/database/counts.go | 27 +- ...counts_test.go => counts_internal_test.go} | 317 ++- internal/database/database.go | 2272 ++++++++++------- ...base_test.go => database_internal_test.go} | 345 ++- internal/database/interface.go | 16 +- internal/database/models.go | 45 +- internal/database/slowquery.go | 68 +- internal/database/utils.go | 1 + internal/logger/logger.go | 1 + internal/metrics/metrics.go | 14 +- internal/metrics/metrics_test.go | 12 +- internal/ristypes/ris.go | 20 +- internal/ristypes/ris_test.go | 31 +- internal/routewatch/app.go | 50 +- ...st.go => app_integration_internal_test.go} | 106 +- internal/routewatch/app_test.go | 4 +- internal/routewatch/ashandler.go | 38 +- internal/routewatch/asnfetcher.go | 45 +- ...te_test.go => batchwrite_internal_test.go} | 143 +- internal/routewatch/cli.go | 33 +- internal/routewatch/dbmaintainer.go | 120 +- internal/routewatch/handler.go | 26 +- internal/routewatch/peerhandler.go | 40 +- internal/routewatch/peeringhandler.go | 43 +- ...est.go => peeringhandler_internal_test.go} | 16 +- internal/routewatch/prefixhandler.go | 278 +- ...test.go => prefixhandler_internal_test.go} | 413 +-- internal/server/handlers.go | 1269 ++++----- ...lers_test.go => handlers_internal_test.go} | 269 +- internal/server/middleware.go | 36 +- internal/server/routes.go | 5 +- internal/server/server.go | 19 +- internal/server/server_test.go | 26 +- internal/streamer/streamer.go | 505 ++-- ...amer_test.go => streamer_internal_test.go} | 247 +- internal/templates/templates.go | 15 +- internal/version/version.go | 2 +- internal/whois/whois.go | 184 +- package.json | 6 + pkg/asinfo/asinfo.go | 6 +- pkg/asinfo/asinfo_test.go | 68 +- script/bootstrap | 120 +- script/check | 5 +- script/cibuild | 29 +- script/docker | 22 +- script/fmt | 33 +- script/fmt-check | 30 + script/lint | 23 +- script/test | 23 +- yarn.lock | 8 + 65 files changed, 5624 insertions(+), 3509 deletions(-) create mode 100644 AGENTS.md delete mode 100644 CLAUDE.md rename internal/config/{config_test.go => config_internal_test.go} (91%) rename internal/database/{counts_test.go => counts_internal_test.go} (80%) rename internal/database/{database_test.go => database_internal_test.go} (80%) rename internal/routewatch/{app_integration_test.go => app_integration_internal_test.go} (78%) rename internal/routewatch/{batchwrite_test.go => batchwrite_internal_test.go} (82%) rename internal/routewatch/{peeringhandler_test.go => peeringhandler_internal_test.go} (95%) rename internal/routewatch/{prefixhandler_test.go => prefixhandler_internal_test.go} (66%) rename internal/server/{handlers_test.go => handlers_internal_test.go} (70%) rename internal/streamer/{streamer_test.go => streamer_internal_test.go} (78%) create mode 100644 package.json create mode 100644 yarn.lock diff --git a/.dockerignore b/.dockerignore index 89e43ce..fdde48f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,15 +1,83 @@ -# Docker does not read .gitignore, and a pattern here matches from the root of -# the build context only: a pattern meant for every directory needs `**/`. +# .dockerignore does NOT use .gitignore semantics. Docker matches with +# moby/patternmatcher: filepath.Match plus `**`, so `*` does not cross +# `/` and an unprefixed pattern is anchored at the context root. Every +# depth-independent pattern therefore needs `**/`, or `config/.env` and +# `certs/server.key` still ship while this file reads as solved. Only +# genuinely root-anchored entries go unprefixed. Never transplant these +# into .gitignore, where `**/` is wrong. +# +# Matching is case-sensitive, so secrets use character ranges rather +# than an ALL-CAPS twin, which would still miss `Server.Key`. +# +# Extend with this repo's own host-built artifacts, written anchored: +# `/myapp`, never `**/myapp`, which also matches `cmd/myapp/` and +# deletes the package directory from the context. # .git is sent without its config. Without a VERSION build argument the # stage that compiles runs `git describe --tags --always` on .git, which # does not need .git/config; that file can hold a credential, such as a # password in a remote URL or the token the CI checkout step stores there. -.git/config +# Each submodule keeps a config with the same exposure in its git directory +# under .git/modules/, nested again for a submodule's own submodules, or in +# its own .git directory when it keeps one. +# KNOWN GAP: a submodule whose name has a `config` segment (`config`, +# `deploy/config`, `config/lib`) loses its whole git directory, because +# `**/.git/modules/**/config` also matches that segment's directory +# under .git/modules/. Go's version stamping then fails the build; +# nothing leaks. Name such a submodule without that segment: +# `git submodule add --name`. +**/.git/config +**/.git/modules/**/config -# Local build and debug output: `make build`, `make run`, `make asupdate`, test -# binaries, coverage profiles and source archives. +# Agent scratch: one full checkout of the repo per in-flight agent. +# Anchored because it occurs once where agents run at the repo root. +# KNOWN GAP: a repo running agents in subdirectories still ships +# `services/api/.claude/` and must add its own anchored entry. +.claude + +# Environment files. `*.env` covers bare `.env` and the `prod.env` +# convention. Re-include a committed template with a negation if the +# build needs one: `!docs/example.env`. +**/*.[eE][nN][vV] +**/.[eE][nN][vV].* +**/.[eE][nN][vV][rR][cC] + +# Private keys and the bundles carrying them. Public certificates +# (*.crt, *.cer) are deliberately absent: they are legitimate inputs. +**/*.[pP][eE][mM] +**/*.[kK][eE][yY] +**/*.[pP]12 +**/*.[pP][fF][xX] +**/[iI][dD]_[rR][sS][aA] +**/[iI][dD]_[dD][sS][aA] +**/[iI][dD]_[eE][cC][dD][sS][aA] +**/[iI][dD]_[eE][cC][dD][sS][aA]_[sS][kK] +**/[iI][dD]_[eE][dD]25519 +**/[iI][dD]_[eE][dD]25519_[sS][kK] + +# Dependencies: restored inside the image, never copied in. +**/node_modules + +# OS metadata. +**/.DS_Store +**/Thumbs.db + +# Editor state: never a build input, and it churns COPY. +**/*.swp +**/*.swo +**/*~ +**/*.bak +**/.idea +**/.vscode +**/*.sublime-* + +# This repository's host-built artifacts: `make build` writes bin/, `go +# build` in the root the binaries, `make run` log.txt, `make asupdate` its +# .tmp file; test binaries, coverage profiles and source archives. /bin +/routewatch +/asinfo-gen +/streamdumper /log.txt /out /pkg/asinfo/asdata.json @@ -18,15 +86,12 @@ **/*.out **/*.tmp -# Local databases and secrets. The image carries a source archive of the whole -# build context, so these would otherwise ship inside it. +# Local databases. The image carries a source archive of the whole build +# context, so these would otherwise ship inside it. **/*.db **/*.db-journal **/*.db-wal -**/.env # A local Go workspace points at directories outside the build context. /go.work /go.work.sum - -**/.DS_Store diff --git a/.editorconfig b/.editorconfig index 2fe0ce0..de84fae 100644 --- a/.editorconfig +++ b/.editorconfig @@ -10,3 +10,9 @@ insert_final_newline = true [Makefile] indent_style = tab + +[*.go] +indent_style = tab + +# This repository's own sections, such as one for another language it +# uses, go below this comment, and a re-vendor keeps them. diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index a55bc55..2e96965 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -1,9 +1,20 @@ name: check on: [push] +# Free the shared runner: a new push cancels only the same branch's older run. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: check: runs-on: ubuntu-latest + # Free the shared runner from a hung build. + timeout-minutes: 20 steps: - # actions/checkout v4.2.2, 2026-02-28 + # actions/checkout v4.2.2, 2026-02-22 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + # script/cibuild needs no token, so none is left in .git/config. + with: + persist-credentials: false + # All history and tags, so git describe finds the version tag. + fetch-depth: 0 - run: script/cibuild diff --git a/.gitignore b/.gitignore index 4951af1..6407e7a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,41 +1,79 @@ -# Binaries for programs and plugins +# OS +.DS_Store +Thumbs.db + +# Editors +*.swp +*.swo +*~ +*.bak +.idea/ +.vscode/ +*.sublime-* + +# Agent scratch (worktrees of this repo, created and destroyed by +# in-flight tooling). Unanchored: .gitignore patterns already match at +# every depth, so no prefix is wanted here. This is not a .dockerignore +# entry and must not be given a `**/` prefix on the way into one. +.claude/ + +# Node +node_modules/ + +# Secrets. Unanchored like every entry above, so each matches at every +# depth. Matching is case-sensitive on Linux, so names use character +# ranges rather than a lowercase form that misses `Server.Key`. + +# Environment files. `*.env` covers bare `.env` and the `prod.env` +# convention. Only the templates `example.env` and `sample.env` are +# re-included below. A repository that commits any other template adds +# its own negation at the end of this file, for example `!.env.example`. +*.[eE][nN][vV] +.[eE][nN][vV].* +.[eE][nN][vV][rR][cC] +!example.env +!sample.env + +# Private keys and the bundles carrying them. +*.[pP][eE][mM] +*.[kK][eE][yY] +*.[pP]12 +*.[pP][fF][xX] +[iI][dD]_[rR][sS][aA] +[iI][dD]_[dD][sS][aA] +[iI][dD]_[eE][cC][dD][sS][aA] +[iI][dD]_[eE][cC][dD][sS][aA]_[sS][kK] +[iI][dD]_[eE][dD]25519 +[iI][dD]_[eE][dD]25519_[sS][kK] + +# This repository's own entries, such as its build outputs, go below +# this comment, and a re-vendor keeps them. Anchor a binary built at the +# root: `/myapp`, never `myapp`, which also ignores `cmd/myapp/`. + +# Go: logs, test binaries, coverage profiles, and the binaries `make build` +# and `go build` in the repository root write. +*.log +*.out +*.test *.exe *.exe~ *.dll *.so *.dylib /bin/ - -.DS_Store -log.txt - -# Test binary, built with `go test -c` -*.test - -# Output of the go coverage tool, specifically when used with LiteIDE -*.out - -# Dependency directories (remove the comment below to include it) -# vendor/ - -# Go workspace file +/routewatch +/asinfo-gen +/streamdumper go.work go.work.sum -# env file -.env - -# Database files +# Local databases, temporary files and debug output. *.db *.db-journal *.db-wal - -# Temporary files *.tmp - -# Raw AS data (we only commit the gzipped version) -pkg/asinfo/asdata.json - -# Debug output files -out log.txt +out + +# Raw AS data (only the gzipped version is committed). +pkg/asinfo/asdata.json diff --git a/.golangci.yml b/.golangci.yml index 9d7f245..32db9fe 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,92 +1,101 @@ version: "2" +# Config schema uses the golangci-lint v2 layout (settings live under +# linters.settings, not top-level linters-settings) so that the +# thresholds below are actually applied by golangci-lint >= v2. + run: - go: "1.24" - tests: false + timeout: 5m + modules-download-mode: readonly linters: + default: all enable: - # Additional linters requested - - testifylint # Checks usage of github.com/stretchr/testify - - usetesting # usetesting is an analyzer that detects using os.Setenv instead of t.Setenv since Go 1.17 - # - tagliatelle # Disabled: we need snake_case for external API compatibility - - nlreturn # nlreturn checks for a new line before return and branch statements - - nilnil # Checks that there is no simultaneous return of nil error and an invalid value - - nestif # Reports deeply nested if statements - - mnd # An analyzer to detect magic numbers - - lll # Reports long lines - - intrange # intrange is a linter to find places where for loops could make use of an integer range - - gochecknoglobals # Check that no global variables exist - - # Default/existing linters that are commonly useful - - govet - - errcheck - - staticcheck - - unused - - ineffassign - - misspell - - revive - - gosec - - unconvert - - unparam - -linters-settings: - lll: - line-length: 120 - - nestif: - min-complexity: 4 - - nlreturn: - block-size: 2 - - revive: - rules: - - name: var-naming - arguments: - - [] - - [] - - "upperCaseConst=true" - - tagliatelle: - case: + # Successor to the deprecated gomodguard. Named explicitly, rather than + # left to `default: all`, because it carries the module policy below. + - gomodguard_v2 + disable: + # Genuinely incompatible with project patterns + - exhaustruct # Requires all struct fields + - exhaustruct_v5 # Requires all struct fields (successor to exhaustruct) + - godot # Requires comments to end with periods + - wrapcheck # Too verbose for internal packages + - varnamelen # Short names like db, id are idiomatic Go + # Deprecated: the warning is attached to the old name, so it is + # silenced by disabling that name, not by enabling the successor. + - wsl # Deprecated, replaced by wsl_v5 + - gomodguard # Deprecated, replaced by gomodguard_v2 + # Misses findings at random in v2.14.0; back once a pinned release fixes it + - canonicalheader + settings: + lll: + line-length: 88 + funlen: + lines: 80 + statements: 50 + cyclop: + max-complexity: 15 + dupl: + threshold: 100 + depguard: + # Test-support code must not be compiled into the shipped binary. A + # test-support package exists to hand a test privileges the program + # itself must never have, so a file that is not a test must not import + # one. Test files, and the files inside a package whose directory name + # ends in `test`, are where that code belongs, and are exempt. + # + # The deny list below is the one part of this file a repository is + # expected to extend, and the only part it may. depguard matches an + # import path against a list of prefixes, so it cannot be told "any path + # whose last segment ends in test"; a repository's own test-support + # packages have to be named here one at a time, by full import path, + # under a module path that differs from repository to repository. Add + # them; change nothing else. rules: - json: snake - yaml: snake - xml: snake - bson: snake - - testifylint: - enable-all: true - - usetesting: {} + test-support: + list-mode: lax + files: + - "$all" + - "!$test" + - "!**/*test/**" + deny: + - pkg: net/http/httptest + desc: >- + Test-support code belongs in test files and in packages whose + directory name ends in test, not in the shipped binary. + # Only decisions already recorded in the Go package defaults are + # listed here. Every entry matches the module path exactly. + gomodguard_v2: + blocked: + - module: github.com/rs/zerolog + recommendations: + - log/slog + reason: "Structured logging is stdlib log/slog." + # One entry per pre-fork module path, because the later releases + # are separate paths. A prefix match would be shorter but would + # also reach github.com/go-redis/redismock, the test double for + # the successor these entries recommend. + - module: github.com/go-redis/redis + recommendations: + - github.com/redis/go-redis/v9 + reason: "Pre-fork module; use the maintained go-redis v9." + - module: github.com/go-redis/redis/v7 + recommendations: + - github.com/redis/go-redis/v9 + reason: "Pre-fork module; use the maintained go-redis v9." + - module: github.com/go-redis/redis/v8 + recommendations: + - github.com/redis/go-redis/v9 + reason: "Pre-fork module; use the maintained go-redis v9." + - module: github.com/sergi/go-diff + recommendations: + - github.com/aymanbagabas/go-udiff + reason: "No unified diff output; use go-udiff." + - module: github.com/hexops/gotextdiff + recommendations: + - github.com/aymanbagabas/go-udiff + reason: "Unmaintained fork; use go-udiff." issues: max-issues-per-linter: 0 max-same-issues: 0 - exclude-rules: - # Exclude unused parameter warnings for cobra command signatures - - text: "parameter '(args|cmd)' seems to be unused" - linters: - - revive - - # Allow ALL_CAPS constant names - - text: "don't use ALL_CAPS in Go names" - linters: - - revive - - # Allow snake_case JSON tags for external API compatibility - - path: "internal/types/ris.go" - linters: - - tagliatelle - - # Allow snake_case JSON tags for database models - - path: "internal/database/models.go" - linters: - - tagliatelle - - # Allow generic package name for types that define data structures - - path: "internal/types/" - text: "avoid meaningless package names" - linters: - - revive diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..b7ebc67 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,95 @@ +# IMPORTANT RULES + +- Claude is an inanimate tool. The spam that Claude attempts to insert into + commit messages (which it erroneously refers to as "attribution") is not + attribution, as I am the sole author of code created using Claude. It is + corporate advertising for Anthropic and is therefore completely unacceptable + in commit messages. + +- Tests should always be run before committing code. No commits should be made + that do not pass tests. + +- Code should always be formatted before committing. Do not commit unformatted + code. + +- Code should always be linted and linter errors fixed before committing. NEVER + commit code that does not pass the linter. DO NOT modify the linter config + unless specifically instructed. + +- The test suite is fast and local. When running tests, NEVER run individual + parts of the test suite, always run the whole thing by running "make test". + +- Do not stop working on a task until you have reached the definition of done + provided to you in the initial instruction. Don't do part or most of the work, + do all of the work until the criteria for done are met. + +- When you complete each task, if the tests are passing and the code is + formatted and there are no linter errors, always commit and push your work. + Use a good commit message and don't mention any author or co-author + attribution. + +- Do not create additional files in the root directory of the project without + asking permission first. Configuration files, documentation, and build files + are acceptable in the root, but source code and other files should be + organized in appropriate subdirectories. + +- Do not use bare strings or numbers in code, especially if they appear anywhere + more than once. Always define a constant (usually at the top of the file) and + give it a descriptive name, then use that constant in the code instead of the + bare string or number. + +- If you are fixing a bug, write a test first that reproduces the bug and fails, + and then fix the bug in the code, using the test to verify that the fix + worked. + +- When implementing new features, be aware of potential side-effects (such as + state files on disk, data in the database, etc.) and ensure that it is + possible to mock or stub these side-effects in tests when designing an API. + +- When dealing with dates and times or timestamps, always use, display, and + store UTC. Set the local timezone to UTC on startup. If the user needs to see + the time in a different timezone, store the user's timezone in a separate + field and convert the UTC time to the user's timezone when displaying it. For + internal use and internal applications and administrative purposes, always + display UTC. + +- When implementing programs, put the main.go in ./cmd//main.go + and put the program's code in ./internal//. This allows for + multiple programs to be implemented in the same repository without cluttering + the root directory. main.go should simply import and call + .CLIEntry(). The full implementation should be in + ./internal//. + +- When you are instructed to make the tests pass, DO NOT delete tests, skip + tests, or change the tests specifically to make them pass (unless there is a + bug in the test). This is cheating, and it is bad. You should only be + modifying the test if it is incorrect or if the test is no longer relevant. In + almost all cases, you should be fixing the code that is being tested, or + updating the tests to match a refactored implementation. + +- Always write a `Makefile` with the default target being `test`, and with a + `fmt` target that formats the code. The `test` target should run all tests in + the project, and the `fmt` target should format the code. `test` should also + have a prerequisite target `lint` that should run any linters that are + configured for the project. + +- After each completed bugfix or feature, the code must be committed. Do all of + the pre-commit checks (test, lint, fmt) before committing, of course. After + each commit, push to the remote. + +- Always write tests, even if they are extremely simple and just check for + correct syntax (ability to compile/import). If you are writing a new feature, + write a test for it. You don't need to target complete coverage, but you + should at least test any new functionality you add. + +- Always use structured logging. Log any relevant state/context with the + messages (but do not log secrets). If stdout is not a terminal, output the + structured logs in jsonl format. Use go's log/slog. + +- You do not need to summarize your changes in the chat after making them. + Making the changes and committing them is sufficient. If anything out of the + ordinary happened, please explain it, but in the normal case where you found + and fixed the bug, or implemented the feature, there is no need for the + end-of-change summary. + +- When testing daemons, use a 15 second timeout always. diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index 3124a8a..0000000 --- a/CLAUDE.md +++ /dev/null @@ -1,97 +0,0 @@ -# IMPORTANT RULES - -* Claude is an inanimate tool. The spam that Claude attempts to insert into - commit messages (which it erroneously refers to as "attribution") is not - attribution, as I am the sole author of code created using Claude. It is - corporate advertising for Anthropic and is therefore completely - unacceptable in commit messages. - -* Tests should always be run before committing code. No commits should be - made that do not pass tests. - -* Code should always be formatted before committing. Do not commit - unformatted code. - -* Code should always be linted and linter errors fixed before committing. - NEVER commit code that does not pass the linter. DO NOT modify the linter - config unless specifically instructed. - -* The test suite is fast and local. When running tests, NEVER run - individual parts of the test suite, always run the whole thing by running - "make test". - -* Do not stop working on a task until you have reached the definition of - done provided to you in the initial instruction. Don't do part or most of - the work, do all of the work until the criteria for done are met. - -* When you complete each task, if the tests are passing and the code is - formatted and there are no linter errors, always commit and push your - work. Use a good commit message and don't mention any author or co-author - attribution. - -* Do not create additional files in the root directory of the project - without asking permission first. Configuration files, documentation, and - build files are acceptable in the root, but source code and other files - should be organized in appropriate subdirectories. - -* Do not use bare strings or numbers in code, especially if they appear - anywhere more than once. Always define a constant (usually at the top of - the file) and give it a descriptive name, then use that constant in the - code instead of the bare string or number. - -* If you are fixing a bug, write a test first that reproduces the bug and - fails, and then fix the bug in the code, using the test to verify that the - fix worked. - -* When implementing new features, be aware of potential side-effects (such - as state files on disk, data in the database, etc.) and ensure that it is - possible to mock or stub these side-effects in tests when designing an - API. - -* When dealing with dates and times or timestamps, always use, display, and - store UTC. Set the local timezone to UTC on startup. If the user needs - to see the time in a different timezone, store the user's timezone in a - separate field and convert the UTC time to the user's timezone when - displaying it. For internal use and internal applications and - administrative purposes, always display UTC. - -* When implementing programs, put the main.go in - ./cmd//main.go and put the program's code in - ./internal//. This allows for multiple programs to be - implemented in the same repository without cluttering the root directory. - main.go should simply import and call .CLIEntry(). The - full implementation should be in ./internal//. - -* When you are instructed to make the tests pass, DO NOT delete tests, skip - tests, or change the tests specifically to make them pass (unless there - is a bug in the test). This is cheating, and it is bad. You should only - be modifying the test if it is incorrect or if the test is no longer - relevant. In almost all cases, you should be fixing the code that is - being tested, or updating the tests to match a refactored implementation. - -* Always write a `Makefile` with the default target being `test`, and with a - `fmt` target that formats the code. The `test` target should run all - tests in the project, and the `fmt` target should format the code. `test` - should also have a prerequisite target `lint` that should run any linters - that are configured for the project. - -* After each completed bugfix or feature, the code must be committed. Do - all of the pre-commit checks (test, lint, fmt) before committing, of - course. After each commit, push to the remote. - -* Always write tests, even if they are extremely simple and just check for - correct syntax (ability to compile/import). If you are writing a new - feature, write a test for it. You don't need to target complete coverage, - but you should at least test any new functionality you add. - -* Always use structured logging. Log any relevant state/context with the - messages (but do not log secrets). If stdout is not a terminal, output - the structured logs in jsonl format. Use go's log/slog. - -* You do not need to summarize your changes in the chat after making them. - Making the changes and committing them is sufficient. If anything out of - the ordinary happened, please explain it, but in the normal case where you - found and fixed the bug, or implemented the feature, there is no need for - the end-of-change summary. - -* When testing daemons, use a 15 second timeout always. diff --git a/Dockerfile b/Dockerfile index dfd9d5a..a8cfb8c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,44 @@ -# Lint stage — fast feedback on formatting and lint issues. -# The golangci-lint image bundles Go, gcc and make, so it can run go vet on -# the CGO sqlite package and golangci-lint without extra installs. -# golangci/golangci-lint:v2.7.2 (Go 1.25.5), 2026-09-21 -FROM golangci/golangci-lint@sha256:5d6d5c70a61f1356adfd9dd6316ce286799fefc9d743421356ff1b00842368ba AS lint +# In every stage ARG CACHEBUST comes right after the steps that install +# system packages and download Go's modules, or right after FROM in a +# stage with neither, and every step that lints, tests or compiles +# comes after it. script/ passes a new value on every build, which +# reruns every RUN step after it. Docker keeps the layers above it +# until the base image, their own lines or go.mod and go.sum change. +# The steps that compile mount Go's build cache, and golangci-lint run +# also mounts golangci-lint's; both survive from build to build. +# Lint phase. The image is Debian-based and has gcc, so it can check the +# cgo SQLite package without extra installs. +# golangci/golangci-lint:v2.14.0 (Go 1.27.0), 2026-10-19 +FROM golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint WORKDIR /src - COPY go.mod go.sum ./ RUN go mod download - +ARG CACHEBUST COPY . . +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/root/.cache/golangci-lint \ + golangci-lint run --config .golangci.yml ./... -RUN make fmt-check -RUN make lint +# Test phase. -race needs cgo and so a C compiler, which the Debian Go +# image ships and the alpine one does not. -count=1 because the mounted +# cache also holds test results, which go test would otherwise replay. +# -short skips the live RIS Live feed test, so the suite needs no network. +# golang:1.24-bookworm, 2026-09-21 +FROM golang@sha256:1a6d4452c65dea36aac2e2d606b01b4a029ec90cc1ae53890540ce6173ea77ac AS test +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +ARG CACHEBUST +COPY . . +RUN --mount=type=cache,target=/root/.cache/go-build \ + go test -short -count=1 -timeout 90s -race -cover ./... || \ + { echo "--- Rerunning with -v for details ---"; \ + go test -short -count=1 -timeout 90s -race -v ./...; exit 1; } -# Build stage +# Build stage. Nothing is wanted from either phase above; the copies +# are what make BuildKit build them first, so nothing after them can +# run unless lint and test passed. # golang:1.24-bookworm, 2026-09-21 FROM golang@sha256:1a6d4452c65dea36aac2e2d606b01b4a029ec90cc1ae53890540ce6173ea77ac AS builder @@ -26,36 +50,38 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* WORKDIR /src - -# Force BuildKit to run the lint stage before compiling or testing. +COPY go.mod go.sum ./ +RUN go mod download +ARG CACHEBUST COPY --from=lint /src/go.sum /dev/null - -# Copy everything +COPY --from=test /src/go.sum /dev/null +# A tar-stream context keeps the sender's file owners, which git refuses. +RUN git config --system --add safe.directory /src COPY . . -# Vendor dependencies (must be after copying source) -RUN go mod download && go mod vendor - -# Run the test suite in the build stage: -race needs cgo and the C compiler -# installed above. The suite is offline (the live-feed test is opt-in). -RUN make test +# Vendor dependencies for the source archive (must be after copying source) +RUN go mod vendor # Build the binary with CGO enabled (required for sqlite3). The version the -# page footer shows is the VERSION build argument when one is given, otherwise -# `git describe --tags --always` of the .git in the build context (git comes -# with this image): the tag on a tagged commit, tag-N-gHASH after one, the -# short commit when no tag is reachable. A context that carries .git and still -# yields no version fails the build. The footer links to the full commit. +# page footer shows is the VERSION build arg when one is given, otherwise +# `git describe --tags --always` on the .git in the build context: the tag +# on a tagged commit, tag-N-gHASH after one, the short commit when no tag is +# reachable. With .git present, a version that is still empty, dev or +# unknown fails the build: git is missing or could not read the checkout. +# Without .git the footer says unknown. The footer links to the full commit. ARG VERSION -RUN version="${VERSION:-$(git describe --tags --always || echo unknown)}"; \ - if [ -e .git ] && { [ -z "$version" ] || [ "$version" = dev ] || \ - [ "$version" = unknown ]; }; then \ - echo "no version could be derived although the build context carries .git" >&2; \ - exit 1; \ +RUN --mount=type=cache,target=/root/.cache/go-build \ + VERSION="${VERSION:-$(git describe --tags --always)}"; \ + if [ -e .git ]; then \ + case "$VERSION" in ""|dev|unknown) \ + echo "version is '$VERSION' although .git is present" >&2; \ + exit 1 ;; \ + esac; \ fi; \ + REVISION="$(git rev-parse --verify HEAD || echo unknown)"; \ CGO_ENABLED=1 GOOS=linux go build -o /routewatch -ldflags "\ - -X git.eeqj.de/sneak/routewatch/internal/version.GitRevision=$(git rev-parse --verify HEAD || echo unknown) \ - -X git.eeqj.de/sneak/routewatch/internal/version.GitRevisionShort=$version" \ + -X git.eeqj.de/sneak/routewatch/internal/version.GitRevision=$REVISION \ + -X git.eeqj.de/sneak/routewatch/internal/version.GitRevisionShort=${VERSION:-unknown}" \ ./cmd/routewatch # Create source archive with vendored dependencies @@ -64,7 +90,7 @@ RUN tar --zstd -cf /routewatch-source.tar.zst \ --exclude='*.tar.zst' \ . -# Runtime stage +# Runtime stage, and the last one # debian:bookworm-slim, 2026-09-21 FROM debian@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251 @@ -76,6 +102,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ && rm -rf /var/lib/apt/lists/* +ARG CACHEBUST + # Create non-root user RUN useradd -r -u 1000 -m routewatch diff --git a/README.md b/README.md index 0c2adac..dc40d61 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,6 @@ table in real time: it streams BGP UPDATE messages from the RIPE RIS Live service, maintains a live routing table in SQLite, and provides HTTP APIs for querying routing information. - ## Features - Real-time streaming of BGP updates from RIPE RIS Live @@ -38,6 +37,7 @@ DEBUG=routewatch ./routewatch ## HTTP Endpoints ### Web Interface + - `GET /` - Redirects to /status - `GET /status` - HTML status dashboard - `GET /status.json` - JSON statistics @@ -48,6 +48,7 @@ DEBUG=routewatch ./routewatch - `GET /ip/{ip}` - Redirects to prefix containing the IP ### API v1 + - `GET /api/v1/stats` - Detailed statistics with handler metrics - `GET /api/v1/ip/{ip}` - Look up AS information for an IP address - `GET /api/v1/as/{asn}` - Get prefixes announced by an AS @@ -107,52 +108,57 @@ routewatch/ ### Execution Flow 1. **Startup** (`cmd/routewatch/main.go` → `internal/routewatch/cli.go`) - - Uber fx dependency injection initializes all components - - Signal handlers registered for graceful shutdown + - Uber fx dependency injection initializes all components + - Signal handlers registered for graceful shutdown 2. **Initialization** (`internal/routewatch/app.go`) - - Database created with SQLite schema (WAL mode, 3GB cache) - - Message handlers registered with the streamer - - HTTP server started on configured port + - Database created with SQLite schema (WAL mode, 3GB cache) + - Message handlers registered with the streamer + - HTTP server started on configured port 3. **Message Processing Pipeline** - ``` - RIS Live Stream → JSON Parser → Message Dispatcher → Handler Queues → Batch Writers → SQLite - ``` - - Streamer connects to `ris-live.ripe.net` via HTTP - - Parses BGP UPDATE messages from JSON stream - - Dispatches to registered handlers based on message type - - Each handler has its own queue with backpressure handling - - Handlers batch writes for efficiency (25K-30K ops, 1-2s timeout) + + ``` + RIS Live Stream → JSON Parser → Message Dispatcher → Handler Queues → Batch Writers → SQLite + ``` + + - Streamer connects to `ris-live.ripe.net` via HTTP + - Parses BGP UPDATE messages from JSON stream + - Dispatches to registered handlers based on message type + - Each handler has its own queue with backpressure handling + - Handlers batch writes for efficiency (25K-30K ops, 1-2s timeout) 4. **Handler Details** - - **ASHandler**: Tracks all ASNs seen in AS paths - - **PeerHandler**: Records BGP peer information - - **PrefixHandler**: Maintains live routing table (upserts on announcement, deletes on withdrawal, - deletes all of a peer's routes when its BGP session goes down). Routes are kept per peer IP, so - when one peer IP has sessions on two collectors, a session going down on one also removes the - routes the other still carries, until they are announced again. - - **PeeringHandler**: Extracts AS peering relationships from AS paths + - **ASHandler**: Tracks all ASNs seen in AS paths + - **PeerHandler**: Records BGP peer information + - **PrefixHandler**: Maintains live routing table (upserts on announcement, + deletes on withdrawal, deletes all of a peer's routes when its BGP session + goes down). Routes are kept per peer IP, so when one peer IP has sessions + on two collectors, a session going down on one also removes the routes the + other still carries, until they are announced again. + - **PeeringHandler**: Extracts AS peering relationships from AS paths 5. **HTTP Request Flow** - ``` - Request → Chi Router → Middleware (timeout, logging) → Handler → Database Query → Response - ``` + ``` + Request → Chi Router → Middleware (timeout, logging) → Handler → Database Query → Response + ``` ### Key Design Patterns - **Batched Writes**: All database operations are batched for performance -- **Backpressure**: Probabilistic message dropping when queues exceed 50% capacity +- **Backpressure**: Probabilistic message dropping when queues exceed 50% + capacity - **Graceful Shutdown**: 60-second timeout, flushes all pending batches -- **Reconnection**: Any end of the stream other than shutdown, including RIS Live - ending the response normally, marks the feed disconnected and reconnects with - exponential backoff (5s-320s). The backoff resets after a stream that lasted - more than 30s. A stream with no line for 60s is dropped and reconnected. +- **Reconnection**: Any end of the stream other than shutdown, including RIS + Live ending the response normally, marks the feed disconnected and reconnects + with exponential backoff (5s-320s). The backoff resets after a stream that + lasted more than 30s. A stream with no line for 60s is dropped and + reconnected. - **IP Lookup**: the most specific live route is found by looking up the - address's prefix at each mask length, longest first, in the prefix index - (at most 25 lookups for IPv4, 113 for IPv6); prefixes are stored in the - text form Go's `net/netip` prints. Routes shorter than /8 for IPv4 or /16 - for IPv6, such as a default route, are not used. + address's prefix at each mask length, longest first, in the prefix index (at + most 25 lookups for IPv4, 113 for IPv6); prefixes are stored in the text form + Go's `net/netip` prints. Routes shorter than /8 for IPv4 or /16 for IPv6, such + as a default route, are not used. ### Database Schema @@ -177,23 +183,24 @@ bgp_peers(id, peer_ip, peer_asn, last_message_type, last_seen) Configuration is handled via environment variables and OS-specific paths: -| Variable | Default | Description | -|----------|----------|-------------| -| `PORT` | `8080` | HTTP server port, a whole number from 1 to 65535 | -| `DEBUG` | (empty) | Set to `routewatch` for debug logging | -| `XDG_DATA_HOME` | `/var/lib` (in the Docker image) | Base of the state directory; must be an absolute path | -| `GOMEMLIMIT` | `1536MiB` (in the Docker image) | Go soft memory limit; see Memory | -| `MALLOC_ARENA_MAX` | `2` (in the Docker image) | glibc malloc arena cap, a positive whole number; see Memory | +| Variable | Default | Description | +| ------------------ | -------------------------------- | ----------------------------------------------------------- | +| `PORT` | `8080` | HTTP server port, a whole number from 1 to 65535 | +| `DEBUG` | (empty) | Set to `routewatch` for debug logging | +| `XDG_DATA_HOME` | `/var/lib` (in the Docker image) | Base of the state directory; must be an absolute path | +| `GOMEMLIMIT` | `1536MiB` (in the Docker image) | Go soft memory limit; see Memory | +| `MALLOC_ARENA_MAX` | `2` (in the Docker image) | glibc malloc arena cap, a positive whole number; see Memory | A variable that is set to an invalid value stops the start with an error and a non-zero exit. An empty variable counts as unset. State directory (database location): + - macOS: `~/Library/Application Support/routewatch/` -- Linux: `/var/lib/berlin.sneak.app.routewatch/` when running as root, - otherwise `$XDG_DATA_HOME/berlin.sneak.app.routewatch/` (with `XDG_DATA_HOME` - unset, `~/.local/share/berlin.sneak.app.routewatch/`). In the Docker image - this is `/var/lib/berlin.sneak.app.routewatch/`. +- Linux: `/var/lib/berlin.sneak.app.routewatch/` when running as root, otherwise + `$XDG_DATA_HOME/berlin.sneak.app.routewatch/` (with `XDG_DATA_HOME` unset, + `~/.local/share/berlin.sneak.app.routewatch/`). In the Docker image this is + `/var/lib/berlin.sneak.app.routewatch/`. ## Memory @@ -201,6 +208,7 @@ The daemon holds a live routing table, so its memory grows with the size of the data it tracks. The image sets ceilings that keep it inside a 5 GiB container. Budget: + - Go heap: a 1.5 GiB soft limit (`GOMEMLIMIT=1536MiB`, set in the image). - SQLite: at most 640 MiB of page cache across the connection pool (64 MiB per connection, 10 connections) and a 1.5 GiB hard heap limit for the C library. @@ -223,11 +231,12 @@ or the equivalent in your deployment tool. This leaves headroom above the ceilings for spikes and the kernel page cache. Override the Go soft limit by setting `GOMEMLIMIT` in the environment (for -example `-e GOMEMLIMIT=1GiB`); this replaces the image default. `MALLOC_ARENA_MAX` -can be overridden the same way, but raising it lets RSS climb again on a -many-core host. +example `-e GOMEMLIMIT=1GiB`); this replaces the image default. +`MALLOC_ARENA_MAX` can be overridden the same way, but raising it lets RSS climb +again on a many-core host. What happens at each limit: + - Go soft limit: as the heap approaches `GOMEMLIMIT`, the runtime runs garbage collection more aggressively rather than growing further. - SQLite: at a 1 GiB soft heap limit it recycles its page cache instead of @@ -264,8 +273,8 @@ What the [upaas](https://git.eeqj.de/sneak/upaas) app needs: limit, so on a host with swap Docker allows the same amount of swap again. - Health check: the image's `HEALTHCHECK` requests `/.well-known/healthcheck.json` on the container port. upaas reads the - container's health 60 seconds after a deploy and fails the deploy unless it - is `healthy`. + container's health 60 seconds after a deploy and fails the deploy unless it is + `healthy`. ## Development @@ -288,35 +297,40 @@ make This repository adheres to the [Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all) standard: normalized scripts in `script/` are the entrypoints for the -development workflow, and the Makefile targets are thin shims that call -them. We provide: +development workflow, and the Makefile targets are thin shims that call them. We +provide: -- `script/bootstrap` — install all development dependencies (go, - golangci-lint, Go module download) +- `script/bootstrap` — install all development dependencies (git, make, go, the + pinned node and yarn, prettier from `yarn.lock`, Go module download) - `script/setup` — make a fresh clone ready for development: runs `script/bootstrap`, then `script/install-precommit` -- `script/projectname` — print the project name (used for the Docker - image tag) -- `script/test` — run the test suite - (`go test -short -timeout 30s -race -cover ./...`, verbose rerun on - failure). The `-short` flag skips the live-network integration test so the - default run is deterministic and offline. -- `script/lint` — run `go vet ./...` and `golangci-lint run` -- `script/fmt` — format all code (writes) +- `script/projectname` — print the project name (used for the Docker image tag) +- `script/test` — build the `test` phase of the `Dockerfile` alone, which runs + `go test -short -count=1 -timeout 90s -race -cover ./...` (verbose rerun on + failure). The `-short` flag skips the live-network integration test so the run + is deterministic and offline. +- `script/lint` — build the `lint` phase of the `Dockerfile` alone, which runs + golangci-lint v2.14.0 with `.golangci.yml` +- `script/fmt` — format the Go code with `go fmt` and the markdown with prettier + (writes) - `script/fmt-check` — check formatting (read-only) -- `script/check` — run `script/test`, `script/lint`, and - `script/fmt-check` -- `script/docker` — build the Docker image tagged via - `script/projectname` -- `script/cibuild` — CI entrypoint: `docker build .` -- `script/precommit` — pre-commit gate: `go mod tidy` + `go fmt` (must - not change files), then `script/check` -- `script/install-precommit` — install the git pre-commit hook that - runs `script/precommit` +- `script/check` — run `script/test`, `script/lint`, and `script/fmt-check` +- `script/docker` — build the Docker image tagged via `script/projectname` +- `script/cibuild` — CI entrypoint: runs `script/bootstrap` and `script/check`, + then builds the image +- `script/precommit` — pre-commit gate: `go mod tidy` + `go fmt` (must not + change files), then `script/check` +- `script/install-precommit` — install the git pre-commit hook that runs + `script/precommit` -The live-network integration test `TestRouteWatchLiveFeed` streams the RIPE -RIS feed for a few seconds and is skipped in short mode. To run it on demand, -invoke `go test` directly without `-short`: +Tests and the linter run only in Docker, as the `test` and `lint` phases of the +`Dockerfile`; the image cannot be built unless both pass. Every `docker build` +in `script/` passes a new `CACHEBUST` value, so each run executes every check, +while Go's build cache and golangci-lint's cache are kept between builds. + +The live-network integration test `TestRouteWatchLiveFeed` streams the RIPE RIS +feed for a few seconds and is skipped in short mode. To run it on demand, invoke +`go test` directly without `-short`: `go test -run TestRouteWatchLiveFeed ./internal/routewatch/`. ## License diff --git a/REPO_POLICIES.md b/REPO_POLICIES.md index bc2f161..1067698 100644 --- a/REPO_POLICIES.md +++ b/REPO_POLICIES.md @@ -1,6 +1,6 @@ --- title: Repository Policies -last_modified: 2026-07-06 +last_modified: 2026-10-18 --- This document covers repository structure, tooling, and workflow standards. Code @@ -54,21 +54,37 @@ style conventions are in separate documents: `cibuild`. `script/bootstrap` installs all dependencies idempotently and assumes nothing is present: base tools come from nix, apt, brew, or apk (detected in that order; apt runs noninteractive). For node it uses the - installed node if present; otherwise it installs a PINNED node version via - nvm, first installing nvm itself if missing — from a hash-verified GitHub - release archive (never `curl | sh`), with bash installed as an explicit - prerequisite since nvm requires bash. yarn is then pinned via - `corepack prepare yarn@ --activate`. Never install "latest" or "lts"; - always exact versions. `script/cibuild` runs the CI build: it changes to the - repo root and runs `docker build .`; the Gitea workflow calls it. Four further - scripts are our own extensions to the standard: `script/check` runs - `script/test`, `script/lint`, and `script/fmt-check`; `script/precommit` is - what the git pre-commit hook runs, and it calls `script/check`; - `script/install-precommit` installs the git pre-commit hook (the `make hooks` - target shims to it); and `script/projectname` (literally that filename) simply - outputs the project's name. Scripts that need the name call - `script/projectname` — e.g. `script/docker` assembles its image tag from it — - so those scripts stay byte-identical across all repos. Repo-type-specific + installed node only when its major version is the pinned one; otherwise (node + missing, or another major, such as the node 24 the CI runner image ships) it + installs the PINNED node version via nvm and installs yarn and the packages + under it. nvm itself is installed first if missing, from a hash-verified + GitHub release archive (never `curl | sh`), with bash installed as an explicit + prerequisite since nvm requires bash. Only the major version is compared + because the `Dockerfile` stages start from a node image whose exact version is + not the pin, and nvm cannot install a prebuilt node on alpine. yarn is pinned + via `corepack prepare yarn@ --activate`. Never install "latest" or + "lts"; always exact versions. `script/cibuild` runs the CI build: it changes + to the repo root, runs `script/bootstrap`, runs `script/check`, and builds the + image with the version; the Gitea workflow calls it. **`script/cibuild` runs + `script/bootstrap` first**, because the workflow checks out the repo and runs + nothing else, while `script/fmt-check` runs the formatter on the host: on a + pristine checkout with nothing installed the run dies there, after the + containerised gates have passed. **The bootstrap alone is not enough**: when + `script/bootstrap` installs node and yarn under nvm it leaves neither on the + `PATH` of the shell that called it, so a bare `yarn` either exits 127 or runs + under another node. The host entrypoints that need yarn — `script/fmt` and + `script/fmt-check` — therefore use the same test as `script/bootstrap`: when + the node on `PATH` has the pinned major version they run the `yarn` on `PATH`, + and otherwise they load nvm through `$HOME/.nvm/nvm.sh` and run yarn under the + pinned node. A runner carrying nothing but docker and git then gets through + `script/check`. Four further scripts are our own extensions to the standard: + `script/check` runs `script/test`, `script/lint` and `script/fmt-check`; + `script/precommit` is what the git pre-commit hook runs, and it calls + `script/check`; `script/install-precommit` installs the git pre-commit hook + (the `make hooks` target shims to it); and `script/projectname` (literally + that filename) simply outputs the project's name. Scripts that need the name + call `script/projectname` — e.g. `script/docker` assembles its image tag from + it — so those scripts stay byte-identical across all repos. Repo-type-specific pre-commit extras (e.g. `go mod tidy` verification in Go repos) belong in `script/precommit`, not in the hook itself. Model scripts are at `https://git.eeqj.de/sneak/prompts/raw/branch/main/script/`. The README @@ -89,87 +105,290 @@ style conventions are in separate documents: contributor should be able to understand the entire development workflow by reading the Makefile. -- Every repo should have a `Dockerfile`. All Dockerfiles must run `make check` - as a build step so the build fails if the branch is not green. For non-server - repos, the Dockerfile should bring up a development environment and run - `make check`. For server repos, `make check` should run as an early build - stage before the final image is assembled. Dockerfiles install development - prerequisites by running `script/bootstrap` rather than duplicating installs - inline; COPY `script/` and the dependency manifests (`package.json` + - `yarn.lock`, `go.mod` + `go.sum`, etc.) before running it so the bootstrap - layer stays cached until dependencies change. +- Every repo should have a `Dockerfile`, and it carries the repo's gates: a + `lint` phase and a `test` phase, with the final stage depending on both so the + image cannot be built unless they pass. For non-server repos the final stage + brings up a development environment; for server repos it is the runtime image. + The gate phases and the build stage start from their pinned base images and + install what those images lack either inline, as the canonical Go `Dockerfile` + below does for `git`, or by running `script/bootstrap`, as the `prompts` + repo's own `Dockerfile` does for its yarn packages. The development + environment stage installs development prerequisites by running + `script/bootstrap` rather than duplicating its installs inline. A stage that + runs `script/bootstrap` COPYs `script/` and the dependency manifests + (`package.json` + `yarn.lock`, `go.mod` + `go.sum`, etc.) before running it. -- **Dockerfiles must use a separate lint stage for fail-fast feedback.** Go - repos use a multistage build where linting runs in an independent stage based - on the `golangci/golangci-lint` image (pinned by hash). This stage runs - `make fmt-check` and `make lint` before the full build begins. The build stage - then declares an explicit dependency on the lint stage via - `COPY --from=lint /src/go.sum /dev/null`, which forces BuildKit to complete - linting before proceeding to compilation and tests. This ensures lint failures - surface in seconds rather than minutes, without blocking on dependency - download or compilation in the build stage. +- **Linting and testing run in Docker, as phases of the `Dockerfile`.** There is + no separate lint file. `script/lint` and `script/test` each build one phase + and nothing else: - The standard pattern for a Go repo Dockerfile is: + ```sh + docker build --build-arg CACHEBUST="$(date +%s)-$$" \ + --target lint --output type=cacheonly . + docker build --build-arg CACHEBUST="$(date +%s)-$$" \ + --target test --output type=cacheonly . + ``` + + **A stage that is not the last one in the file is built only when the final + stage's chain depends on it, or when `--target` names it.** That is why the + two gates are always invoked by name here, and why the final stage carries a + `COPY --from=` of a harmless file from each of them: without that edge a + plain `docker build .` builds the last stage alone and exits 0 having linted + and tested nothing. + + **The gate builds write no image.** With `--output type=cacheonly` the phase + runs and a failing step fails the build, but the result is not exported. + Nothing uses those images, and writing one out is slow: a Go test phase's + image holds the toolchain. A build given neither `--output` nor `-t` writes + an untagged image and leaves it dangling, on every developer host and every + CI runner. `script/cibuild` and `script/docker` build the image that ships + and tag it, so each build replaces the previous image; each assigns the tag + on its own line before the build, so `set -e` stops it where + `script/projectname` fails. + + Inside a phase the tool is invoked directly — `golangci-lint`, `go test`, + `eslint`, `prettier` — never through `make lint` or `script/test`, which are + themselves a `docker build` and would recurse into a daemon that does not + exist in a build step. Formatting is the exception and stays on the host: + `script/fmt` writes the working tree, and `script/fmt-check` is its + read-only twin. + + **No lint verdict may come from a host invocation of the linter.** On a + shared host golangci-lint reads a result cache keyed on file content rather + than location, so a second checkout of the same content is served the first + one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs + exit non-zero with `parallel golangci-lint is running` — a status a caller + cannot tell from real findings. Both have produced wrong verdicts in this + org, in both directions. A container has its own `TMPDIR`, so the lock is + out of reach. The lint phase does keep golangci-lint's cache between check + builds (below), but it lints every checkout at `/src`, and a finding is + taken from that cache only for the same files, settings and golangci-lint + version. + +- **Any build that runs checks is given a new `CACHEBUST` value.** Docker + invalidates a `COPY` layer only when the copied content changes, so on an + unchanged tree the check `RUN` is served from cache, nothing executes, and the + build still exits 0. Every `docker build` in `script/` therefore passes + `--build-arg CACHEBUST="$(date +%s)-$$"`, the time and the script's process + id, so no two builds share a value, not even the three in one + `script/cibuild`. `script/lint`, `script/test`, `script/cibuild` and + `script/docker` are the four, and there is no fifth — `script/check` runs the + two gate phases and `script/fmt-check`, and builds no image of its own. In + every stage of the `Dockerfile`, `ARG CACHEBUST` comes right after the steps + that install system packages and download Go's modules, or right after `FROM` + in a stage with neither, and every step that lints, tests or compiles comes + after it. A step that installs anything else as well, such as a + `script/bootstrap` that also installs node packages, comes after it too. A new + value reruns every `RUN` step after that line; `COPY` steps are compared by + content and need nothing. Docker keeps the layers above the line until the + base image, their own lines or the files they copy change. A stage without the + `ARG` line is served from cache on an unchanged tree, so a repository + re-vendors the scripts and adds the `ARG` lines in one commit. The scripts do + not pass `--no-cache`: it reruns every step too, but it also builds the layers + above the line again and empties every cache mount, so Go's build cache and + golangci-lint's, which the canonical Go `Dockerfile` below mounts so that they + survive between check builds, would start empty every time. A bare + `docker build .` is not evidence that anything ran: a sub-second build + reporting success is a cache hit, not a result. Never invalidate by pruning — + `docker builder prune` and friends destroy a build cache shared with every + other build on the host. When a check is added or changed, prove it works by + planting a defect it must catch and watching the run fail on it, then revert + the defect. A green run alone shows neither that the check ran nor that it + covers what it should. + +- **The gate phases are separate stages, and the build stage depends on both.** + The lint phase is based on the `golangci/golangci-lint` image (pinned by + hash), so lint failures surface in seconds rather than after a full compile, + and the test phase is based on the Debian Go image. The canonical Go repo + `Dockerfile`: ```dockerfile - # Lint stage — fast feedback on formatting and lint issues + # In every stage ARG CACHEBUST comes right after the steps that install + # system packages and download Go's modules, or right after FROM in a + # stage with neither, and every step that lints, tests or compiles + # comes after it. script/ passes a new value on every build, which + # reruns every RUN step after it. Docker keeps the layers above it + # until the base image, their own lines or go.mod and go.sum change. + # The steps that compile mount Go's build cache, and golangci-lint run + # also mounts golangci-lint's; both survive from build to build. + + # Lint phase # golangci/golangci-lint:v2.x.x, YYYY-MM-DD FROM golangci/golangci-lint@sha256:... AS lint WORKDIR /src COPY go.mod go.sum ./ RUN go mod download + ARG CACHEBUST COPY . . - RUN make fmt-check - RUN make lint + RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/root/.cache/golangci-lint \ + golangci-lint run --config .golangci.yml ./... - # Build stage - # golang:1.x-alpine, YYYY-MM-DD - FROM golang@sha256:... AS builder + # Test phase. -race needs cgo and so a C compiler, which the Debian Go + # image ships and the alpine one does not. -count=1 because the mounted + # cache also holds test results, which go test would otherwise replay. + # golang:1.x, YYYY-MM-DD + FROM golang@sha256:... AS test WORKDIR /src - - # Force BuildKit to run the lint stage before proceeding - COPY --from=lint /src/go.sum /dev/null - COPY go.mod go.sum ./ RUN go mod download + ARG CACHEBUST COPY . . - RUN make test + RUN --mount=type=cache,target=/root/.cache/go-build \ + go test -count=1 -timeout 90s -race -cover ./... || \ + { echo "--- Rerunning with -v for details ---"; \ + go test -count=1 -timeout 90s -race -v ./...; exit 1; } - ARG VERSION=dev - RUN CGO_ENABLED=0 go build -trimpath \ - -ldflags="-s -w -X main.Version=${VERSION}" \ - -o /app ./cmd/app/ + # Build stage. Nothing is wanted from either phase above; the copies + # are what make BuildKit build them first, so nothing after them can + # run unless lint and test passed. + # golang:1.x-alpine, YYYY-MM-DD + FROM golang@sha256:... AS builder + RUN apk add --no-cache git + WORKDIR /src + COPY go.mod go.sum ./ + RUN go mod download + ARG CACHEBUST + COPY --from=lint /src/go.sum /dev/null + COPY --from=test /src/go.sum /dev/null + # A tar-stream context keeps the sender's file owners, which git refuses. + RUN git config --system --add safe.directory /src + COPY . . - # Runtime stage + # The VERSION build arg when one is given, otherwise + # `git describe --tags --always` on the .git in the build context. With + # .git present, a version that is still empty, dev or unknown fails the + # build: git is missing or could not read the checkout. + ARG VERSION + RUN --mount=type=cache,target=/root/.cache/go-build \ + VERSION="${VERSION:-$(git describe --tags --always)}"; \ + if [ -e .git ]; then \ + case "$VERSION" in ""|dev|unknown) \ + echo "version is '$VERSION' although .git is present" >&2; \ + exit 1 ;; \ + esac; \ + fi; \ + CGO_ENABLED=0 go build -trimpath \ + -ldflags="-s -w -X main.Version=${VERSION}" \ + -o /app ./cmd/app/ + + # Runtime stage, and the last one FROM alpine@sha256:... + ARG CACHEBUST COPY --from=builder /app /usr/local/bin/app ENTRYPOINT ["app"] ``` Key points: - - The lint stage uses the `golangci/golangci-lint` image directly (it - includes both Go and the linter), so there is no need to install the - linter separately. - - `COPY --from=lint /src/go.sum /dev/null` is a no-op file copy that creates - a stage dependency. BuildKit runs stages in parallel by default; without - this line, the build stage would not wait for lint to finish and a lint - failure might not fail the overall build. + - The lint phase uses the `golangci/golangci-lint` image directly (it has + both Go and the linter), so nothing needs installing. + - `COPY --from= /src/go.sum /dev/null` is a no-op copy whose only + purpose is the ordering edge. BuildKit runs stages in parallel by default, + and a stage nothing depends on is not built at all, so without these two + lines a red gate would not fail the build. + - Keep the runtime stage last, and if you add a stage after it, give it the + same two copies. A plain `docker build .` builds the last stage's chain + and nothing else. + - Check builds keep these, and none of them can hand back a stale result: + - The layers above `ARG CACHEBUST`: the system packages and Go's + modules. Docker installs the packages again when the base image or the + install line changes, and downloads the modules again when `go.mod` or + `go.sum` changes. Go checks every module against `go.sum` as it + downloads it, and no later build can change a kept layer. The modules + are kept in a layer rather than a cache mount because a cache mount is + shared by every build on the host, and Go builds from whatever files + it holds without checking them again (only `go mod verify` does), so a + file changed there would go unnoticed. + - Go's build cache (`/root/.cache/go-build`, where Go keeps it for + root), mounted on the steps that compile: `golangci-lint run`, both + `go test` runs and `go build`. Go reuses a compiled package only when + all its inputs are unchanged. The cache also holds test results, which + is why both `go test` runs carry `-count=1`. + - golangci-lint's cache (`/root/.cache/golangci-lint`, where the pinned + image keeps it), mounted on `golangci-lint run`. golangci-lint reuses + a package's findings only when the package's files and those of every + package it imports, `go.mod`, the linters and their settings in + `.golangci.yml`, and the golangci-lint version are all unchanged. - If the project uses `//go:embed` directives that reference build artifacts - (e.g. a web frontend compiled in a separate stage), the lint stage must + (e.g. a web frontend compiled in a separate stage), the lint phase must create placeholder files so the embed directives resolve. Example: `RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`. - The lint stage should not depend on the actual build output — it exists to - fail fast. - - If the project requires CGO or system libraries for linting (e.g. - `vips-dev`), install them in the lint stage with `apk add`. - - The build stage runs `make test` after compilation setup. Tests run in the - build stage, not the lint stage, because they may require compiled - artifacts or heavier dependencies. + - If the project requires CGO or system libraries for linting, install them + in the lint phase. The `golangci/golangci-lint` image is Debian-based and + has no `apk`, so install with `apt-get` under the Debian package name + (`libvips-dev`, where alpine says `vips-dev`), and delete the package + lists in the same `RUN`, so the layer does not keep them. The test phase, + on the Debian Go image, installs the libraries its tests need the same + way. Either install goes right after `FROM`, above `ARG CACHEBUST`, so + Docker keeps it until the base image or the install line changes: + + ```dockerfile + FROM golangci/golangci-lint@sha256:... AS lint + RUN apt-get update \ + && apt-get install -y --no-install-recommends libvips-dev \ + && rm -rf /var/lib/apt/lists/* + WORKDIR /src + COPY go.mod go.sum ./ + RUN go mod download + ARG CACHEBUST + ``` + + - `.dockerignore` lets `.git` into the build context. It keeps out every git + `config` at any depth (`**/.git/config`, `**/.git/modules/**/config`): the + repository's own, each submodule's under `.git/modules/`, and that of a + submodule keeping its own `.git` directory. `git describe` does not need + them, and each can hold a credential: a password in a remote URL, or the + token the CI checkout step stores there. A submodule whose name has a + `config` segment (`config`, `deploy/config`, `config/lib`) loses its whole + git directory to `**/.git/modules/**/config`, and Go's version stamping + then fails the build: give it a name without that segment + (`git submodule add --name`). The stage that compiles has `git` (the + Debian Go image has it; an alpine one needs `apk add --no-cache git`) and + takes the version from the `VERSION` build argument when one is given, + otherwise from `git describe --tags --always`. That gives the tag on a + tagged commit; on a later commit, the tag, the number of commits since it + and the short commit (`v1.2.3-4-gabc1234`); and the short commit when no + tag is reachable. The stage that compiles also marks its working directory + safe for git (`git config --system --add safe.directory /src`): a context + sent as a tar stream keeps the sender's file owners, and git refuses a + checkout owned by another user, so the version would come out empty. + `ARG VERSION` has no default, and the build fails if the context carries + `.git` and the version still comes out empty, `dev` or `unknown`. A plain + `docker build .` with no build arguments must succeed; a Dockerfile that + refuses an empty build argument drops that refusal and keeps the argument. + A checkout whose `.git` is a file (a linked worktree, or a repository + checked out as a submodule) is the exception: that file points to a git + directory outside the build context, so the build cannot read the version + and a plain `docker build .` fails; pass the version with + `--build-arg VERSION=...`, as `script/docker` and `script/cibuild` already + do. - Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that - runs `script/cibuild` (which runs `docker build .`) on push. Since the - Dockerfile already runs `make check`, a successful build implies all checks - pass. + runs `script/cibuild` on push, and checks out the repo as its only other step, + with `persist-credentials: false`: `script/cibuild` needs no token, and + without it the checkout leaves the job's token in `.git/config` for every + later step. The checkout step also sets `fetch-depth: 0`, which fetches the + tags `git describe` needs: by default it clones shallow with no tags, and a + tagged repository's CI build would stamp a bare short commit id. The + workflow's `concurrency` block groups runs by workflow and branch + (`${{ github.workflow }}-${{ github.ref }}`) with `cancel-in-progress: true`, + so a new push cancels the older run on the same branch, queued or running, and + no other: runs for replaced commits do not hold up the shared runner. + `script/cibuild` bootstraps, runs the gate phases, and then builds the image, + so a successful run means every check passed; a bare `docker build .` does not + carry the same guarantee, because its gate phases may come from the cache. The + image build gets a `CACHEBUST` value of its own and so runs the gate phases a + second time. That is the price of the rule above, and it is worth paying: the + image that ships is built from a run of its own gates rather than from a cache + entry. The `check` job sets `timeout-minutes: 20`, so a hung build frees the + shared runner after 20 minutes. That allows for the three Docker builds + described above (the test phase, the lint phase, then the image), each held to + the 5-minute Docker build limit below, plus the bootstrap. A separate workflow + limited to `main` by a `branches` list under `on: push` cannot be checked by + review: to try a change to it, add the feature branch to that list and push, + then remove the branch from the list again before merging. Keep any job in it + that publishes behind `if: github.ref_name == 'main'`, so the run from the + feature branch publishes nothing. - Use platform-standard formatters: `black` for Python, `prettier` for JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with @@ -189,14 +408,21 @@ style conventions are in separate documents: module under test to verify it compiles/parses. There is no excuse for `make test` to be a no-op. -- `make test` must complete in under 20 seconds. Add a 30-second timeout in the - Makefile. +- `make test` must complete in under 60 seconds. That is the hard cap, and a + suite that exceeds it fails. Under 20 seconds is the target. A suite between + 20 and 60 seconds is still green, but the overage must be filed as an + improvement bug against that repo. Add a 90-second timeout to the test + invocation (`go test -timeout 90s`). The backstop deliberately sits above the + hard cap so that it catches a genuinely hung test rather than a merely slow + one. -- **`make test` should use the conditional verbose rerun pattern.** Run tests - without `-v` (verbose) first. If tests fail, automatically rerun with `-v` to - show full output. This keeps CI logs and `docker build` output clean on - success (just package/suite summaries) while providing full diagnostic detail - on failure (every test case, every assertion). The general shell pattern: +- **The test command should use the conditional verbose rerun pattern.** Run + tests without `-v` (verbose) first. If tests fail, automatically rerun with + `-v` to show full output. This keeps CI logs and `docker build` output clean + on success (just package/suite summaries) while providing full diagnostic + detail on failure (every test case, every assertion). The command lives in the + `test` phase of the `Dockerfile`, since `script/test` builds that phase; the + Makefile form below is the same pattern for any repo-local invocation: ```makefile test: @@ -209,11 +435,25 @@ style conventions are in separate documents: ```makefile test: - @go test -timeout 30s -race -cover ./... || \ + @go test -count=1 -timeout 90s -race -cover ./... || \ { echo "--- Rerunning with -v for details ---"; \ - go test -timeout 30s -race -v ./...; exit 1; } + go test -count=1 -timeout 90s -race -v ./...; exit 1; } ``` + `-count=1` is required on both invocations: it defeats Go's test _result_ + cache, so neither run can report a stored pass in place of running the + tests. It leaves the build cache alone, so it costs the runtime of the suite + and no recompilation. + + That cache is Go's own, separate from Docker's layer cache. Go stores a + passing result in its cache directory (`GOCACHE`), and when the same tests + run again on unchanged code it prints that result, marked `(cached)`, + without running them. The directory lasts from one run to the next on a + developer's machine, where this target runs, and in the `test` phase of the + `Dockerfile`, which mounts it to keep compiled packages between check + builds, so the phase's `go test` runs carry `-count=1` too. A new + `CACHEBUST` value (above) is what makes that step run on an unchanged tree. + Python example: ```makefile @@ -239,10 +479,89 @@ style conventions are in separate documents: must be in `.gitignore`. No exceptions. - `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`), - editor files (`.swp`, `*~`), language build artifacts, and `node_modules/`. - Fetch the standard `.gitignore` from + editor files (`.swp`, `*~`), in-repo agent scratch directories (`.claude/`), + `node_modules/`, and the repo's own build outputs. Fetch the standard + `.gitignore` from `https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when setting up - a new repo. + a new repo. A repo's `.gitignore` is the standard file followed by the repo's + own entries, such as its binaries; a re-vendor replaces the standard part and + keeps those entries. These patterns are written to `.gitignore`'s own + semantics, in which an unanchored pattern already matches at every depth; they + are not a `.dockerignore` and must not be transplanted into one unmodified. + +- **`.dockerignore` does not use `.gitignore` semantics, and copying patterns + across unmodified leaves secrets in the build context.** Docker matches with + `moby/patternmatcher`: `filepath.Match` semantics plus a `**` extension, so + `*` does not cross `/` and a pattern without a leading `**/` is anchored at + the build-context root. A `.dockerignore` listing `.env`, `*.pem` and `*.key` + therefore excludes only the copies at the repository root, while `config/.env` + and `certs/server.key` still reach the context and can land in an image layer + — which is more dangerous than a short file with no secret patterns at all, + because it reads as solved and stops anyone looking. Give every + depth-independent pattern the `**/` prefix and leave only genuinely + root-anchored entries unprefixed: `.claude`, and the repo's own host-built + binary, written `/myapp` and never `**/myapp`, which would also match + `cmd/myapp/` and delete the package directory from the context. Matching is + case-sensitive, and an ALL-CAPS twin per pattern still misses `Server.Key`, so + secret names use character ranges — `**/*.[kK][eE][yY]`, `**/*.[pP][eE][mM]`, + and likewise for `.envrc` and the extensionless SSH keys. Where such a pattern + also catches something the build needs, re-include it with a negation + (`!docs/example.env`); deleting the pattern reopens the exposure for every + other file it covers. Fetch the standard `.dockerignore` from + `https://git.eeqj.de/sneak/prompts/raw/branch/main/.dockerignore` and extend + it with the repo's own artifacts. + +- **In-repo agent scratch belongs in both files, written to each file's own + semantics.** `.claude/` holds one worktree per in-flight agent — an entire + additional checkout of the repo — so under `COPY . .` the build context + inflates by a multiple of the repo and another session's unreviewed work can + be copied into an image layer. In `.gitignore` the entry is `.claude/`, + unanchored. In `.dockerignore` it is `.claude`, anchored and with **no** `**/` + prefix, because the prefixed form would also delete any nested directory of + that name from the build. Anchoring carries a known gap that the canonical + `.dockerignore` states in its own comment, since consuming repos receive the + file and not the tracker: the directory is created in the agent's working + directory, so a repo running agents in subdirectories still ships + `services/api/.claude/` and must add its own anchored entry there. + +- **A plain `docker build .` of a clone stamps the version that + `git describe --tags --always` gives**, derived from the `.git` in the build + context as the canonical `Dockerfile` above shows. Without its failure check, + a missing `git` or an unreadable checkout would leave `-X main.Version=` empty + and the build would still exit 0. `script/docker` and `script/cibuild` pass + the version they compute on the host; it takes precedence. They do this + byte-identically across repos: + + ```sh + # The version and the tag each get their own line: a failing command + # substitution inside an argument does not trip `set -e`, so the inline + # form degrades to an empty constant. + version="$(git describe --tags --always --dirty 2>/dev/null || true)" + [ -n "$version" ] || version="unknown" + tag="$(script/projectname)" + docker build --build-arg CACHEBUST="$(date +%s)-$$" \ + --build-arg VERSION="$version" \ + -t "$tag" . + ``` + + `--always` makes an untagged repo yield an abbreviated commit hash rather + than failing, and the `[ -n "$version" ]` line is the single place the + fallback is applied — a live check that fires on a build from an export with + no `.git` and on a repository with no commits yet. Do not fold it into the + substitution as `|| echo unknown`, which makes the guard unreachable. The + Dockerfile's side is `ARG VERSION` in the stage that compiles, declared + there because `ARG` is stage-scoped; passing `VERSION` to a repo whose + Dockerfile declares no such `ARG` is ignored and costs nothing, which is why + the scripts stay byte-identical. One consequence for CI: the standard + checkout action clones shallow and fetches no tags, so the canonical + `.gitea/workflows/check.yml` sets `fetch-depth: 0` on its checkout step. + +- **Verify `.dockerignore` by enumerating the image, not by reading the + patterns.** Plant files at the root _and_ at least two directories deep, build + a probe image that does `COPY . .`, and list what actually landed + (`docker run --rm --entrypoint find IMAGE /app`). The `transferring context` + size is not a substitute: a nested secret is a few bytes, and BuildKit + transfers only the delta from the previous build. - **No build artifacts in version control.** Code-derived data (compiled bundles, minified output, generated assets) must never be committed to the @@ -258,9 +577,61 @@ style conventions are in separate documents: - Make all changes on a feature branch. You can do whatever you want on a feature branch. -- `.golangci.yml` is standardized and must _NEVER_ be modified by an agent, only - manually by the user. Fetch from - `https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml`. +- `.golangci.yml` is standardized. The vendored copy in a consuming repo must + _NEVER_ be modified by an agent: fetch it from + `https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml` and keep it + byte-identical, so that no repo can quietly loosen its own linting. Linter + configuration changes are made to the canonical copy in the `prompts` repo and + reach consuming repos by re-vendoring; an agent may open a PR against + canonical, which only the user merges. One list is exempt from byte-identity, + because it cannot be written once for every repo: the `deny` list of the + `test-support` depguard rule, where a repo names its own test-support packages + by full import path. A repo adds entries there and changes nothing else, and a + re-vendor carries its entries forward. The canonical golangci-lint version is + v2.14.0 (released 2026-09-24), pinned as the digest of the lint phase's base + image + (`golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f`, + which reports `2.14.0 built with go1.27.0 from 114493f9`). A module's `go` + directive must not name a newer Go minor version than the one golangci-lint + was built with, or golangci-lint refuses to lint it: this release lints + `go 1.27.1` but not `go 1.28`. That digest is the only pin, since no repo + installs golangci-lint on the host. A repo sets the lint phase digest to the + one named here and re-vendors `.golangci.yml` in the same commit, whichever of + the two prompted the change: the canonical copy can name linters that an older + golangci-lint rejects, and a newer golangci-lint can add linters that + `default: all` switches on until the canonical copy disables them. + +- **`script/bootstrap` installs a pinned tool by comparing versions, never by + testing presence.** An `if ! command -v ; then install; fi` guard tests + `PATH` only, so on an already-provisioned machine the pin is inert and a + version bump is a silent no-op — while the Dockerfile, installing into a clean + image, gets the pinned version, so a local `make check` and `make docker` can + disagree about what the tool even is. The canonical form: + - compares the installed version against the pin over the **whole** version + token; a parser that stops at the first `-` reports `2.12.2` for a host + running `2.12.2-rc1` and skips the install; + - treats absent, non-zero, empty or unrecognised `--version` output as a + mismatch, so the failure direction is a redundant install and never a + skipped one; + - after installing, re-resolves the binary the way callers do — `hash -r`, + then through `PATH`, not through the directory the installer wrote to — + and fails naming the resolved path, since an install that a shadowing + binary hides succeeds while changing nothing any caller sees; + - is actually called, and prints the version on both success paths: a + function defined and never invoked has the same exit status and the same + empty output as one that worked. + + Node is the exception to the whole-token comparison: the canonical + `script/bootstrap` compares only its major version, because the node 22 + alpine image the `Dockerfile` stages start from is not the exact pin, and + nvm cannot install a prebuilt node on alpine. + + Keep it POSIX sh: no arrays, no `[[`, no `grep -P`. + + A Go tool a repo needs on the host is installed with `go install` pinned to + a commit hash (`go install @`). It is never tracked as + a `go.mod` tool dependency or through a `tools.go` file, either of which + pulls the tool's own dependencies into the repo's `go.mod` and `go.sum`. - When pinning images or packages by hash, add a comment above the reference with the version and date (YYYY-MM-DD). @@ -371,15 +742,21 @@ style conventions are in separate documents: Never edit existing migrations after release. - All repos should have an `.editorconfig` enforcing the project's indentation - settings. + settings: the standard file from + `https://git.eeqj.de/sneak/prompts/raw/branch/main/.editorconfig`, which sets + tabs for `Makefile` and Go files, followed by the repo's own sections, such as + one for another language it uses. A re-vendor replaces the standard part and + keeps those sections. - Avoid putting files in the repo root unless necessary. Root should contain - only project-level config files (`README.md`, `Makefile`, `Dockerfile`, - `LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, and - language-specific config). Everything else goes in a subdirectory. Canonical - subdirectory names: + only project-level config files (`README.md`, `AGENTS.md`, `Makefile`, + `Dockerfile`, `LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, + and language-specific config). Everything else goes in a subdirectory. + Canonical subdirectory names: - `bin/` — executable scripts and tools - - `cmd/` — Go command entrypoints + - `cmd/` — Go command entrypoints; thin only: one `main.go` per binary whose + body is a single call into `internal/` or `pkg/`, no project logic in + `cmd/` - `configs/` — configuration templates and examples - `deploy/` — deployment manifests (k8s, compose, terraform) - `docs/` — documentation and markdown (README.md stays in root) @@ -406,3 +783,7 @@ style conventions are in separate documents: - Go: `go.mod`, `go.sum`, `.golangci.yml` - JS: `package.json`, `yarn.lock`, `.prettierrc`, `.prettierignore` - Python: `pyproject.toml` + +- Guidance for coding agents lives in one `AGENTS.md` at the repository root. It + is never committed under a file or directory named after one agent tool, such + as `CLAUDE.md` or `.claude/`, and never split into separate memory files. diff --git a/TODO.md b/TODO.md index d0b7920..ade4f7b 100644 --- a/TODO.md +++ b/TODO.md @@ -1,122 +1,121 @@ # Workflow -* branch (from `main`) -* do the work in Next Step -* move Next Step to the top of Completed Steps -* move the top item of Future Steps into Next Step -* commit (`TODO.md` changes in the same commit as the work) -* merge to `main` if the branch is not protected, otherwise open a PR -* push +- branch (from `main`) +- do the work in Next Step +- move Next Step to the top of Completed Steps +- move the top item of Future Steps into Next Step +- commit (`TODO.md` changes in the same commit as the work) +- merge to `main` if the branch is not protected, otherwise open a PR +- push # Status -pre-1.0. No git tags. The Docker build runs the format check, the linter -and the tests, and the Gitea workflow runs that build on every push. The -image sets memory ceilings for a 5 GiB container (README "Memory") and the -README says how to run it under upaas (README "Running under upaas"). A -35-hour run of `3898daa` on the live feed peaked at about 1 GiB, without a -container memory limit. +pre-1.0. No git tags. The linter and the tests are phases of the Docker build, +and the Gitea workflow runs `script/cibuild` (the format check of the Go code +and the markdown, both phases, then the image) on every push. The image sets +memory ceilings for a 5 GiB container (README "Memory") and the README says how +to run it under upaas (README "Running under upaas"). A 35-hour run of `3898daa` +on the live feed peaked at about 1 GiB, without a container memory limit. # Next Step `next` waits for sneak to merge it to `main` through -https://git.eeqj.de/sneak/routewatch/pulls/6. After that, setting -routewatch up under upaas on fsn1app1 and deploying it are his -(https://git.eeqj.de/sneak/routewatch/issues/31), and so is the run under a -real 5 GiB limit (https://git.eeqj.de/sneak/routewatch/issues/3). +https://git.eeqj.de/sneak/routewatch/pulls/6. After that, setting routewatch up +under upaas on fsn1app1 and deploying it are his +(https://git.eeqj.de/sneak/routewatch/issues/31), and so is the run under a real +5 GiB limit (https://git.eeqj.de/sneak/routewatch/issues/3). # Completed Steps -- 2026-10-03: `/api/v1/stats` serves the prefix distribution from memory, - seeded at startup and adjusted on every live-route write, so a request no - longer reads every live route (closes #30) -- 2026-10-03: a new database is created with `auto_vacuum` set to - incremental, through the connection string so it is set before the file - is first written, and each periodic incremental vacuum now returns up to - 1000 free pages; the late `PRAGMA auto_vacuum` in `Initialize`, which - SQLite ignored, is gone (closes #43) -- 2026-10-03: looking up an IP address no longer reads every IPv6 route: - both families find the most specific live route with at most 33 or 129 - lookups on the prefix index, and the IPv4 range columns are gone. Prefixes - from the feed are stored in one text form, so an IPv6 withdrawal, which the - feed sends uncompressed, now removes its route (closes #48) -- 2026-10-02: a plain `docker build .` stamps the commit's tag or short - commit (`git describe --tags --always`) into the page footer instead of - `unknown`: `.dockerignore` sends `.git` without `.git/config`, a `VERSION` - build argument takes precedence, and `make build` stamps the same value - (closes #46) +- 2026-10-19: the shared files are re-vendored from `sneak/prompts` at + `0653579`: golangci-lint v2.14.0 and its configuration, lint and test as + `Dockerfile` phases that keep Go's and golangci-lint's caches, markdown + formatted with prettier, agent guidance in `AGENTS.md`; the code is fixed for + what the new linter configuration finds (closes #52) +- 2026-10-03: `/api/v1/stats` serves the prefix distribution from memory, seeded + at startup and adjusted on every live-route write, so a request no longer + reads every live route (closes #30) +- 2026-10-03: a new database is created with `auto_vacuum` set to incremental, + through the connection string so it is set before the file is first written, + and each periodic incremental vacuum now returns up to 1000 free pages; the + late `PRAGMA auto_vacuum` in `Initialize`, which SQLite ignored, is gone + (closes #43) +- 2026-10-03: looking up an IP address no longer reads every IPv6 route: both + families find the most specific live route with at most 33 or 129 lookups on + the prefix index, and the IPv4 range columns are gone. Prefixes from the feed + are stored in one text form, so an IPv6 withdrawal, which the feed sends + uncompressed, now removes its route (closes #48) +- 2026-10-02: a plain `docker build .` stamps the commit's tag or short commit + (`git describe --tags --always`) into the page footer instead of `unknown`: + `.dockerignore` sends `.git` without `.git/config`, a `VERSION` build argument + takes precedence, and `make build` stamps the same value (closes #46) - 2026-09-29: the entrypoint creates the data directory if it is missing and - stops the start if a step fails; README "Running under upaas" no longer - asks for the host directory to be created first (closes #42) -- 2026-09-29: `.dockerignore` keeps `.git`, local build output, local - databases and `.env` out of the Docker build context, and so out of the - source archive in the image (closes #39) -- 2026-09-29: README first line names the MIT license and the author; - the License section now says MIT and links `LICENSE`, and an Author - section was added; this file brought up to date (closes #38) + stops the start if a step fails; README "Running under upaas" no longer asks + for the host directory to be created first (closes #42) +- 2026-09-29: `.dockerignore` keeps `.git`, local build output, local databases + and `.env` out of the Docker build context, and so out of the source archive + in the image (closes #39) +- 2026-09-29: README first line names the MIT license and the author; the + License section now says MIT and links `LICENSE`, and an Author section was + added; this file brought up to date (closes #38) - 2026-09-29: MIT `LICENSE` (closes #1) -- 2026-09-28: stopping the daemon while the feed is flowing no longer - panics with "send on closed channel": the read loop checks for a stop - just before handing a message to the handler queues, and a second - `Stop` no longer closes the queues again (closes #34) -- 2026-09-28: `docker stop` no longer kills the daemon 2 seconds after the - stop signal: the entrypoint switches to the `routewatch` user with - `setpriv` instead of `runuser`, so the daemon receives the signal itself - and gets the whole wait `docker stop` allows, up to its own 60-second - limit (closes #33) +- 2026-09-28: stopping the daemon while the feed is flowing no longer panics + with "send on closed channel": the read loop checks for a stop just before + handing a message to the handler queues, and a second `Stop` no longer closes + the queues again (closes #34) +- 2026-09-28: `docker stop` no longer kills the daemon 2 seconds after the stop + signal: the entrypoint switches to the `routewatch` user with `setpriv` + instead of `runuser`, so the daemon receives the signal itself and gets the + whole wait `docker stop` allows, up to its own 60-second limit (closes #33) - 2026-09-28: ready to run under upaas: a set but invalid `PORT`, - `XDG_DATA_HOME` or `MALLOC_ARENA_MAX` stops the start, the health - check follows `PORT`, README "Running under upaas" section (closes - #31) -- 2026-09-22: realtime in-memory database statistics: counts seeded at - startup and adjusted on every write, oldest/newest route timestamps via - index-end lookups; `/api/v1/stats` no longer scans the tables (closes - #27) -- 2026-09-21: batch writes take the write lock when their transaction - begins (`_txlock=immediate`), so they wait out a WAL checkpoint instead - of failing with "database is locked" (closes #25) -- 2026-09-21: `MALLOC_ARENA_MAX=2` in the image caps glibc malloc arenas, - so memory outside the Go runtime no longer grows with the core count - (closes #23) -- 2026-09-21: `GOMEMLIMIT=1536MiB` in the image; README Memory section - with the memory budget and the 5 GiB container limit (closes #13) -- 2026-09-21: the four handler queues hold at most 20,000 messages each, - down from 100,000 (closes #11) -- 2026-09-21: two goroutine leaks fixed: the stats handlers after a - timeout and the streamer's tickers on every reconnect (closes #12) -- 2026-09-21: parsed RIS messages no longer keep the unused `Community` - and `Raw` fields (closes #9) -- 2026-09-21: `.editorconfig`, and a Gitea workflow that runs - `script/cibuild` on every push (closes #14) -- 2026-09-21: the peering handler's AS-path map holds at most 500,000 - paths and is swapped for an empty one every 30 seconds instead of copied - (closes #10) -- 2026-09-21: SQLite memory bounded across the whole connection pool: a - 64 MiB page cache on each connection, 1 GiB soft and 1.5 GiB hard heap - limits (closes #8) -- 2026-09-21: the Docker build runs the format check, the linter and the - tests, linting in a separate stage on a golangci-lint image pinned by - digest (closes #5) + `XDG_DATA_HOME` or `MALLOC_ARENA_MAX` stops the start, the health check + follows `PORT`, README "Running under upaas" section (closes #31) +- 2026-09-22: realtime in-memory database statistics: counts seeded at startup + and adjusted on every write, oldest/newest route timestamps via index-end + lookups; `/api/v1/stats` no longer scans the tables (closes #27) +- 2026-09-21: batch writes take the write lock when their transaction begins + (`_txlock=immediate`), so they wait out a WAL checkpoint instead of failing + with "database is locked" (closes #25) +- 2026-09-21: `MALLOC_ARENA_MAX=2` in the image caps glibc malloc arenas, so + memory outside the Go runtime no longer grows with the core count (closes #23) +- 2026-09-21: `GOMEMLIMIT=1536MiB` in the image; README Memory section with the + memory budget and the 5 GiB container limit (closes #13) +- 2026-09-21: the four handler queues hold at most 20,000 messages each, down + from 100,000 (closes #11) +- 2026-09-21: two goroutine leaks fixed: the stats handlers after a timeout and + the streamer's tickers on every reconnect (closes #12) +- 2026-09-21: parsed RIS messages no longer keep the unused `Community` and + `Raw` fields (closes #9) +- 2026-09-21: `.editorconfig`, and a Gitea workflow that runs `script/cibuild` + on every push (closes #14) +- 2026-09-21: the peering handler's AS-path map holds at most 500,000 paths and + is swapped for an empty one every 30 seconds instead of copied (closes #10) +- 2026-09-21: SQLite memory bounded across the whole connection pool: a 64 MiB + page cache on each connection, 1 GiB soft and 1.5 GiB hard heap limits (closes + #8) +- 2026-09-21: the Docker build runs the format check, the linter and the tests, + linting in a separate stage on a golangci-lint image pinned by digest (closes + #5) - 2026-09-21: `make test` skips the live-network feed test (`-short`), so `make check` no longer depends on the network (closes #2) -- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints, - Makefile shims, README Entrypoints section -- 2026-02-22: repo policy compliance: required policy files, .gitignore - update, Makefile fmt-check/check/docker/hooks targets, gofmt pass +- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints, Makefile + shims, README Entrypoints section +- 2026-02-22: repo policy compliance: required policy files, .gitignore update, + Makefile fmt-check/check/docker/hooks targets, gofmt pass (repo-policies-compliance, unmerged) - 2026-01-01: WAL checkpointing: periodic (5s), on startup with logging, TRUNCATE mode; fixed slow queries -- 2025-12-31: status page: navbar, home page with search, oldest/newest - route timestamps, NULL handling in WHOIS stats; container runs as - routewatch user with proper state dir -- 2025-12-30: structured HTTP request logging, increased timeouts, - CIDR prefix URL routing fixes, status page metrics and footer +- 2025-12-31: status page: navbar, home page with search, oldest/newest route + timestamps, NULL handling in WHOIS stats; container runs as routewatch user + with proper state dir +- 2025-12-30: structured HTTP request logging, increased timeouts, CIDR prefix + URL routing fixes, status page metrics and footer - 2025-12-29: Dockerfile multi-stage build with source archive; SQLite incremental vacuum for non-blocking space reclamation - 2025-07-27: initial RouteWatch BGP stream monitor # Future Steps -- Production memory under 5 GiB: whether to test under a real 5 GiB - container limit on fsn1app1 is open for sneak +- Production memory under 5 GiB: whether to test under a real 5 GiB container + limit on fsn1app1 is open for sneak (https://git.eeqj.de/sneak/routewatch/issues/3) diff --git a/cmd/asinfo-gen/main.go b/cmd/asinfo-gen/main.go index 1d895b9..fbc0a50 100644 --- a/cmd/asinfo-gen/main.go +++ b/cmd/asinfo-gen/main.go @@ -2,8 +2,10 @@ package main import ( + "context" "encoding/csv" "encoding/json" + "errors" "io" "log" "net/http" @@ -27,13 +29,17 @@ func main() { // Fetch the CSV data log.Println("Fetching AS CSV data...") - resp, err := http.Get(asCsvURL) + + req, err := http.NewRequestWithContext( + context.Background(), http.MethodGet, asCsvURL, nil) + if err != nil { + log.Fatalf("Failed to fetch CSV: %v", err) + } + + resp, err := http.DefaultClient.Do(req) if err != nil { log.Fatalf("Failed to fetch CSV: %v", err) } - defer func() { - _ = resp.Body.Close() - }() if resp.StatusCode != http.StatusOK { log.Fatalf("Failed to fetch CSV: HTTP %d", resp.StatusCode) @@ -48,17 +54,20 @@ func main() { log.Fatalf("Failed to read CSV header: %v", err) } - if len(header) != 3 || header[0] != "asn" || header[1] != "handle" || header[2] != "description" { + if len(header) != 3 || header[0] != "asn" || header[1] != "handle" || + header[2] != "description" { log.Fatalf("Unexpected CSV header: %v", header) } // Read all records var asInfos []ASInfo + for { record, err := reader.Read() - if err == io.EOF { + if errors.Is(err, io.EOF) { break } + if err != nil { log.Printf("Error reading CSV record: %v", err) @@ -86,12 +95,18 @@ func main() { }) } + // Closed here rather than deferred: log.Fatalf exits without running + // deferred calls. + _ = resp.Body.Close() + log.Printf("Parsed %d AS records", len(asInfos)) // Convert to JSON and output to stdout encoder := json.NewEncoder(os.Stdout) encoder.SetIndent("", "\t") - if err := encoder.Encode(asInfos); err != nil { + + err = encoder.Encode(asInfos) + if err != nil { log.Fatalf("Failed to encode JSON: %v", err) } } diff --git a/cmd/streamdumper/main.go b/cmd/streamdumper/main.go index 7754d7a..95a0722 100644 --- a/cmd/streamdumper/main.go +++ b/cmd/streamdumper/main.go @@ -26,9 +26,17 @@ func main() { // Register raw message handler that prints to stdout s.RegisterRawHandler(func(line string) { - fmt.Println(line) + _, _ = fmt.Fprintln(os.Stdout, line) }) + // Start streaming before any defer: log.Fatal exits without running + // deferred calls. + err := s.Start(context.Background()) + if err != nil { + log.Fatal("Failed to start streamer:", err) + } + defer s.Stop() + // Set up context with cancellation ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -36,18 +44,13 @@ func main() { // Handle signals sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) + go func() { <-sigChan log.Println("Received shutdown signal") cancel() }() - // Start streaming - if err := s.Start(); err != nil { - log.Fatal("Failed to start streamer:", err) - } - defer s.Stop() - // Wait for context cancellation <-ctx.Done() } diff --git a/internal/config/config.go b/internal/config/config.go index 87a6ce3..efb290f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -2,6 +2,7 @@ package config import ( + "errors" "fmt" "os" "path/filepath" @@ -27,6 +28,12 @@ const ( maxPort = 65535 ) +var ( + errRelativeXDGDataHome = errors.New("XDG_DATA_HOME must be an absolute path") + errUnsupportedOS = errors.New("unsupported operating system") + errInvalidPort = errors.New("PORT must be a whole number") +) + // Config holds configuration for the entire application type Config struct { // StateDir is the directory for all application state (database, snapshots) @@ -38,11 +45,14 @@ type Config struct { // MaxRuntime is the maximum runtime (0 = run forever) MaxRuntime time.Duration - // EnableBatchedDatabaseWrites enables batched database operations for better performance + // EnableBatchedDatabaseWrites enables batched database operations for + // better performance EnableBatchedDatabaseWrites bool - // RouteExpirationTimeout is how long a route can go without being refreshed before expiring - // Default is 2 hours which is conservative for BGP (typical BGP hold time is 90-180 seconds) + // RouteExpirationTimeout is how long a route can go without being + // refreshed before expiring + // Default is 2 hours which is conservative for BGP (typical BGP hold + // time is 90-180 seconds) RouteExpirationTimeout time.Duration } @@ -59,11 +69,14 @@ func New() (*Config, error) { } return &Config{ - StateDir: stateDir, - Port: port, - MaxRuntime: 0, // Run forever by default - EnableBatchedDatabaseWrites: true, // Enable batching by default - RouteExpirationTimeout: defaultRouteExpirationMinutes * time.Minute, // For active route monitoring + StateDir: stateDir, + Port: port, + // Run forever by default + MaxRuntime: 0, + // Enable batching by default + EnableBatchedDatabaseWrites: true, + // For active route monitoring + RouteExpirationTimeout: defaultRouteExpirationMinutes * time.Minute, }, nil } @@ -89,7 +102,7 @@ func getStateDirectory() (string, error) { // the database somewhere unexpected. xdgData := os.Getenv("XDG_DATA_HOME") if xdgData != "" && !filepath.IsAbs(xdgData) { - return "", fmt.Errorf("XDG_DATA_HOME must be an absolute path, got %q", xdgData) + return "", fmt.Errorf("%w, got %q", errRelativeXDGDataHome, xdgData) } // Unix-like: /var/lib/berlin.sneak.app.routewatch if root, else XDG_DATA_HOME @@ -111,7 +124,7 @@ func getStateDirectory() (string, error) { return filepath.Join(home, ".local", "share", AppIdentifier), nil default: - return "", fmt.Errorf("unsupported operating system: %s", runtime.GOOS) + return "", fmt.Errorf("%w: %s", errUnsupportedOS, runtime.GOOS) } } @@ -125,7 +138,8 @@ func getPort() (int, error) { // ParseUint, unlike Atoi, refuses a sign: the health check URL cannot use "+9090" port, err := strconv.ParseUint(value, 10, 0) if err != nil || port < 1 || port > maxPort { - return 0, fmt.Errorf("PORT must be a whole number from 1 to %d, got %q", maxPort, value) + return 0, fmt.Errorf("%w from 1 to %d, got %q", + errInvalidPort, maxPort, value) } return int(port), nil @@ -134,7 +148,8 @@ func getPort() (int, error) { // EnsureDirectories creates all necessary directories if they don't exist func (c *Config) EnsureDirectories() error { // Ensure state directory exists - if err := os.MkdirAll(c.StateDir, dirPermissions); err != nil { + err := os.MkdirAll(c.StateDir, dirPermissions) + if err != nil { return fmt.Errorf("failed to create state directory: %w", err) } diff --git a/internal/config/config_test.go b/internal/config/config_internal_test.go similarity index 91% rename from internal/config/config_test.go rename to internal/config/config_internal_test.go index 84f9414..c9f10bd 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_internal_test.go @@ -36,7 +36,8 @@ func TestNewRefusesInvalidPort(t *testing.T) { t.Setenv("PORT", value) t.Setenv("XDG_DATA_HOME", "") - if _, err := New(); err == nil { + _, err := New() + if err == nil { t.Errorf("New() with PORT=%q returned no error", value) } }) @@ -51,12 +52,16 @@ func TestNewRefusesRelativeXDGDataHome(t *testing.T) { t.Setenv("PORT", "") t.Setenv("XDG_DATA_HOME", "relative/path") - if _, err := New(); err == nil { + + _, err := New() + if err == nil { t.Error("New() with a relative XDG_DATA_HOME returned no error") } t.Setenv("XDG_DATA_HOME", "/var/lib") - if _, err := New(); err != nil { + + _, err = New() + if err != nil { t.Errorf("New() with XDG_DATA_HOME=/var/lib: %v", err) } } diff --git a/internal/database/counts.go b/internal/database/counts.go index 71ce82f..5c8d4a5 100644 --- a/internal/database/counts.go +++ b/internal/database/counts.go @@ -33,8 +33,10 @@ type liveCounts struct { // seed sets every count to the value read from the tables at startup. It runs // before any writer, so it needs no coordination with the adjust methods. -func (c *liveCounts) seed(asns, prefixesV4, prefixesV6, peerings, peers, routesV4, routesV6 int, - distributionV4, distributionV6 []PrefixDistribution) { +func (c *liveCounts) seed( + asns, prefixesV4, prefixesV6, peerings, peers, routesV4, routesV6 int, + distributionV4, distributionV6 []PrefixDistribution, +) { c.mu.Lock() defer c.mu.Unlock() @@ -45,9 +47,11 @@ func (c *liveCounts) seed(asns, prefixesV4, prefixesV6, peerings, peers, routesV c.peers = peers c.routesV4 = routesV4 c.routesV6 = routesV6 + for _, entry := range distributionV4 { addAtMaskLength(c.distributionV4[:], entry.MaskLength, entry.Count) } + for _, entry := range distributionV6 { addAtMaskLength(c.distributionV6[:], entry.MaskLength, entry.Count) } @@ -102,6 +106,7 @@ func (c *liveCounts) addToDistribution(maskLengthsV4, maskLengthsV6 []int, n int for _, maskLength := range maskLengthsV4 { addAtMaskLength(c.distributionV4[:], maskLength, n) } + for _, maskLength := range maskLengthsV6 { addAtMaskLength(c.distributionV6[:], maskLength, n) } @@ -138,9 +143,11 @@ func (c *liveCounts) fill(s *Stats) { // ascending order, the way the distribution query returns them. func distributionList(counts []int) []PrefixDistribution { var list []PrefixDistribution + for maskLength, count := range counts { if count > 0 { - list = append(list, PrefixDistribution{MaskLength: maskLength, Count: count}) + list = append(list, + PrefixDistribution{MaskLength: maskLength, Count: count}) } } @@ -152,7 +159,8 @@ func distributionList(counts []int) []PrefixDistribution { func (d *Database) countRows(ctx context.Context, table string) (int, error) { var n int // table is one of a fixed set of literals below, never external input. - if err := d.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM "+table).Scan(&n); err != nil { + err := d.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM "+table).Scan(&n) + if err != nil { return 0, fmt.Errorf("failed to count %s: %w", table, err) } @@ -167,30 +175,37 @@ func (d *Database) seedCounts(ctx context.Context) error { if err != nil { return err } + prefixesV4, err := d.countRows(ctx, "prefixes_v4") if err != nil { return err } + prefixesV6, err := d.countRows(ctx, "prefixes_v6") if err != nil { return err } + peerings, err := d.countRows(ctx, "peerings") if err != nil { return err } + peers, err := d.countRows(ctx, "bgp_peers") if err != nil { return err } - routesV4, err := d.countRows(ctx, "live_routes_v4") + + routesV4, err := d.countRows(ctx, tableLiveRoutesV4) if err != nil { return err } - routesV6, err := d.countRows(ctx, "live_routes_v6") + + routesV6, err := d.countRows(ctx, tableLiveRoutesV6) if err != nil { return err } + distributionV4, distributionV6, err := d.GetPrefixDistributionContext(ctx) if err != nil { return err diff --git a/internal/database/counts_test.go b/internal/database/counts_internal_test.go similarity index 80% rename from internal/database/counts_test.go rename to internal/database/counts_internal_test.go index 5cd62b1..c5decf7 100644 --- a/internal/database/counts_test.go +++ b/internal/database/counts_internal_test.go @@ -12,6 +12,12 @@ import ( "github.com/google/uuid" ) +// testV4PeerIP is the peer the IPv4 routes in these tests are learned from. +const testV4PeerIP = "192.0.2.1" + +// testV6Prefix is the prefix of the IPv6 routes in these tests. +const testV6Prefix = "2001:db8::/32" + // mkV4Route builds an IPv4 live route with its mask length taken from the // prefix. func mkV4Route(t *testing.T, prefix string, asn int, ts time.Time) *LiveRoute { @@ -28,7 +34,7 @@ func mkV4Route(t *testing.T, prefix string, asn int, ts time.Time) *LiveRoute { MaskLength: maskLength, IPVersion: ipVersionV4, OriginASN: asn, - PeerIP: "192.0.2.1", + PeerIP: testV4PeerIP, ASPath: []int{asn}, NextHop: "192.0.2.254", LastUpdated: ts, @@ -56,6 +62,8 @@ func mkV6Route(prefix string, asn int, ts time.Time) *LiveRoute { // deleted. These counts are what /api/v1/stats reports; before this change the // endpoint recomputed them with a COUNT(*) over each table on every request. func TestLiveCountsTrackWritesInRealtime(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -70,6 +78,7 @@ func TestLiveCountsTrackWritesInRealtime(t *testing.T) { if err != nil { t.Fatalf("GetStatsContext on empty database: %v", err) } + if empty.ASNs != 0 || empty.Prefixes != 0 || empty.Peerings != 0 || empty.Peers != 0 || empty.LiveRoutes != 0 { t.Fatalf("empty database counts nonzero: %+v", empty) @@ -77,30 +86,16 @@ func TestLiveCountsTrackWritesInRealtime(t *testing.T) { ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) - if err := db.GetOrCreateASNBatch(map[int]time.Time{64500: ts, 64501: ts}); err != nil { - t.Fatalf("GetOrCreateASNBatch: %v", err) - } - if err := db.UpdatePrefixesBatch(map[string]time.Time{ - "198.51.100.0/24": ts, - "2001:db8::/32": ts, - }); err != nil { - t.Fatalf("UpdatePrefixesBatch: %v", err) - } - if err := db.UpdatePeerBatch(map[string]PeerUpdate{ - "192.0.2.1": {PeerIP: "192.0.2.1", PeerASN: 64500, MessageType: "UPDATE", Timestamp: ts}, - }); err != nil { - t.Fatalf("UpdatePeerBatch: %v", err) - } - if err := db.RecordPeering(64500, 64501, ts); err != nil { - t.Fatalf("RecordPeering: %v", err) - } + writeASNsPrefixesAndPeers(t, db, ts) routes := []*LiveRoute{ mkV4Route(t, "198.51.100.0/24", 64500, ts), mkV4Route(t, "203.0.113.0/24", 64501, ts.Add(time.Minute)), - mkV6Route("2001:db8::/32", 64502, ts.Add(2*time.Minute)), + mkV6Route(testV6Prefix, 64502, ts.Add(2*time.Minute)), } - if err := db.UpsertLiveRouteBatch(routes); err != nil { + + err = db.UpsertLiveRouteBatch(routes) + if err != nil { t.Fatalf("UpsertLiveRouteBatch: %v", err) } @@ -108,44 +103,87 @@ func TestLiveCountsTrackWritesInRealtime(t *testing.T) { if err != nil { t.Fatalf("GetStatsContext: %v", err) } + assertCounts(t, "after inserts", stats, wantCounts{ asns: 2, prefixes: 2, peerings: 1, peers: 1, ipv4Routes: 2, ipv6Routes: 1, liveRoutes: 3, }) // Re-announcing the same routes is an update, not an insert: counts hold. - if err := db.UpsertLiveRouteBatch(routes); err != nil { + err = db.UpsertLiveRouteBatch(routes) + if err != nil { t.Fatalf("UpsertLiveRouteBatch (re-announce): %v", err) } + stats, err = db.GetStatsContext(ctx) if err != nil { t.Fatalf("GetStatsContext: %v", err) } + assertCounts(t, "after re-announce", stats, wantCounts{ asns: 2, prefixes: 2, peerings: 1, peers: 1, ipv4Routes: 2, ipv6Routes: 1, liveRoutes: 3, }) // A withdrawal removes one route. - if err := db.DeleteLiveRouteBatch([]LiveRouteDeletion{ - {Prefix: "203.0.113.0/24", PeerIP: "192.0.2.1", IPVersion: ipVersionV4}, - }); err != nil { + err = db.DeleteLiveRouteBatch([]LiveRouteDeletion{ + {Prefix: "203.0.113.0/24", PeerIP: testV4PeerIP, IPVersion: ipVersionV4}, + }) + if err != nil { t.Fatalf("DeleteLiveRouteBatch: %v", err) } + stats, err = db.GetStatsContext(ctx) if err != nil { t.Fatalf("GetStatsContext: %v", err) } + assertCounts(t, "after delete", stats, wantCounts{ asns: 2, prefixes: 2, peerings: 1, peers: 1, ipv4Routes: 1, ipv6Routes: 1, liveRoutes: 2, }) } +// writeASNsPrefixesAndPeers writes two ASNs, two prefixes, one peer and one +// peering: the rows besides live routes that the stats count. +func writeASNsPrefixesAndPeers(t *testing.T, db *Database, ts time.Time) { + t.Helper() + + err := db.GetOrCreateASNBatch(map[int]time.Time{64500: ts, 64501: ts}) + if err != nil { + t.Fatalf("GetOrCreateASNBatch: %v", err) + } + + err = db.UpdatePrefixesBatch(map[string]time.Time{ + "198.51.100.0/24": ts, + testV6Prefix: ts, + }) + if err != nil { + t.Fatalf("UpdatePrefixesBatch: %v", err) + } + + err = db.UpdatePeerBatch(map[string]PeerUpdate{ + testV4PeerIP: { + PeerIP: testV4PeerIP, PeerASN: 64500, + MessageType: "UPDATE", Timestamp: ts, + }, + }) + if err != nil { + t.Fatalf("UpdatePeerBatch: %v", err) + } + + err = db.RecordPeering(64500, 64501, ts) + if err != nil { + t.Fatalf("RecordPeering: %v", err) + } +} + // TestLiveCountsSeededFromDatabaseAtStartup writes rows, reopens the same // database file, and checks the counts and the prefix distribution come back // from the seed scan rather than starting at zero. func TestLiveCountsSeededFromDatabaseAtStartup(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -154,16 +192,22 @@ func TestLiveCountsSeededFromDatabaseAtStartup(t *testing.T) { } ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) - if err := db.GetOrCreateASNBatch(map[int]time.Time{64500: ts, 64501: ts, 64502: ts}); err != nil { + + err = db.GetOrCreateASNBatch(map[int]time.Time{64500: ts, 64501: ts, 64502: ts}) + if err != nil { t.Fatalf("GetOrCreateASNBatch: %v", err) } - if err := db.UpsertLiveRouteBatch([]*LiveRoute{ + + err = db.UpsertLiveRouteBatch([]*LiveRoute{ mkV4Route(t, "198.51.100.0/24", 64500, ts), - mkV6Route("2001:db8::/32", 64502, ts), - }); err != nil { + mkV6Route(testV6Prefix, 64502, ts), + }) + if err != nil { t.Fatalf("UpsertLiveRouteBatch: %v", err) } - if err := db.Close(); err != nil { + + err = db.Close() + if err != nil { t.Fatalf("Close: %v", err) } @@ -177,13 +221,16 @@ func TestLiveCountsSeededFromDatabaseAtStartup(t *testing.T) { if err != nil { t.Fatalf("GetStatsContext after reopen: %v", err) } + if stats.ASNs != 3 { t.Errorf("seeded ASNs = %d, want 3", stats.ASNs) } + if stats.IPv4Routes != 1 || stats.IPv6Routes != 1 || stats.LiveRoutes != 2 { t.Errorf("seeded routes = (v4 %d, v6 %d, total %d), want (1, 1, 2)", stats.IPv4Routes, stats.IPv6Routes, stats.LiveRoutes) } + assertDistribution(t, "seeded IPv4 distribution", stats.IPv4PrefixDistribution, []PrefixDistribution{{MaskLength: 24, Count: 1}}) assertDistribution(t, "seeded IPv6 distribution", stats.IPv6PrefixDistribution, @@ -196,26 +243,91 @@ func TestLiveCountsSeededFromDatabaseAtStartup(t *testing.T) { // tables. The steps run once through the batch methods the prefix handler uses // and once through the single-route methods. func TestPrefixDistributionTracksWrites(t *testing.T) { + t.Parallel() + + steps := prefixDistributionSteps(t) + + for _, batch := range []bool{true, false} { + name := "single-route writes" + if batch { + name = "batch writes" + } + + t.Run(name, func(t *testing.T) { + t.Parallel() + + db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New()) + if err != nil { + t.Fatalf("failed to create database: %v", err) + } + defer func() { _ = db.Close() }() + + ctx := context.Background() + + for _, step := range steps { + err := announceRoutes(db, batch, step.announce) + if err != nil { + t.Fatalf("%s: announce: %v", step.name, err) + } + + err = withdrawRoutes(db, batch, step.withdraw) + if err != nil { + t.Fatalf("%s: withdraw: %v", step.name, err) + } + + stats, err := db.GetStatsContext(ctx) + if err != nil { + t.Fatalf("%s: GetStatsContext: %v", step.name, err) + } + + queryV4, queryV6, err := db.GetPrefixDistributionContext(ctx) + if err != nil { + t.Fatalf("%s: GetPrefixDistributionContext: %v", step.name, err) + } + + assertDistribution(t, step.name+": IPv4 distribution", + stats.IPv4PrefixDistribution, step.wantV4) + assertDistribution(t, step.name+": IPv6 distribution", + stats.IPv6PrefixDistribution, step.wantV6) + assertDistribution(t, step.name+": IPv4 distribution query", + queryV4, step.wantV4) + assertDistribution(t, step.name+": IPv6 distribution query", + queryV6, step.wantV6) + } + }) + } +} + +// prefixDistributionStep is one step of TestPrefixDistributionTracksWrites: +// the routes it announces, then the routes it withdraws, then the +// distributions both the stats and the distribution query must report. +type prefixDistributionStep struct { + name string + announce []*LiveRoute + withdraw []*LiveRoute + wantV4 []PrefixDistribution + wantV6 []PrefixDistribution +} + +// prefixDistributionSteps returns the steps of +// TestPrefixDistributionTracksWrites, in the order they run. +func prefixDistributionSteps(t *testing.T) []prefixDistributionStep { + t.Helper() + ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) shared := mkV4Route(t, "198.51.100.0/24", 64500, ts) sharedSecondPeer := mkV4Route(t, "198.51.100.0/24", 64500, ts) sharedSecondPeer.PeerIP = "192.0.2.2" other := mkV4Route(t, "203.0.113.0/24", 64501, ts) wide := mkV4Route(t, "172.16.0.0/16", 64502, ts) - v6 := mkV6Route("2001:db8::/32", 64503, ts) - v6SecondPeer := mkV6Route("2001:db8::/32", 64503, ts) + v6 := mkV6Route(testV6Prefix, 64503, ts) + v6SecondPeer := mkV6Route(testV6Prefix, 64503, ts) v6SecondPeer.PeerIP = "2001:db8::2" all := []PrefixDistribution{{MaskLength: 16, Count: 1}, {MaskLength: 24, Count: 2}} v6Only := []PrefixDistribution{{MaskLength: 32, Count: 1}} - steps := []struct { - name string - announce []*LiveRoute - withdraw []*LiveRoute - wantV4 []PrefixDistribution - wantV6 []PrefixDistribution - }{ + return []prefixDistributionStep{ { name: "new routes", announce: []*LiveRoute{shared, other, wide, v6}, @@ -239,7 +351,9 @@ func TestPrefixDistributionTracksWrites(t *testing.T) { { name: "withdrawal of the last route for a prefix", withdraw: []*LiveRoute{sharedSecondPeer, v6SecondPeer}, - wantV4: []PrefixDistribution{{MaskLength: 16, Count: 1}, {MaskLength: 24, Count: 1}}, + wantV4: []PrefixDistribution{ + {MaskLength: 16, Count: 1}, {MaskLength: 24, Count: 1}, + }, }, { name: "withdrawal of every remaining route", @@ -270,45 +384,6 @@ func TestPrefixDistributionTracksWrites(t *testing.T) { wantV6: v6Only, }, } - - for _, batch := range []bool{true, false} { - name := "single-route writes" - if batch { - name = "batch writes" - } - - t.Run(name, func(t *testing.T) { - db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New()) - if err != nil { - t.Fatalf("failed to create database: %v", err) - } - defer func() { _ = db.Close() }() - - ctx := context.Background() - for _, step := range steps { - if err := announceRoutes(db, batch, step.announce); err != nil { - t.Fatalf("%s: announce: %v", step.name, err) - } - if err := withdrawRoutes(db, batch, step.withdraw); err != nil { - t.Fatalf("%s: withdraw: %v", step.name, err) - } - - stats, err := db.GetStatsContext(ctx) - if err != nil { - t.Fatalf("%s: GetStatsContext: %v", step.name, err) - } - queryV4, queryV6, err := db.GetPrefixDistributionContext(ctx) - if err != nil { - t.Fatalf("%s: GetPrefixDistributionContext: %v", step.name, err) - } - - assertDistribution(t, step.name+": IPv4 distribution", stats.IPv4PrefixDistribution, step.wantV4) - assertDistribution(t, step.name+": IPv6 distribution", stats.IPv6PrefixDistribution, step.wantV6) - assertDistribution(t, step.name+": IPv4 distribution query", queryV4, step.wantV4) - assertDistribution(t, step.name+": IPv6 distribution query", queryV6, step.wantV6) - } - }) - } } // announceRoutes writes routes in one UpsertLiveRouteBatch, or with one @@ -317,8 +392,10 @@ func announceRoutes(db *Database, batch bool, routes []*LiveRoute) error { if batch { return db.UpsertLiveRouteBatch(routes) } + for _, route := range routes { - if err := db.UpsertLiveRoute(route); err != nil { + err := db.UpsertLiveRoute(route) + if err != nil { return err } } @@ -331,7 +408,8 @@ func announceRoutes(db *Database, batch bool, routes []*LiveRoute) error { func withdrawRoutes(db *Database, batch bool, routes []*LiveRoute) error { if !batch { for _, route := range routes { - if err := db.DeleteLiveRoute(route.Prefix, route.PeerIP); err != nil { + err := db.DeleteLiveRoute(route.Prefix, route.PeerIP) + if err != nil { return err } } @@ -354,6 +432,8 @@ func withdrawRoutes(db *Database, batch bool, routes []*LiveRoute) error { // routes stay, and the route counts and prefix distribution held in memory // equal what is read from the route tables. func TestDeleteLiveRoutesOfPeer(t *testing.T) { + t.Parallel() + const ( peerIP = "192.0.2.1" otherPeerIP = "192.0.2.2" @@ -375,14 +455,16 @@ func TestDeleteLiveRoutesOfPeer(t *testing.T) { } onlyV6 := mkV6Route("2001:db8:1::/48", 64503, ts) onlyV6.MaskLength = 48 - if err := db.UpsertLiveRouteBatch([]*LiveRoute{ + + err = db.UpsertLiveRouteBatch([]*LiveRoute{ ofPeer(peerIP, mkV4Route(t, sharedV4, 64500, ts)), ofPeer(otherPeerIP, mkV4Route(t, sharedV4, 64500, ts)), ofPeer(peerIP, mkV4Route(t, "203.0.113.0/24", 64501, ts)), ofPeer(peerIP, mkV6Route(sharedV6, 64502, ts)), ofPeer(otherPeerIP, mkV6Route(sharedV6, 64502, ts)), ofPeer(peerIP, onlyV6), - }); err != nil { + }) + if err != nil { t.Fatalf("UpsertLiveRouteBatch: %v", err) } @@ -392,40 +474,61 @@ func TestDeleteLiveRoutesOfPeer(t *testing.T) { if err != nil { t.Fatalf("DeleteLiveRoutesOfPeer: %v", err) } + if deleted != want { - t.Errorf("DeleteLiveRoutesOfPeer deleted %d routes, want %d", deleted, want) + t.Errorf("DeleteLiveRoutesOfPeer deleted %d routes, want %d", + deleted, want) } } - ctx := context.Background() for _, prefix := range []string{sharedV4, sharedV6} { - routes, _, err := db.GetPrefixDetailsContext(ctx, prefix) + routes, _, err := db.GetPrefixDetailsContext(context.Background(), prefix) if err != nil { t.Fatalf("GetPrefixDetailsContext(%s): %v", prefix, err) } + if len(routes) != 1 || routes[0].PeerIP != otherPeerIP { - t.Errorf("live routes for %s = %+v, want only the one from %s", prefix, routes, otherPeerIP) + t.Errorf("live routes for %s = %+v, want only the one from %s", + prefix, routes, otherPeerIP) } } + checkOneLiveRoutePerIPVersion(t, db) +} + +// checkOneLiveRoutePerIPVersion checks that the database has one IPv4 live +// route, for a /24, and one IPv6 live route, for a /32, both as counted in +// memory and as read from the route tables. +func checkOneLiveRoutePerIPVersion(t *testing.T, db *Database) { + t.Helper() + + ctx := context.Background() + stats, err := db.GetStatsContext(ctx) if err != nil { t.Fatalf("GetStatsContext: %v", err) } + tableV4, tableV6, err := db.GetLiveRouteCountsContext(ctx) if err != nil { t.Fatalf("GetLiveRouteCountsContext: %v", err) } - if stats.IPv4Routes != 1 || tableV4 != 1 || stats.IPv6Routes != 1 || tableV6 != 1 { - t.Errorf("live routes (IPv4, IPv6): (%d, %d) in memory, (%d, %d) in the tables, want (1, 1)", + + if stats.IPv4Routes != 1 || tableV4 != 1 || stats.IPv6Routes != 1 || + tableV6 != 1 { + t.Errorf("live routes (IPv4, IPv6): (%d, %d) in memory, (%d, %d) in "+ + "the tables, want (1, 1)", stats.IPv4Routes, stats.IPv6Routes, tableV4, tableV6) } + queryV4, queryV6, err := db.GetPrefixDistributionContext(ctx) if err != nil { t.Fatalf("GetPrefixDistributionContext: %v", err) } + wantV4 := []PrefixDistribution{{MaskLength: 24, Count: 1}} wantV6 := []PrefixDistribution{{MaskLength: 32, Count: 1}} + assertDistribution(t, "IPv4 distribution", stats.IPv4PrefixDistribution, wantV4) assertDistribution(t, "IPv6 distribution", stats.IPv6PrefixDistribution, wantV6) assertDistribution(t, "IPv4 distribution query", queryV4, wantV4) @@ -437,6 +540,8 @@ func TestDeleteLiveRoutesOfPeer(t *testing.T) { // MIN/MAX union query read its result into *time.Time, which the driver could // not parse, so it logged a warning every call and left both timestamps nil. func TestStatsRouteTimestamps(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -451,6 +556,7 @@ func TestStatsRouteTimestamps(t *testing.T) { if err != nil { t.Fatalf("GetStatsContext on empty database: %v", err) } + if empty.OldestRoute != nil || empty.NewestRoute != nil { t.Fatalf("empty database timestamps = (%v, %v), want (nil, nil)", empty.OldestRoute, empty.NewestRoute) @@ -459,11 +565,13 @@ func TestStatsRouteTimestamps(t *testing.T) { base := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) oldest := base newest := base.Add(2 * time.Minute) - if err := db.UpsertLiveRouteBatch([]*LiveRoute{ + + err = db.UpsertLiveRouteBatch([]*LiveRoute{ mkV4Route(t, "198.51.100.0/24", 64500, base.Add(time.Minute)), mkV4Route(t, "203.0.113.0/24", 64501, oldest), - mkV6Route("2001:db8::/32", 64502, newest), - }); err != nil { + mkV6Route(testV6Prefix, 64502, newest), + }) + if err != nil { t.Fatalf("UpsertLiveRouteBatch: %v", err) } @@ -471,9 +579,11 @@ func TestStatsRouteTimestamps(t *testing.T) { if err != nil { t.Fatalf("GetStatsContext: %v", err) } + if stats.OldestRoute == nil || !stats.OldestRoute.Equal(oldest) { t.Errorf("OldestRoute = %v, want %v", stats.OldestRoute, oldest) } + if stats.NewestRoute == nil || !stats.NewestRoute.Equal(newest) { t.Errorf("NewestRoute = %v, want %v", stats.NewestRoute, newest) } @@ -482,6 +592,8 @@ func TestStatsRouteTimestamps(t *testing.T) { // TestLiveCountsConcurrentReadWrite runs writers and stats readers at once so // the race detector proves the counters are safe under concurrent use. func TestLiveCountsConcurrentReadWrite(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -493,17 +605,21 @@ func TestLiveCountsConcurrentReadWrite(t *testing.T) { ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) const writers = 4 - var wg sync.WaitGroup + var wg sync.WaitGroup wg.Add(writers) + for w := range writers { go func(base int) { defer wg.Done() + for i := range 25 { asn := 65000 + base*100 + i - route := mkV6Route("2001:db8::/32", asn, ts) + route := mkV6Route(testV6Prefix, asn, ts) route.PeerIP = "2001:db8::" + uuid.NewString() - if err := db.UpsertLiveRoute(route); err != nil { + + err := db.UpsertLiveRoute(route) + if err != nil { t.Errorf("UpsertLiveRoute: %v", err) return @@ -514,15 +630,19 @@ func TestLiveCountsConcurrentReadWrite(t *testing.T) { var readerWG sync.WaitGroup readerWG.Add(1) + stop := make(chan struct{}) + go func() { defer readerWG.Done() + for { select { case <-stop: return default: - if _, err := db.GetStatsContext(context.Background()); err != nil { + _, err := db.GetStatsContext(context.Background()) + if err != nil { t.Errorf("GetStatsContext: %v", err) return @@ -539,6 +659,7 @@ func TestLiveCountsConcurrentReadWrite(t *testing.T) { if err != nil { t.Fatalf("GetStatsContext: %v", err) } + if want := writers * 25; stats.IPv6Routes != want { t.Errorf("IPv6Routes = %d, want %d", stats.IPv6Routes, want) } @@ -563,21 +684,27 @@ func assertCounts(t *testing.T, when string, got Stats, want wantCounts) { if got.ASNs != want.asns { t.Errorf("%s: ASNs = %d, want %d", when, got.ASNs, want.asns) } + if got.Prefixes != want.prefixes { t.Errorf("%s: Prefixes = %d, want %d", when, got.Prefixes, want.prefixes) } + if got.Peerings != want.peerings { t.Errorf("%s: Peerings = %d, want %d", when, got.Peerings, want.peerings) } + if got.Peers != want.peers { t.Errorf("%s: Peers = %d, want %d", when, got.Peers, want.peers) } + if got.IPv4Routes != want.ipv4Routes { t.Errorf("%s: IPv4Routes = %d, want %d", when, got.IPv4Routes, want.ipv4Routes) } + if got.IPv6Routes != want.ipv6Routes { t.Errorf("%s: IPv6Routes = %d, want %d", when, got.IPv6Routes, want.ipv6Routes) } + if got.LiveRoutes != want.liveRoutes { t.Errorf("%s: LiveRoutes = %d, want %d", when, got.LiveRoutes, want.liveRoutes) } diff --git a/internal/database/database.go b/internal/database/database.go index 7005042..d17a505 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -1,4 +1,3 @@ -// Package database provides SQLite storage for BGP routing data including ASNs, prefixes, announcements and peerings. package database import ( @@ -36,6 +35,13 @@ const ( ipVersionV6 = 6 ipv4Bits = 32 ipv6Bits = 128 + + tableLiveRoutesV4 = "live_routes_v4" + tableLiveRoutesV6 = "live_routes_v6" + + // Names of the IP versions in error and log messages. + ipv4Family = "IPv4" + ipv6Family = "IPv6" ) // The IP lookup does not use routes shorter than these mask lengths. A default @@ -81,6 +87,9 @@ var ( ErrNoRoute = errors.New("no route found") // ErrNoStaleASN is returned when no ASN needs WHOIS refresh ErrNoStaleASN = errors.New("no stale ASN found") + + errInvalidASN = errors.New("invalid ASN") + errSameASNInPeering = errors.New("cannot create peering with same ASN") ) // Database manages the SQLite database connection and operations. @@ -103,7 +112,9 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) { // Ensure directory exists dir := filepath.Dir(dbPath) - if err := os.MkdirAll(dir, dirPermissions); err != nil { + + err := os.MkdirAll(dir, dirPermissions) + if err != nil { return nil, fmt.Errorf("failed to create database directory: %w", err) } @@ -124,12 +135,14 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) { sqliteCacheSizeKiB, sqliteBusyTimeoutMs, ) + db, err := sql.Open("sqlite3", dsn) if err != nil { return nil, fmt.Errorf("failed to open database: %w", err) } - if err := db.Ping(); err != nil { + err = db.PingContext(context.Background()) + if err != nil { return nil, fmt.Errorf("failed to ping database: %w", err) } @@ -140,16 +153,20 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) { db.SetMaxIdleConns(maxConns) db.SetConnMaxLifetime(0) - database := &Database{db: db, logger: logger, path: dbPath, counts: &liveCounts{}} + database := &Database{ + db: db, logger: logger, path: dbPath, counts: &liveCounts{}, + } - if err := database.Initialize(); err != nil { + err = database.Initialize() + if err != nil { return nil, fmt.Errorf("failed to initialize database: %w", err) } // Seed the in-memory statistics counters from the tables once, before the // streamer starts writing. From here on every write keeps them current, so // the stats endpoints never scan the tables to report counts. - if err := database.seedCounts(context.Background()); err != nil { + err = database.seedCounts(context.Background()) + if err != nil { return nil, fmt.Errorf("failed to seed statistics counters: %w", err) } @@ -171,14 +188,18 @@ func (d *Database) Initialize() error { } for _, pragma := range pragmas { - if err := d.exec(pragma); err != nil { + err := d.exec(pragma) + if err != nil { d.logger.Warn("Failed to set pragma", "pragma", pragma, "error", err) } } // Run WAL checkpoint on startup to consolidate any existing WAL var walPages, checkpointedPages, movedPages int - err := d.db.QueryRow("PRAGMA wal_checkpoint(TRUNCATE)").Scan(&walPages, &checkpointedPages, &movedPages) + + err := d.db.QueryRowContext(context.Background(), + "PRAGMA wal_checkpoint(TRUNCATE)", + ).Scan(&walPages, &checkpointedPages, &movedPages) if err != nil { d.logger.Warn("Failed to checkpoint WAL on startup", "error", err) } else { @@ -202,43 +223,6 @@ func (d *Database) Close() error { return d.db.Close() } -// lock acquires the database mutex and logs debug information -func (d *Database) lock(operation string) { - // Get caller information - _, file, line, _ := runtime.Caller(1) - caller := fmt.Sprintf("%s:%d", filepath.Base(file), line) - - d.logger.Debug("Acquiring database lock", "operation", operation, "caller", caller) - - d.mu.Lock() - d.lockedAt = time.Now() - d.lockedBy = fmt.Sprintf("%s (%s)", operation, caller) - - d.logger.Debug("Database lock acquired", "operation", operation, "caller", caller) -} - -// unlock releases the database mutex and logs debug information including hold duration -func (d *Database) unlock() { - holdDuration := time.Since(d.lockedAt) - lockedBy := d.lockedBy - - d.lockedAt = time.Time{} - d.lockedBy = "" - d.mu.Unlock() - - d.logger.Debug("Database lock released", "held_by", lockedBy, "duration_ms", holdDuration.Milliseconds()) -} - -// beginTx starts a new transaction with logging -func (d *Database) beginTx() (*loggingTx, error) { - tx, err := d.db.Begin() - if err != nil { - return nil, err - } - - return &loggingTx{Tx: tx, logger: d.logger}, nil -} - // A live-route upsert is an UPDATE followed, only when no row matched, by an // INSERT. A live route is keyed by prefix and peer, so an announcement replaces // the peer's route for the prefix whatever its origin AS. The UPDATE's @@ -248,52 +232,73 @@ func (d *Database) beginTx() (*loggingTx, error) { // two statements. The id column is set only on INSERT, so an updated route // keeps its original id, exactly as the previous ON CONFLICT upsert did. const ( - updateLiveRouteV4SQL = `UPDATE live_routes_v4 SET mask_length = ?, origin_asn = ?, as_path = ?, - next_hop = ?, last_updated = ? WHERE prefix = ? AND peer_ip = ?` + updateLiveRouteV4SQL = `UPDATE live_routes_v4 + SET mask_length = ?, origin_asn = ?, as_path = ?, next_hop = ?, + last_updated = ? WHERE prefix = ? AND peer_ip = ?` - insertLiveRouteV4SQL = `INSERT INTO live_routes_v4 (id, prefix, mask_length, origin_asn, peer_ip, - as_path, next_hop, last_updated) VALUES (?, ?, ?, ?, ?, ?, ?, ?)` + insertLiveRouteV4SQL = `INSERT INTO live_routes_v4 + (id, prefix, mask_length, origin_asn, peer_ip, as_path, next_hop, + last_updated) VALUES (?, ?, ?, ?, ?, ?, ?, ?)` - updateLiveRouteV6SQL = `UPDATE live_routes_v6 SET mask_length = ?, origin_asn = ?, as_path = ?, - next_hop = ?, last_updated = ? WHERE prefix = ? AND peer_ip = ?` + updateLiveRouteV6SQL = `UPDATE live_routes_v6 + SET mask_length = ?, origin_asn = ?, as_path = ?, next_hop = ?, + last_updated = ? WHERE prefix = ? AND peer_ip = ?` - insertLiveRouteV6SQL = `INSERT INTO live_routes_v6 (id, prefix, mask_length, origin_asn, peer_ip, - as_path, next_hop, last_updated) VALUES (?, ?, ?, ?, ?, ?, ?, ?)` + insertLiveRouteV6SQL = `INSERT INTO live_routes_v6 + (id, prefix, mask_length, origin_asn, peer_ip, as_path, next_hop, + last_updated) VALUES (?, ?, ?, ?, ?, ?, ?, ?)` ) // Before a new route is inserted and after a route is deleted, the write looks // up whether any live route has that prefix, to keep the in-memory prefix // distribution exact. The lookup reads one entry of the prefix index. const ( - prefixHasLiveRouteV4SQL = `SELECT EXISTS (SELECT 1 FROM live_routes_v4 WHERE prefix = ?)` - prefixHasLiveRouteV6SQL = `SELECT EXISTS (SELECT 1 FROM live_routes_v6 WHERE prefix = ?)` + prefixHasLiveRouteV4SQL = `SELECT EXISTS + (SELECT 1 FROM live_routes_v4 WHERE prefix = ?)` + prefixHasLiveRouteV6SQL = `SELECT EXISTS + (SELECT 1 FROM live_routes_v6 WHERE prefix = ?)` ) -// upsertRouteRowV4 updates an IPv4 live route, inserting it when no row matched. -// It reports whether a new row was inserted and whether that row is the first -// live route for its prefix. -func upsertRouteRowV4(upd, ins, has *sql.Stmt, route *LiveRoute, pathJSON string) ( - inserted, newPrefix bool, err error) { - res, err := upd.Exec(route.MaskLength, route.OriginASN, pathJSON, route.NextHop, route.LastUpdated, - route.Prefix, route.PeerIP) +// A deletion removes the live route the peer has for the prefix, if any. +const ( + deleteLiveRouteV4SQL = `DELETE FROM live_routes_v4 + WHERE prefix = ? AND peer_ip = ?` + deleteLiveRouteV6SQL = `DELETE FROM live_routes_v6 + WHERE prefix = ? AND peer_ip = ?` +) + +// upsertRouteRow updates a live route with the update statement of its IP +// version's table, inserting it when no row matched. It reports whether a new +// row was inserted and whether that row is the first live route for its prefix. +func upsertRouteRow( + ctx context.Context, upd, ins, has *sql.Stmt, route *LiveRoute, + pathJSON string, +) (bool, bool, error) { + res, err := upd.ExecContext(ctx, route.MaskLength, route.OriginASN, pathJSON, + route.NextHop, route.LastUpdated, route.Prefix, route.PeerIP) if err != nil { return false, false, err } + affected, err := res.RowsAffected() if err != nil { return false, false, err } + if affected > 0 { return false, false, nil } var prefixHadRoute bool - if err := has.QueryRow(route.Prefix).Scan(&prefixHadRoute); err != nil { + + err = has.QueryRowContext(ctx, route.Prefix).Scan(&prefixHadRoute) + if err != nil { return false, false, err } - _, err = ins.Exec(route.ID.String(), route.Prefix, route.MaskLength, route.OriginASN, - route.PeerIP, pathJSON, route.NextHop, route.LastUpdated) + _, err = ins.ExecContext(ctx, route.ID.String(), route.Prefix, + route.MaskLength, route.OriginASN, route.PeerIP, pathJSON, + route.NextHop, route.LastUpdated) if err != nil { return false, false, err } @@ -301,39 +306,75 @@ func upsertRouteRowV4(upd, ins, has *sql.Stmt, route *LiveRoute, pathJSON string return true, !prefixHadRoute, nil } -// upsertRouteRowV6 updates an IPv6 live route, inserting it when no row matched. -// It reports whether a new row was inserted and whether that row is the first -// live route for its prefix. -func upsertRouteRowV6(upd, ins, has *sql.Stmt, route *LiveRoute, pathJSON string) ( - inserted, newPrefix bool, err error) { - res, err := upd.Exec(route.MaskLength, route.OriginASN, pathJSON, route.NextHop, route.LastUpdated, - route.Prefix, route.PeerIP) - if err != nil { - return false, false, err - } - affected, err := res.RowsAffected() - if err != nil { - return false, false, err - } - if affected > 0 { - return false, false, nil +// upsertRoutesInTable upserts routes, all of one IP version, in the live route +// table of that version. It returns how many new rows it inserted and the mask +// lengths of the prefixes that got their first live route. +func upsertRoutesInTable( + ctx context.Context, tx *loggingTx, routes []*LiveRoute, ipVersion int, +) (int, []int, error) { + if len(routes) == 0 { + return 0, nil, nil } - var prefixHadRoute bool - if err := has.QueryRow(route.Prefix).Scan(&prefixHadRoute); err != nil { - return false, false, err + family, updateSQL, insertSQL, lookupSQL := ipv4Family, updateLiveRouteV4SQL, + insertLiveRouteV4SQL, prefixHasLiveRouteV4SQL + if ipVersion == ipVersionV6 { + family, updateSQL, insertSQL, lookupSQL = ipv6Family, + updateLiveRouteV6SQL, insertLiveRouteV6SQL, prefixHasLiveRouteV6SQL } - _, err = ins.Exec(route.ID.String(), route.Prefix, route.MaskLength, route.OriginASN, - route.PeerIP, pathJSON, route.NextHop, route.LastUpdated) + upd, err := tx.PrepareContext(ctx, updateSQL) if err != nil { - return false, false, err + return 0, nil, fmt.Errorf("failed to prepare %s update statement: %w", + family, err) + } + defer func() { _ = upd.Close() }() + + ins, err := tx.PrepareContext(ctx, insertSQL) + if err != nil { + return 0, nil, fmt.Errorf("failed to prepare %s insert statement: %w", + family, err) + } + defer func() { _ = ins.Close() }() + + has, err := tx.PrepareContext(ctx, lookupSQL) + if err != nil { + return 0, nil, fmt.Errorf( + "failed to prepare %s prefix lookup statement: %w", family, err) + } + defer func() { _ = has.Close() }() + + var newRows int + // Mask lengths of the prefixes that get their first live route. + var newPrefixMaskLengths []int + + for _, route := range routes { + pathJSON, err := json.Marshal(route.ASPath) + if err != nil { + return 0, nil, fmt.Errorf("failed to encode AS path: %w", err) + } + + inserted, newPrefix, err := upsertRouteRow(ctx, upd, ins, has, route, + string(pathJSON)) + if err != nil { + return 0, nil, fmt.Errorf("failed to upsert route %s: %w", + route.Prefix, err) + } + + if inserted { + newRows++ + } + + if newPrefix { + newPrefixMaskLengths = append(newPrefixMaskLengths, route.MaskLength) + } } - return true, !prefixHadRoute, nil + return newRows, newPrefixMaskLengths, nil } -// UpsertLiveRouteBatch inserts or updates multiple live routes in a single transaction +// UpsertLiveRouteBatch inserts or updates multiple live routes in a single +// transaction, the IPv4 routes first and then the IPv6 routes. func (d *Database) UpsertLiveRouteBatch(routes []*LiveRoute) error { if len(routes) == 0 { return nil @@ -346,86 +387,34 @@ func (d *Database) UpsertLiveRouteBatch(routes []*LiveRoute) error { if err != nil { return fmt.Errorf("failed to begin transaction: %w", err) } - defer func() { - if err := tx.Rollback(); err != nil && err != sql.ErrTxDone { - d.logger.Error("Failed to rollback transaction", "error", err) - } - }() + defer d.rollback(tx) - // Prepare the update and insert statements for both tables. - updV4, err := tx.Prepare(updateLiveRouteV4SQL) - if err != nil { - return fmt.Errorf("failed to prepare IPv4 update statement: %w", err) - } - defer func() { _ = updV4.Close() }() + var routesV4, routesV6 []*LiveRoute - insV4, err := tx.Prepare(insertLiveRouteV4SQL) - if err != nil { - return fmt.Errorf("failed to prepare IPv4 insert statement: %w", err) - } - defer func() { _ = insV4.Close() }() - - updV6, err := tx.Prepare(updateLiveRouteV6SQL) - if err != nil { - return fmt.Errorf("failed to prepare IPv6 update statement: %w", err) - } - defer func() { _ = updV6.Close() }() - - insV6, err := tx.Prepare(insertLiveRouteV6SQL) - if err != nil { - return fmt.Errorf("failed to prepare IPv6 insert statement: %w", err) - } - defer func() { _ = insV6.Close() }() - - hasV4, err := tx.Prepare(prefixHasLiveRouteV4SQL) - if err != nil { - return fmt.Errorf("failed to prepare IPv4 prefix lookup statement: %w", err) - } - defer func() { _ = hasV4.Close() }() - - hasV6, err := tx.Prepare(prefixHasLiveRouteV6SQL) - if err != nil { - return fmt.Errorf("failed to prepare IPv6 prefix lookup statement: %w", err) - } - defer func() { _ = hasV6.Close() }() - - var newV4, newV6 int - // Mask lengths of the prefixes that get their first live route in this batch. - var newPrefixMaskLengthsV4, newPrefixMaskLengthsV6 []int for _, route := range routes { - pathJSON, err := json.Marshal(route.ASPath) - if err != nil { - return fmt.Errorf("failed to encode AS path: %w", err) - } - if route.IPVersion == ipVersionV4 { - inserted, newPrefix, err := upsertRouteRowV4(updV4, insV4, hasV4, route, string(pathJSON)) - if err != nil { - return fmt.Errorf("failed to upsert route %s: %w", route.Prefix, err) - } - if inserted { - newV4++ - } - if newPrefix { - newPrefixMaskLengthsV4 = append(newPrefixMaskLengthsV4, route.MaskLength) - } - - continue - } - - inserted, newPrefix, err := upsertRouteRowV6(updV6, insV6, hasV6, route, string(pathJSON)) - if err != nil { - return fmt.Errorf("failed to upsert route %s: %w", route.Prefix, err) - } - if inserted { - newV6++ - } - if newPrefix { - newPrefixMaskLengthsV6 = append(newPrefixMaskLengthsV6, route.MaskLength) + routesV4 = append(routesV4, route) + } else { + routesV6 = append(routesV6, route) } } - if err = tx.Commit(); err != nil { + ctx := context.Background() + + newV4, newPrefixMaskLengthsV4, err := upsertRoutesInTable(ctx, tx, + routesV4, ipVersionV4) + if err != nil { + return err + } + + newV6, newPrefixMaskLengthsV6, err := upsertRoutesInTable(ctx, tx, + routesV6, ipVersionV6) + if err != nil { + return err + } + + err = tx.Commit() + if err != nil { return fmt.Errorf("failed to commit transaction: %w", err) } @@ -435,7 +424,103 @@ func (d *Database) UpsertLiveRouteBatch(routes []*LiveRoute) error { return nil } -// DeleteLiveRouteBatch deletes multiple live routes in a single transaction +// deleteRouteRow runs one deletion with the delete statement of its IP +// version's table. It returns how many rows it removed and whether that left +// the prefix with no live route. +func deleteRouteRow( + ctx context.Context, deleteStmt, has *sql.Stmt, del LiveRouteDeletion, +) (int64, bool, error) { + res, err := deleteStmt.ExecContext(ctx, del.Prefix, del.PeerIP) + if err != nil { + return 0, false, fmt.Errorf("failed to delete route %s: %w", + del.Prefix, err) + } + + // The feed often withdraws a route that is not live, which deletes + // nothing, so the in-memory route counters use the rows-affected count. + affected, err := res.RowsAffected() + if err != nil { + return 0, false, fmt.Errorf("failed to count deleted route %s: %w", + del.Prefix, err) + } + + if affected == 0 { + return 0, false, nil + } + + // The prefix leaves the distribution when no live route has it any more. + var prefixHasRoute bool + + err = has.QueryRowContext(ctx, del.Prefix).Scan(&prefixHasRoute) + if err != nil { + return 0, false, fmt.Errorf("failed to look up prefix %s: %w", + del.Prefix, err) + } + + return affected, !prefixHasRoute, nil +} + +// deleteRoutesInTable runs deletions, all of one IP version, on the live route +// table of that version. It returns how many rows it removed and the mask +// lengths of the prefixes it left with no live route. +func deleteRoutesInTable( + ctx context.Context, tx *loggingTx, deletions []LiveRouteDeletion, + ipVersion int, +) (int64, []int, error) { + if len(deletions) == 0 { + return 0, nil, nil + } + + family, deleteSQL, lookupSQL := ipv4Family, deleteLiveRouteV4SQL, + prefixHasLiveRouteV4SQL + if ipVersion == ipVersionV6 { + family, deleteSQL, lookupSQL = ipv6Family, deleteLiveRouteV6SQL, + prefixHasLiveRouteV6SQL + } + + deleteStmt, err := tx.PrepareContext(ctx, deleteSQL) + if err != nil { + return 0, nil, fmt.Errorf("failed to prepare %s delete statement: %w", + family, err) + } + defer func() { _ = deleteStmt.Close() }() + + has, err := tx.PrepareContext(ctx, lookupSQL) + if err != nil { + return 0, nil, fmt.Errorf( + "failed to prepare %s prefix lookup statement: %w", family, err) + } + defer func() { _ = has.Close() }() + + var deleted int64 + // Mask lengths of the prefixes left with no live route. + var gonePrefixMaskLengths []int + + for _, del := range deletions { + affected, prefixGone, err := deleteRouteRow(ctx, deleteStmt, has, del) + if err != nil { + return 0, nil, err + } + + deleted += affected + + if !prefixGone { + continue + } + + maskLength, err := prefixMaskLength(del.Prefix) + if err != nil { + return 0, nil, err + } + + gonePrefixMaskLengths = append(gonePrefixMaskLengths, maskLength) + } + + return deleted, gonePrefixMaskLengths, nil +} + +// DeleteLiveRouteBatch deletes multiple live routes in a single transaction, +// the IPv4 routes first and then the IPv6 routes. func (d *Database) DeleteLiveRouteBatch(deletions []LiveRouteDeletion) error { if len(deletions) == 0 { return nil @@ -448,91 +533,34 @@ func (d *Database) DeleteLiveRouteBatch(deletions []LiveRouteDeletion) error { if err != nil { return fmt.Errorf("failed to begin transaction: %w", err) } - defer func() { - if err := tx.Rollback(); err != nil && err != sql.ErrTxDone { - d.logger.Error("Failed to rollback transaction", "error", err) - } - }() + defer d.rollback(tx) - // Prepare statements for both IPv4 and IPv6 tables - deleteV4, err := tx.Prepare(`DELETE FROM live_routes_v4 WHERE prefix = ? AND peer_ip = ?`) - if err != nil { - return fmt.Errorf("failed to prepare IPv4 delete statement: %w", err) - } - defer func() { _ = deleteV4.Close() }() + var deletionsV4, deletionsV6 []LiveRouteDeletion - deleteV6, err := tx.Prepare(`DELETE FROM live_routes_v6 WHERE prefix = ? AND peer_ip = ?`) - if err != nil { - return fmt.Errorf("failed to prepare IPv6 delete statement: %w", err) - } - defer func() { _ = deleteV6.Close() }() - - hasV4, err := tx.Prepare(prefixHasLiveRouteV4SQL) - if err != nil { - return fmt.Errorf("failed to prepare IPv4 prefix lookup statement: %w", err) - } - defer func() { _ = hasV4.Close() }() - - hasV6, err := tx.Prepare(prefixHasLiveRouteV6SQL) - if err != nil { - return fmt.Errorf("failed to prepare IPv6 prefix lookup statement: %w", err) - } - defer func() { _ = hasV6.Close() }() - - // Process deletions - var deletedV4, deletedV6 int64 - // Mask lengths of the prefixes this batch leaves with no live route. - var gonePrefixMaskLengthsV4, gonePrefixMaskLengthsV6 []int for _, del := range deletions { - stmt := deleteV4 - if del.IPVersion != ipVersionV4 { - stmt = deleteV6 - } - - res, err := stmt.Exec(del.Prefix, del.PeerIP) - if err != nil { - return fmt.Errorf("failed to delete route %s: %w", del.Prefix, err) - } - - // The feed often withdraws a route that is not live, which deletes - // nothing, so the in-memory route counters use the rows-affected count. - affected, err := res.RowsAffected() - if err != nil { - return fmt.Errorf("failed to count deleted route %s: %w", del.Prefix, err) - } if del.IPVersion == ipVersionV4 { - deletedV4 += affected + deletionsV4 = append(deletionsV4, del) } else { - deletedV6 += affected - } - if affected == 0 { - continue - } - - // The prefix leaves the distribution when no live route has it any more. - has := hasV4 - if del.IPVersion != ipVersionV4 { - has = hasV6 - } - var prefixHasRoute bool - if err := has.QueryRow(del.Prefix).Scan(&prefixHasRoute); err != nil { - return fmt.Errorf("failed to look up prefix %s: %w", del.Prefix, err) - } - if prefixHasRoute { - continue - } - maskLength, err := prefixMaskLength(del.Prefix) - if err != nil { - return err - } - if del.IPVersion == ipVersionV4 { - gonePrefixMaskLengthsV4 = append(gonePrefixMaskLengthsV4, maskLength) - } else { - gonePrefixMaskLengthsV6 = append(gonePrefixMaskLengthsV6, maskLength) + deletionsV6 = append(deletionsV6, del) } } - if err = tx.Commit(); err != nil { + ctx := context.Background() + + deletedV4, gonePrefixMaskLengthsV4, err := deleteRoutesInTable(ctx, tx, + deletionsV4, ipVersionV4) + if err != nil { + return err + } + + deletedV6, gonePrefixMaskLengthsV6, err := deleteRoutesInTable(ctx, tx, + deletionsV6, ipVersionV6) + if err != nil { + return err + } + + err = tx.Commit() + if err != nil { return fmt.Errorf("failed to commit transaction: %w", err) } @@ -547,41 +575,56 @@ func (d *Database) DeleteLiveRouteBatch(deletions []LiveRouteDeletion) error { // live route for: the delete takes those prefixes out of the prefix // distribution. const ( - peerOnlyRouteMaskLengthsV4SQL = `SELECT r.mask_length FROM live_routes_v4 AS r WHERE r.peer_ip = ? - AND NOT EXISTS (SELECT 1 FROM live_routes_v4 AS o WHERE o.prefix = r.prefix AND o.peer_ip != r.peer_ip)` + peerOnlyRouteMaskLengthsV4SQL = `SELECT r.mask_length + FROM live_routes_v4 AS r WHERE r.peer_ip = ? + AND NOT EXISTS (SELECT 1 FROM live_routes_v4 AS o + WHERE o.prefix = r.prefix AND o.peer_ip != r.peer_ip)` deletePeerLiveRoutesV4SQL = `DELETE FROM live_routes_v4 WHERE peer_ip = ?` - peerOnlyRouteMaskLengthsV6SQL = `SELECT r.mask_length FROM live_routes_v6 AS r WHERE r.peer_ip = ? - AND NOT EXISTS (SELECT 1 FROM live_routes_v6 AS o WHERE o.prefix = r.prefix AND o.peer_ip != r.peer_ip)` + peerOnlyRouteMaskLengthsV6SQL = `SELECT r.mask_length + FROM live_routes_v6 AS r WHERE r.peer_ip = ? + AND NOT EXISTS (SELECT 1 FROM live_routes_v6 AS o + WHERE o.prefix = r.prefix AND o.peer_ip != r.peer_ip)` deletePeerLiveRoutesV6SQL = `DELETE FROM live_routes_v6 WHERE peer_ip = ?` ) // deletePeerRoutes deletes all live routes of peerIP from one table, given that // table's two statements above. It returns how many routes it deleted and the // mask length of each prefix it left with no live route. -func deletePeerRoutes(tx *loggingTx, onlyRouteMaskLengthsSQL, deleteSQL, peerIP string) ( - deleted int64, goneMaskLengths []int, err error) { - rows, err := tx.Query(onlyRouteMaskLengthsSQL, peerIP) +func deletePeerRoutes( + ctx context.Context, tx *loggingTx, + onlyRouteMaskLengthsSQL, deleteSQL, peerIP string, +) (int64, []int, error) { + rows, err := tx.QueryContext(ctx, onlyRouteMaskLengthsSQL, peerIP) if err != nil { return 0, nil, err } defer func() { _ = rows.Close() }() + + var goneMaskLengths []int + for rows.Next() { var maskLength int - if err := rows.Scan(&maskLength); err != nil { + + err := rows.Scan(&maskLength) + if err != nil { return 0, nil, err } + goneMaskLengths = append(goneMaskLengths, maskLength) } - if err := rows.Err(); err != nil { - return 0, nil, err - } - res, err := tx.Exec(deleteSQL, peerIP) + err = rows.Err() if err != nil { return 0, nil, err } - deleted, err = res.RowsAffected() + + res, err := tx.ExecContext(ctx, deleteSQL, peerIP) + if err != nil { + return 0, nil, err + } + + deleted, err := res.RowsAffected() if err != nil { return 0, nil, err } @@ -600,24 +643,26 @@ func (d *Database) DeleteLiveRoutesOfPeer(peerIP string) (int, error) { if err != nil { return 0, fmt.Errorf("failed to begin transaction: %w", err) } - defer func() { - if err := tx.Rollback(); err != nil && err != sql.ErrTxDone { - d.logger.Error("Failed to rollback transaction", "error", err) - } - }() + defer d.rollback(tx) - deletedV4, goneMaskLengthsV4, err := deletePeerRoutes(tx, + ctx := context.Background() + + deletedV4, goneMaskLengthsV4, err := deletePeerRoutes(ctx, tx, peerOnlyRouteMaskLengthsV4SQL, deletePeerLiveRoutesV4SQL, peerIP) if err != nil { - return 0, fmt.Errorf("failed to delete the IPv4 routes of peer %s: %w", peerIP, err) - } - deletedV6, goneMaskLengthsV6, err := deletePeerRoutes(tx, - peerOnlyRouteMaskLengthsV6SQL, deletePeerLiveRoutesV6SQL, peerIP) - if err != nil { - return 0, fmt.Errorf("failed to delete the IPv6 routes of peer %s: %w", peerIP, err) + return 0, fmt.Errorf("failed to delete the IPv4 routes of peer %s: %w", + peerIP, err) } - if err := tx.Commit(); err != nil { + deletedV6, goneMaskLengthsV6, err := deletePeerRoutes(ctx, tx, + peerOnlyRouteMaskLengthsV6SQL, deletePeerLiveRoutesV6SQL, peerIP) + if err != nil { + return 0, fmt.Errorf("failed to delete the IPv6 routes of peer %s: %w", + peerIP, err) + } + + err = tx.Commit() + if err != nil { return 0, fmt.Errorf("failed to commit transaction: %w", err) } @@ -627,7 +672,107 @@ func (d *Database) DeleteLiveRoutesOfPeer(peerIP string) (int, error) { return int(deletedV4 + deletedV6), nil } -// UpdatePrefixesBatch updates the last_seen time for multiple prefixes in a single transaction +// upsertPrefix updates the last_seen time of prefix with the statements of its +// IP version's table, inserting the prefix when it is new. It reports whether +// it inserted it. +func upsertPrefix( + ctx context.Context, selectStmt, updateStmt, insertStmt *sql.Stmt, + prefix string, timestamp time.Time, +) (bool, error) { + var id string + + err := selectStmt.QueryRowContext(ctx, prefix).Scan(&id) + + switch { + case err == nil: + // Prefix exists, update last_seen + _, err = updateStmt.ExecContext(ctx, timestamp, prefix) + if err != nil { + return false, fmt.Errorf("failed to update prefix %s: %w", + prefix, err) + } + + return false, nil + case errors.Is(err, sql.ErrNoRows): + // Prefix doesn't exist, create it + _, err = insertStmt.ExecContext(ctx, generateUUID().String(), prefix, + timestamp, timestamp) + if err != nil { + return false, fmt.Errorf("failed to insert prefix %s: %w", + prefix, err) + } + + return true, nil + default: + return false, fmt.Errorf("failed to query prefix %s: %w", prefix, err) + } +} + +// upsertPrefixesInTable updates the last_seen time of prefixes, all of one IP +// version, in the prefix table of that version, inserting the ones that are +// new. It returns how many it inserted. +func upsertPrefixesInTable( + ctx context.Context, tx *loggingTx, prefixes map[string]time.Time, + ipVersion int, +) (int, error) { + if len(prefixes) == 0 { + return 0, nil + } + + family := ipv4Family + selectSQL := "SELECT id FROM prefixes_v4 WHERE prefix = ?" + updateSQL := "UPDATE prefixes_v4 SET last_seen = ? WHERE prefix = ?" + insertSQL := "INSERT INTO prefixes_v4 (id, prefix, first_seen, last_seen) " + + "VALUES (?, ?, ?, ?)" + + if ipVersion == ipVersionV6 { + family = ipv6Family + selectSQL = "SELECT id FROM prefixes_v6 WHERE prefix = ?" + updateSQL = "UPDATE prefixes_v6 SET last_seen = ? WHERE prefix = ?" + insertSQL = "INSERT INTO prefixes_v6 (id, prefix, first_seen, last_seen) " + + "VALUES (?, ?, ?, ?)" + } + + selectStmt, err := tx.PrepareContext(ctx, selectSQL) + if err != nil { + return 0, fmt.Errorf("failed to prepare %s select statement: %w", + family, err) + } + defer func() { _ = selectStmt.Close() }() + + updateStmt, err := tx.PrepareContext(ctx, updateSQL) + if err != nil { + return 0, fmt.Errorf("failed to prepare %s update statement: %w", + family, err) + } + defer func() { _ = updateStmt.Close() }() + + insertStmt, err := tx.PrepareContext(ctx, insertSQL) + if err != nil { + return 0, fmt.Errorf("failed to prepare %s insert statement: %w", + family, err) + } + defer func() { _ = insertStmt.Close() }() + + var newPrefixes int + + for prefix, timestamp := range prefixes { + inserted, err := upsertPrefix(ctx, selectStmt, updateStmt, insertStmt, + prefix, timestamp) + if err != nil { + return 0, err + } + + if inserted { + newPrefixes++ + } + } + + return newPrefixes, nil +} + +// UpdatePrefixesBatch updates the last_seen time for multiple prefixes in a +// single transaction, the IPv4 prefixes first and then the IPv6 prefixes. func (d *Database) UpdatePrefixesBatch(prefixes map[string]time.Time) error { if len(prefixes) == 0 { return nil @@ -640,87 +785,33 @@ func (d *Database) UpdatePrefixesBatch(prefixes map[string]time.Time) error { if err != nil { return fmt.Errorf("failed to begin transaction: %w", err) } - defer func() { - if err := tx.Rollback(); err != nil && err != sql.ErrTxDone { - d.logger.Error("Failed to rollback transaction", "error", err) - } - }() + defer d.rollback(tx) - // Prepare statements for both IPv4 and IPv6 tables - selectV4Stmt, err := tx.Prepare("SELECT id FROM prefixes_v4 WHERE prefix = ?") - if err != nil { - return fmt.Errorf("failed to prepare IPv4 select statement: %w", err) - } - defer func() { _ = selectV4Stmt.Close() }() + prefixesV4 := make(map[string]time.Time) + prefixesV6 := make(map[string]time.Time) - updateV4Stmt, err := tx.Prepare("UPDATE prefixes_v4 SET last_seen = ? WHERE prefix = ?") - if err != nil { - return fmt.Errorf("failed to prepare IPv4 update statement: %w", err) - } - defer func() { _ = updateV4Stmt.Close() }() - - insertV4Stmt, err := tx.Prepare("INSERT INTO prefixes_v4 (id, prefix, first_seen, last_seen) VALUES (?, ?, ?, ?)") - if err != nil { - return fmt.Errorf("failed to prepare IPv4 insert statement: %w", err) - } - defer func() { _ = insertV4Stmt.Close() }() - - selectV6Stmt, err := tx.Prepare("SELECT id FROM prefixes_v6 WHERE prefix = ?") - if err != nil { - return fmt.Errorf("failed to prepare IPv6 select statement: %w", err) - } - defer func() { _ = selectV6Stmt.Close() }() - - updateV6Stmt, err := tx.Prepare("UPDATE prefixes_v6 SET last_seen = ? WHERE prefix = ?") - if err != nil { - return fmt.Errorf("failed to prepare IPv6 update statement: %w", err) - } - defer func() { _ = updateV6Stmt.Close() }() - - insertV6Stmt, err := tx.Prepare("INSERT INTO prefixes_v6 (id, prefix, first_seen, last_seen) VALUES (?, ?, ?, ?)") - if err != nil { - return fmt.Errorf("failed to prepare IPv6 insert statement: %w", err) - } - defer func() { _ = insertV6Stmt.Close() }() - - var newV4, newV6 int for prefix, timestamp := range prefixes { - ipVersion := detectIPVersion(prefix) - - var selectStmt, updateStmt, insertStmt *sql.Stmt - if ipVersion == ipVersionV4 { - selectStmt, updateStmt, insertStmt = selectV4Stmt, updateV4Stmt, insertV4Stmt + if detectIPVersion(prefix) == ipVersionV4 { + prefixesV4[prefix] = timestamp } else { - selectStmt, updateStmt, insertStmt = selectV6Stmt, updateV6Stmt, insertV6Stmt - } - - var id string - err = selectStmt.QueryRow(prefix).Scan(&id) - - switch err { - case nil: - // Prefix exists, update last_seen - _, err = updateStmt.Exec(timestamp, prefix) - if err != nil { - return fmt.Errorf("failed to update prefix %s: %w", prefix, err) - } - case sql.ErrNoRows: - // Prefix doesn't exist, create it - _, err = insertStmt.Exec(generateUUID().String(), prefix, timestamp, timestamp) - if err != nil { - return fmt.Errorf("failed to insert prefix %s: %w", prefix, err) - } - if ipVersion == ipVersionV4 { - newV4++ - } else { - newV6++ - } - default: - return fmt.Errorf("failed to query prefix %s: %w", prefix, err) + prefixesV6[prefix] = timestamp } } - if err = tx.Commit(); err != nil { + ctx := context.Background() + + newV4, err := upsertPrefixesInTable(ctx, tx, prefixesV4, ipVersionV4) + if err != nil { + return err + } + + newV6, err := upsertPrefixesInTable(ctx, tx, prefixesV6, ipVersionV6) + if err != nil { + return err + } + + err = tx.Commit() + if err != nil { return fmt.Errorf("failed to commit transaction: %w", err) } @@ -729,145 +820,10 @@ func (d *Database) UpdatePrefixesBatch(prefixes map[string]time.Time) error { return nil } -// GetOrCreateASNBatch creates or updates multiple ASNs in a single transaction -func (d *Database) GetOrCreateASNBatch(asns map[int]time.Time) error { - if len(asns) == 0 { - return nil - } - - d.lock("GetOrCreateASNBatch") - defer d.unlock() - - tx, err := d.beginTx() - if err != nil { - return fmt.Errorf("failed to begin transaction: %w", err) - } - defer func() { - if err := tx.Rollback(); err != nil && err != sql.ErrTxDone { - d.logger.Error("Failed to rollback transaction", "error", err) - } - }() - - // Prepare statements - selectStmt, err := tx.Prepare( - "SELECT asn, handle, description, first_seen, last_seen FROM asns WHERE asn = ?") - if err != nil { - return fmt.Errorf("failed to prepare select statement: %w", err) - } - defer func() { _ = selectStmt.Close() }() - - updateStmt, err := tx.Prepare("UPDATE asns SET last_seen = ? WHERE asn = ?") - if err != nil { - return fmt.Errorf("failed to prepare update statement: %w", err) - } - defer func() { _ = updateStmt.Close() }() - - insertStmt, err := tx.Prepare( - "INSERT INTO asns (asn, handle, description, first_seen, last_seen) VALUES (?, ?, ?, ?, ?)") - if err != nil { - return fmt.Errorf("failed to prepare insert statement: %w", err) - } - defer func() { _ = insertStmt.Close() }() - - var newASNs int - for number, timestamp := range asns { - var asn ASN - var handle, description sql.NullString - - err = selectStmt.QueryRow(number).Scan(&asn.ASN, &handle, &description, &asn.FirstSeen, &asn.LastSeen) - - if err == nil { - // ASN exists, update last_seen - _, err = updateStmt.Exec(timestamp, number) - if err != nil { - return fmt.Errorf("failed to update ASN %d: %w", number, err) - } - - continue - } - - if err == sql.ErrNoRows { - // ASN doesn't exist, create it - asn = ASN{ - ASN: number, - FirstSeen: timestamp, - LastSeen: timestamp, - } - - // Look up ASN info - if info, ok := asinfo.Get(number); ok { - asn.Handle = info.Handle - asn.Description = info.Description - } - - _, err = insertStmt.Exec(asn.ASN, asn.Handle, asn.Description, asn.FirstSeen, asn.LastSeen) - if err != nil { - return fmt.Errorf("failed to insert ASN %d: %w", number, err) - } - newASNs++ - - continue - } - - if err != nil { - return fmt.Errorf("failed to query ASN %d: %w", number, err) - } - } - - if err = tx.Commit(); err != nil { - return fmt.Errorf("failed to commit transaction: %w", err) - } - - d.counts.addASNs(newASNs) - - return nil -} - -// GetOrCreateASN retrieves an existing ASN or creates a new one if it doesn't exist. -func (d *Database) GetOrCreateASN(number int, timestamp time.Time) (*ASN, error) { - d.lock("GetOrCreateASN") - defer d.unlock() - - tx, err := d.beginTx() - if err != nil { - return nil, err - } - defer func() { - if err := tx.Rollback(); err != nil && err != sql.ErrTxDone { - d.logger.Error("Failed to rollback transaction", "error", err) - } - }() - - var asn ASN - var handle, description sql.NullString - err = tx.QueryRow("SELECT asn, handle, description, first_seen, last_seen FROM asns WHERE asn = ?", number). - Scan(&asn.ASN, &handle, &description, &asn.FirstSeen, &asn.LastSeen) - - if err == nil { - // ASN exists, update last_seen - asn.Handle = handle.String - asn.Description = description.String - _, err = tx.Exec("UPDATE asns SET last_seen = ? WHERE asn = ?", timestamp, number) - if err != nil { - return nil, err - } - asn.LastSeen = timestamp - - if err = tx.Commit(); err != nil { - d.logger.Error("Failed to commit transaction for ASN update", "asn", number, "error", err) - - return nil, err - } - - return &asn, nil - } - - if err != sql.ErrNoRows { - return nil, err - } - - // ASN doesn't exist, create it with ASN info lookup - asn = ASN{ +// newASN returns a new ASN record first and last seen at timestamp, with the +// handle and description from the AS info lookup when it knows the number. +func newASN(number int, timestamp time.Time) ASN { + asn := ASN{ ASN: number, FirstSeen: timestamp, LastSeen: timestamp, @@ -879,14 +835,177 @@ func (d *Database) GetOrCreateASN(number int, timestamp time.Time) (*ASN, error) asn.Description = info.Description } - _, err = tx.Exec("INSERT INTO asns (asn, handle, description, first_seen, last_seen) VALUES (?, ?, ?, ?, ?)", + return asn +} + +// upsertASN updates the last_seen time of ASN number with the given +// statements, inserting the ASN when it is new. It reports whether it inserted +// it. +func upsertASN( + ctx context.Context, selectStmt, updateStmt, insertStmt *sql.Stmt, + number int, timestamp time.Time, +) (bool, error) { + var ( + asn ASN + handle, description sql.NullString + ) + + err := selectStmt.QueryRowContext(ctx, number).Scan(&asn.ASN, &handle, + &description, &asn.FirstSeen, &asn.LastSeen) + if err == nil { + // ASN exists, update last_seen + _, err = updateStmt.ExecContext(ctx, timestamp, number) + if err != nil { + return false, fmt.Errorf("failed to update ASN %d: %w", number, err) + } + + return false, nil + } + + if !errors.Is(err, sql.ErrNoRows) { + return false, fmt.Errorf("failed to query ASN %d: %w", number, err) + } + + // ASN doesn't exist, create it + asn = newASN(number, timestamp) + + _, err = insertStmt.ExecContext(ctx, asn.ASN, asn.Handle, + asn.Description, asn.FirstSeen, asn.LastSeen) + if err != nil { + return false, fmt.Errorf("failed to insert ASN %d: %w", number, err) + } + + return true, nil +} + +// GetOrCreateASNBatch creates or updates multiple ASNs in a single transaction +func (d *Database) GetOrCreateASNBatch(asns map[int]time.Time) error { + if len(asns) == 0 { + return nil + } + + d.lock("GetOrCreateASNBatch") + defer d.unlock() + + ctx := context.Background() + + tx, err := d.beginTx() + if err != nil { + return fmt.Errorf("failed to begin transaction: %w", err) + } + defer d.rollback(tx) + + // Prepare statements + selectStmt, err := tx.PrepareContext(ctx, + "SELECT asn, handle, description, first_seen, last_seen "+ + "FROM asns WHERE asn = ?") + if err != nil { + return fmt.Errorf("failed to prepare select statement: %w", err) + } + defer func() { _ = selectStmt.Close() }() + + updateStmt, err := tx.PrepareContext(ctx, + "UPDATE asns SET last_seen = ? WHERE asn = ?") + if err != nil { + return fmt.Errorf("failed to prepare update statement: %w", err) + } + defer func() { _ = updateStmt.Close() }() + + insertStmt, err := tx.PrepareContext(ctx, + "INSERT INTO asns (asn, handle, description, first_seen, last_seen) "+ + "VALUES (?, ?, ?, ?, ?)") + if err != nil { + return fmt.Errorf("failed to prepare insert statement: %w", err) + } + defer func() { _ = insertStmt.Close() }() + + var newASNs int + + for number, timestamp := range asns { + inserted, err := upsertASN(ctx, selectStmt, updateStmt, insertStmt, + number, timestamp) + if err != nil { + return err + } + + if inserted { + newASNs++ + } + } + + err = tx.Commit() + if err != nil { + return fmt.Errorf("failed to commit transaction: %w", err) + } + + d.counts.addASNs(newASNs) + + return nil +} + +// GetOrCreateASN retrieves an existing ASN or creates a new one if it doesn't +// exist. +func (d *Database) GetOrCreateASN(number int, timestamp time.Time) (*ASN, error) { + d.lock("GetOrCreateASN") + defer d.unlock() + + tx, err := d.beginTx() + if err != nil { + return nil, err + } + defer d.rollback(tx) + + var ( + asn ASN + handle, description sql.NullString + ) + + err = tx.QueryRow("SELECT asn, handle, description, first_seen, last_seen "+ + "FROM asns WHERE asn = ?", number). + Scan(&asn.ASN, &handle, &description, &asn.FirstSeen, &asn.LastSeen) + if err == nil { + // ASN exists, update last_seen + asn.Handle = handle.String + asn.Description = description.String + + _, err = tx.Exec("UPDATE asns SET last_seen = ? WHERE asn = ?", + timestamp, number) + if err != nil { + return nil, err + } + + asn.LastSeen = timestamp + + err = tx.Commit() + if err != nil { + d.logger.Error("Failed to commit transaction for ASN update", + "asn", number, "error", err) + + return nil, err + } + + return &asn, nil + } + + if !errors.Is(err, sql.ErrNoRows) { + return nil, err + } + + // ASN doesn't exist, create it with ASN info lookup + asn = newASN(number, timestamp) + + _, err = tx.Exec("INSERT INTO asns "+ + "(asn, handle, description, first_seen, last_seen) "+ + "VALUES (?, ?, ?, ?, ?)", asn.ASN, asn.Handle, asn.Description, asn.FirstSeen, asn.LastSeen) if err != nil { return nil, err } - if err = tx.Commit(); err != nil { - d.logger.Error("Failed to commit transaction for ASN creation", "asn", number, "error", err) + err = tx.Commit() + if err != nil { + d.logger.Error("Failed to commit transaction for ASN creation", + "asn", number, "error", err) return nil, err } @@ -896,8 +1015,10 @@ func (d *Database) GetOrCreateASN(number int, timestamp time.Time) (*ASN, error) return &asn, nil } -// GetOrCreatePrefix retrieves an existing prefix or creates a new one if it doesn't exist. -func (d *Database) GetOrCreatePrefix(prefix string, timestamp time.Time) (*Prefix, error) { +// GetOrCreatePrefix retrieves an existing prefix or creates a new one if it +// doesn't exist. +func (d *Database) GetOrCreatePrefix( + prefix string, timestamp time.Time) (*Prefix, error) { d.lock("GetOrCreatePrefix") defer d.unlock() @@ -905,38 +1026,42 @@ func (d *Database) GetOrCreatePrefix(prefix string, timestamp time.Time) (*Prefi if err != nil { return nil, err } - defer func() { - if err := tx.Rollback(); err != nil && err != sql.ErrTxDone { - d.logger.Error("Failed to rollback transaction", "error", err) - } - }() + defer d.rollback(tx) // Determine table based on IP version ipVersion := detectIPVersion(prefix) + tableName := "prefixes_v4" if ipVersion == ipVersionV6 { tableName = "prefixes_v6" } - var p Prefix - var idStr string - query := fmt.Sprintf("SELECT id, prefix, first_seen, last_seen FROM %s WHERE prefix = ?", tableName) - err = tx.QueryRow(query, prefix). - Scan(&idStr, &p.Prefix, &p.FirstSeen, &p.LastSeen) + var ( + p Prefix + idStr string + ) + query := fmt.Sprintf("SELECT id, prefix, first_seen, last_seen "+ + "FROM %s WHERE prefix = ?", tableName) + + err = tx.QueryRow(query, prefix).Scan(&idStr, &p.Prefix, &p.FirstSeen, &p.LastSeen) if err == nil { // Prefix exists, update last_seen p.ID, _ = uuid.Parse(idStr) p.IPVersion = ipVersion updateQuery := fmt.Sprintf("UPDATE %s SET last_seen = ? WHERE id = ?", tableName) + _, err = tx.Exec(updateQuery, timestamp, p.ID.String()) if err != nil { return nil, err } + p.LastSeen = timestamp - if err = tx.Commit(); err != nil { - d.logger.Error("Failed to commit transaction for prefix update", "prefix", prefix, "error", err) + err = tx.Commit() + if err != nil { + d.logger.Error("Failed to commit transaction for prefix update", + "prefix", prefix, "error", err) return nil, err } @@ -944,26 +1069,25 @@ func (d *Database) GetOrCreatePrefix(prefix string, timestamp time.Time) (*Prefi return &p, nil } - if err != sql.ErrNoRows { + if !errors.Is(err, sql.ErrNoRows) { return nil, err } // Prefix doesn't exist, create it - p = Prefix{ - ID: generateUUID(), - Prefix: prefix, - IPVersion: ipVersion, - FirstSeen: timestamp, - LastSeen: timestamp, - } - insertQuery := fmt.Sprintf("INSERT INTO %s (id, prefix, first_seen, last_seen) VALUES (?, ?, ?, ?)", tableName) + p = Prefix{ID: generateUUID(), Prefix: prefix, IPVersion: ipVersion, + FirstSeen: timestamp, LastSeen: timestamp} + insertQuery := fmt.Sprintf("INSERT INTO %s "+ + "(id, prefix, first_seen, last_seen) VALUES (?, ?, ?, ?)", tableName) + _, err = tx.Exec(insertQuery, p.ID.String(), p.Prefix, p.FirstSeen, p.LastSeen) if err != nil { return nil, err } - if err = tx.Commit(); err != nil { - d.logger.Error("Failed to commit transaction for prefix creation", "prefix", prefix, "error", err) + err = tx.Commit() + if err != nil { + d.logger.Error("Failed to commit transaction for prefix creation", + "prefix", prefix, "error", err) return nil, err } @@ -977,17 +1101,20 @@ func (d *Database) GetOrCreatePrefix(prefix string, timestamp time.Time) (*Prefi return &p, nil } -// RecordAnnouncement inserts a new BGP announcement or withdrawal into the database. +// RecordAnnouncement inserts a new BGP announcement or withdrawal into the +// database. func (d *Database) RecordAnnouncement(announcement *Announcement) error { d.lock("RecordAnnouncement") defer d.unlock() err := d.exec(` - INSERT INTO announcements (id, prefix_id, peer_asn, origin_asn, path, next_hop, timestamp, is_withdrawal) + INSERT INTO announcements (id, prefix_id, peer_asn, origin_asn, path, + next_hop, timestamp, is_withdrawal) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, announcement.ID.String(), announcement.PrefixID.String(), announcement.PeerASN, announcement.OriginASN, - announcement.Path, announcement.NextHop, announcement.Timestamp, announcement.IsWithdrawal) + announcement.Path, announcement.NextHop, announcement.Timestamp, + announcement.IsWithdrawal) return err } @@ -996,10 +1123,11 @@ func (d *Database) RecordAnnouncement(announcement *Announcement) error { func (d *Database) RecordPeering(asA, asB int, timestamp time.Time) error { // Validate ASNs if asA <= 0 || asB <= 0 { - return fmt.Errorf("invalid ASN: asA=%d, asB=%d", asA, asB) + return fmt.Errorf("%w: asA=%d, asB=%d", errInvalidASN, asA, asB) } + if asA == asB { - return fmt.Errorf("cannot create peering with same ASN: %d", asA) + return fmt.Errorf("%w: %d", errSameASNInPeering, asA) } // Normalize: ensure asA < asB @@ -1014,21 +1142,20 @@ func (d *Database) RecordPeering(asA, asB int, timestamp time.Time) error { if err != nil { return err } - defer func() { - if err := tx.Rollback(); err != nil && err != sql.ErrTxDone { - d.logger.Error("Failed to rollback transaction", "error", err) - } - }() + defer d.rollback(tx) var exists bool - err = tx.QueryRow("SELECT EXISTS(SELECT 1 FROM peerings WHERE as_a = ? AND as_b = ?)", + + err = tx.QueryRow( + "SELECT EXISTS(SELECT 1 FROM peerings WHERE as_a = ? AND as_b = ?)", asA, asB).Scan(&exists) if err != nil { return err } if exists { - _, err = tx.Exec("UPDATE peerings SET last_seen = ? WHERE as_a = ? AND as_b = ?", + _, err = tx.Exec( + "UPDATE peerings SET last_seen = ? WHERE as_a = ? AND as_b = ?", timestamp, asA, asB) } else { _, err = tx.Exec(` @@ -1036,11 +1163,13 @@ func (d *Database) RecordPeering(asA, asB int, timestamp time.Time) error { VALUES (?, ?, ?, ?, ?)`, generateUUID().String(), asA, asB, timestamp, timestamp) } + if err != nil { return err } - if err = tx.Commit(); err != nil { + err = tx.Commit() + if err != nil { d.logger.Error("Failed to commit transaction for peering", "as_a", asA, "as_b", asB, @@ -1066,61 +1195,68 @@ func (d *Database) UpdatePeerBatch(peers map[string]PeerUpdate) error { d.lock("UpdatePeerBatch") defer d.unlock() + ctx := context.Background() + tx, err := d.beginTx() if err != nil { return fmt.Errorf("failed to begin transaction: %w", err) } - defer func() { - if err := tx.Rollback(); err != nil && err != sql.ErrTxDone { - d.logger.Error("Failed to rollback transaction", "error", err) - } - }() + defer d.rollback(tx) // Prepare statements - checkStmt, err := tx.Prepare("SELECT EXISTS(SELECT 1 FROM bgp_peers WHERE peer_ip = ?)") + checkStmt, err := tx.PrepareContext(ctx, + "SELECT EXISTS(SELECT 1 FROM bgp_peers WHERE peer_ip = ?)") if err != nil { return fmt.Errorf("failed to prepare check statement: %w", err) } defer func() { _ = checkStmt.Close() }() - updateStmt, err := tx.Prepare( - "UPDATE bgp_peers SET peer_asn = ?, last_seen = ?, last_message_type = ? WHERE peer_ip = ?") + updateStmt, err := tx.PrepareContext(ctx, + "UPDATE bgp_peers SET peer_asn = ?, last_seen = ?, "+ + "last_message_type = ? WHERE peer_ip = ?") if err != nil { return fmt.Errorf("failed to prepare update statement: %w", err) } defer func() { _ = updateStmt.Close() }() - insertStmt, err := tx.Prepare( - "INSERT INTO bgp_peers (id, peer_ip, peer_asn, first_seen, last_seen, last_message_type) VALUES (?, ?, ?, ?, ?, ?)") + insertStmt, err := tx.PrepareContext(ctx, + "INSERT INTO bgp_peers (id, peer_ip, peer_asn, first_seen, "+ + "last_seen, last_message_type) VALUES (?, ?, ?, ?, ?, ?)") if err != nil { return fmt.Errorf("failed to prepare insert statement: %w", err) } defer func() { _ = insertStmt.Close() }() var newPeers int + for _, update := range peers { var exists bool - err = checkStmt.QueryRow(update.PeerIP).Scan(&exists) + + err = checkStmt.QueryRowContext(ctx, update.PeerIP).Scan(&exists) if err != nil { return fmt.Errorf("failed to check peer %s: %w", update.PeerIP, err) } if exists { - _, err = updateStmt.Exec(update.PeerASN, update.Timestamp, update.MessageType, update.PeerIP) + _, err = updateStmt.ExecContext(ctx, update.PeerASN, + update.Timestamp, update.MessageType, update.PeerIP) } else { - _, err = insertStmt.Exec( + _, err = insertStmt.ExecContext(ctx, generateUUID().String(), update.PeerIP, update.PeerASN, update.Timestamp, update.Timestamp, update.MessageType) } + if err != nil { return fmt.Errorf("failed to update peer %s: %w", update.PeerIP, err) } + if !exists { newPeers++ } } - if err = tx.Commit(); err != nil { + err = tx.Commit() + if err != nil { return fmt.Errorf("failed to commit transaction: %w", err) } @@ -1130,7 +1266,9 @@ func (d *Database) UpdatePeerBatch(peers map[string]PeerUpdate) error { } // UpdatePeer updates or creates a BGP peer record -func (d *Database) UpdatePeer(peerIP string, peerASN int, messageType string, timestamp time.Time) error { +func (d *Database) UpdatePeer( + peerIP string, peerASN int, messageType string, timestamp time.Time, +) error { d.lock("UpdatePeer") defer d.unlock() @@ -1138,34 +1276,38 @@ func (d *Database) UpdatePeer(peerIP string, peerASN int, messageType string, ti if err != nil { return err } - defer func() { - if err := tx.Rollback(); err != nil && err != sql.ErrTxDone { - d.logger.Error("Failed to rollback transaction", "error", err) - } - }() + defer d.rollback(tx) var exists bool - err = tx.QueryRow("SELECT EXISTS(SELECT 1 FROM bgp_peers WHERE peer_ip = ?)", peerIP).Scan(&exists) + + err = tx.QueryRow( + "SELECT EXISTS(SELECT 1 FROM bgp_peers WHERE peer_ip = ?)", peerIP, + ).Scan(&exists) if err != nil { return err } if exists { _, err = tx.Exec( - "UPDATE bgp_peers SET peer_asn = ?, last_seen = ?, last_message_type = ? WHERE peer_ip = ?", + "UPDATE bgp_peers SET peer_asn = ?, last_seen = ?, "+ + "last_message_type = ? WHERE peer_ip = ?", peerASN, timestamp, messageType, peerIP, ) } else { _, err = tx.Exec( - "INSERT INTO bgp_peers (id, peer_ip, peer_asn, first_seen, last_seen, last_message_type) VALUES (?, ?, ?, ?, ?, ?)", - generateUUID().String(), peerIP, peerASN, timestamp, timestamp, messageType, + "INSERT INTO bgp_peers (id, peer_ip, peer_asn, first_seen, "+ + "last_seen, last_message_type) VALUES (?, ?, ?, ?, ?, ?)", + generateUUID().String(), peerIP, peerASN, timestamp, timestamp, + messageType, ) } + if err != nil { return err } - if err = tx.Commit(); err != nil { + err = tx.Commit() + if err != nil { d.logger.Error("Failed to commit transaction for peer update", "peer_ip", peerIP, "peer_asn", peerASN, @@ -1202,7 +1344,8 @@ func (d *Database) GetStatsContext(ctx context.Context) (Stats, error) { d.counts.fill(&stats) // Database file size is a cheap stat() on the file. - if fileInfo, err := os.Stat(d.path); err != nil { + fileInfo, err := os.Stat(d.path) + if err != nil { d.logger.Warn("Failed to get database file size", "error", err) } else { stats.FileSizeBytes = fileInfo.Size() @@ -1225,64 +1368,6 @@ func (d *Database) GetStatsContext(ctx context.Context) (Stats, error) { return stats, nil } -// routeTimestampRange returns the earliest and latest last_updated across both -// live route tables, or nil values when both tables are empty. Each query reads -// one row from an end of the last_updated index rather than scanning the tables. -func (d *Database) routeTimestampRange(ctx context.Context) (oldest, newest *time.Time, err error) { - oldestV4, ok, err := d.scanRouteTimestamp(ctx, - "SELECT last_updated FROM live_routes_v4 ORDER BY last_updated ASC LIMIT 1") - if err != nil { - return nil, nil, err - } - if ok { - oldest = &oldestV4 - } - - oldestV6, ok, err := d.scanRouteTimestamp(ctx, - "SELECT last_updated FROM live_routes_v6 ORDER BY last_updated ASC LIMIT 1") - if err != nil { - return nil, nil, err - } - if ok && (oldest == nil || oldestV6.Before(*oldest)) { - oldest = &oldestV6 - } - - newestV4, ok, err := d.scanRouteTimestamp(ctx, - "SELECT last_updated FROM live_routes_v4 ORDER BY last_updated DESC LIMIT 1") - if err != nil { - return nil, nil, err - } - if ok { - newest = &newestV4 - } - - newestV6, ok, err := d.scanRouteTimestamp(ctx, - "SELECT last_updated FROM live_routes_v6 ORDER BY last_updated DESC LIMIT 1") - if err != nil { - return nil, nil, err - } - if ok && (newest == nil || newestV6.After(*newest)) { - newest = &newestV6 - } - - return oldest, newest, nil -} - -// scanRouteTimestamp runs a single-row timestamp query. ok is false when the -// table is empty. The query selects the last_updated column directly so the -// driver parses the DATETIME value into a time.Time. -func (d *Database) scanRouteTimestamp(ctx context.Context, query string) (ts time.Time, ok bool, err error) { - err = d.db.QueryRowContext(ctx, query).Scan(&ts) - switch { - case errors.Is(err, sql.ErrNoRows): - return time.Time{}, false, nil - case err != nil: - return time.Time{}, false, err - default: - return ts, true, nil - } -} - // UpsertLiveRoute inserts or updates a live route func (d *Database) UpsertLiveRoute(route *LiveRoute) error { d.lock("UpsertLiveRoute") @@ -1293,37 +1378,37 @@ func (d *Database) UpsertLiveRoute(route *LiveRoute) error { return fmt.Errorf("failed to encode AS path: %w", err) } - updateSQL, insertSQL, lookupSQL := updateLiveRouteV4SQL, insertLiveRouteV4SQL, prefixHasLiveRouteV4SQL + updateSQL, insertSQL, lookupSQL := updateLiveRouteV4SQL, + insertLiveRouteV4SQL, prefixHasLiveRouteV4SQL if route.IPVersion == ipVersionV6 { - updateSQL, insertSQL, lookupSQL = updateLiveRouteV6SQL, insertLiveRouteV6SQL, prefixHasLiveRouteV6SQL + updateSQL, insertSQL, lookupSQL = updateLiveRouteV6SQL, + insertLiveRouteV6SQL, prefixHasLiveRouteV6SQL } + ctx := context.Background() + // The write lock is held, so no other writer can insert this key between the // update and the insert even though they are separate autocommit statements. - upd, err := d.db.Prepare(updateSQL) + upd, err := d.db.PrepareContext(ctx, updateSQL) if err != nil { return fmt.Errorf("failed to prepare update statement: %w", err) } defer func() { _ = upd.Close() }() - ins, err := d.db.Prepare(insertSQL) + ins, err := d.db.PrepareContext(ctx, insertSQL) if err != nil { return fmt.Errorf("failed to prepare insert statement: %w", err) } defer func() { _ = ins.Close() }() - has, err := d.db.Prepare(lookupSQL) + has, err := d.db.PrepareContext(ctx, lookupSQL) if err != nil { return fmt.Errorf("failed to prepare prefix lookup statement: %w", err) } defer func() { _ = has.Close() }() - var inserted, newPrefix bool - if route.IPVersion == ipVersionV4 { - inserted, newPrefix, err = upsertRouteRowV4(upd, ins, has, route, string(pathJSON)) - } else { - inserted, newPrefix, err = upsertRouteRowV6(upd, ins, has, route, string(pathJSON)) - } + inserted, newPrefix, err := upsertRouteRow(ctx, upd, ins, has, route, + string(pathJSON)) if err != nil { return fmt.Errorf("failed to upsert route %s: %w", route.Prefix, err) } @@ -1335,6 +1420,7 @@ func (d *Database) UpsertLiveRoute(route *LiveRoute) error { d.counts.addRoutes(0, 1) } } + if newPrefix { if route.IPVersion == ipVersionV4 { d.counts.addToDistribution([]int{route.MaskLength}, nil, 1) @@ -1365,20 +1451,16 @@ func (d *Database) DeleteLiveRoute(prefix, peerIP string) error { if err != nil { return fmt.Errorf("failed to begin transaction: %w", err) } - defer func() { - if err := tx.Rollback(); err != nil && err != sql.ErrTxDone { - d.logger.Error("Failed to rollback transaction", "error", err) - } - }() + defer d.rollback(tx) // Literal per-table queries (rather than one formatted with the table name) // so the delete carries no dynamically built SQL. - var res sql.Result + deleteSQL := deleteLiveRouteV6SQL if isV4 { - res, err = tx.Exec(`DELETE FROM live_routes_v4 WHERE prefix = ? AND peer_ip = ?`, prefix, peerIP) - } else { - res, err = tx.Exec(`DELETE FROM live_routes_v6 WHERE prefix = ? AND peer_ip = ?`, prefix, peerIP) + deleteSQL = deleteLiveRouteV4SQL } + + res, err := tx.Exec(deleteSQL, prefix, peerIP) if err != nil { return err } @@ -1387,6 +1469,7 @@ func (d *Database) DeleteLiveRoute(prefix, peerIP string) error { if err != nil { return err } + if affected == 0 { return nil } @@ -1396,12 +1479,16 @@ func (d *Database) DeleteLiveRoute(prefix, peerIP string) error { if isV4 { lookupSQL = prefixHasLiveRouteV4SQL } + var prefixHasRoute bool - if err := tx.QueryRow(lookupSQL, prefix).Scan(&prefixHasRoute); err != nil { + + err = tx.QueryRow(lookupSQL, prefix).Scan(&prefixHasRoute) + if err != nil { return err } - if err := tx.Commit(); err != nil { + err = tx.Commit() + if err != nil { return fmt.Errorf("failed to commit transaction: %w", err) } @@ -1410,6 +1497,7 @@ func (d *Database) DeleteLiveRoute(prefix, peerIP string) error { } else { d.counts.addRoutes(0, -int(affected)) } + if !prefixHasRoute { maskLength, _ := ipnet.Mask.Size() if isV4 { @@ -1422,16 +1510,22 @@ func (d *Database) DeleteLiveRoute(prefix, peerIP string) error { return nil } -// GetPrefixDistribution returns the distribution of unique prefixes by mask length -func (d *Database) GetPrefixDistribution() (ipv4 []PrefixDistribution, ipv6 []PrefixDistribution, err error) { +// GetPrefixDistribution returns the distribution of unique prefixes by mask +// length +func (d *Database) GetPrefixDistribution() ( + []PrefixDistribution, []PrefixDistribution, error, +) { return d.GetPrefixDistributionContext(context.Background()) } -// GetPrefixDistributionContext returns the distribution of unique prefixes by mask length with context support. -// It reads every live route, so the stats read does not call it; it seeds the -// in-memory distribution once at startup. +// GetPrefixDistributionContext returns the distribution of unique prefixes by +// mask length with context support. It reads every live route, so the stats +// read does not call it; it seeds the in-memory distribution once at startup. func (d *Database) GetPrefixDistributionContext(ctx context.Context) ( - ipv4 []PrefixDistribution, ipv6 []PrefixDistribution, err error) { + []PrefixDistribution, []PrefixDistribution, error, +) { + var ipv4, ipv6 []PrefixDistribution + // IPv4 distribution - count unique prefixes from v4 table query := ` SELECT mask_length, COUNT(DISTINCT prefix) as count @@ -1439,6 +1533,7 @@ func (d *Database) GetPrefixDistributionContext(ctx context.Context) ( GROUP BY mask_length ORDER BY mask_length ` + rows4, err := d.db.QueryContext(ctx, query) if err != nil { return nil, nil, fmt.Errorf("failed to query IPv4 distribution: %w", err) @@ -1451,13 +1546,19 @@ func (d *Database) GetPrefixDistributionContext(ctx context.Context) ( for rows4.Next() { var dist PrefixDistribution - if err := rows4.Scan(&dist.MaskLength, &dist.Count); err != nil { - return nil, nil, fmt.Errorf("failed to scan IPv4 distribution: %w", err) + + err := rows4.Scan(&dist.MaskLength, &dist.Count) + if err != nil { + return nil, nil, fmt.Errorf("failed to scan IPv4 distribution: %w", + err) } + ipv4 = append(ipv4, dist) } + // A read that stops partway ends the loop without an error of its own. - if err := rows4.Err(); err != nil { + err = rows4.Err() + if err != nil { return nil, nil, fmt.Errorf("failed to read IPv4 distribution: %w", err) } @@ -1468,6 +1569,7 @@ func (d *Database) GetPrefixDistributionContext(ctx context.Context) ( GROUP BY mask_length ORDER BY mask_length ` + rows6, err := d.db.QueryContext(ctx, query) if err != nil { return nil, nil, fmt.Errorf("failed to query IPv6 distribution: %w", err) @@ -1480,12 +1582,18 @@ func (d *Database) GetPrefixDistributionContext(ctx context.Context) ( for rows6.Next() { var dist PrefixDistribution - if err := rows6.Scan(&dist.MaskLength, &dist.Count); err != nil { - return nil, nil, fmt.Errorf("failed to scan IPv6 distribution: %w", err) + + err := rows6.Scan(&dist.MaskLength, &dist.Count) + if err != nil { + return nil, nil, fmt.Errorf("failed to scan IPv6 distribution: %w", + err) } + ipv6 = append(ipv6, dist) } - if err := rows6.Err(); err != nil { + + err = rows6.Err() + if err != nil { return nil, nil, fmt.Errorf("failed to read IPv6 distribution: %w", err) } @@ -1493,20 +1601,27 @@ func (d *Database) GetPrefixDistributionContext(ctx context.Context) ( } // GetLiveRouteCounts returns the count of IPv4 and IPv6 routes -func (d *Database) GetLiveRouteCounts() (ipv4Count, ipv6Count int, err error) { +func (d *Database) GetLiveRouteCounts() (int, int, error) { return d.GetLiveRouteCountsContext(context.Background()) } -// GetLiveRouteCountsContext returns the count of IPv4 and IPv6 routes with context support -func (d *Database) GetLiveRouteCountsContext(ctx context.Context) (ipv4Count, ipv6Count int, err error) { +// GetLiveRouteCountsContext returns the count of IPv4 and IPv6 routes with +// context support +func (d *Database) GetLiveRouteCountsContext( + ctx context.Context, +) (int, int, error) { + var ipv4Count, ipv6Count int + // Get IPv4 count from dedicated table - err = d.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM live_routes_v4").Scan(&ipv4Count) + err := d.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM live_routes_v4"). + Scan(&ipv4Count) if err != nil { return 0, 0, fmt.Errorf("failed to count IPv4 routes: %w", err) } // Get IPv6 count from dedicated table - err = d.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM live_routes_v6").Scan(&ipv6Count) + err = d.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM live_routes_v6"). + Scan(&ipv6Count) if err != nil { return 0, 0, fmt.Errorf("failed to count IPv6 routes: %w", err) } @@ -1519,18 +1634,25 @@ func (d *Database) GetASDetails(asn int) (*ASN, []LiveRoute, error) { return d.GetASDetailsContext(context.Background(), asn) } -// GetASDetailsContext returns detailed information about an AS including prefixes with context support -func (d *Database) GetASDetailsContext(ctx context.Context, asn int) (*ASN, []LiveRoute, error) { +// GetASDetailsContext returns detailed information about an AS including +// prefixes with context support +func (d *Database) GetASDetailsContext( + ctx context.Context, asn int, +) (*ASN, []LiveRoute, error) { // Get AS information - var asnInfo ASN - var handle, description sql.NullString - err := d.db.QueryRowContext(ctx, - "SELECT asn, handle, description, first_seen, last_seen FROM asns WHERE asn = ?", - asn, - ).Scan(&asnInfo.ASN, &handle, &description, &asnInfo.FirstSeen, &asnInfo.LastSeen) + var ( + asnInfo ASN + handle, description sql.NullString + ) + err := d.db.QueryRowContext(ctx, + "SELECT asn, handle, description, first_seen, last_seen "+ + "FROM asns WHERE asn = ?", + asn, + ).Scan(&asnInfo.ASN, &handle, &description, &asnInfo.FirstSeen, + &asnInfo.LastSeen) if err != nil { - if err == sql.ErrNoRows { + if errors.Is(err, sql.ErrNoRows) { return nil, nil, fmt.Errorf("%w: AS%d", ErrNoRoute, asn) } @@ -1541,9 +1663,6 @@ func (d *Database) GetASDetailsContext(ctx context.Context, asn int) (*ASN, []Li asnInfo.Description = description.String // Get prefixes announced by this AS from both tables - var allPrefixes []LiveRoute - - // Query IPv4 prefixes queryV4 := ` SELECT prefix, mask_length, MAX(last_updated) as last_updated FROM live_routes_v4 @@ -1551,38 +1670,11 @@ func (d *Database) GetASDetailsContext(ctx context.Context, asn int) (*ASN, []Li GROUP BY prefix, mask_length ` - rows4, err := d.db.QueryContext(ctx, queryV4, asn) + allPrefixes, err := d.originPrefixes(ctx, queryV4, asn, ipVersionV4) if err != nil { - return &asnInfo, nil, fmt.Errorf("failed to query IPv4 prefixes: %w", err) - } - defer func() { _ = rows4.Close() }() - - for rows4.Next() { - var route LiveRoute - var lastUpdatedStr string - err := rows4.Scan(&route.Prefix, &route.MaskLength, &lastUpdatedStr) - if err != nil { - d.logger.Error("Failed to scan IPv4 prefix row", "error", err, "asn", asn) - - continue - } - // Parse the timestamp string - route.LastUpdated, err = time.Parse("2006-01-02 15:04:05-07:00", lastUpdatedStr) - if err != nil { - // Try without timezone - route.LastUpdated, err = time.Parse("2006-01-02 15:04:05", lastUpdatedStr) - if err != nil { - d.logger.Error("Failed to parse timestamp", "error", err, "timestamp", lastUpdatedStr) - - continue - } - } - route.OriginASN = asn - route.IPVersion = ipVersionV4 - allPrefixes = append(allPrefixes, route) + return &asnInfo, nil, err } - // Query IPv6 prefixes queryV6 := ` SELECT prefix, mask_length, MAX(last_updated) as last_updated FROM live_routes_v6 @@ -1590,41 +1682,18 @@ func (d *Database) GetASDetailsContext(ctx context.Context, asn int) (*ASN, []Li GROUP BY prefix, mask_length ` - rows6, err := d.db.QueryContext(ctx, queryV6, asn) + prefixesV6, err := d.originPrefixes(ctx, queryV6, asn, ipVersionV6) if err != nil { - return &asnInfo, allPrefixes, fmt.Errorf("failed to query IPv6 prefixes: %w", err) - } - defer func() { _ = rows6.Close() }() - - for rows6.Next() { - var route LiveRoute - var lastUpdatedStr string - err := rows6.Scan(&route.Prefix, &route.MaskLength, &lastUpdatedStr) - if err != nil { - d.logger.Error("Failed to scan IPv6 prefix row", "error", err, "asn", asn) - - continue - } - // Parse the timestamp string - route.LastUpdated, err = time.Parse("2006-01-02 15:04:05-07:00", lastUpdatedStr) - if err != nil { - // Try without timezone - route.LastUpdated, err = time.Parse("2006-01-02 15:04:05", lastUpdatedStr) - if err != nil { - d.logger.Error("Failed to parse timestamp", "error", err, "timestamp", lastUpdatedStr) - - continue - } - } - route.OriginASN = asn - route.IPVersion = ipVersionV6 - allPrefixes = append(allPrefixes, route) + return &asnInfo, allPrefixes, err } - return &asnInfo, allPrefixes, nil + return &asnInfo, append(allPrefixes, prefixesV6...), nil } -// ASPeer represents a peering relationship with another AS including handle, description, and timestamps. +// ASPeer represents a peering relationship with another AS including handle, +// description, and timestamps. +// +//nolint:tagliatelle // field names are the existing JSON API type ASPeer struct { ASN int `json:"asn"` Handle string `json:"handle"` @@ -1638,8 +1707,11 @@ func (d *Database) GetASPeers(asn int) ([]ASPeer, error) { return d.GetASPeersContext(context.Background(), asn) } -// GetASPeersContext returns all ASes that peer with the given AS with context support -func (d *Database) GetASPeersContext(ctx context.Context, asn int) ([]ASPeer, error) { +// GetASPeersContext returns all ASes that peer with the given AS with context +// support +func (d *Database) GetASPeersContext( + ctx context.Context, asn int, +) ([]ASPeer, error) { query := ` SELECT CASE @@ -1666,18 +1738,23 @@ func (d *Database) GetASPeersContext(ctx context.Context, asn int) ([]ASPeer, er defer func() { _ = rows.Close() }() var peers []ASPeer + for rows.Next() { var peer ASPeer - err := rows.Scan(&peer.ASN, &peer.Handle, &peer.Description, &peer.FirstSeen, &peer.LastSeen) + + err := rows.Scan(&peer.ASN, &peer.Handle, &peer.Description, + &peer.FirstSeen, &peer.LastSeen) if err != nil { d.logger.Error("Failed to scan peer row", "error", err, "asn", asn) continue } + peers = append(peers, peer) } - if err := rows.Err(); err != nil { + err = rows.Err() + if err != nil { return nil, fmt.Errorf("error iterating AS peers: %w", err) } @@ -1685,30 +1762,34 @@ func (d *Database) GetASPeersContext(ctx context.Context, asn int) ([]ASPeer, er } // GetPrefixDetails returns detailed information about a prefix -func (d *Database) GetPrefixDetails(prefix string) ([]LiveRoute, map[int]ASN, error) { +func (d *Database) GetPrefixDetails( + prefix string, +) ([]LiveRoute, map[int]ASN, error) { return d.GetPrefixDetailsContext(context.Background(), prefix) } // GetPrefixDetailsContext returns the live routes for a prefix and, keyed by // AS number, the handle and description of each AS that originates it. -func (d *Database) GetPrefixDetailsContext(ctx context.Context, prefix string) ([]LiveRoute, map[int]ASN, error) { +func (d *Database) GetPrefixDetailsContext( + ctx context.Context, prefix string, +) ([]LiveRoute, map[int]ASN, error) { // Determine if it's IPv4 or IPv6 by parsing the prefix _, ipnet, err := net.ParseCIDR(prefix) if err != nil { return nil, nil, fmt.Errorf("invalid prefix format: %w", err) } - tableName := "live_routes_v4" + tableName := tableLiveRoutesV4 ipVersion := ipVersionV4 + if ipnet.IP.To4() == nil { - tableName = "live_routes_v6" + tableName = tableLiveRoutesV6 ipVersion = ipVersionV6 } - //nolint:gosec // Table name is hardcoded based on IP version query := fmt.Sprintf(` - SELECT lr.origin_asn, lr.peer_ip, lr.as_path, lr.next_hop, lr.last_updated, - a.handle, a.description + SELECT lr.origin_asn, lr.peer_ip, lr.as_path, lr.next_hop, + lr.last_updated, a.handle, a.description FROM %s lr LEFT JOIN asns a ON a.asn = lr.origin_asn WHERE lr.prefix = ? @@ -1722,11 +1803,15 @@ func (d *Database) GetPrefixDetailsContext(ctx context.Context, prefix string) ( defer func() { _ = rows.Close() }() var routes []LiveRoute + origins := make(map[int]ASN) + for rows.Next() { - var route LiveRoute - var pathJSON string - var handle, description sql.NullString + var ( + route LiveRoute + pathJSON string + handle, description sql.NullString + ) err := rows.Scan( &route.OriginASN, &route.PeerIP, &pathJSON, &route.NextHop, @@ -1737,7 +1822,8 @@ func (d *Database) GetPrefixDetailsContext(ctx context.Context, prefix string) ( } // Decode AS path - if err := json.Unmarshal([]byte(pathJSON), &route.ASPath); err != nil { + err = json.Unmarshal([]byte(pathJSON), &route.ASPath) + if err != nil { route.ASPath = []int{} } @@ -1752,6 +1838,11 @@ func (d *Database) GetPrefixDetailsContext(ctx context.Context, prefix string) ( } } + err = rows.Err() + if err != nil { + return nil, nil, fmt.Errorf("failed to read prefix details: %w", err) + } + if len(routes) == 0 { return nil, nil, fmt.Errorf("%w: %s", ErrNoRoute, prefix) } @@ -1759,9 +1850,13 @@ func (d *Database) GetPrefixDetailsContext(ctx context.Context, prefix string) ( return routes, origins, nil } -// GetRandomPrefixesByLength returns a random sample of prefixes with the specified mask length -func (d *Database) GetRandomPrefixesByLength(maskLength, ipVersion, limit int) ([]LiveRoute, error) { - return d.GetRandomPrefixesByLengthContext(context.Background(), maskLength, ipVersion, limit) +// GetRandomPrefixesByLength returns a random sample of prefixes with the +// specified mask length +func (d *Database) GetRandomPrefixesByLength( + maskLength, ipVersion, limit int, +) ([]LiveRoute, error) { + return d.GetRandomPrefixesByLengthContext(context.Background(), + maskLength, ipVersion, limit) } // GetRandomPrefixesByLengthContext returns limit prefixes with the given mask @@ -1773,205 +1868,41 @@ func (d *Database) GetRandomPrefixesByLength(maskLength, ipVersion, limit int) ( // index search for each listed prefix's newest route. func (d *Database) GetRandomPrefixesByLengthContext( ctx context.Context, maskLength, ipVersion, limit int) ([]LiveRoute, error) { - tableName := "live_routes_v4" + tableName := tableLiveRoutesV4 if ipVersion == ipVersionV6 { - tableName = "live_routes_v6" + tableName = tableLiveRoutesV6 } // Reading one prefix more than maxPrefixesToShuffle tells whether these // are all the prefixes with this mask length. - prefixes, err := d.firstPrefixesByLength(ctx, tableName, maskLength, maxPrefixesToShuffle+1) + prefixes, err := d.firstPrefixesByLength(ctx, tableName, maskLength, + maxPrefixesToShuffle+1) if err != nil { return nil, err } + + //nolint:gosec // The sample needs variety, not unpredictability. rand.Shuffle(len(prefixes), func(i, j int) { prefixes[i], prefixes[j] = prefixes[j], prefixes[i] }) + if len(prefixes) > maxPrefixesToShuffle { - prefixes, err = d.samplePrefixesByLength(ctx, tableName, maskLength, limit, prefixes) + prefixes, err = d.samplePrefixesByLength(ctx, tableName, maskLength, + limit, prefixes) if err != nil { return nil, err } } + prefixes = prefixes[:min(limit, len(prefixes))] return d.newestRouteOfEachPrefix(ctx, tableName, ipVersion, prefixes) } -// firstPrefixesByLength returns the first n prefixes, in text order, with the -// given mask length. Each prefix is one search of the mask length and prefix -// index for the prefix after the previous one, so the read does not grow with -// the number of peers that carry a prefix, as a SELECT DISTINCT would. -func (d *Database) firstPrefixesByLength( - ctx context.Context, tableName string, maskLength, n int) ([]string, error) { - //nolint:gosec // Table name is hardcoded based on IP version - query := fmt.Sprintf(` - SELECT prefix FROM %s - WHERE mask_length = ? AND prefix > ? - ORDER BY prefix - LIMIT 1 - `, tableName) - - stmt, err := d.db.PrepareContext(ctx, query) - if err != nil { - return nil, fmt.Errorf("failed to prepare prefix query: %w", err) - } - defer func() { _ = stmt.Close() }() - - var prefixes []string - previous := "" // every prefix sorts after the empty string - for len(prefixes) < n { - var prefix string - err := stmt.QueryRowContext(ctx, maskLength, previous).Scan(&prefix) - if errors.Is(err, sql.ErrNoRows) { - break - } - if err != nil { - return nil, fmt.Errorf("failed to list prefixes: %w", err) - } - prefixes = append(prefixes, prefix) - previous = prefix - } - - return prefixes, nil -} - -// samplePrefixesByLength picks limit distinct prefixes with the given mask -// length at random, in at most sampleLookupsPerPrefix lookups for each prefix -// asked for. ORDER BY RANDOM() would read every route with that mask length. -// Instead each lookup draws a row id between those of the first and last route -// with that mask length and takes the prefix of the first such route at or -// after it. SQLite keeps the row id at the end of every index entry, so the -// mask length index is ordered by mask length, then row id, and the lookup is -// one search of it. -// -// The sample is uneven, as the page needs a varied sample, not an even one: a -// route is picked in proportion to the row ids between it and the previous -// route with that mask length, including the row ids of routes with other mask -// lengths and of withdrawn routes. A route that follows a long run of -// withdrawn routes is picked far more often than one that directly follows -// another route with that mask length. -// -// Should the lookups keep landing on prefixes already picked, the rest are -// taken from shuffled, prefixes of that length in random order. -func (d *Database) samplePrefixesByLength( - ctx context.Context, tableName string, maskLength, limit int, shuffled []string) ([]string, error) { - //nolint:gosec // Table name is hardcoded based on IP version - rangeQuery := fmt.Sprintf(` - SELECT - (SELECT MIN(rowid) FROM %[1]s WHERE mask_length = ?), - (SELECT MAX(rowid) FROM %[1]s WHERE mask_length = ?) - `, tableName) - - var first, last sql.NullInt64 - if err := d.db.QueryRowContext(ctx, rangeQuery, maskLength, maskLength).Scan(&first, &last); err != nil { - return nil, fmt.Errorf("failed to read row id range: %w", err) - } - if !first.Valid { - // Every route with this mask length was withdrawn since they were listed. - return nil, nil - } - - //nolint:gosec // Table name is hardcoded based on IP version - lookupQuery := fmt.Sprintf(` - SELECT prefix FROM %s - WHERE mask_length = ? AND rowid >= ? - ORDER BY rowid - LIMIT 1 - `, tableName) - - stmt, err := d.db.PrepareContext(ctx, lookupQuery) - if err != nil { - return nil, fmt.Errorf("failed to prepare prefix lookup: %w", err) - } - defer func() { _ = stmt.Close() }() - - picked := make(map[string]bool, limit) - prefixes := make([]string, 0, limit) - pick := func(prefix string) { - if !picked[prefix] { - picked[prefix] = true - prefixes = append(prefixes, prefix) - } - } - - for lookups := 0; lookups < limit*sampleLookupsPerPrefix && len(prefixes) < limit; lookups++ { - //nolint:gosec // The sample needs variety, not unpredictability. - rowID := first.Int64 + rand.Int64N(last.Int64-first.Int64+1) - - var prefix string - err := stmt.QueryRowContext(ctx, maskLength, rowID).Scan(&prefix) - if errors.Is(err, sql.ErrNoRows) { - // The routes from rowID on were withdrawn since the range was read. - continue - } - if err != nil { - return nil, fmt.Errorf("failed to look up prefix: %w", err) - } - pick(prefix) - } - - for _, prefix := range shuffled { - if len(prefixes) == limit { - break - } - pick(prefix) - } - - return prefixes, nil -} - -// newestRouteOfEachPrefix returns the most recently updated live route of each -// prefix, one search of the prefix and last_updated index per prefix however -// many peers carry it. A prefix whose routes were all withdrawn since it was -// picked is left out. -func (d *Database) newestRouteOfEachPrefix( - ctx context.Context, tableName string, ipVersion int, prefixes []string) ([]LiveRoute, error) { - //nolint:gosec // Table name is hardcoded based on IP version - query := fmt.Sprintf(` - SELECT prefix, mask_length, origin_asn, as_path, peer_ip, last_updated - FROM %s - WHERE prefix = ? - ORDER BY last_updated DESC - LIMIT 1 - `, tableName) - - stmt, err := d.db.PrepareContext(ctx, query) - if err != nil { - return nil, fmt.Errorf("failed to prepare route query: %w", err) - } - defer func() { _ = stmt.Close() }() - - routes := make([]LiveRoute, 0, len(prefixes)) - for _, prefix := range prefixes { - route := LiveRoute{IPVersion: ipVersion} - var pathJSON string - err := stmt.QueryRowContext(ctx, prefix).Scan( - &route.Prefix, - &route.MaskLength, - &route.OriginASN, - &pathJSON, - &route.PeerIP, - &route.LastUpdated, - ) - if errors.Is(err, sql.ErrNoRows) { - continue - } - if err != nil { - return nil, fmt.Errorf("failed to query route of %s: %w", prefix, err) - } - - if err := json.Unmarshal([]byte(pathJSON), &route.ASPath); err != nil { - route.ASPath = []int{} - } - routes = append(routes, route) - } - - return routes, nil -} - // GetNextStaleASN returns a random ASN that needs WHOIS data refresh. -func (d *Database) GetNextStaleASN(ctx context.Context, staleThreshold time.Duration) (int, error) { +func (d *Database) GetNextStaleASN( + ctx context.Context, staleThreshold time.Duration, +) (int, error) { cutoff := time.Now().Add(-staleThreshold) // Select a random stale ASN using ORDER BY RANDOM() @@ -1984,9 +1915,10 @@ func (d *Database) GetNextStaleASN(ctx context.Context, staleThreshold time.Dura ` var asn int + err := d.db.QueryRowContext(ctx, query, cutoff).Scan(&asn) if err != nil { - if err == sql.ErrNoRows { + if errors.Is(err, sql.ErrNoRows) { return 0, ErrNoStaleASN } @@ -1997,6 +1929,8 @@ func (d *Database) GetNextStaleASN(ctx context.Context, staleThreshold time.Dura } // WHOISStats contains statistics about WHOIS data freshness. +// +//nolint:tagliatelle // field names are the existing JSON API type WHOISStats struct { TotalASNs int `json:"total_asns"` StaleASNs int `json:"stale_asns"` @@ -2005,19 +1939,25 @@ type WHOISStats struct { } // GetWHOISStats returns statistics about WHOIS data freshness. -func (d *Database) GetWHOISStats(ctx context.Context, staleThreshold time.Duration) (*WHOISStats, error) { +func (d *Database) GetWHOISStats( + ctx context.Context, staleThreshold time.Duration, +) (*WHOISStats, error) { cutoff := time.Now().Add(-staleThreshold) query := ` SELECT COUNT(*) as total, - COALESCE(SUM(CASE WHEN whois_updated_at IS NULL THEN 1 ELSE 0 END), 0) as never_fetched, - COALESCE(SUM(CASE WHEN whois_updated_at IS NOT NULL AND whois_updated_at < ? THEN 1 ELSE 0 END), 0) as stale, - COALESCE(SUM(CASE WHEN whois_updated_at IS NOT NULL AND whois_updated_at >= ? THEN 1 ELSE 0 END), 0) as fresh + COALESCE(SUM(CASE WHEN whois_updated_at IS NULL + THEN 1 ELSE 0 END), 0) as never_fetched, + COALESCE(SUM(CASE WHEN whois_updated_at IS NOT NULL + AND whois_updated_at < ? THEN 1 ELSE 0 END), 0) as stale, + COALESCE(SUM(CASE WHEN whois_updated_at IS NOT NULL + AND whois_updated_at >= ? THEN 1 ELSE 0 END), 0) as fresh FROM asns ` var stats WHOISStats + err := d.db.QueryRowContext(ctx, query, cutoff, cutoff).Scan( &stats.TotalASNs, &stats.NeverFetched, @@ -2032,7 +1972,9 @@ func (d *Database) GetWHOISStats(ctx context.Context, staleThreshold time.Durati } // UpdateASNWHOIS updates an ASN record with WHOIS data. -func (d *Database) UpdateASNWHOIS(ctx context.Context, update *ASNWHOISUpdate) error { +func (d *Database) UpdateASNWHOIS( + ctx context.Context, update *ASNWHOISUpdate, +) error { d.lock("UpdateASNWHOIS") defer d.unlock() @@ -2084,41 +2026,17 @@ func (d *Database) GetIPInfo(ip string) (*IPInfo, error) { return d.GetIPInfoContext(context.Background(), ip) } -// GetIPInfoContext returns comprehensive IP information with context support. -// -// It finds the most specific live route for the address with point lookups on -// the prefix index: for each mask length from the longest (32 or 128) down to -// minLookupMaskLengthV4 or minLookupMaskLengthV6 it looks up the address's -// network prefix at that length, and the first length with a live route wins. -// That is at most 25 or 113 indexed lookups, however many routes are live. The -// prefix is looked up in the text form net/netip prints, which is the form the -// prefix handler stores. -func (d *Database) GetIPInfoContext(ctx context.Context, ip string) (*IPInfo, error) { - addr, err := netip.ParseAddr(ip) - if err != nil { - return nil, fmt.Errorf("%w: %s", ErrInvalidIP, ip) - } - // An IPv4 address written in IPv6 form (::ffff:192.0.2.1) is an IPv4 lookup. - addr = addr.Unmap() - - info := &IPInfo{IP: ip, IPVersion: ipVersionV6} - routesTable, prefixesTable := "live_routes_v6", "prefixes_v6" - minMaskLength := minLookupMaskLengthV6 - if addr.Is4() { - info.IPVersion = ipVersionV4 - routesTable, prefixesTable = "live_routes_v4", "prefixes_v4" - minMaskLength = minLookupMaskLengthV4 - } - - // Route info with peer count and prefix first_seen - //nolint:gosec // Table names are hardcoded based on IP version - query := fmt.Sprintf(` +// ipInfoQuery reads the live route for one prefix with its peer count, the +// prefix's first_seen and the origin AS's details. %[1]s is the live route +// table and %[2]s the prefix table of the address's IP version. +const ipInfoQuery = ` SELECT lr.prefix, lr.mask_length, lr.origin_asn, lr.last_updated, - (SELECT COUNT(DISTINCT peer_ip) FROM %[1]s WHERE prefix = lr.prefix) as num_peers, + (SELECT COUNT(DISTINCT peer_ip) FROM %[1]s + WHERE prefix = lr.prefix) as num_peers, p.first_seen, a.handle, a.description, @@ -2135,7 +2053,39 @@ func (d *Database) GetIPInfoContext(ctx context.Context, ip string) (*IPInfo, er LEFT JOIN asns a ON a.asn = lr.origin_asn WHERE lr.prefix = ? LIMIT 1 - `, routesTable, prefixesTable) + ` + +// GetIPInfoContext returns comprehensive IP information with context support. +// +// It finds the most specific live route for the address with point lookups on +// the prefix index: for each mask length from the longest (32 or 128) down to +// minLookupMaskLengthV4 or minLookupMaskLengthV6 it looks up the address's +// network prefix at that length, and the first length with a live route wins. +// That is at most 25 or 113 indexed lookups, however many routes are live. The +// prefix is looked up in the text form net/netip prints, which is the form the +// prefix handler stores. +func (d *Database) GetIPInfoContext( + ctx context.Context, ip string, +) (*IPInfo, error) { + addr, err := netip.ParseAddr(ip) + if err != nil { + return nil, fmt.Errorf("%w: %s", ErrInvalidIP, ip) + } + // An IPv4 address written in IPv6 form (::ffff:192.0.2.1) is an IPv4 lookup. + addr = addr.Unmap() + + info := &IPInfo{IP: ip, IPVersion: ipVersionV6} + minMaskLength := minLookupMaskLengthV6 + + routesTable, prefixesTable := tableLiveRoutesV6, "prefixes_v6" + if addr.Is4() { + routesTable, prefixesTable = tableLiveRoutesV4, "prefixes_v4" + minMaskLength = minLookupMaskLengthV4 + info.IPVersion = ipVersionV4 + } + + // Route info with peer count and prefix first_seen + query := fmt.Sprintf(ipInfoQuery, routesTable, prefixesTable) stmt, err := d.db.PrepareContext(ctx, query) if err != nil { @@ -2143,41 +2093,36 @@ func (d *Database) GetIPInfoContext(ctx context.Context, ip string) (*IPInfo, er } defer func() { _ = stmt.Close() }() - var handle, description, asName, orgName, orgID, address, countryCode, abuseEmail, rir sql.NullString - var prefixFirstSeen sql.NullTime - var whoisUpdatedAt sql.NullTime + var ( + handle, description, asName, orgName, orgID sql.NullString + address, countryCode, abuseEmail, rir sql.NullString + prefixFirstSeen, whoisUpdatedAt sql.NullTime + ) found := false + for bits := addr.BitLen(); bits >= minMaskLength; bits-- { prefix := netip.PrefixFrom(addr, bits).Masked().String() + err := stmt.QueryRowContext(ctx, prefix).Scan( - &info.Netblock, - &info.MaskLength, - &info.ASN, - &info.LastSeen, - &info.NumPeers, - &prefixFirstSeen, - &handle, - &description, - &asName, - &orgName, - &orgID, - &address, - &countryCode, - &abuseEmail, - &rir, - &whoisUpdatedAt, + &info.Netblock, &info.MaskLength, &info.ASN, &info.LastSeen, + &info.NumPeers, &prefixFirstSeen, &handle, &description, + &asName, &orgName, &orgID, &address, &countryCode, &abuseEmail, + &rir, &whoisUpdatedAt, ) if errors.Is(err, sql.ErrNoRows) { continue } + if err != nil { return nil, fmt.Errorf("failed to query routes: %w", err) } + found = true break } + if !found { return nil, fmt.Errorf("%w for IP %s", ErrNoRoute, ip) } @@ -2198,18 +2143,23 @@ func (d *Database) GetIPInfoContext(ctx context.Context, ip string) (*IPInfo, er // Check if WHOIS data needs refresh (never fetched or older than 30 days) const staleThreshold = 30 * 24 * time.Hour - info.NeedsWHOISRefresh = !whoisUpdatedAt.Valid || time.Since(whoisUpdatedAt.Time) > staleThreshold + + info.NeedsWHOISRefresh = !whoisUpdatedAt.Valid || + time.Since(whoisUpdatedAt.Time) > staleThreshold return info, nil } -// Vacuum runs incremental vacuum to reclaim unused pages without blocking writes. -// It frees up to the specified number of pages per call (0 = all freeable pages). +// Vacuum runs incremental vacuum to reclaim unused pages without blocking +// writes. It frees up to the specified number of pages per call (0 = all +// freeable pages). func (d *Database) Vacuum(ctx context.Context) error { // Free up to 1000 pages per call (~4MB with default 4KB page size) // This keeps each vacuum operation quick and non-blocking const pagesToFree = 1000 - rows, err := d.db.QueryContext(ctx, fmt.Sprintf("PRAGMA incremental_vacuum(%d)", pagesToFree)) + + rows, err := d.db.QueryContext(ctx, + fmt.Sprintf("PRAGMA incremental_vacuum(%d)", pagesToFree)) if err != nil { return fmt.Errorf("failed to run incremental vacuum: %w", err) } @@ -2220,7 +2170,9 @@ func (d *Database) Vacuum(ctx context.Context) error { // than one page. for rows.Next() { } - if err := rows.Err(); err != nil { + + err = rows.Err() + if err != nil { return fmt.Errorf("failed to run incremental vacuum: %w", err) } @@ -2252,6 +2204,7 @@ func (d *Database) Checkpoint(ctx context.Context) error { // Ping checks if the database connection is alive with a lightweight query. func (d *Database) Ping(ctx context.Context) error { var result int + err := d.db.QueryRowContext(ctx, "SELECT 1").Scan(&result) if err != nil { return fmt.Errorf("database ping failed: %w", err) @@ -2259,3 +2212,380 @@ func (d *Database) Ping(ctx context.Context) error { return nil } + +// lock acquires the database mutex and logs debug information +func (d *Database) lock(operation string) { + // Get caller information + _, file, line, _ := runtime.Caller(1) + caller := fmt.Sprintf("%s:%d", filepath.Base(file), line) + + d.logger.Debug("Acquiring database lock", + "operation", operation, "caller", caller) + + d.mu.Lock() + d.lockedAt = time.Now() + d.lockedBy = fmt.Sprintf("%s (%s)", operation, caller) + + d.logger.Debug("Database lock acquired", + "operation", operation, "caller", caller) +} + +// unlock releases the database mutex and logs debug information including +// hold duration +func (d *Database) unlock() { + holdDuration := time.Since(d.lockedAt) + lockedBy := d.lockedBy + + d.lockedAt = time.Time{} + d.lockedBy = "" + d.mu.Unlock() + + d.logger.Debug("Database lock released", + "held_by", lockedBy, "duration_ms", holdDuration.Milliseconds()) +} + +// beginTx starts a new transaction with logging +func (d *Database) beginTx() (*loggingTx, error) { + tx, err := d.db.BeginTx(context.Background(), nil) + if err != nil { + return nil, err + } + + return &loggingTx{Tx: tx, logger: d.logger}, nil +} + +// rollback rolls back tx unless it has already been committed, and logs a +// failure. Every write defers it right after beginning its transaction. +func (d *Database) rollback(tx *loggingTx) { + err := tx.Rollback() + if err != nil && !errors.Is(err, sql.ErrTxDone) { + d.logger.Error("Failed to rollback transaction", "error", err) + } +} + +// routeTimestampRange returns the earliest and latest last_updated across both +// live route tables, or nil values when both tables are empty. Each query reads +// one row from an end of the last_updated index rather than scanning the tables. +func (d *Database) routeTimestampRange( + ctx context.Context, +) (*time.Time, *time.Time, error) { + var oldest, newest *time.Time + + oldestV4, ok, err := d.scanRouteTimestamp(ctx, "SELECT last_updated "+ + "FROM live_routes_v4 ORDER BY last_updated ASC LIMIT 1") + if err != nil { + return nil, nil, err + } + + if ok { + oldest = &oldestV4 + } + + oldestV6, ok, err := d.scanRouteTimestamp(ctx, "SELECT last_updated "+ + "FROM live_routes_v6 ORDER BY last_updated ASC LIMIT 1") + if err != nil { + return nil, nil, err + } + + if ok && (oldest == nil || oldestV6.Before(*oldest)) { + oldest = &oldestV6 + } + + newestV4, ok, err := d.scanRouteTimestamp(ctx, "SELECT last_updated "+ + "FROM live_routes_v4 ORDER BY last_updated DESC LIMIT 1") + if err != nil { + return nil, nil, err + } + + if ok { + newest = &newestV4 + } + + newestV6, ok, err := d.scanRouteTimestamp(ctx, "SELECT last_updated "+ + "FROM live_routes_v6 ORDER BY last_updated DESC LIMIT 1") + if err != nil { + return nil, nil, err + } + + if ok && (newest == nil || newestV6.After(*newest)) { + newest = &newestV6 + } + + return oldest, newest, nil +} + +// scanRouteTimestamp runs a single-row timestamp query. The bool is false when +// the table is empty. The query selects the last_updated column directly so the +// driver parses the DATETIME value into a time.Time. +func (d *Database) scanRouteTimestamp( + ctx context.Context, query string, +) (time.Time, bool, error) { + var ts time.Time + + err := d.db.QueryRowContext(ctx, query).Scan(&ts) + switch { + case errors.Is(err, sql.ErrNoRows): + return time.Time{}, false, nil + case err != nil: + return time.Time{}, false, err + default: + return ts, true, nil + } +} + +// originPrefixes runs query, which selects the prefix, mask length and latest +// last_updated of every prefix asn originates in the live route table of +// ipVersion. Rows that fail to scan or whose timestamp does not parse are +// logged and skipped. +func (d *Database) originPrefixes( + ctx context.Context, query string, asn, ipVersion int, +) ([]LiveRoute, error) { + family := ipv4Family + if ipVersion == ipVersionV6 { + family = ipv6Family + } + + rows, err := d.db.QueryContext(ctx, query, asn) + if err != nil { + return nil, fmt.Errorf("failed to query %s prefixes: %w", family, err) + } + defer func() { _ = rows.Close() }() + + var routes []LiveRoute + + for rows.Next() { + var ( + route LiveRoute + lastUpdatedStr string + ) + + err := rows.Scan(&route.Prefix, &route.MaskLength, &lastUpdatedStr) + if err != nil { + d.logger.Error("Failed to scan "+family+" prefix row", + "error", err, "asn", asn) + + continue + } + + // Parse the timestamp string + route.LastUpdated, err = time.Parse("2006-01-02 15:04:05-07:00", + lastUpdatedStr) + if err != nil { + // Try without timezone + route.LastUpdated, err = time.Parse("2006-01-02 15:04:05", + lastUpdatedStr) + if err != nil { + d.logger.Error("Failed to parse timestamp", + "error", err, "timestamp", lastUpdatedStr) + + continue + } + } + + route.OriginASN = asn + route.IPVersion = ipVersion + routes = append(routes, route) + } + + err = rows.Err() + if err != nil { + return nil, fmt.Errorf("failed to read %s prefixes: %w", family, err) + } + + return routes, nil +} + +// firstPrefixesByLength returns the first n prefixes, in text order, with the +// given mask length. Each prefix is one search of the mask length and prefix +// index for the prefix after the previous one, so the read does not grow with +// the number of peers that carry a prefix, as a SELECT DISTINCT would. +func (d *Database) firstPrefixesByLength( + ctx context.Context, tableName string, maskLength, n int) ([]string, error) { + //nolint:gosec // Table name is hardcoded based on IP version + query := fmt.Sprintf(` + SELECT prefix FROM %s + WHERE mask_length = ? AND prefix > ? + ORDER BY prefix + LIMIT 1 + `, tableName) + + stmt, err := d.db.PrepareContext(ctx, query) + if err != nil { + return nil, fmt.Errorf("failed to prepare prefix query: %w", err) + } + defer func() { _ = stmt.Close() }() + + var prefixes []string + + previous := "" // every prefix sorts after the empty string + + for len(prefixes) < n { + var prefix string + + err := stmt.QueryRowContext(ctx, maskLength, previous).Scan(&prefix) + if errors.Is(err, sql.ErrNoRows) { + break + } + + if err != nil { + return nil, fmt.Errorf("failed to list prefixes: %w", err) + } + + prefixes = append(prefixes, prefix) + previous = prefix + } + + return prefixes, nil +} + +// samplePrefixesByLength picks limit distinct prefixes with the given mask +// length at random, in at most sampleLookupsPerPrefix lookups for each prefix +// asked for. ORDER BY RANDOM() would read every route with that mask length. +// Instead each lookup draws a row id between those of the first and last route +// with that mask length and takes the prefix of the first such route at or +// after it. SQLite keeps the row id at the end of every index entry, so the +// mask length index is ordered by mask length, then row id, and the lookup is +// one search of it. +// +// The sample is uneven, as the page needs a varied sample, not an even one: a +// route is picked in proportion to the row ids between it and the previous +// route with that mask length, including the row ids of routes with other mask +// lengths and of withdrawn routes. A route that follows a long run of +// withdrawn routes is picked far more often than one that directly follows +// another route with that mask length. +// +// Should the lookups keep landing on prefixes already picked, the rest are +// taken from shuffled, prefixes of that length in random order. +func (d *Database) samplePrefixesByLength( + ctx context.Context, tableName string, maskLength, limit int, + shuffled []string, +) ([]string, error) { + //nolint:gosec // Table name is hardcoded based on IP version + rangeQuery := fmt.Sprintf(` + SELECT + (SELECT MIN(rowid) FROM %[1]s WHERE mask_length = ?), + (SELECT MAX(rowid) FROM %[1]s WHERE mask_length = ?) + `, tableName) + + var first, last sql.NullInt64 + + err := d.db.QueryRowContext(ctx, rangeQuery, maskLength, maskLength). + Scan(&first, &last) + if err != nil { + return nil, fmt.Errorf("failed to read row id range: %w", err) + } + + if !first.Valid { + // Every route with this mask length was withdrawn since they were listed. + return nil, nil + } + + //nolint:gosec // Table name is hardcoded based on IP version + lookupQuery := fmt.Sprintf(` + SELECT prefix FROM %s + WHERE mask_length = ? AND rowid >= ? + ORDER BY rowid + LIMIT 1 + `, tableName) + + stmt, err := d.db.PrepareContext(ctx, lookupQuery) + if err != nil { + return nil, fmt.Errorf("failed to prepare prefix lookup: %w", err) + } + defer func() { _ = stmt.Close() }() + + picked := make(map[string]bool, limit) + prefixes := make([]string, 0, limit) + pick := func(prefix string) { + if !picked[prefix] { + picked[prefix] = true + prefixes = append(prefixes, prefix) + } + } + + maxLookups := limit * sampleLookupsPerPrefix + for lookups := 0; lookups < maxLookups && len(prefixes) < limit; lookups++ { + //nolint:gosec // The sample needs variety, not unpredictability. + rowID := first.Int64 + rand.Int64N(last.Int64-first.Int64+1) + + var prefix string + + err := stmt.QueryRowContext(ctx, maskLength, rowID).Scan(&prefix) + if errors.Is(err, sql.ErrNoRows) { + // The routes from rowID on were withdrawn since the range was read. + continue + } + + if err != nil { + return nil, fmt.Errorf("failed to look up prefix: %w", err) + } + + pick(prefix) + } + + for _, prefix := range shuffled { + if len(prefixes) == limit { + break + } + + pick(prefix) + } + + return prefixes, nil +} + +// newestRouteOfEachPrefix returns the most recently updated live route of each +// prefix, one search of the prefix and last_updated index per prefix however +// many peers carry it. A prefix whose routes were all withdrawn since it was +// picked is left out. +func (d *Database) newestRouteOfEachPrefix( + ctx context.Context, tableName string, ipVersion int, prefixes []string, +) ([]LiveRoute, error) { + //nolint:gosec // Table name is hardcoded based on IP version + query := fmt.Sprintf(` + SELECT prefix, mask_length, origin_asn, as_path, peer_ip, last_updated + FROM %s + WHERE prefix = ? + ORDER BY last_updated DESC + LIMIT 1 + `, tableName) + + stmt, err := d.db.PrepareContext(ctx, query) + if err != nil { + return nil, fmt.Errorf("failed to prepare route query: %w", err) + } + defer func() { _ = stmt.Close() }() + + routes := make([]LiveRoute, 0, len(prefixes)) + + for _, prefix := range prefixes { + route := LiveRoute{IPVersion: ipVersion} + + var pathJSON string + + err := stmt.QueryRowContext(ctx, prefix).Scan( + &route.Prefix, + &route.MaskLength, + &route.OriginASN, + &pathJSON, + &route.PeerIP, + &route.LastUpdated, + ) + if errors.Is(err, sql.ErrNoRows) { + continue + } + + if err != nil { + return nil, fmt.Errorf("failed to query route of %s: %w", prefix, err) + } + + err = json.Unmarshal([]byte(pathJSON), &route.ASPath) + if err != nil { + route.ASPath = []int{} + } + + routes = append(routes, route) + } + + return routes, nil +} diff --git a/internal/database/database_test.go b/internal/database/database_internal_test.go similarity index 80% rename from internal/database/database_test.go rename to internal/database/database_internal_test.go index 3628feb..50f3f77 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_internal_test.go @@ -78,6 +78,8 @@ const ( // address, ErrNoRoute when none covers it, and the next less specific prefix // once the only route of the most specific one is withdrawn. func TestGetIPInfoFindsMostSpecificLiveRoute(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -95,24 +97,7 @@ func TestGetIPInfoFindsMostSpecificLiveRoute(t *testing.T) { "2001:db8:1::/48": 64501, "2001:db8:1:2::/64": 64502, } - ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) - routes := make([]*LiveRoute, 0, len(origins)) - for prefix, asn := range origins { - routes = append(routes, &LiveRoute{ - ID: uuid.New(), - Prefix: prefix, - MaskLength: netip.MustParsePrefix(prefix).Bits(), - IPVersion: detectIPVersion(prefix), - OriginASN: asn, - PeerIP: testPeerIP, - ASPath: []int{asn}, - NextHop: testPeerIP, - LastUpdated: ts, - }) - } - if err := db.UpsertLiveRouteBatch(routes); err != nil { - t.Fatalf("UpsertLiveRouteBatch: %v", err) - } + writeOriginRoutes(t, db, origins) // lookup checks that ip resolves to the live prefix want, or to ErrNoRoute // when want is empty. @@ -127,11 +112,13 @@ func TestGetIPInfoFindsMostSpecificLiveRoute(t *testing.T) { return } + if err != nil { t.Errorf("GetIPInfo(%s): %v", ip, err) return } + if info.Netblock != want || info.MaskLength != netip.MustParsePrefix(want).Bits() || info.ASN != origins[want] { t.Errorf("GetIPInfo(%s) = %s (mask %d) AS%d, want %s AS%d", @@ -166,6 +153,8 @@ func TestGetIPInfoFindsMostSpecificLiveRoute(t *testing.T) { // route for an address, and that a route of exactly /8 or /16, or longer, still // is. The prefixes are written out so that a wrong limit fails the test. func TestGetIPInfoIgnoresShortRoutes(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -174,24 +163,11 @@ func TestGetIPInfoIgnoresShortRoutes(t *testing.T) { } defer func() { _ = db.Close() }() - ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + // announce writes one live route for prefix, originated by asn. announce := func(prefix string, asn int) { t.Helper() - err := db.UpsertLiveRouteBatch([]*LiveRoute{{ - ID: uuid.New(), - Prefix: prefix, - MaskLength: netip.MustParsePrefix(prefix).Bits(), - IPVersion: detectIPVersion(prefix), - OriginASN: asn, - PeerIP: testPeerIP, - ASPath: []int{asn}, - NextHop: testPeerIP, - LastUpdated: ts, - }}) - if err != nil { - t.Fatalf("UpsertLiveRouteBatch(%s): %v", prefix, err) - } + writeOriginRoutes(t, db, map[string]int{prefix: asn}) } // lookup checks that ip resolves to the live prefix netblock from asn. @@ -204,8 +180,10 @@ func TestGetIPInfoIgnoresShortRoutes(t *testing.T) { return } + if info.Netblock != netblock || info.ASN != asn { - t.Errorf("GetIPInfo(%s) = %s AS%d, want %s AS%d", ip, info.Netblock, info.ASN, netblock, asn) + t.Errorf("GetIPInfo(%s) = %s AS%d, want %s AS%d", + ip, info.Netblock, info.ASN, netblock, asn) } } @@ -216,8 +194,10 @@ func TestGetIPInfoIgnoresShortRoutes(t *testing.T) { announce("192.0.0.0/7", 64500) for _, ip := range []string{"2001:db8::1", "192.0.2.1"} { - if info, err := db.GetIPInfo(ip); !errors.Is(err, ErrNoRoute) { - t.Errorf("GetIPInfo(%s) with only short routes = %+v, %v; want ErrNoRoute", ip, info, err) + info, err := db.GetIPInfo(ip) + if !errors.Is(err, ErrNoRoute) { + t.Errorf("GetIPInfo(%s) with only short routes = %+v, %v; "+ + "want ErrNoRoute", ip, info, err) } } @@ -238,6 +218,8 @@ func TestGetIPInfoIgnoresShortRoutes(t *testing.T) { // origin AS. An upsert updates the peer's route before it would insert one, so // only an insert straight into the table shows what the table allows. func TestLiveRouteTablesHoldOneRoutePerPrefixAndPeer(t *testing.T) { + t.Parallel() + db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New()) if err != nil { t.Fatalf("failed to create database: %v", err) @@ -245,33 +227,76 @@ func TestLiveRouteTablesHoldOneRoutePerPrefixAndPeer(t *testing.T) { defer func() { _ = db.Close() }() const otherOriginASN = 64501 + ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + for _, tt := range []struct { route *LiveRoute insertSQL string }{ - {route: mkV4Route(t, "198.51.100.0/24", 64500, ts), insertSQL: insertLiveRouteV4SQL}, - {route: mkV6Route("2001:db8::/32", 64500, ts), insertSQL: insertLiveRouteV6SQL}, + { + route: mkV4Route(t, "198.51.100.0/24", 64500, ts), + insertSQL: insertLiveRouteV4SQL, + }, + { + route: mkV6Route(testV6Prefix, 64500, ts), + insertSQL: insertLiveRouteV6SQL, + }, } { - if err := db.UpsertLiveRoute(tt.route); err != nil { + err := db.UpsertLiveRoute(tt.route) + if err != nil { t.Fatalf("UpsertLiveRoute(%s): %v", tt.route.Prefix, err) } - _, err := db.db.Exec(tt.insertSQL, uuid.NewString(), tt.route.Prefix, tt.route.MaskLength, - otherOriginASN, tt.route.PeerIP, "[]", tt.route.NextHop, ts) + _, err = db.db.ExecContext(t.Context(), tt.insertSQL, uuid.NewString(), + tt.route.Prefix, tt.route.MaskLength, otherOriginASN, + tt.route.PeerIP, "[]", tt.route.NextHop, ts) + var sqliteErr sqlite3.Error - if !errors.As(err, &sqliteErr) || sqliteErr.ExtendedCode != sqlite3.ErrConstraintUnique { - t.Errorf("second route from peer %s for %s: error %v, want a UNIQUE constraint error", + if !errors.As(err, &sqliteErr) || + sqliteErr.ExtendedCode != sqlite3.ErrConstraintUnique { + t.Errorf("second route from peer %s for %s: error %v, "+ + "want a UNIQUE constraint error", tt.route.PeerIP, tt.route.Prefix, err) } } } +// writeOriginRoutes writes one live route from testPeerIP for each prefix in +// origins, originated by the AS it maps to. +func writeOriginRoutes(t *testing.T, db *Database, origins map[string]int) { + t.Helper() + + ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + routes := make([]*LiveRoute, 0, len(origins)) + + for prefix, asn := range origins { + routes = append(routes, &LiveRoute{ + ID: uuid.New(), + Prefix: prefix, + MaskLength: netip.MustParsePrefix(prefix).Bits(), + IPVersion: detectIPVersion(prefix), + OriginASN: asn, + PeerIP: testPeerIP, + ASPath: []int{asn}, + NextHop: testPeerIP, + LastUpdated: ts, + }) + } + + err := db.UpsertLiveRouteBatch(routes) + if err != nil { + t.Fatalf("UpsertLiveRouteBatch: %v", err) + } +} + // TestConnectionPoolPragmas holds several pooled connections open at once and // checks each one carries the per-connection settings from the DSN, plus the // process-wide hard heap limit, and sees the new file with auto_vacuum // incremental. func TestConnectionPoolPragmas(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -291,57 +316,64 @@ func TestConnectionPoolPragmas(t *testing.T) { } }() - for i := 0; i < heldConnections; i++ { + for i := range heldConnections { c, err := db.db.Conn(ctx) if err != nil { t.Fatalf("failed to open connection %d: %v", i, err) } + conns = append(conns, c) } for i, c := range conns { - var cacheSize int - if err := c.QueryRowContext(ctx, "PRAGMA cache_size").Scan(&cacheSize); err != nil { - t.Fatalf("conn %d: failed to read cache_size: %v", i, err) - } + cacheSize := readPragma(ctx, t, c, i, "cache_size") if cacheSize != sqliteCacheSizeKiB { - t.Errorf("conn %d: cache_size = %d, want %d", i, cacheSize, sqliteCacheSizeKiB) + t.Errorf("conn %d: cache_size = %d, want %d", + i, cacheSize, sqliteCacheSizeKiB) } - var busyTimeout int - if err := c.QueryRowContext(ctx, "PRAGMA busy_timeout").Scan(&busyTimeout); err != nil { - t.Fatalf("conn %d: failed to read busy_timeout: %v", i, err) - } + busyTimeout := readPragma(ctx, t, c, i, "busy_timeout") if busyTimeout != sqliteBusyTimeoutMs { - t.Errorf("conn %d: busy_timeout = %d, want %d", i, busyTimeout, sqliteBusyTimeoutMs) + t.Errorf("conn %d: busy_timeout = %d, want %d", + i, busyTimeout, sqliteBusyTimeoutMs) } - var tempStore int - if err := c.QueryRowContext(ctx, "PRAGMA temp_store").Scan(&tempStore); err != nil { - t.Fatalf("conn %d: failed to read temp_store: %v", i, err) - } + tempStore := readPragma(ctx, t, c, i, "temp_store") if tempStore == tempStoreMemory { - t.Errorf("conn %d: temp_store = %d, want anything but %d (MEMORY)", i, tempStore, tempStoreMemory) + t.Errorf("conn %d: temp_store = %d, want anything but %d (MEMORY)", + i, tempStore, tempStoreMemory) } - var hardHeapLimit int64 - if err := c.QueryRowContext(ctx, "PRAGMA hard_heap_limit").Scan(&hardHeapLimit); err != nil { - t.Fatalf("conn %d: failed to read hard_heap_limit: %v", i, err) - } + hardHeapLimit := readPragma(ctx, t, c, i, "hard_heap_limit") if hardHeapLimit != sqliteHardHeapLimitBytes { - t.Errorf("conn %d: hard_heap_limit = %d, want %d", i, hardHeapLimit, sqliteHardHeapLimitBytes) + t.Errorf("conn %d: hard_heap_limit = %d, want %d", + i, hardHeapLimit, sqliteHardHeapLimitBytes) } - var autoVacuum int - if err := c.QueryRowContext(ctx, "PRAGMA auto_vacuum").Scan(&autoVacuum); err != nil { - t.Fatalf("conn %d: failed to read auto_vacuum: %v", i, err) - } + autoVacuum := readPragma(ctx, t, c, i, "auto_vacuum") if autoVacuum != autoVacuumIncremental { - t.Errorf("conn %d: auto_vacuum = %d, want %d (incremental)", i, autoVacuum, autoVacuumIncremental) + t.Errorf("conn %d: auto_vacuum = %d, want %d (incremental)", + i, autoVacuum, autoVacuumIncremental) } } } +// readPragma reads the integer value of PRAGMA name on connection i, c. +func readPragma( + ctx context.Context, t *testing.T, c *sql.Conn, i int, name string, +) int64 { + t.Helper() + + var value int64 + + err := c.QueryRowContext(ctx, "PRAGMA "+name).Scan(&value) + if err != nil { + t.Fatalf("conn %d: failed to read %s: %v", i, name, err) + } + + return value +} + // TestVacuumReturnsFreePages checks that after routes are deleted from a new // database, one Vacuum call returns every page they used. The deletes leave // fewer free pages than the 1000 Vacuum frees per call, so none may remain. @@ -349,6 +381,8 @@ func TestConnectionPoolPragmas(t *testing.T) { // the free pages stayed in the file, and with the PRAGMA run by ExecContext // Vacuum freed only one page per call. func TestVacuumReturnsFreePages(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -359,10 +393,12 @@ func TestVacuumReturnsFreePages(t *testing.T) { ctx := context.Background() ts := time.Now().UTC() + const asn = 64500 routes := make([]*LiveRoute, 0, vacuumTestRoutes) deletions := make([]LiveRouteDeletion, 0, vacuumTestRoutes) + for i := range vacuumTestRoutes { route := mkV4Route(t, fmt.Sprintf("10.%d.%d.0/24", i/256, i%256), asn, ts) routes = append(routes, route) @@ -373,29 +409,39 @@ func TestVacuumReturnsFreePages(t *testing.T) { }) } - if err := db.UpsertLiveRouteBatch(routes); err != nil { + err = db.UpsertLiveRouteBatch(routes) + if err != nil { t.Fatalf("failed to write routes: %v", err) } - if err := db.DeleteLiveRouteBatch(deletions); err != nil { + + err = db.DeleteLiveRouteBatch(deletions) + if err != nil { t.Fatalf("failed to delete routes: %v", err) } var before int - if err := db.db.QueryRowContext(ctx, "PRAGMA freelist_count").Scan(&before); err != nil { + + err = db.db.QueryRowContext(ctx, "PRAGMA freelist_count").Scan(&before) + if err != nil { t.Fatalf("failed to read freelist_count: %v", err) } + if before == 0 { t.Fatalf("no free pages after deleting %d routes", vacuumTestRoutes) } - if err := db.Vacuum(ctx); err != nil { + err = db.Vacuum(ctx) + if err != nil { t.Fatalf("Vacuum failed: %v", err) } var after int - if err := db.db.QueryRowContext(ctx, "PRAGMA freelist_count").Scan(&after); err != nil { + + err = db.db.QueryRowContext(ctx, "PRAGMA freelist_count").Scan(&after) + if err != nil { t.Fatalf("failed to read freelist_count: %v", err) } + if after != 0 { t.Errorf("free pages after Vacuum = %d of %d, want 0", after, before) } @@ -410,6 +456,8 @@ func TestVacuumReturnsFreePages(t *testing.T) { // no batch is dropped. The checkpoint runs without the Database mutex, exactly // as the background maintainer does in production. func TestBatchWriteDuringCheckpoint(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -422,8 +470,10 @@ func TestBatchWriteDuringCheckpoint(t *testing.T) { var wg sync.WaitGroup wg.Add(1) + go func() { defer wg.Done() + for { select { case <-ctx.Done(): @@ -435,12 +485,14 @@ func TestBatchWriteDuringCheckpoint(t *testing.T) { }() ts := time.Now().UTC() - for i := 0; i < contentionIterations; i++ { + for i := range contentionIterations { asns := map[int]time.Time{ i % contendedASNCount: ts, (i % contendedASNCount) + asnSecondBand: ts, } - if err := db.GetOrCreateASNBatch(asns); err != nil { + + err := db.GetOrCreateASNBatch(asns) + if err != nil { cancel() wg.Wait() t.Fatalf("batch write failed under checkpoint contention: %v", err) @@ -455,6 +507,8 @@ func TestBatchWriteDuringCheckpoint(t *testing.T) { // family learned from two peers in the same second and checks that the // prefix-length sample lists each of them once. func TestGetRandomPrefixesByLengthListsPrefixOnce(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -467,18 +521,24 @@ func TestGetRandomPrefixesByLengthListsPrefixOnce(t *testing.T) { v4First := mkV4Route(t, "198.51.100.0/24", 64500, ts) v4Second := mkV4Route(t, "198.51.100.0/24", 64500, ts) v4Second.PeerIP = secondV4PeerIP - v6First := mkV6Route("2001:db8::/32", 64500, ts) - v6Second := mkV6Route("2001:db8::/32", 64500, ts) + v6First := mkV6Route(testV6Prefix, 64500, ts) + v6Second := mkV6Route(testV6Prefix, 64500, ts) v6Second.PeerIP = secondV6PeerIP - if err := db.UpsertLiveRouteBatch([]*LiveRoute{v4First, v4Second, v6First, v6Second}); err != nil { + + err = db.UpsertLiveRouteBatch( + []*LiveRoute{v4First, v4Second, v6First, v6Second}) + if err != nil { t.Fatalf("UpsertLiveRouteBatch: %v", err) } for _, want := range []*LiveRoute{v4First, v6First} { - routes, err := db.GetRandomPrefixesByLength(want.MaskLength, want.IPVersion, sampleLimit) + routes, err := db.GetRandomPrefixesByLength(want.MaskLength, + want.IPVersion, sampleLimit) if err != nil { - t.Fatalf("GetRandomPrefixesByLength(%d, %d): %v", want.MaskLength, want.IPVersion, err) + t.Fatalf("GetRandomPrefixesByLength(%d, %d): %v", + want.MaskLength, want.IPVersion, err) } + if len(routes) != 1 || routes[0].Prefix != want.Prefix { t.Errorf("GetRandomPrefixesByLength(%d, %d) = %+v, want %s listed once", want.MaskLength, want.IPVersion, routes, want.Prefix) @@ -490,6 +550,8 @@ func TestGetRandomPrefixesByLengthListsPrefixOnce(t *testing.T) { // two peers at different times, the older route first, and checks that the // prefix-length sample lists the newer route. func TestGetRandomPrefixesByLengthListsNewestRoute(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -502,17 +564,21 @@ func TestGetRandomPrefixesByLengthListsNewestRoute(t *testing.T) { olderRoute := mkV4Route(t, "198.51.100.0/24", 64500, older) newerRoute := mkV4Route(t, "198.51.100.0/24", 64500, older.Add(time.Minute)) newerRoute.PeerIP = secondV4PeerIP - if err := db.UpsertLiveRouteBatch([]*LiveRoute{olderRoute, newerRoute}); err != nil { + + err = db.UpsertLiveRouteBatch([]*LiveRoute{olderRoute, newerRoute}) + if err != nil { t.Fatalf("UpsertLiveRouteBatch: %v", err) } - routes, err := db.GetRandomPrefixesByLength(newerRoute.MaskLength, ipVersionV4, sampleLimit) + routes, err := db.GetRandomPrefixesByLength(newerRoute.MaskLength, + ipVersionV4, sampleLimit) if err != nil { t.Fatalf("GetRandomPrefixesByLength: %v", err) } + if len(routes) != 1 || routes[0].PeerIP != newerRoute.PeerIP { - t.Errorf("GetRandomPrefixesByLength = %+v, want only the newer route, from %s", - routes, newerRoute.PeerIP) + t.Errorf("GetRandomPrefixesByLength = %+v, want only the newer route, "+ + "from %s", routes, newerRoute.PeerIP) } } @@ -523,6 +589,8 @@ func TestGetRandomPrefixesByLengthListsNewestRoute(t *testing.T) { // shuffled prefixes, and the random lookups used for a mask length with more // than maxPrefixesToShuffle prefixes. func TestGetRandomPrefixesByLengthSamplesManyPrefixes(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -532,8 +600,11 @@ func TestGetRandomPrefixesByLengthSamplesManyPrefixes(t *testing.T) { defer func() { _ = db.Close() }() ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + var routes []*LiveRoute - var stored []string + + stored := make([]string, 0, sampledPrefixCount) + for i := range sampledPrefixCount { prefix := fmt.Sprintf("10.0.%d.0/24", i) for _, p := range []string{prefix, fmt.Sprintf("172.%d.0.0/16", i)} { @@ -542,22 +613,29 @@ func TestGetRandomPrefixesByLengthSamplesManyPrefixes(t *testing.T) { second.PeerIP = secondV4PeerIP routes = append(routes, first, second) } + stored = append(stored, prefix) } - if err := db.UpsertLiveRouteBatch(routes); err != nil { + + err = db.UpsertLiveRouteBatch(routes) + if err != nil { t.Fatalf("UpsertLiveRouteBatch: %v", err) } + slices.Sort(stored) maskLength := routes[0].MaskLength + sample, err := db.GetRandomPrefixesByLength(maskLength, ipVersionV4, sampleLimit) if err != nil { t.Fatalf("GetRandomPrefixesByLength: %v", err) } - var listed []string + + listed := make([]string, 0, len(sample)) for _, route := range sample { listed = append(listed, route.Prefix) } + checkPrefixSample(t, "GetRandomPrefixesByLength", listed, stored) looked, err := db.samplePrefixesByLength( @@ -565,6 +643,7 @@ func TestGetRandomPrefixesByLengthSamplesManyPrefixes(t *testing.T) { if err != nil { t.Fatalf("samplePrefixesByLength: %v", err) } + checkPrefixSample(t, "samplePrefixesByLength", looked, stored) } @@ -577,15 +656,20 @@ func checkPrefixSample(t *testing.T, name string, sample, stored []string) { if len(sample) != sampleLimit { t.Errorf("%s listed %d prefixes, want %d", name, len(sample), sampleLimit) } + listed := make(map[string]bool) + for _, prefix := range sample { if listed[prefix] || !slices.Contains(stored, prefix) { t.Errorf("%s lists %s twice or it is not one of the /24s", name, prefix) } + listed[prefix] = true } + if slices.Equal(slices.Sorted(slices.Values(sample)), stored[:sampleLimit]) { - t.Errorf("%s lists just the first %d prefixes in text order", name, sampleLimit) + t.Errorf("%s lists just the first %d prefixes in text order", + name, sampleLimit) } } @@ -595,6 +679,8 @@ func checkPrefixSample(t *testing.T, name string, sample, stored []string) { // order, and that the sample lists the prefixes asked for, each once, some of // them after those first ones, which only the random lookups reach. func TestGetRandomPrefixesByLengthSamplesPastFirstPrefixes(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -604,51 +690,64 @@ func TestGetRandomPrefixesByLengthSamplesPastFirstPrefixes(t *testing.T) { defer func() { _ = db.Close() }() ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) - var routes []*LiveRoute - var stored []string + routes := make([]*LiveRoute, 0, largeSamplePrefixCount) + stored := make([]string, 0, largeSamplePrefixCount) + for i := range largeSamplePrefixCount { route := mkV6Route(fmt.Sprintf("2001:%x::/32", i), 64500, ts) routes = append(routes, route) stored = append(stored, route.Prefix) } - if err := db.UpsertLiveRouteBatch(routes); err != nil { + + err = db.UpsertLiveRouteBatch(routes) + if err != nil { t.Fatalf("UpsertLiveRouteBatch: %v", err) } + slices.Sort(stored) firstPrefixes := stored[:maxPrefixesToShuffle+1] - maskLength := routes[0].MaskLength - read, err := db.firstPrefixesByLength( - context.Background(), liveRoutesV6Table, maskLength, maxPrefixesToShuffle+1) + + read, err := db.firstPrefixesByLength(context.Background(), + liveRoutesV6Table, maskLength, maxPrefixesToShuffle+1) if err != nil { t.Fatalf("firstPrefixesByLength: %v", err) } + if !slices.Equal(read, firstPrefixes) { - t.Errorf("firstPrefixesByLength read %d prefixes, want the first %d in text order", - len(read), len(firstPrefixes)) + t.Errorf("firstPrefixesByLength read %d prefixes, want the first %d "+ + "in text order", len(read), len(firstPrefixes)) } - sample, err := db.GetRandomPrefixesByLength(maskLength, ipVersionV6, largeSampleLimit) + sample, err := db.GetRandomPrefixesByLength(maskLength, ipVersionV6, + largeSampleLimit) if err != nil { t.Fatalf("GetRandomPrefixesByLength: %v", err) } + if len(sample) != largeSampleLimit { - t.Errorf("GetRandomPrefixesByLength listed %d prefixes, want %d", len(sample), largeSampleLimit) + t.Errorf("GetRandomPrefixesByLength listed %d prefixes, want %d", + len(sample), largeSampleLimit) } + listed := make(map[string]bool) pastFirstPrefixes := 0 + for _, route := range sample { if listed[route.Prefix] { t.Errorf("GetRandomPrefixesByLength lists %s twice", route.Prefix) } + listed[route.Prefix] = true + if !slices.Contains(firstPrefixes, route.Prefix) { pastFirstPrefixes++ } } + if pastFirstPrefixes == 0 { - t.Errorf("GetRandomPrefixesByLength lists only prefixes among the first %d in text order", - len(firstPrefixes)) + t.Errorf("GetRandomPrefixesByLength lists only prefixes among the first "+ + "%d in text order", len(firstPrefixes)) } } @@ -657,6 +756,8 @@ func TestGetRandomPrefixesByLengthSamplesPastFirstPrefixes(t *testing.T) { // sampleLimit prefixes: the stored one, then the shuffled ones in order. The // stored prefix is the second shuffled one, so the fill-up must skip it. func TestSamplePrefixesByLengthFillsUpFromShuffled(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -665,20 +766,25 @@ func TestSamplePrefixesByLengthFillsUpFromShuffled(t *testing.T) { } defer func() { _ = db.Close() }() - var shuffled []string + shuffled := make([]string, 0, sampledPrefixCount) for i := range sampledPrefixCount { shuffled = append(shuffled, fmt.Sprintf("10.0.%d.0/24", i)) } - route := mkV4Route(t, shuffled[1], 64500, time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)) - if err := db.UpsertLiveRouteBatch([]*LiveRoute{route}); err != nil { + + ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + route := mkV4Route(t, shuffled[1], 64500, ts) + + err = db.UpsertLiveRouteBatch([]*LiveRoute{route}) + if err != nil { t.Fatalf("UpsertLiveRouteBatch: %v", err) } - sample, err := db.samplePrefixesByLength( - context.Background(), liveRoutesV4Table, route.MaskLength, sampleLimit, shuffled) + sample, err := db.samplePrefixesByLength(context.Background(), + liveRoutesV4Table, route.MaskLength, sampleLimit, shuffled) if err != nil { t.Fatalf("samplePrefixesByLength: %v", err) } + want := append([]string{shuffled[1], shuffled[0]}, shuffled[2:sampleLimit]...) if !slices.Equal(sample, want) { t.Errorf("samplePrefixesByLength = %v, want %v", sample, want) @@ -691,6 +797,8 @@ func TestSamplePrefixesByLengthFillsUpFromShuffled(t *testing.T) { // left, and that only the prefixes that still have a route get their newest // route listed. func TestGetRandomPrefixesByLengthSkipsWithdrawnRoutes(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -702,29 +810,38 @@ func TestGetRandomPrefixesByLengthSkipsWithdrawnRoutes(t *testing.T) { ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) kept := mkV4Route(t, "198.51.100.0/24", 64500, ts) withdrawn := mkV4Route(t, "172.16.0.0/16", 64500, ts) - if err := db.UpsertLiveRouteBatch([]*LiveRoute{kept, withdrawn}); err != nil { + + err = db.UpsertLiveRouteBatch([]*LiveRoute{kept, withdrawn}) + if err != nil { t.Fatalf("UpsertLiveRouteBatch: %v", err) } - if err := db.DeleteLiveRoute(withdrawn.Prefix, withdrawn.PeerIP); err != nil { + + err = db.DeleteLiveRoute(withdrawn.Prefix, withdrawn.PeerIP) + if err != nil { t.Fatalf("DeleteLiveRoute: %v", err) } ctx := context.Background() - sample, err := db.samplePrefixesByLength( - ctx, liveRoutesV4Table, withdrawn.MaskLength, sampleLimit, []string{withdrawn.Prefix}) + + sample, err := db.samplePrefixesByLength(ctx, liveRoutesV4Table, + withdrawn.MaskLength, sampleLimit, []string{withdrawn.Prefix}) if err != nil { t.Fatalf("samplePrefixesByLength: %v", err) } + if len(sample) != 0 { - t.Errorf("samplePrefixesByLength = %v, want none for a mask length with no route left", sample) + t.Errorf("samplePrefixesByLength = %v, want none for a mask length "+ + "with no route left", sample) } - routes, err := db.newestRouteOfEachPrefix( - ctx, liveRoutesV4Table, ipVersionV4, []string{withdrawn.Prefix, kept.Prefix}) + routes, err := db.newestRouteOfEachPrefix(ctx, liveRoutesV4Table, + ipVersionV4, []string{withdrawn.Prefix, kept.Prefix}) if err != nil { t.Fatalf("newestRouteOfEachPrefix: %v", err) } + if len(routes) != 1 || routes[0].Prefix != kept.Prefix { - t.Errorf("newestRouteOfEachPrefix = %+v, want only the route of %s", routes, kept.Prefix) + t.Errorf("newestRouteOfEachPrefix = %+v, want only the route of %s", + routes, kept.Prefix) } } diff --git a/internal/database/interface.go b/internal/database/interface.go index 0311f05..2f5f8ac 100644 --- a/internal/database/interface.go +++ b/internal/database/interface.go @@ -29,6 +29,8 @@ type Stats struct { // Store defines the interface for database operations. It provides methods for // managing ASNs, prefixes, announcements, peerings, BGP peers, and live routes. // Implementations must be safe for concurrent use. +// +//nolint:interfacebloat // the one seam handlers and tests depend on type Store interface { // ASN operations GetOrCreateASN(number int, timestamp time.Time) (*ASN, error) @@ -58,8 +60,10 @@ type Store interface { DeleteLiveRoute(prefix, peerIP string) error DeleteLiveRouteBatch(deletions []LiveRouteDeletion) error DeleteLiveRoutesOfPeer(peerIP string) (deleted int, err error) - GetPrefixDistribution() (ipv4 []PrefixDistribution, ipv6 []PrefixDistribution, err error) - GetPrefixDistributionContext(ctx context.Context) (ipv4 []PrefixDistribution, ipv6 []PrefixDistribution, err error) + GetPrefixDistribution() ( + ipv4 []PrefixDistribution, ipv6 []PrefixDistribution, err error) + GetPrefixDistributionContext(ctx context.Context) ( + ipv4 []PrefixDistribution, ipv6 []PrefixDistribution, err error) GetLiveRouteCounts() (ipv4Count, ipv6Count int, err error) GetLiveRouteCountsContext(ctx context.Context) (ipv4Count, ipv6Count int, err error) @@ -78,9 +82,13 @@ type Store interface { GetASPeers(asn int) ([]ASPeer, error) GetASPeersContext(ctx context.Context, asn int) ([]ASPeer, error) GetPrefixDetails(prefix string) ([]LiveRoute, map[int]ASN, error) - GetPrefixDetailsContext(ctx context.Context, prefix string) ([]LiveRoute, map[int]ASN, error) + GetPrefixDetailsContext( + ctx context.Context, prefix string, + ) ([]LiveRoute, map[int]ASN, error) GetRandomPrefixesByLength(maskLength, ipVersion, limit int) ([]LiveRoute, error) - GetRandomPrefixesByLengthContext(ctx context.Context, maskLength, ipVersion, limit int) ([]LiveRoute, error) + GetRandomPrefixesByLengthContext( + ctx context.Context, maskLength, ipVersion, limit int, + ) ([]LiveRoute, error) // Lifecycle Close() error diff --git a/internal/database/models.go b/internal/database/models.go index 54c54f8..3872f95 100644 --- a/internal/database/models.go +++ b/internal/database/models.go @@ -1,4 +1,3 @@ -// Package database provides SQLite storage for BGP routing data. package database import ( @@ -9,6 +8,8 @@ import ( // ASN represents an Autonomous System Number with its metadata including // handle, description, WHOIS data, and first/last seen timestamps. +// +//nolint:tagliatelle // field names are the existing JSON API type ASN struct { ASN int `json:"asn"` Handle string `json:"handle"` @@ -36,37 +37,39 @@ type ASN struct { // Prefix represents an IP prefix (CIDR block) with its IP version (4 or 6) // and first/last seen timestamps. type Prefix struct { - ID uuid.UUID `json:"id"` - Prefix string `json:"prefix"` - IPVersion int `json:"ip_version"` // 4 or 6 - FirstSeen time.Time `json:"first_seen"` - LastSeen time.Time `json:"last_seen"` + ID uuid.UUID + Prefix string + IPVersion int // 4 or 6 + FirstSeen time.Time + LastSeen time.Time } // Announcement represents a BGP announcement or withdrawal event, // containing the prefix, AS path, origin ASN, peer ASN, next hop, and timestamp. type Announcement struct { - ID uuid.UUID `json:"id"` - PrefixID uuid.UUID `json:"prefix_id"` - PeerASN int `json:"peer_asn"` - OriginASN int `json:"origin_asn"` - Path string `json:"path"` // JSON-encoded AS path - NextHop string `json:"next_hop"` - Timestamp time.Time `json:"timestamp"` - IsWithdrawal bool `json:"is_withdrawal"` + ID uuid.UUID + PrefixID uuid.UUID + PeerASN int + OriginASN int + Path string // JSON-encoded AS path + NextHop string + Timestamp time.Time + IsWithdrawal bool } // ASNPeering represents a peering relationship between two ASNs, // stored with the lower ASN as ASA and the higher as ASB. type ASNPeering struct { - ID uuid.UUID `json:"id"` - ASA int `json:"as_a"` - ASB int `json:"as_b"` - FirstSeen time.Time `json:"first_seen"` - LastSeen time.Time `json:"last_seen"` + ID uuid.UUID + ASA int + ASB int + FirstSeen time.Time + LastSeen time.Time } // LiveRoute represents a route in the live routing table +// +//nolint:tagliatelle // field names are the existing JSON API type LiveRoute struct { ID uuid.UUID `json:"id"` Prefix string `json:"prefix"` @@ -80,12 +83,16 @@ type LiveRoute struct { } // PrefixDistribution represents the distribution of prefixes by mask length +// +//nolint:tagliatelle // field names are the existing JSON API type PrefixDistribution struct { MaskLength int `json:"mask_length"` Count int `json:"count"` } // IPInfo represents comprehensive IP information for the /ip endpoint +// +//nolint:tagliatelle // field names are the existing JSON API type IPInfo struct { IP string `json:"ip"` PTR []string `json:"ptr,omitempty"` diff --git a/internal/database/slowquery.go b/internal/database/slowquery.go index 1ac302a..d027211 100644 --- a/internal/database/slowquery.go +++ b/internal/database/slowquery.go @@ -18,30 +18,12 @@ func logSlowQuery(logger *logger.Logger, query string, start time.Time) { } } -// queryRow wraps QueryRow with slow query logging -// nolint:unused // kept for consistency with other query wrappers -func (d *Database) queryRow(query string, args ...interface{}) *sql.Row { - start := time.Now() - defer logSlowQuery(d.logger, query, start) - - return d.db.QueryRow(query, args...) -} - -// query wraps Query with slow query logging -// nolint:unused // kept for future use to ensure all queries go through slow query logging -func (d *Database) query(query string, args ...interface{}) (*sql.Rows, error) { - start := time.Now() - defer logSlowQuery(d.logger, query, start) - - return d.db.Query(query, args...) -} - // exec wraps Exec with slow query logging -func (d *Database) exec(query string, args ...interface{}) error { +func (d *Database) exec(query string, args ...any) error { start := time.Now() defer logSlowQuery(d.logger, query, start) - _, err := d.db.Exec(query, args...) + _, err := d.db.ExecContext(context.Background(), query, args...) return err } @@ -49,51 +31,51 @@ func (d *Database) exec(query string, args ...interface{}) error { // loggingTx wraps sql.Tx to log slow queries type loggingTx struct { *sql.Tx + logger *logger.Logger } // QueryRow wraps sql.Tx.QueryRow to log slow queries -func (tx *loggingTx) QueryRow(query string, args ...interface{}) *sql.Row { +func (tx *loggingTx) QueryRow(query string, args ...any) *sql.Row { start := time.Now() defer logSlowQuery(tx.logger, query, start) - return tx.Tx.QueryRow(query, args...) -} - -// Query wraps sql.Tx.Query to log slow queries -func (tx *loggingTx) Query(query string, args ...interface{}) (*sql.Rows, error) { - start := time.Now() - defer logSlowQuery(tx.logger, query, start) - - return tx.Tx.Query(query, args...) -} - -// QueryContext wraps sql.Tx.QueryContext to log slow queries -func (tx *loggingTx) QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error) { - start := time.Now() - defer logSlowQuery(tx.logger, query, start) - - return tx.Tx.QueryContext(ctx, query, args...) + return tx.Tx.QueryRowContext(context.Background(), query, args...) } // QueryRowContext wraps sql.Tx.QueryRowContext to log slow queries -func (tx *loggingTx) QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row { +func (tx *loggingTx) QueryRowContext( + ctx context.Context, query string, args ...any, +) *sql.Row { start := time.Now() defer logSlowQuery(tx.logger, query, start) return tx.Tx.QueryRowContext(ctx, query, args...) } -// Exec wraps sql.Tx.Exec to log slow queries -func (tx *loggingTx) Exec(query string, args ...interface{}) (sql.Result, error) { +// QueryContext wraps sql.Tx.QueryContext to log slow queries +func (tx *loggingTx) QueryContext( + ctx context.Context, query string, args ...any, +) (*sql.Rows, error) { start := time.Now() defer logSlowQuery(tx.logger, query, start) - return tx.Tx.Exec(query, args...) + //nolint:sqlclosecheck // returns the rows, which the caller closes + return tx.Tx.QueryContext(ctx, query, args...) +} + +// Exec wraps sql.Tx.Exec to log slow queries +func (tx *loggingTx) Exec(query string, args ...any) (sql.Result, error) { + start := time.Now() + defer logSlowQuery(tx.logger, query, start) + + return tx.Tx.ExecContext(context.Background(), query, args...) } // ExecContext wraps sql.Tx.ExecContext to log slow queries -func (tx *loggingTx) ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error) { +func (tx *loggingTx) ExecContext( + ctx context.Context, query string, args ...any, +) (sql.Result, error) { start := time.Now() defer logSlowQuery(tx.logger, query, start) diff --git a/internal/database/utils.go b/internal/database/utils.go index 8094c78..4ba0b0d 100644 --- a/internal/database/utils.go +++ b/internal/database/utils.go @@ -27,6 +27,7 @@ func prefixMaskLength(prefix string) (int, error) { if err != nil { return 0, fmt.Errorf("invalid prefix %s: %w", prefix, err) } + maskLength, _ := network.Mask.Size() return maskLength, nil diff --git a/internal/logger/logger.go b/internal/logger/logger.go index 3e90ad8..d46d523 100644 --- a/internal/logger/logger.go +++ b/internal/logger/logger.go @@ -74,6 +74,7 @@ func getSourceAttrs() []slog.Attr { // Get the function name fn := runtime.FuncForPC(pc) + var funcName string if fn != nil { funcName = filepath.Base(fn.Name()) diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 25aef45..ad04309 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -64,6 +64,7 @@ func New() *Tracker { // first counts one reconnect. func (t *Tracker) SetConnected(connected bool) { t.isConnected.Store(connected) + if !connected { return } @@ -120,9 +121,11 @@ func (t *Tracker) GetStreamMetrics() StreamMetrics { if msgCount >= 0 { totalMessages = uint64(msgCount) } + if byteCount >= 0 { totalBytes = uint64(byteCount) } + if wireByteCount >= 0 { totalWireBytes = uint64(wireByteCount) } @@ -166,7 +169,7 @@ func (t *Tracker) RecordWithdrawal() { func (t *Tracker) SetBGPPeerCount(count int) { // BGP peer count is always small (< 1000), so int32 is safe if count > 0 && count < 1<<31 { - t.bgpPeerCount.Store(int32(count)) //nolint:gosec // count is validated + t.bgpPeerCount.Store(int32(count)) } } @@ -214,7 +217,8 @@ type StreamMetrics struct { TotalMessages uint64 // TotalBytes is the total number of decompressed bytes received since startup TotalBytes uint64 - // TotalWireBytes is the total number of bytes received on the wire (before decompression) + // TotalWireBytes is the total number of bytes received on the wire (before + // decompression) TotalWireBytes uint64 // ConnectedSince is the time when the current connection was established ConnectedSince time.Time @@ -222,9 +226,11 @@ type StreamMetrics struct { Connected bool // MessagesPerSec is the rate of messages received per second (1-minute average) MessagesPerSec float64 - // BitsPerSec is the rate of decompressed bits received per second (1-minute average) + // BitsPerSec is the rate of decompressed bits received per second (1-minute + // average) BitsPerSec float64 - // WireBitsPerSec is the rate of bits received on the wire per second (1-minute average) + // WireBitsPerSec is the rate of bits received on the wire per second + // (1-minute average) WireBitsPerSec float64 // ReconnectCount is the number of reconnections since startup ReconnectCount uint64 diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go index e8776fd..c301122 100644 --- a/internal/metrics/metrics_test.go +++ b/internal/metrics/metrics_test.go @@ -1,12 +1,18 @@ -package metrics +package metrics_test -import "testing" +import ( + "testing" + + "git.eeqj.de/sneak/routewatch/internal/metrics" +) // TestReconnectCount connects three times with a disconnect between each, the // way the streamer does, and expects every connection after the first to count // one reconnect. func TestReconnectCount(t *testing.T) { - tracker := New() + t.Parallel() + + tracker := metrics.New() tracker.SetConnected(true) tracker.SetConnected(false) diff --git a/internal/ristypes/ris.go b/internal/ristypes/ris.go index e8130b6..4b67ee5 100644 --- a/internal/ristypes/ris.go +++ b/internal/ristypes/ris.go @@ -1,4 +1,5 @@ -// Package ristypes defines the data structures for RIS Live BGP messages and announcements. +// Package ristypes defines the data structures for RIS Live BGP messages and +// announcements. package ristypes import ( @@ -33,25 +34,30 @@ type ASPath []int func (p *ASPath) UnmarshalJSON(data []byte) error { // First try to unmarshal as a simple array of integers var simple []int - if err := json.Unmarshal(data, &simple); err == nil { + + err := json.Unmarshal(data, &simple) + if err == nil { *p = ASPath(simple) return nil } // If that fails, unmarshal as array of interfaces and flatten - var raw []interface{} - if err := json.Unmarshal(data, &raw); err != nil { + var raw []any + + err = json.Unmarshal(data, &raw) + if err != nil { return err } // Flatten the array result := make([]int, 0) + for _, item := range raw { switch v := item.(type) { case float64: result = append(result, int(v)) - case []interface{}: + case []any: // Nested array - flatten it for _, nested := range v { if num, ok := nested.(float64); ok { @@ -78,6 +84,8 @@ type RISLiveMessage struct { // It contains metadata about the BGP session (peer, ASN, host) along with // the actual BGP update data including AS path, communities, announcements, // and withdrawals. +// +//nolint:tagliatelle // field names are the RIS Live feed's type RISMessage struct { Type string `json:"type"` Timestamp float64 `json:"timestamp"` @@ -108,6 +116,8 @@ type RISMessage struct { // RISAnnouncement represents a BGP route announcement within a RIS message. // It contains the next hop IP address and the list of prefixes being announced // via that next hop. +// +//nolint:tagliatelle // field names are the RIS Live feed's type RISAnnouncement struct { NextHop string `json:"next_hop"` Prefixes []string `json:"prefixes"` diff --git a/internal/ristypes/ris_test.go b/internal/ristypes/ris_test.go index 6a3b7b0..2b3f7d2 100644 --- a/internal/ristypes/ris_test.go +++ b/internal/ristypes/ris_test.go @@ -1,10 +1,12 @@ -package ristypes +package ristypes_test import ( "bufio" "encoding/json" "os" "testing" + + "git.eeqj.de/sneak/routewatch/internal/ristypes" ) // messageExamplesPath is the captured RIS Live feed used as decode fixtures, @@ -15,32 +17,43 @@ const messageExamplesPath = "../../docs/message-examples.json" // streamer does and checks that the fields no handler reads (Community, Raw) // stay empty while the fields handlers use (Path, Announcements) still decode. func TestDecodeDropsCommunityAndRaw(t *testing.T) { + t.Parallel() + f, err := os.Open(messageExamplesPath) if err != nil { t.Fatalf("open fixtures: %v", err) } - defer f.Close() + + defer func() { + _ = f.Close() + }() var messages, withPath, withAnnouncements int scanner := bufio.NewScanner(f) scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024) + for scanner.Scan() { line := scanner.Bytes() if len(line) == 0 { continue } - var wrapper RISLiveMessage - if err := json.Unmarshal(line, &wrapper); err != nil { + var wrapper ristypes.RISLiveMessage + + err := json.Unmarshal(line, &wrapper) + if err != nil { t.Fatalf("unmarshal message %d: %v", messages+1, err) } + messages++ msg := wrapper.Data if msg.Community != nil { - t.Errorf("message %d: Community decoded, want empty: %v", messages, msg.Community) + t.Errorf("message %d: Community decoded, want empty: %v", + messages, msg.Community) } + if msg.Raw != "" { t.Errorf("message %d: Raw decoded, want empty", messages) } @@ -48,11 +61,14 @@ func TestDecodeDropsCommunityAndRaw(t *testing.T) { if len(msg.Path) > 0 { withPath++ } + if len(msg.Announcements) > 0 { withAnnouncements++ } } - if err := scanner.Err(); err != nil { + + err = scanner.Err() + if err != nil { t.Fatalf("scan fixtures: %v", err) } @@ -62,6 +78,7 @@ func TestDecodeDropsCommunityAndRaw(t *testing.T) { // The fixtures include announcement messages; a used field must still decode, // otherwise an empty Community/Raw would prove nothing. if withPath == 0 || withAnnouncements == 0 { - t.Fatalf("used fields did not decode: withPath=%d withAnnouncements=%d", withPath, withAnnouncements) + t.Fatalf("used fields did not decode: withPath=%d withAnnouncements=%d", + withPath, withAnnouncements) } } diff --git a/internal/routewatch/app.go b/internal/routewatch/app.go index 0c29d75..9e79f11 100644 --- a/internal/routewatch/app.go +++ b/internal/routewatch/app.go @@ -1,10 +1,13 @@ -// Package routewatch contains the primary RouteWatch type that represents a running instance -// of the application and contains pointers to its core dependencies, and is responsible for initialization. +// Package routewatch contains the primary RouteWatch type that represents a +// running instance of the application and contains pointers to its core +// dependencies, and is responsible for initialization. It also contains the +// ASN WHOIS fetcher for background updates and the database maintainer for +// background maintenance tasks. package routewatch import ( "context" - "fmt" + "errors" "sync" "time" @@ -18,6 +21,10 @@ import ( "go.uber.org/fx" ) +// errNonBatchedNotImplemented is returned by Run when batched database writes +// are disabled, since only the batched handlers exist. +var errNonBatchedNotImplemented = errors.New("non-batched handlers not implemented") + // Dependencies contains all dependencies for RouteWatch type Dependencies struct { fx.In @@ -68,8 +75,10 @@ func (rw *RouteWatch) Run(ctx context.Context) error { // Apply runtime limit if specified if rw.maxRuntime > 0 { var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, rw.maxRuntime) defer cancel() + rw.logger.Info("Running with time limit", "max_runtime", rw.maxRuntime) } @@ -96,29 +105,31 @@ func (rw *RouteWatch) Run(ctx context.Context) error { // Non-batched handlers not implemented yet rw.logger.Error("Non-batched handlers not implemented") - return fmt.Errorf("non-batched handlers not implemented") + return errNonBatchedNotImplemented } // No longer need routing table handler - PrefixHandler maintains live_routes table // Start streaming - if err := rw.streamer.Start(); err != nil { + err := rw.streamer.Start(ctx) + if err != nil { return err } // Start HTTP server - if err := rw.server.Start(); err != nil { + err = rw.server.Start(ctx) + if err != nil { return err } // Start ASN WHOIS fetcher for background updates rw.asnFetcher = NewASNFetcher(rw.db, rw.logger.Logger) - rw.asnFetcher.Start() + rw.asnFetcher.Start(ctx) rw.server.SetASNFetcher(rw.asnFetcher) // Start database maintenance goroutine rw.dbMaintainer = NewDBMaintainer(rw.db, rw.logger.Logger) - rw.dbMaintainer.Start() + rw.dbMaintainer.Start(ctx) // Wait for context cancellation <-ctx.Done() @@ -127,13 +138,14 @@ func (rw *RouteWatch) Run(ctx context.Context) error { } // Shutdown performs graceful shutdown of all services -func (rw *RouteWatch) Shutdown() { +func (rw *RouteWatch) Shutdown(ctx context.Context) { rw.mu.Lock() if rw.shutdown { rw.mu.Unlock() return } + rw.shutdown = true rw.mu.Unlock() @@ -142,14 +154,17 @@ func (rw *RouteWatch) Shutdown() { rw.logger.Info("Flushing database handler") rw.dbHandler.Stop() } + if rw.peerHandler != nil { rw.logger.Info("Flushing peer handler") rw.peerHandler.Stop() } + if rw.prefixHandler != nil { rw.logger.Info("Flushing prefix handler") rw.prefixHandler.Stop() } + if rw.peeringHandler != nil { rw.logger.Info("Flushing peering handler") rw.peeringHandler.Stop() @@ -168,11 +183,15 @@ func (rw *RouteWatch) Shutdown() { // Stop services rw.streamer.Stop() - // Stop HTTP server with a timeout + // Stop HTTP server with a timeout of its own, however much of ctx is left const serverStopTimeout = 5 * time.Second - stopCtx, cancel := context.WithTimeout(context.Background(), serverStopTimeout) + + stopCtx, cancel := context.WithTimeout( + context.WithoutCancel(ctx), serverStopTimeout) defer cancel() - if err := rw.server.Stop(stopCtx); err != nil { + + err := rw.server.Stop(stopCtx) + if err != nil { rw.logger.Error("Failed to stop HTTP server gracefully", "error", err) } @@ -185,12 +204,11 @@ func (rw *RouteWatch) Shutdown() { "bits_per_sec", metrics.BitsPerSec, "duration", time.Since(metrics.ConnectedSince), ) - } // getModule provides all fx dependencies -func getModule() fx.Option { - return fx.Options( +func getModule() []fx.Option { + return []fx.Option{ fx.Provide( logger.New, config.New, @@ -203,5 +221,5 @@ func getModule() fx.Option { server.New, New, ), - ) + } } diff --git a/internal/routewatch/app_integration_test.go b/internal/routewatch/app_integration_internal_test.go similarity index 78% rename from internal/routewatch/app_integration_test.go rename to internal/routewatch/app_integration_internal_test.go index a23d39a..13bb7aa 100644 --- a/internal/routewatch/app_integration_test.go +++ b/internal/routewatch/app_integration_internal_test.go @@ -50,7 +50,9 @@ func newMockStore() *mockStore { } // GetOrCreateASN mock implementation -func (m *mockStore) GetOrCreateASN(number int, timestamp time.Time) (*database.ASN, error) { +func (m *mockStore) GetOrCreateASN( + number int, timestamp time.Time, +) (*database.ASN, error) { m.mu.Lock() defer m.mu.Unlock() @@ -99,11 +101,14 @@ func (m *mockStore) UpdatePrefixesBatch(prefixes map[string]time.Time) error { } } } + return nil } // GetOrCreatePrefix mock implementation -func (m *mockStore) GetOrCreatePrefix(prefix string, timestamp time.Time) (*database.Prefix, error) { +func (m *mockStore) GetOrCreatePrefix( + prefix string, timestamp time.Time, +) (*database.Prefix, error) { m.mu.Lock() defer m.mu.Unlock() @@ -168,7 +173,7 @@ func (m *mockStore) RecordPeering(asA, asB int, _ time.Time) error { } // UpdatePeer mock implementation -func (m *mockStore) UpdatePeer(peerIP string, peerASN int, messageType string, timestamp time.Time) error { +func (m *mockStore) UpdatePeer(_ string, _ int, _ string, _ time.Time) error { // Simple mock - just return nil return nil } @@ -194,41 +199,45 @@ func (m *mockStore) GetStats() (database.Stats, error) { } // GetStatsContext returns statistics about the mock store with context support -func (m *mockStore) GetStatsContext(ctx context.Context) (database.Stats, error) { +func (m *mockStore) GetStatsContext(_ context.Context) (database.Stats, error) { return m.GetStats() } // UpsertLiveRoute mock implementation -func (m *mockStore) UpsertLiveRoute(route *database.LiveRoute) error { +func (m *mockStore) UpsertLiveRoute(_ *database.LiveRoute) error { // Simple mock - just return nil return nil } // DeleteLiveRoute mock implementation -func (m *mockStore) DeleteLiveRoute(prefix, peerIP string) error { +func (m *mockStore) DeleteLiveRoute(_, _ string) error { // Simple mock - just return nil return nil } // GetPrefixDistribution mock implementation -func (m *mockStore) GetPrefixDistribution() (ipv4 []database.PrefixDistribution, ipv6 []database.PrefixDistribution, err error) { +func (m *mockStore) GetPrefixDistribution() ( + []database.PrefixDistribution, []database.PrefixDistribution, error, +) { // Return empty distributions for now return nil, nil, nil } // GetPrefixDistributionContext mock implementation with context support -func (m *mockStore) GetPrefixDistributionContext(ctx context.Context) (ipv4 []database.PrefixDistribution, ipv6 []database.PrefixDistribution, err error) { +func (m *mockStore) GetPrefixDistributionContext(_ context.Context) ( + []database.PrefixDistribution, []database.PrefixDistribution, error, +) { return m.GetPrefixDistribution() } // GetLiveRouteCounts mock implementation -func (m *mockStore) GetLiveRouteCounts() (ipv4Count, ipv6Count int, err error) { +func (m *mockStore) GetLiveRouteCounts() (int, int, error) { // Return mock counts return m.RouteCount / 2, m.RouteCount / 2, nil } // GetLiveRouteCountsContext mock implementation with context support -func (m *mockStore) GetLiveRouteCountsContext(ctx context.Context) (ipv4Count, ipv6Count int, err error) { +func (m *mockStore) GetLiveRouteCountsContext(_ context.Context) (int, int, error) { return m.GetLiveRouteCounts() } @@ -247,41 +256,51 @@ func (m *mockStore) GetASDetails(asn int) (*database.ASN, []database.LiveRoute, } // GetASDetailsContext mock implementation with context support -func (m *mockStore) GetASDetailsContext(ctx context.Context, asn int) (*database.ASN, []database.LiveRoute, error) { +func (m *mockStore) GetASDetailsContext( + _ context.Context, asn int, +) (*database.ASN, []database.LiveRoute, error) { return m.GetASDetails(asn) } // GetPrefixDetails mock implementation -func (m *mockStore) GetPrefixDetails(prefix string) ([]database.LiveRoute, map[int]database.ASN, error) { +func (m *mockStore) GetPrefixDetails( + _ string, +) ([]database.LiveRoute, map[int]database.ASN, error) { // Return empty routes for now return []database.LiveRoute{}, map[int]database.ASN{}, nil } // GetPrefixDetailsContext mock implementation with context support func (m *mockStore) GetPrefixDetailsContext( - ctx context.Context, prefix string, + _ context.Context, prefix string, ) ([]database.LiveRoute, map[int]database.ASN, error) { return m.GetPrefixDetails(prefix) } -func (m *mockStore) GetRandomPrefixesByLength(maskLength, ipVersion, limit int) ([]database.LiveRoute, error) { +func (m *mockStore) GetRandomPrefixesByLength( + _, _, _ int, +) ([]database.LiveRoute, error) { // Return empty routes for now return []database.LiveRoute{}, nil } // GetRandomPrefixesByLengthContext mock implementation with context support -func (m *mockStore) GetRandomPrefixesByLengthContext(ctx context.Context, maskLength, ipVersion, limit int) ([]database.LiveRoute, error) { +func (m *mockStore) GetRandomPrefixesByLengthContext( + _ context.Context, maskLength, ipVersion, limit int, +) ([]database.LiveRoute, error) { return m.GetRandomPrefixesByLength(maskLength, ipVersion, limit) } // GetASPeers mock implementation -func (m *mockStore) GetASPeers(asn int) ([]database.ASPeer, error) { +func (m *mockStore) GetASPeers(_ int) ([]database.ASPeer, error) { // Return empty peers for now return []database.ASPeer{}, nil } // GetASPeersContext mock implementation with context support -func (m *mockStore) GetASPeersContext(ctx context.Context, asn int) ([]database.ASPeer, error) { +func (m *mockStore) GetASPeersContext( + _ context.Context, asn int, +) ([]database.ASPeer, error) { return m.GetASPeers(asn) } @@ -291,8 +310,11 @@ func (m *mockStore) GetIPInfo(ip string) (*database.IPInfo, error) { } // GetIPInfoContext mock implementation with context support -func (m *mockStore) GetIPInfoContext(ctx context.Context, ip string) (*database.IPInfo, error) { +func (m *mockStore) GetIPInfoContext( + _ context.Context, ip string, +) (*database.IPInfo, error) { now := time.Now() + return &database.IPInfo{ IP: ip, Netblock: "8.8.8.0/24", @@ -309,17 +331,21 @@ func (m *mockStore) GetIPInfoContext(ctx context.Context, ip string) (*database. } // GetNextStaleASN mock implementation -func (m *mockStore) GetNextStaleASN(ctx context.Context, staleThreshold time.Duration) (int, error) { +func (m *mockStore) GetNextStaleASN(_ context.Context, _ time.Duration) (int, error) { return 0, database.ErrNoStaleASN } // UpdateASNWHOIS mock implementation -func (m *mockStore) UpdateASNWHOIS(ctx context.Context, update *database.ASNWHOISUpdate) error { +func (m *mockStore) UpdateASNWHOIS( + _ context.Context, _ *database.ASNWHOISUpdate, +) error { return nil } // GetWHOISStats mock implementation -func (m *mockStore) GetWHOISStats(ctx context.Context, staleThreshold time.Duration) (*database.WHOISStats, error) { +func (m *mockStore) GetWHOISStats( + _ context.Context, _ time.Duration, +) (*database.WHOISStats, error) { m.mu.Lock() defer m.mu.Unlock() @@ -347,25 +373,28 @@ func (m *mockStore) UpsertLiveRouteBatch(routes []*database.LiveRoute) error { LastSeen: route.LastUpdated, } m.PrefixCount++ + if route.IPVersion == 4 { m.IPv4Prefixes++ } else { m.IPv6Prefixes++ } } + m.RouteCount++ } + return nil } // DeleteLiveRouteBatch mock implementation -func (m *mockStore) DeleteLiveRouteBatch(deletions []database.LiveRouteDeletion) error { +func (m *mockStore) DeleteLiveRouteBatch(_ []database.LiveRouteDeletion) error { // Simple mock - just return nil return nil } // DeleteLiveRoutesOfPeer mock implementation -func (m *mockStore) DeleteLiveRoutesOfPeer(peerIP string) (int, error) { +func (m *mockStore) DeleteLiveRoutesOfPeer(_ string) (int, error) { // Simple mock - deletes nothing return 0, nil } @@ -385,43 +414,53 @@ func (m *mockStore) GetOrCreateASNBatch(asns map[int]time.Time) error { m.ASNCount++ } } + return nil } // UpdatePeerBatch mock implementation -func (m *mockStore) UpdatePeerBatch(peers map[string]database.PeerUpdate) error { +func (m *mockStore) UpdatePeerBatch(_ map[string]database.PeerUpdate) error { // Simple mock - just return nil return nil } // Vacuum mock implementation -func (m *mockStore) Vacuum(ctx context.Context) error { +func (m *mockStore) Vacuum(_ context.Context) error { return nil } // Analyze mock implementation -func (m *mockStore) Analyze(ctx context.Context) error { +func (m *mockStore) Analyze(_ context.Context) error { return nil } // Checkpoint mock implementation -func (m *mockStore) Checkpoint(ctx context.Context) error { +func (m *mockStore) Checkpoint(_ context.Context) error { return nil } // Ping mock implementation -func (m *mockStore) Ping(ctx context.Context) error { +func (m *mockStore) Ping(_ context.Context) error { return nil } func TestRouteWatchLiveFeed(t *testing.T) { + t.Parallel() + if testing.Short() { - t.Skip("skipping live RIPE RIS network feed test in short mode; run without -short to include it") + t.Skip("skipping live RIPE RIS network feed test in short mode; " + + "run without -short to include it") } // Create mock database mockDB := newMockStore() - defer mockDB.Close() + + defer func() { + err := mockDB.Close() + if err != nil { + t.Errorf("failed to close mock database: %v", err) + } + }() logger := logger.New() @@ -474,16 +513,19 @@ func TestRouteWatchLiveFeed(t *testing.T) { if stats.ASNs == 0 { t.Error("Expected to receive some ASNs from live feed") } + t.Logf("Received %d unique ASNs in 5 seconds", stats.ASNs) if stats.Prefixes == 0 { t.Error("Expected to receive some prefixes from live feed") } - t.Logf("Received %d unique prefixes (%d IPv4, %d IPv6) in 5 seconds", stats.Prefixes, stats.IPv4Prefixes, stats.IPv6Prefixes) + + t.Logf("Received %d unique prefixes (%d IPv4, %d IPv6) in 5 seconds", + stats.Prefixes, stats.IPv4Prefixes, stats.IPv6Prefixes) if stats.Peerings == 0 { t.Error("Expected to receive some peerings from live feed") } - t.Logf("Recorded %d AS peering relationships in 5 seconds", stats.Peerings) + t.Logf("Recorded %d AS peering relationships in 5 seconds", stats.Peerings) } diff --git a/internal/routewatch/app_test.go b/internal/routewatch/app_test.go index 6adfec8..9e52502 100644 --- a/internal/routewatch/app_test.go +++ b/internal/routewatch/app_test.go @@ -1,3 +1,3 @@ -package routewatch +package routewatch_test -// Tests for routewatch package are in app_integration_test.go +// Tests for routewatch package are in app_integration_internal_test.go diff --git a/internal/routewatch/ashandler.go b/internal/routewatch/ashandler.go index 4ff8920..27a003f 100644 --- a/internal/routewatch/ashandler.go +++ b/internal/routewatch/ashandler.go @@ -41,9 +41,11 @@ type ASHandler struct { // Batching. mu guards batch only: the flush goroutine takes the batch // under mu and writes it after releasing mu, so HandleMessage waits for // a database write only while batch is at asnBatchMaxSize. - mu sync.Mutex - batch []asnOp - batchTaken sync.Cond // signalled when the flush goroutine takes batch; its L is &mu + mu sync.Mutex + batch []asnOp + // batchTaken is signalled when the flush goroutine takes batch; its L is + // &mu. + batchTaken sync.Cond batchFullCh chan struct{} // wakes the flush goroutine when batch is full lastFlush time.Time // used only by the flush goroutine stopCh chan struct{} @@ -83,7 +85,7 @@ func NewASHandler(db database.Store, logger *logger.Logger) *ASHandler { // the AS path information needed to track autonomous systems. func (h *ASHandler) WantsMessage(messageType string) bool { // We only care about UPDATE messages for the database - return messageType == "UPDATE" + return messageType == ristypes.MessageTypeUpdate } // QueueCapacity returns the recommended message queue size for this handler. @@ -148,10 +150,21 @@ func (h *ASHandler) HandleMessage(msg *ristypes.RISMessage) { } } +// Stop gracefully shuts down the ASHandler by signaling the background flush +// goroutine to terminate and waiting for it to complete. Any pending ASN +// operations in the current batch are flushed to the database before Stop +// returns. This method should be called during application shutdown to ensure +// no data is lost. +func (h *ASHandler) Stop() { + close(h.stopCh) + h.wg.Wait() +} + // flushLoop runs in a goroutine, the only one that writes the batch: it // flushes when the batch is full, periodically, and once more on Stop. func (h *ASHandler) flushLoop() { defer h.wg.Done() + ticker := time.NewTicker(asnBatchTimeout) defer ticker.Stop() @@ -181,6 +194,7 @@ func (h *ASHandler) takeBatch() []asnOp { if len(h.batch) == 0 { return nil } + batch := h.batch h.batch = make([]asnOp, 0, asnBatchSize) h.batchTaken.Broadcast() @@ -205,19 +219,11 @@ func (h *ASHandler) flushBatch() { } // Process all ASNs in a single batch transaction - if err := h.db.GetOrCreateASNBatch(asnMap); err != nil { - h.logger.Error("Failed to process ASN batch", "error", err, "count", len(asnMap)) + err := h.db.GetOrCreateASNBatch(asnMap) + if err != nil { + h.logger.Error("Failed to process ASN batch", + "error", err, "count", len(asnMap)) } h.lastFlush = time.Now() } - -// Stop gracefully shuts down the ASHandler by signaling the background flush -// goroutine to terminate and waiting for it to complete. Any pending ASN -// operations in the current batch are flushed to the database before Stop -// returns. This method should be called during application shutdown to ensure -// no data is lost. -func (h *ASHandler) Stop() { - close(h.stopCh) - h.wg.Wait() -} diff --git a/internal/routewatch/asnfetcher.go b/internal/routewatch/asnfetcher.go index 7a0b246..7320276 100644 --- a/internal/routewatch/asnfetcher.go +++ b/internal/routewatch/asnfetcher.go @@ -1,8 +1,8 @@ -// Package routewatch contains the ASN WHOIS fetcher for background updates. package routewatch import ( "context" + "errors" "log/slog" "sync" "time" @@ -73,10 +73,13 @@ func NewASNFetcher(db database.Store, logger *slog.Logger) *ASNFetcher { } } -// Start begins the background ASN fetcher goroutine. -func (f *ASNFetcher) Start() { +// Start begins the background ASN fetcher goroutine. It runs until Stop: +// the end of ctx does not stop it. +func (f *ASNFetcher) Start(ctx context.Context) { f.wg.Add(1) - go f.run() + + go f.run(context.WithoutCancel(ctx)) + f.logger.Info("ASN fetcher started", "base_interval", baseInterval, "min_interval", minInterval, @@ -151,10 +154,7 @@ func (f *ASNFetcher) recordSuccess() { f.consecutiveFails = 0 // Decrease interval by half, but not below minimum - newInterval := f.currentInterval / backoffMultiplier - if newInterval < minInterval { - newInterval = minInterval - } + newInterval := max(f.currentInterval/backoffMultiplier, minInterval) if newInterval != f.currentInterval { f.logger.Debug("Decreased fetch interval", @@ -177,10 +177,7 @@ func (f *ASNFetcher) recordFailure() { f.consecutiveFails++ // Exponential backoff: multiply by 2, capped at max - newInterval := f.currentInterval * backoffMultiplier - if newInterval > maxInterval { - newInterval = maxInterval - } + newInterval := min(f.currentInterval*backoffMultiplier, maxInterval) if newInterval != f.currentInterval { f.logger.Debug("Increased fetch interval due to failure", @@ -199,7 +196,7 @@ func (f *ASNFetcher) recordFailure() { } // run is the main background loop. -func (f *ASNFetcher) run() { +func (f *ASNFetcher) run(ctx context.Context) { defer f.wg.Done() timer := time.NewTimer(f.getInterval()) @@ -212,13 +209,13 @@ func (f *ASNFetcher) run() { case asn := <-f.immediateQueue: // Process immediate request (respects lock) - f.tryFetch(asn) + f.tryFetch(ctx, asn) // Reset timer after immediate fetch timer.Reset(f.getInterval()) case <-timer.C: // Background fetch of stale/missing ASN - f.fetchNextStale() + f.fetchNextStale(ctx) // Reset timer with potentially updated interval timer.Reset(f.getInterval()) } @@ -227,7 +224,7 @@ func (f *ASNFetcher) run() { // tryFetch attempts to fetch and update an ASN, respecting the fetch lock. // Returns true if fetch was successful. -func (f *ASNFetcher) tryFetch(asn int) bool { +func (f *ASNFetcher) tryFetch(ctx context.Context, asn int) bool { // Try to acquire lock, skip if another fetch is running if !f.fetchMu.TryLock() { f.logger.Debug("Skipping fetch, another fetch in progress", "asn", asn) @@ -236,11 +233,11 @@ func (f *ASNFetcher) tryFetch(asn int) bool { } defer f.fetchMu.Unlock() - return f.fetchAndUpdate(asn) + return f.fetchAndUpdate(ctx, asn) } // fetchNextStale finds and fetches the next ASN needing WHOIS data. -func (f *ASNFetcher) fetchNextStale() { +func (f *ASNFetcher) fetchNextStale(ctx context.Context) { // Try to acquire lock, skip if another fetch is running if !f.fetchMu.TryLock() { f.logger.Debug("Skipping stale fetch, another fetch in progress") @@ -249,12 +246,12 @@ func (f *ASNFetcher) fetchNextStale() { } defer f.fetchMu.Unlock() - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + lookupCtx, cancel := context.WithTimeout(ctx, time.Minute) defer cancel() - asn, err := f.db.GetNextStaleASN(ctx, whoisStaleThreshold) + asn, err := f.db.GetNextStaleASN(lookupCtx, whoisStaleThreshold) if err != nil { - if err != database.ErrNoStaleASN { + if !errors.Is(err, database.ErrNoStaleASN) { f.logger.Error("Failed to get stale ASN", "error", err) f.recordFailure() } @@ -263,13 +260,13 @@ func (f *ASNFetcher) fetchNextStale() { return } - f.fetchAndUpdate(asn) + f.fetchAndUpdate(ctx, asn) } // fetchAndUpdate performs a WHOIS lookup and updates the database. // Returns true if successful. -func (f *ASNFetcher) fetchAndUpdate(asn int) bool { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) +func (f *ASNFetcher) fetchAndUpdate(ctx context.Context, asn int) bool { + ctx, cancel := context.WithTimeout(ctx, time.Minute) defer cancel() f.logger.Info("Fetching WHOIS data", "asn", asn) diff --git a/internal/routewatch/batchwrite_test.go b/internal/routewatch/batchwrite_internal_test.go similarity index 82% rename from internal/routewatch/batchwrite_test.go rename to internal/routewatch/batchwrite_internal_test.go index 4551eae..d6e6d1d 100644 --- a/internal/routewatch/batchwrite_test.go +++ b/internal/routewatch/batchwrite_internal_test.go @@ -49,21 +49,11 @@ func newBlockingStore() *blockingStore { } } -func (s *blockingStore) waitForRelease() { - s.startOnce.Do(func() { close(s.writeStarted) }) - <-s.release -} - -// releaseWrites lets the held writes, and all later ones, finish. A second call -// does nothing. -func (s *blockingStore) releaseWrites() { - s.releaseOnce.Do(func() { close(s.release) }) -} - func (s *blockingStore) UpsertLiveRouteBatch(routes []*database.LiveRoute) error { s.waitForRelease() s.mu.Lock() defer s.mu.Unlock() + for _, route := range routes { s.prefixes[route.Prefix] = true } @@ -75,6 +65,7 @@ func (s *blockingStore) GetOrCreateASNBatch(asns map[int]time.Time) error { s.waitForRelease() s.mu.Lock() defer s.mu.Unlock() + for asn := range asns { s.asns[asn] = true } @@ -86,6 +77,7 @@ func (s *blockingStore) UpdatePeerBatch(peers map[string]database.PeerUpdate) er s.waitForRelease() s.mu.Lock() defer s.mu.Unlock() + for peerIP := range peers { s.peers[peerIP] = true } @@ -93,9 +85,21 @@ func (s *blockingStore) UpdatePeerBatch(peers map[string]database.PeerUpdate) er return nil } +func (s *blockingStore) waitForRelease() { + s.startOnce.Do(func() { close(s.writeStarted) }) + <-s.release +} + +// releaseWrites lets the held writes, and all later ones, finish. A second call +// does nothing. +func (s *blockingStore) releaseWrites() { + s.releaseOnce.Do(func() { close(s.release) }) +} + // returnsInTime reports whether f returns within handleMessageTimeout. func returnsInTime(f func()) bool { done := make(chan struct{}) + go func() { f() close(done) @@ -115,7 +119,9 @@ func returnsInTime(f func()) bool { func startBlockedWrite( t *testing.T, store *blockingStore, - h interface{ HandleMessage(*ristypes.RISMessage) }, + h interface { + HandleMessage(msg *ristypes.RISMessage) + }, fill []*ristypes.RISMessage, ) { t.Helper() @@ -125,7 +131,8 @@ func startBlockedWrite( h.HandleMessage(msg) } }) { - t.Fatal("HandleMessage did not return after filling the batch: it waits for the database write") + t.Fatal("HandleMessage did not return after filling the batch: " + + "it waits for the database write") } select { @@ -143,13 +150,14 @@ func handleWhileAWriteIsBlocked( t *testing.T, store *blockingStore, h interface { - HandleMessage(*ristypes.RISMessage) + HandleMessage(msg *ristypes.RISMessage) Stop() }, fill []*ristypes.RISMessage, extra *ristypes.RISMessage, ) { t.Helper() + defer h.Stop() defer store.releaseWrites() @@ -169,22 +177,25 @@ func handleAtMaxBatchSize( t *testing.T, store *blockingStore, h interface { - HandleMessage(*ristypes.RISMessage) + HandleMessage(msg *ristypes.RISMessage) Stop() }, fill, toMax []*ristypes.RISMessage, extra *ristypes.RISMessage, ) { t.Helper() + defer h.Stop() defer store.releaseWrites() startBlockedWrite(t, store, h, fill) + for _, msg := range toMax { h.HandleMessage(msg) } returned := make(chan struct{}) + go func() { h.HandleMessage(extra) close(returned) @@ -192,11 +203,13 @@ func handleAtMaxBatchSize( select { case <-returned: - t.Fatal("HandleMessage returned while the batch was at its maximum size and a write was blocked") + t.Fatal("HandleMessage returned while the batch was at its maximum " + + "size and a write was blocked") case <-time.After(stillWaitingAfter): } store.releaseWrites() + select { case <-returned: case <-time.After(handleMessageTimeout): @@ -208,46 +221,57 @@ func handleAtMaxBatchSize( // with origin AS testASNA. func prefixAnnouncement(prefixes []string) *ristypes.RISMessage { return &ristypes.RISMessage{ - Peer: testPeerIP, - Path: ristypes.ASPath{testASNA}, - Announcements: []ristypes.RISAnnouncement{{NextHop: testPeerIP, Prefixes: prefixes}}, + Peer: testPeerIP, + Path: ristypes.ASPath{testASNA}, + Announcements: []ristypes.RISAnnouncement{ + {NextHop: testPeerIP, Prefixes: prefixes}, + }, } } func TestPrefixHandlerHandlesMessagesWhileAWriteIsBlocked(t *testing.T) { - const ( - fillPrefix = "192.0.2.0/24" - extraPrefix = "198.51.100.0/24" - ) + t.Parallel() + + const extraPrefix = "198.51.100.0/24" store := newBlockingStore() h := NewPrefixHandler(store, logger.New()) + fill := slices.Repeat([]string{testV4Prefix}, prefixBatchSize) handleWhileAWriteIsBlocked(t, store, h, - []*ristypes.RISMessage{prefixAnnouncement(slices.Repeat([]string{fillPrefix}, prefixBatchSize))}, + []*ristypes.RISMessage{prefixAnnouncement(fill)}, prefixAnnouncement([]string{extraPrefix})) store.mu.Lock() defer store.mu.Unlock() + if !store.prefixes[extraPrefix] { - t.Errorf("Stop did not write %s, handled while a write was blocked", extraPrefix) + t.Errorf("Stop did not write %s, handled while a write was blocked", + extraPrefix) } } func TestASHandlerHandlesMessagesWhileAWriteIsBlocked(t *testing.T) { + t.Parallel() + store := newBlockingStore() h := NewASHandler(store, logger.New()) + fill := slices.Repeat(ristypes.ASPath{testASNA}, asnBatchSize) handleWhileAWriteIsBlocked(t, store, h, - []*ristypes.RISMessage{{Path: slices.Repeat(ristypes.ASPath{testASNA}, asnBatchSize)}}, + []*ristypes.RISMessage{{Path: fill}}, &ristypes.RISMessage{Path: ristypes.ASPath{testASNB}}) store.mu.Lock() defer store.mu.Unlock() + if !store.asns[testASNB] { - t.Errorf("Stop did not write AS%d, handled while a write was blocked", testASNB) + t.Errorf("Stop did not write AS%d, handled while a write was blocked", + testASNB) } } func TestPeerHandlerHandlesMessagesWhileAWriteIsBlocked(t *testing.T) { + t.Parallel() + const otherPeerIP = "2001:db8:ffff::2" store := newBlockingStore() @@ -258,60 +282,76 @@ func TestPeerHandlerHandlesMessagesWhileAWriteIsBlocked(t *testing.T) { store.mu.Lock() defer store.mu.Unlock() + if !store.peers[otherPeerIP] { - t.Errorf("Stop did not write peer %s, handled while a write was blocked", otherPeerIP) + t.Errorf("Stop did not write peer %s, handled while a write was blocked", + otherPeerIP) } } func TestPrefixHandlerWaitsAtTheMaximumBatchSize(t *testing.T) { - const ( - fillPrefix = "192.0.2.0/24" - extraPrefix = "198.51.100.0/24" - ) + t.Parallel() + + const extraPrefix = "198.51.100.0/24" store := newBlockingStore() h := NewPrefixHandler(store, logger.New()) + fill := slices.Repeat([]string{testV4Prefix}, prefixBatchSize) + toMax := slices.Repeat([]string{testV4Prefix}, prefixBatchMaxSize) handleAtMaxBatchSize(t, store, h, - []*ristypes.RISMessage{prefixAnnouncement(slices.Repeat([]string{fillPrefix}, prefixBatchSize))}, - []*ristypes.RISMessage{prefixAnnouncement(slices.Repeat([]string{fillPrefix}, prefixBatchMaxSize))}, + []*ristypes.RISMessage{prefixAnnouncement(fill)}, + []*ristypes.RISMessage{prefixAnnouncement(toMax)}, prefixAnnouncement([]string{extraPrefix})) store.mu.Lock() defer store.mu.Unlock() + if !store.prefixes[extraPrefix] { - t.Errorf("Stop did not write %s, handled after the batch reached its maximum size", extraPrefix) + t.Errorf("Stop did not write %s, handled after the batch reached its "+ + "maximum size", extraPrefix) } } func TestASHandlerWaitsAtTheMaximumBatchSize(t *testing.T) { + t.Parallel() + store := newBlockingStore() h := NewASHandler(store, logger.New()) + fill := slices.Repeat(ristypes.ASPath{testASNA}, asnBatchSize) + toMax := slices.Repeat(ristypes.ASPath{testASNA}, asnBatchMaxSize) handleAtMaxBatchSize(t, store, h, - []*ristypes.RISMessage{{Path: slices.Repeat(ristypes.ASPath{testASNA}, asnBatchSize)}}, - []*ristypes.RISMessage{{Path: slices.Repeat(ristypes.ASPath{testASNA}, asnBatchMaxSize)}}, + []*ristypes.RISMessage{{Path: fill}}, + []*ristypes.RISMessage{{Path: toMax}}, &ristypes.RISMessage{Path: ristypes.ASPath{testASNB}}) store.mu.Lock() defer store.mu.Unlock() + if !store.asns[testASNB] { - t.Errorf("Stop did not write AS%d, handled after the batch reached its maximum size", testASNB) + t.Errorf("Stop did not write AS%d, handled after the batch reached its "+ + "maximum size", testASNB) } } func TestPeerHandlerWaitsAtTheMaximumBatchSize(t *testing.T) { + t.Parallel() + const otherPeerIP = "2001:db8:ffff::2" store := newBlockingStore() h := NewPeerHandler(store, logger.New()) + message := []*ristypes.RISMessage{{Peer: testPeerIP}} handleAtMaxBatchSize(t, store, h, - slices.Repeat([]*ristypes.RISMessage{{Peer: testPeerIP}}, peerBatchSize), - slices.Repeat([]*ristypes.RISMessage{{Peer: testPeerIP}}, peerBatchMaxSize), + slices.Repeat(message, peerBatchSize), + slices.Repeat(message, peerBatchMaxSize), &ristypes.RISMessage{Peer: otherPeerIP}) store.mu.Lock() defer store.mu.Unlock() + if !store.peers[otherPeerIP] { - t.Errorf("Stop did not write peer %s, handled after the batch reached its maximum size", otherPeerIP) + t.Errorf("Stop did not write peer %s, handled after the batch reached "+ + "its maximum size", otherPeerIP) } } @@ -320,6 +360,8 @@ func TestPeerHandlerWaitsAtTheMaximumBatchSize(t *testing.T) { // of a full batch, and a second one exactly a full batch. Only the full batch // leaves a wake-up in batchFullCh. func TestHandlersWakeTheFlushGoroutineWhenTheBatchIsFull(t *testing.T) { + t.Parallel() + tests := []struct { name string batchSize int @@ -331,7 +373,8 @@ func TestHandlersWakeTheFlushGoroutineWhenTheBatchIsFull(t *testing.T) { batchSize: prefixBatchSize, handle: func(n int) chan struct{} { h := &PrefixHandler{batchFullCh: make(chan struct{}, 1)} - h.HandleMessage(prefixAnnouncement(slices.Repeat([]string{"192.0.2.0/24"}, n))) + h.HandleMessage(prefixAnnouncement( + slices.Repeat([]string{testV4Prefix}, n))) return h.batchFullCh }, @@ -342,7 +385,9 @@ func TestHandlersWakeTheFlushGoroutineWhenTheBatchIsFull(t *testing.T) { handle: func(n int) chan struct{} { h := &ASHandler{batchFullCh: make(chan struct{}, 1)} // The origin AS and each AS in the path are one update each. - h.HandleMessage(&ristypes.RISMessage{Path: slices.Repeat(ristypes.ASPath{testASNA}, n-1)}) + h.HandleMessage(&ristypes.RISMessage{ + Path: slices.Repeat(ristypes.ASPath{testASNA}, n-1), + }) return h.batchFullCh }, @@ -363,10 +408,16 @@ func TestHandlersWakeTheFlushGoroutineWhenTheBatchIsFull(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := len(tt.handle(tt.batchSize - 1)); got != 0 { - t.Errorf("one update short of a full batch left %d wake-ups, want none", got) + t.Parallel() + + got := len(tt.handle(tt.batchSize - 1)) + if got != 0 { + t.Errorf("one update short of a full batch left %d wake-ups, "+ + "want none", got) } - if got := len(tt.handle(tt.batchSize)); got != 1 { + + got = len(tt.handle(tt.batchSize)) + if got != 1 { t.Errorf("a full batch left %d wake-ups, want 1", got) } }) diff --git a/internal/routewatch/cli.go b/internal/routewatch/cli.go index 6c2bd6c..ea7d475 100644 --- a/internal/routewatch/cli.go +++ b/internal/routewatch/cli.go @@ -57,14 +57,19 @@ func logDebugStats(logger *logger.Logger) { // CLIEntry is the main entry point for the routewatch command-line interface. // It initializes the application using the fx dependency injection framework, -// sets up signal handling for graceful shutdown, and starts the RouteWatch service. -// This function blocks until the application receives a shutdown signal or encounters -// a fatal error. +// sets up signal handling for graceful shutdown, and starts the RouteWatch +// service. This function blocks until the application receives a shutdown +// signal or encounters a fatal error. func CLIEntry() { app := fx.New( - getModule(), + fx.Options(getModule()...), fx.StopTimeout(shutdownTimeout), // Allow 60 seconds for graceful shutdown - fx.Invoke(func(lc fx.Lifecycle, rw *RouteWatch, logger *logger.Logger, shutdowner fx.Shutdowner) { + fx.Invoke(func( + lc fx.Lifecycle, + rw *RouteWatch, + logger *logger.Logger, + shutdowner fx.Shutdowner, + ) { lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { // Start debug stats logging @@ -77,25 +82,31 @@ func CLIEntry() { go func() { <-sigCh logger.Info("Received shutdown signal") - if err := shutdowner.Shutdown(); err != nil { + + err := shutdowner.Shutdown() + if err != nil { logger.Error("Failed to shutdown gracefully", "error", err) } }() go func() { - if err := rw.Run(ctx); err != nil { + err := rw.Run(ctx) + if err != nil { logger.Error("RouteWatch error", "error", err) - if err := shutdowner.Shutdown(fx.ExitCode(failureExitCode)); err != nil { - logger.Error("Failed to shutdown gracefully", "error", err) + + err = shutdowner.Shutdown(fx.ExitCode(failureExitCode)) + if err != nil { + logger.Error("Failed to shutdown gracefully", + "error", err) } } }() return nil }, - OnStop: func(_ context.Context) error { + OnStop: func(ctx context.Context) error { logger.Info("Shutting down RouteWatch") - rw.Shutdown() + rw.Shutdown(ctx) return nil }, diff --git a/internal/routewatch/dbmaintainer.go b/internal/routewatch/dbmaintainer.go index 993d616..8859a46 100644 --- a/internal/routewatch/dbmaintainer.go +++ b/internal/routewatch/dbmaintainer.go @@ -1,4 +1,3 @@ -// Package routewatch contains the database maintainer for background maintenance tasks. package routewatch import ( @@ -64,10 +63,13 @@ func NewDBMaintainer(db database.Store, logger *slog.Logger) *DBMaintainer { } } -// Start begins the background maintenance goroutine. -func (m *DBMaintainer) Start() { +// Start begins the background maintenance goroutine. It runs until Stop: +// the end of ctx does not stop it. +func (m *DBMaintainer) Start(ctx context.Context) { m.wg.Add(1) - go m.run() + + go m.run(context.WithoutCancel(ctx)) + m.logger.Info("Database maintainer started", "checkpoint_interval", checkpointInterval, "vacuum_interval", vacuumInterval, @@ -83,15 +85,17 @@ func (m *DBMaintainer) Stop() { } // run is the main background loop. -func (m *DBMaintainer) run() { +func (m *DBMaintainer) run(ctx context.Context) { defer m.wg.Done() // Use different timers for each task checkpointTimer := time.NewTimer(checkpointInterval) - vacuumTimer := time.NewTimer(vacuumInterval) - analyzeTimer := time.NewTimer(analyzeInterval) defer checkpointTimer.Stop() + + vacuumTimer := time.NewTimer(vacuumInterval) defer vacuumTimer.Stop() + + analyzeTimer := time.NewTimer(analyzeInterval) defer analyzeTimer.Stop() for { @@ -100,90 +104,88 @@ func (m *DBMaintainer) run() { return case <-checkpointTimer.C: - m.runCheckpoint() + m.runCheckpoint(ctx) checkpointTimer.Reset(checkpointInterval) case <-vacuumTimer.C: - m.runVacuum() + m.runVacuum(ctx) vacuumTimer.Reset(vacuumInterval) case <-analyzeTimer.C: - m.runAnalyze() + m.runAnalyze(ctx) analyzeTimer.Reset(analyzeInterval) } } } // runCheckpoint performs a WAL checkpoint to keep the WAL file small. -func (m *DBMaintainer) runCheckpoint() { - ctx, cancel := context.WithTimeout(context.Background(), checkpointTimeout) - defer cancel() - - startTime := time.Now() - - err := m.db.Checkpoint(ctx) - - m.statsMu.Lock() - m.lastCheckpoint = time.Now() - m.lastCheckpointError = err - if err == nil { - m.checkpointCount++ - } - m.statsMu.Unlock() +func (m *DBMaintainer) runCheckpoint(ctx context.Context) { + duration, err := runWithTimeout(ctx, checkpointTimeout, m.db.Checkpoint) + m.recordResult(&m.lastCheckpoint, &m.lastCheckpointError, &m.checkpointCount, err) if err != nil { - m.logger.Error("WAL checkpoint failed", "error", err, "duration", time.Since(startTime)) + m.logger.Error("WAL checkpoint failed", "error", err, "duration", duration) } else { - m.logger.Debug("WAL checkpoint completed", "duration", time.Since(startTime)) + m.logger.Debug("WAL checkpoint completed", "duration", duration) } } // runVacuum performs an incremental vacuum operation on the database. -func (m *DBMaintainer) runVacuum() { - ctx, cancel := context.WithTimeout(context.Background(), vacuumTimeout) - defer cancel() - +func (m *DBMaintainer) runVacuum(ctx context.Context) { m.logger.Debug("Running incremental vacuum") - startTime := time.Now() - err := m.db.Vacuum(ctx) - - m.statsMu.Lock() - m.lastVacuum = time.Now() - m.lastVacuumError = err - if err == nil { - m.vacuumCount++ - } - m.statsMu.Unlock() + duration, err := runWithTimeout(ctx, vacuumTimeout, m.db.Vacuum) + m.recordResult(&m.lastVacuum, &m.lastVacuumError, &m.vacuumCount, err) if err != nil { - m.logger.Error("Incremental vacuum failed", "error", err, "duration", time.Since(startTime)) + m.logger.Error("Incremental vacuum failed", "error", err, "duration", duration) } else { - m.logger.Debug("Incremental vacuum completed", "duration", time.Since(startTime)) + m.logger.Debug("Incremental vacuum completed", "duration", duration) } } // runAnalyze performs an ANALYZE operation on the database. -func (m *DBMaintainer) runAnalyze() { - ctx, cancel := context.WithTimeout(context.Background(), analyzeTimeout) - defer cancel() - +func (m *DBMaintainer) runAnalyze(ctx context.Context) { m.logger.Info("Starting database ANALYZE") - startTime := time.Now() - err := m.db.Analyze(ctx) - - m.statsMu.Lock() - m.lastAnalyze = time.Now() - m.lastAnalyzeError = err - if err == nil { - m.analyzeCount++ - } - m.statsMu.Unlock() + duration, err := runWithTimeout(ctx, analyzeTimeout, m.db.Analyze) + m.recordResult(&m.lastAnalyze, &m.lastAnalyzeError, &m.analyzeCount, err) if err != nil { - m.logger.Error("ANALYZE failed", "error", err, "duration", time.Since(startTime)) + m.logger.Error("ANALYZE failed", "error", err, "duration", duration) } else { - m.logger.Info("ANALYZE completed", "duration", time.Since(startTime)) + m.logger.Info("ANALYZE completed", "duration", duration) + } +} + +// runWithTimeout runs operation with a context that ends after timeout and +// returns how long it took and its error. +func runWithTimeout( + ctx context.Context, + timeout time.Duration, + operation func(context.Context) error, +) (time.Duration, error) { + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + startTime := time.Now() + err := operation(ctx) + + return time.Since(startTime), err +} + +// recordResult stores when a maintenance task last ran and its error, and +// counts it when it succeeded. +func (m *DBMaintainer) recordResult( + lastRun *time.Time, lastErr *error, successCount *int, err error, +) { + m.statsMu.Lock() + defer m.statsMu.Unlock() + + *lastRun = time.Now() + *lastErr = err + + if err == nil { + *successCount++ } } diff --git a/internal/routewatch/handler.go b/internal/routewatch/handler.go index 328b7ff..b4a7441 100644 --- a/internal/routewatch/handler.go +++ b/internal/routewatch/handler.go @@ -1,6 +1,8 @@ package routewatch import ( + "slices" + "git.eeqj.de/sneak/routewatch/internal/logger" "git.eeqj.de/sneak/routewatch/internal/ristypes" ) @@ -15,10 +17,11 @@ type SimpleHandler struct { callback func(*ristypes.RISMessage) } -// NewSimpleHandler creates a new SimpleHandler that accepts specific message types. -// The messageTypes parameter specifies which RIS message types this handler will process. -// If messageTypes is empty, the handler will accept all message types. -// The callback function is invoked for each message that passes the type filter. +// NewSimpleHandler creates a new SimpleHandler that accepts specific message +// types. The messageTypes parameter specifies which RIS message types this +// handler will process. If messageTypes is empty, the handler will accept all +// message types. The callback function is invoked for each message that +// passes the type filter. func NewSimpleHandler( logger *logger.Logger, messageTypes []string, @@ -31,22 +34,17 @@ func NewSimpleHandler( } } -// WantsMessage returns true if this handler wants to process messages of the given type. -// It checks whether messageType is in the handler's configured list of accepted types. -// If no specific types were configured (empty messageTypes slice), it returns true for all types. +// WantsMessage returns true if this handler wants to process messages of the +// given type. It checks whether messageType is in the handler's configured +// list of accepted types. If no specific types were configured (empty +// messageTypes slice), it returns true for all types. func (h *SimpleHandler) WantsMessage(messageType string) bool { // If no specific types are set, accept all messages if len(h.messageTypes) == 0 { return true } - for _, t := range h.messageTypes { - if t == messageType { - return true - } - } - - return false + return slices.Contains(h.messageTypes, messageType) } // HandleMessage processes a RIS message by invoking the configured callback function. diff --git a/internal/routewatch/peerhandler.go b/internal/routewatch/peerhandler.go index 61510f0..dcb1bc4 100644 --- a/internal/routewatch/peerhandler.go +++ b/internal/routewatch/peerhandler.go @@ -1,7 +1,8 @@ package routewatch -// peerhandler.go provides batched peer tracking functionality for BGP route monitoring. -// It tracks BGP peers from incoming UPDATE messages and maintains peer state in the database. +// peerhandler.go provides batched peer tracking functionality for BGP route +// monitoring. It tracks BGP peers from incoming UPDATE messages and maintains +// peer state in the database. import ( "strconv" @@ -44,9 +45,11 @@ type PeerHandler struct { // Batching. mu guards peerBatch only: the flush goroutine takes the batch // under mu and writes it after releasing mu, so HandleMessage waits for // a database write only while peerBatch is at peerBatchMaxSize. - mu sync.Mutex - peerBatch []peerUpdate - batchTaken sync.Cond // signalled when the flush goroutine takes peerBatch; its L is &mu + mu sync.Mutex + peerBatch []peerUpdate + // batchTaken is signalled when the flush goroutine takes peerBatch; its L + // is &mu. + batchTaken sync.Cond batchFullCh chan struct{} // wakes the flush goroutine when peerBatch is full lastFlush time.Time // used only by the flush goroutine stopCh chan struct{} @@ -104,8 +107,10 @@ func (h *PeerHandler) QueueCapacity() int { func (h *PeerHandler) HandleMessage(msg *ristypes.RISMessage) { // Parse peer ASN from string peerASN := 0 + if msg.PeerASN != "" { - if asn, err := strconv.Atoi(msg.PeerASN); err == nil { + asn, err := strconv.Atoi(msg.PeerASN) + if err == nil { peerASN = asn } } @@ -139,10 +144,17 @@ func (h *PeerHandler) HandleMessage(msg *ristypes.RISMessage) { } } +// Stop gracefully stops the handler and flushes remaining batches +func (h *PeerHandler) Stop() { + close(h.stopCh) + h.wg.Wait() +} + // flushLoop runs in a goroutine, the only one that writes the batch: it // flushes when the batch is full, periodically, and once more on Stop. func (h *PeerHandler) flushLoop() { defer h.wg.Done() + ticker := time.NewTicker(peerBatchTimeout) defer ticker.Stop() @@ -172,6 +184,7 @@ func (h *PeerHandler) takeBatch() []peerUpdate { if len(h.peerBatch) == 0 { return nil } + batch := h.peerBatch h.peerBatch = make([]peerUpdate, 0, peerBatchSize) h.batchTaken.Broadcast() @@ -190,7 +203,8 @@ func (h *PeerHandler) flushBatch() { // Deduplicate by peer IP, keeping the latest update for each peer peerMap := make(map[string]peerUpdate) for _, update := range batch { - if existing, ok := peerMap[update.peerIP]; !ok || update.timestamp.After(existing.timestamp) { + existing, ok := peerMap[update.peerIP] + if !ok || update.timestamp.After(existing.timestamp) { peerMap[update.peerIP] = update } } @@ -207,15 +221,11 @@ func (h *PeerHandler) flushBatch() { } // Process all peers in a single batch transaction - if err := h.db.UpdatePeerBatch(dbPeerMap); err != nil { - h.logger.Error("Failed to process peer batch", "error", err, "count", len(dbPeerMap)) + err := h.db.UpdatePeerBatch(dbPeerMap) + if err != nil { + h.logger.Error("Failed to process peer batch", + "error", err, "count", len(dbPeerMap)) } h.lastFlush = time.Now() } - -// Stop gracefully stops the handler and flushes remaining batches -func (h *PeerHandler) Stop() { - close(h.stopCh) - h.wg.Wait() -} diff --git a/internal/routewatch/peeringhandler.go b/internal/routewatch/peeringhandler.go index b4f3277..25df1d7 100644 --- a/internal/routewatch/peeringhandler.go +++ b/internal/routewatch/peeringhandler.go @@ -72,7 +72,7 @@ func NewPeeringHandler(db database.Store, logger *logger.Logger) *PeeringHandler // the AS path information needed to extract peering relationships. func (h *PeeringHandler) WantsMessage(messageType string) bool { // We only care about UPDATE messages that have AS paths - return messageType == "UPDATE" + return messageType == ristypes.MessageTypeUpdate } // QueueCapacity returns the buffer size for the handler's message queue. @@ -101,6 +101,7 @@ func (h *PeeringHandler) HandleMessage(msg *ristypes.RISMessage) { return } + key := string(pathJSON) h.mu.Lock() @@ -116,6 +117,23 @@ func (h *PeeringHandler) HandleMessage(msg *ristypes.RISMessage) { h.mu.Unlock() } +// ProcessPeeringsNow triggers immediate processing of all accumulated AS +// paths into peering records. This bypasses the normal periodic processing +// schedule and is primarily intended for testing purposes. +func (h *PeeringHandler) ProcessPeeringsNow() { + h.processPeerings() +} + +// Stop gracefully shuts down the handler by signaling the background +// goroutine to stop and performing a final synchronous processing of +// any remaining AS paths. This ensures no peering data is lost during +// shutdown. +func (h *PeeringHandler) Stop() { + close(h.stopCh) + // Process any remaining peerings synchronously + h.processPeerings() +} + // processLoop runs periodically to process AS paths into peerings func (h *PeeringHandler) processLoop() { ticker := time.NewTicker(peeringProcessInterval) @@ -134,13 +152,6 @@ func (h *PeeringHandler) processLoop() { } } -// ProcessPeeringsNow triggers immediate processing of all accumulated AS -// paths into peering records. This bypasses the normal periodic processing -// schedule and is primarily intended for testing purposes. -func (h *PeeringHandler) ProcessPeeringsNow() { - h.processPeerings() -} - // processPeerings extracts peerings from AS paths and writes to database func (h *PeeringHandler) processPeerings() { // Take the accumulated paths and replace the map with a fresh empty one @@ -160,11 +171,14 @@ func (h *PeeringHandler) processPeerings() { type peeringKey struct { low, high int } + peerings := make(map[peeringKey]time.Time) for pathJSON, timestamp := range paths { var path []int - if err := json.Unmarshal([]byte(pathJSON), &path); err != nil { + + err := json.Unmarshal([]byte(pathJSON), &path) + if err != nil { h.logger.Error("Failed to decode AS path", "error", err) continue @@ -197,6 +211,7 @@ func (h *PeeringHandler) processPeerings() { // Record peerings in database start := time.Now() successCount := 0 + for key, ts := range peerings { err := h.db.RecordPeering(key.low, key.high, ts) if err != nil { @@ -218,13 +233,3 @@ func (h *PeeringHandler) processPeerings() { "duration", time.Since(start), ) } - -// Stop gracefully shuts down the handler by signaling the background -// goroutine to stop and performing a final synchronous processing of -// any remaining AS paths. This ensures no peering data is lost during -// shutdown. -func (h *PeeringHandler) Stop() { - close(h.stopCh) - // Process any remaining peerings synchronously - h.processPeerings() -} diff --git a/internal/routewatch/peeringhandler_test.go b/internal/routewatch/peeringhandler_internal_test.go similarity index 95% rename from internal/routewatch/peeringhandler_test.go rename to internal/routewatch/peeringhandler_internal_test.go index 364fa46..de4f5dd 100644 --- a/internal/routewatch/peeringhandler_test.go +++ b/internal/routewatch/peeringhandler_internal_test.go @@ -53,7 +53,8 @@ func newTestHandler(db database.Store) *PeeringHandler { } } -func snapshot(h *PeeringHandler) (tracked, dropped int) { +// snapshot returns how many paths the handler tracks and how many it dropped. +func snapshot(h *PeeringHandler) (int, int) { h.mu.Lock() defer h.mu.Unlock() @@ -82,6 +83,8 @@ func handle(h *PeeringHandler, ts time.Time, asns ...int) { // empties the path map (the swap) and that a path seen again after a run is // recorded in the next run too. func TestPeeringHandlerProcessesEachRunAndEmpties(t *testing.T) { + t.Parallel() + store := &recordingStore{mockStore: newMockStore()} h := newTestHandler(store) @@ -94,6 +97,7 @@ func TestPeeringHandlerProcessesEachRunAndEmpties(t *testing.T) { if tracked, _ := snapshot(h); tracked != 0 { t.Fatalf("map not empty after first run: %d paths remain", tracked) } + if got := store.callCount(); got != 1 { t.Fatalf("want 1 RecordPeering call after first run, got %d", got) } @@ -105,6 +109,7 @@ func TestPeeringHandlerProcessesEachRunAndEmpties(t *testing.T) { if tracked, _ := snapshot(h); tracked != 0 { t.Fatalf("map not empty after second run: %d paths remain", tracked) } + if got := store.callCount(); got != 2 { t.Fatalf("want 2 RecordPeering calls after second run, got %d", got) } @@ -114,6 +119,8 @@ func TestPeeringHandlerProcessesEachRunAndEmpties(t *testing.T) { // and counts them, while a path already tracked is refreshed rather than // dropped. func TestPeeringHandlerCapDropsAndCounts(t *testing.T) { + t.Parallel() + store := &recordingStore{mockStore: newMockStore()} h := newTestHandler(store) @@ -126,6 +133,7 @@ func TestPeeringHandlerCapDropsAndCounts(t *testing.T) { h.mu.Lock() h.asPaths[existing] = now + for i := 0; len(h.asPaths) < maxTrackedPaths; i++ { h.asPaths[strconv.Itoa(i)] = now } @@ -138,6 +146,7 @@ func TestPeeringHandlerCapDropsAndCounts(t *testing.T) { if tracked != maxTrackedPaths { t.Fatalf("want map size %d after drop, got %d", maxTrackedPaths, tracked) } + if dropped != 1 { t.Fatalf("want dropped count 1, got %d", dropped) } @@ -150,6 +159,7 @@ func TestPeeringHandlerCapDropsAndCounts(t *testing.T) { if tracked != maxTrackedPaths { t.Fatalf("want map size %d after refresh, got %d", maxTrackedPaths, tracked) } + if dropped != 1 { t.Fatalf("want dropped count still 1 after refresh, got %d", dropped) } @@ -157,7 +167,9 @@ func TestPeeringHandlerCapDropsAndCounts(t *testing.T) { h.mu.Lock() gotTS := h.asPaths[existing] h.mu.Unlock() + if !gotTS.Equal(refreshed) { - t.Fatalf("existing path timestamp not refreshed: want %v, got %v", refreshed, gotTS) + t.Fatalf("existing path timestamp not refreshed: want %v, got %v", + refreshed, gotTS) } } diff --git a/internal/routewatch/prefixhandler.go b/internal/routewatch/prefixhandler.go index 4aec6ca..a79238c 100644 --- a/internal/routewatch/prefixhandler.go +++ b/internal/routewatch/prefixhandler.go @@ -41,9 +41,9 @@ const ( ipv6Version = 6 ) -// PrefixHandler tracks BGP prefixes and maintains a live routing table in the database. -// Routes are added on announcement and deleted on withdrawal, and all routes of -// a peer are deleted when its BGP session goes down. +// PrefixHandler tracks BGP prefixes and maintains a live routing table in the +// database. Routes are added on announcement and deleted on withdrawal, and all +// routes of a peer are deleted when its BGP session goes down. type PrefixHandler struct { db database.Store logger *logger.Logger @@ -52,10 +52,14 @@ type PrefixHandler struct { // Batching. mu guards batch and peersDown only: the flush goroutine takes // both under mu and writes them after releasing mu, so HandleMessage // waits for a database write only while batch is at prefixBatchMaxSize. - mu sync.Mutex - batch []prefixUpdate - peersDown []string // peers whose BGP session went down since the batch was taken - batchTaken sync.Cond // signalled when the flush goroutine takes batch; its L is &mu + mu sync.Mutex + batch []prefixUpdate + // peersDown holds the peers whose BGP session went down since the batch + // was taken. + peersDown []string + // batchTaken is signalled when the flush goroutine takes batch; its L is + // &mu. + batchTaken sync.Cond batchFullCh chan struct{} // wakes the flush goroutine when batch is full lastFlush time.Time // used only by the flush goroutine stopCh chan struct{} @@ -95,11 +99,13 @@ func (h *PrefixHandler) SetMetricsTracker(metrics *metrics.Tracker) { h.metrics = metrics } -// WantsMessage returns true if this handler wants to process messages of the given type +// WantsMessage returns true if this handler wants to process messages of the +// given type func (h *PrefixHandler) WantsMessage(messageType string) bool { // UPDATE messages change routes; a STATE message can say a peer's // session went down, which removes all of its routes. - return messageType == ristypes.MessageTypeUpdate || messageType == ristypes.MessageTypeState + return messageType == ristypes.MessageTypeUpdate || + messageType == ristypes.MessageTypeState } // QueueCapacity returns the desired queue capacity for this handler @@ -183,6 +189,12 @@ func (h *PrefixHandler) HandleMessage(msg *ristypes.RISMessage) { } } +// Stop gracefully stops the handler and flushes remaining batches +func (h *PrefixHandler) Stop() { + close(h.stopCh) + h.wg.Wait() +} + // peerSessionDown notes that the peer's BGP session went down, so that the next // flush deletes all of the peer's live routes before it writes the batch. The // peer's updates still in the batch came before the down, which replaces them, @@ -192,7 +204,9 @@ func (h *PrefixHandler) peerSessionDown(peer string) { h.mu.Lock() defer h.mu.Unlock() - h.batch = slices.DeleteFunc(h.batch, func(update prefixUpdate) bool { return update.peer == peer }) + h.batch = slices.DeleteFunc(h.batch, func(update prefixUpdate) bool { + return update.peer == peer + }) h.peersDown = append(h.peersDown, peer) } @@ -200,6 +214,7 @@ func (h *PrefixHandler) peerSessionDown(peer string) { // flushes when the batch is full, periodically, and once more on Stop. func (h *PrefixHandler) flushLoop() { defer h.wg.Done() + ticker := time.NewTicker(prefixBatchTimeout) defer ticker.Stop() @@ -230,6 +245,7 @@ func (h *PrefixHandler) takeBatch() ([]prefixUpdate, []string) { if len(h.batch) == 0 && len(h.peersDown) == 0 { return nil, nil } + batch, peersDown := h.batch, h.peersDown h.batch = make([]prefixUpdate, 0, prefixBatchSize) h.peersDown = nil @@ -250,43 +266,34 @@ func (h *PrefixHandler) flushBatch() { batchSize := len(batch) // The routes of a peer whose session went down are deleted first: the - // batch holds only the updates the peer sent after the down. Most downs - // come from peers that have no routes, so only a delete that removed - // routes is logged. - for _, peer := range peersDown { - deleted, err := h.db.DeleteLiveRoutesOfPeer(peer) - if err != nil { - h.logger.Error("Failed to delete the live routes of a peer whose session went down", - "peer", peer, "error", err) - - continue - } - if deleted > 0 { - h.logger.Info("Deleted the live routes of a peer whose session went down", - "peer", peer, "routes", deleted) - } - } + // batch holds only the updates the peer sent after the down. + h.deleteLiveRoutesOfPeers(peersDown) // For each prefix and peer, keep only the last update the feed delivered. // Updates from different peers for the same prefix are all kept. Arrival // order decides, not the timestamp: timestamps are whole seconds, so // updates within one second carry the same one. type prefixAndPeer struct{ prefix, peer string } + lastUpdates := make(map[prefixAndPeer]prefixUpdate) + for _, update := range batch { lastUpdates[prefixAndPeer{update.prefix, update.peer}] = update } // Collect routes to upsert and delete - var routesToUpsert []*database.LiveRoute - var routesToDelete []database.LiveRouteDeletion + var ( + routesToUpsert []*database.LiveRoute + routesToDelete []database.LiveRouteDeletion + ) // Collect unique prefixes to update prefixesToUpdate := make(map[string]time.Time) for _, update := range lastUpdates { // Track prefix for both announcements and withdrawals - if _, exists := prefixesToUpdate[update.prefix]; !exists || update.timestamp.After(prefixesToUpdate[update.prefix]) { + seen, exists := prefixesToUpdate[update.prefix] + if !exists || update.timestamp.After(seen) { prefixesToUpdate[update.prefix] = update.timestamp } @@ -300,7 +307,8 @@ func (h *PrefixHandler) flushBatch() { // Parse CIDR to get IP version _, ipVersion, err := parseCIDR(update.prefix) if err != nil { - h.logger.Error("Failed to parse CIDR for withdrawal", "prefix", update.prefix, "error", err) + h.logger.Error("Failed to parse CIDR for withdrawal", + "prefix", update.prefix, "error", err) continue } @@ -314,30 +322,7 @@ func (h *PrefixHandler) flushBatch() { } } - // Process batch operations - successCount := 0 - if len(routesToUpsert) > 0 { - if err := h.db.UpsertLiveRouteBatch(routesToUpsert); err != nil { - h.logger.Error("Failed to upsert route batch", "error", err, "count", len(routesToUpsert)) - } else { - successCount += len(routesToUpsert) - } - } - - if len(routesToDelete) > 0 { - if err := h.db.DeleteLiveRouteBatch(routesToDelete); err != nil { - h.logger.Error("Failed to delete route batch", "error", err, "count", len(routesToDelete)) - } else { - successCount += len(routesToDelete) - } - } - - // Update prefix tables - if len(prefixesToUpdate) > 0 { - if err := h.db.UpdatePrefixesBatch(prefixesToUpdate); err != nil { - h.logger.Error("Failed to update prefix batch", "error", err, "count", len(prefixesToUpdate)) - } - } + successCount := h.writeBatch(routesToUpsert, routesToDelete, prefixesToUpdate) elapsed := time.Since(startTime) h.logger.Debug("Flushed prefix batch", @@ -351,6 +336,69 @@ func (h *PrefixHandler) flushBatch() { h.lastFlush = time.Now() } +// deleteLiveRoutesOfPeers deletes all live routes of each of peers, the peers +// whose BGP session went down. Most downs come from peers that have no routes, +// so only a delete that removed routes is logged. +func (h *PrefixHandler) deleteLiveRoutesOfPeers(peers []string) { + for _, peer := range peers { + deleted, err := h.db.DeleteLiveRoutesOfPeer(peer) + if err != nil { + h.logger.Error( + "Failed to delete the live routes of a peer whose session went down", + "peer", peer, "error", err) + + continue + } + + if deleted > 0 { + h.logger.Info( + "Deleted the live routes of a peer whose session went down", + "peer", peer, "routes", deleted) + } + } +} + +// writeBatch writes the collected route upserts, route deletions and prefix +// updates to the database and returns how many routes it wrote. +func (h *PrefixHandler) writeBatch( + routesToUpsert []*database.LiveRoute, + routesToDelete []database.LiveRouteDeletion, + prefixesToUpdate map[string]time.Time, +) int { + successCount := 0 + + if len(routesToUpsert) > 0 { + err := h.db.UpsertLiveRouteBatch(routesToUpsert) + if err != nil { + h.logger.Error("Failed to upsert route batch", + "error", err, "count", len(routesToUpsert)) + } else { + successCount += len(routesToUpsert) + } + } + + if len(routesToDelete) > 0 { + err := h.db.DeleteLiveRouteBatch(routesToDelete) + if err != nil { + h.logger.Error("Failed to delete route batch", + "error", err, "count", len(routesToDelete)) + } else { + successCount += len(routesToDelete) + } + } + + // Update prefix tables + if len(prefixesToUpdate) > 0 { + err := h.db.UpdatePrefixesBatch(prefixesToUpdate) + if err != nil { + h.logger.Error("Failed to update prefix batch", + "error", err, "count", len(prefixesToUpdate)) + } + } + + return successCount +} + // canonicalPrefix returns prefix in the text form net/netip prints, the form // the IP lookup builds when it looks a prefix up. The feed sends IPv6 // announcements compressed ("2001:db8::/32") but IPv6 withdrawals uncompressed @@ -366,8 +414,8 @@ func canonicalPrefix(prefix string) string { return p.Masked().String() } -// parseCIDR extracts the mask length and IP version from a prefix string -func parseCIDR(prefix string) (maskLength int, ipVersion int, err error) { +// parseCIDR returns the mask length and the IP version of a prefix string +func parseCIDR(prefix string) (int, int, error) { _, ipNet, err := net.ParseCIDR(prefix) if err != nil { return 0, 0, err @@ -381,50 +429,6 @@ func parseCIDR(prefix string) (maskLength int, ipVersion int, err error) { return ones, ipv4Version, nil } -// processAnnouncement handles storing an announcement in the database -// nolint:unused // kept for potential future use -func (h *PrefixHandler) processAnnouncement(_ *database.Prefix, update prefixUpdate) { - // Parse CIDR to get mask length - maskLength, ipVersion, err := parseCIDR(update.prefix) - if err != nil { - h.logger.Error("Failed to parse CIDR", - "prefix", update.prefix, - "error", err, - ) - - return - } - - // Track route update metrics - if h.metrics != nil { - if ipVersion == ipv4Version { - h.metrics.RecordIPv4Update() - } else { - h.metrics.RecordIPv6Update() - } - } - - // Create live route record - liveRoute := &database.LiveRoute{ - ID: uuid.New(), - Prefix: update.prefix, - MaskLength: maskLength, - IPVersion: ipVersion, - OriginASN: update.originASN, - PeerIP: update.peer, - ASPath: update.path, - NextHop: update.peer, // Using peer as next hop - LastUpdated: update.timestamp, - } - - if err := h.db.UpsertLiveRoute(liveRoute); err != nil { - h.logger.Error("Failed to upsert live route", - "prefix", update.prefix, - "error", err, - ) - } -} - // createLiveRoute creates a LiveRoute from a prefix update func (h *PrefixHandler) createLiveRoute(update prefixUpdate) *database.LiveRoute { // Parse CIDR to get mask length @@ -462,77 +466,3 @@ func (h *PrefixHandler) createLiveRoute(update prefixUpdate) *database.LiveRoute return liveRoute } - -// processAnnouncementDirect handles storing an announcement directly without prefix table -// nolint:unused // kept for potential future use -func (h *PrefixHandler) processAnnouncementDirect(update prefixUpdate) { - // Parse CIDR to get mask length - maskLength, ipVersion, err := parseCIDR(update.prefix) - if err != nil { - h.logger.Error("Failed to parse CIDR", - "prefix", update.prefix, - "error", err, - ) - - return - } - - // Track route update metrics - if h.metrics != nil { - if ipVersion == ipv4Version { - h.metrics.RecordIPv4Update() - } else { - h.metrics.RecordIPv6Update() - } - } - - // Create live route record - liveRoute := &database.LiveRoute{ - ID: uuid.New(), - Prefix: update.prefix, - MaskLength: maskLength, - IPVersion: ipVersion, - OriginASN: update.originASN, - PeerIP: update.peer, - ASPath: update.path, - NextHop: update.peer, // Using peer as next hop - LastUpdated: update.timestamp, - } - - if err := h.db.UpsertLiveRoute(liveRoute); err != nil { - h.logger.Error("Failed to upsert live route", - "prefix", update.prefix, - "error", err, - ) - } -} - -// processWithdrawalDirect handles removing a route directly without prefix table -// nolint:unused // kept for potential future use -func (h *PrefixHandler) processWithdrawalDirect(update prefixUpdate) { - if err := h.db.DeleteLiveRoute(update.prefix, update.peer); err != nil { - h.logger.Error("Failed to delete live route", - "prefix", update.prefix, - "peer", update.peer, - "error", err, - ) - } -} - -// processWithdrawal handles removing a route from the live routing table -// nolint:unused // kept for potential future use -func (h *PrefixHandler) processWithdrawal(_ *database.Prefix, update prefixUpdate) { - if err := h.db.DeleteLiveRoute(update.prefix, update.peer); err != nil { - h.logger.Error("Failed to delete live route", - "prefix", update.prefix, - "peer", update.peer, - "error", err, - ) - } -} - -// Stop gracefully stops the handler and flushes remaining batches -func (h *PrefixHandler) Stop() { - close(h.stopCh) - h.wg.Wait() -} diff --git a/internal/routewatch/prefixhandler_test.go b/internal/routewatch/prefixhandler_internal_test.go similarity index 66% rename from internal/routewatch/prefixhandler_test.go rename to internal/routewatch/prefixhandler_internal_test.go index 2575a49..82c425f 100644 --- a/internal/routewatch/prefixhandler_test.go +++ b/internal/routewatch/prefixhandler_internal_test.go @@ -17,6 +17,12 @@ import ( const testPeerIP = "2001:db8:ffff::1" +// An IPv4 and an IPv6 prefix the tests announce. +const ( + testV4Prefix = "192.0.2.0/24" + testV6Prefix = "2001:db8:1::/48" +) + // TestPrefixHandlerStoresPrefixesTheIPLookupFinds runs announcements and // withdrawals through the prefix handler into a real database and looks the // addresses up. The prefixes are written the way the feed sends them: IPv6 @@ -24,11 +30,9 @@ const testPeerIP = "2001:db8:ffff::1" // remove the route the announcement stored, and the IP lookup must find the // stored prefix. func TestPrefixHandlerStoresPrefixesTheIPLookupFinds(t *testing.T) { - db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New()) - if err != nil { - t.Fatalf("failed to create database: %v", err) - } - defer func() { _ = db.Close() }() + t.Parallel() + + db := newTestDatabase(t) // Built without NewPrefixHandler's flush timer, so the test flushes itself. h := &PrefixHandler{db: db, logger: logger.New()} @@ -40,34 +44,38 @@ func TestPrefixHandlerStoresPrefixesTheIPLookupFinds(t *testing.T) { ParsedTimestamp: ts, Announcements: []ristypes.RISAnnouncement{{ NextHop: testPeerIP, - Prefixes: []string{"2001:db8:1::/48", "192.0.2.0/24"}, + Prefixes: []string{testV6Prefix, testV4Prefix}, }}, }) h.flushBatch() for ip, want := range map[string]string{ - "2001:db8:1::1": "2001:db8:1::/48", - "192.0.2.1": "192.0.2.0/24", + "2001:db8:1::1": testV6Prefix, + "192.0.2.1": testV4Prefix, } { info, err := db.GetIPInfo(ip) if err != nil { t.Fatalf("GetIPInfo(%s) after announcement: %v", ip, err) } + if info.Netblock != want || info.ASN != testASNB { - t.Errorf("GetIPInfo(%s) = %s AS%d, want %s AS%d", ip, info.Netblock, info.ASN, want, testASNB) + t.Errorf("GetIPInfo(%s) = %s AS%d, want %s AS%d", + ip, info.Netblock, info.ASN, want, testASNB) } } h.HandleMessage(&ristypes.RISMessage{ Peer: testPeerIP, ParsedTimestamp: ts.Add(time.Minute), - Withdrawals: []string{"2001:db8:1:0:0:0:0:0/48", "192.0.2.0/24"}, + Withdrawals: []string{"2001:db8:1:0:0:0:0:0/48", testV4Prefix}, }) h.flushBatch() for _, ip := range []string{"2001:db8:1::1", "192.0.2.1"} { - if info, err := db.GetIPInfo(ip); !errors.Is(err, database.ErrNoRoute) { - t.Errorf("GetIPInfo(%s) after withdrawal = %+v, %v; want ErrNoRoute", ip, info, err) + info, err := db.GetIPInfo(ip) + if !errors.Is(err, database.ErrNoRoute) { + t.Errorf("GetIPInfo(%s) after withdrawal = %+v, %v; want ErrNoRoute", + ip, info, err) } } } @@ -79,6 +87,8 @@ func TestPrefixHandlerStoresPrefixesTheIPLookupFinds(t *testing.T) { // live route for it, and updates from different peers are all applied. The // in-memory route count and prefix distribution must match the route table. func TestPrefixHandlerAppliesEachPeersLastUpdateInABatch(t *testing.T) { + t.Parallel() + const ( prefix = "198.51.100.0/24" prefixMaskLength = 24 @@ -91,11 +101,14 @@ func TestPrefixHandlerAppliesEachPeersLastUpdateInABatch(t *testing.T) { Peer: peer, Path: ristypes.ASPath{testASNA, testASNB}, ParsedTimestamp: ts, - Announcements: []ristypes.RISAnnouncement{{NextHop: peer, Prefixes: []string{prefix}}}, + Announcements: []ristypes.RISAnnouncement{ + {NextHop: peer, Prefixes: []string{prefix}}, + }, } } withdraw := func(peer string) *ristypes.RISMessage { - return &ristypes.RISMessage{Peer: peer, ParsedTimestamp: ts, Withdrawals: []string{prefix}} + return &ristypes.RISMessage{Peer: peer, ParsedTimestamp: ts, + Withdrawals: []string{prefix}} } announceAndWithdraw := func(peer string) *ristypes.RISMessage { msg := announce(peer) @@ -112,8 +125,10 @@ func TestPrefixHandlerAppliesEachPeersLastUpdateInABatch(t *testing.T) { wantPeers []string }{ { - name: "two peers announce the prefix", - batches: [][]*ristypes.RISMessage{{announce(testPeerIP), announce(otherPeerIP)}}, + name: "two peers announce the prefix", + batches: [][]*ristypes.RISMessage{ + {announce(testPeerIP), announce(otherPeerIP)}, + }, wantPeers: []string{testPeerIP, otherPeerIP}, }, { @@ -149,39 +164,10 @@ func TestPrefixHandlerAppliesEachPeersLastUpdateInABatch(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New()) - if err != nil { - t.Fatalf("failed to create database: %v", err) - } - defer func() { _ = db.Close() }() + t.Parallel() - // Built without NewPrefixHandler's flush timer, so the test flushes itself. - h := &PrefixHandler{db: db, logger: logger.New()} - for _, batch := range tt.batches { - for _, msg := range batch { - h.HandleMessage(msg) - } - h.flushBatch() - } - - ctx := context.Background() - routes, _, err := db.GetPrefixDetailsContext(ctx, prefix) - if err != nil && !errors.Is(err, database.ErrNoRoute) { - t.Fatalf("GetPrefixDetailsContext: %v", err) - } - var gotPeers []string - for _, route := range routes { - gotPeers = append(gotPeers, route.PeerIP) - } - if !slices.Equal(gotPeers, tt.wantPeers) { - t.Errorf("peers with a live route for %s = %v, want %v", prefix, gotPeers, tt.wantPeers) - } - - var wantDistribution []database.PrefixDistribution - if len(tt.wantPeers) > 0 { - wantDistribution = []database.PrefixDistribution{{MaskLength: prefixMaskLength, Count: 1}} - } - assertCounts(t, db, ipv4Version, len(tt.wantPeers), wantDistribution) + db := handleBatches(t, tt.batches) + checkLivePeers(t, db, prefix, prefixMaskLength, tt.wantPeers) }) } } @@ -194,14 +180,22 @@ func TestPrefixHandlerAppliesEachPeersLastUpdateInABatch(t *testing.T) { // in. The in-memory route count and prefix distribution must match the route // table. func TestPrefixHandlerKeepsOneRoutePerPrefixAndPeer(t *testing.T) { + t.Parallel() + families := []struct { ipVersion int prefix, otherPrefix string // The mask length of both prefixes. maskLength int }{ - {ipVersion: ipv4Version, prefix: "198.51.100.0/24", otherPrefix: "203.0.113.0/24", maskLength: 24}, - {ipVersion: ipv6Version, prefix: "2001:db8:1::/48", otherPrefix: "2001:db8:2::/48", maskLength: 48}, + { + ipVersion: ipv4Version, maskLength: 24, + prefix: "198.51.100.0/24", otherPrefix: "203.0.113.0/24", + }, + { + ipVersion: ipv6Version, maskLength: 48, + prefix: testV6Prefix, otherPrefix: "2001:db8:2::/48", + }, } ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) @@ -210,7 +204,9 @@ func TestPrefixHandlerKeepsOneRoutePerPrefixAndPeer(t *testing.T) { Peer: testPeerIP, Path: path, ParsedTimestamp: ts, - Announcements: []ristypes.RISAnnouncement{{NextHop: testPeerIP, Prefixes: []string{announced}}}, + Announcements: []ristypes.RISAnnouncement{ + {NextHop: testPeerIP, Prefixes: []string{announced}}, + }, } } @@ -219,71 +215,50 @@ func TestPrefixHandlerKeepsOneRoutePerPrefixAndPeer(t *testing.T) { // One message that announces otherPrefix with origin AS testASNB and // withdraws prefix. - announceOtherAndWithdraw := announce(ristypes.ASPath{testASNA, testASNB}, otherPrefix) + announceOtherAndWithdraw := announce( + ristypes.ASPath{testASNA, testASNB}, otherPrefix) announceOtherAndWithdraw.Withdrawals = []string{prefix} tests := []struct { - name string - messages []*ristypes.RISMessage + name string + // Each message is handled and then flushed as its own batch. + batches [][]*ristypes.RISMessage // The origin AS of the peer's live route for each prefix. A prefix // left out must have no live route. wantOrigins map[string]int }{ { name: "the peer announces the prefix with a new origin AS", - messages: []*ristypes.RISMessage{ - announce(ristypes.ASPath{testASNA}, prefix), - announce(ristypes.ASPath{testASNA, testASNB}, prefix), + batches: [][]*ristypes.RISMessage{ + {announce(ristypes.ASPath{testASNA}, prefix)}, + {announce(ristypes.ASPath{testASNA, testASNB}, prefix)}, }, wantOrigins: map[string]int{prefix: testASNB}, }, { - name: "one message announces a prefix with another origin AS and withdraws the prefix", - messages: []*ristypes.RISMessage{ - announce(ristypes.ASPath{testASNA}, prefix), - announceOtherAndWithdraw, + name: "one message announces a prefix with another origin AS " + + "and withdraws the prefix", + batches: [][]*ristypes.RISMessage{ + {announce(ristypes.ASPath{testASNA}, prefix)}, + {announceOtherAndWithdraw}, }, wantOrigins: map[string]int{otherPrefix: testASNB}, }, } for _, tt := range tests { - t.Run(fmt.Sprintf("IPv%d %s", family.ipVersion, tt.name), func(t *testing.T) { - db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New()) - if err != nil { - t.Fatalf("failed to create database: %v", err) - } - defer func() { _ = db.Close() }() + name := fmt.Sprintf("IPv%d %s", family.ipVersion, tt.name) + t.Run(name, func(t *testing.T) { + t.Parallel() - // Built without NewPrefixHandler's flush timer, so the test flushes itself. - h := &PrefixHandler{db: db, logger: logger.New()} - for _, msg := range tt.messages { - h.HandleMessage(msg) - h.flushBatch() - } - - for _, p := range []string{prefix, otherPrefix} { - routes, _, err := db.GetPrefixDetailsContext(context.Background(), p) - if err != nil && !errors.Is(err, database.ErrNoRoute) { - t.Fatalf("GetPrefixDetailsContext(%s): %v", p, err) - } - var gotOrigins []int - for _, route := range routes { - gotOrigins = append(gotOrigins, route.OriginASN) - } - var wantOrigins []int - if origin, ok := tt.wantOrigins[p]; ok { - wantOrigins = []int{origin} - } - if !slices.Equal(gotOrigins, wantOrigins) { - t.Errorf("origin ASes of the live routes for %s = %v, want %v", p, gotOrigins, wantOrigins) - } - } + db := handleBatches(t, tt.batches) + checkLiveOrigins(t, db, tt.wantOrigins, prefix, otherPrefix) wantDistribution := []database.PrefixDistribution{ {MaskLength: family.maskLength, Count: len(tt.wantOrigins)}, } - assertCounts(t, db, family.ipVersion, len(tt.wantOrigins), wantDistribution) + assertCounts(t, db, family.ipVersion, len(tt.wantOrigins), + wantDistribution) }) } } @@ -293,12 +268,16 @@ func TestPrefixHandlerKeepsOneRoutePerPrefixAndPeer(t *testing.T) { // messages go to the prefix handler, which acts on a down, and not to the peer // handler, which would record a peer whose session never comes up as active. func TestStateMessagesGoToThePrefixHandlerNotThePeerHandler(t *testing.T) { + t.Parallel() + if !(&PrefixHandler{}).WantsMessage(ristypes.MessageTypeState) { t.Error("the prefix handler does not want STATE messages") } + if (&PeerHandler{}).WantsMessage(ristypes.MessageTypeState) { t.Error("the peer handler wants STATE messages") } + if !(&PeerHandler{}).WantsMessage(ristypes.MessageTypeUpdate) { t.Error("the peer handler does not want UPDATE messages") } @@ -313,6 +292,8 @@ func TestStateMessagesGoToThePrefixHandlerNotThePeerHandler(t *testing.T) { // in the same batch, is kept. The in-memory route counts and prefix // distribution must match the route tables. func TestPrefixHandlerDeletesAPeersRoutesWhenItsSessionGoesDown(t *testing.T) { + t.Parallel() + const ( otherPeerIP = "2001:db8:ffff::2" // Both peers announce the shared prefixes, only testPeerIP the others. @@ -335,14 +316,17 @@ func TestPrefixHandlerDeletesAPeersRoutesWhenItsSessionGoesDown(t *testing.T) { wantAfterDown bool }{ { - name: "the down ends the batch", - batch: []*ristypes.RISMessage{peerAnnouncement(testPeerIP, beforeDown), sessionDown(testPeerIP)}, + name: "the down ends the batch", + batch: []*ristypes.RISMessage{ + peerAnnouncement(testPeerIP, beforeDown), + sessionDown(), + }, }, { name: "the peer announces a prefix after the down", batch: []*ristypes.RISMessage{ peerAnnouncement(testPeerIP, beforeDown), - sessionDown(testPeerIP), + sessionDown(), peerAnnouncement(testPeerIP, afterDown), }, wantAfterDown: true, @@ -351,21 +335,16 @@ func TestPrefixHandlerDeletesAPeersRoutesWhenItsSessionGoesDown(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New()) - if err != nil { - t.Fatalf("failed to create database: %v", err) - } - defer func() { _ = db.Close() }() + t.Parallel() - // Built without NewPrefixHandler's flush timer, so the test flushes itself. - h := &PrefixHandler{db: db, logger: logger.New()} - h.HandleMessage(peerAnnouncement(testPeerIP, sharedV4, sharedV6, onlyV4, onlyV6)) - h.HandleMessage(peerAnnouncement(otherPeerIP, sharedV4, sharedV6)) - h.flushBatch() - for _, msg := range tt.batch { - h.HandleMessage(msg) - } - h.flushBatch() + db := handleBatches(t, [][]*ristypes.RISMessage{ + { + peerAnnouncement(testPeerIP, + sharedV4, sharedV6, onlyV4, onlyV6), + peerAnnouncement(otherPeerIP, sharedV4, sharedV6), + }, + tt.batch, + }) // The peers left with a live route for each prefix. wantPeers := map[string][]string{ @@ -377,16 +356,20 @@ func TestPrefixHandlerDeletesAPeersRoutesWhenItsSessionGoesDown(t *testing.T) { afterDown: nil, } wantRoutesV6 := 1 + if tt.wantAfterDown { wantPeers[afterDown] = []string{testPeerIP} wantRoutesV6++ } assertPeers(t, db, wantPeers) - assertCounts(t, db, ipv4Version, 1, - []database.PrefixDistribution{{MaskLength: maskLengthV4, Count: 1}}) + assertCounts(t, db, ipv4Version, 1, []database.PrefixDistribution{ + {MaskLength: maskLengthV4, Count: 1}, + }) assertCounts(t, db, ipv6Version, wantRoutesV6, - []database.PrefixDistribution{{MaskLength: maskLengthV6, Count: wantRoutesV6}}) + []database.PrefixDistribution{ + {MaskLength: maskLengthV6, Count: wantRoutesV6}, + }) }) } } @@ -398,6 +381,8 @@ func TestPrefixHandlerDeletesAPeersRoutesWhenItsSessionGoesDown(t *testing.T) { // that handles it and on no later flush, so the route the peer announced after // it is kept. func TestPrefixHandlerKeepsWhatAPeerAnnouncesAfterTheFlushOfItsDown(t *testing.T) { + t.Parallel() + const ( otherPeerIP = "2001:db8:ffff::2" beforeDown = "192.0.2.0/24" @@ -406,30 +391,22 @@ func TestPrefixHandlerKeepsWhatAPeerAnnouncesAfterTheFlushOfItsDown(t *testing.T maskLength = 24 ) - db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New()) - if err != nil { - t.Fatalf("failed to create database: %v", err) - } - defer func() { _ = db.Close() }() - - // Built without NewPrefixHandler's flush timer, so the test flushes itself. - h := &PrefixHandler{db: db, logger: logger.New()} - for _, msg := range []*ristypes.RISMessage{ - peerAnnouncement(testPeerIP, beforeDown), - sessionDown(testPeerIP), - peerAnnouncement(testPeerIP, afterDown), - peerAnnouncement(otherPeerIP, otherPeers), - } { - h.HandleMessage(msg) - h.flushBatch() - } + // Each message is handled and then flushed as its own batch. + db := handleBatches(t, [][]*ristypes.RISMessage{ + {peerAnnouncement(testPeerIP, beforeDown)}, + {sessionDown()}, + {peerAnnouncement(testPeerIP, afterDown)}, + {peerAnnouncement(otherPeerIP, otherPeers)}, + }) assertPeers(t, db, map[string][]string{ beforeDown: nil, afterDown: {testPeerIP}, otherPeers: {otherPeerIP}, }) - assertCounts(t, db, ipv4Version, 2, []database.PrefixDistribution{{MaskLength: maskLength, Count: 2}}) + assertCounts(t, db, ipv4Version, 2, []database.PrefixDistribution{ + {MaskLength: maskLength, Count: 2}, + }) } // firstWriteHeldStore is a real database whose first live route batch write @@ -442,7 +419,11 @@ type firstWriteHeldStore struct { once sync.Once } -func (s *firstWriteHeldStore) UpsertLiveRouteBatch(routes []*database.LiveRoute) error { +// UpsertLiveRouteBatch holds the first call until release is closed, then +// writes routes to the database. +func (s *firstWriteHeldStore) UpsertLiveRouteBatch( + routes []*database.LiveRoute, +) error { s.once.Do(func() { close(s.writeStarted) <-s.release @@ -457,27 +438,32 @@ func (s *firstWriteHeldStore) UpsertLiveRouteBatch(routes []*database.LiveRoute) // must delete the route the held write stored and keep the one announced after // the down. func TestPrefixHandlerHandlesADownWhileAFlushWrites(t *testing.T) { + t.Parallel() + const ( beforeDown = "192.0.2.0/24" afterDown = "198.51.100.0/24" maskLength = 24 ) - db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New()) - if err != nil { - t.Fatalf("failed to create database: %v", err) + db := newTestDatabase(t) + store := &firstWriteHeldStore{ + Database: db, + writeStarted: make(chan struct{}), + release: make(chan struct{}), } - defer func() { _ = db.Close() }() - store := &firstWriteHeldStore{Database: db, writeStarted: make(chan struct{}), release: make(chan struct{})} // Built without NewPrefixHandler's flush timer, so the test flushes itself. h := &PrefixHandler{db: store, logger: logger.New()} h.HandleMessage(peerAnnouncement(testPeerIP, beforeDown)) + flushed := make(chan struct{}) + go func() { h.flushBatch() close(flushed) }() + select { case <-store.writeStarted: case <-time.After(handleMessageTimeout): @@ -485,21 +471,67 @@ func TestPrefixHandlerHandlesADownWhileAFlushWrites(t *testing.T) { } if !returnsInTime(func() { - h.HandleMessage(sessionDown(testPeerIP)) + h.HandleMessage(sessionDown()) h.HandleMessage(peerAnnouncement(testPeerIP, afterDown)) }) { t.Fatal("HandleMessage did not return while a database write was held") } + close(store.release) + select { case <-flushed: case <-time.After(handleMessageTimeout): t.Fatal("the flush did not finish after its write was let go") } + h.flushBatch() - assertPeers(t, db, map[string][]string{beforeDown: nil, afterDown: {testPeerIP}}) - assertCounts(t, db, ipv4Version, 1, []database.PrefixDistribution{{MaskLength: maskLength, Count: 1}}) + assertPeers(t, db, map[string][]string{ + beforeDown: nil, + afterDown: {testPeerIP}, + }) + assertCounts(t, db, ipv4Version, 1, []database.PrefixDistribution{ + {MaskLength: maskLength, Count: 1}, + }) +} + +// newTestDatabase opens a database in a new temporary directory. It is closed +// when the test ends. +func newTestDatabase(t *testing.T) *database.Database { + t.Helper() + + db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New()) + if err != nil { + t.Fatalf("failed to create database: %v", err) + } + + t.Cleanup(func() { _ = db.Close() }) + + return db +} + +// handleBatches runs each batch of messages through a prefix handler into a +// new database, flushing after each batch, and returns the database. +func handleBatches( + t *testing.T, batches [][]*ristypes.RISMessage, +) *database.Database { + t.Helper() + + db := newTestDatabase(t) + + // Built without NewPrefixHandler's flush timer, so the test flushes itself. + h := &PrefixHandler{db: db, logger: logger.New()} + + for _, batch := range batches { + for _, msg := range batch { + h.HandleMessage(msg) + } + + h.flushBatch() + } + + return db } // peerAnnouncement returns an UPDATE message in which peer announces prefixes @@ -510,18 +542,48 @@ func peerAnnouncement(peer string, prefixes ...string) *ristypes.RISMessage { Peer: peer, Path: ristypes.ASPath{testASNA, testASNB}, ParsedTimestamp: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC), - Announcements: []ristypes.RISAnnouncement{{NextHop: peer, Prefixes: prefixes}}, + Announcements: []ristypes.RISAnnouncement{ + {NextHop: peer, Prefixes: prefixes}, + }, } } -// sessionDown returns the STATE message that says peer's BGP session went down. -func sessionDown(peer string) *ristypes.RISMessage { - return &ristypes.RISMessage{Type: ristypes.MessageTypeState, Peer: peer, State: ristypes.StateDown} +// sessionDown returns the STATE message that says the BGP session of the peer +// testPeerIP went down. +func sessionDown() *ristypes.RISMessage { + return &ristypes.RISMessage{ + Type: ristypes.MessageTypeState, + Peer: testPeerIP, + State: ristypes.StateDown, + } +} + +// checkLivePeers checks that exactly wantPeers have a live route for the IPv4 +// prefix, and that the in-memory route count and prefix distribution match +// the route table. +func checkLivePeers( + t *testing.T, db *database.Database, prefix string, maskLength int, + wantPeers []string, +) { + t.Helper() + + assertPeers(t, db, map[string][]string{prefix: wantPeers}) + + var wantDistribution []database.PrefixDistribution + if len(wantPeers) > 0 { + wantDistribution = []database.PrefixDistribution{ + {MaskLength: maskLength, Count: 1}, + } + } + + assertCounts(t, db, ipv4Version, len(wantPeers), wantDistribution) } // assertPeers checks, for each prefix in wantPeers, that the peers with a live // route for it are the ones listed, in peer IP order; nil means none. -func assertPeers(t *testing.T, db *database.Database, wantPeers map[string][]string) { +func assertPeers( + t *testing.T, db *database.Database, wantPeers map[string][]string, +) { t.Helper() for prefix, want := range wantPeers { @@ -529,12 +591,46 @@ func assertPeers(t *testing.T, db *database.Database, wantPeers map[string][]str if err != nil && !errors.Is(err, database.ErrNoRoute) { t.Fatalf("GetPrefixDetailsContext(%s): %v", prefix, err) } - var got []string + + got := make([]string, 0, len(routes)) for _, route := range routes { got = append(got, route.PeerIP) } + if !slices.Equal(got, want) { - t.Errorf("peers with a live route for %s = %v, want %v", prefix, got, want) + t.Errorf("peers with a live route for %s = %v, want %v", + prefix, got, want) + } + } +} + +// checkLiveOrigins checks that each of prefixes has one live route, with the +// origin AS wantOrigins gives for it, or none when wantOrigins leaves it out. +func checkLiveOrigins( + t *testing.T, db *database.Database, wantOrigins map[string]int, + prefixes ...string, +) { + t.Helper() + + for _, prefix := range prefixes { + routes, _, err := db.GetPrefixDetailsContext(context.Background(), prefix) + if err != nil && !errors.Is(err, database.ErrNoRoute) { + t.Fatalf("GetPrefixDetailsContext(%s): %v", prefix, err) + } + + gotOrigins := make([]int, 0, len(routes)) + for _, route := range routes { + gotOrigins = append(gotOrigins, route.OriginASN) + } + + var want []int + if origin, ok := wantOrigins[prefix]; ok { + want = []int{origin} + } + + if !slices.Equal(gotOrigins, want) { + t.Errorf("origin ASes of the live routes for %s = %v, want %v", + prefix, gotOrigins, want) } } } @@ -542,38 +638,49 @@ func assertPeers(t *testing.T, db *database.Database, wantPeers map[string][]str // assertCounts checks that the live route count and prefix distribution of one // IP version are wantRoutes and wantDistribution, both as held in memory and as // read from the route table. -func assertCounts(t *testing.T, db *database.Database, ipVersion, wantRoutes int, - wantDistribution []database.PrefixDistribution) { +func assertCounts( + t *testing.T, db *database.Database, ipVersion, wantRoutes int, + wantDistribution []database.PrefixDistribution, +) { t.Helper() ctx := context.Background() + stats, err := db.GetStatsContext(ctx) if err != nil { t.Fatalf("GetStatsContext: %v", err) } + tableRoutesV4, tableRoutesV6, err := db.GetLiveRouteCountsContext(ctx) if err != nil { t.Fatalf("GetLiveRouteCountsContext: %v", err) } - queryDistributionV4, queryDistributionV6, err := db.GetPrefixDistributionContext(ctx) + + queryDistributionV4, queryDistributionV6, err := + db.GetPrefixDistributionContext(ctx) if err != nil { t.Fatalf("GetPrefixDistributionContext: %v", err) } memoryRoutes, tableRoutes := stats.IPv4Routes, tableRoutesV4 - memoryDistribution, queryDistribution := stats.IPv4PrefixDistribution, queryDistributionV4 + memoryDistribution := stats.IPv4PrefixDistribution + queryDistribution := queryDistributionV4 + if ipVersion == ipv6Version { memoryRoutes, tableRoutes = stats.IPv6Routes, tableRoutesV6 - memoryDistribution, queryDistribution = stats.IPv6PrefixDistribution, queryDistributionV6 + memoryDistribution = stats.IPv6PrefixDistribution + queryDistribution = queryDistributionV6 } if memoryRoutes != wantRoutes || tableRoutes != wantRoutes { t.Errorf("IPv%d live routes: %d in memory, %d in the table, want %d", ipVersion, memoryRoutes, tableRoutes, wantRoutes) } + if !slices.Equal(memoryDistribution, wantDistribution) || !slices.Equal(queryDistribution, wantDistribution) { - t.Errorf("IPv%d prefix distribution: %v in memory, %v from the table, want %v", - ipVersion, memoryDistribution, queryDistribution, wantDistribution) + t.Errorf("IPv%d prefix distribution: %v in memory, %v from the table, "+ + "want %v", ipVersion, memoryDistribution, queryDistribution, + wantDistribution) } } diff --git a/internal/server/handlers.go b/internal/server/handlers.go index 37c5c5b..2c436ae 100644 --- a/internal/server/handlers.go +++ b/internal/server/handlers.go @@ -15,6 +15,8 @@ import ( "time" "git.eeqj.de/sneak/routewatch/internal/database" + "git.eeqj.de/sneak/routewatch/internal/metrics" + "git.eeqj.de/sneak/routewatch/internal/streamer" "git.eeqj.de/sneak/routewatch/internal/templates" asinfo "git.eeqj.de/sneak/routewatch/pkg/asinfo" "github.com/dustin/go-humanize" @@ -29,6 +31,22 @@ const ( healthCheckTimeout = 2 * time.Second ) +// Keys of the JSON envelope the handlers and middleware write, and the status +// value of an error answer. +const ( + jsonKeyStatus = "status" + jsonKeyError = "error" + jsonKeyMsg = "msg" + jsonKeyCode = "code" + jsonStatusError = "error" +) + +// IP versions as database.LiveRoute.IPVersion records them. +const ( + ipVersionV4 = 4 + ipVersionV6 = 6 +) + // HealthCheckResponse represents the health check response. type HealthCheckResponse struct { Status string `json:"status"` @@ -68,7 +86,7 @@ func (s *Server) handleHealthCheck() http.HandlerFunc { // Build response status := "ok" if !healthy { - status = "error" + status = jsonStatusError } response := HealthCheckResponse{ @@ -81,7 +99,8 @@ func (s *Server) handleHealthCheck() http.HandlerFunc { w.WriteHeader(http.StatusServiceUnavailable) } - if err := writeJSONSuccess(w, response); err != nil { + err = writeJSONSuccess(w, response) + if err != nil { s.logger.Error("Failed to encode health check response", "error", err) } } @@ -93,7 +112,9 @@ func (s *Server) handleIndex() http.HandlerFunc { w.Header().Set("Content-Type", "text/html; charset=utf-8") tmpl := templates.IndexTemplate() - if err := tmpl.Execute(w, nil); err != nil { + + err := tmpl.Execute(w, nil) + if err != nil { s.logger.Error("Failed to render index template", "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) } @@ -102,30 +123,36 @@ func (s *Server) handleIndex() http.HandlerFunc { // writeJSONError writes a standardized JSON error response with the given // status code and error message. -func writeJSONError(w http.ResponseWriter, statusCode int, message string) { +func (s *Server) writeJSONError(w http.ResponseWriter, statusCode int, message string) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(statusCode) - _ = json.NewEncoder(w).Encode(map[string]interface{}{ - "status": "error", - "error": map[string]interface{}{ - "msg": message, - "code": statusCode, + + err := json.NewEncoder(w).Encode(map[string]any{ + jsonKeyStatus: jsonStatusError, + jsonKeyError: map[string]any{ + jsonKeyMsg: message, + jsonKeyCode: statusCode, }, }) + if err != nil { + s.logger.Error("Failed to encode error response", "error", err) + } } // writeJSONSuccess writes a standardized JSON success response containing // the provided data wrapped in a status envelope. -func writeJSONSuccess(w http.ResponseWriter, data interface{}) error { +func writeJSONSuccess(w http.ResponseWriter, data any) error { w.Header().Set("Content-Type", "application/json") - return json.NewEncoder(w).Encode(map[string]interface{}{ - "status": "ok", - "data": data, + return json.NewEncoder(w).Encode(map[string]any{ + jsonKeyStatus: "ok", + "data": data, }) } // WHOISStatsInfo contains WHOIS fetcher statistics for the status page. +// +//nolint:tagliatelle // field names are the existing JSON API type WHOISStatsInfo struct { TotalASNs int `json:"total_asns"` FreshASNs int `json:"fresh_asns"` @@ -138,41 +165,75 @@ type WHOISStatsInfo struct { FreshPercent float64 `json:"fresh_percent"` } +// Stats is the /status.json answer: uptime, message counts, database stats, +// and route information. +// +//nolint:tagliatelle // field names are the existing JSON API +type Stats struct { + Uptime string `json:"uptime"` + TotalMessages uint64 `json:"total_messages"` + TotalBytes uint64 `json:"total_bytes"` + TotalWireBytes uint64 `json:"total_wire_bytes"` + MessagesPerSec float64 `json:"messages_per_sec"` + MbitsPerSec float64 `json:"mbits_per_sec"` + WireMbitsPerSec float64 `json:"wire_mbits_per_sec"` + Connected bool `json:"connected"` + GoVersion string `json:"go_version"` + Goroutines int `json:"goroutines"` + MemoryUsage string `json:"memory_usage"` + ASNs int `json:"asns"` + Prefixes int `json:"prefixes"` + IPv4Prefixes int `json:"ipv4_prefixes"` + IPv6Prefixes int `json:"ipv6_prefixes"` + Peerings int `json:"peerings"` + Peers int `json:"peers"` + DatabaseSizeBytes int64 `json:"database_size_bytes"` + LiveRoutes int `json:"live_routes"` + IPv4Routes int `json:"ipv4_routes"` + IPv6Routes int `json:"ipv6_routes"` + OldestRoute *time.Time `json:"oldest_route,omitempty"` + NewestRoute *time.Time `json:"newest_route,omitempty"` + IPv4UpdatesPerSec float64 `json:"ipv4_updates_per_sec"` + IPv6UpdatesPerSec float64 `json:"ipv6_updates_per_sec"` + IPv4PrefixDistribution []database.PrefixDistribution `json:"ipv4_prefix_distribution"` + IPv6PrefixDistribution []database.PrefixDistribution `json:"ipv6_prefix_distribution"` + WHOISStats *WHOISStatsInfo `json:"whois_stats,omitempty"` +} + +// databaseStats gets the database stats, giving up when ctx is done; it then +// returns ctx's error. The channels are buffered so the goroutine's send never +// blocks if the timeout wins and nothing here receives; otherwise it would +// block forever and leak. +func (s *Server) databaseStats(ctx context.Context) (database.Stats, error) { + statsChan := make(chan database.Stats, 1) + errChan := make(chan error, 1) + + go func() { + dbStats, err := s.db.GetStatsContext(ctx) + if err != nil { + s.logger.Debug("Database stats query failed", "error", err) + + errChan <- err + + return + } + + statsChan <- dbStats + }() + + select { + case <-ctx.Done(): + return database.Stats{}, ctx.Err() + case err := <-errChan: + return database.Stats{}, err + case dbStats := <-statsChan: + return dbStats, nil + } +} + // handleStatusJSON returns a handler that serves JSON statistics including // uptime, message counts, database stats, and route information. func (s *Server) handleStatusJSON() http.HandlerFunc { - // Stats represents the statistics response - type Stats struct { - Uptime string `json:"uptime"` - TotalMessages uint64 `json:"total_messages"` - TotalBytes uint64 `json:"total_bytes"` - TotalWireBytes uint64 `json:"total_wire_bytes"` - MessagesPerSec float64 `json:"messages_per_sec"` - MbitsPerSec float64 `json:"mbits_per_sec"` - WireMbitsPerSec float64 `json:"wire_mbits_per_sec"` - Connected bool `json:"connected"` - GoVersion string `json:"go_version"` - Goroutines int `json:"goroutines"` - MemoryUsage string `json:"memory_usage"` - ASNs int `json:"asns"` - Prefixes int `json:"prefixes"` - IPv4Prefixes int `json:"ipv4_prefixes"` - IPv6Prefixes int `json:"ipv6_prefixes"` - Peerings int `json:"peerings"` - Peers int `json:"peers"` - DatabaseSizeBytes int64 `json:"database_size_bytes"` - LiveRoutes int `json:"live_routes"` - IPv4Routes int `json:"ipv4_routes"` - IPv6Routes int `json:"ipv6_routes"` - OldestRoute *time.Time `json:"oldest_route,omitempty"` - NewestRoute *time.Time `json:"newest_route,omitempty"` - IPv4UpdatesPerSec float64 `json:"ipv4_updates_per_sec"` - IPv6UpdatesPerSec float64 `json:"ipv6_updates_per_sec"` - IPv4PrefixDistribution []database.PrefixDistribution `json:"ipv4_prefix_distribution"` - IPv6PrefixDistribution []database.PrefixDistribution `json:"ipv6_prefix_distribution"` - WHOISStats *WHOISStatsInfo `json:"whois_stats,omitempty"` - } - return func(w http.ResponseWriter, r *http.Request) { // Create a 4 second timeout context for this request ctx, cancel := context.WithTimeout(r.Context(), statsContextTimeout) @@ -180,49 +241,21 @@ func (s *Server) handleStatusJSON() http.HandlerFunc { metrics := s.streamer.GetMetrics() - // Get database stats with timeout. The channels are buffered so the - // goroutine's send never blocks if the timeout wins and nothing here - // receives; otherwise it would block forever and leak. - statsChan := make(chan database.Stats, 1) - errChan := make(chan error, 1) - - go func() { - dbStats, err := s.db.GetStatsContext(ctx) - if err != nil { - s.logger.Debug("Database stats query failed", "error", err) - errChan <- err - - return - } - statsChan <- dbStats - }() - - var dbStats database.Stats - var err error - select { - case <-ctx.Done(): - err = ctx.Err() - case err = <-errChan: - case dbStats = <-statsChan: - } + dbStats, err := s.databaseStats(ctx) if deadlinePassed(ctx) { s.logger.Error("Database stats timeout in status.json") - writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) + s.writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) return } + if err != nil { s.logger.Error("Failed to get database stats", "error", err) - writeJSONError(w, http.StatusInternalServerError, err.Error()) + s.writeJSONError(w, http.StatusInternalServerError, err.Error()) return } - uptime := time.Since(metrics.ConnectedSince).Truncate(time.Second).String() - if metrics.ConnectedSince.IsZero() { - uptime = "0s" - } - const bitsPerMegabit = 1000000.0 // Get route update metrics @@ -237,14 +270,14 @@ func (s *Server) handleStatusJSON() http.HandlerFunc { if s.asnFetcher != nil { whoisStats = s.getWHOISStats(ctx) if deadlinePassed(ctx) { - writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) + s.writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) return } } stats := Stats{ - Uptime: uptime, + Uptime: streamUptime(metrics.ConnectedSince), TotalMessages: metrics.TotalMessages, TotalBytes: metrics.TotalBytes, TotalWireBytes: metrics.TotalWireBytes, @@ -274,7 +307,8 @@ func (s *Server) handleStatusJSON() http.HandlerFunc { WHOISStats: whoisStats, } - if err := writeJSONSuccess(w, stats); err != nil { + err = writeJSONSuccess(w, stats) + if err != nil { s.logger.Error("Failed to encode stats", "error", err) } } @@ -296,7 +330,8 @@ func (s *Server) getWHOISStats(ctx context.Context) *WHOISStatsInfo { // Calculate fresh percentage var freshPercent float64 if dbStats.TotalASNs > 0 { - freshPercent = float64(dbStats.FreshASNs) / float64(dbStats.TotalASNs) * percentMultiplier + freshPercent = float64(dbStats.FreshASNs) / float64(dbStats.TotalASNs) * + percentMultiplier } return &WHOISStatsInfo{ @@ -312,82 +347,174 @@ func (s *Server) getWHOISStats(ctx context.Context) *WHOISStatsInfo { } } +// streamUptime is how long the stream has been connected, to the second, or +// "0s" if it has never connected. +func streamUptime(connectedSince time.Time) string { + if connectedSince.IsZero() { + return "0s" + } + + return time.Since(connectedSince).Truncate(time.Second).String() +} + +// connectionDuration is how long the stream has been connected, to the +// second, or "disconnected". +func connectionDuration(connected bool, connectedSince time.Time) string { + if !connected || connectedSince.IsZero() { + return "disconnected" + } + + return time.Since(connectedSince).Truncate(time.Second).String() +} + // whoisStaleThreshold matches the fetcher's threshold for consistency. const whoisStaleThreshold = 30 * 24 * time.Hour // percentMultiplier converts a ratio to a percentage. const percentMultiplier = 100 +// HandlerStatsInfo represents handler statistics in the API response +// +//nolint:tagliatelle // field names are the existing JSON API +type HandlerStatsInfo struct { + Name string `json:"name"` + QueueLength int `json:"queue_length"` + QueueCapacity int `json:"queue_capacity"` + QueueHighWaterMark int `json:"queue_high_water_mark"` + ProcessedCount uint64 `json:"processed_count"` + DroppedCount uint64 `json:"dropped_count"` + AvgProcessTimeMs float64 `json:"avg_process_time_ms"` + MinProcessTimeMs float64 `json:"min_process_time_ms"` + MaxProcessTimeMs float64 `json:"max_process_time_ms"` +} + +// GCStats represents garbage collection statistics +// +//nolint:tagliatelle // field names are the existing JSON API +type GCStats struct { + NumGC uint32 `json:"num_gc"` + TotalPauseMs uint64 `json:"total_pause_ms"` + LastPauseMs float64 `json:"last_pause_ms"` + HeapAllocBytes uint64 `json:"heap_alloc_bytes"` + HeapSysBytes uint64 `json:"heap_sys_bytes"` +} + +// StreamStats represents stream statistics including announcements/withdrawals +// +//nolint:tagliatelle // field names are the existing JSON API +type StreamStats struct { + Announcements uint64 `json:"announcements"` + Withdrawals uint64 `json:"withdrawals"` + RouteChurnPerSec float64 `json:"route_churn_per_sec"` + BGPPeerCount int `json:"bgp_peer_count"` +} + +// StatsResponse represents the API statistics response +// +//nolint:tagliatelle // field names are the existing JSON API +type StatsResponse struct { + Uptime string `json:"uptime"` + TotalMessages uint64 `json:"total_messages"` + TotalBytes uint64 `json:"total_bytes"` + TotalWireBytes uint64 `json:"total_wire_bytes"` + MessagesPerSec float64 `json:"messages_per_sec"` + MbitsPerSec float64 `json:"mbits_per_sec"` + WireMbitsPerSec float64 `json:"wire_mbits_per_sec"` + Connected bool `json:"connected"` + ConnectionDuration string `json:"connection_duration"` + ReconnectCount uint64 `json:"reconnect_count"` + GoVersion string `json:"go_version"` + Goroutines int `json:"goroutines"` + MemoryUsage string `json:"memory_usage"` + GC GCStats `json:"gc"` + Stream StreamStats `json:"stream"` + ASNs int `json:"asns"` + Prefixes int `json:"prefixes"` + IPv4Prefixes int `json:"ipv4_prefixes"` + IPv6Prefixes int `json:"ipv6_prefixes"` + Peerings int `json:"peerings"` + Peers int `json:"peers"` + DatabaseSizeBytes int64 `json:"database_size_bytes"` + LiveRoutes int `json:"live_routes"` + IPv4Routes int `json:"ipv4_routes"` + IPv6Routes int `json:"ipv6_routes"` + OldestRoute *time.Time `json:"oldest_route,omitempty"` + NewestRoute *time.Time `json:"newest_route,omitempty"` + IPv4UpdatesPerSec float64 `json:"ipv4_updates_per_sec"` + IPv6UpdatesPerSec float64 `json:"ipv6_updates_per_sec"` + HandlerStats []HandlerStatsInfo `json:"handler_stats"` + IPv4PrefixDistribution []database.PrefixDistribution `json:"ipv4_prefix_distribution"` + IPv6PrefixDistribution []database.PrefixDistribution `json:"ipv6_prefix_distribution"` + WHOISStats *WHOISStatsInfo `json:"whois_stats,omitempty"` +} + +// handlerStatsInfo converts the streamer's handler statistics for the API +// response. +func handlerStatsInfo(handlerStats []streamer.HandlerStats) []HandlerStatsInfo { + const microsecondsPerMillisecond = 1000.0 + + handlerStatsInfo := make([]HandlerStatsInfo, 0, len(handlerStats)) + for _, hs := range handlerStats { + handlerStatsInfo = append(handlerStatsInfo, HandlerStatsInfo{ + Name: hs.Name, + QueueLength: hs.QueueLength, + QueueCapacity: hs.QueueCapacity, + QueueHighWaterMark: hs.QueueHighWaterMark, + ProcessedCount: hs.ProcessedCount, + DroppedCount: hs.DroppedCount, + AvgProcessTimeMs: float64(hs.AvgProcessTime.Microseconds()) / + microsecondsPerMillisecond, + MinProcessTimeMs: float64(hs.MinProcessTime.Microseconds()) / + microsecondsPerMillisecond, + MaxProcessTimeMs: float64(hs.MaxProcessTime.Microseconds()) / + microsecondsPerMillisecond, + }) + } + + return handlerStatsInfo +} + +// gcStats reports the garbage collector's figures from memStats. +func gcStats(memStats *runtime.MemStats) GCStats { + const ( + nanosecondsPerMillisecond = 1e6 + gcPauseHistorySize = 256 // Size of runtime.MemStats.PauseNs circular buffer + ) + + // Calculate last GC pause + var lastPauseMs float64 + + if memStats.NumGC > 0 { + // PauseNs is a circular buffer, get the most recent pause + lastPauseIdx := (memStats.NumGC + gcPauseHistorySize - 1) % gcPauseHistorySize + lastPauseMs = float64(memStats.PauseNs[lastPauseIdx]) / nanosecondsPerMillisecond + } + + return GCStats{ + NumGC: memStats.NumGC, + TotalPauseMs: memStats.PauseTotalNs / uint64(nanosecondsPerMillisecond), + LastPauseMs: lastPauseMs, + HeapAllocBytes: memStats.HeapAlloc, + HeapSysBytes: memStats.HeapSys, + } +} + +// streamStats reports the announcement, withdrawal and BGP peer figures of the +// streamer's metrics tracker. +func (s *Server) streamStats() StreamStats { + metricsTracker := s.streamer.GetMetricsTracker() + + return StreamStats{ + Announcements: metricsTracker.GetAnnouncementCount(), + Withdrawals: metricsTracker.GetWithdrawalCount(), + RouteChurnPerSec: metricsTracker.GetChurnRate(), + BGPPeerCount: metricsTracker.GetBGPPeerCount(), + } +} + // handleStats returns a handler that serves API v1 statistics including // detailed handler queue statistics and performance metrics. func (s *Server) handleStats() http.HandlerFunc { - // HandlerStatsInfo represents handler statistics in the API response - type HandlerStatsInfo struct { - Name string `json:"name"` - QueueLength int `json:"queue_length"` - QueueCapacity int `json:"queue_capacity"` - QueueHighWaterMark int `json:"queue_high_water_mark"` - ProcessedCount uint64 `json:"processed_count"` - DroppedCount uint64 `json:"dropped_count"` - AvgProcessTimeMs float64 `json:"avg_process_time_ms"` - MinProcessTimeMs float64 `json:"min_process_time_ms"` - MaxProcessTimeMs float64 `json:"max_process_time_ms"` - } - - // GCStats represents garbage collection statistics - type GCStats struct { - NumGC uint32 `json:"num_gc"` - TotalPauseMs uint64 `json:"total_pause_ms"` - LastPauseMs float64 `json:"last_pause_ms"` - HeapAllocBytes uint64 `json:"heap_alloc_bytes"` - HeapSysBytes uint64 `json:"heap_sys_bytes"` - } - - // StreamStats represents stream statistics including announcements/withdrawals - type StreamStats struct { - Announcements uint64 `json:"announcements"` - Withdrawals uint64 `json:"withdrawals"` - RouteChurnPerSec float64 `json:"route_churn_per_sec"` - BGPPeerCount int `json:"bgp_peer_count"` - } - - // StatsResponse represents the API statistics response - type StatsResponse struct { - Uptime string `json:"uptime"` - TotalMessages uint64 `json:"total_messages"` - TotalBytes uint64 `json:"total_bytes"` - TotalWireBytes uint64 `json:"total_wire_bytes"` - MessagesPerSec float64 `json:"messages_per_sec"` - MbitsPerSec float64 `json:"mbits_per_sec"` - WireMbitsPerSec float64 `json:"wire_mbits_per_sec"` - Connected bool `json:"connected"` - ConnectionDuration string `json:"connection_duration"` - ReconnectCount uint64 `json:"reconnect_count"` - GoVersion string `json:"go_version"` - Goroutines int `json:"goroutines"` - MemoryUsage string `json:"memory_usage"` - GC GCStats `json:"gc"` - Stream StreamStats `json:"stream"` - ASNs int `json:"asns"` - Prefixes int `json:"prefixes"` - IPv4Prefixes int `json:"ipv4_prefixes"` - IPv6Prefixes int `json:"ipv6_prefixes"` - Peerings int `json:"peerings"` - Peers int `json:"peers"` - DatabaseSizeBytes int64 `json:"database_size_bytes"` - LiveRoutes int `json:"live_routes"` - IPv4Routes int `json:"ipv4_routes"` - IPv6Routes int `json:"ipv6_routes"` - OldestRoute *time.Time `json:"oldest_route,omitempty"` - NewestRoute *time.Time `json:"newest_route,omitempty"` - IPv4UpdatesPerSec float64 `json:"ipv4_updates_per_sec"` - IPv6UpdatesPerSec float64 `json:"ipv6_updates_per_sec"` - HandlerStats []HandlerStatsInfo `json:"handler_stats"` - IPv4PrefixDistribution []database.PrefixDistribution `json:"ipv4_prefix_distribution"` - IPv6PrefixDistribution []database.PrefixDistribution `json:"ipv6_prefix_distribution"` - WHOISStats *WHOISStatsInfo `json:"whois_stats,omitempty"` - } - return func(w http.ResponseWriter, r *http.Request) { // Create a 4 second timeout context for this request ctx, cancel := context.WithTimeout(r.Context(), statsContextTimeout) @@ -396,7 +523,7 @@ func (s *Server) handleStats() http.HandlerFunc { // Check if context is already cancelled select { case <-ctx.Done(): - writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) + s.writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) return default: @@ -404,165 +531,94 @@ func (s *Server) handleStats() http.HandlerFunc { metrics := s.streamer.GetMetrics() - // Get database stats with timeout. The channels are buffered so the - // goroutine's send never blocks if the timeout wins and nothing here - // receives; otherwise it would block forever and leak. - statsChan := make(chan database.Stats, 1) - errChan := make(chan error, 1) - - go func() { - dbStats, err := s.db.GetStatsContext(ctx) - if err != nil { - s.logger.Debug("Database stats query failed", "error", err) - errChan <- err - - return - } - statsChan <- dbStats - }() - - var dbStats database.Stats - var err error - select { - case <-ctx.Done(): - err = ctx.Err() - case err = <-errChan: - case dbStats = <-statsChan: - } + dbStats, err := s.databaseStats(ctx) if deadlinePassed(ctx) { s.logger.Error("Database stats timeout") - writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) + s.writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) return } + if err != nil { s.logger.Error("Failed to get database stats", "error", err) - writeJSONError(w, http.StatusInternalServerError, err.Error()) + s.writeJSONError(w, http.StatusInternalServerError, err.Error()) return } - uptime := time.Since(metrics.ConnectedSince).Truncate(time.Second).String() - if metrics.ConnectedSince.IsZero() { - uptime = "0s" - } - - const bitsPerMegabit = 1000000.0 - - // Get route update metrics - routeMetrics := s.streamer.GetMetricsTracker().GetRouteMetrics() - - // Get handler stats - handlerStats := s.streamer.GetHandlerStats() - handlerStatsInfo := make([]HandlerStatsInfo, 0, len(handlerStats)) - const microsecondsPerMillisecond = 1000.0 - for _, hs := range handlerStats { - handlerStatsInfo = append(handlerStatsInfo, HandlerStatsInfo{ - Name: hs.Name, - QueueLength: hs.QueueLength, - QueueCapacity: hs.QueueCapacity, - QueueHighWaterMark: hs.QueueHighWaterMark, - ProcessedCount: hs.ProcessedCount, - DroppedCount: hs.DroppedCount, - AvgProcessTimeMs: float64(hs.AvgProcessTime.Microseconds()) / microsecondsPerMillisecond, - MinProcessTimeMs: float64(hs.MinProcessTime.Microseconds()) / microsecondsPerMillisecond, - MaxProcessTimeMs: float64(hs.MaxProcessTime.Microseconds()) / microsecondsPerMillisecond, - }) - } - - // Get memory stats - var memStats runtime.MemStats - runtime.ReadMemStats(&memStats) - // Get WHOIS stats if fetcher is available var whoisStats *WHOISStatsInfo if s.asnFetcher != nil { whoisStats = s.getWHOISStats(ctx) if deadlinePassed(ctx) { - writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) + s.writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) return } } - // Calculate connection duration - connectionDuration := "disconnected" - if metrics.Connected && !metrics.ConnectedSince.IsZero() { - connectionDuration = time.Since(metrics.ConnectedSince).Truncate(time.Second).String() - } - - // Get announcement/withdrawal stats from metrics tracker - metricsTracker := s.streamer.GetMetricsTracker() - announcements := metricsTracker.GetAnnouncementCount() - withdrawals := metricsTracker.GetWithdrawalCount() - churnRate := metricsTracker.GetChurnRate() - bgpPeerCount := metricsTracker.GetBGPPeerCount() - - // Calculate last GC pause - const ( - nanosecondsPerMillisecond = 1e6 - gcPauseHistorySize = 256 // Size of runtime.MemStats.PauseNs circular buffer - ) - var lastPauseMs float64 - if memStats.NumGC > 0 { - // PauseNs is a circular buffer, get the most recent pause - lastPauseIdx := (memStats.NumGC + gcPauseHistorySize - 1) % gcPauseHistorySize - lastPauseMs = float64(memStats.PauseNs[lastPauseIdx]) / nanosecondsPerMillisecond - } - - stats := StatsResponse{ - Uptime: uptime, - TotalMessages: metrics.TotalMessages, - TotalBytes: metrics.TotalBytes, - TotalWireBytes: metrics.TotalWireBytes, - MessagesPerSec: metrics.MessagesPerSec, - MbitsPerSec: metrics.BitsPerSec / bitsPerMegabit, - WireMbitsPerSec: metrics.WireBitsPerSec / bitsPerMegabit, - Connected: metrics.Connected, - ConnectionDuration: connectionDuration, - ReconnectCount: metrics.ReconnectCount, - GoVersion: runtime.Version(), - Goroutines: runtime.NumGoroutine(), - MemoryUsage: humanize.Bytes(memStats.Alloc), - GC: GCStats{ - NumGC: memStats.NumGC, - TotalPauseMs: memStats.PauseTotalNs / uint64(nanosecondsPerMillisecond), - LastPauseMs: lastPauseMs, - HeapAllocBytes: memStats.HeapAlloc, - HeapSysBytes: memStats.HeapSys, - }, - Stream: StreamStats{ - Announcements: announcements, - Withdrawals: withdrawals, - RouteChurnPerSec: churnRate, - BGPPeerCount: bgpPeerCount, - }, - ASNs: dbStats.ASNs, - Prefixes: dbStats.Prefixes, - IPv4Prefixes: dbStats.IPv4Prefixes, - IPv6Prefixes: dbStats.IPv6Prefixes, - Peerings: dbStats.Peerings, - Peers: dbStats.Peers, - DatabaseSizeBytes: dbStats.FileSizeBytes, - LiveRoutes: dbStats.LiveRoutes, - IPv4Routes: dbStats.IPv4Routes, - IPv6Routes: dbStats.IPv6Routes, - OldestRoute: dbStats.OldestRoute, - NewestRoute: dbStats.NewestRoute, - IPv4UpdatesPerSec: routeMetrics.IPv4UpdatesPerSec, - IPv6UpdatesPerSec: routeMetrics.IPv6UpdatesPerSec, - HandlerStats: handlerStatsInfo, - IPv4PrefixDistribution: dbStats.IPv4PrefixDistribution, - IPv6PrefixDistribution: dbStats.IPv6PrefixDistribution, - WHOISStats: whoisStats, - } - - if err := writeJSONSuccess(w, stats); err != nil { + err = writeJSONSuccess(w, s.statsResponse(metrics, dbStats, whoisStats)) + if err != nil { s.logger.Error("Failed to encode stats", "error", err) } } } +// statsResponse builds the /api/v1/stats answer from the stream metrics, the +// database stats and the WHOIS stats, with the route update rates, memory use +// and handler queues as they are now. +func (s *Server) statsResponse( + streamMetrics metrics.StreamMetrics, + dbStats database.Stats, + whoisStats *WHOISStatsInfo, +) StatsResponse { + const bitsPerMegabit = 1000000.0 + + // Get route update metrics + routeMetrics := s.streamer.GetMetricsTracker().GetRouteMetrics() + + // Get memory stats + var memStats runtime.MemStats + runtime.ReadMemStats(&memStats) + + return StatsResponse{ + Uptime: streamUptime(streamMetrics.ConnectedSince), + TotalMessages: streamMetrics.TotalMessages, + TotalBytes: streamMetrics.TotalBytes, + TotalWireBytes: streamMetrics.TotalWireBytes, + MessagesPerSec: streamMetrics.MessagesPerSec, + MbitsPerSec: streamMetrics.BitsPerSec / bitsPerMegabit, + WireMbitsPerSec: streamMetrics.WireBitsPerSec / bitsPerMegabit, + Connected: streamMetrics.Connected, + ConnectionDuration: connectionDuration( + streamMetrics.Connected, streamMetrics.ConnectedSince), + ReconnectCount: streamMetrics.ReconnectCount, + GoVersion: runtime.Version(), + Goroutines: runtime.NumGoroutine(), + MemoryUsage: humanize.Bytes(memStats.Alloc), + GC: gcStats(&memStats), + Stream: s.streamStats(), + ASNs: dbStats.ASNs, + Prefixes: dbStats.Prefixes, + IPv4Prefixes: dbStats.IPv4Prefixes, + IPv6Prefixes: dbStats.IPv6Prefixes, + Peerings: dbStats.Peerings, + Peers: dbStats.Peers, + DatabaseSizeBytes: dbStats.FileSizeBytes, + LiveRoutes: dbStats.LiveRoutes, + IPv4Routes: dbStats.IPv4Routes, + IPv6Routes: dbStats.IPv6Routes, + OldestRoute: dbStats.OldestRoute, + NewestRoute: dbStats.NewestRoute, + IPv4UpdatesPerSec: routeMetrics.IPv4UpdatesPerSec, + IPv6UpdatesPerSec: routeMetrics.IPv6UpdatesPerSec, + HandlerStats: handlerStatsInfo(s.streamer.GetHandlerStats()), + IPv4PrefixDistribution: dbStats.IPv4PrefixDistribution, + IPv6PrefixDistribution: dbStats.IPv6PrefixDistribution, + WHOISStats: whoisStats, + } +} + // handleStatusHTML returns a handler that serves the HTML status page, // which displays real-time statistics fetched via JavaScript. func (s *Server) handleStatusHTML() http.HandlerFunc { @@ -570,7 +626,9 @@ func (s *Server) handleStatusHTML() http.HandlerFunc { w.Header().Set("Content-Type", "text/html; charset=utf-8") tmpl := templates.StatusTemplate() - if err := tmpl.Execute(w, nil); err != nil { + + err := tmpl.Execute(w, nil) + if err != nil { s.logger.Error("Failed to render template", "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) } @@ -595,18 +653,9 @@ type IPLookupResponse struct { // Always returns the same response structure with PTR records for each IP. func (s *Server) handleIPInfo() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - // Get IP/hostname from URL param, falling back to client IP - target := chi.URLParam(r, "ip") + target := lookupTarget(r) if target == "" { - target = chi.URLParam(r, "addr") - } - if target == "" { - // Use client IP (RealIP middleware has already processed this) - target = extractClientIP(r) - } - - if target == "" { - writeJSONError(w, http.StatusBadRequest, "Could not determine IP address") + s.writeJSONError(w, http.StatusBadRequest, "Could not determine IP address") return } @@ -618,38 +667,17 @@ func (s *Server) handleIPInfo() http.HandlerFunc { } // Collect all IPs to look up - var ipsToLookup []string + ipsToLookup, resolveErrors := resolveLookupTargets(ctx, target) + if deadlinePassed(ctx) { + s.writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) - // Check if target contains commas (multiple IPs) - targets := strings.Split(target, ",") - for _, t := range targets { - t = strings.TrimSpace(t) - if t == "" { - continue - } - - // Check if this target is an IP address - if parsedIP := net.ParseIP(t); parsedIP != nil { - ipsToLookup = append(ipsToLookup, t) - } else { - // It's a hostname - resolve it - resolved, err := net.DefaultResolver.LookupHost(ctx, t) - if deadlinePassed(ctx) { - writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) - - return - } - if err != nil { - response.Errors = append(response.Errors, t+": "+err.Error()) - - continue - } - ipsToLookup = append(ipsToLookup, resolved...) - } + return } + response.Errors = append(response.Errors, resolveErrors...) + if len(ipsToLookup) == 0 { - writeJSONError(w, http.StatusBadRequest, "No valid IPs or hostnames provided") + s.writeJSONError(w, http.StatusBadRequest, "No valid IPs or hostnames provided") return } @@ -661,10 +689,11 @@ func (s *Server) handleIPInfo() http.HandlerFunc { for _, ip := range ipsToLookup { ipInfo, err := s.db.GetIPInfoContext(ctx, ip) if deadlinePassed(ctx) { - writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) + s.writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) return } + if err != nil { response.Errors = append(response.Errors, ip+": "+err.Error()) @@ -672,17 +701,14 @@ func (s *Server) handleIPInfo() http.HandlerFunc { } // Do PTR lookup for this IP - ptrs, err := net.DefaultResolver.LookupAddr(ctx, ip) + ptrs := lookupPTRNames(ctx, ip) if deadlinePassed(ctx) { - writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) + s.writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) return } - if err == nil && len(ptrs) > 0 { - // Remove trailing dots from PTR records - for i, ptr := range ptrs { - ptrs[i] = strings.TrimSuffix(ptr, ".") - } + + if len(ptrs) > 0 { ipInfo.PTR = ptrs } @@ -694,25 +720,104 @@ func (s *Server) handleIPInfo() http.HandlerFunc { } // Queue WHOIS refresh for stale ASNs (non-blocking) - if s.asnFetcher != nil { - for asn := range refreshASNs { - s.asnFetcher.QueueImmediate(asn) - } - } + s.queueWHOISRefresh(refreshASNs) // Return response (even if no results, include errors) if len(response.Results) == 0 && len(response.Errors) > 0 { - writeJSONError(w, http.StatusNotFound, "No routes found: "+response.Errors[0]) + s.writeJSONError(w, http.StatusNotFound, "No routes found: "+response.Errors[0]) return } - if err := writeJSONSuccess(w, response); err != nil { + err := writeJSONSuccess(w, response) + if err != nil { s.logger.Error("Failed to encode IP lookup response", "error", err) } } } +// lookupTarget returns what an IP lookup request asks for: the ip or addr URL +// parameter, or else the client's own address. +func lookupTarget(r *http.Request) string { + target := chi.URLParam(r, "ip") + if target == "" { + target = chi.URLParam(r, "addr") + } + + if target == "" { + // Use client IP (RealIP middleware has already processed this) + target = extractClientIP(r) + } + + return target +} + +// queueWHOISRefresh asks the WHOIS fetcher, when there is one, to refresh +// each of asns now. +func (s *Server) queueWHOISRefresh(asns map[int]bool) { + if s.asnFetcher == nil { + return + } + + for asn := range asns { + s.asnFetcher.QueueImmediate(asn) + } +} + +// resolveLookupTargets splits a comma-separated list of IP addresses and +// hostnames and returns the IP addresses to look up, resolving each hostname. +// A hostname that does not resolve is reported in the returned errors instead. +// It stops once ctx's deadline has passed; the caller then answers 504. +func resolveLookupTargets(ctx context.Context, target string) ([]string, []string) { + var ipsToLookup, resolveErrors []string + + for t := range strings.SplitSeq(target, ",") { + t = strings.TrimSpace(t) + if t == "" { + continue + } + + // Check if this target is an IP address + if net.ParseIP(t) != nil { + ipsToLookup = append(ipsToLookup, t) + + continue + } + + // It's a hostname - resolve it + resolved, err := net.DefaultResolver.LookupHost(ctx, t) + if deadlinePassed(ctx) { + return nil, nil + } + + if err != nil { + resolveErrors = append(resolveErrors, t+": "+err.Error()) + + continue + } + + ipsToLookup = append(ipsToLookup, resolved...) + } + + return ipsToLookup, resolveErrors +} + +// lookupPTRNames returns the PTR records of ip without their trailing dots, or +// nil when the lookup fails. +func lookupPTRNames(ctx context.Context, ip string) []string { + ptrs, err := net.DefaultResolver.LookupAddr(ctx, ip) + if err != nil { + return nil + } + + // Remove trailing dots from PTR records + for i, ptr := range ptrs { + ptrs[i] = strings.TrimSuffix(ptr, ".") + } + + return ptrs +} + // extractClientIP extracts the client IP from the request. // Works with chi's RealIP middleware which sets RemoteAddr. func extractClientIP(r *http.Request) string { @@ -730,53 +835,86 @@ func extractClientIP(r *http.Request) string { func (s *Server) handleASDetailJSON() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { asnStr := chi.URLParam(r, "asn") + asn, err := strconv.Atoi(asnStr) if err != nil { - writeJSONError(w, http.StatusBadRequest, "Invalid ASN") + s.writeJSONError(w, http.StatusBadRequest, "Invalid ASN") return } asInfo, prefixes, err := s.db.GetASDetailsContext(r.Context(), asn) if deadlinePassed(r.Context()) { - writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) + s.writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) return } + if err != nil { if errors.Is(err, database.ErrNoRoute) { - writeJSONError(w, http.StatusNotFound, err.Error()) + s.writeJSONError(w, http.StatusNotFound, err.Error()) } else { - writeJSONError(w, http.StatusInternalServerError, err.Error()) + s.writeJSONError(w, http.StatusInternalServerError, err.Error()) } return } // Group prefixes by IP version - const ipVersionV4 = 4 - var ipv4Prefixes, ipv6Prefixes []database.LiveRoute - for _, p := range prefixes { - if p.IPVersion == ipVersionV4 { - ipv4Prefixes = append(ipv4Prefixes, p) - } else { - ipv6Prefixes = append(ipv6Prefixes, p) - } - } + ipv4Prefixes, ipv6Prefixes := splitRoutesByIPVersion(prefixes) - response := map[string]interface{}{ + response := map[string]any{ "asn": asInfo, "ipv4_prefixes": ipv4Prefixes, "ipv6_prefixes": ipv6Prefixes, "total_count": len(prefixes), } - if err := writeJSONSuccess(w, response); err != nil { + err = writeJSONSuccess(w, response) + if err != nil { s.logger.Error("Failed to encode AS details", "error", err) } } } +// splitRoutesByIPVersion groups routes into the IPv4 and the IPv6 ones. +func splitRoutesByIPVersion( + routes []database.LiveRoute, +) ([]database.LiveRoute, []database.LiveRoute) { + var ipv4Routes, ipv6Routes []database.LiveRoute + + for _, route := range routes { + if route.IPVersion == ipVersionV4 { + ipv4Routes = append(ipv4Routes, route) + } else { + ipv6Routes = append(ipv6Routes, route) + } + } + + return ipv4Routes, ipv6Routes +} + +// sortRoutesByNetwork sorts routes by network address, then by mask length. +func sortRoutesByNetwork(routes []database.LiveRoute) { + sort.Slice(routes, func(i, j int) bool { + // Parse the prefixes to compare network addresses + ipI, netI, _ := net.ParseCIDR(routes[i].Prefix) + ipJ, netJ, _ := net.ParseCIDR(routes[j].Prefix) + + // Compare by network address first + cmp := bytes.Compare(ipI.To16(), ipJ.To16()) + if cmp != 0 { + return cmp < 0 + } + + // If network addresses are equal, compare by mask length + onesI, _ := netI.Mask.Size() + onesJ, _ := netJ.Mask.Size() + + return onesI < onesJ + }) +} + // handlePrefixDetailJSON returns prefix details as JSON func (s *Server) handlePrefixDetailJSON() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { @@ -785,7 +923,8 @@ func (s *Server) handlePrefixDetailJSON() http.HandlerFunc { lenParam := chi.URLParam(r, "len") if prefixParam == "" || lenParam == "" { - writeJSONError(w, http.StatusBadRequest, "Prefix and length parameters are required") + s.writeJSONError(w, http.StatusBadRequest, + "Prefix and length parameters are required") return } @@ -794,23 +933,25 @@ func (s *Server) handlePrefixDetailJSON() http.HandlerFunc { // tables store, so that any valid way of writing it finds the route. parsed, err := netip.ParsePrefix(prefixParam + "/" + lenParam) if err != nil { - writeJSONError(w, http.StatusBadRequest, err.Error()) + s.writeJSONError(w, http.StatusBadRequest, err.Error()) return } + prefix := parsed.Masked().String() routes, _, err := s.db.GetPrefixDetailsContext(r.Context(), prefix) if deadlinePassed(r.Context()) { - writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) + s.writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) return } + if err != nil { if errors.Is(err, database.ErrNoRoute) { - writeJSONError(w, http.StatusNotFound, err.Error()) + s.writeJSONError(w, http.StatusNotFound, err.Error()) } else { - writeJSONError(w, http.StatusInternalServerError, err.Error()) + s.writeJSONError(w, http.StatusInternalServerError, err.Error()) } return @@ -822,7 +963,7 @@ func (s *Server) handlePrefixDetailJSON() http.HandlerFunc { originMap[route.OriginASN] = append(originMap[route.OriginASN], route) } - response := map[string]interface{}{ + response := map[string]any{ "prefix": prefix, "routes": routes, "origins": originMap, @@ -830,7 +971,8 @@ func (s *Server) handlePrefixDetailJSON() http.HandlerFunc { "origin_count": len(originMap), } - if err := writeJSONSuccess(w, response); err != nil { + err = writeJSONSuccess(w, response) + if err != nil { s.logger.Error("Failed to encode prefix details", "error", err) } } @@ -840,6 +982,7 @@ func (s *Server) handlePrefixDetailJSON() http.HandlerFunc { func (s *Server) handleASDetail() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { asnStr := chi.URLParam(r, "asn") + asn, err := strconv.Atoi(asnStr) if err != nil { http.Error(w, "Invalid ASN", http.StatusBadRequest) @@ -853,6 +996,7 @@ func (s *Server) handleASDetail() http.HandlerFunc { return } + if err != nil { if errors.Is(err, database.ErrNoRoute) { http.Error(w, "AS not found", http.StatusNotFound) @@ -871,6 +1015,7 @@ func (s *Server) handleASDetail() http.HandlerFunc { return } + if err != nil { s.logger.Error("Failed to get AS peers", "error", err) // Continue without peers rather than failing the whole request @@ -878,52 +1023,11 @@ func (s *Server) handleASDetail() http.HandlerFunc { } // Group prefixes by IP version - const ipVersionV4 = 4 - var ipv4Prefixes, ipv6Prefixes []database.LiveRoute - for _, p := range prefixes { - if p.IPVersion == ipVersionV4 { - ipv4Prefixes = append(ipv4Prefixes, p) - } else { - ipv6Prefixes = append(ipv6Prefixes, p) - } - } + ipv4Prefixes, ipv6Prefixes := splitRoutesByIPVersion(prefixes) // Sort prefixes by network address - sort.Slice(ipv4Prefixes, func(i, j int) bool { - // Parse the prefixes to compare network addresses - ipI, netI, _ := net.ParseCIDR(ipv4Prefixes[i].Prefix) - ipJ, netJ, _ := net.ParseCIDR(ipv4Prefixes[j].Prefix) - - // Compare by network address first - cmp := bytes.Compare(ipI.To4(), ipJ.To4()) - if cmp != 0 { - return cmp < 0 - } - - // If network addresses are equal, compare by mask length - onesI, _ := netI.Mask.Size() - onesJ, _ := netJ.Mask.Size() - - return onesI < onesJ - }) - - sort.Slice(ipv6Prefixes, func(i, j int) bool { - // Parse the prefixes to compare network addresses - ipI, netI, _ := net.ParseCIDR(ipv6Prefixes[i].Prefix) - ipJ, netJ, _ := net.ParseCIDR(ipv6Prefixes[j].Prefix) - - // Compare by network address first - cmp := bytes.Compare(ipI.To16(), ipJ.To16()) - if cmp != 0 { - return cmp < 0 - } - - // If network addresses are equal, compare by mask length - onesI, _ := netI.Mask.Size() - onesJ, _ := netJ.Mask.Size() - - return onesI < onesJ - }) + sortRoutesByNetwork(ipv4Prefixes) + sortRoutesByNetwork(ipv6Prefixes) // Prepare template data data := struct { @@ -947,16 +1051,17 @@ func (s *Server) handleASDetail() http.HandlerFunc { } // Check if context is still valid before writing response - select { - case <-r.Context().Done(): + if r.Context().Err() != nil { // Request was cancelled, don't write response return - default: } w.Header().Set("Content-Type", "text/html; charset=utf-8") + tmpl := templates.ASDetailTemplate() - if err := tmpl.Execute(w, data); err != nil { + + err = tmpl.Execute(w, data) + if err != nil { s.logger.Error("Failed to render AS detail template", "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) } @@ -984,14 +1089,16 @@ func (s *Server) handlePrefixDetail() http.HandlerFunc { return } + prefix := parsed.Masked().String() - routes, originASNs, err := s.db.GetPrefixDetailsContext(r.Context(), prefix) + routes, originNames, err := s.db.GetPrefixDetailsContext(r.Context(), prefix) if deadlinePassed(r.Context()) { http.Error(w, requestTimeoutMessage, http.StatusGatewayTimeout) return } + if err != nil { if errors.Is(err, database.ErrNoRoute) { http.Error(w, "Prefix not found", http.StatusNotFound) @@ -1004,75 +1111,16 @@ func (s *Server) handlePrefixDetail() http.HandlerFunc { } // Group by origin AS and collect unique AS info - type ASNInfo struct { - ASN int - Handle string - Description string - PeerCount int - } - originMap := make(map[int]*ASNInfo) - for _, route := range routes { - if _, exists := originMap[route.OriginASN]; !exists { - originMap[route.OriginASN] = &ASNInfo{ - ASN: route.OriginASN, - Handle: originASNs[route.OriginASN].Handle, - Description: originASNs[route.OriginASN].Description, - PeerCount: 0, - } - } - originMap[route.OriginASN].PeerCount++ - } + origins := originASNs(routes, originNames) // Get the first route to extract some common info var maskLength, ipVersion int if len(routes) > 0 { - // Parse CIDR to get mask length and IP version - _, ipNet, err := net.ParseCIDR(prefix) - if err == nil { - ones, _ := ipNet.Mask.Size() - maskLength = ones - if ipNet.IP.To4() != nil { - ipVersion = 4 - } else { - ipVersion = 6 - } - } - } - - // Convert origin map to sorted slice - var origins []*ASNInfo - for _, origin := range originMap { - origins = append(origins, origin) + maskLength, ipVersion = prefixMaskLengthAndIPVersion(prefix) } // Create enhanced routes with AS path handles - type ASPathEntry struct { - ASN int - Handle string - } - type EnhancedRoute struct { - database.LiveRoute - ASPathWithHandle []ASPathEntry - } - - enhancedRoutes := make([]EnhancedRoute, len(routes)) - for i, route := range routes { - enhancedRoute := EnhancedRoute{ - LiveRoute: route, - ASPathWithHandle: make([]ASPathEntry, len(route.ASPath)), - } - - // Look up handle for each AS in the path - for j, asn := range route.ASPath { - handle := asinfo.GetHandle(asn) - enhancedRoute.ASPathWithHandle[j] = ASPathEntry{ - ASN: asn, - Handle: handle, - } - } - - enhancedRoutes[i] = enhancedRoute - } + enhancedRoutes := routesWithASPathHandles(routes) // Prepare template data data := struct { @@ -1090,7 +1138,7 @@ func (s *Server) handlePrefixDetail() http.HandlerFunc { Routes: enhancedRoutes, Origins: origins, PeerCount: len(routes), - OriginCount: len(originMap), + OriginCount: len(origins), } // Check if context is still valid before writing response @@ -1102,16 +1150,123 @@ func (s *Server) handlePrefixDetail() http.HandlerFunc { } w.Header().Set("Content-Type", "text/html; charset=utf-8") + tmpl := templates.PrefixDetailTemplate() - if err := tmpl.Execute(w, data); err != nil { + + err = tmpl.Execute(w, data) + if err != nil { s.logger.Error("Failed to render prefix detail template", "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) } } } -// handlePrefixLength shows a random sample of IPv4 prefixes with the specified mask length +// ASNInfo is one origin AS of a prefix on the prefix detail page. +type ASNInfo struct { + ASN int + Handle string + Description string + PeerCount int +} + +// originASNs groups routes by origin AS, with each AS's handle and description +// from names, as the prefix query returned them, and the number of routes it +// originates. The origin ASes come in no particular order. +func originASNs( + routes []database.LiveRoute, + names map[int]database.ASN, +) []*ASNInfo { + originMap := make(map[int]*ASNInfo) + + for _, route := range routes { + if _, exists := originMap[route.OriginASN]; !exists { + originMap[route.OriginASN] = &ASNInfo{ + ASN: route.OriginASN, + Handle: names[route.OriginASN].Handle, + Description: names[route.OriginASN].Description, + PeerCount: 0, + } + } + + originMap[route.OriginASN].PeerCount++ + } + + origins := make([]*ASNInfo, 0, len(originMap)) + for _, origin := range originMap { + origins = append(origins, origin) + } + + return origins +} + +// ASPathEntry is one AS of a route's AS path, with its handle. +type ASPathEntry struct { + ASN int + Handle string +} + +// EnhancedRoute is a live route with the handle of each AS in its path. +type EnhancedRoute struct { + database.LiveRoute + + ASPathWithHandle []ASPathEntry +} + +// routesWithASPathHandles looks up the handle of each AS in the path of each +// route. +func routesWithASPathHandles(routes []database.LiveRoute) []EnhancedRoute { + enhancedRoutes := make([]EnhancedRoute, len(routes)) + + for i, route := range routes { + enhancedRoute := EnhancedRoute{ + LiveRoute: route, + ASPathWithHandle: make([]ASPathEntry, len(route.ASPath)), + } + + // Look up handle for each AS in the path + for j, asn := range route.ASPath { + handle := asinfo.GetHandle(asn) + enhancedRoute.ASPathWithHandle[j] = ASPathEntry{ + ASN: asn, + Handle: handle, + } + } + + enhancedRoutes[i] = enhancedRoute + } + + return enhancedRoutes +} + +// prefixMaskLengthAndIPVersion returns the mask length and IP version of a +// prefix in CIDR notation, or zeros if it does not parse. +func prefixMaskLengthAndIPVersion(prefix string) (int, int) { + _, ipNet, err := net.ParseCIDR(prefix) + if err != nil { + return 0, 0 + } + + maskLength, _ := ipNet.Mask.Size() + + if ipNet.IP.To4() != nil { + return maskLength, ipVersionV4 + } + + return maskLength, ipVersionV6 +} + +// handlePrefixLength shows a random sample of IPv4 prefixes with the +// specified mask length func (s *Server) handlePrefixLength() http.HandlerFunc { + const maxIPv4MaskLength = 32 + + return s.prefixLengthHandler(ipVersionV4, maxIPv4MaskLength) +} + +// prefixLengthHandler returns a handler that shows a random sample of the +// prefixes of one IP version with the specified mask length, which may be at +// most maxMaskLength. +func (s *Server) prefixLengthHandler(ipVersion, maxMaskLength int) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { lengthStr := chi.URLParam(r, "length") if lengthStr == "" { @@ -1127,24 +1282,25 @@ func (s *Server) handlePrefixLength() http.HandlerFunc { return } - // Validate IPv4 mask length - const maxIPv4MaskLength = 32 - if maskLength < 0 || maskLength > maxIPv4MaskLength { - http.Error(w, "Invalid IPv4 mask length", http.StatusBadRequest) + // Validate mask length + if maskLength < 0 || maskLength > maxMaskLength { + http.Error(w, "Invalid IPv"+strconv.Itoa(ipVersion)+" mask length", + http.StatusBadRequest) return } - const ipVersion = 4 - // Get random sample of prefixes const maxPrefixes = 500 - prefixes, err := s.db.GetRandomPrefixesByLengthContext(r.Context(), maskLength, ipVersion, maxPrefixes) + + prefixes, err := s.db.GetRandomPrefixesByLengthContext( + r.Context(), maskLength, ipVersion, maxPrefixes) if deadlinePassed(r.Context()) { http.Error(w, requestTimeoutMessage, http.StatusGatewayTimeout) return } + if err != nil { s.logger.Error("Failed to get prefixes by length", "error", err) http.Error(w, "Internal server error", http.StatusInternalServerError) @@ -1162,31 +1318,11 @@ func (s *Server) handlePrefixLength() http.HandlerFunc { return prefixes[i].Prefix < prefixes[j].Prefix }) - // Create enhanced prefixes with AS descriptions - type EnhancedPrefix struct { - database.LiveRoute - OriginASDescription string - Age string - } - - enhancedPrefixes := make([]EnhancedPrefix, len(prefixes)) - for i, prefix := range prefixes { - enhancedPrefixes[i] = EnhancedPrefix{ - LiveRoute: prefix, - Age: formatAge(prefix.LastUpdated), - } - - // Get AS description - if asInfo, ok := asinfo.Get(prefix.OriginASN); ok { - enhancedPrefixes[i].OriginASDescription = asInfo.Description - } - } - // Render template - data := map[string]interface{}{ + data := map[string]any{ "MaskLength": maskLength, "IPVersion": ipVersion, - "Prefixes": enhancedPrefixes, + "Prefixes": prefixesWithASDescriptions(prefixes), "Count": len(prefixes), } @@ -1199,107 +1335,49 @@ func (s *Server) handlePrefixLength() http.HandlerFunc { } tmpl := templates.PrefixLengthTemplate() - if err := tmpl.Execute(w, data); err != nil { + + err = tmpl.Execute(w, data) + if err != nil { s.logger.Error("Failed to render prefix length template", "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) } } } -// handlePrefixLength6 shows a random sample of IPv6 prefixes with the specified mask length -func (s *Server) handlePrefixLength6() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - lengthStr := chi.URLParam(r, "length") - if lengthStr == "" { - http.Error(w, "Length parameter is required", http.StatusBadRequest) +// EnhancedPrefix is a live route on the prefix length page, with the +// description of its origin AS and its age. +type EnhancedPrefix struct { + database.LiveRoute - return + OriginASDescription string + Age string +} + +// prefixesWithASDescriptions looks up the description of the origin AS of +// each prefix and gives each one's age. +func prefixesWithASDescriptions(prefixes []database.LiveRoute) []EnhancedPrefix { + enhancedPrefixes := make([]EnhancedPrefix, len(prefixes)) + for i, prefix := range prefixes { + enhancedPrefixes[i] = EnhancedPrefix{ + LiveRoute: prefix, + Age: formatAge(prefix.LastUpdated), } - maskLength, err := strconv.Atoi(lengthStr) - if err != nil { - http.Error(w, "Invalid mask length", http.StatusBadRequest) - - return - } - - // Validate IPv6 mask length - const maxIPv6MaskLength = 128 - if maskLength < 0 || maskLength > maxIPv6MaskLength { - http.Error(w, "Invalid IPv6 mask length", http.StatusBadRequest) - - return - } - - const ipVersion = 6 - - // Get random sample of prefixes - const maxPrefixes = 500 - prefixes, err := s.db.GetRandomPrefixesByLengthContext(r.Context(), maskLength, ipVersion, maxPrefixes) - if deadlinePassed(r.Context()) { - http.Error(w, requestTimeoutMessage, http.StatusGatewayTimeout) - - return - } - if err != nil { - s.logger.Error("Failed to get prefixes by length", "error", err) - http.Error(w, "Internal server error", http.StatusInternalServerError) - - return - } - - // Sort prefixes for display - sort.Slice(prefixes, func(i, j int) bool { - // First compare by IP version - if prefixes[i].IPVersion != prefixes[j].IPVersion { - return prefixes[i].IPVersion < prefixes[j].IPVersion - } - // Then by prefix - return prefixes[i].Prefix < prefixes[j].Prefix - }) - - // Create enhanced prefixes with AS descriptions - type EnhancedPrefix struct { - database.LiveRoute - OriginASDescription string - Age string - } - - enhancedPrefixes := make([]EnhancedPrefix, len(prefixes)) - for i, prefix := range prefixes { - enhancedPrefixes[i] = EnhancedPrefix{ - LiveRoute: prefix, - Age: formatAge(prefix.LastUpdated), - } - - // Get AS description - if asInfo, ok := asinfo.Get(prefix.OriginASN); ok { - enhancedPrefixes[i].OriginASDescription = asInfo.Description - } - } - - // Render template - data := map[string]interface{}{ - "MaskLength": maskLength, - "IPVersion": ipVersion, - "Prefixes": enhancedPrefixes, - "Count": len(prefixes), - } - - // Check if context is still valid before writing response - select { - case <-r.Context().Done(): - // Request was cancelled, don't write response - return - default: - } - - tmpl := templates.PrefixLengthTemplate() - if err := tmpl.Execute(w, data); err != nil { - s.logger.Error("Failed to render prefix length template", "error", err) - http.Error(w, "Internal Server Error", http.StatusInternalServerError) + // Get AS description + if asInfo, ok := asinfo.Get(prefix.OriginASN); ok { + enhancedPrefixes[i].OriginASDescription = asInfo.Description } } + + return enhancedPrefixes +} + +// handlePrefixLength6 shows a random sample of IPv6 prefixes with the +// specified mask length +func (s *Server) handlePrefixLength6() http.HandlerFunc { + const maxIPv6MaskLength = 128 + + return s.prefixLengthHandler(ipVersionV6, maxIPv6MaskLength) } // formatAge returns a human-readable age string @@ -1308,13 +1386,14 @@ func formatAge(timestamp time.Time) string { const hoursPerDay = 24 - if age < time.Minute { + switch { + case age < time.Minute: return "< 1m" - } else if age < time.Hour { + case age < time.Hour: minutes := int(age.Minutes()) return strconv.Itoa(minutes) + "m" - } else if age < hoursPerDay*time.Hour { + case age < hoursPerDay*time.Hour: hours := int(age.Hours()) return strconv.Itoa(hours) + "h" diff --git a/internal/server/handlers_test.go b/internal/server/handlers_internal_test.go similarity index 70% rename from internal/server/handlers_test.go rename to internal/server/handlers_internal_test.go index 1691c0f..7932601 100644 --- a/internal/server/handlers_test.go +++ b/internal/server/handlers_internal_test.go @@ -22,12 +22,26 @@ import ( "github.com/google/uuid" ) +// The prefix and peer addresses the prefix page tests store routes for. +const ( + testPrefixV4 = "192.0.2.0/24" + testPeerV4 = "198.51.100.1" + testPeerV6 = "2001:db8::1" +) + +// The names the stats handler tests report the two stats handlers by. +const ( + statusJSONHandlerName = "status.json" + statsHandlerName = "stats" +) + // blockingStatsDB embeds database.Store (left nil) and overrides only // GetStatsContext, which blocks until release is closed. The stats handlers // call it in a goroutine; every other Store method is unused on the timeout // path and would panic if called. type blockingStatsDB struct { database.Store + release chan struct{} } @@ -43,14 +57,16 @@ func (d blockingStatsDB) GetStatsContext(_ context.Context) (database.Stats, err // value. Before the fix the per-request goroutine sent on an unbuffered channel // that nothing received once the timeout won, so it blocked forever and every // poll leaked one goroutine. +// +//nolint:paralleltest // counts goroutines, so no other test may run alongside func TestStatsHandlersDoNotLeakOnTimeout(t *testing.T) { release := make(chan struct{}) db := blockingStatsDB{release: release} s := New(db, streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{}) handlers := map[string]http.HandlerFunc{ - "status.json": s.handleStatusJSON(), - "stats": s.handleStats(), + statusJSONHandlerName: s.handleStatusJSON(), + statsHandlerName: s.handleStats(), } baseline := settledGoroutineCount() @@ -62,7 +78,7 @@ func TestStatsHandlersDoNotLeakOnTimeout(t *testing.T) { for _, handler := range handlers { for range iterations { ctx, cancel := context.WithTimeout(context.Background(), requestTimeout) - req := httptest.NewRequest(http.MethodGet, "/", nil).WithContext(ctx) + req := httptest.NewRequestWithContext(ctx, http.MethodGet, "/", nil) handler(httptest.NewRecorder(), req) cancel() } @@ -87,67 +103,83 @@ func TestStatsHandlersDoNotLeakOnTimeout(t *testing.T) { // The oldest and newest route times still come from one-row lookups at the ends // of an index; with the database closed they are left out of the answer. func TestStatsHandlersAnswerFromMemory(t *testing.T) { + t.Parallel() + db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New()) if err != nil { t.Fatalf("database.New: %v", err) } ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) - if err := db.UpsertLiveRouteBatch([]*database.LiveRoute{ + + err = db.UpsertLiveRouteBatch([]*database.LiveRoute{ { ID: uuid.New(), Prefix: "198.51.100.0/24", MaskLength: 24, IPVersion: 4, - OriginASN: 64500, PeerIP: "192.0.2.1", ASPath: []int{64500}, NextHop: "192.0.2.1", - LastUpdated: ts, + OriginASN: 64500, PeerIP: "192.0.2.1", ASPath: []int{64500}, + NextHop: "192.0.2.1", LastUpdated: ts, }, { ID: uuid.New(), Prefix: "2001:db8::/32", MaskLength: 32, IPVersion: 6, - OriginASN: 64501, PeerIP: "2001:db8::1", ASPath: []int{64501}, NextHop: "2001:db8::1", - LastUpdated: ts, + OriginASN: 64501, PeerIP: testPeerV6, ASPath: []int{64501}, + NextHop: testPeerV6, LastUpdated: ts, }, - }); err != nil { + }) + if err != nil { t.Fatalf("UpsertLiveRouteBatch: %v", err) } - if err := db.Close(); err != nil { + + err = db.Close() + if err != nil { t.Fatalf("Close: %v", err) } s := New(db, streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{}) handlers := map[string]http.HandlerFunc{ - "status.json": s.handleStatusJSON(), - "stats": s.handleStats(), + statusJSONHandlerName: s.handleStatusJSON(), + statsHandlerName: s.handleStats(), } for name, handler := range handlers { rec := httptest.NewRecorder() - handler(rec, httptest.NewRequest(http.MethodGet, "/", nil)) + handler(rec, httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil)) + if rec.Code != http.StatusOK { - t.Errorf("%s: status %d, want %d; body %s", name, rec.Code, http.StatusOK, rec.Body) + t.Errorf("%s: status %d, want %d; body %s", + name, rec.Code, http.StatusOK, rec.Body) continue } + //nolint:tagliatelle // field names are the existing JSON API var body struct { Data struct { - IPv4Routes int `json:"ipv4_routes"` - IPv6Routes int `json:"ipv6_routes"` - IPv4PrefixDistribution []database.PrefixDistribution `json:"ipv4_prefix_distribution"` - IPv6PrefixDistribution []database.PrefixDistribution `json:"ipv6_prefix_distribution"` + IPv4Routes int `json:"ipv4_routes"` + IPv6Routes int `json:"ipv6_routes"` + IPv4Distribution []database.PrefixDistribution `json:"ipv4_prefix_distribution"` + IPv6Distribution []database.PrefixDistribution `json:"ipv6_prefix_distribution"` } `json:"data"` } - if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + + err := json.Unmarshal(rec.Body.Bytes(), &body) + if err != nil { t.Fatalf("%s: decoding the answer: %v", name, err) } if body.Data.IPv4Routes != 1 || body.Data.IPv6Routes != 1 { - t.Errorf("%s: routes = (v4 %d, v6 %d), want (1, 1)", name, body.Data.IPv4Routes, body.Data.IPv6Routes) + t.Errorf("%s: routes = (v4 %d, v6 %d), want (1, 1)", + name, body.Data.IPv4Routes, body.Data.IPv6Routes) } + wantV4 := []database.PrefixDistribution{{MaskLength: 24, Count: 1}} - if !slices.Equal(body.Data.IPv4PrefixDistribution, wantV4) { - t.Errorf("%s: IPv4 distribution = %v, want %v", name, body.Data.IPv4PrefixDistribution, wantV4) + if !slices.Equal(body.Data.IPv4Distribution, wantV4) { + t.Errorf("%s: IPv4 distribution = %v, want %v", + name, body.Data.IPv4Distribution, wantV4) } + wantV6 := []database.PrefixDistribution{{MaskLength: 32, Count: 1}} - if !slices.Equal(body.Data.IPv6PrefixDistribution, wantV6) { - t.Errorf("%s: IPv6 distribution = %v, want %v", name, body.Data.IPv6PrefixDistribution, wantV6) + if !slices.Equal(body.Data.IPv6Distribution, wantV6) { + t.Errorf("%s: IPv6 distribution = %v, want %v", + name, body.Data.IPv6Distribution, wantV6) } } } @@ -159,25 +191,30 @@ func TestStatsHandlersAnswerFromMemory(t *testing.T) { // "2001:DB8:0:0::/32" was not found although "2001:db8::/32" was stored, and // "foo/24" answered 500 (https://git.eeqj.de/sneak/routewatch/issues/71). func TestPrefixDetailHandlersParsePrefix(t *testing.T) { + t.Parallel() + db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New()) if err != nil { t.Fatalf("database.New: %v", err) } + t.Cleanup(func() { _ = db.Close() }) ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) - if err := db.UpsertLiveRouteBatch([]*database.LiveRoute{ + + err = db.UpsertLiveRouteBatch([]*database.LiveRoute{ { - ID: uuid.New(), Prefix: "192.0.2.0/24", MaskLength: 24, IPVersion: 4, - OriginASN: 64500, PeerIP: "198.51.100.1", ASPath: []int{64500}, NextHop: "198.51.100.1", - LastUpdated: ts, + ID: uuid.New(), Prefix: testPrefixV4, MaskLength: 24, IPVersion: 4, + OriginASN: 64500, PeerIP: testPeerV4, ASPath: []int{64500}, + NextHop: testPeerV4, LastUpdated: ts, }, { ID: uuid.New(), Prefix: "2001:db8::/32", MaskLength: 32, IPVersion: 6, - OriginASN: 64501, PeerIP: "2001:db8::1", ASPath: []int{64501}, NextHop: "2001:db8::1", - LastUpdated: ts, + OriginASN: 64501, PeerIP: testPeerV6, ASPath: []int{64501}, + NextHop: testPeerV6, LastUpdated: ts, }, - }); err != nil { + }) + if err != nil { t.Fatalf("UpsertLiveRouteBatch: %v", err) } @@ -197,20 +234,29 @@ func TestPrefixDetailHandlersParsePrefix(t *testing.T) { for _, tt := range tests { rec := httptest.NewRecorder() - s.router.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, tt.path, nil)) + s.router.ServeHTTP(rec, httptest.NewRequestWithContext( + t.Context(), http.MethodGet, tt.path, nil)) + if rec.Code != tt.want { - t.Errorf("%s: status %d, want %d; body %s", tt.path, rec.Code, tt.want, rec.Body) + t.Errorf("%s: status %d, want %d; body %s", + tt.path, rec.Code, tt.want, rec.Body) } } } +// errASDetailsLookupFailed is what failingASDetailsDB answers every AS details +// lookup with. +var errASDetailsLookupFailed = errors.New("AS details lookup failed") + // failingASDetailsDB wraps a real store and fails every AS details lookup. type failingASDetailsDB struct { database.Store } -func (failingASDetailsDB) GetASDetailsContext(_ context.Context, _ int) (*database.ASN, []database.LiveRoute, error) { - return nil, nil, errors.New("AS details lookup failed") +func (failingASDetailsDB) GetASDetailsContext( + _ context.Context, _ int, +) (*database.ASN, []database.LiveRoute, error) { + return nil, nil, errASDetailsLookupFailed } // TestPrefixDetailShowsOriginNames checks that the prefix page shows each @@ -220,51 +266,66 @@ func (failingASDetailsDB) GetASDetailsContext(_ context.Context, _ int) (*databa // (https://git.eeqj.de/sneak/routewatch/issues/67). The AS path column shows // handles too, so the test looks for the origin list's "(handle)" form. func TestPrefixDetailShowsOriginNames(t *testing.T) { + t.Parallel() + db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New()) if err != nil { t.Fatalf("database.New: %v", err) } + t.Cleanup(func() { _ = db.Close() }) ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) origins := []int{15169, 13335} + for _, asn := range origins { - if _, err := db.GetOrCreateASN(asn, ts); err != nil { + _, err := db.GetOrCreateASN(asn, ts) + if err != nil { t.Fatalf("GetOrCreateASN(%d): %v", asn, err) } } - if err := db.UpsertLiveRouteBatch([]*database.LiveRoute{ + + err = db.UpsertLiveRouteBatch([]*database.LiveRoute{ { - ID: uuid.New(), Prefix: "192.0.2.0/24", MaskLength: 24, IPVersion: 4, - OriginASN: origins[0], PeerIP: "198.51.100.1", ASPath: []int{origins[0]}, NextHop: "198.51.100.1", - LastUpdated: ts, + ID: uuid.New(), Prefix: testPrefixV4, MaskLength: 24, IPVersion: 4, + OriginASN: origins[0], PeerIP: testPeerV4, + ASPath: []int{origins[0]}, NextHop: testPeerV4, LastUpdated: ts, }, { - ID: uuid.New(), Prefix: "192.0.2.0/24", MaskLength: 24, IPVersion: 4, - OriginASN: origins[1], PeerIP: "198.51.100.2", ASPath: []int{origins[1]}, NextHop: "198.51.100.2", - LastUpdated: ts, + ID: uuid.New(), Prefix: testPrefixV4, MaskLength: 24, IPVersion: 4, + OriginASN: origins[1], PeerIP: "198.51.100.2", + ASPath: []int{origins[1]}, NextHop: "198.51.100.2", LastUpdated: ts, }, - }); err != nil { + }) + if err != nil { t.Fatalf("UpsertLiveRouteBatch: %v", err) } - s := New(failingASDetailsDB{db}, streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{}) + s := New(failingASDetailsDB{db}, streamer.New(logger.New(), metrics.New()), + logger.New(), &config.Config{}) rec := httptest.NewRecorder() - s.router.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/prefix/192.0.2.0/24", nil)) + s.router.ServeHTTP(rec, httptest.NewRequestWithContext( + t.Context(), http.MethodGet, "/prefix/"+testPrefixV4, nil)) + if rec.Code != http.StatusOK { t.Fatalf("status %d, want %d; body %s", rec.Code, http.StatusOK, rec.Body) } page := rec.Body.String() + for _, asn := range origins { info, ok := asinfo.Get(asn) if !ok { t.Fatalf("AS%d is not in the AS info data", asn) } - if want := "(" + html.EscapeString(info.Handle) + ")"; !strings.Contains(page, want) { + + want := "(" + html.EscapeString(info.Handle) + ")" + if !strings.Contains(page, want) { t.Errorf("page does not show AS%d's handle %q", asn, want) } - if want := html.EscapeString(info.Description); !strings.Contains(page, want) { + + want = html.EscapeString(info.Description) + if !strings.Contains(page, want) { t.Errorf("page does not show AS%d's description %q", asn, want) } } @@ -282,14 +343,18 @@ const shortRequestDeadline = 50 * time.Millisecond // written by the time limit middleware from its own goroutine while the // handler could still be writing. func TestRequestPastTimeLimitAnswers504(t *testing.T) { - s := New(nil, streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{}) + t.Parallel() + + s := New(nil, streamer.New(logger.New(), metrics.New()), logger.New(), + &config.Config{}) waitForTimeLimit := func(_ http.ResponseWriter, r *http.Request) { <-r.Context().Done() } // Registered the way setupRoutes registers pages and JSON routes. s.router.Get("/slow-page", waitForTimeLimit) - s.router.Get("/slow.json", JSONValidationMiddleware(http.HandlerFunc(waitForTimeLimit)).ServeHTTP) + s.router.Get("/slow.json", + JSONValidationMiddleware(http.HandlerFunc(waitForTimeLimit)).ServeHTTP) tests := []struct { path string @@ -300,14 +365,18 @@ func TestRequestPastTimeLimitAnswers504(t *testing.T) { } for _, tt := range tests { - ctx, cancel := context.WithTimeout(context.Background(), shortRequestDeadline) + ctx, cancel := context.WithTimeout(context.Background(), + shortRequestDeadline) rec := httptest.NewRecorder() - s.router.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, tt.path, nil).WithContext(ctx)) + s.router.ServeHTTP(rec, + httptest.NewRequestWithContext(ctx, http.MethodGet, tt.path, nil)) cancel() if rec.Code != http.StatusGatewayTimeout { - t.Errorf("%s: status %d, want %d; body %s", tt.path, rec.Code, http.StatusGatewayTimeout, rec.Body) + t.Errorf("%s: status %d, want %d; body %s", + tt.path, rec.Code, http.StatusGatewayTimeout, rec.Body) } + if tt.wantJSONBody { checkJSONErrorCode(t, tt.path, rec, http.StatusGatewayTimeout) } @@ -320,25 +389,32 @@ func TestRequestPastTimeLimitAnswers504(t *testing.T) { // body. Before the fix status.json answered 408, and /api/v1/stats wrote // nothing, which the JSON check on its route turned into 500. func TestStatsHandlersAnswer504WhenDatabaseReadRunsOut(t *testing.T) { + t.Parallel() + release := make(chan struct{}) defer close(release) + db := blockingStatsDB{release: release} - s := New(db, streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{}) + s := New(db, streamer.New(logger.New(), metrics.New()), logger.New(), + &config.Config{}) handlers := map[string]http.HandlerFunc{ - "status.json": s.handleStatusJSON(), - "stats": s.handleStats(), + statusJSONHandlerName: s.handleStatusJSON(), + statsHandlerName: s.handleStats(), } for name, handler := range handlers { - ctx, cancel := context.WithTimeout(context.Background(), shortRequestDeadline) + ctx, cancel := context.WithTimeout(context.Background(), + shortRequestDeadline) rec := httptest.NewRecorder() - handler(rec, httptest.NewRequest(http.MethodGet, "/", nil).WithContext(ctx)) + handler(rec, httptest.NewRequestWithContext(ctx, http.MethodGet, "/", nil)) cancel() if rec.Code != http.StatusGatewayTimeout { - t.Errorf("%s: status %d, want %d; body %s", name, rec.Code, http.StatusGatewayTimeout, rec.Body) + t.Errorf("%s: status %d, want %d; body %s", + name, rec.Code, http.StatusGatewayTimeout, rec.Body) } + checkJSONErrorCode(t, name, rec, http.StatusGatewayTimeout) } } @@ -360,25 +436,33 @@ func (slowDB) GetStatsContext(ctx context.Context) (database.Stats, error) { return database.Stats{}, nil } -func (slowDB) GetIPInfoContext(ctx context.Context, _ string) (*database.IPInfo, error) { +func (slowDB) GetIPInfoContext( + ctx context.Context, _ string, +) (*database.IPInfo, error) { <-ctx.Done() return nil, ctx.Err() } -func (slowDB) GetASDetailsContext(ctx context.Context, asn int) (*database.ASN, []database.LiveRoute, error) { +func (slowDB) GetASDetailsContext( + ctx context.Context, asn int, +) (*database.ASN, []database.LiveRoute, error) { <-ctx.Done() return &database.ASN{ASN: asn}, nil, nil } -func (slowDB) GetPrefixDetailsContext(ctx context.Context, _ string) ([]database.LiveRoute, map[int]database.ASN, error) { +func (slowDB) GetPrefixDetailsContext( + ctx context.Context, _ string, +) ([]database.LiveRoute, map[int]database.ASN, error) { <-ctx.Done() return nil, nil, database.ErrNoRoute } -func (slowDB) GetRandomPrefixesByLengthContext(ctx context.Context, _, _, _ int) ([]database.LiveRoute, error) { +func (slowDB) GetRandomPrefixesByLengthContext( + ctx context.Context, _, _, _ int, +) ([]database.LiveRoute, error) { <-ctx.Done() return nil, nil @@ -393,7 +477,10 @@ func (slowDB) GetRandomPrefixesByLengthContext(ctx context.Context, _, _, _ int) // body. The health check is left out: it answers 503 when its own 2-second // ping runs out. func TestRoutesAnswer504WhenDatabaseReadIsCutOff(t *testing.T) { - s := New(slowDB{}, streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{}) + t.Parallel() + + s := New(slowDB{}, streamer.New(logger.New(), metrics.New()), logger.New(), + &config.Config{}) tests := []struct { path string @@ -425,11 +512,15 @@ type slowSecondReadDB struct { database.Store } -func (slowSecondReadDB) GetASDetailsContext(_ context.Context, asn int) (*database.ASN, []database.LiveRoute, error) { +func (slowSecondReadDB) GetASDetailsContext( + _ context.Context, asn int, +) (*database.ASN, []database.LiveRoute, error) { return &database.ASN{ASN: asn}, nil, nil } -func (slowSecondReadDB) GetASPeersContext(ctx context.Context, _ int) ([]database.ASPeer, error) { +func (slowSecondReadDB) GetASPeersContext( + ctx context.Context, _ int, +) ([]database.ASPeer, error) { <-ctx.Done() return nil, ctx.Err() @@ -439,7 +530,9 @@ func (slowSecondReadDB) GetStatsContext(_ context.Context) (database.Stats, erro return database.Stats{}, nil } -func (slowSecondReadDB) GetWHOISStats(ctx context.Context, _ time.Duration) (*database.WHOISStats, error) { +func (slowSecondReadDB) GetWHOISStats( + ctx context.Context, _ time.Duration, +) (*database.WHOISStats, error) { <-ctx.Done() return nil, ctx.Err() @@ -462,7 +555,10 @@ func (idleASNFetcher) GetStats() ASNFetcherStats { // message. Without the check after the second read, the AS page answers 504 // with an empty body and the stats routes answer 200 without the WHOIS stats. func TestRoutesAnswer504WhenSecondDatabaseReadIsCutOff(t *testing.T) { - s := New(slowSecondReadDB{}, streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{}) + t.Parallel() + + s := New(slowSecondReadDB{}, streamer.New(logger.New(), metrics.New()), + logger.New(), &config.Config{}) s.SetASNFetcher(idleASNFetcher{}) tests := []struct { @@ -482,27 +578,40 @@ func TestRoutesAnswer504WhenSecondDatabaseReadIsCutOff(t *testing.T) { // checkRouteAnswers504 sends a request for path through the server's router // with a short deadline and checks that it answers 504 with the time limit // message: the JSON error body on JSON routes, the plain message on pages. -func checkRouteAnswers504(t *testing.T, s *Server, path string, wantJSONBody bool) { +func checkRouteAnswers504( + t *testing.T, s *Server, path string, wantJSONBody bool, +) { t.Helper() ctx, cancel := context.WithTimeout(context.Background(), shortRequestDeadline) defer cancel() + rec := httptest.NewRecorder() - s.router.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil).WithContext(ctx)) + s.router.ServeHTTP(rec, + httptest.NewRequestWithContext(ctx, http.MethodGet, path, nil)) if rec.Code != http.StatusGatewayTimeout { - t.Errorf("%s: status %d, want %d; body %s", path, rec.Code, http.StatusGatewayTimeout, rec.Body) + t.Errorf("%s: status %d, want %d; body %s", + path, rec.Code, http.StatusGatewayTimeout, rec.Body) } + if wantJSONBody { checkJSONErrorCode(t, path, rec, http.StatusGatewayTimeout) - } else if body := strings.TrimSpace(rec.Body.String()); body != requestTimeoutMessage { + + return + } + + body := strings.TrimSpace(rec.Body.String()) + if body != requestTimeoutMessage { t.Errorf("%s: body %q, want %q", path, body, requestTimeoutMessage) } } // checkJSONErrorCode checks that rec holds the JSON error body with the given // code. -func checkJSONErrorCode(t *testing.T, name string, rec *httptest.ResponseRecorder, want int) { +func checkJSONErrorCode( + t *testing.T, name string, rec *httptest.ResponseRecorder, want int, +) { t.Helper() var body struct { @@ -511,13 +620,17 @@ func checkJSONErrorCode(t *testing.T, name string, rec *httptest.ResponseRecorde Code int `json:"code"` } `json:"error"` } - if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + + err := json.Unmarshal(rec.Body.Bytes(), &body) + if err != nil { t.Errorf("%s: decoding the answer %q: %v", name, rec.Body, err) return } - if body.Status != "error" || body.Error.Code != want { - t.Errorf("%s: answer %s, want status \"error\" and error code %d", name, rec.Body, want) + + if body.Status != jsonStatusError || body.Error.Code != want { + t.Errorf("%s: answer %s, want status \"error\" and error code %d", + name, rec.Body, want) } } @@ -525,12 +638,15 @@ func checkJSONErrorCode(t *testing.T, name string, rec *httptest.ResponseRecorde // current count. func settledGoroutineCount() int { prev := runtime.NumGoroutine() + for range 20 { time.Sleep(10 * time.Millisecond) + cur := runtime.NumGoroutine() if cur == prev { return cur } + prev = cur } @@ -543,6 +659,7 @@ func waitForGoroutines(target int) bool { if runtime.NumGoroutine() <= target { return true } + time.Sleep(10 * time.Millisecond) } diff --git a/internal/server/middleware.go b/internal/server/middleware.go index bd336fb..d2a185d 100644 --- a/internal/server/middleware.go +++ b/internal/server/middleware.go @@ -15,6 +15,7 @@ import ( // responseWriter wraps http.ResponseWriter to capture the response type responseWriter struct { http.ResponseWriter + body *bytes.Buffer statusCode int written bool @@ -87,8 +88,10 @@ func JSONResponseMiddleware(next http.Handler) http.Handler { } // Parse the original response - var originalResponse map[string]interface{} - if err := json.Unmarshal(rw.body.Bytes(), &originalResponse); err != nil { + var originalResponse map[string]any + + err := json.Unmarshal(rw.body.Bytes(), &originalResponse) + if err != nil { // If we can't parse it, just write original w.WriteHeader(rw.statusCode) _, _ = w.Write(rw.body.Bytes()) @@ -97,7 +100,7 @@ func JSONResponseMiddleware(next http.Handler) http.Handler { } // Add @meta field - originalResponse["@meta"] = map[string]interface{}{ + originalResponse["@meta"] = map[string]any{ "time_zone": "UTC", "api_version": 1, "execution_time": fmt.Sprintf("%d ms", execTime.Milliseconds()), @@ -160,8 +163,10 @@ func JSONValidationMiddleware(next http.Handler) http.Handler { // For JSON responses, validate the JSON if rw.body.Len() > 0 { - var testParse interface{} - if err := json.Unmarshal(rw.body.Bytes(), &testParse); err == nil { + var testParse any + + err := json.Unmarshal(rw.body.Bytes(), &testParse) + if err == nil { // Valid JSON, write it out w.Header().Set("Content-Type", "application/json") w.WriteHeader(rw.statusCode) @@ -180,9 +185,11 @@ func JSONValidationMiddleware(next http.Handler) http.Handler { if statusCode == http.StatusOK { statusCode = http.StatusInternalServerError } + if deadlinePassed(r.Context()) { statusCode = http.StatusGatewayTimeout } + w.WriteHeader(statusCode) errorMsg := "Internal server error" @@ -190,11 +197,11 @@ func JSONValidationMiddleware(next http.Handler) http.Handler { errorMsg = requestTimeoutMessage } - response := map[string]interface{}{ - "status": "error", - "error": map[string]interface{}{ - "msg": errorMsg, - "code": statusCode, + response := map[string]any{ + jsonKeyStatus: jsonStatusError, + jsonKeyError: map[string]any{ + jsonKeyMsg: errorMsg, + jsonKeyCode: statusCode, }, } _ = json.NewEncoder(w).Encode(response) @@ -204,6 +211,7 @@ func JSONValidationMiddleware(next http.Handler) http.Handler { // statusWriter wraps http.ResponseWriter to capture the status code type statusWriter struct { http.ResponseWriter + statusCode int written bool } @@ -213,6 +221,7 @@ func (sw *statusWriter) WriteHeader(statusCode int) { sw.statusCode = statusCode sw.written = true } + sw.ResponseWriter.WriteHeader(statusCode) } @@ -252,11 +261,12 @@ func RequestLoggerMiddleware(logger *slog.Logger) func(http.Handler) http.Handle // Slow query threshold (1 second) const slowQueryThreshold = 1 * time.Second - if sw.statusCode >= http.StatusInternalServerError { + switch { + case sw.statusCode >= http.StatusInternalServerError: logLevel = slog.LevelError - } else if sw.statusCode >= http.StatusBadRequest { + case sw.statusCode >= http.StatusBadRequest: logLevel = slog.LevelWarn - } else if duration >= slowQueryThreshold { + case duration >= slowQueryThreshold: logLevel = slog.LevelWarn } diff --git a/internal/server/routes.go b/internal/server/routes.go index 9227772..aca1005 100644 --- a/internal/server/routes.go +++ b/internal/server/routes.go @@ -27,7 +27,10 @@ func (s *Server) setupRoutes() { r.Get("/", s.handleIndex()) r.Get("/status", s.handleStatusHTML()) r.Get("/status.json", JSONValidationMiddleware(s.handleStatusJSON()).ServeHTTP) - r.Get("/.well-known/healthcheck.json", JSONValidationMiddleware(s.handleHealthCheck()).ServeHTTP) + r.Get( + "/.well-known/healthcheck.json", + JSONValidationMiddleware(s.handleHealthCheck()).ServeHTTP, + ) // AS and prefix detail pages r.Get("/as/{asn}", s.handleASDetail()) diff --git a/internal/server/server.go b/internal/server/server.go index 16ba2b3..d709b7a 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -3,6 +3,7 @@ package server import ( "context" + "errors" "fmt" "net" "net/http" @@ -42,7 +43,12 @@ type Server struct { } // New creates a new HTTP server -func New(db database.Store, streamer *streamer.Streamer, logger *logger.Logger, cfg *config.Config) *Server { +func New( + db database.Store, + streamer *streamer.Streamer, + logger *logger.Logger, + cfg *config.Config, +) *Server { s := &Server{ db: db, streamer: streamer, @@ -57,8 +63,8 @@ func New(db database.Store, streamer *streamer.Streamer, logger *logger.Logger, // Start listens on the configured port and serves HTTP in the background. // It returns an error if it cannot listen, for example because the port is -// already in use. -func (s *Server) Start() error { +// already in use. ctx bounds only opening the port, not serving. +func (s *Server) Start(ctx context.Context) error { const ( network = "tcp" readHeaderTimeout = 40 * time.Second @@ -78,7 +84,9 @@ func (s *Server) Start() error { s.logger.Info("Starting HTTP server", "port", s.port, "addr", s.srv.Addr) - listener, err := net.Listen(network, s.srv.Addr) + var listenConfig net.ListenConfig + + listener, err := listenConfig.Listen(ctx, network, s.srv.Addr) if err != nil { return fmt.Errorf("starting HTTP server: %w", err) } @@ -86,7 +94,8 @@ func (s *Server) Start() error { s.logger.Info("HTTP server listening", "addr", listener.Addr().String()) go func() { - if err := s.srv.Serve(listener); err != nil && err != http.ErrServerClosed { + err := s.srv.Serve(listener) + if err != nil && !errors.Is(err, http.ErrServerClosed) { s.logger.Error("HTTP server error", "error", err) } }() diff --git a/internal/server/server_test.go b/internal/server/server_test.go index ff7c7d2..1f6964b 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -1,4 +1,4 @@ -package server +package server_test import ( "context" @@ -8,6 +8,7 @@ import ( "git.eeqj.de/sneak/routewatch/internal/config" "git.eeqj.de/sneak/routewatch/internal/logger" "git.eeqj.de/sneak/routewatch/internal/metrics" + "git.eeqj.de/sneak/routewatch/internal/server" "git.eeqj.de/sneak/routewatch/internal/streamer" ) @@ -16,18 +17,31 @@ import ( // listened in a goroutine, only logged the failure and returned nil, so the // daemon kept running without its web server. func TestStartFailsWhenPortIsHeld(t *testing.T) { - held, err := net.Listen("tcp", ":0") + t.Parallel() + + var listenConfig net.ListenConfig + + held, err := listenConfig.Listen(t.Context(), "tcp", ":0") if err != nil { t.Fatalf("failed to open the test listener: %v", err) } defer func() { _ = held.Close() }() - port := held.Addr().(*net.TCPAddr).Port + addr, ok := held.Addr().(*net.TCPAddr) + if !ok { + t.Fatalf("test listener address %v is not a TCP address", held.Addr()) + } + + port := addr.Port log := logger.New() - s := New(nil, streamer.New(log, metrics.New()), log, &config.Config{Port: port}) + s := server.New(nil, streamer.New(log, metrics.New()), log, + &config.Config{Port: port}) + defer func() { _ = s.Stop(context.Background()) }() - if err := s.Start(); err == nil { - t.Fatalf("Start returned nil while port %d was held by another listener", port) + err = s.Start(t.Context()) + if err == nil { + t.Fatalf("Start returned nil while port %d was held by another listener", + port) } } diff --git a/internal/streamer/streamer.go b/internal/streamer/streamer.go index 7c475c4..4b43ff1 100644 --- a/internal/streamer/streamer.go +++ b/internal/streamer/streamer.go @@ -1,5 +1,6 @@ -// Package streamer implements an HTTP client that connects to the RIPE RIS Live streaming API, -// parses BGP UPDATE messages from the JSON stream, and dispatches them to registered handlers. +// Package streamer implements an HTTP client that connects to the RIPE RIS Live +// streaming API, parses BGP UPDATE messages from the JSON stream, and +// dispatches them to registered handlers. package streamer import ( @@ -25,50 +26,60 @@ import ( // countingReader wraps an io.Reader and counts bytes read type countingReader struct { reader io.Reader - count int64 + count atomic.Int64 } // Read implements io.Reader and counts bytes func (c *countingReader) Read(p []byte) (int, error) { n, err := c.reader.Read(p) - atomic.AddInt64(&c.count, int64(n)) + c.count.Add(int64(n)) return n, err } // Count returns the total bytes read func (c *countingReader) Count() int64 { - return atomic.LoadInt64(&c.count) + return c.count.Load() } // Configuration constants for the RIS Live streamer. const ( risLiveURL = "https://ris-live.ripe.net/v1/stream/?format=json&" + "client=https%3A%2F%2Fgit.eeqj.de%2Fsneak%2Froutewatch" - metricsWindowSize = 60 // seconds for rolling average - metricsUpdateRate = time.Second - minBackoffDelay = 5 * time.Second - maxBackoffDelay = 320 * time.Second - streamIdleTimeout = 60 * time.Second // Reconnect when no line arrives for this long - metricsLogInterval = 10 * time.Second - bytesPerKB = 1024 - bytesPerMB = 1024 * 1024 - maxLineSize = bytesPerMB // Well above an extended BGP message (RFC 8654) in hex plus its prefixes - maxConcurrentHandlers = 800 // Maximum number of concurrent message handlers + metricsWindowSize = 60 // seconds for rolling average + metricsUpdateRate = time.Second + minBackoffDelay = 5 * time.Second + maxBackoffDelay = 320 * time.Second + // Reconnect when no line arrives for this long + streamIdleTimeout = 60 * time.Second + metricsLogInterval = 10 * time.Second + bytesPerKB = 1024 + bytesPerMB = 1024 * 1024 + // Well above an extended BGP message (RFC 8654) in hex plus its prefixes + maxLineSize = bytesPerMB + maxConcurrentHandlers = 800 // Maximum number of concurrent message handlers // Backpressure constants backpressureThreshold = 0.5 // Start dropping at 50% queue utilization backpressureSlope = 2.0 // Slope for linear drop probability increase ) -// errStreamEnded is returned by stream when RIS Live ends the response. -var errStreamEnded = errors.New("RIS Live ended the stream") +var ( + errAlreadyRunning = errors.New("streamer already running") + errUnexpectedStatus = errors.New("unexpected status code") + // errStreamEnded is returned by stream when RIS Live ends the response. + errStreamEnded = errors.New("RIS Live ended the stream") + // errStreamIdle is returned by stream when no line arrived for the idle + // timeout. + errStreamIdle = errors.New("no line received") +) // MessageHandler defines the interface for processing RIS messages. // Implementations must specify which message types they want to receive, // how to process messages, and their desired queue capacity. type MessageHandler interface { - // WantsMessage returns true if this handler wants to process messages of the given type. + // WantsMessage returns true if this handler wants to process messages of + // the given type. WantsMessage(messageType string) bool // HandleMessage processes a RIS message. This method is called from a dedicated @@ -114,18 +125,20 @@ type handlerInfo struct { // backoff, dispatches messages to registered handlers via per-handler queues, and // implements backpressure to prevent queue overflow during high traffic periods. type Streamer struct { - logger *logger.Logger - client *http.Client - url string - idleTimeout time.Duration // Drop the connection when no line arrives for this long - minBackoff time.Duration // First wait before reconnecting; doubles up to maxBackoffDelay + logger *logger.Logger + client *http.Client + url string + // Drop the connection when no line arrives for this long + idleTimeout time.Duration + // First wait before reconnecting; doubles up to maxBackoffDelay + minBackoff time.Duration handlers []*handlerInfo rawHandler RawMessageHandler mu sync.RWMutex cancel context.CancelFunc running bool metrics *metrics.Tracker - totalDropped uint64 // Total dropped messages across all handlers + totalDropped atomic.Uint64 // Total dropped messages across all handlers random *rand.Rand // Random number generator for backpressure drops bgpPeers map[string]bool // Track active BGP peers by peer IP bgpPeersMu sync.RWMutex // Protects bgpPeers map @@ -133,7 +146,8 @@ type Streamer struct { // New creates a new Streamer instance configured to connect to the RIS Live API. // The logger is used for structured logging of connection events and errors. -// The metrics tracker is used to record message counts, bytes received, and connection status. +// The metrics tracker is used to record message counts, bytes received, and +// connection status. func New(logger *logger.Logger, metrics *metrics.Tracker) *Streamer { return &Streamer{ logger: logger, @@ -167,7 +181,8 @@ func (s *Streamer) RegisterHandler(handler MessageHandler) { handler: handler, queue: make(chan *ristypes.RISMessage, handler.QueueCapacity()), metrics: handlerMetrics{ - minTime: time.Duration(math.MaxInt64), // Initialize to max so first value sets the floor + // Initialize to max so first value sets the floor + minTime: time.Duration(math.MaxInt64), }, } @@ -185,22 +200,24 @@ func (s *Streamer) RegisterHandler(handler MessageHandler) { func (s *Streamer) RegisterRawHandler(handler RawMessageHandler) { s.mu.Lock() defer s.mu.Unlock() + s.rawHandler = handler } // Start begins streaming BGP updates from the RIS Live API in a background goroutine. // It starts worker goroutines for each registered handler and manages automatic // reconnection with exponential backoff on connection failures. -// Returns an error if the streamer is already running. -func (s *Streamer) Start() error { +// Returns an error if the streamer is already running. It streams until Stop: +// the end of ctx does not stop it. +func (s *Streamer) Start(ctx context.Context) error { s.mu.Lock() defer s.mu.Unlock() if s.running { - return fmt.Errorf("streamer already running") + return errAlreadyRunning } - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(context.WithoutCancel(ctx)) s.cancel = cancel s.running = true @@ -230,44 +247,21 @@ func (s *Streamer) Stop() { return } + s.cancel() s.cancel = nil // Close all handler queues to signal workers to stop for _, info := range s.handlers { close(info.queue) } + s.running = false s.mu.Unlock() s.metrics.SetConnected(false) } -// runHandlerWorker processes messages for a specific handler -func (s *Streamer) runHandlerWorker(info *handlerInfo) { - for msg := range info.queue { - start := time.Now() - info.handler.HandleMessage(msg) - elapsed := time.Since(start) - - // Update metrics - info.metrics.mu.Lock() - info.metrics.processedCount++ - info.metrics.totalTime += elapsed - - // Update min time - if elapsed < info.metrics.minTime { - info.metrics.minTime = elapsed - } - - // Update max time - if elapsed > info.metrics.maxTime { - info.metrics.maxTime = elapsed - } - info.metrics.mu.Unlock() - } -} - -// IsRunning reports whether the streamer is currently connected and processing messages. -// This is safe to call concurrently from multiple goroutines. +// IsRunning reports whether the streamer is currently connected and processing +// messages. This is safe to call concurrently from multiple goroutines. func (s *Streamer) IsRunning() bool { s.mu.RLock() defer s.mu.RUnlock() @@ -328,12 +322,15 @@ func (s *Streamer) GetHandlerStats() []HandlerStats { // Calculate average time if info.metrics.processedCount > 0 { processedCount := info.metrics.processedCount + const maxInt64 = 1<<63 - 1 + if processedCount > maxInt64 { processedCount = maxInt64 } - //nolint:gosec // processedCount is explicitly bounded above - hs.AvgProcessTime = info.metrics.totalTime / time.Duration(processedCount) + + hs.AvgProcessTime = info.metrics.totalTime / + time.Duration(processedCount) } info.metrics.mu.Unlock() @@ -344,11 +341,38 @@ func (s *Streamer) GetHandlerStats() []HandlerStats { return stats } -// GetDroppedMessages returns the total number of messages dropped across all handlers -// due to queue overflow or backpressure. This counter is monotonically increasing -// and is safe to call concurrently. +// GetDroppedMessages returns the total number of messages dropped across all +// handlers due to queue overflow or backpressure. This counter is monotonically +// increasing and is safe to call concurrently. func (s *Streamer) GetDroppedMessages() uint64 { - return atomic.LoadUint64(&s.totalDropped) + return s.totalDropped.Load() +} + +// runHandlerWorker processes messages for a specific handler +func (s *Streamer) runHandlerWorker(info *handlerInfo) { + for msg := range info.queue { + start := time.Now() + + info.handler.HandleMessage(msg) + + elapsed := time.Since(start) + + // Update metrics + info.metrics.mu.Lock() + info.metrics.processedCount++ + info.metrics.totalTime += elapsed + + // Update min time + if elapsed < info.metrics.minTime { + info.metrics.minTime = elapsed + } + + // Update max time + if elapsed > info.metrics.maxTime { + info.metrics.maxTime = elapsed + } + info.metrics.mu.Unlock() + } } // logMetrics logs the current streaming statistics @@ -357,7 +381,8 @@ func (s *Streamer) logMetrics() { uptime := time.Since(metrics.ConnectedSince) const bitsPerMegabit = 1000000 - totalDropped := atomic.LoadUint64(&s.totalDropped) + + totalDropped := s.totalDropped.Load() s.logger.Info( "Stream statistics", @@ -383,16 +408,19 @@ func (s *Streamer) logMetrics() { // Log per-handler statistics s.mu.RLock() + for i, info := range s.handlers { info.metrics.mu.Lock() if info.metrics.processedCount > 0 { // Safe conversion: processedCount is bounded by maxInt64 processedCount := info.metrics.processedCount + const maxInt64 = 1<<63 - 1 + if processedCount > maxInt64 { processedCount = maxInt64 } - //nolint:gosec // processedCount is explicitly bounded above + avgTime := info.metrics.totalTime / time.Duration(processedCount) s.logger.Info( "Handler statistics", @@ -409,6 +437,7 @@ func (s *Streamer) logMetrics() { } info.metrics.mu.Unlock() } + s.mu.RUnlock() } @@ -417,7 +446,8 @@ func (s *Streamer) updateMetrics(messageBytes int) { s.metrics.RecordMessage(int64(messageBytes)) } -// streamWithReconnect handles streaming with automatic reconnection and exponential backoff +// streamWithReconnect handles streaming with automatic reconnection and +// exponential backoff func (s *Streamer) streamWithReconnect(ctx context.Context) { backoffDelay := s.minBackoff consecutiveFailures := 0 @@ -455,6 +485,7 @@ func (s *Streamer) streamWithReconnect(ctx context.Context) { if streamDuration > 30*time.Second { s.logger.Info("Resetting backoff delay due to successful connection", "stream_duration", streamDuration) + backoffDelay = s.minBackoff consecutiveFailures = 0 } else { @@ -494,7 +525,7 @@ func (s *Streamer) stream(ctx context.Context) error { idleTimer := time.AfterFunc(s.idleTimeout, connCancel) defer idleTimer.Stop() - req, err := http.NewRequestWithContext(connCtx, "GET", s.url, nil) + req, err := http.NewRequestWithContext(connCtx, http.MethodGet, s.url, nil) if err != nil { return fmt.Errorf("failed to create request: %w", err) } @@ -507,13 +538,14 @@ func (s *Streamer) stream(ctx context.Context) error { return fmt.Errorf("failed to connect to RIS Live: %w", err) } defer func() { - if err := resp.Body.Close(); err != nil { + err := resp.Body.Close() + if err != nil { s.logger.Error("Failed to close response body", "error", err) } }() if resp.StatusCode != http.StatusOK { - return fmt.Errorf("unexpected status code: %d", resp.StatusCode) + return fmt.Errorf("%w: %d", errUnexpectedStatus, resp.StatusCode) } // Wrap body with counting reader to track actual wire bytes @@ -527,7 +559,9 @@ func (s *Streamer) stream(ctx context.Context) error { return fmt.Errorf("failed to create gzip reader: %w", err) } defer func() { _ = gzReader.Close() }() + reader = gzReader + s.logger.Info("Connected to RIS Live stream", "compression", "gzip") } else { s.logger.Info("Connected to RIS Live stream", "compression", "none") @@ -535,9 +569,6 @@ func (s *Streamer) stream(ctx context.Context) error { s.metrics.SetConnected(true) - // Track wire bytes for metrics updates - var lastWireBytes int64 - // Start metrics logging goroutine metricsTicker := time.NewTicker(metricsLogInterval) defer metricsTicker.Stop() @@ -554,25 +585,44 @@ func (s *Streamer) stream(ctx context.Context) error { }() // Wire byte update ticker - update metrics with actual wire bytes periodically + go s.recordWireBytes(connCtx, wireCounter) + + return s.readStream(ctx, connCtx, reader, idleTimer) +} + +// recordWireBytes adds the bytes wireCounter has counted to the metrics once a +// second until ctx is cancelled. +func (s *Streamer) recordWireBytes( + ctx context.Context, wireCounter *countingReader, +) { wireUpdateTicker := time.NewTicker(time.Second) defer wireUpdateTicker.Stop() - go func() { - for { - select { - case <-wireUpdateTicker.C: - currentBytes := wireCounter.Count() - delta := currentBytes - lastWireBytes - if delta > 0 { - s.metrics.RecordWireBytes(delta) - lastWireBytes = currentBytes - } - case <-connCtx.Done(): - return - } - } - }() + var lastWireBytes int64 + for { + select { + case <-wireUpdateTicker.C: + currentBytes := wireCounter.Count() + + delta := currentBytes - lastWireBytes + if delta > 0 { + s.metrics.RecordWireBytes(delta) + + lastWireBytes = currentBytes + } + case <-ctx.Done(): + return + } + } +} + +// readStream hands each line of the stream to processLine until the stream +// ends, a line fails, or ctx is cancelled. Each line restarts idleTimer, which +// cancels connCtx when it fires. +func (s *Streamer) readStream( + ctx, connCtx context.Context, reader io.Reader, idleTimer *time.Timer, +) error { scanner := bufio.NewScanner(reader) scanner.Buffer(nil, maxLineSize) @@ -592,129 +642,10 @@ func (s *Streamer) stream(ctx context.Context) error { continue } - // Update metrics with decompressed message size - s.updateMetrics(len(line)) - - // Call raw handler if registered - s.mu.RLock() - rawHandler := s.rawHandler - s.mu.RUnlock() - - if rawHandler != nil { - // Call raw handler synchronously to preserve order - rawHandler(string(line)) + err := s.processLine(ctx, line) + if err != nil { + return err } - - // Parse the message first - var wrapper ristypes.RISLiveMessage - if err := json.Unmarshal(line, &wrapper); err != nil { - // Log the error and return to trigger reconnection - s.logger.Error("Failed to parse JSON", - "error", err, - "line", string(line), - "line_length", len(line)) - - return fmt.Errorf("JSON parse error: %w", err) - } - - // Check if it's a ris_message wrapper - if wrapper.Type != "ris_message" { - s.logger.Error("Unexpected wrapper type", - "type", wrapper.Type, - "line", string(line), - ) - - continue - } - - // Get the actual message - msg := wrapper.Data - - // Parse the timestamp - msg.ParsedTimestamp = time.Unix(int64(msg.Timestamp), 0).UTC() - - // Process based on message type - switch msg.Type { - case ristypes.MessageTypeUpdate: - // Process BGP UPDATE messages - // Will be dispatched to handlers - case "RIS_PEER_STATE": - // RIS peer state messages - silently ignore - continue - case "KEEPALIVE": - // BGP keepalive messages - silently process - continue - case "OPEN": - // BGP open messages - track peer as active - s.bgpPeersMu.Lock() - s.bgpPeers[msg.Peer] = true - peerCount := len(s.bgpPeers) - s.bgpPeersMu.Unlock() - s.metrics.SetBGPPeerCount(peerCount) - - s.logger.Info("BGP session opened", - "peer", msg.Peer, - "peer_asn", msg.PeerASN, - "total_peers", peerCount, - ) - - continue - case "NOTIFICATION": - // BGP notification messages (session closed) - s.bgpPeersMu.Lock() - delete(s.bgpPeers, msg.Peer) - peerCount := len(s.bgpPeers) - s.bgpPeersMu.Unlock() - s.metrics.SetBGPPeerCount(peerCount) - - s.logger.Warn("BGP notification", - "peer", msg.Peer, - "peer_asn", msg.PeerASN, - "total_peers", peerCount, - ) - - continue - case ristypes.MessageTypeState: - // The BGP session with the peer changed state, for example went - // down. Dispatched to the handlers that want it. - default: - s.logger.Warn("Unknown message type, skipping", - "type", msg.Type, - ) - - continue - } - - // Dispatch to interested handlers. Stop cancels ctx and closes the - // queues under the write lock, so if ctx is cancelled here, under the - // read lock, the queues are closed and must not be sent to. - s.mu.RLock() - if ctx.Err() != nil { - s.mu.RUnlock() - - return ctx.Err() - } - for _, info := range s.handlers { - if !info.handler.WantsMessage(msg.Type) { - continue - } - - if !s.queueMessage(info, &msg) { - atomic.AddUint64(&info.metrics.droppedCount, 1) - atomic.AddUint64(&s.totalDropped, 1) - - continue - } - - // Update high water mark if needed - queueLen := len(info.queue) - info.metrics.mu.Lock() - if queueLen > info.metrics.queueHighWaterMark { - info.metrics.queueHighWaterMark = queueLen - } - info.metrics.mu.Unlock() - } - s.mu.RUnlock() } if ctx.Err() != nil { @@ -722,16 +653,161 @@ func (s *Streamer) stream(ctx context.Context) error { } if connCtx.Err() != nil { - return fmt.Errorf("no line received for %s", s.idleTimeout) + return fmt.Errorf("%w for %s", errStreamIdle, s.idleTimeout) } - if err := scanner.Err(); err != nil { + err := scanner.Err() + if err != nil { return fmt.Errorf("scanner error: %w", err) } return errStreamEnded } +// processLine passes one line of the stream to the raw handler, parses it, +// and dispatches an UPDATE or STATE message to the handlers. An error ends the +// connection. +func (s *Streamer) processLine(ctx context.Context, line []byte) error { + // Update metrics with decompressed message size + s.updateMetrics(len(line)) + + // Call raw handler if registered + s.mu.RLock() + rawHandler := s.rawHandler + s.mu.RUnlock() + + if rawHandler != nil { + // Call raw handler synchronously to preserve order + rawHandler(string(line)) + } + + // Parse the message first + var wrapper ristypes.RISLiveMessage + + err := json.Unmarshal(line, &wrapper) + if err != nil { + // Log the error and return to trigger reconnection + s.logger.Error("Failed to parse JSON", + "error", err, + "line", string(line), + "line_length", len(line)) + + return fmt.Errorf("JSON parse error: %w", err) + } + + // Check if it's a ris_message wrapper + if wrapper.Type != "ris_message" { + s.logger.Error("Unexpected wrapper type", + "type", wrapper.Type, + "line", string(line), + ) + + return nil + } + + // Get the actual message + msg := wrapper.Data + + // Parse the timestamp + msg.ParsedTimestamp = time.Unix(int64(msg.Timestamp), 0).UTC() + + // Process based on message type + switch msg.Type { + case ristypes.MessageTypeUpdate: + // Process BGP UPDATE messages + // Will be dispatched to handlers + case "RIS_PEER_STATE": + // RIS peer state messages - silently ignore + return nil + case "KEEPALIVE": + // BGP keepalive messages - silently process + return nil + case "OPEN": + // BGP open messages - track peer as active + s.bgpPeersMu.Lock() + s.bgpPeers[msg.Peer] = true + peerCount := len(s.bgpPeers) + s.bgpPeersMu.Unlock() + s.metrics.SetBGPPeerCount(peerCount) + + s.logger.Info("BGP session opened", + "peer", msg.Peer, + "peer_asn", msg.PeerASN, + "total_peers", peerCount, + ) + + return nil + case "NOTIFICATION": + // BGP notification messages (session closed) + s.bgpPeersMu.Lock() + delete(s.bgpPeers, msg.Peer) + peerCount := len(s.bgpPeers) + s.bgpPeersMu.Unlock() + s.metrics.SetBGPPeerCount(peerCount) + + s.logger.Warn("BGP notification", + "peer", msg.Peer, + "peer_asn", msg.PeerASN, + "total_peers", peerCount, + ) + + return nil + case ristypes.MessageTypeState: + // The BGP session with the peer changed state, for example went + // down. Dispatched to the handlers that want it. + default: + s.logger.Warn("Unknown message type, skipping", + "type", msg.Type, + ) + + return nil + } + + return s.dispatchMessage(ctx, &msg) +} + +// dispatchMessage queues msg for each handler that wants it, unless +// queueMessage drops it for that handler. +func (s *Streamer) dispatchMessage( + ctx context.Context, msg *ristypes.RISMessage, +) error { + // Stop cancels ctx and closes the queues under the write lock, so if ctx + // is cancelled here, under the read lock, the queues are closed and must + // not be sent to. + s.mu.RLock() + + if ctx.Err() != nil { + s.mu.RUnlock() + + return ctx.Err() + } + + for _, info := range s.handlers { + if !info.handler.WantsMessage(msg.Type) { + continue + } + + if !s.queueMessage(info, msg) { + atomic.AddUint64(&info.metrics.droppedCount, 1) + s.totalDropped.Add(1) + + continue + } + + // Update high water mark if needed + queueLen := len(info.queue) + info.metrics.mu.Lock() + if queueLen > info.metrics.queueHighWaterMark { + info.metrics.queueHighWaterMark = queueLen + } + info.metrics.mu.Unlock() + } + + s.mu.RUnlock() + + return nil +} + // queueMessage puts msg on the handler's queue and reports false if it dropped // msg instead. It never waits for room, so a full queue holds up neither the // read loop nor Stop. @@ -746,7 +822,9 @@ func (s *Streamer) stream(ctx context.Context) error { // // The held list stays small: a STATE message is a few hundred bytes and the // feed sends about ten a second, so a queue full for a minute holds about 600. -func (s *Streamer) queueMessage(info *handlerInfo, msg *ristypes.RISMessage) bool { +func (s *Streamer) queueMessage( + info *handlerInfo, msg *ristypes.RISMessage, +) bool { for len(info.heldState) > 0 && trySend(info.queue, info.heldState[0]) { info.heldState = info.heldState[1:] } @@ -776,7 +854,8 @@ func trySend(queue chan *ristypes.RISMessage, msg *ristypes.RISMessage) bool { } } -// shouldDropForBackpressure determines if a message should be dropped based on queue utilization +// shouldDropForBackpressure determines if a message should be dropped based on +// queue utilization func (s *Streamer) shouldDropForBackpressure(info *handlerInfo) bool { // Calculate queue utilization queueLen := len(info.queue) diff --git a/internal/streamer/streamer_test.go b/internal/streamer/streamer_internal_test.go similarity index 78% rename from internal/streamer/streamer_test.go rename to internal/streamer/streamer_internal_test.go index 42e1f18..e3b545e 100644 --- a/internal/streamer/streamer_test.go +++ b/internal/streamer/streamer_internal_test.go @@ -18,6 +18,8 @@ import ( ) func TestNewStreamer(t *testing.T) { + t.Parallel() + logger := logger.New() metricsTracker := metrics.New() s := New(logger, metricsTracker) @@ -56,10 +58,13 @@ func TestNewStreamer(t *testing.T) { // goroutine count returns to its starting value. Each connection starts two // ticker goroutines; before the fix they lived until the streamer's lifetime // context was cancelled, so every reconnect leaked two. +// +//nolint:paralleltest // counts goroutines, so no other test may run alongside func TestStreamDoesNotLeakTickersAcrossReconnects(t *testing.T) { // The handler returns immediately, so the response body is empty and each // stream call ends at once, standing in for a dropped connection. - srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {})) + srv := httptest.NewServer(http.HandlerFunc( + func(_ http.ResponseWriter, _ *http.Request) {})) defer srv.Close() s := New(logger.New(), metrics.New()) @@ -67,19 +72,23 @@ func TestStreamDoesNotLeakTickersAcrossReconnects(t *testing.T) { // One warm-up connection so any persistent HTTP transport goroutine exists // before we take the baseline. - if err := s.stream(context.Background()); !errors.Is(err, errStreamEnded) { + err := s.stream(context.Background()) + if !errors.Is(err, errStreamEnded) { t.Fatalf("warm-up stream returned %v, want %v", err, errStreamEnded) } + s.client.CloseIdleConnections() baseline := settledGoroutineCount() const reconnects = 20 for range reconnects { - if err := s.stream(context.Background()); !errors.Is(err, errStreamEnded) { + err := s.stream(context.Background()) + if !errors.Is(err, errStreamEnded) { t.Fatalf("stream returned %v, want %v", err, errStreamEnded) } } + s.client.CloseIdleConnections() if !waitForGoroutines(baseline) { @@ -89,8 +98,8 @@ func TestStreamDoesNotLeakTickersAcrossReconnects(t *testing.T) { } // updateLine is one UPDATE message as RIS Live sends it. -const updateLine = `{"type":"ris_message","data":{"type":"UPDATE","peer":"192.0.2.1",` + - `"peer_asn":"64496","timestamp":1700000000}}` + "\n" +const updateLine = `{"type":"ris_message","data":{"type":"UPDATE",` + + `"peer":"192.0.2.1","peer_asn":"64496","timestamp":1700000000}}` + "\n" // updateHandler wants UPDATE messages. For each one it handles it sends on // handled if there is room, and otherwise does nothing; a nil handled is @@ -99,8 +108,11 @@ type updateHandler struct { handled chan struct{} } -func (updateHandler) WantsMessage(messageType string) bool { return messageType == "UPDATE" } -func (updateHandler) QueueCapacity() int { return 10 } +func (updateHandler) WantsMessage(messageType string) bool { + return messageType == "UPDATE" +} + +func (updateHandler) QueueCapacity() int { return 10 } func (h updateHandler) HandleMessage(*ristypes.RISMessage) { select { @@ -109,11 +121,21 @@ func (h updateHandler) HandleMessage(*ristypes.RISMessage) { } } +// flush sends what w has buffered to the client. The writers httptest servers +// hand to handlers can always do that. +func flush(w http.ResponseWriter) { + flusher, ok := w.(http.Flusher) + if ok { + flusher.Flush() + } +} + // stateDownLine is the STATE message RIS Live sends when the collector's BGP // session with the peer stateDownPeer goes down. const ( stateDownPeer = "192.0.2.1" - stateDownLine = `{"type":"ris_message","data":{"type":"STATE","peer":"` + stateDownPeer + `",` + + stateDownLine = `{"type":"ris_message","data":{"type":"STATE",` + + `"peer":"` + stateDownPeer + `",` + `"peer_asn":"64496","timestamp":1700000000,"state":"down"}}` + "\n" ) @@ -124,31 +146,39 @@ type stateHandler struct{} func (stateHandler) WantsMessage(messageType string) bool { return messageType == ristypes.MessageTypeState } -func (stateHandler) QueueCapacity() int { return 10 } + +func (stateHandler) QueueCapacity() int { return 10 } + func (stateHandler) HandleMessage(*ristypes.RISMessage) {} // TestStreamHandsStateMessagesToHandlersThatWantThem has the server send one // STATE message whose state is down. The streamer must queue it, with its // peer and state, for a handler that wants STATE messages. func TestStreamHandsStateMessagesToHandlersThatWantThem(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = io.WriteString(w, stateDownLine) - })) + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, stateDownLine) + })) defer srv.Close() s := New(logger.New(), metrics.New()) s.url = srv.URL s.RegisterHandler(stateHandler{}) - if err := s.stream(context.Background()); !errors.Is(err, errStreamEnded) { + err := s.stream(context.Background()) + if !errors.Is(err, errStreamEnded) { t.Fatalf("stream returned %v, want %v", err, errStreamEnded) } select { case msg := <-s.handlers[0].queue: - if msg.Type != ristypes.MessageTypeState || msg.Peer != stateDownPeer || msg.State != ristypes.StateDown { - t.Errorf("queued message has type %q, peer %q, state %q; want %q, %q, %q", - msg.Type, msg.Peer, msg.State, ristypes.MessageTypeState, stateDownPeer, ristypes.StateDown) + if msg.Type != ristypes.MessageTypeState || msg.Peer != stateDownPeer || + msg.State != ristypes.StateDown { + t.Errorf("queued message has type %q, peer %q, state %q; "+ + "want %q, %q, %q", msg.Type, msg.Peer, msg.State, + ristypes.MessageTypeState, stateDownPeer, ristypes.StateDown) } default: t.Fatal("the STATE message was not queued for the handler that wants it") @@ -161,26 +191,37 @@ func TestStreamHandsStateMessagesToHandlersThatWantThem(t *testing.T) { // everything queued before it and before anything handed over after it. The // update handed over while a STATE message is held is dropped. func TestHeldStateMessagesKeepTheirPlaceInTheQueue(t *testing.T) { + t.Parallel() + s := New(logger.New(), metrics.New()) s.RegisterHandler(stateHandler{}) + info := s.handlers[0] before := &ristypes.RISMessage{Type: ristypes.MessageTypeUpdate} for len(info.queue) < cap(info.queue) { info.queue <- before } - firstDown := &ristypes.RISMessage{Type: ristypes.MessageTypeState, State: ristypes.StateDown} - secondDown := &ristypes.RISMessage{Type: ristypes.MessageTypeState, State: ristypes.StateDown} + + firstDown := &ristypes.RISMessage{ + Type: ristypes.MessageTypeState, State: ristypes.StateDown, + } + secondDown := &ristypes.RISMessage{ + Type: ristypes.MessageTypeState, State: ristypes.StateDown, + } updateWhileHeld := &ristypes.RISMessage{Type: ristypes.MessageTypeUpdate} after := &ristypes.RISMessage{Type: ristypes.MessageTypeUpdate} if !s.queueMessage(info, firstDown) { t.Fatal("the first STATE message was dropped on a full queue") } + <-info.queue // the handler makes room for one message + if !s.queueMessage(info, secondDown) { t.Fatal("the second STATE message was dropped") } + if s.queueMessage(info, updateWhileHeld) { t.Fatal("an update was queued while a STATE message was held") } @@ -191,6 +232,7 @@ func TestHeldStateMessagesKeepTheirPlaceInTheQueue(t *testing.T) { t.Fatal("a message overtook those queued before the first STATE message") } } + if !s.queueMessage(info, after) { t.Fatal("an update was dropped on a queue with room") } @@ -198,12 +240,16 @@ func TestHeldStateMessagesKeepTheirPlaceInTheQueue(t *testing.T) { if <-info.queue != firstDown { t.Fatal("the first STATE message did not come next") } + if <-info.queue != secondDown { t.Fatal("the second STATE message did not come after the first") } + if <-info.queue != after { - t.Fatal("the update handed over after the STATE messages did not come after them") + t.Fatal("the update handed over after the STATE messages did not come " + + "after them") } + if len(info.queue) != 0 { t.Errorf("%d messages left on the queue, want none", len(info.queue)) } @@ -215,6 +261,7 @@ func newStreamerWithFullStateQueue(url string) *Streamer { s := New(logger.New(), metrics.New()) s.url = url s.RegisterHandler(stateHandler{}) + queue := s.handlers[0].queue for len(queue) < cap(queue) { queue <- &ristypes.RISMessage{Type: ristypes.MessageTypeState} @@ -229,24 +276,33 @@ func newStreamerWithFullStateQueue(url string) *Streamer { // for that handler, and the read loop must go on to queue the UPDATE for the // handler that wants it. func TestOtherHandlersKeepReceivingWhileAStateMessageIsHeld(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = io.WriteString(w, stateDownLine+updateLine) - })) + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, stateDownLine+updateLine) + })) defer srv.Close() s := newStreamerWithFullStateQueue(srv.URL) s.RegisterHandler(updateHandler{}) - if err := s.stream(context.Background()); !errors.Is(err, errStreamEnded) { + err := s.stream(context.Background()) + if !errors.Is(err, errStreamEnded) { t.Fatalf("stream returned %v, want %v", err, errStreamEnded) } held := s.handlers[0].heldState - if len(held) != 1 || held[0].Peer != stateDownPeer || held[0].State != ristypes.StateDown { - t.Errorf("held %d STATE messages, want the down from %s alone", len(held), stateDownPeer) + if len(held) != 1 || held[0].Peer != stateDownPeer || + held[0].State != ristypes.StateDown { + t.Errorf("held %d STATE messages, want the down from %s alone", + len(held), stateDownPeer) } - if got := len(s.handlers[1].queue); got != 1 { - t.Errorf("%d messages queued for the handler that wants UPDATE messages, want 1", got) + + got := len(s.handlers[1].queue) + if got != 1 { + t.Errorf("%d messages queued for the handler that wants UPDATE "+ + "messages, want 1", got) } } @@ -254,21 +310,27 @@ func TestOtherHandlersKeepReceivingWhileAStateMessageIsHeld(t *testing.T) { // messages and hold the response open while the queue of the handler that // wants them is full. Stop must return at once and end the stream. func TestStopReturnsAtOnceWithStateMessagesHeld(t *testing.T) { + t.Parallel() + const ( lines = 2 maxWait = 5 * time.Second ) - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = io.WriteString(w, stateDownLine+stateDownLine) - w.(http.Flusher).Flush() - <-r.Context().Done() - })) + + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, stateDownLine+stateDownLine) + + flush(w) + <-r.Context().Done() + })) defer srv.Close() s := newStreamerWithFullStateQueue(srv.URL) // The raw handler sees each line before the read loop hands it to the // queues, so once it has seen the second line, the first is held. linesSeen := make(chan struct{}, lines) + s.RegisterRawHandler(func(string) { linesSeen <- struct{}{} }) // As in TestStopBeforeMessageReachesQueues, setting cancel as Start does @@ -276,25 +338,30 @@ func TestStopReturnsAtOnceWithStateMessagesHeld(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) s.cancel = cancel streamErr := make(chan error, 1) + go func() { streamErr <- s.stream(ctx) }() for range lines { select { case <-linesSeen: case <-time.After(maxWait): - t.Fatalf("the read loop did not read %d lines within %s", lines, maxWait) + t.Fatalf("the read loop did not read %d lines within %s", + lines, maxWait) } } stopped := make(chan struct{}) + go func() { s.Stop() close(stopped) }() + select { case <-stopped: case <-time.After(maxWait): - t.Fatalf("Stop did not return within %s while a STATE message was held", maxWait) + t.Fatalf("Stop did not return within %s while a STATE message was held", + maxWait) } select { @@ -313,9 +380,12 @@ func TestStopReturnsAtOnceWithStateMessagesHeld(t *testing.T) { // used to end in "send on closed channel". The raw handler runs in that gap on // the read loop itself, so calling Stop from it hits the gap every time. func TestStopBeforeMessageReachesQueues(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = io.WriteString(w, updateLine) - })) + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, updateLine) + })) defer srv.Close() s := New(logger.New(), metrics.New()) @@ -329,7 +399,8 @@ func TestStopBeforeMessageReachesQueues(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) s.cancel = cancel - if err := s.stream(ctx); !errors.Is(err, context.Canceled) { + err := s.stream(ctx) + if !errors.Is(err, context.Canceled) { t.Fatalf("stream returned %v, want %v", err, context.Canceled) } @@ -341,6 +412,8 @@ func TestStopBeforeMessageReachesQueues(t *testing.T) { // normally, as when RIS Live restarts. The streamer must treat that as a // disconnect and connect again. func TestReconnectWhenStreamEnds(t *testing.T) { + t.Parallel() + endResponse := func(http.ResponseWriter, *http.Request) {} checkReconnect(t, endResponse, streamIdleTimeout) } @@ -349,9 +422,12 @@ func TestReconnectWhenStreamEnds(t *testing.T) { // open without sending anything. The streamer must drop it once the idle // timeout passes and connect again. func TestReconnectWhenStreamGoesIdle(t *testing.T) { + t.Parallel() + const idleTimeout = 50 * time.Millisecond + holdResponse := func(w http.ResponseWriter, r *http.Request) { - w.(http.Flusher).Flush() + flush(w) <-r.Context().Done() } checkReconnect(t, holdResponse, idleTimeout) @@ -361,7 +437,10 @@ func TestReconnectWhenStreamGoesIdle(t *testing.T) { // request and never answer it, not even with headers. The streamer must give // up on it once the idle timeout passes and connect again. func TestReconnectWhenServerNeverAnswers(t *testing.T) { + t.Parallel() + const idleTimeout = 50 * time.Millisecond + neverAnswer := func(_ http.ResponseWriter, r *http.Request) { <-r.Context().Done() } @@ -373,26 +452,32 @@ func TestReconnectWhenServerNeverAnswers(t *testing.T) { // line must start the idle timeout over, so the stream lasts until the server // ends it instead of being dropped as idle. func TestStreamNotDroppedWhileLinesArrive(t *testing.T) { + t.Parallel() + const ( idleTimeout = 100 * time.Millisecond lineInterval = idleTimeout / 10 sendFor = 5 * idleTimeout ) - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - deadline := time.Now().Add(sendFor) - for time.Now().Before(deadline) { - _, _ = io.WriteString(w, updateLine) - w.(http.Flusher).Flush() - time.Sleep(lineInterval) - } - })) + + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + deadline := time.Now().Add(sendFor) + for time.Now().Before(deadline) { + _, _ = io.WriteString(w, updateLine) + + flush(w) + time.Sleep(lineInterval) + } + })) defer srv.Close() s := New(logger.New(), metrics.New()) s.url = srv.URL s.idleTimeout = idleTimeout - if err := s.stream(context.Background()); !errors.Is(err, errStreamEnded) { + err := s.stream(context.Background()) + if !errors.Is(err, errStreamEnded) { t.Fatalf("stream returned %v, want %v", err, errStreamEnded) } } @@ -404,24 +489,31 @@ func TestStreamNotDroppedWhileLinesArrive(t *testing.T) { // handler's queue and read on until the server ends the stream, instead of // dropping the connection at the long line. func TestStreamReadsLongLine(t *testing.T) { + t.Parallel() + const rawHexLength = 200 * bytesPerKB + longLine := `{"type":"ris_message","data":{"type":"UPDATE","peer":"192.0.2.1",` + `"peer_asn":"64496","timestamp":1700000000,"raw":"` + strings.Repeat("0", rawHexLength) + `"}}` + "\n" - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = io.WriteString(w, longLine) - })) + + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, longLine) + })) defer srv.Close() s := New(logger.New(), metrics.New()) s.url = srv.URL s.RegisterHandler(updateHandler{}) - if err := s.stream(context.Background()); !errors.Is(err, errStreamEnded) { + err := s.stream(context.Background()) + if !errors.Is(err, errStreamEnded) { t.Fatalf("stream returned %v, want %v", err, errStreamEnded) } - if queued := len(s.handlers[0].queue); queued != 1 { + queued := len(s.handlers[0].queue) + if queued != 1 { t.Errorf("handler queue holds %d messages, want 1", queued) } } @@ -431,28 +523,36 @@ func TestStreamReadsLongLine(t *testing.T) { // line and then stays open. The streamer must come back for a second // response, must not be marked connected when it does, and must handle the // UPDATE in it. -func checkReconnect(t *testing.T, firstResponse http.HandlerFunc, idleTimeout time.Duration) { +func checkReconnect( + t *testing.T, firstResponse http.HandlerFunc, idleTimeout time.Duration, +) { t.Helper() const maxWait = 5 * time.Second tracker := metrics.New() connectedAtReconnect := make(chan bool, 1) - var requests atomic.Int32 - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if requests.Add(1) == 1 { - firstResponse(w, r) - return - } - select { - case connectedAtReconnect <- tracker.IsConnected(): - default: - } - _, _ = io.WriteString(w, updateLine) - w.(http.Flusher).Flush() - <-r.Context().Done() - })) + var requests atomic.Int32 + + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + if requests.Add(1) == 1 { + firstResponse(w, r) + + return + } + + select { + case connectedAtReconnect <- tracker.IsConnected(): + default: + } + + _, _ = io.WriteString(w, updateLine) + + flush(w) + <-r.Context().Done() + })) defer srv.Close() s := New(logger.New(), tracker) @@ -461,7 +561,9 @@ func checkReconnect(t *testing.T, firstResponse http.HandlerFunc, idleTimeout ti s.minBackoff = time.Millisecond handler := updateHandler{handled: make(chan struct{}, 1)} s.RegisterHandler(handler) - if err := s.Start(); err != nil { + + err := s.Start(t.Context()) + if err != nil { t.Fatalf("Start returned error: %v", err) } defer s.Stop() @@ -473,7 +575,8 @@ func checkReconnect(t *testing.T, firstResponse http.HandlerFunc, idleTimeout ti } if <-connectedAtReconnect { - t.Error("streamer was still marked connected when it came back for a second response") + t.Error("streamer was still marked connected when it came back " + + "for a second response") } } @@ -481,12 +584,15 @@ func checkReconnect(t *testing.T, firstResponse http.HandlerFunc, idleTimeout ti // current count. func settledGoroutineCount() int { prev := runtime.NumGoroutine() + for range 20 { time.Sleep(10 * time.Millisecond) + cur := runtime.NumGoroutine() if cur == prev { return cur } + prev = cur } @@ -499,6 +605,7 @@ func waitForGoroutines(target int) bool { if runtime.NumGoroutine() <= target { return true } + time.Sleep(10 * time.Millisecond) } diff --git a/internal/templates/templates.go b/internal/templates/templates.go index b388343..16d6737 100644 --- a/internal/templates/templates.go +++ b/internal/templates/templates.go @@ -60,6 +60,7 @@ func timeSince(t time.Time) string { if duration < time.Minute { return "just now" } + if duration < time.Hour { minutes := int(duration.Minutes()) if minutes == 1 { @@ -68,6 +69,7 @@ func timeSince(t time.Time) string { return duration.Truncate(time.Minute).String() + " ago" } + if duration < hoursPerDay*time.Hour { hours := int(duration.Hours()) if hours == 1 { @@ -76,10 +78,12 @@ func timeSince(t time.Time) string { return duration.Truncate(time.Hour).String() + " ago" } + days := int(duration.Hours() / hoursPerDay) if days == 1 { return "1 day ago" } + if days < daysPerMonth { return duration.Truncate(hoursPerDay*time.Hour).String() + " ago" } @@ -118,7 +122,7 @@ func initTemplates() { "appLicense": func() string { return version.License }, "appRepoURL": func() string { return version.RepoURL }, "appGitRevision": func() string { return version.GitRevisionShort }, - "appGitCommitURL": func() string { return version.CommitURL() }, + "appGitCommitURL": version.CommitURL, } // Parse index template @@ -134,19 +138,22 @@ func initTemplates() { } // Parse AS detail template - defaultTemplates.ASDetail, err = template.New("asDetail").Funcs(funcs).Parse(asDetailHTML) + defaultTemplates.ASDetail, err = template.New("asDetail"). + Funcs(funcs).Parse(asDetailHTML) if err != nil { panic("failed to parse AS detail template: " + err.Error()) } // Parse prefix detail template - defaultTemplates.PrefixDetail, err = template.New("prefixDetail").Funcs(funcs).Parse(prefixDetailHTML) + defaultTemplates.PrefixDetail, err = template.New("prefixDetail"). + Funcs(funcs).Parse(prefixDetailHTML) if err != nil { panic("failed to parse prefix detail template: " + err.Error()) } // Parse prefix length template - defaultTemplates.PrefixLength, err = template.New("prefixLength").Funcs(funcs).Parse(prefixLengthHTML) + defaultTemplates.PrefixLength, err = template.New("prefixLength"). + Funcs(funcs).Parse(prefixLengthHTML) if err != nil { panic("failed to parse prefix length template: " + err.Error()) } diff --git a/internal/version/version.go b/internal/version/version.go index b32b2d8..20c44d5 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -3,7 +3,7 @@ package version // Build-time variables set via ldflags // -//nolint:gochecknoglobals // These must be variables to allow ldflags injection at build time +//nolint:gochecknoglobals // variables so that ldflags can set them at build time var ( // GitRevision is the git commit hash GitRevision = "unknown" diff --git a/internal/whois/whois.go b/internal/whois/whois.go index 1c4e55b..3713c9f 100644 --- a/internal/whois/whois.go +++ b/internal/whois/whois.go @@ -98,7 +98,8 @@ func (c *Client) LookupASN(ctx context.Context, asn int) (*ASNInfo, error) { if err != nil { // Return partial data from IANA if RIR query fails info := c.parseResponse(asn, rir, ianaResp) - info.RawResponse = ianaResp + "\n--- RIR query failed: " + err.Error() + " ---\n" + info.RawResponse = fmt.Sprintf("%s\n--- RIR query failed: %v ---\n", + ianaResp, err) return info, nil } @@ -120,28 +121,33 @@ func (c *Client) query(ctx context.Context, server, query string) (string, error defer func() { _ = conn.Close() }() // Set deadlines - if err := conn.SetWriteDeadline(time.Now().Add(writeTimeout)); err != nil { + err = conn.SetWriteDeadline(time.Now().Add(writeTimeout)) + if err != nil { return "", fmt.Errorf("set write deadline: %w", err) } // Send query - if _, err := fmt.Fprintf(conn, "%s\r\n", query); err != nil { + _, err = fmt.Fprintf(conn, "%s\r\n", query) + if err != nil { return "", fmt.Errorf("write query: %w", err) } // Read response - if err := conn.SetReadDeadline(time.Now().Add(readTimeout)); err != nil { + err = conn.SetReadDeadline(time.Now().Add(readTimeout)) + if err != nil { return "", fmt.Errorf("set read deadline: %w", err) } var sb strings.Builder + scanner := bufio.NewScanner(conn) for scanner.Scan() { sb.WriteString(scanner.Text()) sb.WriteString("\n") } - if err := scanner.Err(); err != nil { + err = scanner.Err() + if err != nil { return sb.String(), fmt.Errorf("read response: %w", err) } @@ -149,10 +155,10 @@ func (c *Client) query(ctx context.Context, server, query string) (string, error } // parseIANAReferral extracts the RIR and WHOIS server from an IANA response. -func (c *Client) parseIANAReferral(response string) (rir, whoisServer string) { - lines := strings.Split(response, "\n") +func (c *Client) parseIANAReferral(response string) (string, string) { + var rir string - for _, line := range lines { + for line := range strings.SplitSeq(response, "\n") { line = strings.TrimSpace(line) // Look for "refer:" line @@ -183,18 +189,9 @@ func (c *Client) parseIANAReferral(response string) (rir, whoisServer string) { // Also check organisation line for RIR hints if strings.HasPrefix(strings.ToLower(line), "organisation:") { - org := strings.ToLower(line) - switch { - case strings.Contains(org, "arin"): - rir = RIRARIN - case strings.Contains(org, "ripe"): - rir = RIRRIPE - case strings.Contains(org, "apnic"): - rir = RIRAPNIC - case strings.Contains(org, "lacnic"): - rir = RIRLACNIC - case strings.Contains(org, "afrinic"): - rir = RIRAFRNIC + orgRIR := rirInOrganisationLine(strings.ToLower(line)) + if orgRIR != "" { + rir = orgRIR } } } @@ -202,6 +199,25 @@ func (c *Client) parseIANAReferral(response string) (rir, whoisServer string) { return rir, "" } +// rirInOrganisationLine returns the RIR a lowercased IANA "organisation:" +// line names, or "" when it names none. +func rirInOrganisationLine(org string) string { + switch { + case strings.Contains(org, "arin"): + return RIRARIN + case strings.Contains(org, "ripe"): + return RIRRIPE + case strings.Contains(org, "apnic"): + return RIRAPNIC + case strings.Contains(org, "lacnic"): + return RIRLACNIC + case strings.Contains(org, "afrinic"): + return RIRAFRNIC + default: + return "" + } +} + // parseResponse extracts ASN information from a WHOIS response. func (c *Client) parseResponse(asn int, rir, response string) *ASNInfo { info := &ASNInfo{ @@ -211,11 +227,13 @@ func (c *Client) parseResponse(asn int, rir, response string) *ASNInfo { } lines := strings.Split(response, "\n") + var addressLines []string for _, line := range lines { line = strings.TrimSpace(line) - if line == "" || strings.HasPrefix(line, "%") || strings.HasPrefix(line, "#") { + if line == "" || strings.HasPrefix(line, "%") || + strings.HasPrefix(line, "#") { continue } @@ -232,63 +250,14 @@ func (c *Client) parseResponse(asn int, rir, response string) *ASNInfo { continue } - switch key { - // AS Name (varies by RIR) - case "asname", "as-name": - if info.ASName == "" { - info.ASName = value - } - - // Organization - case "orgname", "org-name", "owner": - if info.OrgName == "" { - info.OrgName = value - } - case "orgid", "org-id", "org": - if info.OrgID == "" { - info.OrgID = value - } - // Address (collect multiple lines) - case "address": + if key == "address" { addressLines = append(addressLines, value) - // Country - case "country": - if info.CountryCode == "" && len(value) == 2 { - info.CountryCode = strings.ToUpper(value) - } - - // Abuse contact - case "orgabuseemail", "abuse-mailbox": - if info.AbuseEmail == "" { - info.AbuseEmail = value - } - case "orgabusephone": - if info.AbusePhone == "" { - info.AbusePhone = value - } - - // Tech contact - case "orgtechemail": - if info.TechEmail == "" { - info.TechEmail = value - } - case "orgtechphone": - if info.TechPhone == "" { - info.TechPhone = value - } - - // Registration dates - case "regdate", "created": - if info.RegDate == nil { - info.RegDate = c.parseDate(value) - } - case "updated", "last-modified", "changed": - if info.LastMod == nil { - info.LastMod = c.parseDate(value) - } + continue } + + c.setResponseField(info, key, value) } // Combine address lines @@ -304,6 +273,65 @@ func (c *Client) parseResponse(asn int, rir, response string) *ASNInfo { return info } +// setResponseField stores the value of one "key: value" line of a WHOIS +// response in info, keeping the first value seen for each field. +func (c *Client) setResponseField(info *ASNInfo, key, value string) { + switch key { + // Country + case "country": + if info.CountryCode == "" && len(value) == 2 { + info.CountryCode = strings.ToUpper(value) + } + + // Registration dates + case "regdate", "created": + if info.RegDate == nil { + info.RegDate = c.parseDate(value) + } + case "updated", "last-modified", "changed": + if info.LastMod == nil { + info.LastMod = c.parseDate(value) + } + + default: + field := textFieldForKey(info, key) + if field != nil && *field == "" { + *field = value + } + } +} + +// textFieldForKey returns the text field of info that a WHOIS response key +// fills, or nil when the key fills none. +func textFieldForKey(info *ASNInfo, key string) *string { + switch key { + // AS Name (varies by RIR) + case "asname", "as-name": + return &info.ASName + + // Organization + case "orgname", "org-name", "owner": + return &info.OrgName + case "orgid", "org-id", "org": + return &info.OrgID + + // Abuse contact + case "orgabuseemail", "abuse-mailbox": + return &info.AbuseEmail + case "orgabusephone": + return &info.AbusePhone + + // Tech contact + case "orgtechemail": + return &info.TechEmail + case "orgtechphone": + return &info.TechPhone + + default: + return nil + } +} + // parseDate attempts to parse various date formats used in WHOIS responses. func (c *Client) parseDate(value string) *time.Time { // Common formats @@ -319,15 +347,15 @@ func (c *Client) parseDate(value string) *time.Time { value = strings.TrimSpace(value) // Handle "YYYYMMDD" format from LACNIC if len(value) == lacnicDateFormatLen { - if _, err := time.Parse("20060102", value); err == nil { - t, _ := time.Parse("20060102", value) - + t, err := time.Parse("20060102", value) + if err == nil { return &t } } for _, format := range formats { - if t, err := time.Parse(format, value); err == nil { + t, err := time.Parse(format, value) + if err == nil { return &t } } diff --git a/package.json b/package.json new file mode 100644 index 0000000..fb8b622 --- /dev/null +++ b/package.json @@ -0,0 +1,6 @@ +{ + "private": true, + "devDependencies": { + "prettier": "3.8.1" + } +} diff --git a/pkg/asinfo/asinfo.go b/pkg/asinfo/asinfo.go index 500ec20..372f2e1 100644 --- a/pkg/asinfo/asinfo.go +++ b/pkg/asinfo/asinfo.go @@ -1,4 +1,3 @@ -// Package asinfo provides information about Autonomous Systems (AS) package asinfo import ( @@ -54,7 +53,9 @@ func initRegistry() { } var asInfos []Info - if err := json.Unmarshal(jsonData, &asInfos); err != nil { + + err = json.Unmarshal(jsonData, &asInfos) + if err != nil { panic("failed to unmarshal embedded AS data: " + err.Error()) } @@ -137,6 +138,7 @@ func Search(query string) []Info { defer defaultRegistry.mu.RUnlock() var results []Info + for _, info := range defaultRegistry.byASN { if contains(info.Handle, query) || contains(info.Description, query) { results = append(results, *info) diff --git a/pkg/asinfo/asinfo_test.go b/pkg/asinfo/asinfo_test.go index 5d8c205..3dabfa6 100644 --- a/pkg/asinfo/asinfo_test.go +++ b/pkg/asinfo/asinfo_test.go @@ -1,10 +1,15 @@ -package asinfo +package asinfo_test import ( + "strings" "testing" + + "git.eeqj.de/sneak/routewatch/pkg/asinfo" ) func TestGet(t *testing.T) { + t.Parallel() + tests := []struct { name string asn int @@ -38,20 +43,26 @@ func TestGet(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - info, ok := Get(tt.asn) + t.Parallel() + + info, ok := asinfo.Get(tt.asn) if ok != tt.wantOK { t.Errorf("Get(%d) ok = %v, want %v", tt.asn, ok, tt.wantOK) + return } if ok && info.Description != tt.wantDesc { - t.Errorf("Get(%d) description = %q, want %q", tt.asn, info.Description, tt.wantDesc) + t.Errorf("Get(%d) description = %q, want %q", + tt.asn, info.Description, tt.wantDesc) } }) } } func TestGetDescription(t *testing.T) { + t.Parallel() + tests := []struct { asn int want string @@ -62,7 +73,10 @@ func TestGetDescription(t *testing.T) { for _, tt := range tests { t.Run("", func(t *testing.T) { - if got := GetDescription(tt.asn); got != tt.want { + t.Parallel() + + got := asinfo.GetDescription(tt.asn) + if got != tt.want { t.Errorf("GetDescription(%d) = %q, want %q", tt.asn, got, tt.want) } }) @@ -70,6 +84,8 @@ func TestGetDescription(t *testing.T) { } func TestGetHandle(t *testing.T) { + t.Parallel() + tests := []struct { asn int want string @@ -81,7 +97,10 @@ func TestGetHandle(t *testing.T) { for _, tt := range tests { t.Run("", func(t *testing.T) { - if got := GetHandle(tt.asn); got != tt.want { + t.Parallel() + + got := asinfo.GetHandle(tt.asn) + if got != tt.want { t.Errorf("GetHandle(%d) = %q, want %q", tt.asn, got, tt.want) } }) @@ -89,18 +108,23 @@ func TestGetHandle(t *testing.T) { } func TestTotal(t *testing.T) { - total := Total() + t.Parallel() + + total := asinfo.Total() if total < 100000 { t.Errorf("Total() = %d, expected > 100000", total) } // Verify it's consistent - if total2 := Total(); total2 != total { + total2 := asinfo.Total() + if total2 != total { t.Errorf("Total() returned different values: %d vs %d", total, total2) } } func TestSearch(t *testing.T) { + t.Parallel() + tests := []struct { name string query string @@ -130,16 +154,21 @@ func TestSearch(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - results := Search(tt.query) + t.Parallel() + + results := asinfo.Search(tt.query) if len(results) < tt.wantMin { - t.Errorf("Search(%q) returned %d results, want at least %d", tt.query, len(results), tt.wantMin) + t.Errorf("Search(%q) returned %d results, want at least %d", + tt.query, len(results), tt.wantMin) } // Verify all results contain the query if tt.query != "" { for _, r := range results { - if !contains(r.Handle, tt.query) && !contains(r.Description, tt.query) { - t.Errorf("Search result ASN %d doesn't contain query %q", r.ASN, tt.query) + if !strings.Contains(r.Handle, tt.query) && + !strings.Contains(r.Description, tt.query) { + t.Errorf("Search result ASN %d doesn't contain query %q", + r.ASN, tt.query) } } } @@ -148,13 +177,17 @@ func TestSearch(t *testing.T) { } func TestDataIntegrity(t *testing.T) { + t.Parallel() + // Verify no duplicate ASNs - all := All() + all := asinfo.All() seen := make(map[int]bool) + for _, info := range all { if seen[info.ASN] { t.Errorf("Duplicate ASN found: %d", info.ASN) } + seen[info.ASN] = true } @@ -163,6 +196,7 @@ func TestDataIntegrity(t *testing.T) { if info.Handle == "" && info.ASN != 0 { t.Errorf("ASN %d has empty handle", info.ASN) } + if info.Description == "" { t.Errorf("ASN %d has empty description", info.ASN) } @@ -174,8 +208,9 @@ func BenchmarkGet(b *testing.B) { asns := []int{1, 15169, 13335, 32934, 8075, 16509} b.ResetTimer() - for i := 0; i < b.N; i++ { - Get(asns[i%len(asns)]) + + for i := range b.N { + asinfo.Get(asns[i%len(asns)]) } } @@ -183,7 +218,8 @@ func BenchmarkSearch(b *testing.B) { queries := []string{"Google", "Amazon", "Microsoft", "University"} b.ResetTimer() - for i := 0; i < b.N; i++ { - Search(queries[i%len(queries)]) + + for i := range b.N { + asinfo.Search(queries[i%len(queries)]) } } diff --git a/script/bootstrap b/script/bootstrap index 3a156e5..79ebb69 100755 --- a/script/bootstrap +++ b/script/bootstrap @@ -2,14 +2,27 @@ # script/bootstrap: install all dependencies needed to build and develop # this repo. Idempotent: every install is guarded by a check so already # installed tools are skipped. Base tooling comes from nix, apt, brew, -# or apk (detected in that order); assumes NOTHING is present (not git, -# make, or go). +# or apk (detected in that order); assumes nothing is present. The +# installed node is used only when its major version is the pinned one; +# otherwise (node missing, or another major) the pinned version is +# installed via nvm (installing nvm itself first, from a hash-verified +# release archive, never curl | sh), and yarn and the packages are +# installed under it. Go formats the code and builds it on the host; the +# linter and the tests run only in Docker, as phases of the Dockerfile. set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" +# Pinned versions, 2026-07-06 +NODE_VERSION="22.17.0" +NVM_VERSION="0.40.3" +# sha256 of https://github.com/nvm-sh/nvm/archive/refs/tags/v0.40.3.tar.gz +NVM_SHA256="5f4d6aaa04a177dc93c985e31dbc411ab6b8c6e1e21d8015dbc1372625fcd1d0" +YARN_VERSION="1.22.22" + PKGMGR="" SUDO="" +APT_UPDATED="" detect_pkgmgr() { [ -n "$PKGMGR" ] && return 0 @@ -38,7 +51,14 @@ pkg_install() { detect_pkgmgr case "$PKGMGR" in nix) nix-env -iA "nixpkgs.$1" ;; - apt) $SUDO env DEBIAN_FRONTEND=noninteractive apt-get install -y "$2" ;; + apt) + # Package lists may be empty (fresh images); refresh once per run. + if [ -z "$APT_UPDATED" ]; then + $SUDO env DEBIAN_FRONTEND=noninteractive apt-get update + APT_UPDATED=1 + fi + $SUDO env DEBIAN_FRONTEND=noninteractive apt-get install -y "$2" + ;; brew) brew install "$3" ;; apk) apk add --no-cache "$4" ;; esac @@ -48,22 +68,100 @@ missing() { ! command -v "$1" >/dev/null 2>&1 } +# verify_sha256 +verify_sha256() { + if command -v sha256sum >/dev/null 2>&1; then + actual="$(sha256sum "$1" | cut -d' ' -f1)" + else + actual="$(shasum -a 256 "$1" | cut -d' ' -f1)" + fi + if [ "$actual" != "$2" ]; then + echo "bootstrap: sha256 mismatch for $1" >&2 + echo " expected: $2" >&2 + echo " actual: $actual" >&2 + exit 1 + fi +} + +# nvm is a bash script; run a command in a bash with nvm loaded +nvm_sh() { + bash -c ". \"\$HOME/.nvm/nvm.sh\" && $*" +} + +ensure_nvm() { + [ -s "$HOME/.nvm/nvm.sh" ] && return 0 + # nvm prerequisites; nvm itself requires bash + if missing bash; then pkg_install bash bash bash bash; fi + if missing curl; then pkg_install curl curl curl curl; fi + if missing git; then pkg_install git git git git; fi + tmp="$(mktemp -d)" + curl -fsSL -o "$tmp/nvm.tar.gz" \ + "https://github.com/nvm-sh/nvm/archive/refs/tags/v${NVM_VERSION}.tar.gz" + verify_sha256 "$tmp/nvm.tar.gz" "$NVM_SHA256" + mkdir -p "$HOME/.nvm" + tar -xzf "$tmp/nvm.tar.gz" -C "$HOME/.nvm" --strip-components=1 + rm -rf "$tmp" +} + +# True when the node on PATH has the pinned major version. Only the +# major is compared, as in the shared script/bootstrap of sneak/prompts. +# Another major is not used: the pinned yarn 1 prints a deprecation +# warning under node 24. script/fmt and script/fmt-check carry this +# function unchanged, to run the yarn installed here. +node_is_pinned_major() { + if ! command -v node >/dev/null 2>&1; then return 1; fi + installed="$(node --version)" + installed="${installed#v}" + [ "${installed%%.*}" = "${NODE_VERSION%%.*}" ] +} + +ensure_node() { + if node_is_pinned_major; then + echo "bootstrap: using node $(node --version)" + return 0 + fi + ensure_nvm + nvm_sh "nvm install $NODE_VERSION" + nvm_sh "nvm use $NODE_VERSION >/dev/null && \ + echo \"bootstrap: using node \$(node --version)\"" +} + +ensure_yarn() { + if ! node_is_pinned_major; then + nvm_sh "nvm use $NODE_VERSION >/dev/null && corepack enable && \ + corepack prepare yarn@$YARN_VERSION --activate" + return 0 + fi + if ! missing yarn; then return 0; fi + if ! missing corepack; then + corepack enable + corepack prepare "yarn@$YARN_VERSION" --activate + else + npm install -g "yarn@$YARN_VERSION" + fi +} + +install_js_deps() { + if node_is_pinned_major; then + yarn install --frozen-lockfile + else + nvm_sh "nvm use $NODE_VERSION >/dev/null && cd \"$ROOT\" && \ + yarn install --frozen-lockfile" + fi +} + main() { cd "$ROOT" - # Base tooling (every repo) - if missing git; then pkg_install git git git git; fi if missing make; then pkg_install gnumake make make make; fi + if missing git; then pkg_install git git git git; fi # Go toolchain if missing go; then pkg_install go golang go go; fi - # golangci-lint: packaged in nix, brew, and apk. There is no apt - # package; on apt systems install it manually from a hash-verified - # GitHub release archive (never curl | sh). - if missing golangci-lint; then - pkg_install golangci-lint golangci-lint golangci-lint golangci-lint - fi + ensure_node + ensure_yarn + install_js_deps go mod download diff --git a/script/check b/script/check index cc046f7..92875f7 100755 --- a/script/check +++ b/script/check @@ -1,7 +1,8 @@ #!/bin/sh # script/check: run all checks (test, lint, fmt-check). Our own -# extension to scripts-to-rule-them-all. Must not modify any files. -# Generic: usually needs no adaptation. +# extension to scripts-to-rule-them-all. test and lint are Docker +# phases; fmt-check is native, because a formatter writes the working +# tree. Must not modify any files. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" diff --git a/script/cibuild b/script/cibuild index 3da5857..2656771 100755 --- a/script/cibuild +++ b/script/cibuild @@ -1,14 +1,33 @@ #!/bin/sh -# script/cibuild: run the CI build. The Dockerfile runs script/check -# (via make check), so a successful build implies all checks pass. -# Generic: needs no adaptation. The Gitea workflow runs this on push. +# script/cibuild: run the CI build. It bootstraps first: a CI runner +# checks out and runs this and nothing else, and script/fmt-check runs +# the formatter on the host, which a pristine checkout cannot do. +# A new CACHEBUST value reruns every RUN step after the ARG CACHEBUST +# line, since a cached gate phase is a check that did not run; unlike +# --no-cache, it keeps the layers above that line and the cache mounts. +# This relies on every stage of the Dockerfile declaring ARG CACHEBUST +# before every step that lints, tests or compiles; a stage without that +# line is served from cache on an unchanged tree, with no warning. set -eu -ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" - docker build . + "$SCRIPT_DIR/bootstrap" + "$SCRIPT_DIR/check" + # The version and the tag each get their own line: a failing + # command substitution inside an argument does not trip `set -e`, + # so the inline form degrades silently to an empty constant. The + # VERSION build argument takes precedence over the version a build + # stage derives from the .git in the context. + version="$(git describe --tags --always --dirty 2>/dev/null || true)" + [ -n "$version" ] || version="unknown" + tag="$("$SCRIPT_DIR/projectname")" + docker build --build-arg CACHEBUST="$(date +%s)-$$" \ + --build-arg VERSION="$version" \ + -t "$tag" . } main "$@" diff --git a/script/docker b/script/docker index 07b626c..cc00d9c 100755 --- a/script/docker +++ b/script/docker @@ -1,8 +1,12 @@ #!/bin/sh # script/docker: build the Docker image tagged with the project name. # Identical in all repos; the tag comes from script/projectname. -# --no-cache because the gate phases the final stage depends on are RUN -# steps, and a cached one is a check that did not run. +# A new CACHEBUST value reruns every RUN step after the ARG CACHEBUST +# line, since a cached gate phase is a check that did not run; unlike +# --no-cache, it keeps the layers above that line and the cache mounts. +# This relies on every stage of the Dockerfile declaring ARG CACHEBUST +# before every step that lints, tests or compiles; a stage without that +# line is served from cache on an unchanged tree, with no warning. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" @@ -10,15 +14,17 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" - # Own line: a failing command substitution inside an argument does - # not trip `set -e`, so the inline form degrades silently to an - # empty constant. The VERSION build argument takes precedence over - # the version a build stage derives from the .git in the context. + # The version and the tag each get their own line: a failing + # command substitution inside an argument does not trip `set -e`, + # so the inline form degrades silently to an empty constant. The + # VERSION build argument takes precedence over the version a build + # stage derives from the .git in the context. version="$(git describe --tags --always --dirty 2>/dev/null || true)" [ -n "$version" ] || version="unknown" - docker build --no-cache \ + tag="$("$SCRIPT_DIR/projectname")" + docker build --build-arg CACHEBUST="$(date +%s)-$$" \ --build-arg VERSION="$version" \ - -t "$("$SCRIPT_DIR/projectname")" . + -t "$tag" . } main "$@" diff --git a/script/fmt b/script/fmt index e95d111..9906883 100755 --- a/script/fmt +++ b/script/fmt @@ -1,12 +1,43 @@ #!/bin/sh -# script/fmt: format all files (writes). +# script/fmt: format all files (writes): Go with go fmt, markdown with +# prettier. set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" +# Must match the pin in script/bootstrap. +NODE_VERSION="22.17.0" + +# True when the node on PATH has the pinned major version: the same +# function as in script/bootstrap, which uses it to decide where it +# installs yarn. +node_is_pinned_major() { + if ! command -v node >/dev/null 2>&1; then return 1; fi + installed="$(node --version)" + installed="${installed#v}" + [ "${installed%%.*}" = "${NODE_VERSION%%.*}" ] +} + +# Run the yarn script/bootstrap installed: the yarn on PATH when the +# node on PATH has the pinned major version, and otherwise yarn under +# the pinned node in nvm, which bootstrap leaves off the PATH of the +# shell that called it. nvm is a bash script, hence the subshell. +run_yarn() { + if node_is_pinned_major && command -v yarn >/dev/null 2>&1; then + exec yarn "$@" + fi + if node_is_pinned_major || [ ! -s "$HOME/.nvm/nvm.sh" ]; then + echo "fmt: no yarn; run script/bootstrap first" >&2 + exit 1 + fi + exec bash -c '. "$HOME/.nvm/nvm.sh" && nvm use "$1" >/dev/null && + shift && exec yarn "$@"' bash "$NODE_VERSION" "$@" +} + main() { cd "$ROOT" go fmt ./... + run_yarn run prettier --write '**/*.md' --tab-width 4 --prose-wrap always } main "$@" diff --git a/script/fmt-check b/script/fmt-check index 57e8e5c..924ccad 100755 --- a/script/fmt-check +++ b/script/fmt-check @@ -5,6 +5,35 @@ set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" +# Must match the pin in script/bootstrap. +NODE_VERSION="22.17.0" + +# True when the node on PATH has the pinned major version: the same +# function as in script/bootstrap, which uses it to decide where it +# installs yarn. +node_is_pinned_major() { + if ! command -v node >/dev/null 2>&1; then return 1; fi + installed="$(node --version)" + installed="${installed#v}" + [ "${installed%%.*}" = "${NODE_VERSION%%.*}" ] +} + +# Run the yarn script/bootstrap installed: the yarn on PATH when the +# node on PATH has the pinned major version, and otherwise yarn under +# the pinned node in nvm, which bootstrap leaves off the PATH of the +# shell that called it. nvm is a bash script, hence the subshell. +run_yarn() { + if node_is_pinned_major && command -v yarn >/dev/null 2>&1; then + exec yarn "$@" + fi + if node_is_pinned_major || [ ! -s "$HOME/.nvm/nvm.sh" ]; then + echo "fmt-check: no yarn; run script/bootstrap first" >&2 + exit 1 + fi + exec bash -c '. "$HOME/.nvm/nvm.sh" && nvm use "$1" >/dev/null && + shift && exec yarn "$@"' bash "$NODE_VERSION" "$@" +} + main() { cd "$ROOT" unformatted="$(gofmt -l .)" @@ -13,6 +42,7 @@ main() { echo "$unformatted" >&2 exit 1 fi + run_yarn run prettier --check '**/*.md' --tab-width 4 --prose-wrap always } main "$@" diff --git a/script/lint b/script/lint index a675833..451a067 100755 --- a/script/lint +++ b/script/lint @@ -1,13 +1,28 @@ #!/bin/sh -# script/lint: run the linters. +# script/lint: run the linter. Linting is a phase of the Dockerfile and +# this builds that phase alone; the linter is never installed or run on +# a developer host, where a shared result cache and a host-global lock +# make its answer untrustworthy. +# +# The phase is not the last stage in the file, so it is built only when +# --target names it. A new CACHEBUST value reruns every RUN step after +# the ARG CACHEBUST line, since a cached lint layer is a lint that did +# not run; unlike --no-cache, it keeps the layers above that line and +# the cache mounts. This relies on every stage of the Dockerfile +# declaring ARG CACHEBUST before every step that lints, tests or +# compiles; a stage without that line is served from cache on an +# unchanged tree, with no warning. +# --output type=cacheonly writes no image, since nothing uses one. set -eu -ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" - go vet ./... - golangci-lint run + docker build --build-arg CACHEBUST="$(date +%s)-$$" \ + --target lint \ + --output type=cacheonly . } main "$@" diff --git a/script/test b/script/test index bffb59e..1622a11 100755 --- a/script/test +++ b/script/test @@ -1,17 +1,24 @@ #!/bin/sh -# script/test: run the test suite. On failure, rerun verbosely so the -# failing tests are visible in the output. +# script/test: run the test suite. Testing is a phase of the Dockerfile +# and this builds that phase alone, on the same terms as script/lint: +# --target because a phase that is not the last stage is built only when +# named, and a new CACHEBUST value because a cached test layer is a test +# that did not run; unlike --no-cache, it keeps the layers above the ARG +# CACHEBUST line and the cache mounts. This relies on every stage of the +# Dockerfile declaring ARG CACHEBUST before every step that lints, tests +# or compiles; a stage without that line is served from cache on an +# unchanged tree, with no warning. --output type=cacheonly writes no +# image, since nothing uses one. set -eu -ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" - go test -short -timeout 30s -race -cover ./... || { - echo "--- Rerunning with -v for details ---" - go test -short -timeout 30s -race -v ./... - exit 1 - } + docker build --build-arg CACHEBUST="$(date +%s)-$$" \ + --target test \ + --output type=cacheonly . } main "$@" diff --git a/yarn.lock b/yarn.lock new file mode 100644 index 0000000..d846639 --- /dev/null +++ b/yarn.lock @@ -0,0 +1,8 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +prettier@3.8.1: + version "3.8.1" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.8.1.tgz#edf48977cf991558f4fcbd8a3ba6015ba2a3a173" + integrity sha512-UOnG6LftzbdaHZcKoPFtOcCKztrQ57WkHDeRD9t/PTQtmT0NHSeWWepj6pS0z/N7+08BHFDQVUrfmfMRcZwbMg==