From 793c75751d23f1cf346ce34769730ea361aa9d3b Mon Sep 17 00:00:00 2001 From: sneak Date: Wed, 14 Oct 2026 13:57:50 +0000 Subject: [PATCH] Re-vendor the shared files from sneak/prompts at dd4027b (closes #52) Takes .golangci.yml, .dockerignore, .gitignore, .editorconfig, the workflow and REPO_POLICIES.md from sneak/prompts dd4027b, keeping this repository's own entries after the shared content. Lint (golangci-lint v2.14.0) and test become Dockerfile phases that script/lint and script/test build alone and uncached; the image build depends on both, and script/cibuild bootstraps, runs script/check, then builds the image. The agent guidance file moves to AGENTS.md. The Go code is fixed for what the new configuration finds; reads that iterate rows now also report an iteration error. Model: opus-5-5 --- .dockerignore | 85 +- .editorconfig | 3 + .gitea/workflows/check.yml | 2 +- .gitignore | 90 +- .golangci.yml | 169 +- CLAUDE.md => AGENTS.md | 0 Dockerfile | 47 +- README.md | 22 +- REPO_POLICIES.md | 443 +++- TODO.md | 9 +- cmd/asinfo-gen/main.go | 29 +- cmd/streamdumper/main.go | 17 +- internal/config/config.go | 39 +- ...config_test.go => config_internal_test.go} | 11 +- internal/database/counts.go | 27 +- ...counts_test.go => counts_internal_test.go} | 283 ++- internal/database/database.go | 1954 ++++++++++------- ...base_test.go => database_internal_test.go} | 185 +- internal/database/interface.go | 16 +- internal/database/models.go | 45 +- internal/database/slowquery.go | 53 +- internal/database/utils.go | 1 + internal/logger/logger.go | 1 + internal/metrics/metrics.go | 14 +- internal/metrics/metrics_test.go | 12 +- internal/ristypes/ris.go | 20 +- internal/ristypes/ris_test.go | 31 +- internal/routewatch/app.go | 48 +- ...st.go => app_integration_internal_test.go} | 104 +- internal/routewatch/app_test.go | 2 +- internal/routewatch/ashandler.go | 32 +- internal/routewatch/asnfetcher.go | 45 +- internal/routewatch/cli.go | 33 +- internal/routewatch/dbmaintainer.go | 120 +- internal/routewatch/handler.go | 30 +- internal/routewatch/peerhandler.go | 34 +- internal/routewatch/peeringhandler.go | 43 +- ...est.go => peeringhandler_internal_test.go} | 16 +- internal/routewatch/prefixhandler.go | 248 +-- ...test.go => prefixhandler_internal_test.go} | 183 +- internal/server/handlers.go | 1080 ++++----- ...lers_test.go => handlers_internal_test.go} | 147 +- internal/server/middleware.go | 57 +- internal/server/routes.go | 7 +- internal/server/server.go | 19 +- internal/server/server_test.go | 26 +- internal/streamer/streamer.go | 510 +++-- ...amer_test.go => streamer_internal_test.go} | 130 +- internal/templates/templates.go | 15 +- internal/version/version.go | 2 +- internal/whois/whois.go | 184 +- pkg/asinfo/asinfo.go | 6 +- pkg/asinfo/asinfo_test.go | 68 +- script/bootstrap | 12 +- script/check | 5 +- script/cibuild | 24 +- script/lint | 18 +- script/test | 18 +- 58 files changed, 4119 insertions(+), 2755 deletions(-) rename CLAUDE.md => AGENTS.md (100%) rename internal/config/{config_test.go => config_internal_test.go} (91%) rename internal/database/{counts_test.go => counts_internal_test.go} (80%) rename internal/database/{database_test.go => database_internal_test.go} (78%) rename internal/routewatch/{app_integration_test.go => app_integration_internal_test.go} (78%) rename internal/routewatch/{peeringhandler_test.go => peeringhandler_internal_test.go} (95%) rename internal/routewatch/{prefixhandler_test.go => prefixhandler_internal_test.go} (56%) rename internal/server/{handlers_test.go => handlers_internal_test.go} (68%) rename internal/streamer/{streamer_test.go => streamer_internal_test.go} (78%) diff --git a/.dockerignore b/.dockerignore index 89e43ce..fdde48f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,15 +1,83 @@ -# Docker does not read .gitignore, and a pattern here matches from the root of -# the build context only: a pattern meant for every directory needs `**/`. +# .dockerignore does NOT use .gitignore semantics. Docker matches with +# moby/patternmatcher: filepath.Match plus `**`, so `*` does not cross +# `/` and an unprefixed pattern is anchored at the context root. Every +# depth-independent pattern therefore needs `**/`, or `config/.env` and +# `certs/server.key` still ship while this file reads as solved. Only +# genuinely root-anchored entries go unprefixed. Never transplant these +# into .gitignore, where `**/` is wrong. +# +# Matching is case-sensitive, so secrets use character ranges rather +# than an ALL-CAPS twin, which would still miss `Server.Key`. +# +# Extend with this repo's own host-built artifacts, written anchored: +# `/myapp`, never `**/myapp`, which also matches `cmd/myapp/` and +# deletes the package directory from the context. # .git is sent without its config. Without a VERSION build argument the # stage that compiles runs `git describe --tags --always` on .git, which # does not need .git/config; that file can hold a credential, such as a # password in a remote URL or the token the CI checkout step stores there. -.git/config +# Each submodule keeps a config with the same exposure in its git directory +# under .git/modules/, nested again for a submodule's own submodules, or in +# its own .git directory when it keeps one. +# KNOWN GAP: a submodule whose name has a `config` segment (`config`, +# `deploy/config`, `config/lib`) loses its whole git directory, because +# `**/.git/modules/**/config` also matches that segment's directory +# under .git/modules/. Go's version stamping then fails the build; +# nothing leaks. Name such a submodule without that segment: +# `git submodule add --name`. +**/.git/config +**/.git/modules/**/config -# Local build and debug output: `make build`, `make run`, `make asupdate`, test -# binaries, coverage profiles and source archives. +# Agent scratch: one full checkout of the repo per in-flight agent. +# Anchored because it occurs once where agents run at the repo root. +# KNOWN GAP: a repo running agents in subdirectories still ships +# `services/api/.claude/` and must add its own anchored entry. +.claude + +# Environment files. `*.env` covers bare `.env` and the `prod.env` +# convention. Re-include a committed template with a negation if the +# build needs one: `!docs/example.env`. +**/*.[eE][nN][vV] +**/.[eE][nN][vV].* +**/.[eE][nN][vV][rR][cC] + +# Private keys and the bundles carrying them. Public certificates +# (*.crt, *.cer) are deliberately absent: they are legitimate inputs. +**/*.[pP][eE][mM] +**/*.[kK][eE][yY] +**/*.[pP]12 +**/*.[pP][fF][xX] +**/[iI][dD]_[rR][sS][aA] +**/[iI][dD]_[dD][sS][aA] +**/[iI][dD]_[eE][cC][dD][sS][aA] +**/[iI][dD]_[eE][cC][dD][sS][aA]_[sS][kK] +**/[iI][dD]_[eE][dD]25519 +**/[iI][dD]_[eE][dD]25519_[sS][kK] + +# Dependencies: restored inside the image, never copied in. +**/node_modules + +# OS metadata. +**/.DS_Store +**/Thumbs.db + +# Editor state: never a build input, and it churns COPY. +**/*.swp +**/*.swo +**/*~ +**/*.bak +**/.idea +**/.vscode +**/*.sublime-* + +# This repository's host-built artifacts: `make build` writes bin/, `go +# build` in the root the binaries, `make run` log.txt, `make asupdate` its +# .tmp file; test binaries, coverage profiles and source archives. /bin +/routewatch +/asinfo-gen +/streamdumper /log.txt /out /pkg/asinfo/asdata.json @@ -18,15 +86,12 @@ **/*.out **/*.tmp -# Local databases and secrets. The image carries a source archive of the whole -# build context, so these would otherwise ship inside it. +# Local databases. The image carries a source archive of the whole build +# context, so these would otherwise ship inside it. **/*.db **/*.db-journal **/*.db-wal -**/.env # A local Go workspace points at directories outside the build context. /go.work /go.work.sum - -**/.DS_Store diff --git a/.editorconfig b/.editorconfig index 2fe0ce0..92ec261 100644 --- a/.editorconfig +++ b/.editorconfig @@ -10,3 +10,6 @@ insert_final_newline = true [Makefile] indent_style = tab + +[*.go] +indent_style = tab diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index a55bc55..ee73864 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -4,6 +4,6 @@ jobs: check: runs-on: ubuntu-latest steps: - # actions/checkout v4.2.2, 2026-02-28 + # actions/checkout v4.2.2, 2026-02-22 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - run: script/cibuild diff --git a/.gitignore b/.gitignore index 4951af1..3d7057b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,41 +1,77 @@ -# Binaries for programs and plugins +# OS +.DS_Store +Thumbs.db + +# Editors +*.swp +*.swo +*~ +*.bak +.idea/ +.vscode/ +*.sublime-* + +# Agent scratch (worktrees of this repo, created and destroyed by +# in-flight tooling). Unanchored: .gitignore patterns already match at +# every depth, so no prefix is wanted here. This is not a .dockerignore +# entry and must not be given a `**/` prefix on the way into one. +.claude/ + +# Node +node_modules/ + +# Secrets. Unanchored like every entry above, so each matches at every +# depth. Matching is case-sensitive on Linux, so names use character +# ranges rather than a lowercase form that misses `Server.Key`. + +# Environment files. `*.env` covers bare `.env` and the `prod.env` +# convention. Only the templates `example.env` and `sample.env` are +# re-included below. A repository that commits any other template adds +# its own negation after these lines, for example `!.env.example`. +*.[eE][nN][vV] +.[eE][nN][vV].* +.[eE][nN][vV][rR][cC] +!example.env +!sample.env + +# Private keys and the bundles carrying them. +*.[pP][eE][mM] +*.[kK][eE][yY] +*.[pP]12 +*.[pP][fF][xX] +[iI][dD]_[rR][sS][aA] +[iI][dD]_[dD][sS][aA] +[iI][dD]_[eE][cC][dD][sS][aA] +[iI][dD]_[eE][cC][dD][sS][aA]_[sS][kK] +[iI][dD]_[eE][dD]25519 +[iI][dD]_[eE][dD]25519_[sS][kK] + +# This repository's own entries, kept after the canonical content above. + +# Go: logs, test binaries, coverage profiles, and the binaries `make build` +# and `go build` in the repository root write. +*.log +*.out +*.test *.exe *.exe~ *.dll *.so *.dylib /bin/ - -.DS_Store -log.txt - -# Test binary, built with `go test -c` -*.test - -# Output of the go coverage tool, specifically when used with LiteIDE -*.out - -# Dependency directories (remove the comment below to include it) -# vendor/ - -# Go workspace file +/routewatch +/asinfo-gen +/streamdumper go.work go.work.sum -# env file -.env - -# Database files +# Local databases, temporary files and debug output. *.db *.db-journal *.db-wal - -# Temporary files *.tmp - -# Raw AS data (we only commit the gzipped version) -pkg/asinfo/asdata.json - -# Debug output files -out log.txt +out + +# Raw AS data (only the gzipped version is committed). +pkg/asinfo/asdata.json diff --git a/.golangci.yml b/.golangci.yml index 9d7f245..1b73eb9 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,92 +1,99 @@ version: "2" +# Config schema uses the golangci-lint v2 layout (settings live under +# linters.settings, not top-level linters-settings) so that the +# thresholds below are actually applied by golangci-lint >= v2. + run: - go: "1.24" - tests: false + timeout: 5m + modules-download-mode: readonly linters: + default: all enable: - # Additional linters requested - - testifylint # Checks usage of github.com/stretchr/testify - - usetesting # usetesting is an analyzer that detects using os.Setenv instead of t.Setenv since Go 1.17 - # - tagliatelle # Disabled: we need snake_case for external API compatibility - - nlreturn # nlreturn checks for a new line before return and branch statements - - nilnil # Checks that there is no simultaneous return of nil error and an invalid value - - nestif # Reports deeply nested if statements - - mnd # An analyzer to detect magic numbers - - lll # Reports long lines - - intrange # intrange is a linter to find places where for loops could make use of an integer range - - gochecknoglobals # Check that no global variables exist - - # Default/existing linters that are commonly useful - - govet - - errcheck - - staticcheck - - unused - - ineffassign - - misspell - - revive - - gosec - - unconvert - - unparam - -linters-settings: - lll: - line-length: 120 - - nestif: - min-complexity: 4 - - nlreturn: - block-size: 2 - - revive: - rules: - - name: var-naming - arguments: - - [] - - [] - - "upperCaseConst=true" - - tagliatelle: - case: + # Successor to the deprecated gomodguard. Named explicitly, rather than + # left to `default: all`, because it carries the module policy below. + - gomodguard_v2 + disable: + # Genuinely incompatible with project patterns + - exhaustruct # Requires all struct fields + - exhaustruct_v5 # Requires all struct fields (successor to exhaustruct) + - godot # Requires comments to end with periods + - wrapcheck # Too verbose for internal packages + - varnamelen # Short names like db, id are idiomatic Go + # Deprecated: the warning is attached to the old name, so it is + # silenced by disabling that name, not by enabling the successor. + - wsl # Deprecated, replaced by wsl_v5 + - gomodguard # Deprecated, replaced by gomodguard_v2 + settings: + lll: + line-length: 88 + funlen: + lines: 80 + statements: 50 + cyclop: + max-complexity: 15 + dupl: + threshold: 100 + depguard: + # Test-support code must not be compiled into the shipped binary. A + # test-support package exists to hand a test privileges the program + # itself must never have, so a file that is not a test must not import + # one. Test files, and the files inside a package whose directory name + # ends in `test`, are where that code belongs, and are exempt. + # + # The deny list below is the one part of this file a repository is + # expected to extend, and the only part it may. depguard matches an + # import path against a list of prefixes, so it cannot be told "any path + # whose last segment ends in test"; a repository's own test-support + # packages have to be named here one at a time, by full import path, + # under a module path that differs from repository to repository. Add + # them; change nothing else. rules: - json: snake - yaml: snake - xml: snake - bson: snake - - testifylint: - enable-all: true - - usetesting: {} + test-support: + list-mode: lax + files: + - "$all" + - "!$test" + - "!**/*test/**" + deny: + - pkg: net/http/httptest + desc: >- + Test-support code belongs in test files and in packages whose + directory name ends in test, not in the shipped binary. + # Only decisions already recorded in the Go package defaults are + # listed here. Every entry matches the module path exactly. + gomodguard_v2: + blocked: + - module: github.com/rs/zerolog + recommendations: + - log/slog + reason: "Structured logging is stdlib log/slog." + # One entry per pre-fork module path, because the later releases + # are separate paths. A prefix match would be shorter but would + # also reach github.com/go-redis/redismock, the test double for + # the successor these entries recommend. + - module: github.com/go-redis/redis + recommendations: + - github.com/redis/go-redis/v9 + reason: "Pre-fork module; use the maintained go-redis v9." + - module: github.com/go-redis/redis/v7 + recommendations: + - github.com/redis/go-redis/v9 + reason: "Pre-fork module; use the maintained go-redis v9." + - module: github.com/go-redis/redis/v8 + recommendations: + - github.com/redis/go-redis/v9 + reason: "Pre-fork module; use the maintained go-redis v9." + - module: github.com/sergi/go-diff + recommendations: + - github.com/aymanbagabas/go-udiff + reason: "No unified diff output; use go-udiff." + - module: github.com/hexops/gotextdiff + recommendations: + - github.com/aymanbagabas/go-udiff + reason: "Unmaintained fork; use go-udiff." issues: max-issues-per-linter: 0 max-same-issues: 0 - exclude-rules: - # Exclude unused parameter warnings for cobra command signatures - - text: "parameter '(args|cmd)' seems to be unused" - linters: - - revive - - # Allow ALL_CAPS constant names - - text: "don't use ALL_CAPS in Go names" - linters: - - revive - - # Allow snake_case JSON tags for external API compatibility - - path: "internal/types/ris.go" - linters: - - tagliatelle - - # Allow snake_case JSON tags for database models - - path: "internal/database/models.go" - linters: - - tagliatelle - - # Allow generic package name for types that define data structures - - path: "internal/types/" - text: "avoid meaningless package names" - linters: - - revive diff --git a/CLAUDE.md b/AGENTS.md similarity index 100% rename from CLAUDE.md rename to AGENTS.md diff --git a/Dockerfile b/Dockerfile index dfd9d5a..52c9cf2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,22 +1,37 @@ -# Lint stage — fast feedback on formatting and lint issues. -# The golangci-lint image bundles Go, gcc and make, so it can run go vet on -# the CGO sqlite package and golangci-lint without extra installs. -# golangci/golangci-lint:v2.7.2 (Go 1.25.5), 2026-09-21 -FROM golangci/golangci-lint@sha256:5d6d5c70a61f1356adfd9dd6316ce286799fefc9d743421356ff1b00842368ba AS lint - +# Lint phase, built alone by script/lint. The linter is invoked directly +# rather than through `make lint`, which is itself a docker build and +# would recurse into a daemon that does not exist in a build step. The +# image is Debian-based and has Go and gcc, so it can check the cgo +# SQLite package without extra installs. +# golangci/golangci-lint:v2.14.0 (Go 1.27.0), 2026-10-14 +FROM golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint WORKDIR /src - COPY go.mod go.sum ./ RUN go mod download - COPY . . +RUN golangci-lint run --config .golangci.yml ./... -RUN make fmt-check -RUN make lint +# Test phase, built alone by script/test. -race needs cgo and so a C +# compiler, which the Debian Go image ships. -short skips the live RIPE RIS +# feed test, so the suite needs no network. The rerun with -v only shows +# details: the build fails however it ends, because the first run failed. +# golang:1.24-bookworm, 2026-09-21 +FROM golang@sha256:1a6d4452c65dea36aac2e2d606b01b4a029ec90cc1ae53890540ce6173ea77ac AS test +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +RUN go test -short -timeout 90s -race -cover ./... || \ + { echo "--- Rerunning with -v for details ---"; \ + go test -short -timeout 90s -race -v ./...; exit 1; } -# Build stage +# Build stage. Nothing is wanted from the lint and test phases; the copies +# are what make BuildKit build them first, so this stage cannot run unless +# lint and test passed. # golang:1.24-bookworm, 2026-09-21 FROM golang@sha256:1a6d4452c65dea36aac2e2d606b01b4a029ec90cc1ae53890540ce6173ea77ac AS builder +COPY --from=lint /src/go.sum /dev/null +COPY --from=test /src/go.sum /dev/null # Install build dependencies (zstd for archive, gcc for CGO/sqlite3) RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -25,10 +40,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libc6-dev \ && rm -rf /var/lib/apt/lists/* -WORKDIR /src +# A tar-stream context keeps the sender's file owners, which git refuses. +RUN git config --system --add safe.directory /src -# Force BuildKit to run the lint stage before compiling or testing. -COPY --from=lint /src/go.sum /dev/null +WORKDIR /src # Copy everything COPY . . @@ -36,10 +51,6 @@ COPY . . # Vendor dependencies (must be after copying source) RUN go mod download && go mod vendor -# Run the test suite in the build stage: -race needs cgo and the C compiler -# installed above. The suite is offline (the live-feed test is opt-in). -RUN make test - # Build the binary with CGO enabled (required for sqlite3). The version the # page footer shows is the VERSION build argument when one is given, otherwise # `git describe --tags --always` of the .git in the build context (git comes diff --git a/README.md b/README.md index 05615d5..af2705e 100644 --- a/README.md +++ b/README.md @@ -277,29 +277,35 @@ standard: normalized scripts in `script/` are the entrypoints for the development workflow, and the Makefile targets are thin shims that call them. We provide: -- `script/bootstrap` — install all development dependencies (go, - golangci-lint, Go module download) +- `script/bootstrap` — install all development dependencies (git, make, go, Go + module download); warns when docker is missing - `script/setup` — make a fresh clone ready for development: runs `script/bootstrap`, then `script/install-precommit` - `script/projectname` — print the project name (used for the Docker image tag) -- `script/test` — run the test suite - (`go test -short -timeout 30s -race -cover ./...`, verbose rerun on - failure). The `-short` flag skips the live-network integration test so the - default run is deterministic and offline. -- `script/lint` — run `go vet ./...` and `golangci-lint run` +- `script/test` — build the `test` phase alone, which runs + `go test -short -timeout 90s -race -cover ./...` (verbose rerun on failure). + The `-short` flag skips the live-network integration test so the run is + deterministic and offline. +- `script/lint` — build the `lint` phase alone, which runs golangci-lint + v2.14.0 with `.golangci.yml` - `script/fmt` — format all code (writes) - `script/fmt-check` — check formatting (read-only) - `script/check` — run `script/test`, `script/lint`, and `script/fmt-check` - `script/docker` — build the Docker image tagged via `script/projectname` -- `script/cibuild` — CI entrypoint: `docker build .` +- `script/cibuild` — CI entrypoint: runs `script/bootstrap` and + `script/check`, then builds the image - `script/precommit` — pre-commit gate: `go mod tidy` + `go fmt` (must not change files), then `script/check` - `script/install-precommit` — install the git pre-commit hook that runs `script/precommit` +Tests and the linter run only in Docker, as the `test` and `lint` phases of the +`Dockerfile`; the image cannot be built unless both pass. Every `docker build` +in `script/` passes `--no-cache`, so each run executes every check. + The live-network integration test `TestRouteWatchLiveFeed` streams the RIPE RIS feed for a few seconds and is skipped in short mode. To run it on demand, invoke `go test` directly without `-short`: diff --git a/REPO_POLICIES.md b/REPO_POLICIES.md index bc2f161..20382d1 100644 --- a/REPO_POLICIES.md +++ b/REPO_POLICIES.md @@ -1,6 +1,6 @@ --- title: Repository Policies -last_modified: 2026-07-06 +last_modified: 2026-10-04 --- This document covers repository structure, tooling, and workflow standards. Code @@ -60,17 +60,28 @@ style conventions are in separate documents: prerequisite since nvm requires bash. yarn is then pinned via `corepack prepare yarn@ --activate`. Never install "latest" or "lts"; always exact versions. `script/cibuild` runs the CI build: it changes to the - repo root and runs `docker build .`; the Gitea workflow calls it. Four further - scripts are our own extensions to the standard: `script/check` runs - `script/test`, `script/lint`, and `script/fmt-check`; `script/precommit` is - what the git pre-commit hook runs, and it calls `script/check`; - `script/install-precommit` installs the git pre-commit hook (the `make hooks` - target shims to it); and `script/projectname` (literally that filename) simply - outputs the project's name. Scripts that need the name call - `script/projectname` — e.g. `script/docker` assembles its image tag from it — - so those scripts stay byte-identical across all repos. Repo-type-specific - pre-commit extras (e.g. `go mod tidy` verification in Go repos) belong in - `script/precommit`, not in the hook itself. Model scripts are at + repo root, runs `script/bootstrap`, runs `script/check`, and builds the image + with the version; the Gitea workflow calls it. **`script/cibuild` runs + `script/bootstrap` first**, because the workflow checks out the repo and runs + nothing else, while `script/fmt-check` runs the formatter on the host: on a + pristine checkout with nothing installed the run dies there, after the + containerised gates have passed. **The bootstrap alone is not enough**: + `script/bootstrap` installs node and yarn under nvm and leaves neither on the + `PATH` of the shell that called it, so a bare `yarn` still exits 127. The host + entrypoints that need yarn — `script/fmt` and `script/fmt-check` — therefore + source nvm for the pinned node version before invoking it, exactly as + `script/bootstrap`'s own install step does. A runner carrying nothing but + docker and git then gets through `script/check`. Four further scripts are our + own extensions to the standard: `script/check` runs `script/test`, + `script/lint` and `script/fmt-check`; `script/precommit` is what the git + pre-commit hook runs, and it calls `script/check`; `script/install-precommit` + installs the git pre-commit hook (the `make hooks` target shims to it); and + `script/projectname` (literally that filename) simply outputs the project's + name. Scripts that need the name call `script/projectname` — e.g. + `script/docker` assembles its image tag from it — so those scripts stay + byte-identical across all repos. Repo-type-specific pre-commit extras (e.g. + `go mod tidy` verification in Go repos) belong in `script/precommit`, not in + the hook itself. Model scripts are at `https://git.eeqj.de/sneak/prompts/raw/branch/main/script/`. The README must document the provided scripts in an **Entrypoints** section (see the README requirements below). @@ -89,87 +100,198 @@ style conventions are in separate documents: contributor should be able to understand the entire development workflow by reading the Makefile. -- Every repo should have a `Dockerfile`. All Dockerfiles must run `make check` - as a build step so the build fails if the branch is not green. For non-server - repos, the Dockerfile should bring up a development environment and run - `make check`. For server repos, `make check` should run as an early build - stage before the final image is assembled. Dockerfiles install development - prerequisites by running `script/bootstrap` rather than duplicating installs - inline; COPY `script/` and the dependency manifests (`package.json` + - `yarn.lock`, `go.mod` + `go.sum`, etc.) before running it so the bootstrap - layer stays cached until dependencies change. +- Every repo should have a `Dockerfile`, and it carries the repo's gates: a + `lint` phase and a `test` phase, with the final stage depending on both so the + image cannot be built unless they pass. For non-server repos the final stage + brings up a development environment; for server repos it is the runtime image. + The gate phases and the build stage start from their pinned base images and + install what those images lack either inline, as the canonical Go `Dockerfile` + below does for `git`, or by running `script/bootstrap`, as the `prompts` + repo's own `Dockerfile` does for its yarn packages. The development + environment stage installs development prerequisites by running + `script/bootstrap` rather than duplicating its installs inline. A stage that + runs `script/bootstrap` COPYs `script/` and the dependency manifests + (`package.json` + `yarn.lock`, `go.mod` + `go.sum`, etc.) before running it. -- **Dockerfiles must use a separate lint stage for fail-fast feedback.** Go - repos use a multistage build where linting runs in an independent stage based - on the `golangci/golangci-lint` image (pinned by hash). This stage runs - `make fmt-check` and `make lint` before the full build begins. The build stage - then declares an explicit dependency on the lint stage via - `COPY --from=lint /src/go.sum /dev/null`, which forces BuildKit to complete - linting before proceeding to compilation and tests. This ensures lint failures - surface in seconds rather than minutes, without blocking on dependency - download or compilation in the build stage. +- **Linting and testing run in Docker, as phases of the `Dockerfile`.** There is + no separate lint file. `script/lint` and `script/test` each build one phase + and nothing else: - The standard pattern for a Go repo Dockerfile is: + ```sh + docker build --no-cache --target lint -t "$(script/projectname)-lint" . + docker build --no-cache --target test -t "$(script/projectname)-test" . + ``` + + **A stage that is not the last one in the file is built only when the final + stage's chain depends on it, or when `--target` names it.** That is why the + two gates are always invoked by name here, and why the final stage carries a + `COPY --from=` of a harmless file from each of them: without that edge a + plain `docker build .` builds the last stage alone and exits 0 having linted + and tested nothing. + + **Every `docker build` in `script/` is tagged**, here and in + `script/cibuild` and `script/docker`. An untagged build leaves a dangling + image behind on every invocation, on every developer host and every CI + runner; a tagged one replaces the previous image. + + Inside a phase the tool is invoked directly — `golangci-lint`, `go test`, + `eslint`, `prettier` — never through `make lint` or `script/test`, which are + themselves a `docker build` and would recurse into a daemon that does not + exist in a build step. Formatting is the exception and stays on the host: + `script/fmt` writes the working tree, and `script/fmt-check` is its + read-only twin. + + **No lint verdict may come from a host invocation of the linter.** On a + shared host golangci-lint reads a result cache keyed on file content rather + than location, so a second checkout of the same content is served the first + one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs + exit non-zero with `parallel golangci-lint is running` — a status a caller + cannot tell from real findings. Both have produced wrong verdicts in this + org, in both directions. A container has its own cache, its own `TMPDIR` and + a digest-pinned binary, so neither is reachable. + +- **Any build that runs checks is built with `--no-cache`.** Docker invalidates + a `COPY` layer only when the copied content changes, so on an unchanged tree + the check `RUN` is served from cache, nothing executes, and the build still + exits 0. Every `docker build` in `script/` therefore passes `--no-cache`: + `script/lint`, `script/test`, `script/cibuild` and `script/docker` are the + four, and there is no fifth — `script/check` runs the two gate phases and + `script/fmt-check`, and builds no image of its own. A bare `docker build .` is + not evidence that anything ran: a sub-second build reporting success is a + cache hit, not a result. Never invalidate by pruning — `docker builder prune` + and friends destroy a build cache shared with every other build on the host. + When a check is added or changed, prove it works by planting a defect it must + catch and watching the run fail on it, then revert the defect. A green run + alone shows neither that the check ran nor that it covers what it should. + +- **The gate phases are separate stages, and the build stage depends on both.** + The lint phase is based on the `golangci/golangci-lint` image (pinned by + hash), so lint failures surface in seconds rather than after a full compile, + and the test phase is based on the Debian Go image. The canonical Go repo + `Dockerfile`: ```dockerfile - # Lint stage — fast feedback on formatting and lint issues + # Lint phase # golangci/golangci-lint:v2.x.x, YYYY-MM-DD FROM golangci/golangci-lint@sha256:... AS lint WORKDIR /src COPY go.mod go.sum ./ RUN go mod download COPY . . - RUN make fmt-check - RUN make lint + RUN golangci-lint run --config .golangci.yml ./... - # Build stage - # golang:1.x-alpine, YYYY-MM-DD - FROM golang@sha256:... AS builder + # Test phase. -race needs cgo and so a C compiler, which the Debian Go + # image ships and the alpine one does not. + # golang:1.x, YYYY-MM-DD + FROM golang@sha256:... AS test WORKDIR /src - - # Force BuildKit to run the lint stage before proceeding - COPY --from=lint /src/go.sum /dev/null - COPY go.mod go.sum ./ RUN go mod download COPY . . - RUN make test + RUN go test -timeout 90s -race -cover ./... || \ + { echo "--- Rerunning with -v for details ---"; \ + go test -timeout 90s -race -v ./...; exit 1; } - ARG VERSION=dev - RUN CGO_ENABLED=0 go build -trimpath \ - -ldflags="-s -w -X main.Version=${VERSION}" \ - -o /app ./cmd/app/ + # Build stage. Nothing is wanted from either phase above; the copies + # are what make BuildKit build them first, so this stage cannot run + # unless lint and test passed. + # golang:1.x-alpine, YYYY-MM-DD + FROM golang@sha256:... AS builder + COPY --from=lint /src/go.sum /dev/null + COPY --from=test /src/go.sum /dev/null + RUN apk add --no-cache git + # A tar-stream context keeps the sender's file owners, which git refuses. + RUN git config --system --add safe.directory /src + WORKDIR /src + COPY go.mod go.sum ./ + RUN go mod download + COPY . . - # Runtime stage + # The VERSION build arg when one is given, otherwise + # `git describe --tags --always` on the .git in the build context. With + # .git present, a version that is still empty, dev or unknown fails the + # build: git is missing or could not read the checkout. + ARG VERSION + RUN VERSION="${VERSION:-$(git describe --tags --always)}"; \ + if [ -e .git ]; then \ + case "$VERSION" in ""|dev|unknown) \ + echo "version is '$VERSION' although .git is present" >&2; \ + exit 1 ;; \ + esac; \ + fi; \ + CGO_ENABLED=0 go build -trimpath \ + -ldflags="-s -w -X main.Version=${VERSION}" \ + -o /app ./cmd/app/ + + # Runtime stage, and the last one FROM alpine@sha256:... COPY --from=builder /app /usr/local/bin/app ENTRYPOINT ["app"] ``` Key points: - - The lint stage uses the `golangci/golangci-lint` image directly (it - includes both Go and the linter), so there is no need to install the - linter separately. - - `COPY --from=lint /src/go.sum /dev/null` is a no-op file copy that creates - a stage dependency. BuildKit runs stages in parallel by default; without - this line, the build stage would not wait for lint to finish and a lint - failure might not fail the overall build. + - The lint phase uses the `golangci/golangci-lint` image directly (it has + both Go and the linter), so nothing needs installing. + - `COPY --from= /src/go.sum /dev/null` is a no-op copy whose only + purpose is the ordering edge. BuildKit runs stages in parallel by default, + and a stage nothing depends on is not built at all, so without these two + lines a red gate would not fail the build. + - Keep the runtime stage last, and if you add a stage after it, give it the + same two copies. A plain `docker build .` builds the last stage's chain + and nothing else. - If the project uses `//go:embed` directives that reference build artifacts - (e.g. a web frontend compiled in a separate stage), the lint stage must + (e.g. a web frontend compiled in a separate stage), the lint phase must create placeholder files so the embed directives resolve. Example: `RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`. - The lint stage should not depend on the actual build output — it exists to - fail fast. - - If the project requires CGO or system libraries for linting (e.g. - `vips-dev`), install them in the lint stage with `apk add`. - - The build stage runs `make test` after compilation setup. Tests run in the - build stage, not the lint stage, because they may require compiled - artifacts or heavier dependencies. + - If the project requires CGO or system libraries for linting, install them + in the lint phase. The `golangci/golangci-lint` image is Debian-based and + has no `apk`, so install with `apt-get` under the Debian package name + (`libvips-dev`, where alpine says `vips-dev`), and delete the package + lists in the same `RUN`, so the layer does not keep them: + + ```dockerfile + RUN apt-get update \ + && apt-get install -y --no-install-recommends libvips-dev \ + && rm -rf /var/lib/apt/lists/* + ``` + + - `.dockerignore` lets `.git` into the build context. It keeps out every git + `config` at any depth (`**/.git/config`, `**/.git/modules/**/config`): the + repository's own, each submodule's under `.git/modules/`, and that of a + submodule keeping its own `.git` directory. `git describe` does not need + them, and each can hold a credential: a password in a remote URL, or the + token the CI checkout step stores there. A submodule whose name has a + `config` segment (`config`, `deploy/config`, `config/lib`) loses its whole + git directory to `**/.git/modules/**/config`, and Go's version stamping + then fails the build: give it a name without that segment + (`git submodule add --name`). The stage that compiles has `git` (the + Debian Go image has it; an alpine one needs `apk add --no-cache git`) and + takes the version from the `VERSION` build argument when one is given, + otherwise from `git describe --tags --always`. That gives the tag on a + tagged commit; on a later commit, the tag, the number of commits since it + and the short commit (`v1.2.3-4-gabc1234`); and the short commit when no + tag is reachable. The stage that compiles also marks its working directory + safe for git (`git config --system --add safe.directory /src`): a context + sent as a tar stream keeps the sender's file owners, and git refuses a + checkout owned by another user, so the version would come out empty. + `ARG VERSION` has no default, and the build fails if the context carries + `.git` and the version still comes out empty, `dev` or `unknown`. A plain + `docker build .` with no build arguments must succeed; a Dockerfile that + refuses an empty build argument drops that refusal and keeps the argument. - Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that - runs `script/cibuild` (which runs `docker build .`) on push. Since the - Dockerfile already runs `make check`, a successful build implies all checks - pass. + runs `script/cibuild` on push, and checks out the repo as its only other step. + That script bootstraps, runs the gate phases, and then builds the image, so a + successful run means every check passed; a bare `docker build .` does not + carry the same guarantee, because its gate phases may come from the cache. The + image build is uncached and so runs the gate phases a second time. That is the + price of the rule above, and it is worth paying: the image that ships is built + from a run of its own gates rather than from a cache entry. A separate + workflow limited to `main` by a `branches` list under `on: push` cannot be + checked by review: to try a change to it, add the feature branch to that list + and push, then remove the branch from the list again before merging. Keep any + job in it that publishes behind `if: github.ref_name == 'main'`, so the run + from the feature branch publishes nothing. - Use platform-standard formatters: `black` for Python, `prettier` for JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with @@ -189,14 +311,21 @@ style conventions are in separate documents: module under test to verify it compiles/parses. There is no excuse for `make test` to be a no-op. -- `make test` must complete in under 20 seconds. Add a 30-second timeout in the - Makefile. +- `make test` must complete in under 60 seconds. That is the hard cap, and a + suite that exceeds it fails. Under 20 seconds is the target. A suite between + 20 and 60 seconds is still green, but the overage must be filed as an + improvement bug against that repo. Add a 90-second timeout to the test + invocation (`go test -timeout 90s`). The backstop deliberately sits above the + hard cap so that it catches a genuinely hung test rather than a merely slow + one. -- **`make test` should use the conditional verbose rerun pattern.** Run tests - without `-v` (verbose) first. If tests fail, automatically rerun with `-v` to - show full output. This keeps CI logs and `docker build` output clean on - success (just package/suite summaries) while providing full diagnostic detail - on failure (every test case, every assertion). The general shell pattern: +- **The test command should use the conditional verbose rerun pattern.** Run + tests without `-v` (verbose) first. If tests fail, automatically rerun with + `-v` to show full output. This keeps CI logs and `docker build` output clean + on success (just package/suite summaries) while providing full diagnostic + detail on failure (every test case, every assertion). The command lives in the + `test` phase of the `Dockerfile`, since `script/test` builds that phase; the + Makefile form below is the same pattern for any repo-local invocation: ```makefile test: @@ -209,11 +338,26 @@ style conventions are in separate documents: ```makefile test: - @go test -timeout 30s -race -cover ./... || \ + @go test -count=1 -timeout 90s -race -cover ./... || \ { echo "--- Rerunning with -v for details ---"; \ - go test -timeout 30s -race -v ./...; exit 1; } + go test -count=1 -timeout 90s -race -v ./...; exit 1; } ``` + `-count=1` is required on both invocations: it defeats Go's test _result_ + cache, so neither run can report a stored pass in place of running the + tests. It leaves the build cache alone, so it costs the runtime of the suite + and no recompilation. + + That cache is Go's own, separate from Docker's layer cache. Go stores a + passing result in its cache directory (`GOCACHE`), and when the same tests + run again on unchanged code it prints that result, marked `(cached)`, + without running them. That matters on a developer's machine, where this + target runs and the directory lasts from one run to the next. The `test` + phase of the `Dockerfile` needs no `-count=1`: its base image holds no + result for this repo's tests and nothing before its `go test` step runs a + test, so there is nothing to replay. `--no-cache` (above) is what makes that + step run on an unchanged tree. + Python example: ```makefile @@ -239,10 +383,84 @@ style conventions are in separate documents: must be in `.gitignore`. No exceptions. - `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`), - editor files (`.swp`, `*~`), language build artifacts, and `node_modules/`. - Fetch the standard `.gitignore` from - `https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when setting up - a new repo. + editor files (`.swp`, `*~`), in-repo agent scratch directories (`.claude/`), + language build artifacts, and `node_modules/`. Fetch the standard `.gitignore` + from `https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when + setting up a new repo. These patterns are written to `.gitignore`'s own + semantics, in which an unanchored pattern already matches at every depth; they + are not a `.dockerignore` and must not be transplanted into one unmodified. + +- **`.dockerignore` does not use `.gitignore` semantics, and copying patterns + across unmodified leaves secrets in the build context.** Docker matches with + `moby/patternmatcher`: `filepath.Match` semantics plus a `**` extension, so + `*` does not cross `/` and a pattern without a leading `**/` is anchored at + the build-context root. A `.dockerignore` listing `.env`, `*.pem` and `*.key` + therefore excludes only the copies at the repository root, while `config/.env` + and `certs/server.key` still reach the context and can land in an image layer + — which is more dangerous than a short file with no secret patterns at all, + because it reads as solved and stops anyone looking. Give every + depth-independent pattern the `**/` prefix and leave only genuinely + root-anchored entries unprefixed: `.claude`, and the repo's own host-built + binary, written `/myapp` and never `**/myapp`, which would also match + `cmd/myapp/` and delete the package directory from the context. Matching is + case-sensitive, and an ALL-CAPS twin per pattern still misses `Server.Key`, so + secret names use character ranges — `**/*.[kK][eE][yY]`, `**/*.[pP][eE][mM]`, + and likewise for `.envrc` and the extensionless SSH keys. Where such a pattern + also catches something the build needs, re-include it with a negation + (`!docs/example.env`); deleting the pattern reopens the exposure for every + other file it covers. Fetch the standard `.dockerignore` from + `https://git.eeqj.de/sneak/prompts/raw/branch/main/.dockerignore` and extend + it with the repo's own artifacts. + +- **In-repo agent scratch belongs in both files, written to each file's own + semantics.** `.claude/` holds one worktree per in-flight agent — an entire + additional checkout of the repo — so under `COPY . .` the build context + inflates by a multiple of the repo and another session's unreviewed work can + be copied into an image layer. In `.gitignore` the entry is `.claude/`, + unanchored. In `.dockerignore` it is `.claude`, anchored and with **no** `**/` + prefix, because the prefixed form would also delete any nested directory of + that name from the build. Anchoring carries a known gap that the canonical + `.dockerignore` states in its own comment, since consuming repos receive the + file and not the tracker: the directory is created in the agent's working + directory, so a repo running agents in subdirectories still ships + `services/api/.claude/` and must add its own anchored entry there. + +- **A plain `docker build .` of a clone stamps the version that + `git describe --tags --always` gives**, derived from the `.git` in the build + context as the canonical `Dockerfile` above shows. Without its failure check, + a missing `git` or an unreadable checkout would leave `-X main.Version=` empty + and the build would still exit 0. `script/docker` and `script/cibuild` pass + the version they compute on the host; it takes precedence. They do this + byte-identically across repos: + + ```sh + # Own line: a failing command substitution inside an argument does not + # trip `set -e`, so the inline form degrades to an empty constant. + version="$(git describe --tags --always --dirty 2>/dev/null || true)" + [ -n "$version" ] || version="unknown" + docker build --no-cache \ + --build-arg VERSION="$version" \ + -t "$(script/projectname)" . + ``` + + `--always` makes an untagged repo yield an abbreviated commit hash rather + than failing, and the `[ -n "$version" ]` line is the single place the + fallback is applied — a live check that fires on a build from an export with + no `.git` and on a repository with no commits yet. Do not fold it into the + substitution as `|| echo unknown`, which makes the guard unreachable. The + Dockerfile's side is `ARG VERSION` in the stage that compiles, declared + there because `ARG` is stage-scoped; passing `VERSION` to a repo whose + Dockerfile declares no such `ARG` is ignored and costs nothing, which is why + the scripts stay byte-identical. One consequence for CI: the standard + checkout action clones shallow and fetches no tags, so a repo that embeds a + tag-derived version must set `fetch-depth: 0` on its checkout step. + +- **Verify `.dockerignore` by enumerating the image, not by reading the + patterns.** Plant files at the root _and_ at least two directories deep, build + a probe image that does `COPY . .`, and list what actually landed + (`docker run --rm --entrypoint find IMAGE /app`). The `transferring context` + size is not a substitute: a nested secret is a few bytes, and BuildKit + transfers only the delta from the previous build. - **No build artifacts in version control.** Code-derived data (compiled bundles, minified output, generated assets) must never be committed to the @@ -258,9 +476,56 @@ style conventions are in separate documents: - Make all changes on a feature branch. You can do whatever you want on a feature branch. -- `.golangci.yml` is standardized and must _NEVER_ be modified by an agent, only - manually by the user. Fetch from - `https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml`. +- `.golangci.yml` is standardized. The vendored copy in a consuming repo must + _NEVER_ be modified by an agent: fetch it from + `https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml` and keep it + byte-identical, so that no repo can quietly loosen its own linting. Linter + configuration changes are made to the canonical copy in the `prompts` repo and + reach consuming repos by re-vendoring; an agent may open a PR against + canonical, which only the user merges. One list is exempt from byte-identity, + because it cannot be written once for every repo: the `deny` list of the + `test-support` depguard rule, where a repo names its own test-support packages + by full import path. A repo adds entries there and changes nothing else, and a + re-vendor carries its entries forward. The canonical golangci-lint version is + v2.14.0 (released 2026-09-24), pinned as the digest of the lint phase's base + image + (`golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f`, + which reports `2.14.0 built with go1.27.0 from 114493f9`). A module's `go` + directive must not name a newer Go minor version than the one golangci-lint + was built with, or golangci-lint refuses to lint it: this release lints + `go 1.27.1` but not `go 1.28`. That digest is the only pin, since no repo + installs golangci-lint on the host. A repo sets the lint phase digest to the + one named here and re-vendors `.golangci.yml` in the same commit, whichever of + the two prompted the change: the canonical copy can name linters that an older + golangci-lint rejects, and a newer golangci-lint can add linters that + `default: all` switches on until the canonical copy disables them. + +- **`script/bootstrap` installs a pinned tool by comparing versions, never by + testing presence.** An `if ! command -v ; then install; fi` guard tests + `PATH` only, so on an already-provisioned machine the pin is inert and a + version bump is a silent no-op — while the Dockerfile, installing into a clean + image, gets the pinned version, so a local `make check` and `make docker` can + disagree about what the tool even is. The canonical form: + - compares the installed version against the pin over the **whole** version + token; a parser that stops at the first `-` reports `2.12.2` for a host + running `2.12.2-rc1` and skips the install; + - treats absent, non-zero, empty or unrecognised `--version` output as a + mismatch, so the failure direction is a redundant install and never a + skipped one; + - after installing, re-resolves the binary the way callers do — `hash -r`, + then through `PATH`, not through the directory the installer wrote to — + and fails naming the resolved path, since an install that a shadowing + binary hides succeeds while changing nothing any caller sees; + - is actually called, and prints the version on both success paths: a + function defined and never invoked has the same exit status and the same + empty output as one that worked. + + Keep it POSIX sh: no arrays, no `[[`, no `grep -P`. + + A Go tool a repo needs on the host is installed with `go install` pinned to + a commit hash (`go install @`). It is never tracked as + a `go.mod` tool dependency or through a `tools.go` file, either of which + pulls the tool's own dependencies into the repo's `go.mod` and `go.sum`. - When pinning images or packages by hash, add a comment above the reference with the version and date (YYYY-MM-DD). @@ -374,12 +639,14 @@ style conventions are in separate documents: settings. - Avoid putting files in the repo root unless necessary. Root should contain - only project-level config files (`README.md`, `Makefile`, `Dockerfile`, - `LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, and - language-specific config). Everything else goes in a subdirectory. Canonical - subdirectory names: + only project-level config files (`README.md`, `AGENTS.md`, `Makefile`, + `Dockerfile`, `LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, + and language-specific config). Everything else goes in a subdirectory. + Canonical subdirectory names: - `bin/` — executable scripts and tools - - `cmd/` — Go command entrypoints + - `cmd/` — Go command entrypoints; thin only: one `main.go` per binary whose + body is a single call into `internal/` or `pkg/`, no project logic in + `cmd/` - `configs/` — configuration templates and examples - `deploy/` — deployment manifests (k8s, compose, terraform) - `docs/` — documentation and markdown (README.md stays in root) @@ -406,3 +673,7 @@ style conventions are in separate documents: - Go: `go.mod`, `go.sum`, `.golangci.yml` - JS: `package.json`, `yarn.lock`, `.prettierrc`, `.prettierignore` - Python: `pyproject.toml` + +- Guidance for coding agents lives in one `AGENTS.md` at the repository root. It + is never committed under a file or directory named after one agent tool, such + as `CLAUDE.md` or `.claude/`, and never split into separate memory files. diff --git a/TODO.md b/TODO.md index d0b7920..c3c80d2 100644 --- a/TODO.md +++ b/TODO.md @@ -10,8 +10,9 @@ # Status -pre-1.0. No git tags. The Docker build runs the format check, the linter -and the tests, and the Gitea workflow runs that build on every push. The +pre-1.0. No git tags. The linter and the tests are phases of the Docker +build, and the Gitea workflow runs `script/cibuild` (the format check, both +phases, then the image) on every push. The image sets memory ceilings for a 5 GiB container (README "Memory") and the README says how to run it under upaas (README "Running under upaas"). A 35-hour run of `3898daa` on the live feed peaked at about 1 GiB, without a @@ -27,6 +28,10 @@ real 5 GiB limit (https://git.eeqj.de/sneak/routewatch/issues/3). # Completed Steps +- 2026-10-14: the shared files are re-vendored from `sneak/prompts` at + `dd4027b`: golangci-lint v2.14.0 and its configuration, lint and test as + separate uncached Dockerfile phases, agent guidance in `AGENTS.md`; the code + is fixed for what the new linter configuration finds (closes #52) - 2026-10-03: `/api/v1/stats` serves the prefix distribution from memory, seeded at startup and adjusted on every live-route write, so a request no longer reads every live route (closes #30) diff --git a/cmd/asinfo-gen/main.go b/cmd/asinfo-gen/main.go index 1d895b9..fbc0a50 100644 --- a/cmd/asinfo-gen/main.go +++ b/cmd/asinfo-gen/main.go @@ -2,8 +2,10 @@ package main import ( + "context" "encoding/csv" "encoding/json" + "errors" "io" "log" "net/http" @@ -27,13 +29,17 @@ func main() { // Fetch the CSV data log.Println("Fetching AS CSV data...") - resp, err := http.Get(asCsvURL) + + req, err := http.NewRequestWithContext( + context.Background(), http.MethodGet, asCsvURL, nil) + if err != nil { + log.Fatalf("Failed to fetch CSV: %v", err) + } + + resp, err := http.DefaultClient.Do(req) if err != nil { log.Fatalf("Failed to fetch CSV: %v", err) } - defer func() { - _ = resp.Body.Close() - }() if resp.StatusCode != http.StatusOK { log.Fatalf("Failed to fetch CSV: HTTP %d", resp.StatusCode) @@ -48,17 +54,20 @@ func main() { log.Fatalf("Failed to read CSV header: %v", err) } - if len(header) != 3 || header[0] != "asn" || header[1] != "handle" || header[2] != "description" { + if len(header) != 3 || header[0] != "asn" || header[1] != "handle" || + header[2] != "description" { log.Fatalf("Unexpected CSV header: %v", header) } // Read all records var asInfos []ASInfo + for { record, err := reader.Read() - if err == io.EOF { + if errors.Is(err, io.EOF) { break } + if err != nil { log.Printf("Error reading CSV record: %v", err) @@ -86,12 +95,18 @@ func main() { }) } + // Closed here rather than deferred: log.Fatalf exits without running + // deferred calls. + _ = resp.Body.Close() + log.Printf("Parsed %d AS records", len(asInfos)) // Convert to JSON and output to stdout encoder := json.NewEncoder(os.Stdout) encoder.SetIndent("", "\t") - if err := encoder.Encode(asInfos); err != nil { + + err = encoder.Encode(asInfos) + if err != nil { log.Fatalf("Failed to encode JSON: %v", err) } } diff --git a/cmd/streamdumper/main.go b/cmd/streamdumper/main.go index 7754d7a..95a0722 100644 --- a/cmd/streamdumper/main.go +++ b/cmd/streamdumper/main.go @@ -26,9 +26,17 @@ func main() { // Register raw message handler that prints to stdout s.RegisterRawHandler(func(line string) { - fmt.Println(line) + _, _ = fmt.Fprintln(os.Stdout, line) }) + // Start streaming before any defer: log.Fatal exits without running + // deferred calls. + err := s.Start(context.Background()) + if err != nil { + log.Fatal("Failed to start streamer:", err) + } + defer s.Stop() + // Set up context with cancellation ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -36,18 +44,13 @@ func main() { // Handle signals sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) + go func() { <-sigChan log.Println("Received shutdown signal") cancel() }() - // Start streaming - if err := s.Start(); err != nil { - log.Fatal("Failed to start streamer:", err) - } - defer s.Stop() - // Wait for context cancellation <-ctx.Done() } diff --git a/internal/config/config.go b/internal/config/config.go index 87a6ce3..efb290f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -2,6 +2,7 @@ package config import ( + "errors" "fmt" "os" "path/filepath" @@ -27,6 +28,12 @@ const ( maxPort = 65535 ) +var ( + errRelativeXDGDataHome = errors.New("XDG_DATA_HOME must be an absolute path") + errUnsupportedOS = errors.New("unsupported operating system") + errInvalidPort = errors.New("PORT must be a whole number") +) + // Config holds configuration for the entire application type Config struct { // StateDir is the directory for all application state (database, snapshots) @@ -38,11 +45,14 @@ type Config struct { // MaxRuntime is the maximum runtime (0 = run forever) MaxRuntime time.Duration - // EnableBatchedDatabaseWrites enables batched database operations for better performance + // EnableBatchedDatabaseWrites enables batched database operations for + // better performance EnableBatchedDatabaseWrites bool - // RouteExpirationTimeout is how long a route can go without being refreshed before expiring - // Default is 2 hours which is conservative for BGP (typical BGP hold time is 90-180 seconds) + // RouteExpirationTimeout is how long a route can go without being + // refreshed before expiring + // Default is 2 hours which is conservative for BGP (typical BGP hold + // time is 90-180 seconds) RouteExpirationTimeout time.Duration } @@ -59,11 +69,14 @@ func New() (*Config, error) { } return &Config{ - StateDir: stateDir, - Port: port, - MaxRuntime: 0, // Run forever by default - EnableBatchedDatabaseWrites: true, // Enable batching by default - RouteExpirationTimeout: defaultRouteExpirationMinutes * time.Minute, // For active route monitoring + StateDir: stateDir, + Port: port, + // Run forever by default + MaxRuntime: 0, + // Enable batching by default + EnableBatchedDatabaseWrites: true, + // For active route monitoring + RouteExpirationTimeout: defaultRouteExpirationMinutes * time.Minute, }, nil } @@ -89,7 +102,7 @@ func getStateDirectory() (string, error) { // the database somewhere unexpected. xdgData := os.Getenv("XDG_DATA_HOME") if xdgData != "" && !filepath.IsAbs(xdgData) { - return "", fmt.Errorf("XDG_DATA_HOME must be an absolute path, got %q", xdgData) + return "", fmt.Errorf("%w, got %q", errRelativeXDGDataHome, xdgData) } // Unix-like: /var/lib/berlin.sneak.app.routewatch if root, else XDG_DATA_HOME @@ -111,7 +124,7 @@ func getStateDirectory() (string, error) { return filepath.Join(home, ".local", "share", AppIdentifier), nil default: - return "", fmt.Errorf("unsupported operating system: %s", runtime.GOOS) + return "", fmt.Errorf("%w: %s", errUnsupportedOS, runtime.GOOS) } } @@ -125,7 +138,8 @@ func getPort() (int, error) { // ParseUint, unlike Atoi, refuses a sign: the health check URL cannot use "+9090" port, err := strconv.ParseUint(value, 10, 0) if err != nil || port < 1 || port > maxPort { - return 0, fmt.Errorf("PORT must be a whole number from 1 to %d, got %q", maxPort, value) + return 0, fmt.Errorf("%w from 1 to %d, got %q", + errInvalidPort, maxPort, value) } return int(port), nil @@ -134,7 +148,8 @@ func getPort() (int, error) { // EnsureDirectories creates all necessary directories if they don't exist func (c *Config) EnsureDirectories() error { // Ensure state directory exists - if err := os.MkdirAll(c.StateDir, dirPermissions); err != nil { + err := os.MkdirAll(c.StateDir, dirPermissions) + if err != nil { return fmt.Errorf("failed to create state directory: %w", err) } diff --git a/internal/config/config_test.go b/internal/config/config_internal_test.go similarity index 91% rename from internal/config/config_test.go rename to internal/config/config_internal_test.go index 84f9414..c9f10bd 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_internal_test.go @@ -36,7 +36,8 @@ func TestNewRefusesInvalidPort(t *testing.T) { t.Setenv("PORT", value) t.Setenv("XDG_DATA_HOME", "") - if _, err := New(); err == nil { + _, err := New() + if err == nil { t.Errorf("New() with PORT=%q returned no error", value) } }) @@ -51,12 +52,16 @@ func TestNewRefusesRelativeXDGDataHome(t *testing.T) { t.Setenv("PORT", "") t.Setenv("XDG_DATA_HOME", "relative/path") - if _, err := New(); err == nil { + + _, err := New() + if err == nil { t.Error("New() with a relative XDG_DATA_HOME returned no error") } t.Setenv("XDG_DATA_HOME", "/var/lib") - if _, err := New(); err != nil { + + _, err = New() + if err != nil { t.Errorf("New() with XDG_DATA_HOME=/var/lib: %v", err) } } diff --git a/internal/database/counts.go b/internal/database/counts.go index 71ce82f..5c8d4a5 100644 --- a/internal/database/counts.go +++ b/internal/database/counts.go @@ -33,8 +33,10 @@ type liveCounts struct { // seed sets every count to the value read from the tables at startup. It runs // before any writer, so it needs no coordination with the adjust methods. -func (c *liveCounts) seed(asns, prefixesV4, prefixesV6, peerings, peers, routesV4, routesV6 int, - distributionV4, distributionV6 []PrefixDistribution) { +func (c *liveCounts) seed( + asns, prefixesV4, prefixesV6, peerings, peers, routesV4, routesV6 int, + distributionV4, distributionV6 []PrefixDistribution, +) { c.mu.Lock() defer c.mu.Unlock() @@ -45,9 +47,11 @@ func (c *liveCounts) seed(asns, prefixesV4, prefixesV6, peerings, peers, routesV c.peers = peers c.routesV4 = routesV4 c.routesV6 = routesV6 + for _, entry := range distributionV4 { addAtMaskLength(c.distributionV4[:], entry.MaskLength, entry.Count) } + for _, entry := range distributionV6 { addAtMaskLength(c.distributionV6[:], entry.MaskLength, entry.Count) } @@ -102,6 +106,7 @@ func (c *liveCounts) addToDistribution(maskLengthsV4, maskLengthsV6 []int, n int for _, maskLength := range maskLengthsV4 { addAtMaskLength(c.distributionV4[:], maskLength, n) } + for _, maskLength := range maskLengthsV6 { addAtMaskLength(c.distributionV6[:], maskLength, n) } @@ -138,9 +143,11 @@ func (c *liveCounts) fill(s *Stats) { // ascending order, the way the distribution query returns them. func distributionList(counts []int) []PrefixDistribution { var list []PrefixDistribution + for maskLength, count := range counts { if count > 0 { - list = append(list, PrefixDistribution{MaskLength: maskLength, Count: count}) + list = append(list, + PrefixDistribution{MaskLength: maskLength, Count: count}) } } @@ -152,7 +159,8 @@ func distributionList(counts []int) []PrefixDistribution { func (d *Database) countRows(ctx context.Context, table string) (int, error) { var n int // table is one of a fixed set of literals below, never external input. - if err := d.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM "+table).Scan(&n); err != nil { + err := d.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM "+table).Scan(&n) + if err != nil { return 0, fmt.Errorf("failed to count %s: %w", table, err) } @@ -167,30 +175,37 @@ func (d *Database) seedCounts(ctx context.Context) error { if err != nil { return err } + prefixesV4, err := d.countRows(ctx, "prefixes_v4") if err != nil { return err } + prefixesV6, err := d.countRows(ctx, "prefixes_v6") if err != nil { return err } + peerings, err := d.countRows(ctx, "peerings") if err != nil { return err } + peers, err := d.countRows(ctx, "bgp_peers") if err != nil { return err } - routesV4, err := d.countRows(ctx, "live_routes_v4") + + routesV4, err := d.countRows(ctx, tableLiveRoutesV4) if err != nil { return err } - routesV6, err := d.countRows(ctx, "live_routes_v6") + + routesV6, err := d.countRows(ctx, tableLiveRoutesV6) if err != nil { return err } + distributionV4, distributionV6, err := d.GetPrefixDistributionContext(ctx) if err != nil { return err diff --git a/internal/database/counts_test.go b/internal/database/counts_internal_test.go similarity index 80% rename from internal/database/counts_test.go rename to internal/database/counts_internal_test.go index 8922619..c3f0a0e 100644 --- a/internal/database/counts_test.go +++ b/internal/database/counts_internal_test.go @@ -12,6 +12,12 @@ import ( "github.com/google/uuid" ) +// testV4PeerIP is the peer the IPv4 routes in these tests are learned from. +const testV4PeerIP = "192.0.2.1" + +// testV6Prefix is the prefix of every IPv6 route in these tests. +const testV6Prefix = "2001:db8::/32" + // mkV4Route builds an IPv4 live route with its mask length taken from the // prefix. func mkV4Route(t *testing.T, prefix string, asn int, ts time.Time) *LiveRoute { @@ -28,18 +34,18 @@ func mkV4Route(t *testing.T, prefix string, asn int, ts time.Time) *LiveRoute { MaskLength: maskLength, IPVersion: ipVersionV4, OriginASN: asn, - PeerIP: "192.0.2.1", + PeerIP: testV4PeerIP, ASPath: []int{asn}, NextHop: "192.0.2.254", LastUpdated: ts, } } -// mkV6Route builds an IPv6 live route. -func mkV6Route(prefix string, asn int, ts time.Time) *LiveRoute { +// mkV6Route builds an IPv6 live route for testV6Prefix. +func mkV6Route(asn int, ts time.Time) *LiveRoute { return &LiveRoute{ ID: uuid.New(), - Prefix: prefix, + Prefix: testV6Prefix, MaskLength: 32, IPVersion: ipVersionV6, OriginASN: asn, @@ -56,6 +62,8 @@ func mkV6Route(prefix string, asn int, ts time.Time) *LiveRoute { // deleted. These counts are what /api/v1/stats reports; before this change the // endpoint recomputed them with a COUNT(*) over each table on every request. func TestLiveCountsTrackWritesInRealtime(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -70,6 +78,7 @@ func TestLiveCountsTrackWritesInRealtime(t *testing.T) { if err != nil { t.Fatalf("GetStatsContext on empty database: %v", err) } + if empty.ASNs != 0 || empty.Prefixes != 0 || empty.Peerings != 0 || empty.Peers != 0 || empty.LiveRoutes != 0 { t.Fatalf("empty database counts nonzero: %+v", empty) @@ -77,30 +86,16 @@ func TestLiveCountsTrackWritesInRealtime(t *testing.T) { ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) - if err := db.GetOrCreateASNBatch(map[int]time.Time{64500: ts, 64501: ts}); err != nil { - t.Fatalf("GetOrCreateASNBatch: %v", err) - } - if err := db.UpdatePrefixesBatch(map[string]time.Time{ - "198.51.100.0/24": ts, - "2001:db8::/32": ts, - }); err != nil { - t.Fatalf("UpdatePrefixesBatch: %v", err) - } - if err := db.UpdatePeerBatch(map[string]PeerUpdate{ - "192.0.2.1": {PeerIP: "192.0.2.1", PeerASN: 64500, MessageType: "UPDATE", Timestamp: ts}, - }); err != nil { - t.Fatalf("UpdatePeerBatch: %v", err) - } - if err := db.RecordPeering(64500, 64501, ts); err != nil { - t.Fatalf("RecordPeering: %v", err) - } + writeASNsPrefixesAndPeers(t, db, ts) routes := []*LiveRoute{ mkV4Route(t, "198.51.100.0/24", 64500, ts), mkV4Route(t, "203.0.113.0/24", 64501, ts.Add(time.Minute)), - mkV6Route("2001:db8::/32", 64502, ts.Add(2*time.Minute)), + mkV6Route(64502, ts.Add(2*time.Minute)), } - if err := db.UpsertLiveRouteBatch(routes); err != nil { + + err = db.UpsertLiveRouteBatch(routes) + if err != nil { t.Fatalf("UpsertLiveRouteBatch: %v", err) } @@ -108,44 +103,88 @@ func TestLiveCountsTrackWritesInRealtime(t *testing.T) { if err != nil { t.Fatalf("GetStatsContext: %v", err) } + assertCounts(t, "after inserts", stats, wantCounts{ asns: 2, prefixes: 2, peerings: 1, peers: 1, ipv4Routes: 2, ipv6Routes: 1, liveRoutes: 3, }) // Re-announcing the same routes is an update, not an insert: counts hold. - if err := db.UpsertLiveRouteBatch(routes); err != nil { + err = db.UpsertLiveRouteBatch(routes) + if err != nil { t.Fatalf("UpsertLiveRouteBatch (re-announce): %v", err) } + stats, err = db.GetStatsContext(ctx) if err != nil { t.Fatalf("GetStatsContext: %v", err) } + assertCounts(t, "after re-announce", stats, wantCounts{ asns: 2, prefixes: 2, peerings: 1, peers: 1, ipv4Routes: 2, ipv6Routes: 1, liveRoutes: 3, }) // A withdrawal removes one route. - if err := db.DeleteLiveRouteBatch([]LiveRouteDeletion{ - {Prefix: "203.0.113.0/24", OriginASN: 64501, PeerIP: "192.0.2.1", IPVersion: ipVersionV4}, - }); err != nil { + err = db.DeleteLiveRouteBatch([]LiveRouteDeletion{{ + Prefix: "203.0.113.0/24", OriginASN: 64501, + PeerIP: testV4PeerIP, IPVersion: ipVersionV4, + }}) + if err != nil { t.Fatalf("DeleteLiveRouteBatch: %v", err) } + stats, err = db.GetStatsContext(ctx) if err != nil { t.Fatalf("GetStatsContext: %v", err) } + assertCounts(t, "after delete", stats, wantCounts{ asns: 2, prefixes: 2, peerings: 1, peers: 1, ipv4Routes: 1, ipv6Routes: 1, liveRoutes: 2, }) } +// writeASNsPrefixesAndPeers writes two ASNs, two prefixes, one peer and one +// peering: the rows besides live routes that the stats count. +func writeASNsPrefixesAndPeers(t *testing.T, db *Database, ts time.Time) { + t.Helper() + + err := db.GetOrCreateASNBatch(map[int]time.Time{64500: ts, 64501: ts}) + if err != nil { + t.Fatalf("GetOrCreateASNBatch: %v", err) + } + + err = db.UpdatePrefixesBatch(map[string]time.Time{ + "198.51.100.0/24": ts, + testV6Prefix: ts, + }) + if err != nil { + t.Fatalf("UpdatePrefixesBatch: %v", err) + } + + err = db.UpdatePeerBatch(map[string]PeerUpdate{ + testV4PeerIP: { + PeerIP: testV4PeerIP, PeerASN: 64500, + MessageType: "UPDATE", Timestamp: ts, + }, + }) + if err != nil { + t.Fatalf("UpdatePeerBatch: %v", err) + } + + err = db.RecordPeering(64500, 64501, ts) + if err != nil { + t.Fatalf("RecordPeering: %v", err) + } +} + // TestLiveCountsSeededFromDatabaseAtStartup writes rows, reopens the same // database file, and checks the counts and the prefix distribution come back // from the seed scan rather than starting at zero. func TestLiveCountsSeededFromDatabaseAtStartup(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -154,16 +193,22 @@ func TestLiveCountsSeededFromDatabaseAtStartup(t *testing.T) { } ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) - if err := db.GetOrCreateASNBatch(map[int]time.Time{64500: ts, 64501: ts, 64502: ts}); err != nil { + + err = db.GetOrCreateASNBatch(map[int]time.Time{64500: ts, 64501: ts, 64502: ts}) + if err != nil { t.Fatalf("GetOrCreateASNBatch: %v", err) } - if err := db.UpsertLiveRouteBatch([]*LiveRoute{ + + err = db.UpsertLiveRouteBatch([]*LiveRoute{ mkV4Route(t, "198.51.100.0/24", 64500, ts), - mkV6Route("2001:db8::/32", 64502, ts), - }); err != nil { + mkV6Route(64502, ts), + }) + if err != nil { t.Fatalf("UpsertLiveRouteBatch: %v", err) } - if err := db.Close(); err != nil { + + err = db.Close() + if err != nil { t.Fatalf("Close: %v", err) } @@ -177,13 +222,16 @@ func TestLiveCountsSeededFromDatabaseAtStartup(t *testing.T) { if err != nil { t.Fatalf("GetStatsContext after reopen: %v", err) } + if stats.ASNs != 3 { t.Errorf("seeded ASNs = %d, want 3", stats.ASNs) } + if stats.IPv4Routes != 1 || stats.IPv6Routes != 1 || stats.LiveRoutes != 2 { t.Errorf("seeded routes = (v4 %d, v6 %d, total %d), want (1, 1, 2)", stats.IPv4Routes, stats.IPv6Routes, stats.LiveRoutes) } + assertDistribution(t, "seeded IPv4 distribution", stats.IPv4PrefixDistribution, []PrefixDistribution{{MaskLength: 24, Count: 1}}) assertDistribution(t, "seeded IPv6 distribution", stats.IPv6PrefixDistribution, @@ -196,26 +244,91 @@ func TestLiveCountsSeededFromDatabaseAtStartup(t *testing.T) { // tables. The steps run once through the batch methods the prefix handler uses // and once through the single-route methods. func TestPrefixDistributionTracksWrites(t *testing.T) { + t.Parallel() + + steps := prefixDistributionSteps(t) + + for _, batch := range []bool{true, false} { + name := "single-route writes" + if batch { + name = "batch writes" + } + + t.Run(name, func(t *testing.T) { + t.Parallel() + + db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New()) + if err != nil { + t.Fatalf("failed to create database: %v", err) + } + defer func() { _ = db.Close() }() + + ctx := context.Background() + + for _, step := range steps { + err := announceRoutes(db, batch, step.announce) + if err != nil { + t.Fatalf("%s: announce: %v", step.name, err) + } + + err = withdrawRoutes(db, batch, step.withdraw) + if err != nil { + t.Fatalf("%s: withdraw: %v", step.name, err) + } + + stats, err := db.GetStatsContext(ctx) + if err != nil { + t.Fatalf("%s: GetStatsContext: %v", step.name, err) + } + + queryV4, queryV6, err := db.GetPrefixDistributionContext(ctx) + if err != nil { + t.Fatalf("%s: GetPrefixDistributionContext: %v", step.name, err) + } + + assertDistribution(t, step.name+": IPv4 distribution", + stats.IPv4PrefixDistribution, step.wantV4) + assertDistribution(t, step.name+": IPv6 distribution", + stats.IPv6PrefixDistribution, step.wantV6) + assertDistribution(t, step.name+": IPv4 distribution query", + queryV4, step.wantV4) + assertDistribution(t, step.name+": IPv6 distribution query", + queryV6, step.wantV6) + } + }) + } +} + +// prefixDistributionStep is one step of TestPrefixDistributionTracksWrites: +// the routes it announces, then the routes it withdraws, then the +// distributions both the stats and the distribution query must report. +type prefixDistributionStep struct { + name string + announce []*LiveRoute + withdraw []*LiveRoute + wantV4 []PrefixDistribution + wantV6 []PrefixDistribution +} + +// prefixDistributionSteps returns the steps of +// TestPrefixDistributionTracksWrites, in the order they run. +func prefixDistributionSteps(t *testing.T) []prefixDistributionStep { + t.Helper() + ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) shared := mkV4Route(t, "198.51.100.0/24", 64500, ts) sharedSecondPeer := mkV4Route(t, "198.51.100.0/24", 64500, ts) sharedSecondPeer.PeerIP = "192.0.2.2" other := mkV4Route(t, "203.0.113.0/24", 64501, ts) wide := mkV4Route(t, "172.16.0.0/16", 64502, ts) - v6 := mkV6Route("2001:db8::/32", 64503, ts) - v6SecondPeer := mkV6Route("2001:db8::/32", 64503, ts) + v6 := mkV6Route(64503, ts) + v6SecondPeer := mkV6Route(64503, ts) v6SecondPeer.PeerIP = "2001:db8::2" all := []PrefixDistribution{{MaskLength: 16, Count: 1}, {MaskLength: 24, Count: 2}} v6Only := []PrefixDistribution{{MaskLength: 32, Count: 1}} - steps := []struct { - name string - announce []*LiveRoute - withdraw []*LiveRoute - wantV4 []PrefixDistribution - wantV6 []PrefixDistribution - }{ + return []prefixDistributionStep{ { name: "new routes", announce: []*LiveRoute{shared, other, wide, v6}, @@ -239,7 +352,9 @@ func TestPrefixDistributionTracksWrites(t *testing.T) { { name: "withdrawal of the last route for a prefix", withdraw: []*LiveRoute{sharedSecondPeer, v6SecondPeer}, - wantV4: []PrefixDistribution{{MaskLength: 16, Count: 1}, {MaskLength: 24, Count: 1}}, + wantV4: []PrefixDistribution{ + {MaskLength: 16, Count: 1}, {MaskLength: 24, Count: 1}, + }, }, { name: "withdrawal of every remaining route", @@ -270,45 +385,6 @@ func TestPrefixDistributionTracksWrites(t *testing.T) { wantV6: v6Only, }, } - - for _, batch := range []bool{true, false} { - name := "single-route writes" - if batch { - name = "batch writes" - } - - t.Run(name, func(t *testing.T) { - db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New()) - if err != nil { - t.Fatalf("failed to create database: %v", err) - } - defer func() { _ = db.Close() }() - - ctx := context.Background() - for _, step := range steps { - if err := announceRoutes(db, batch, step.announce); err != nil { - t.Fatalf("%s: announce: %v", step.name, err) - } - if err := withdrawRoutes(db, batch, step.withdraw); err != nil { - t.Fatalf("%s: withdraw: %v", step.name, err) - } - - stats, err := db.GetStatsContext(ctx) - if err != nil { - t.Fatalf("%s: GetStatsContext: %v", step.name, err) - } - queryV4, queryV6, err := db.GetPrefixDistributionContext(ctx) - if err != nil { - t.Fatalf("%s: GetPrefixDistributionContext: %v", step.name, err) - } - - assertDistribution(t, step.name+": IPv4 distribution", stats.IPv4PrefixDistribution, step.wantV4) - assertDistribution(t, step.name+": IPv6 distribution", stats.IPv6PrefixDistribution, step.wantV6) - assertDistribution(t, step.name+": IPv4 distribution query", queryV4, step.wantV4) - assertDistribution(t, step.name+": IPv6 distribution query", queryV6, step.wantV6) - } - }) - } } // announceRoutes writes routes in one UpsertLiveRouteBatch, or with one @@ -317,8 +393,10 @@ func announceRoutes(db *Database, batch bool, routes []*LiveRoute) error { if batch { return db.UpsertLiveRouteBatch(routes) } + for _, route := range routes { - if err := db.UpsertLiveRoute(route); err != nil { + err := db.UpsertLiveRoute(route) + if err != nil { return err } } @@ -333,7 +411,8 @@ func announceRoutes(db *Database, batch bool, routes []*LiveRoute) error { func withdrawRoutes(db *Database, batch bool, routes []*LiveRoute) error { if !batch { for _, route := range routes { - if err := db.DeleteLiveRoute(route.Prefix, 0, route.PeerIP); err != nil { + err := db.DeleteLiveRoute(route.Prefix, 0, route.PeerIP) + if err != nil { return err } } @@ -356,6 +435,8 @@ func withdrawRoutes(db *Database, batch bool, routes []*LiveRoute) error { // MIN/MAX union query read its result into *time.Time, which the driver could // not parse, so it logged a warning every call and left both timestamps nil. func TestStatsRouteTimestamps(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -370,6 +451,7 @@ func TestStatsRouteTimestamps(t *testing.T) { if err != nil { t.Fatalf("GetStatsContext on empty database: %v", err) } + if empty.OldestRoute != nil || empty.NewestRoute != nil { t.Fatalf("empty database timestamps = (%v, %v), want (nil, nil)", empty.OldestRoute, empty.NewestRoute) @@ -378,11 +460,13 @@ func TestStatsRouteTimestamps(t *testing.T) { base := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) oldest := base newest := base.Add(2 * time.Minute) - if err := db.UpsertLiveRouteBatch([]*LiveRoute{ + + err = db.UpsertLiveRouteBatch([]*LiveRoute{ mkV4Route(t, "198.51.100.0/24", 64500, base.Add(time.Minute)), mkV4Route(t, "203.0.113.0/24", 64501, oldest), - mkV6Route("2001:db8::/32", 64502, newest), - }); err != nil { + mkV6Route(64502, newest), + }) + if err != nil { t.Fatalf("UpsertLiveRouteBatch: %v", err) } @@ -390,9 +474,11 @@ func TestStatsRouteTimestamps(t *testing.T) { if err != nil { t.Fatalf("GetStatsContext: %v", err) } + if stats.OldestRoute == nil || !stats.OldestRoute.Equal(oldest) { t.Errorf("OldestRoute = %v, want %v", stats.OldestRoute, oldest) } + if stats.NewestRoute == nil || !stats.NewestRoute.Equal(newest) { t.Errorf("NewestRoute = %v, want %v", stats.NewestRoute, newest) } @@ -401,6 +487,8 @@ func TestStatsRouteTimestamps(t *testing.T) { // TestLiveCountsConcurrentReadWrite runs writers and stats readers at once so // the race detector proves the counters are safe under concurrent use. func TestLiveCountsConcurrentReadWrite(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -412,17 +500,21 @@ func TestLiveCountsConcurrentReadWrite(t *testing.T) { ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) const writers = 4 - var wg sync.WaitGroup + var wg sync.WaitGroup wg.Add(writers) + for w := range writers { go func(base int) { defer wg.Done() + for i := range 25 { asn := 65000 + base*100 + i - route := mkV6Route("2001:db8::/32", asn, ts) + route := mkV6Route(asn, ts) route.PeerIP = "2001:db8::" + uuid.NewString() - if err := db.UpsertLiveRoute(route); err != nil { + + err := db.UpsertLiveRoute(route) + if err != nil { t.Errorf("UpsertLiveRoute: %v", err) return @@ -433,15 +525,19 @@ func TestLiveCountsConcurrentReadWrite(t *testing.T) { var readerWG sync.WaitGroup readerWG.Add(1) + stop := make(chan struct{}) + go func() { defer readerWG.Done() + for { select { case <-stop: return default: - if _, err := db.GetStatsContext(context.Background()); err != nil { + _, err := db.GetStatsContext(context.Background()) + if err != nil { t.Errorf("GetStatsContext: %v", err) return @@ -458,6 +554,7 @@ func TestLiveCountsConcurrentReadWrite(t *testing.T) { if err != nil { t.Fatalf("GetStatsContext: %v", err) } + if want := writers * 25; stats.IPv6Routes != want { t.Errorf("IPv6Routes = %d, want %d", stats.IPv6Routes, want) } @@ -482,21 +579,27 @@ func assertCounts(t *testing.T, when string, got Stats, want wantCounts) { if got.ASNs != want.asns { t.Errorf("%s: ASNs = %d, want %d", when, got.ASNs, want.asns) } + if got.Prefixes != want.prefixes { t.Errorf("%s: Prefixes = %d, want %d", when, got.Prefixes, want.prefixes) } + if got.Peerings != want.peerings { t.Errorf("%s: Peerings = %d, want %d", when, got.Peerings, want.peerings) } + if got.Peers != want.peers { t.Errorf("%s: Peers = %d, want %d", when, got.Peers, want.peers) } + if got.IPv4Routes != want.ipv4Routes { t.Errorf("%s: IPv4Routes = %d, want %d", when, got.IPv4Routes, want.ipv4Routes) } + if got.IPv6Routes != want.ipv6Routes { t.Errorf("%s: IPv6Routes = %d, want %d", when, got.IPv6Routes, want.ipv6Routes) } + if got.LiveRoutes != want.liveRoutes { t.Errorf("%s: LiveRoutes = %d, want %d", when, got.LiveRoutes, want.liveRoutes) } diff --git a/internal/database/database.go b/internal/database/database.go index 468bd2b..3d197d1 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -1,4 +1,3 @@ -// Package database provides SQLite storage for BGP routing data including ASNs, prefixes, announcements and peerings. package database import ( @@ -35,6 +34,13 @@ const ( ipVersionV6 = 6 ipv4Bits = 32 ipv6Bits = 128 + + tableLiveRoutesV4 = "live_routes_v4" + tableLiveRoutesV6 = "live_routes_v6" + + // Names of the IP versions in error and log messages. + ipv4Family = "IPv4" + ipv6Family = "IPv6" ) // The IP lookup does not use routes shorter than these mask lengths. A default @@ -69,6 +75,9 @@ var ( ErrNoRoute = errors.New("no route found") // ErrNoStaleASN is returned when no ASN needs WHOIS refresh ErrNoStaleASN = errors.New("no stale ASN found") + + errInvalidASN = errors.New("invalid ASN") + errSameASNInPeering = errors.New("cannot create peering with same ASN") ) // Database manages the SQLite database connection and operations. @@ -91,7 +100,9 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) { // Ensure directory exists dir := filepath.Dir(dbPath) - if err := os.MkdirAll(dir, dirPermissions); err != nil { + + err := os.MkdirAll(dir, dirPermissions) + if err != nil { return nil, fmt.Errorf("failed to create database directory: %w", err) } @@ -112,12 +123,14 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) { sqliteCacheSizeKiB, sqliteBusyTimeoutMs, ) + db, err := sql.Open("sqlite3", dsn) if err != nil { return nil, fmt.Errorf("failed to open database: %w", err) } - if err := db.Ping(); err != nil { + err = db.PingContext(context.Background()) + if err != nil { return nil, fmt.Errorf("failed to ping database: %w", err) } @@ -128,16 +141,20 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) { db.SetMaxIdleConns(maxConns) db.SetConnMaxLifetime(0) - database := &Database{db: db, logger: logger, path: dbPath, counts: &liveCounts{}} + database := &Database{ + db: db, logger: logger, path: dbPath, counts: &liveCounts{}, + } - if err := database.Initialize(); err != nil { + err = database.Initialize() + if err != nil { return nil, fmt.Errorf("failed to initialize database: %w", err) } // Seed the in-memory statistics counters from the tables once, before the // streamer starts writing. From here on every write keeps them current, so // the stats endpoints never scan the tables to report counts. - if err := database.seedCounts(context.Background()); err != nil { + err = database.seedCounts(context.Background()) + if err != nil { return nil, fmt.Errorf("failed to seed statistics counters: %w", err) } @@ -159,14 +176,18 @@ func (d *Database) Initialize() error { } for _, pragma := range pragmas { - if err := d.exec(pragma); err != nil { + err := d.exec(pragma) + if err != nil { d.logger.Warn("Failed to set pragma", "pragma", pragma, "error", err) } } // Run WAL checkpoint on startup to consolidate any existing WAL var walPages, checkpointedPages, movedPages int - err := d.db.QueryRow("PRAGMA wal_checkpoint(TRUNCATE)").Scan(&walPages, &checkpointedPages, &movedPages) + + err := d.db.QueryRowContext(context.Background(), + "PRAGMA wal_checkpoint(TRUNCATE)", + ).Scan(&walPages, &checkpointedPages, &movedPages) if err != nil { d.logger.Warn("Failed to checkpoint WAL on startup", "error", err) } else { @@ -190,43 +211,6 @@ func (d *Database) Close() error { return d.db.Close() } -// lock acquires the database mutex and logs debug information -func (d *Database) lock(operation string) { - // Get caller information - _, file, line, _ := runtime.Caller(1) - caller := fmt.Sprintf("%s:%d", filepath.Base(file), line) - - d.logger.Debug("Acquiring database lock", "operation", operation, "caller", caller) - - d.mu.Lock() - d.lockedAt = time.Now() - d.lockedBy = fmt.Sprintf("%s (%s)", operation, caller) - - d.logger.Debug("Database lock acquired", "operation", operation, "caller", caller) -} - -// unlock releases the database mutex and logs debug information including hold duration -func (d *Database) unlock() { - holdDuration := time.Since(d.lockedAt) - lockedBy := d.lockedBy - - d.lockedAt = time.Time{} - d.lockedBy = "" - d.mu.Unlock() - - d.logger.Debug("Database lock released", "held_by", lockedBy, "duration_ms", holdDuration.Milliseconds()) -} - -// beginTx starts a new transaction with logging -func (d *Database) beginTx() (*loggingTx, error) { - tx, err := d.db.Begin() - if err != nil { - return nil, err - } - - return &loggingTx{Tx: tx, logger: d.logger}, nil -} - // A live-route upsert is an UPDATE followed, only when no row matched, by an // INSERT. The UPDATE's rows-affected count (1 for an existing key, 0 for a new // one) is what lets the in-memory route counters stay exact without a COUNT(*). @@ -235,52 +219,80 @@ func (d *Database) beginTx() (*loggingTx, error) { // updated route keeps its original id, exactly as the previous ON CONFLICT // upsert did. const ( - updateLiveRouteV4SQL = `UPDATE live_routes_v4 SET mask_length = ?, as_path = ?, next_hop = ?, - last_updated = ? WHERE prefix = ? AND origin_asn = ? AND peer_ip = ?` + updateLiveRouteV4SQL = `UPDATE live_routes_v4 + SET mask_length = ?, as_path = ?, next_hop = ?, last_updated = ? + WHERE prefix = ? AND origin_asn = ? AND peer_ip = ?` - insertLiveRouteV4SQL = `INSERT INTO live_routes_v4 (id, prefix, mask_length, origin_asn, peer_ip, - as_path, next_hop, last_updated) VALUES (?, ?, ?, ?, ?, ?, ?, ?)` + insertLiveRouteV4SQL = `INSERT INTO live_routes_v4 + (id, prefix, mask_length, origin_asn, peer_ip, as_path, next_hop, + last_updated) VALUES (?, ?, ?, ?, ?, ?, ?, ?)` - updateLiveRouteV6SQL = `UPDATE live_routes_v6 SET mask_length = ?, as_path = ?, next_hop = ?, - last_updated = ? WHERE prefix = ? AND origin_asn = ? AND peer_ip = ?` + updateLiveRouteV6SQL = `UPDATE live_routes_v6 + SET mask_length = ?, as_path = ?, next_hop = ?, last_updated = ? + WHERE prefix = ? AND origin_asn = ? AND peer_ip = ?` - insertLiveRouteV6SQL = `INSERT INTO live_routes_v6 (id, prefix, mask_length, origin_asn, peer_ip, - as_path, next_hop, last_updated) VALUES (?, ?, ?, ?, ?, ?, ?, ?)` + insertLiveRouteV6SQL = `INSERT INTO live_routes_v6 + (id, prefix, mask_length, origin_asn, peer_ip, as_path, next_hop, + last_updated) VALUES (?, ?, ?, ?, ?, ?, ?, ?)` ) // Before a new route is inserted and after a route is deleted, the write looks // up whether any live route has that prefix, to keep the in-memory prefix // distribution exact. The lookup reads one entry of the prefix index. const ( - prefixHasLiveRouteV4SQL = `SELECT EXISTS (SELECT 1 FROM live_routes_v4 WHERE prefix = ?)` - prefixHasLiveRouteV6SQL = `SELECT EXISTS (SELECT 1 FROM live_routes_v6 WHERE prefix = ?)` + prefixHasLiveRouteV4SQL = `SELECT EXISTS + (SELECT 1 FROM live_routes_v4 WHERE prefix = ?)` + prefixHasLiveRouteV6SQL = `SELECT EXISTS + (SELECT 1 FROM live_routes_v6 WHERE prefix = ?)` ) -// upsertRouteRowV4 updates an IPv4 live route, inserting it when no row matched. -// It reports whether a new row was inserted and whether that row is the first -// live route for its prefix. -func upsertRouteRowV4(upd, ins, has *sql.Stmt, route *LiveRoute, pathJSON string) ( - inserted, newPrefix bool, err error) { - res, err := upd.Exec(route.MaskLength, pathJSON, route.NextHop, route.LastUpdated, - route.Prefix, route.OriginASN, route.PeerIP) +// A deletion that names an origin ASN removes at most one row. One without an +// origin ASN removes the prefix's routes from that peer whatever their origin, +// which can be several rows. +const ( + deleteLiveRouteV4SQL = `DELETE FROM live_routes_v4 + WHERE prefix = ? AND origin_asn = ? AND peer_ip = ?` + deleteLiveRouteAnyOriginV4SQL = `DELETE FROM live_routes_v4 + WHERE prefix = ? AND peer_ip = ?` + + deleteLiveRouteV6SQL = `DELETE FROM live_routes_v6 + WHERE prefix = ? AND origin_asn = ? AND peer_ip = ?` + deleteLiveRouteAnyOriginV6SQL = `DELETE FROM live_routes_v6 + WHERE prefix = ? AND peer_ip = ?` +) + +// upsertRouteRow updates a live route with the update statement of its IP +// version's table, inserting it when no row matched. It reports whether a new +// row was inserted and whether that row is the first live route for its prefix. +func upsertRouteRow( + ctx context.Context, upd, ins, has *sql.Stmt, route *LiveRoute, + pathJSON string, +) (bool, bool, error) { + res, err := upd.ExecContext(ctx, route.MaskLength, pathJSON, route.NextHop, + route.LastUpdated, route.Prefix, route.OriginASN, route.PeerIP) if err != nil { return false, false, err } + affected, err := res.RowsAffected() if err != nil { return false, false, err } + if affected > 0 { return false, false, nil } var prefixHadRoute bool - if err := has.QueryRow(route.Prefix).Scan(&prefixHadRoute); err != nil { + + err = has.QueryRowContext(ctx, route.Prefix).Scan(&prefixHadRoute) + if err != nil { return false, false, err } - _, err = ins.Exec(route.ID.String(), route.Prefix, route.MaskLength, route.OriginASN, - route.PeerIP, pathJSON, route.NextHop, route.LastUpdated) + _, err = ins.ExecContext(ctx, route.ID.String(), route.Prefix, + route.MaskLength, route.OriginASN, route.PeerIP, pathJSON, + route.NextHop, route.LastUpdated) if err != nil { return false, false, err } @@ -288,39 +300,75 @@ func upsertRouteRowV4(upd, ins, has *sql.Stmt, route *LiveRoute, pathJSON string return true, !prefixHadRoute, nil } -// upsertRouteRowV6 updates an IPv6 live route, inserting it when no row matched. -// It reports whether a new row was inserted and whether that row is the first -// live route for its prefix. -func upsertRouteRowV6(upd, ins, has *sql.Stmt, route *LiveRoute, pathJSON string) ( - inserted, newPrefix bool, err error) { - res, err := upd.Exec(route.MaskLength, pathJSON, route.NextHop, route.LastUpdated, - route.Prefix, route.OriginASN, route.PeerIP) - if err != nil { - return false, false, err - } - affected, err := res.RowsAffected() - if err != nil { - return false, false, err - } - if affected > 0 { - return false, false, nil +// upsertRoutesInTable upserts routes, all of one IP version, in the live route +// table of that version. It returns how many new rows it inserted and the mask +// lengths of the prefixes that got their first live route. +func upsertRoutesInTable( + ctx context.Context, tx *loggingTx, routes []*LiveRoute, ipVersion int, +) (int, []int, error) { + if len(routes) == 0 { + return 0, nil, nil } - var prefixHadRoute bool - if err := has.QueryRow(route.Prefix).Scan(&prefixHadRoute); err != nil { - return false, false, err + family, updateSQL, insertSQL, lookupSQL := ipv4Family, updateLiveRouteV4SQL, + insertLiveRouteV4SQL, prefixHasLiveRouteV4SQL + if ipVersion == ipVersionV6 { + family, updateSQL, insertSQL, lookupSQL = ipv6Family, + updateLiveRouteV6SQL, insertLiveRouteV6SQL, prefixHasLiveRouteV6SQL } - _, err = ins.Exec(route.ID.String(), route.Prefix, route.MaskLength, route.OriginASN, - route.PeerIP, pathJSON, route.NextHop, route.LastUpdated) + upd, err := tx.PrepareContext(ctx, updateSQL) if err != nil { - return false, false, err + return 0, nil, fmt.Errorf("failed to prepare %s update statement: %w", + family, err) + } + defer func() { _ = upd.Close() }() + + ins, err := tx.PrepareContext(ctx, insertSQL) + if err != nil { + return 0, nil, fmt.Errorf("failed to prepare %s insert statement: %w", + family, err) + } + defer func() { _ = ins.Close() }() + + has, err := tx.PrepareContext(ctx, lookupSQL) + if err != nil { + return 0, nil, fmt.Errorf( + "failed to prepare %s prefix lookup statement: %w", family, err) + } + defer func() { _ = has.Close() }() + + var newRows int + // Mask lengths of the prefixes that get their first live route. + var newPrefixMaskLengths []int + + for _, route := range routes { + pathJSON, err := json.Marshal(route.ASPath) + if err != nil { + return 0, nil, fmt.Errorf("failed to encode AS path: %w", err) + } + + inserted, newPrefix, err := upsertRouteRow(ctx, upd, ins, has, route, + string(pathJSON)) + if err != nil { + return 0, nil, fmt.Errorf("failed to upsert route %s: %w", + route.Prefix, err) + } + + if inserted { + newRows++ + } + + if newPrefix { + newPrefixMaskLengths = append(newPrefixMaskLengths, route.MaskLength) + } } - return true, !prefixHadRoute, nil + return newRows, newPrefixMaskLengths, nil } -// UpsertLiveRouteBatch inserts or updates multiple live routes in a single transaction +// UpsertLiveRouteBatch inserts or updates multiple live routes in a single +// transaction, the IPv4 routes first and then the IPv6 routes. func (d *Database) UpsertLiveRouteBatch(routes []*LiveRoute) error { if len(routes) == 0 { return nil @@ -333,86 +381,34 @@ func (d *Database) UpsertLiveRouteBatch(routes []*LiveRoute) error { if err != nil { return fmt.Errorf("failed to begin transaction: %w", err) } - defer func() { - if err := tx.Rollback(); err != nil && err != sql.ErrTxDone { - d.logger.Error("Failed to rollback transaction", "error", err) - } - }() + defer d.rollback(tx) - // Prepare the update and insert statements for both tables. - updV4, err := tx.Prepare(updateLiveRouteV4SQL) - if err != nil { - return fmt.Errorf("failed to prepare IPv4 update statement: %w", err) - } - defer func() { _ = updV4.Close() }() + var routesV4, routesV6 []*LiveRoute - insV4, err := tx.Prepare(insertLiveRouteV4SQL) - if err != nil { - return fmt.Errorf("failed to prepare IPv4 insert statement: %w", err) - } - defer func() { _ = insV4.Close() }() - - updV6, err := tx.Prepare(updateLiveRouteV6SQL) - if err != nil { - return fmt.Errorf("failed to prepare IPv6 update statement: %w", err) - } - defer func() { _ = updV6.Close() }() - - insV6, err := tx.Prepare(insertLiveRouteV6SQL) - if err != nil { - return fmt.Errorf("failed to prepare IPv6 insert statement: %w", err) - } - defer func() { _ = insV6.Close() }() - - hasV4, err := tx.Prepare(prefixHasLiveRouteV4SQL) - if err != nil { - return fmt.Errorf("failed to prepare IPv4 prefix lookup statement: %w", err) - } - defer func() { _ = hasV4.Close() }() - - hasV6, err := tx.Prepare(prefixHasLiveRouteV6SQL) - if err != nil { - return fmt.Errorf("failed to prepare IPv6 prefix lookup statement: %w", err) - } - defer func() { _ = hasV6.Close() }() - - var newV4, newV6 int - // Mask lengths of the prefixes that get their first live route in this batch. - var newPrefixMaskLengthsV4, newPrefixMaskLengthsV6 []int for _, route := range routes { - pathJSON, err := json.Marshal(route.ASPath) - if err != nil { - return fmt.Errorf("failed to encode AS path: %w", err) - } - if route.IPVersion == ipVersionV4 { - inserted, newPrefix, err := upsertRouteRowV4(updV4, insV4, hasV4, route, string(pathJSON)) - if err != nil { - return fmt.Errorf("failed to upsert route %s: %w", route.Prefix, err) - } - if inserted { - newV4++ - } - if newPrefix { - newPrefixMaskLengthsV4 = append(newPrefixMaskLengthsV4, route.MaskLength) - } - - continue - } - - inserted, newPrefix, err := upsertRouteRowV6(updV6, insV6, hasV6, route, string(pathJSON)) - if err != nil { - return fmt.Errorf("failed to upsert route %s: %w", route.Prefix, err) - } - if inserted { - newV6++ - } - if newPrefix { - newPrefixMaskLengthsV6 = append(newPrefixMaskLengthsV6, route.MaskLength) + routesV4 = append(routesV4, route) + } else { + routesV6 = append(routesV6, route) } } - if err = tx.Commit(); err != nil { + ctx := context.Background() + + newV4, newPrefixMaskLengthsV4, err := upsertRoutesInTable(ctx, tx, + routesV4, ipVersionV4) + if err != nil { + return err + } + + newV6, newPrefixMaskLengthsV6, err := upsertRoutesInTable(ctx, tx, + routesV6, ipVersionV6) + if err != nil { + return err + } + + err = tx.Commit() + if err != nil { return fmt.Errorf("failed to commit transaction: %w", err) } @@ -422,7 +418,127 @@ func (d *Database) UpsertLiveRouteBatch(routes []*LiveRoute) error { return nil } -// DeleteLiveRouteBatch deletes multiple live routes in a single transaction +// deleteRouteRows runs one deletion with the delete statements of its IP +// version's table: withOrigin when it names an origin ASN, otherwise +// withoutOrigin. It returns how many rows it removed and whether that left the +// prefix with no live route. +func deleteRouteRows( + ctx context.Context, withOrigin, withoutOrigin, has *sql.Stmt, + del LiveRouteDeletion, +) (int64, bool, error) { + var ( + res sql.Result + err error + ) + + if del.OriginASN == 0 { + res, err = withoutOrigin.ExecContext(ctx, del.Prefix, del.PeerIP) + } else { + res, err = withOrigin.ExecContext(ctx, del.Prefix, del.OriginASN, + del.PeerIP) + } + + if err != nil { + return 0, false, fmt.Errorf("failed to delete route %s: %w", + del.Prefix, err) + } + + // A deletion with no origin ASN can remove several rows, so use the + // exact rows-affected count to keep the in-memory route counters right. + affected, err := res.RowsAffected() + if err != nil { + return 0, false, fmt.Errorf("failed to count deleted route %s: %w", + del.Prefix, err) + } + + if affected == 0 { + return 0, false, nil + } + + // The prefix leaves the distribution when no live route has it any more. + var prefixHasRoute bool + + err = has.QueryRowContext(ctx, del.Prefix).Scan(&prefixHasRoute) + if err != nil { + return 0, false, fmt.Errorf("failed to look up prefix %s: %w", + del.Prefix, err) + } + + return affected, !prefixHasRoute, nil +} + +// deleteRoutesInTable runs deletions, all of one IP version, on the live route +// table of that version. It returns how many rows it removed and the mask +// lengths of the prefixes it left with no live route. +func deleteRoutesInTable( + ctx context.Context, tx *loggingTx, deletions []LiveRouteDeletion, + ipVersion int, +) (int64, []int, error) { + if len(deletions) == 0 { + return 0, nil, nil + } + + family, withOriginSQL, withoutOriginSQL, lookupSQL := ipv4Family, + deleteLiveRouteV4SQL, deleteLiveRouteAnyOriginV4SQL, + prefixHasLiveRouteV4SQL + if ipVersion == ipVersionV6 { + family, withOriginSQL, withoutOriginSQL, lookupSQL = ipv6Family, + deleteLiveRouteV6SQL, deleteLiveRouteAnyOriginV6SQL, + prefixHasLiveRouteV6SQL + } + + withOrigin, err := tx.PrepareContext(ctx, withOriginSQL) + if err != nil { + return 0, nil, fmt.Errorf( + "failed to prepare %s delete with origin statement: %w", family, err) + } + defer func() { _ = withOrigin.Close() }() + + withoutOrigin, err := tx.PrepareContext(ctx, withoutOriginSQL) + if err != nil { + return 0, nil, fmt.Errorf( + "failed to prepare %s delete without origin statement: %w", + family, err) + } + defer func() { _ = withoutOrigin.Close() }() + + has, err := tx.PrepareContext(ctx, lookupSQL) + if err != nil { + return 0, nil, fmt.Errorf( + "failed to prepare %s prefix lookup statement: %w", family, err) + } + defer func() { _ = has.Close() }() + + var deleted int64 + // Mask lengths of the prefixes left with no live route. + var gonePrefixMaskLengths []int + + for _, del := range deletions { + affected, prefixGone, err := deleteRouteRows(ctx, + withOrigin, withoutOrigin, has, del) + if err != nil { + return 0, nil, err + } + + deleted += affected + + if !prefixGone { + continue + } + + maskLength, err := prefixMaskLength(del.Prefix) + if err != nil { + return 0, nil, err + } + + gonePrefixMaskLengths = append(gonePrefixMaskLengths, maskLength) + } + + return deleted, gonePrefixMaskLengths, nil +} + +// DeleteLiveRouteBatch deletes multiple live routes in a single transaction, +// the IPv4 routes first and then the IPv6 routes. func (d *Database) DeleteLiveRouteBatch(deletions []LiveRouteDeletion) error { if len(deletions) == 0 { return nil @@ -435,125 +551,34 @@ func (d *Database) DeleteLiveRouteBatch(deletions []LiveRouteDeletion) error { if err != nil { return fmt.Errorf("failed to begin transaction: %w", err) } - defer func() { - if err := tx.Rollback(); err != nil && err != sql.ErrTxDone { - d.logger.Error("Failed to rollback transaction", "error", err) - } - }() + defer d.rollback(tx) - // No longer need to separate deletions since we handle them in the loop below + var deletionsV4, deletionsV6 []LiveRouteDeletion - // Prepare statements for both IPv4 and IPv6 tables - stmtV4WithOrigin, err := tx.Prepare(`DELETE FROM live_routes_v4 WHERE prefix = ? AND origin_asn = ? AND peer_ip = ?`) - if err != nil { - return fmt.Errorf("failed to prepare IPv4 delete with origin statement: %w", err) - } - defer func() { _ = stmtV4WithOrigin.Close() }() - - stmtV4WithoutOrigin, err := tx.Prepare(`DELETE FROM live_routes_v4 WHERE prefix = ? AND peer_ip = ?`) - if err != nil { - return fmt.Errorf("failed to prepare IPv4 delete without origin statement: %w", err) - } - defer func() { _ = stmtV4WithoutOrigin.Close() }() - - stmtV6WithOrigin, err := tx.Prepare(`DELETE FROM live_routes_v6 WHERE prefix = ? AND origin_asn = ? AND peer_ip = ?`) - if err != nil { - return fmt.Errorf("failed to prepare IPv6 delete with origin statement: %w", err) - } - defer func() { _ = stmtV6WithOrigin.Close() }() - - stmtV6WithoutOrigin, err := tx.Prepare(`DELETE FROM live_routes_v6 WHERE prefix = ? AND peer_ip = ?`) - if err != nil { - return fmt.Errorf("failed to prepare IPv6 delete without origin statement: %w", err) - } - defer func() { _ = stmtV6WithoutOrigin.Close() }() - - hasV4, err := tx.Prepare(prefixHasLiveRouteV4SQL) - if err != nil { - return fmt.Errorf("failed to prepare IPv4 prefix lookup statement: %w", err) - } - defer func() { _ = hasV4.Close() }() - - hasV6, err := tx.Prepare(prefixHasLiveRouteV6SQL) - if err != nil { - return fmt.Errorf("failed to prepare IPv6 prefix lookup statement: %w", err) - } - defer func() { _ = hasV6.Close() }() - - // Process deletions - var deletedV4, deletedV6 int64 - // Mask lengths of the prefixes this batch leaves with no live route. - var gonePrefixMaskLengthsV4, gonePrefixMaskLengthsV6 []int for _, del := range deletions { - var stmt *sql.Stmt - - // Select appropriate statement based on IP version and whether we have origin ASN - //nolint:nestif // Clear logic for selecting the right statement if del.IPVersion == ipVersionV4 { - if del.OriginASN == 0 { - stmt = stmtV4WithoutOrigin - } else { - stmt = stmtV4WithOrigin - } + deletionsV4 = append(deletionsV4, del) } else { - if del.OriginASN == 0 { - stmt = stmtV6WithoutOrigin - } else { - stmt = stmtV6WithOrigin - } - } - - // Execute deletion - var res sql.Result - if del.OriginASN == 0 { - res, err = stmt.Exec(del.Prefix, del.PeerIP) - } else { - res, err = stmt.Exec(del.Prefix, del.OriginASN, del.PeerIP) - } - - if err != nil { - return fmt.Errorf("failed to delete route %s: %w", del.Prefix, err) - } - - // A deletion with no origin ASN can remove several rows, so use the - // exact rows-affected count to keep the in-memory route counters right. - affected, err := res.RowsAffected() - if err != nil { - return fmt.Errorf("failed to count deleted route %s: %w", del.Prefix, err) - } - if del.IPVersion == ipVersionV4 { - deletedV4 += affected - } else { - deletedV6 += affected - } - if affected == 0 { - continue - } - - // The prefix leaves the distribution when no live route has it any more. - has := hasV4 - if del.IPVersion != ipVersionV4 { - has = hasV6 - } - var prefixHasRoute bool - if err := has.QueryRow(del.Prefix).Scan(&prefixHasRoute); err != nil { - return fmt.Errorf("failed to look up prefix %s: %w", del.Prefix, err) - } - if prefixHasRoute { - continue - } - maskLength, err := prefixMaskLength(del.Prefix) - if err != nil { - return err - } - if del.IPVersion == ipVersionV4 { - gonePrefixMaskLengthsV4 = append(gonePrefixMaskLengthsV4, maskLength) - } else { - gonePrefixMaskLengthsV6 = append(gonePrefixMaskLengthsV6, maskLength) + deletionsV6 = append(deletionsV6, del) } } - if err = tx.Commit(); err != nil { + ctx := context.Background() + + deletedV4, gonePrefixMaskLengthsV4, err := deleteRoutesInTable(ctx, tx, + deletionsV4, ipVersionV4) + if err != nil { + return err + } + + deletedV6, gonePrefixMaskLengthsV6, err := deleteRoutesInTable(ctx, tx, + deletionsV6, ipVersionV6) + if err != nil { + return err + } + + err = tx.Commit() + if err != nil { return fmt.Errorf("failed to commit transaction: %w", err) } @@ -563,7 +588,107 @@ func (d *Database) DeleteLiveRouteBatch(deletions []LiveRouteDeletion) error { return nil } -// UpdatePrefixesBatch updates the last_seen time for multiple prefixes in a single transaction +// upsertPrefix updates the last_seen time of prefix with the statements of its +// IP version's table, inserting the prefix when it is new. It reports whether +// it inserted it. +func upsertPrefix( + ctx context.Context, selectStmt, updateStmt, insertStmt *sql.Stmt, + prefix string, timestamp time.Time, +) (bool, error) { + var id string + + err := selectStmt.QueryRowContext(ctx, prefix).Scan(&id) + + switch { + case err == nil: + // Prefix exists, update last_seen + _, err = updateStmt.ExecContext(ctx, timestamp, prefix) + if err != nil { + return false, fmt.Errorf("failed to update prefix %s: %w", + prefix, err) + } + + return false, nil + case errors.Is(err, sql.ErrNoRows): + // Prefix doesn't exist, create it + _, err = insertStmt.ExecContext(ctx, generateUUID().String(), prefix, + timestamp, timestamp) + if err != nil { + return false, fmt.Errorf("failed to insert prefix %s: %w", + prefix, err) + } + + return true, nil + default: + return false, fmt.Errorf("failed to query prefix %s: %w", prefix, err) + } +} + +// upsertPrefixesInTable updates the last_seen time of prefixes, all of one IP +// version, in the prefix table of that version, inserting the ones that are +// new. It returns how many it inserted. +func upsertPrefixesInTable( + ctx context.Context, tx *loggingTx, prefixes map[string]time.Time, + ipVersion int, +) (int, error) { + if len(prefixes) == 0 { + return 0, nil + } + + family := ipv4Family + selectSQL := "SELECT id FROM prefixes_v4 WHERE prefix = ?" + updateSQL := "UPDATE prefixes_v4 SET last_seen = ? WHERE prefix = ?" + insertSQL := "INSERT INTO prefixes_v4 (id, prefix, first_seen, last_seen) " + + "VALUES (?, ?, ?, ?)" + + if ipVersion == ipVersionV6 { + family = ipv6Family + selectSQL = "SELECT id FROM prefixes_v6 WHERE prefix = ?" + updateSQL = "UPDATE prefixes_v6 SET last_seen = ? WHERE prefix = ?" + insertSQL = "INSERT INTO prefixes_v6 (id, prefix, first_seen, last_seen) " + + "VALUES (?, ?, ?, ?)" + } + + selectStmt, err := tx.PrepareContext(ctx, selectSQL) + if err != nil { + return 0, fmt.Errorf("failed to prepare %s select statement: %w", + family, err) + } + defer func() { _ = selectStmt.Close() }() + + updateStmt, err := tx.PrepareContext(ctx, updateSQL) + if err != nil { + return 0, fmt.Errorf("failed to prepare %s update statement: %w", + family, err) + } + defer func() { _ = updateStmt.Close() }() + + insertStmt, err := tx.PrepareContext(ctx, insertSQL) + if err != nil { + return 0, fmt.Errorf("failed to prepare %s insert statement: %w", + family, err) + } + defer func() { _ = insertStmt.Close() }() + + var newPrefixes int + + for prefix, timestamp := range prefixes { + inserted, err := upsertPrefix(ctx, selectStmt, updateStmt, insertStmt, + prefix, timestamp) + if err != nil { + return 0, err + } + + if inserted { + newPrefixes++ + } + } + + return newPrefixes, nil +} + +// UpdatePrefixesBatch updates the last_seen time for multiple prefixes in a +// single transaction, the IPv4 prefixes first and then the IPv6 prefixes. func (d *Database) UpdatePrefixesBatch(prefixes map[string]time.Time) error { if len(prefixes) == 0 { return nil @@ -576,87 +701,33 @@ func (d *Database) UpdatePrefixesBatch(prefixes map[string]time.Time) error { if err != nil { return fmt.Errorf("failed to begin transaction: %w", err) } - defer func() { - if err := tx.Rollback(); err != nil && err != sql.ErrTxDone { - d.logger.Error("Failed to rollback transaction", "error", err) - } - }() + defer d.rollback(tx) - // Prepare statements for both IPv4 and IPv6 tables - selectV4Stmt, err := tx.Prepare("SELECT id FROM prefixes_v4 WHERE prefix = ?") - if err != nil { - return fmt.Errorf("failed to prepare IPv4 select statement: %w", err) - } - defer func() { _ = selectV4Stmt.Close() }() + prefixesV4 := make(map[string]time.Time) + prefixesV6 := make(map[string]time.Time) - updateV4Stmt, err := tx.Prepare("UPDATE prefixes_v4 SET last_seen = ? WHERE prefix = ?") - if err != nil { - return fmt.Errorf("failed to prepare IPv4 update statement: %w", err) - } - defer func() { _ = updateV4Stmt.Close() }() - - insertV4Stmt, err := tx.Prepare("INSERT INTO prefixes_v4 (id, prefix, first_seen, last_seen) VALUES (?, ?, ?, ?)") - if err != nil { - return fmt.Errorf("failed to prepare IPv4 insert statement: %w", err) - } - defer func() { _ = insertV4Stmt.Close() }() - - selectV6Stmt, err := tx.Prepare("SELECT id FROM prefixes_v6 WHERE prefix = ?") - if err != nil { - return fmt.Errorf("failed to prepare IPv6 select statement: %w", err) - } - defer func() { _ = selectV6Stmt.Close() }() - - updateV6Stmt, err := tx.Prepare("UPDATE prefixes_v6 SET last_seen = ? WHERE prefix = ?") - if err != nil { - return fmt.Errorf("failed to prepare IPv6 update statement: %w", err) - } - defer func() { _ = updateV6Stmt.Close() }() - - insertV6Stmt, err := tx.Prepare("INSERT INTO prefixes_v6 (id, prefix, first_seen, last_seen) VALUES (?, ?, ?, ?)") - if err != nil { - return fmt.Errorf("failed to prepare IPv6 insert statement: %w", err) - } - defer func() { _ = insertV6Stmt.Close() }() - - var newV4, newV6 int for prefix, timestamp := range prefixes { - ipVersion := detectIPVersion(prefix) - - var selectStmt, updateStmt, insertStmt *sql.Stmt - if ipVersion == ipVersionV4 { - selectStmt, updateStmt, insertStmt = selectV4Stmt, updateV4Stmt, insertV4Stmt + if detectIPVersion(prefix) == ipVersionV4 { + prefixesV4[prefix] = timestamp } else { - selectStmt, updateStmt, insertStmt = selectV6Stmt, updateV6Stmt, insertV6Stmt - } - - var id string - err = selectStmt.QueryRow(prefix).Scan(&id) - - switch err { - case nil: - // Prefix exists, update last_seen - _, err = updateStmt.Exec(timestamp, prefix) - if err != nil { - return fmt.Errorf("failed to update prefix %s: %w", prefix, err) - } - case sql.ErrNoRows: - // Prefix doesn't exist, create it - _, err = insertStmt.Exec(generateUUID().String(), prefix, timestamp, timestamp) - if err != nil { - return fmt.Errorf("failed to insert prefix %s: %w", prefix, err) - } - if ipVersion == ipVersionV4 { - newV4++ - } else { - newV6++ - } - default: - return fmt.Errorf("failed to query prefix %s: %w", prefix, err) + prefixesV6[prefix] = timestamp } } - if err = tx.Commit(); err != nil { + ctx := context.Background() + + newV4, err := upsertPrefixesInTable(ctx, tx, prefixesV4, ipVersionV4) + if err != nil { + return err + } + + newV6, err := upsertPrefixesInTable(ctx, tx, prefixesV6, ipVersionV6) + if err != nil { + return err + } + + err = tx.Commit() + if err != nil { return fmt.Errorf("failed to commit transaction: %w", err) } @@ -665,145 +736,10 @@ func (d *Database) UpdatePrefixesBatch(prefixes map[string]time.Time) error { return nil } -// GetOrCreateASNBatch creates or updates multiple ASNs in a single transaction -func (d *Database) GetOrCreateASNBatch(asns map[int]time.Time) error { - if len(asns) == 0 { - return nil - } - - d.lock("GetOrCreateASNBatch") - defer d.unlock() - - tx, err := d.beginTx() - if err != nil { - return fmt.Errorf("failed to begin transaction: %w", err) - } - defer func() { - if err := tx.Rollback(); err != nil && err != sql.ErrTxDone { - d.logger.Error("Failed to rollback transaction", "error", err) - } - }() - - // Prepare statements - selectStmt, err := tx.Prepare( - "SELECT asn, handle, description, first_seen, last_seen FROM asns WHERE asn = ?") - if err != nil { - return fmt.Errorf("failed to prepare select statement: %w", err) - } - defer func() { _ = selectStmt.Close() }() - - updateStmt, err := tx.Prepare("UPDATE asns SET last_seen = ? WHERE asn = ?") - if err != nil { - return fmt.Errorf("failed to prepare update statement: %w", err) - } - defer func() { _ = updateStmt.Close() }() - - insertStmt, err := tx.Prepare( - "INSERT INTO asns (asn, handle, description, first_seen, last_seen) VALUES (?, ?, ?, ?, ?)") - if err != nil { - return fmt.Errorf("failed to prepare insert statement: %w", err) - } - defer func() { _ = insertStmt.Close() }() - - var newASNs int - for number, timestamp := range asns { - var asn ASN - var handle, description sql.NullString - - err = selectStmt.QueryRow(number).Scan(&asn.ASN, &handle, &description, &asn.FirstSeen, &asn.LastSeen) - - if err == nil { - // ASN exists, update last_seen - _, err = updateStmt.Exec(timestamp, number) - if err != nil { - return fmt.Errorf("failed to update ASN %d: %w", number, err) - } - - continue - } - - if err == sql.ErrNoRows { - // ASN doesn't exist, create it - asn = ASN{ - ASN: number, - FirstSeen: timestamp, - LastSeen: timestamp, - } - - // Look up ASN info - if info, ok := asinfo.Get(number); ok { - asn.Handle = info.Handle - asn.Description = info.Description - } - - _, err = insertStmt.Exec(asn.ASN, asn.Handle, asn.Description, asn.FirstSeen, asn.LastSeen) - if err != nil { - return fmt.Errorf("failed to insert ASN %d: %w", number, err) - } - newASNs++ - - continue - } - - if err != nil { - return fmt.Errorf("failed to query ASN %d: %w", number, err) - } - } - - if err = tx.Commit(); err != nil { - return fmt.Errorf("failed to commit transaction: %w", err) - } - - d.counts.addASNs(newASNs) - - return nil -} - -// GetOrCreateASN retrieves an existing ASN or creates a new one if it doesn't exist. -func (d *Database) GetOrCreateASN(number int, timestamp time.Time) (*ASN, error) { - d.lock("GetOrCreateASN") - defer d.unlock() - - tx, err := d.beginTx() - if err != nil { - return nil, err - } - defer func() { - if err := tx.Rollback(); err != nil && err != sql.ErrTxDone { - d.logger.Error("Failed to rollback transaction", "error", err) - } - }() - - var asn ASN - var handle, description sql.NullString - err = tx.QueryRow("SELECT asn, handle, description, first_seen, last_seen FROM asns WHERE asn = ?", number). - Scan(&asn.ASN, &handle, &description, &asn.FirstSeen, &asn.LastSeen) - - if err == nil { - // ASN exists, update last_seen - asn.Handle = handle.String - asn.Description = description.String - _, err = tx.Exec("UPDATE asns SET last_seen = ? WHERE asn = ?", timestamp, number) - if err != nil { - return nil, err - } - asn.LastSeen = timestamp - - if err = tx.Commit(); err != nil { - d.logger.Error("Failed to commit transaction for ASN update", "asn", number, "error", err) - - return nil, err - } - - return &asn, nil - } - - if err != sql.ErrNoRows { - return nil, err - } - - // ASN doesn't exist, create it with ASN info lookup - asn = ASN{ +// newASN returns a new ASN record first and last seen at timestamp, with the +// handle and description from the AS info lookup when it knows the number. +func newASN(number int, timestamp time.Time) ASN { + asn := ASN{ ASN: number, FirstSeen: timestamp, LastSeen: timestamp, @@ -815,14 +751,177 @@ func (d *Database) GetOrCreateASN(number int, timestamp time.Time) (*ASN, error) asn.Description = info.Description } - _, err = tx.Exec("INSERT INTO asns (asn, handle, description, first_seen, last_seen) VALUES (?, ?, ?, ?, ?)", + return asn +} + +// upsertASN updates the last_seen time of ASN number with the given +// statements, inserting the ASN when it is new. It reports whether it inserted +// it. +func upsertASN( + ctx context.Context, selectStmt, updateStmt, insertStmt *sql.Stmt, + number int, timestamp time.Time, +) (bool, error) { + var ( + asn ASN + handle, description sql.NullString + ) + + err := selectStmt.QueryRowContext(ctx, number).Scan(&asn.ASN, &handle, + &description, &asn.FirstSeen, &asn.LastSeen) + if err == nil { + // ASN exists, update last_seen + _, err = updateStmt.ExecContext(ctx, timestamp, number) + if err != nil { + return false, fmt.Errorf("failed to update ASN %d: %w", number, err) + } + + return false, nil + } + + if !errors.Is(err, sql.ErrNoRows) { + return false, fmt.Errorf("failed to query ASN %d: %w", number, err) + } + + // ASN doesn't exist, create it + asn = newASN(number, timestamp) + + _, err = insertStmt.ExecContext(ctx, asn.ASN, asn.Handle, + asn.Description, asn.FirstSeen, asn.LastSeen) + if err != nil { + return false, fmt.Errorf("failed to insert ASN %d: %w", number, err) + } + + return true, nil +} + +// GetOrCreateASNBatch creates or updates multiple ASNs in a single transaction +func (d *Database) GetOrCreateASNBatch(asns map[int]time.Time) error { + if len(asns) == 0 { + return nil + } + + d.lock("GetOrCreateASNBatch") + defer d.unlock() + + ctx := context.Background() + + tx, err := d.beginTx() + if err != nil { + return fmt.Errorf("failed to begin transaction: %w", err) + } + defer d.rollback(tx) + + // Prepare statements + selectStmt, err := tx.PrepareContext(ctx, + "SELECT asn, handle, description, first_seen, last_seen "+ + "FROM asns WHERE asn = ?") + if err != nil { + return fmt.Errorf("failed to prepare select statement: %w", err) + } + defer func() { _ = selectStmt.Close() }() + + updateStmt, err := tx.PrepareContext(ctx, + "UPDATE asns SET last_seen = ? WHERE asn = ?") + if err != nil { + return fmt.Errorf("failed to prepare update statement: %w", err) + } + defer func() { _ = updateStmt.Close() }() + + insertStmt, err := tx.PrepareContext(ctx, + "INSERT INTO asns (asn, handle, description, first_seen, last_seen) "+ + "VALUES (?, ?, ?, ?, ?)") + if err != nil { + return fmt.Errorf("failed to prepare insert statement: %w", err) + } + defer func() { _ = insertStmt.Close() }() + + var newASNs int + + for number, timestamp := range asns { + inserted, err := upsertASN(ctx, selectStmt, updateStmt, insertStmt, + number, timestamp) + if err != nil { + return err + } + + if inserted { + newASNs++ + } + } + + err = tx.Commit() + if err != nil { + return fmt.Errorf("failed to commit transaction: %w", err) + } + + d.counts.addASNs(newASNs) + + return nil +} + +// GetOrCreateASN retrieves an existing ASN or creates a new one if it doesn't +// exist. +func (d *Database) GetOrCreateASN(number int, timestamp time.Time) (*ASN, error) { + d.lock("GetOrCreateASN") + defer d.unlock() + + tx, err := d.beginTx() + if err != nil { + return nil, err + } + defer d.rollback(tx) + + var ( + asn ASN + handle, description sql.NullString + ) + + err = tx.QueryRow("SELECT asn, handle, description, first_seen, last_seen "+ + "FROM asns WHERE asn = ?", number). + Scan(&asn.ASN, &handle, &description, &asn.FirstSeen, &asn.LastSeen) + if err == nil { + // ASN exists, update last_seen + asn.Handle = handle.String + asn.Description = description.String + + _, err = tx.Exec("UPDATE asns SET last_seen = ? WHERE asn = ?", + timestamp, number) + if err != nil { + return nil, err + } + + asn.LastSeen = timestamp + + err = tx.Commit() + if err != nil { + d.logger.Error("Failed to commit transaction for ASN update", + "asn", number, "error", err) + + return nil, err + } + + return &asn, nil + } + + if !errors.Is(err, sql.ErrNoRows) { + return nil, err + } + + // ASN doesn't exist, create it with ASN info lookup + asn = newASN(number, timestamp) + + _, err = tx.Exec("INSERT INTO asns "+ + "(asn, handle, description, first_seen, last_seen) "+ + "VALUES (?, ?, ?, ?, ?)", asn.ASN, asn.Handle, asn.Description, asn.FirstSeen, asn.LastSeen) if err != nil { return nil, err } - if err = tx.Commit(); err != nil { - d.logger.Error("Failed to commit transaction for ASN creation", "asn", number, "error", err) + err = tx.Commit() + if err != nil { + d.logger.Error("Failed to commit transaction for ASN creation", + "asn", number, "error", err) return nil, err } @@ -832,8 +931,10 @@ func (d *Database) GetOrCreateASN(number int, timestamp time.Time) (*ASN, error) return &asn, nil } -// GetOrCreatePrefix retrieves an existing prefix or creates a new one if it doesn't exist. -func (d *Database) GetOrCreatePrefix(prefix string, timestamp time.Time) (*Prefix, error) { +// GetOrCreatePrefix retrieves an existing prefix or creates a new one if it +// doesn't exist. +func (d *Database) GetOrCreatePrefix( + prefix string, timestamp time.Time) (*Prefix, error) { d.lock("GetOrCreatePrefix") defer d.unlock() @@ -841,38 +942,42 @@ func (d *Database) GetOrCreatePrefix(prefix string, timestamp time.Time) (*Prefi if err != nil { return nil, err } - defer func() { - if err := tx.Rollback(); err != nil && err != sql.ErrTxDone { - d.logger.Error("Failed to rollback transaction", "error", err) - } - }() + defer d.rollback(tx) // Determine table based on IP version ipVersion := detectIPVersion(prefix) + tableName := "prefixes_v4" if ipVersion == ipVersionV6 { tableName = "prefixes_v6" } - var p Prefix - var idStr string - query := fmt.Sprintf("SELECT id, prefix, first_seen, last_seen FROM %s WHERE prefix = ?", tableName) - err = tx.QueryRow(query, prefix). - Scan(&idStr, &p.Prefix, &p.FirstSeen, &p.LastSeen) + var ( + p Prefix + idStr string + ) + query := fmt.Sprintf("SELECT id, prefix, first_seen, last_seen "+ + "FROM %s WHERE prefix = ?", tableName) + + err = tx.QueryRow(query, prefix).Scan(&idStr, &p.Prefix, &p.FirstSeen, &p.LastSeen) if err == nil { // Prefix exists, update last_seen p.ID, _ = uuid.Parse(idStr) p.IPVersion = ipVersion updateQuery := fmt.Sprintf("UPDATE %s SET last_seen = ? WHERE id = ?", tableName) + _, err = tx.Exec(updateQuery, timestamp, p.ID.String()) if err != nil { return nil, err } + p.LastSeen = timestamp - if err = tx.Commit(); err != nil { - d.logger.Error("Failed to commit transaction for prefix update", "prefix", prefix, "error", err) + err = tx.Commit() + if err != nil { + d.logger.Error("Failed to commit transaction for prefix update", + "prefix", prefix, "error", err) return nil, err } @@ -880,26 +985,25 @@ func (d *Database) GetOrCreatePrefix(prefix string, timestamp time.Time) (*Prefi return &p, nil } - if err != sql.ErrNoRows { + if !errors.Is(err, sql.ErrNoRows) { return nil, err } // Prefix doesn't exist, create it - p = Prefix{ - ID: generateUUID(), - Prefix: prefix, - IPVersion: ipVersion, - FirstSeen: timestamp, - LastSeen: timestamp, - } - insertQuery := fmt.Sprintf("INSERT INTO %s (id, prefix, first_seen, last_seen) VALUES (?, ?, ?, ?)", tableName) + p = Prefix{ID: generateUUID(), Prefix: prefix, IPVersion: ipVersion, + FirstSeen: timestamp, LastSeen: timestamp} + insertQuery := fmt.Sprintf("INSERT INTO %s "+ + "(id, prefix, first_seen, last_seen) VALUES (?, ?, ?, ?)", tableName) + _, err = tx.Exec(insertQuery, p.ID.String(), p.Prefix, p.FirstSeen, p.LastSeen) if err != nil { return nil, err } - if err = tx.Commit(); err != nil { - d.logger.Error("Failed to commit transaction for prefix creation", "prefix", prefix, "error", err) + err = tx.Commit() + if err != nil { + d.logger.Error("Failed to commit transaction for prefix creation", + "prefix", prefix, "error", err) return nil, err } @@ -913,17 +1017,20 @@ func (d *Database) GetOrCreatePrefix(prefix string, timestamp time.Time) (*Prefi return &p, nil } -// RecordAnnouncement inserts a new BGP announcement or withdrawal into the database. +// RecordAnnouncement inserts a new BGP announcement or withdrawal into the +// database. func (d *Database) RecordAnnouncement(announcement *Announcement) error { d.lock("RecordAnnouncement") defer d.unlock() err := d.exec(` - INSERT INTO announcements (id, prefix_id, peer_asn, origin_asn, path, next_hop, timestamp, is_withdrawal) + INSERT INTO announcements (id, prefix_id, peer_asn, origin_asn, path, + next_hop, timestamp, is_withdrawal) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, announcement.ID.String(), announcement.PrefixID.String(), announcement.PeerASN, announcement.OriginASN, - announcement.Path, announcement.NextHop, announcement.Timestamp, announcement.IsWithdrawal) + announcement.Path, announcement.NextHop, announcement.Timestamp, + announcement.IsWithdrawal) return err } @@ -932,10 +1039,11 @@ func (d *Database) RecordAnnouncement(announcement *Announcement) error { func (d *Database) RecordPeering(asA, asB int, timestamp time.Time) error { // Validate ASNs if asA <= 0 || asB <= 0 { - return fmt.Errorf("invalid ASN: asA=%d, asB=%d", asA, asB) + return fmt.Errorf("%w: asA=%d, asB=%d", errInvalidASN, asA, asB) } + if asA == asB { - return fmt.Errorf("cannot create peering with same ASN: %d", asA) + return fmt.Errorf("%w: %d", errSameASNInPeering, asA) } // Normalize: ensure asA < asB @@ -950,21 +1058,20 @@ func (d *Database) RecordPeering(asA, asB int, timestamp time.Time) error { if err != nil { return err } - defer func() { - if err := tx.Rollback(); err != nil && err != sql.ErrTxDone { - d.logger.Error("Failed to rollback transaction", "error", err) - } - }() + defer d.rollback(tx) var exists bool - err = tx.QueryRow("SELECT EXISTS(SELECT 1 FROM peerings WHERE as_a = ? AND as_b = ?)", + + err = tx.QueryRow( + "SELECT EXISTS(SELECT 1 FROM peerings WHERE as_a = ? AND as_b = ?)", asA, asB).Scan(&exists) if err != nil { return err } if exists { - _, err = tx.Exec("UPDATE peerings SET last_seen = ? WHERE as_a = ? AND as_b = ?", + _, err = tx.Exec( + "UPDATE peerings SET last_seen = ? WHERE as_a = ? AND as_b = ?", timestamp, asA, asB) } else { _, err = tx.Exec(` @@ -972,11 +1079,13 @@ func (d *Database) RecordPeering(asA, asB int, timestamp time.Time) error { VALUES (?, ?, ?, ?, ?)`, generateUUID().String(), asA, asB, timestamp, timestamp) } + if err != nil { return err } - if err = tx.Commit(); err != nil { + err = tx.Commit() + if err != nil { d.logger.Error("Failed to commit transaction for peering", "as_a", asA, "as_b", asB, @@ -1002,61 +1111,68 @@ func (d *Database) UpdatePeerBatch(peers map[string]PeerUpdate) error { d.lock("UpdatePeerBatch") defer d.unlock() + ctx := context.Background() + tx, err := d.beginTx() if err != nil { return fmt.Errorf("failed to begin transaction: %w", err) } - defer func() { - if err := tx.Rollback(); err != nil && err != sql.ErrTxDone { - d.logger.Error("Failed to rollback transaction", "error", err) - } - }() + defer d.rollback(tx) // Prepare statements - checkStmt, err := tx.Prepare("SELECT EXISTS(SELECT 1 FROM bgp_peers WHERE peer_ip = ?)") + checkStmt, err := tx.PrepareContext(ctx, + "SELECT EXISTS(SELECT 1 FROM bgp_peers WHERE peer_ip = ?)") if err != nil { return fmt.Errorf("failed to prepare check statement: %w", err) } defer func() { _ = checkStmt.Close() }() - updateStmt, err := tx.Prepare( - "UPDATE bgp_peers SET peer_asn = ?, last_seen = ?, last_message_type = ? WHERE peer_ip = ?") + updateStmt, err := tx.PrepareContext(ctx, + "UPDATE bgp_peers SET peer_asn = ?, last_seen = ?, "+ + "last_message_type = ? WHERE peer_ip = ?") if err != nil { return fmt.Errorf("failed to prepare update statement: %w", err) } defer func() { _ = updateStmt.Close() }() - insertStmt, err := tx.Prepare( - "INSERT INTO bgp_peers (id, peer_ip, peer_asn, first_seen, last_seen, last_message_type) VALUES (?, ?, ?, ?, ?, ?)") + insertStmt, err := tx.PrepareContext(ctx, + "INSERT INTO bgp_peers (id, peer_ip, peer_asn, first_seen, "+ + "last_seen, last_message_type) VALUES (?, ?, ?, ?, ?, ?)") if err != nil { return fmt.Errorf("failed to prepare insert statement: %w", err) } defer func() { _ = insertStmt.Close() }() var newPeers int + for _, update := range peers { var exists bool - err = checkStmt.QueryRow(update.PeerIP).Scan(&exists) + + err = checkStmt.QueryRowContext(ctx, update.PeerIP).Scan(&exists) if err != nil { return fmt.Errorf("failed to check peer %s: %w", update.PeerIP, err) } if exists { - _, err = updateStmt.Exec(update.PeerASN, update.Timestamp, update.MessageType, update.PeerIP) + _, err = updateStmt.ExecContext(ctx, update.PeerASN, + update.Timestamp, update.MessageType, update.PeerIP) } else { - _, err = insertStmt.Exec( + _, err = insertStmt.ExecContext(ctx, generateUUID().String(), update.PeerIP, update.PeerASN, update.Timestamp, update.Timestamp, update.MessageType) } + if err != nil { return fmt.Errorf("failed to update peer %s: %w", update.PeerIP, err) } + if !exists { newPeers++ } } - if err = tx.Commit(); err != nil { + err = tx.Commit() + if err != nil { return fmt.Errorf("failed to commit transaction: %w", err) } @@ -1066,7 +1182,9 @@ func (d *Database) UpdatePeerBatch(peers map[string]PeerUpdate) error { } // UpdatePeer updates or creates a BGP peer record -func (d *Database) UpdatePeer(peerIP string, peerASN int, messageType string, timestamp time.Time) error { +func (d *Database) UpdatePeer( + peerIP string, peerASN int, messageType string, timestamp time.Time, +) error { d.lock("UpdatePeer") defer d.unlock() @@ -1074,34 +1192,38 @@ func (d *Database) UpdatePeer(peerIP string, peerASN int, messageType string, ti if err != nil { return err } - defer func() { - if err := tx.Rollback(); err != nil && err != sql.ErrTxDone { - d.logger.Error("Failed to rollback transaction", "error", err) - } - }() + defer d.rollback(tx) var exists bool - err = tx.QueryRow("SELECT EXISTS(SELECT 1 FROM bgp_peers WHERE peer_ip = ?)", peerIP).Scan(&exists) + + err = tx.QueryRow( + "SELECT EXISTS(SELECT 1 FROM bgp_peers WHERE peer_ip = ?)", peerIP, + ).Scan(&exists) if err != nil { return err } if exists { _, err = tx.Exec( - "UPDATE bgp_peers SET peer_asn = ?, last_seen = ?, last_message_type = ? WHERE peer_ip = ?", + "UPDATE bgp_peers SET peer_asn = ?, last_seen = ?, "+ + "last_message_type = ? WHERE peer_ip = ?", peerASN, timestamp, messageType, peerIP, ) } else { _, err = tx.Exec( - "INSERT INTO bgp_peers (id, peer_ip, peer_asn, first_seen, last_seen, last_message_type) VALUES (?, ?, ?, ?, ?, ?)", - generateUUID().String(), peerIP, peerASN, timestamp, timestamp, messageType, + "INSERT INTO bgp_peers (id, peer_ip, peer_asn, first_seen, "+ + "last_seen, last_message_type) VALUES (?, ?, ?, ?, ?, ?)", + generateUUID().String(), peerIP, peerASN, timestamp, timestamp, + messageType, ) } + if err != nil { return err } - if err = tx.Commit(); err != nil { + err = tx.Commit() + if err != nil { d.logger.Error("Failed to commit transaction for peer update", "peer_ip", peerIP, "peer_asn", peerASN, @@ -1138,7 +1260,8 @@ func (d *Database) GetStatsContext(ctx context.Context) (Stats, error) { d.counts.fill(&stats) // Database file size is a cheap stat() on the file. - if fileInfo, err := os.Stat(d.path); err != nil { + fileInfo, err := os.Stat(d.path) + if err != nil { d.logger.Warn("Failed to get database file size", "error", err) } else { stats.FileSizeBytes = fileInfo.Size() @@ -1161,64 +1284,6 @@ func (d *Database) GetStatsContext(ctx context.Context) (Stats, error) { return stats, nil } -// routeTimestampRange returns the earliest and latest last_updated across both -// live route tables, or nil values when both tables are empty. Each query reads -// one row from an end of the last_updated index rather than scanning the tables. -func (d *Database) routeTimestampRange(ctx context.Context) (oldest, newest *time.Time, err error) { - oldestV4, ok, err := d.scanRouteTimestamp(ctx, - "SELECT last_updated FROM live_routes_v4 ORDER BY last_updated ASC LIMIT 1") - if err != nil { - return nil, nil, err - } - if ok { - oldest = &oldestV4 - } - - oldestV6, ok, err := d.scanRouteTimestamp(ctx, - "SELECT last_updated FROM live_routes_v6 ORDER BY last_updated ASC LIMIT 1") - if err != nil { - return nil, nil, err - } - if ok && (oldest == nil || oldestV6.Before(*oldest)) { - oldest = &oldestV6 - } - - newestV4, ok, err := d.scanRouteTimestamp(ctx, - "SELECT last_updated FROM live_routes_v4 ORDER BY last_updated DESC LIMIT 1") - if err != nil { - return nil, nil, err - } - if ok { - newest = &newestV4 - } - - newestV6, ok, err := d.scanRouteTimestamp(ctx, - "SELECT last_updated FROM live_routes_v6 ORDER BY last_updated DESC LIMIT 1") - if err != nil { - return nil, nil, err - } - if ok && (newest == nil || newestV6.After(*newest)) { - newest = &newestV6 - } - - return oldest, newest, nil -} - -// scanRouteTimestamp runs a single-row timestamp query. ok is false when the -// table is empty. The query selects the last_updated column directly so the -// driver parses the DATETIME value into a time.Time. -func (d *Database) scanRouteTimestamp(ctx context.Context, query string) (ts time.Time, ok bool, err error) { - err = d.db.QueryRowContext(ctx, query).Scan(&ts) - switch { - case errors.Is(err, sql.ErrNoRows): - return time.Time{}, false, nil - case err != nil: - return time.Time{}, false, err - default: - return ts, true, nil - } -} - // UpsertLiveRoute inserts or updates a live route func (d *Database) UpsertLiveRoute(route *LiveRoute) error { d.lock("UpsertLiveRoute") @@ -1229,37 +1294,37 @@ func (d *Database) UpsertLiveRoute(route *LiveRoute) error { return fmt.Errorf("failed to encode AS path: %w", err) } - updateSQL, insertSQL, lookupSQL := updateLiveRouteV4SQL, insertLiveRouteV4SQL, prefixHasLiveRouteV4SQL + updateSQL, insertSQL, lookupSQL := updateLiveRouteV4SQL, + insertLiveRouteV4SQL, prefixHasLiveRouteV4SQL if route.IPVersion == ipVersionV6 { - updateSQL, insertSQL, lookupSQL = updateLiveRouteV6SQL, insertLiveRouteV6SQL, prefixHasLiveRouteV6SQL + updateSQL, insertSQL, lookupSQL = updateLiveRouteV6SQL, + insertLiveRouteV6SQL, prefixHasLiveRouteV6SQL } + ctx := context.Background() + // The write lock is held, so no other writer can insert this key between the // update and the insert even though they are separate autocommit statements. - upd, err := d.db.Prepare(updateSQL) + upd, err := d.db.PrepareContext(ctx, updateSQL) if err != nil { return fmt.Errorf("failed to prepare update statement: %w", err) } defer func() { _ = upd.Close() }() - ins, err := d.db.Prepare(insertSQL) + ins, err := d.db.PrepareContext(ctx, insertSQL) if err != nil { return fmt.Errorf("failed to prepare insert statement: %w", err) } defer func() { _ = ins.Close() }() - has, err := d.db.Prepare(lookupSQL) + has, err := d.db.PrepareContext(ctx, lookupSQL) if err != nil { return fmt.Errorf("failed to prepare prefix lookup statement: %w", err) } defer func() { _ = has.Close() }() - var inserted, newPrefix bool - if route.IPVersion == ipVersionV4 { - inserted, newPrefix, err = upsertRouteRowV4(upd, ins, has, route, string(pathJSON)) - } else { - inserted, newPrefix, err = upsertRouteRowV6(upd, ins, has, route, string(pathJSON)) - } + inserted, newPrefix, err := upsertRouteRow(ctx, upd, ins, has, route, + string(pathJSON)) if err != nil { return fmt.Errorf("failed to upsert route %s: %w", route.Prefix, err) } @@ -1271,6 +1336,7 @@ func (d *Database) UpsertLiveRoute(route *LiveRoute) error { d.counts.addRoutes(0, 1) } } + if newPrefix { if route.IPVersion == ipVersionV4 { d.counts.addToDistribution([]int{route.MaskLength}, nil, 1) @@ -1282,9 +1348,30 @@ func (d *Database) UpsertLiveRoute(route *LiveRoute) error { return nil } +// deleteLiveRouteRows deletes the live routes for prefix and peerIP from the +// table of the prefix's IP version, only those from originASN unless it is 0. +// Literal per-table queries (rather than one formatted with the table name) +// keep the delete free of dynamically built SQL. +func deleteLiveRouteRows( + tx *loggingTx, isV4 bool, prefix string, originASN int, peerIP string, +) (sql.Result, error) { + switch { + case isV4 && originASN == 0: + return tx.Exec(deleteLiveRouteAnyOriginV4SQL, prefix, peerIP) + case isV4: + return tx.Exec(deleteLiveRouteV4SQL, prefix, originASN, peerIP) + case originASN == 0: + return tx.Exec(deleteLiveRouteAnyOriginV6SQL, prefix, peerIP) + default: + return tx.Exec(deleteLiveRouteV6SQL, prefix, originASN, peerIP) + } +} + // DeleteLiveRoute deletes a live route // If originASN is 0, deletes all routes for the prefix/peer combination -func (d *Database) DeleteLiveRoute(prefix string, originASN int, peerIP string) error { +func (d *Database) DeleteLiveRoute( + prefix string, originASN int, peerIP string, +) error { d.lock("DeleteLiveRoute") defer d.unlock() @@ -1302,30 +1389,9 @@ func (d *Database) DeleteLiveRoute(prefix string, originASN int, peerIP string) if err != nil { return fmt.Errorf("failed to begin transaction: %w", err) } - defer func() { - if err := tx.Rollback(); err != nil && err != sql.ErrTxDone { - d.logger.Error("Failed to rollback transaction", "error", err) - } - }() + defer d.rollback(tx) - // Literal per-table queries (rather than one formatted with the table name) - // so the delete carries no dynamically built SQL. A delete with no origin - // ASN can remove several rows. - var res sql.Result - switch { - case isV4 && originASN == 0: - res, err = tx.Exec(`DELETE FROM live_routes_v4 WHERE prefix = ? AND peer_ip = ?`, prefix, peerIP) - case isV4: - res, err = tx.Exec( - `DELETE FROM live_routes_v4 WHERE prefix = ? AND origin_asn = ? AND peer_ip = ?`, - prefix, originASN, peerIP) - case originASN == 0: - res, err = tx.Exec(`DELETE FROM live_routes_v6 WHERE prefix = ? AND peer_ip = ?`, prefix, peerIP) - default: - res, err = tx.Exec( - `DELETE FROM live_routes_v6 WHERE prefix = ? AND origin_asn = ? AND peer_ip = ?`, - prefix, originASN, peerIP) - } + res, err := deleteLiveRouteRows(tx, isV4, prefix, originASN, peerIP) if err != nil { return err } @@ -1334,6 +1400,7 @@ func (d *Database) DeleteLiveRoute(prefix string, originASN int, peerIP string) if err != nil { return err } + if affected == 0 { return nil } @@ -1343,12 +1410,16 @@ func (d *Database) DeleteLiveRoute(prefix string, originASN int, peerIP string) if isV4 { lookupSQL = prefixHasLiveRouteV4SQL } + var prefixHasRoute bool - if err := tx.QueryRow(lookupSQL, prefix).Scan(&prefixHasRoute); err != nil { + + err = tx.QueryRow(lookupSQL, prefix).Scan(&prefixHasRoute) + if err != nil { return err } - if err := tx.Commit(); err != nil { + err = tx.Commit() + if err != nil { return fmt.Errorf("failed to commit transaction: %w", err) } @@ -1357,6 +1428,7 @@ func (d *Database) DeleteLiveRoute(prefix string, originASN int, peerIP string) } else { d.counts.addRoutes(0, -int(affected)) } + if !prefixHasRoute { maskLength, _ := ipnet.Mask.Size() if isV4 { @@ -1369,16 +1441,22 @@ func (d *Database) DeleteLiveRoute(prefix string, originASN int, peerIP string) return nil } -// GetPrefixDistribution returns the distribution of unique prefixes by mask length -func (d *Database) GetPrefixDistribution() (ipv4 []PrefixDistribution, ipv6 []PrefixDistribution, err error) { +// GetPrefixDistribution returns the distribution of unique prefixes by mask +// length +func (d *Database) GetPrefixDistribution() ( + []PrefixDistribution, []PrefixDistribution, error, +) { return d.GetPrefixDistributionContext(context.Background()) } -// GetPrefixDistributionContext returns the distribution of unique prefixes by mask length with context support. -// It reads every live route, so the stats read does not call it; it seeds the -// in-memory distribution once at startup. +// GetPrefixDistributionContext returns the distribution of unique prefixes by +// mask length with context support. It reads every live route, so the stats +// read does not call it; it seeds the in-memory distribution once at startup. func (d *Database) GetPrefixDistributionContext(ctx context.Context) ( - ipv4 []PrefixDistribution, ipv6 []PrefixDistribution, err error) { + []PrefixDistribution, []PrefixDistribution, error, +) { + var ipv4, ipv6 []PrefixDistribution + // IPv4 distribution - count unique prefixes from v4 table query := ` SELECT mask_length, COUNT(DISTINCT prefix) as count @@ -1386,6 +1464,7 @@ func (d *Database) GetPrefixDistributionContext(ctx context.Context) ( GROUP BY mask_length ORDER BY mask_length ` + rows4, err := d.db.QueryContext(ctx, query) if err != nil { return nil, nil, fmt.Errorf("failed to query IPv4 distribution: %w", err) @@ -1398,13 +1477,19 @@ func (d *Database) GetPrefixDistributionContext(ctx context.Context) ( for rows4.Next() { var dist PrefixDistribution - if err := rows4.Scan(&dist.MaskLength, &dist.Count); err != nil { - return nil, nil, fmt.Errorf("failed to scan IPv4 distribution: %w", err) + + err := rows4.Scan(&dist.MaskLength, &dist.Count) + if err != nil { + return nil, nil, fmt.Errorf("failed to scan IPv4 distribution: %w", + err) } + ipv4 = append(ipv4, dist) } + // A read that stops partway ends the loop without an error of its own. - if err := rows4.Err(); err != nil { + err = rows4.Err() + if err != nil { return nil, nil, fmt.Errorf("failed to read IPv4 distribution: %w", err) } @@ -1415,6 +1500,7 @@ func (d *Database) GetPrefixDistributionContext(ctx context.Context) ( GROUP BY mask_length ORDER BY mask_length ` + rows6, err := d.db.QueryContext(ctx, query) if err != nil { return nil, nil, fmt.Errorf("failed to query IPv6 distribution: %w", err) @@ -1427,12 +1513,18 @@ func (d *Database) GetPrefixDistributionContext(ctx context.Context) ( for rows6.Next() { var dist PrefixDistribution - if err := rows6.Scan(&dist.MaskLength, &dist.Count); err != nil { - return nil, nil, fmt.Errorf("failed to scan IPv6 distribution: %w", err) + + err := rows6.Scan(&dist.MaskLength, &dist.Count) + if err != nil { + return nil, nil, fmt.Errorf("failed to scan IPv6 distribution: %w", + err) } + ipv6 = append(ipv6, dist) } - if err := rows6.Err(); err != nil { + + err = rows6.Err() + if err != nil { return nil, nil, fmt.Errorf("failed to read IPv6 distribution: %w", err) } @@ -1440,20 +1532,27 @@ func (d *Database) GetPrefixDistributionContext(ctx context.Context) ( } // GetLiveRouteCounts returns the count of IPv4 and IPv6 routes -func (d *Database) GetLiveRouteCounts() (ipv4Count, ipv6Count int, err error) { +func (d *Database) GetLiveRouteCounts() (int, int, error) { return d.GetLiveRouteCountsContext(context.Background()) } -// GetLiveRouteCountsContext returns the count of IPv4 and IPv6 routes with context support -func (d *Database) GetLiveRouteCountsContext(ctx context.Context) (ipv4Count, ipv6Count int, err error) { +// GetLiveRouteCountsContext returns the count of IPv4 and IPv6 routes with +// context support +func (d *Database) GetLiveRouteCountsContext( + ctx context.Context, +) (int, int, error) { + var ipv4Count, ipv6Count int + // Get IPv4 count from dedicated table - err = d.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM live_routes_v4").Scan(&ipv4Count) + err := d.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM live_routes_v4"). + Scan(&ipv4Count) if err != nil { return 0, 0, fmt.Errorf("failed to count IPv4 routes: %w", err) } // Get IPv6 count from dedicated table - err = d.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM live_routes_v6").Scan(&ipv6Count) + err = d.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM live_routes_v6"). + Scan(&ipv6Count) if err != nil { return 0, 0, fmt.Errorf("failed to count IPv6 routes: %w", err) } @@ -1466,18 +1565,25 @@ func (d *Database) GetASDetails(asn int) (*ASN, []LiveRoute, error) { return d.GetASDetailsContext(context.Background(), asn) } -// GetASDetailsContext returns detailed information about an AS including prefixes with context support -func (d *Database) GetASDetailsContext(ctx context.Context, asn int) (*ASN, []LiveRoute, error) { +// GetASDetailsContext returns detailed information about an AS including +// prefixes with context support +func (d *Database) GetASDetailsContext( + ctx context.Context, asn int, +) (*ASN, []LiveRoute, error) { // Get AS information - var asnInfo ASN - var handle, description sql.NullString - err := d.db.QueryRowContext(ctx, - "SELECT asn, handle, description, first_seen, last_seen FROM asns WHERE asn = ?", - asn, - ).Scan(&asnInfo.ASN, &handle, &description, &asnInfo.FirstSeen, &asnInfo.LastSeen) + var ( + asnInfo ASN + handle, description sql.NullString + ) + err := d.db.QueryRowContext(ctx, + "SELECT asn, handle, description, first_seen, last_seen "+ + "FROM asns WHERE asn = ?", + asn, + ).Scan(&asnInfo.ASN, &handle, &description, &asnInfo.FirstSeen, + &asnInfo.LastSeen) if err != nil { - if err == sql.ErrNoRows { + if errors.Is(err, sql.ErrNoRows) { return nil, nil, fmt.Errorf("%w: AS%d", ErrNoRoute, asn) } @@ -1488,9 +1594,6 @@ func (d *Database) GetASDetailsContext(ctx context.Context, asn int) (*ASN, []Li asnInfo.Description = description.String // Get prefixes announced by this AS from both tables - var allPrefixes []LiveRoute - - // Query IPv4 prefixes queryV4 := ` SELECT prefix, mask_length, MAX(last_updated) as last_updated FROM live_routes_v4 @@ -1498,38 +1601,11 @@ func (d *Database) GetASDetailsContext(ctx context.Context, asn int) (*ASN, []Li GROUP BY prefix, mask_length ` - rows4, err := d.db.QueryContext(ctx, queryV4, asn) + allPrefixes, err := d.originPrefixes(ctx, queryV4, asn, ipVersionV4) if err != nil { - return &asnInfo, nil, fmt.Errorf("failed to query IPv4 prefixes: %w", err) - } - defer func() { _ = rows4.Close() }() - - for rows4.Next() { - var route LiveRoute - var lastUpdatedStr string - err := rows4.Scan(&route.Prefix, &route.MaskLength, &lastUpdatedStr) - if err != nil { - d.logger.Error("Failed to scan IPv4 prefix row", "error", err, "asn", asn) - - continue - } - // Parse the timestamp string - route.LastUpdated, err = time.Parse("2006-01-02 15:04:05-07:00", lastUpdatedStr) - if err != nil { - // Try without timezone - route.LastUpdated, err = time.Parse("2006-01-02 15:04:05", lastUpdatedStr) - if err != nil { - d.logger.Error("Failed to parse timestamp", "error", err, "timestamp", lastUpdatedStr) - - continue - } - } - route.OriginASN = asn - route.IPVersion = ipVersionV4 - allPrefixes = append(allPrefixes, route) + return &asnInfo, nil, err } - // Query IPv6 prefixes queryV6 := ` SELECT prefix, mask_length, MAX(last_updated) as last_updated FROM live_routes_v6 @@ -1537,41 +1613,18 @@ func (d *Database) GetASDetailsContext(ctx context.Context, asn int) (*ASN, []Li GROUP BY prefix, mask_length ` - rows6, err := d.db.QueryContext(ctx, queryV6, asn) + prefixesV6, err := d.originPrefixes(ctx, queryV6, asn, ipVersionV6) if err != nil { - return &asnInfo, allPrefixes, fmt.Errorf("failed to query IPv6 prefixes: %w", err) - } - defer func() { _ = rows6.Close() }() - - for rows6.Next() { - var route LiveRoute - var lastUpdatedStr string - err := rows6.Scan(&route.Prefix, &route.MaskLength, &lastUpdatedStr) - if err != nil { - d.logger.Error("Failed to scan IPv6 prefix row", "error", err, "asn", asn) - - continue - } - // Parse the timestamp string - route.LastUpdated, err = time.Parse("2006-01-02 15:04:05-07:00", lastUpdatedStr) - if err != nil { - // Try without timezone - route.LastUpdated, err = time.Parse("2006-01-02 15:04:05", lastUpdatedStr) - if err != nil { - d.logger.Error("Failed to parse timestamp", "error", err, "timestamp", lastUpdatedStr) - - continue - } - } - route.OriginASN = asn - route.IPVersion = ipVersionV6 - allPrefixes = append(allPrefixes, route) + return &asnInfo, allPrefixes, err } - return &asnInfo, allPrefixes, nil + return &asnInfo, append(allPrefixes, prefixesV6...), nil } -// ASPeer represents a peering relationship with another AS including handle, description, and timestamps. +// ASPeer represents a peering relationship with another AS including handle, +// description, and timestamps. +// +//nolint:tagliatelle // field names are the existing JSON API type ASPeer struct { ASN int `json:"asn"` Handle string `json:"handle"` @@ -1585,8 +1638,11 @@ func (d *Database) GetASPeers(asn int) ([]ASPeer, error) { return d.GetASPeersContext(context.Background(), asn) } -// GetASPeersContext returns all ASes that peer with the given AS with context support -func (d *Database) GetASPeersContext(ctx context.Context, asn int) ([]ASPeer, error) { +// GetASPeersContext returns all ASes that peer with the given AS with context +// support +func (d *Database) GetASPeersContext( + ctx context.Context, asn int, +) ([]ASPeer, error) { query := ` SELECT CASE @@ -1613,18 +1669,23 @@ func (d *Database) GetASPeersContext(ctx context.Context, asn int) ([]ASPeer, er defer func() { _ = rows.Close() }() var peers []ASPeer + for rows.Next() { var peer ASPeer - err := rows.Scan(&peer.ASN, &peer.Handle, &peer.Description, &peer.FirstSeen, &peer.LastSeen) + + err := rows.Scan(&peer.ASN, &peer.Handle, &peer.Description, + &peer.FirstSeen, &peer.LastSeen) if err != nil { d.logger.Error("Failed to scan peer row", "error", err, "asn", asn) continue } + peers = append(peers, peer) } - if err := rows.Err(); err != nil { + err = rows.Err() + if err != nil { return nil, fmt.Errorf("error iterating AS peers: %w", err) } @@ -1632,30 +1693,34 @@ func (d *Database) GetASPeersContext(ctx context.Context, asn int) ([]ASPeer, er } // GetPrefixDetails returns detailed information about a prefix -func (d *Database) GetPrefixDetails(prefix string) ([]LiveRoute, map[int]ASN, error) { +func (d *Database) GetPrefixDetails( + prefix string, +) ([]LiveRoute, map[int]ASN, error) { return d.GetPrefixDetailsContext(context.Background(), prefix) } // GetPrefixDetailsContext returns the live routes for a prefix and, keyed by // AS number, the handle and description of each AS that originates it. -func (d *Database) GetPrefixDetailsContext(ctx context.Context, prefix string) ([]LiveRoute, map[int]ASN, error) { +func (d *Database) GetPrefixDetailsContext( + ctx context.Context, prefix string, +) ([]LiveRoute, map[int]ASN, error) { // Determine if it's IPv4 or IPv6 by parsing the prefix _, ipnet, err := net.ParseCIDR(prefix) if err != nil { return nil, nil, fmt.Errorf("invalid prefix format: %w", err) } - tableName := "live_routes_v4" + tableName := tableLiveRoutesV4 ipVersion := ipVersionV4 + if ipnet.IP.To4() == nil { - tableName = "live_routes_v6" + tableName = tableLiveRoutesV6 ipVersion = ipVersionV6 } - //nolint:gosec // Table name is hardcoded based on IP version query := fmt.Sprintf(` - SELECT lr.origin_asn, lr.peer_ip, lr.as_path, lr.next_hop, lr.last_updated, - a.handle, a.description + SELECT lr.origin_asn, lr.peer_ip, lr.as_path, lr.next_hop, + lr.last_updated, a.handle, a.description FROM %s lr LEFT JOIN asns a ON a.asn = lr.origin_asn WHERE lr.prefix = ? @@ -1669,11 +1734,15 @@ func (d *Database) GetPrefixDetailsContext(ctx context.Context, prefix string) ( defer func() { _ = rows.Close() }() var routes []LiveRoute + origins := make(map[int]ASN) + for rows.Next() { - var route LiveRoute - var pathJSON string - var handle, description sql.NullString + var ( + route LiveRoute + pathJSON string + handle, description sql.NullString + ) err := rows.Scan( &route.OriginASN, &route.PeerIP, &pathJSON, &route.NextHop, @@ -1684,7 +1753,8 @@ func (d *Database) GetPrefixDetailsContext(ctx context.Context, prefix string) ( } // Decode AS path - if err := json.Unmarshal([]byte(pathJSON), &route.ASPath); err != nil { + err = json.Unmarshal([]byte(pathJSON), &route.ASPath) + if err != nil { route.ASPath = []int{} } @@ -1699,6 +1769,11 @@ func (d *Database) GetPrefixDetailsContext(ctx context.Context, prefix string) ( } } + err = rows.Err() + if err != nil { + return nil, nil, fmt.Errorf("failed to read prefix details: %w", err) + } + if len(routes) == 0 { return nil, nil, fmt.Errorf("%w: %s", ErrNoRoute, prefix) } @@ -1706,21 +1781,26 @@ func (d *Database) GetPrefixDetailsContext(ctx context.Context, prefix string) ( return routes, origins, nil } -// GetRandomPrefixesByLength returns a random sample of prefixes with the specified mask length -func (d *Database) GetRandomPrefixesByLength(maskLength, ipVersion, limit int) ([]LiveRoute, error) { - return d.GetRandomPrefixesByLengthContext(context.Background(), maskLength, ipVersion, limit) +// GetRandomPrefixesByLength returns a random sample of prefixes with the +// specified mask length +func (d *Database) GetRandomPrefixesByLength( + maskLength, ipVersion, limit int, +) ([]LiveRoute, error) { + return d.GetRandomPrefixesByLengthContext(context.Background(), + maskLength, ipVersion, limit) } -// GetRandomPrefixesByLengthContext returns a random sample of prefixes with context support +// GetRandomPrefixesByLengthContext returns a random sample of prefixes with +// context support func (d *Database) GetRandomPrefixesByLengthContext( - ctx context.Context, maskLength, ipVersion, limit int) ([]LiveRoute, error) { + ctx context.Context, maskLength, ipVersion, limit int, +) ([]LiveRoute, error) { // Select unique prefixes with their most recent route information - tableName := "live_routes_v4" + tableName := tableLiveRoutesV4 if ipVersion == ipVersionV6 { - tableName = "live_routes_v6" + tableName = tableLiveRoutesV6 } - //nolint:gosec // Table name is hardcoded based on IP version query := fmt.Sprintf(` WITH unique_prefixes AS ( SELECT prefix, MAX(last_updated) as max_updated @@ -1730,10 +1810,11 @@ func (d *Database) GetRandomPrefixesByLengthContext( ORDER BY RANDOM() LIMIT ? ) - SELECT lr.prefix, lr.mask_length, lr.origin_asn, lr.as_path, + SELECT lr.prefix, lr.mask_length, lr.origin_asn, lr.as_path, lr.peer_ip, lr.last_updated FROM %s lr - INNER JOIN unique_prefixes up ON lr.prefix = up.prefix AND lr.last_updated = up.max_updated + INNER JOIN unique_prefixes up + ON lr.prefix = up.prefix AND lr.last_updated = up.max_updated WHERE lr.mask_length = ? `, tableName, tableName) @@ -1746,9 +1827,13 @@ func (d *Database) GetRandomPrefixesByLengthContext( }() var routes []LiveRoute + for rows.Next() { - var route LiveRoute - var pathJSON string + var ( + route LiveRoute + pathJSON string + ) + err := rows.Scan( &route.Prefix, &route.MaskLength, @@ -1759,23 +1844,32 @@ func (d *Database) GetRandomPrefixesByLengthContext( ) // Set IP version based on which table we queried route.IPVersion = ipVersion + if err != nil { continue } // Decode AS path - if err := json.Unmarshal([]byte(pathJSON), &route.ASPath); err != nil { + err = json.Unmarshal([]byte(pathJSON), &route.ASPath) + if err != nil { route.ASPath = []int{} } routes = append(routes, route) } + err = rows.Err() + if err != nil { + return nil, fmt.Errorf("failed to read random prefixes: %w", err) + } + return routes, nil } // GetNextStaleASN returns a random ASN that needs WHOIS data refresh. -func (d *Database) GetNextStaleASN(ctx context.Context, staleThreshold time.Duration) (int, error) { +func (d *Database) GetNextStaleASN( + ctx context.Context, staleThreshold time.Duration, +) (int, error) { cutoff := time.Now().Add(-staleThreshold) // Select a random stale ASN using ORDER BY RANDOM() @@ -1788,9 +1882,10 @@ func (d *Database) GetNextStaleASN(ctx context.Context, staleThreshold time.Dura ` var asn int + err := d.db.QueryRowContext(ctx, query, cutoff).Scan(&asn) if err != nil { - if err == sql.ErrNoRows { + if errors.Is(err, sql.ErrNoRows) { return 0, ErrNoStaleASN } @@ -1801,6 +1896,8 @@ func (d *Database) GetNextStaleASN(ctx context.Context, staleThreshold time.Dura } // WHOISStats contains statistics about WHOIS data freshness. +// +//nolint:tagliatelle // field names are the existing JSON API type WHOISStats struct { TotalASNs int `json:"total_asns"` StaleASNs int `json:"stale_asns"` @@ -1809,19 +1906,25 @@ type WHOISStats struct { } // GetWHOISStats returns statistics about WHOIS data freshness. -func (d *Database) GetWHOISStats(ctx context.Context, staleThreshold time.Duration) (*WHOISStats, error) { +func (d *Database) GetWHOISStats( + ctx context.Context, staleThreshold time.Duration, +) (*WHOISStats, error) { cutoff := time.Now().Add(-staleThreshold) query := ` SELECT COUNT(*) as total, - COALESCE(SUM(CASE WHEN whois_updated_at IS NULL THEN 1 ELSE 0 END), 0) as never_fetched, - COALESCE(SUM(CASE WHEN whois_updated_at IS NOT NULL AND whois_updated_at < ? THEN 1 ELSE 0 END), 0) as stale, - COALESCE(SUM(CASE WHEN whois_updated_at IS NOT NULL AND whois_updated_at >= ? THEN 1 ELSE 0 END), 0) as fresh + COALESCE(SUM(CASE WHEN whois_updated_at IS NULL + THEN 1 ELSE 0 END), 0) as never_fetched, + COALESCE(SUM(CASE WHEN whois_updated_at IS NOT NULL + AND whois_updated_at < ? THEN 1 ELSE 0 END), 0) as stale, + COALESCE(SUM(CASE WHEN whois_updated_at IS NOT NULL + AND whois_updated_at >= ? THEN 1 ELSE 0 END), 0) as fresh FROM asns ` var stats WHOISStats + err := d.db.QueryRowContext(ctx, query, cutoff, cutoff).Scan( &stats.TotalASNs, &stats.NeverFetched, @@ -1836,7 +1939,9 @@ func (d *Database) GetWHOISStats(ctx context.Context, staleThreshold time.Durati } // UpdateASNWHOIS updates an ASN record with WHOIS data. -func (d *Database) UpdateASNWHOIS(ctx context.Context, update *ASNWHOISUpdate) error { +func (d *Database) UpdateASNWHOIS( + ctx context.Context, update *ASNWHOISUpdate, +) error { d.lock("UpdateASNWHOIS") defer d.unlock() @@ -1888,41 +1993,17 @@ func (d *Database) GetIPInfo(ip string) (*IPInfo, error) { return d.GetIPInfoContext(context.Background(), ip) } -// GetIPInfoContext returns comprehensive IP information with context support. -// -// It finds the most specific live route for the address with point lookups on -// the prefix index: for each mask length from the longest (32 or 128) down to -// minLookupMaskLengthV4 or minLookupMaskLengthV6 it looks up the address's -// network prefix at that length, and the first length with a live route wins. -// That is at most 25 or 113 indexed lookups, however many routes are live. The -// prefix is looked up in the text form net/netip prints, which is the form the -// prefix handler stores. -func (d *Database) GetIPInfoContext(ctx context.Context, ip string) (*IPInfo, error) { - addr, err := netip.ParseAddr(ip) - if err != nil { - return nil, fmt.Errorf("%w: %s", ErrInvalidIP, ip) - } - // An IPv4 address written in IPv6 form (::ffff:192.0.2.1) is an IPv4 lookup. - addr = addr.Unmap() - - info := &IPInfo{IP: ip, IPVersion: ipVersionV6} - routesTable, prefixesTable := "live_routes_v6", "prefixes_v6" - minMaskLength := minLookupMaskLengthV6 - if addr.Is4() { - info.IPVersion = ipVersionV4 - routesTable, prefixesTable = "live_routes_v4", "prefixes_v4" - minMaskLength = minLookupMaskLengthV4 - } - - // Route info with peer count and prefix first_seen - //nolint:gosec // Table names are hardcoded based on IP version - query := fmt.Sprintf(` +// ipInfoQuery reads the live route for one prefix with its peer count, the +// prefix's first_seen and the origin AS's details. %[1]s is the live route +// table and %[2]s the prefix table of the address's IP version. +const ipInfoQuery = ` SELECT lr.prefix, lr.mask_length, lr.origin_asn, lr.last_updated, - (SELECT COUNT(DISTINCT peer_ip) FROM %[1]s WHERE prefix = lr.prefix) as num_peers, + (SELECT COUNT(DISTINCT peer_ip) FROM %[1]s + WHERE prefix = lr.prefix) as num_peers, p.first_seen, a.handle, a.description, @@ -1939,7 +2020,39 @@ func (d *Database) GetIPInfoContext(ctx context.Context, ip string) (*IPInfo, er LEFT JOIN asns a ON a.asn = lr.origin_asn WHERE lr.prefix = ? LIMIT 1 - `, routesTable, prefixesTable) + ` + +// GetIPInfoContext returns comprehensive IP information with context support. +// +// It finds the most specific live route for the address with point lookups on +// the prefix index: for each mask length from the longest (32 or 128) down to +// minLookupMaskLengthV4 or minLookupMaskLengthV6 it looks up the address's +// network prefix at that length, and the first length with a live route wins. +// That is at most 25 or 113 indexed lookups, however many routes are live. The +// prefix is looked up in the text form net/netip prints, which is the form the +// prefix handler stores. +func (d *Database) GetIPInfoContext( + ctx context.Context, ip string, +) (*IPInfo, error) { + addr, err := netip.ParseAddr(ip) + if err != nil { + return nil, fmt.Errorf("%w: %s", ErrInvalidIP, ip) + } + // An IPv4 address written in IPv6 form (::ffff:192.0.2.1) is an IPv4 lookup. + addr = addr.Unmap() + + info := &IPInfo{IP: ip, IPVersion: ipVersionV6} + minMaskLength := minLookupMaskLengthV6 + + routesTable, prefixesTable := tableLiveRoutesV6, "prefixes_v6" + if addr.Is4() { + routesTable, prefixesTable = tableLiveRoutesV4, "prefixes_v4" + minMaskLength = minLookupMaskLengthV4 + info.IPVersion = ipVersionV4 + } + + // Route info with peer count and prefix first_seen + query := fmt.Sprintf(ipInfoQuery, routesTable, prefixesTable) stmt, err := d.db.PrepareContext(ctx, query) if err != nil { @@ -1947,41 +2060,36 @@ func (d *Database) GetIPInfoContext(ctx context.Context, ip string) (*IPInfo, er } defer func() { _ = stmt.Close() }() - var handle, description, asName, orgName, orgID, address, countryCode, abuseEmail, rir sql.NullString - var prefixFirstSeen sql.NullTime - var whoisUpdatedAt sql.NullTime + var ( + handle, description, asName, orgName, orgID sql.NullString + address, countryCode, abuseEmail, rir sql.NullString + prefixFirstSeen, whoisUpdatedAt sql.NullTime + ) found := false + for bits := addr.BitLen(); bits >= minMaskLength; bits-- { prefix := netip.PrefixFrom(addr, bits).Masked().String() + err := stmt.QueryRowContext(ctx, prefix).Scan( - &info.Netblock, - &info.MaskLength, - &info.ASN, - &info.LastSeen, - &info.NumPeers, - &prefixFirstSeen, - &handle, - &description, - &asName, - &orgName, - &orgID, - &address, - &countryCode, - &abuseEmail, - &rir, - &whoisUpdatedAt, + &info.Netblock, &info.MaskLength, &info.ASN, &info.LastSeen, + &info.NumPeers, &prefixFirstSeen, &handle, &description, + &asName, &orgName, &orgID, &address, &countryCode, &abuseEmail, + &rir, &whoisUpdatedAt, ) if errors.Is(err, sql.ErrNoRows) { continue } + if err != nil { return nil, fmt.Errorf("failed to query routes: %w", err) } + found = true break } + if !found { return nil, fmt.Errorf("%w for IP %s", ErrNoRoute, ip) } @@ -2002,18 +2110,23 @@ func (d *Database) GetIPInfoContext(ctx context.Context, ip string) (*IPInfo, er // Check if WHOIS data needs refresh (never fetched or older than 30 days) const staleThreshold = 30 * 24 * time.Hour - info.NeedsWHOISRefresh = !whoisUpdatedAt.Valid || time.Since(whoisUpdatedAt.Time) > staleThreshold + + info.NeedsWHOISRefresh = !whoisUpdatedAt.Valid || + time.Since(whoisUpdatedAt.Time) > staleThreshold return info, nil } -// Vacuum runs incremental vacuum to reclaim unused pages without blocking writes. -// It frees up to the specified number of pages per call (0 = all freeable pages). +// Vacuum runs incremental vacuum to reclaim unused pages without blocking +// writes. It frees up to the specified number of pages per call (0 = all +// freeable pages). func (d *Database) Vacuum(ctx context.Context) error { // Free up to 1000 pages per call (~4MB with default 4KB page size) // This keeps each vacuum operation quick and non-blocking const pagesToFree = 1000 - rows, err := d.db.QueryContext(ctx, fmt.Sprintf("PRAGMA incremental_vacuum(%d)", pagesToFree)) + + rows, err := d.db.QueryContext(ctx, + fmt.Sprintf("PRAGMA incremental_vacuum(%d)", pagesToFree)) if err != nil { return fmt.Errorf("failed to run incremental vacuum: %w", err) } @@ -2024,7 +2137,9 @@ func (d *Database) Vacuum(ctx context.Context) error { // than one page. for rows.Next() { } - if err := rows.Err(); err != nil { + + err = rows.Err() + if err != nil { return fmt.Errorf("failed to run incremental vacuum: %w", err) } @@ -2056,6 +2171,7 @@ func (d *Database) Checkpoint(ctx context.Context) error { // Ping checks if the database connection is alive with a lightweight query. func (d *Database) Ping(ctx context.Context) error { var result int + err := d.db.QueryRowContext(ctx, "SELECT 1").Scan(&result) if err != nil { return fmt.Errorf("database ping failed: %w", err) @@ -2063,3 +2179,185 @@ func (d *Database) Ping(ctx context.Context) error { return nil } + +// lock acquires the database mutex and logs debug information +func (d *Database) lock(operation string) { + // Get caller information + _, file, line, _ := runtime.Caller(1) + caller := fmt.Sprintf("%s:%d", filepath.Base(file), line) + + d.logger.Debug("Acquiring database lock", + "operation", operation, "caller", caller) + + d.mu.Lock() + d.lockedAt = time.Now() + d.lockedBy = fmt.Sprintf("%s (%s)", operation, caller) + + d.logger.Debug("Database lock acquired", + "operation", operation, "caller", caller) +} + +// unlock releases the database mutex and logs debug information including +// hold duration +func (d *Database) unlock() { + holdDuration := time.Since(d.lockedAt) + lockedBy := d.lockedBy + + d.lockedAt = time.Time{} + d.lockedBy = "" + d.mu.Unlock() + + d.logger.Debug("Database lock released", + "held_by", lockedBy, "duration_ms", holdDuration.Milliseconds()) +} + +// beginTx starts a new transaction with logging +func (d *Database) beginTx() (*loggingTx, error) { + tx, err := d.db.BeginTx(context.Background(), nil) + if err != nil { + return nil, err + } + + return &loggingTx{Tx: tx, logger: d.logger}, nil +} + +// rollback rolls back tx unless it has already been committed, and logs a +// failure. Every write defers it right after beginning its transaction. +func (d *Database) rollback(tx *loggingTx) { + err := tx.Rollback() + if err != nil && !errors.Is(err, sql.ErrTxDone) { + d.logger.Error("Failed to rollback transaction", "error", err) + } +} + +// routeTimestampRange returns the earliest and latest last_updated across both +// live route tables, or nil values when both tables are empty. Each query reads +// one row from an end of the last_updated index rather than scanning the tables. +func (d *Database) routeTimestampRange( + ctx context.Context, +) (*time.Time, *time.Time, error) { + var oldest, newest *time.Time + + oldestV4, ok, err := d.scanRouteTimestamp(ctx, "SELECT last_updated "+ + "FROM live_routes_v4 ORDER BY last_updated ASC LIMIT 1") + if err != nil { + return nil, nil, err + } + + if ok { + oldest = &oldestV4 + } + + oldestV6, ok, err := d.scanRouteTimestamp(ctx, "SELECT last_updated "+ + "FROM live_routes_v6 ORDER BY last_updated ASC LIMIT 1") + if err != nil { + return nil, nil, err + } + + if ok && (oldest == nil || oldestV6.Before(*oldest)) { + oldest = &oldestV6 + } + + newestV4, ok, err := d.scanRouteTimestamp(ctx, "SELECT last_updated "+ + "FROM live_routes_v4 ORDER BY last_updated DESC LIMIT 1") + if err != nil { + return nil, nil, err + } + + if ok { + newest = &newestV4 + } + + newestV6, ok, err := d.scanRouteTimestamp(ctx, "SELECT last_updated "+ + "FROM live_routes_v6 ORDER BY last_updated DESC LIMIT 1") + if err != nil { + return nil, nil, err + } + + if ok && (newest == nil || newestV6.After(*newest)) { + newest = &newestV6 + } + + return oldest, newest, nil +} + +// scanRouteTimestamp runs a single-row timestamp query. The bool is false when +// the table is empty. The query selects the last_updated column directly so the +// driver parses the DATETIME value into a time.Time. +func (d *Database) scanRouteTimestamp( + ctx context.Context, query string, +) (time.Time, bool, error) { + var ts time.Time + + err := d.db.QueryRowContext(ctx, query).Scan(&ts) + switch { + case errors.Is(err, sql.ErrNoRows): + return time.Time{}, false, nil + case err != nil: + return time.Time{}, false, err + default: + return ts, true, nil + } +} + +// originPrefixes runs query, which selects the prefix, mask length and latest +// last_updated of every prefix asn originates in the live route table of +// ipVersion. Rows that fail to scan or whose timestamp does not parse are +// logged and skipped. +func (d *Database) originPrefixes( + ctx context.Context, query string, asn, ipVersion int, +) ([]LiveRoute, error) { + family := ipv4Family + if ipVersion == ipVersionV6 { + family = ipv6Family + } + + rows, err := d.db.QueryContext(ctx, query, asn) + if err != nil { + return nil, fmt.Errorf("failed to query %s prefixes: %w", family, err) + } + defer func() { _ = rows.Close() }() + + var routes []LiveRoute + + for rows.Next() { + var ( + route LiveRoute + lastUpdatedStr string + ) + + err := rows.Scan(&route.Prefix, &route.MaskLength, &lastUpdatedStr) + if err != nil { + d.logger.Error("Failed to scan "+family+" prefix row", + "error", err, "asn", asn) + + continue + } + + // Parse the timestamp string + route.LastUpdated, err = time.Parse("2006-01-02 15:04:05-07:00", + lastUpdatedStr) + if err != nil { + // Try without timezone + route.LastUpdated, err = time.Parse("2006-01-02 15:04:05", + lastUpdatedStr) + if err != nil { + d.logger.Error("Failed to parse timestamp", + "error", err, "timestamp", lastUpdatedStr) + + continue + } + } + + route.OriginASN = asn + route.IPVersion = ipVersion + routes = append(routes, route) + } + + err = rows.Err() + if err != nil { + return nil, fmt.Errorf("failed to read %s prefixes: %w", family, err) + } + + return routes, nil +} diff --git a/internal/database/database_test.go b/internal/database/database_internal_test.go similarity index 78% rename from internal/database/database_test.go rename to internal/database/database_internal_test.go index 1b52c08..1477cb4 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_internal_test.go @@ -50,6 +50,8 @@ const ( // address, ErrNoRoute when none covers it, and the next less specific prefix // once the only route of the most specific one is withdrawn. func TestGetIPInfoFindsMostSpecificLiveRoute(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -67,24 +69,7 @@ func TestGetIPInfoFindsMostSpecificLiveRoute(t *testing.T) { "2001:db8:1::/48": 64501, "2001:db8:1:2::/64": 64502, } - ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) - routes := make([]*LiveRoute, 0, len(origins)) - for prefix, asn := range origins { - routes = append(routes, &LiveRoute{ - ID: uuid.New(), - Prefix: prefix, - MaskLength: netip.MustParsePrefix(prefix).Bits(), - IPVersion: detectIPVersion(prefix), - OriginASN: asn, - PeerIP: testPeerIP, - ASPath: []int{asn}, - NextHop: testPeerIP, - LastUpdated: ts, - }) - } - if err := db.UpsertLiveRouteBatch(routes); err != nil { - t.Fatalf("UpsertLiveRouteBatch: %v", err) - } + writeOriginRoutes(t, db, origins) // lookup checks that ip resolves to the live prefix want, or to ErrNoRoute // when want is empty. @@ -99,11 +84,13 @@ func TestGetIPInfoFindsMostSpecificLiveRoute(t *testing.T) { return } + if err != nil { t.Errorf("GetIPInfo(%s): %v", ip, err) return } + if info.Netblock != want || info.MaskLength != netip.MustParsePrefix(want).Bits() || info.ASN != origins[want] { t.Errorf("GetIPInfo(%s) = %s (mask %d) AS%d, want %s AS%d", @@ -122,8 +109,14 @@ func TestGetIPInfoFindsMostSpecificLiveRoute(t *testing.T) { lookup("2001:db9::1", "") err = db.DeleteLiveRouteBatch([]LiveRouteDeletion{ - {Prefix: "10.1.2.0/24", OriginASN: 64502, PeerIP: testPeerIP, IPVersion: ipVersionV4}, - {Prefix: "2001:db8:1:2::/64", OriginASN: 64502, PeerIP: testPeerIP, IPVersion: ipVersionV6}, + { + Prefix: "10.1.2.0/24", OriginASN: 64502, + PeerIP: testPeerIP, IPVersion: ipVersionV4, + }, + { + Prefix: "2001:db8:1:2::/64", OriginASN: 64502, + PeerIP: testPeerIP, IPVersion: ipVersionV6, + }, }) if err != nil { t.Fatalf("DeleteLiveRouteBatch: %v", err) @@ -138,6 +131,8 @@ func TestGetIPInfoFindsMostSpecificLiveRoute(t *testing.T) { // route for an address, and that a route of exactly /8 or /16, or longer, still // is. The prefixes are written out so that a wrong limit fails the test. func TestGetIPInfoIgnoresShortRoutes(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -146,24 +141,11 @@ func TestGetIPInfoIgnoresShortRoutes(t *testing.T) { } defer func() { _ = db.Close() }() - ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + // announce writes one live route for prefix, originated by asn. announce := func(prefix string, asn int) { t.Helper() - err := db.UpsertLiveRouteBatch([]*LiveRoute{{ - ID: uuid.New(), - Prefix: prefix, - MaskLength: netip.MustParsePrefix(prefix).Bits(), - IPVersion: detectIPVersion(prefix), - OriginASN: asn, - PeerIP: testPeerIP, - ASPath: []int{asn}, - NextHop: testPeerIP, - LastUpdated: ts, - }}) - if err != nil { - t.Fatalf("UpsertLiveRouteBatch(%s): %v", prefix, err) - } + writeOriginRoutes(t, db, map[string]int{prefix: asn}) } // lookup checks that ip resolves to the live prefix netblock from asn. @@ -176,8 +158,10 @@ func TestGetIPInfoIgnoresShortRoutes(t *testing.T) { return } + if info.Netblock != netblock || info.ASN != asn { - t.Errorf("GetIPInfo(%s) = %s AS%d, want %s AS%d", ip, info.Netblock, info.ASN, netblock, asn) + t.Errorf("GetIPInfo(%s) = %s AS%d, want %s AS%d", + ip, info.Netblock, info.ASN, netblock, asn) } } @@ -188,8 +172,10 @@ func TestGetIPInfoIgnoresShortRoutes(t *testing.T) { announce("192.0.0.0/7", 64500) for _, ip := range []string{"2001:db8::1", "192.0.2.1"} { - if info, err := db.GetIPInfo(ip); !errors.Is(err, ErrNoRoute) { - t.Errorf("GetIPInfo(%s) with only short routes = %+v, %v; want ErrNoRoute", ip, info, err) + info, err := db.GetIPInfo(ip) + if !errors.Is(err, ErrNoRoute) { + t.Errorf("GetIPInfo(%s) with only short routes = %+v, %v; "+ + "want ErrNoRoute", ip, info, err) } } @@ -205,11 +191,41 @@ func TestGetIPInfoIgnoresShortRoutes(t *testing.T) { lookup("2001:db8::1", "2001:db8::/32", 64502) } +// writeOriginRoutes writes one live route from testPeerIP for each prefix in +// origins, originated by the AS it maps to. +func writeOriginRoutes(t *testing.T, db *Database, origins map[string]int) { + t.Helper() + + ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + routes := make([]*LiveRoute, 0, len(origins)) + + for prefix, asn := range origins { + routes = append(routes, &LiveRoute{ + ID: uuid.New(), + Prefix: prefix, + MaskLength: netip.MustParsePrefix(prefix).Bits(), + IPVersion: detectIPVersion(prefix), + OriginASN: asn, + PeerIP: testPeerIP, + ASPath: []int{asn}, + NextHop: testPeerIP, + LastUpdated: ts, + }) + } + + err := db.UpsertLiveRouteBatch(routes) + if err != nil { + t.Fatalf("UpsertLiveRouteBatch: %v", err) + } +} + // TestConnectionPoolPragmas holds several pooled connections open at once and // checks each one carries the per-connection settings from the DSN, plus the // process-wide hard heap limit, and sees the new file with auto_vacuum // incremental. func TestConnectionPoolPragmas(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -229,57 +245,64 @@ func TestConnectionPoolPragmas(t *testing.T) { } }() - for i := 0; i < heldConnections; i++ { + for i := range heldConnections { c, err := db.db.Conn(ctx) if err != nil { t.Fatalf("failed to open connection %d: %v", i, err) } + conns = append(conns, c) } for i, c := range conns { - var cacheSize int - if err := c.QueryRowContext(ctx, "PRAGMA cache_size").Scan(&cacheSize); err != nil { - t.Fatalf("conn %d: failed to read cache_size: %v", i, err) - } + cacheSize := readPragma(ctx, t, c, i, "cache_size") if cacheSize != sqliteCacheSizeKiB { - t.Errorf("conn %d: cache_size = %d, want %d", i, cacheSize, sqliteCacheSizeKiB) + t.Errorf("conn %d: cache_size = %d, want %d", + i, cacheSize, sqliteCacheSizeKiB) } - var busyTimeout int - if err := c.QueryRowContext(ctx, "PRAGMA busy_timeout").Scan(&busyTimeout); err != nil { - t.Fatalf("conn %d: failed to read busy_timeout: %v", i, err) - } + busyTimeout := readPragma(ctx, t, c, i, "busy_timeout") if busyTimeout != sqliteBusyTimeoutMs { - t.Errorf("conn %d: busy_timeout = %d, want %d", i, busyTimeout, sqliteBusyTimeoutMs) + t.Errorf("conn %d: busy_timeout = %d, want %d", + i, busyTimeout, sqliteBusyTimeoutMs) } - var tempStore int - if err := c.QueryRowContext(ctx, "PRAGMA temp_store").Scan(&tempStore); err != nil { - t.Fatalf("conn %d: failed to read temp_store: %v", i, err) - } + tempStore := readPragma(ctx, t, c, i, "temp_store") if tempStore == tempStoreMemory { - t.Errorf("conn %d: temp_store = %d, want anything but %d (MEMORY)", i, tempStore, tempStoreMemory) + t.Errorf("conn %d: temp_store = %d, want anything but %d (MEMORY)", + i, tempStore, tempStoreMemory) } - var hardHeapLimit int64 - if err := c.QueryRowContext(ctx, "PRAGMA hard_heap_limit").Scan(&hardHeapLimit); err != nil { - t.Fatalf("conn %d: failed to read hard_heap_limit: %v", i, err) - } + hardHeapLimit := readPragma(ctx, t, c, i, "hard_heap_limit") if hardHeapLimit != sqliteHardHeapLimitBytes { - t.Errorf("conn %d: hard_heap_limit = %d, want %d", i, hardHeapLimit, sqliteHardHeapLimitBytes) + t.Errorf("conn %d: hard_heap_limit = %d, want %d", + i, hardHeapLimit, sqliteHardHeapLimitBytes) } - var autoVacuum int - if err := c.QueryRowContext(ctx, "PRAGMA auto_vacuum").Scan(&autoVacuum); err != nil { - t.Fatalf("conn %d: failed to read auto_vacuum: %v", i, err) - } + autoVacuum := readPragma(ctx, t, c, i, "auto_vacuum") if autoVacuum != autoVacuumIncremental { - t.Errorf("conn %d: auto_vacuum = %d, want %d (incremental)", i, autoVacuum, autoVacuumIncremental) + t.Errorf("conn %d: auto_vacuum = %d, want %d (incremental)", + i, autoVacuum, autoVacuumIncremental) } } } +// readPragma reads the integer value of PRAGMA name on connection i, c. +func readPragma( + ctx context.Context, t *testing.T, c *sql.Conn, i int, name string, +) int64 { + t.Helper() + + var value int64 + + err := c.QueryRowContext(ctx, "PRAGMA "+name).Scan(&value) + if err != nil { + t.Fatalf("conn %d: failed to read %s: %v", i, name, err) + } + + return value +} + // TestVacuumReturnsFreePages checks that after routes are deleted from a new // database, one Vacuum call returns every page they used. The deletes leave // fewer free pages than the 1000 Vacuum frees per call, so none may remain. @@ -287,6 +310,8 @@ func TestConnectionPoolPragmas(t *testing.T) { // the free pages stayed in the file, and with the PRAGMA run by ExecContext // Vacuum freed only one page per call. func TestVacuumReturnsFreePages(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -297,10 +322,12 @@ func TestVacuumReturnsFreePages(t *testing.T) { ctx := context.Background() ts := time.Now().UTC() + const asn = 64500 routes := make([]*LiveRoute, 0, vacuumTestRoutes) deletions := make([]LiveRouteDeletion, 0, vacuumTestRoutes) + for i := range vacuumTestRoutes { route := mkV4Route(t, fmt.Sprintf("10.%d.%d.0/24", i/256, i%256), asn, ts) routes = append(routes, route) @@ -312,29 +339,39 @@ func TestVacuumReturnsFreePages(t *testing.T) { }) } - if err := db.UpsertLiveRouteBatch(routes); err != nil { + err = db.UpsertLiveRouteBatch(routes) + if err != nil { t.Fatalf("failed to write routes: %v", err) } - if err := db.DeleteLiveRouteBatch(deletions); err != nil { + + err = db.DeleteLiveRouteBatch(deletions) + if err != nil { t.Fatalf("failed to delete routes: %v", err) } var before int - if err := db.db.QueryRowContext(ctx, "PRAGMA freelist_count").Scan(&before); err != nil { + + err = db.db.QueryRowContext(ctx, "PRAGMA freelist_count").Scan(&before) + if err != nil { t.Fatalf("failed to read freelist_count: %v", err) } + if before == 0 { t.Fatalf("no free pages after deleting %d routes", vacuumTestRoutes) } - if err := db.Vacuum(ctx); err != nil { + err = db.Vacuum(ctx) + if err != nil { t.Fatalf("Vacuum failed: %v", err) } var after int - if err := db.db.QueryRowContext(ctx, "PRAGMA freelist_count").Scan(&after); err != nil { + + err = db.db.QueryRowContext(ctx, "PRAGMA freelist_count").Scan(&after) + if err != nil { t.Fatalf("failed to read freelist_count: %v", err) } + if after != 0 { t.Errorf("free pages after Vacuum = %d of %d, want 0", after, before) } @@ -349,6 +386,8 @@ func TestVacuumReturnsFreePages(t *testing.T) { // no batch is dropped. The checkpoint runs without the Database mutex, exactly // as the background maintainer does in production. func TestBatchWriteDuringCheckpoint(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -361,8 +400,10 @@ func TestBatchWriteDuringCheckpoint(t *testing.T) { var wg sync.WaitGroup wg.Add(1) + go func() { defer wg.Done() + for { select { case <-ctx.Done(): @@ -374,12 +415,14 @@ func TestBatchWriteDuringCheckpoint(t *testing.T) { }() ts := time.Now().UTC() - for i := 0; i < contentionIterations; i++ { + for i := range contentionIterations { asns := map[int]time.Time{ i % contendedASNCount: ts, (i % contendedASNCount) + asnSecondBand: ts, } - if err := db.GetOrCreateASNBatch(asns); err != nil { + + err := db.GetOrCreateASNBatch(asns) + if err != nil { cancel() wg.Wait() t.Fatalf("batch write failed under checkpoint contention: %v", err) diff --git a/internal/database/interface.go b/internal/database/interface.go index 504c252..285b47e 100644 --- a/internal/database/interface.go +++ b/internal/database/interface.go @@ -29,6 +29,8 @@ type Stats struct { // Store defines the interface for database operations. It provides methods for // managing ASNs, prefixes, announcements, peerings, BGP peers, and live routes. // Implementations must be safe for concurrent use. +// +//nolint:interfacebloat // the one seam handlers and tests depend on type Store interface { // ASN operations GetOrCreateASN(number int, timestamp time.Time) (*ASN, error) @@ -57,8 +59,10 @@ type Store interface { UpsertLiveRouteBatch(routes []*LiveRoute) error DeleteLiveRoute(prefix string, originASN int, peerIP string) error DeleteLiveRouteBatch(deletions []LiveRouteDeletion) error - GetPrefixDistribution() (ipv4 []PrefixDistribution, ipv6 []PrefixDistribution, err error) - GetPrefixDistributionContext(ctx context.Context) (ipv4 []PrefixDistribution, ipv6 []PrefixDistribution, err error) + GetPrefixDistribution() ( + ipv4 []PrefixDistribution, ipv6 []PrefixDistribution, err error) + GetPrefixDistributionContext(ctx context.Context) ( + ipv4 []PrefixDistribution, ipv6 []PrefixDistribution, err error) GetLiveRouteCounts() (ipv4Count, ipv6Count int, err error) GetLiveRouteCountsContext(ctx context.Context) (ipv4Count, ipv6Count int, err error) @@ -77,9 +81,13 @@ type Store interface { GetASPeers(asn int) ([]ASPeer, error) GetASPeersContext(ctx context.Context, asn int) ([]ASPeer, error) GetPrefixDetails(prefix string) ([]LiveRoute, map[int]ASN, error) - GetPrefixDetailsContext(ctx context.Context, prefix string) ([]LiveRoute, map[int]ASN, error) + GetPrefixDetailsContext( + ctx context.Context, prefix string, + ) ([]LiveRoute, map[int]ASN, error) GetRandomPrefixesByLength(maskLength, ipVersion, limit int) ([]LiveRoute, error) - GetRandomPrefixesByLengthContext(ctx context.Context, maskLength, ipVersion, limit int) ([]LiveRoute, error) + GetRandomPrefixesByLengthContext( + ctx context.Context, maskLength, ipVersion, limit int, + ) ([]LiveRoute, error) // Lifecycle Close() error diff --git a/internal/database/models.go b/internal/database/models.go index a8e98b0..ed2274c 100644 --- a/internal/database/models.go +++ b/internal/database/models.go @@ -1,4 +1,3 @@ -// Package database provides SQLite storage for BGP routing data. package database import ( @@ -9,6 +8,8 @@ import ( // ASN represents an Autonomous System Number with its metadata including // handle, description, WHOIS data, and first/last seen timestamps. +// +//nolint:tagliatelle // field names are the existing JSON API type ASN struct { ASN int `json:"asn"` Handle string `json:"handle"` @@ -36,37 +37,39 @@ type ASN struct { // Prefix represents an IP prefix (CIDR block) with its IP version (4 or 6) // and first/last seen timestamps. type Prefix struct { - ID uuid.UUID `json:"id"` - Prefix string `json:"prefix"` - IPVersion int `json:"ip_version"` // 4 or 6 - FirstSeen time.Time `json:"first_seen"` - LastSeen time.Time `json:"last_seen"` + ID uuid.UUID + Prefix string + IPVersion int // 4 or 6 + FirstSeen time.Time + LastSeen time.Time } // Announcement represents a BGP announcement or withdrawal event, // containing the prefix, AS path, origin ASN, peer ASN, next hop, and timestamp. type Announcement struct { - ID uuid.UUID `json:"id"` - PrefixID uuid.UUID `json:"prefix_id"` - PeerASN int `json:"peer_asn"` - OriginASN int `json:"origin_asn"` - Path string `json:"path"` // JSON-encoded AS path - NextHop string `json:"next_hop"` - Timestamp time.Time `json:"timestamp"` - IsWithdrawal bool `json:"is_withdrawal"` + ID uuid.UUID + PrefixID uuid.UUID + PeerASN int + OriginASN int + Path string // JSON-encoded AS path + NextHop string + Timestamp time.Time + IsWithdrawal bool } // ASNPeering represents a peering relationship between two ASNs, // stored with the lower ASN as ASA and the higher as ASB. type ASNPeering struct { - ID uuid.UUID `json:"id"` - ASA int `json:"as_a"` - ASB int `json:"as_b"` - FirstSeen time.Time `json:"first_seen"` - LastSeen time.Time `json:"last_seen"` + ID uuid.UUID + ASA int + ASB int + FirstSeen time.Time + LastSeen time.Time } // LiveRoute represents a route in the live routing table +// +//nolint:tagliatelle // field names are the existing JSON API type LiveRoute struct { ID uuid.UUID `json:"id"` Prefix string `json:"prefix"` @@ -80,12 +83,16 @@ type LiveRoute struct { } // PrefixDistribution represents the distribution of prefixes by mask length +// +//nolint:tagliatelle // field names are the existing JSON API type PrefixDistribution struct { MaskLength int `json:"mask_length"` Count int `json:"count"` } // IPInfo represents comprehensive IP information for the /ip endpoint +// +//nolint:tagliatelle // field names are the existing JSON API type IPInfo struct { IP string `json:"ip"` PTR []string `json:"ptr,omitempty"` diff --git a/internal/database/slowquery.go b/internal/database/slowquery.go index 1ac302a..c082a5e 100644 --- a/internal/database/slowquery.go +++ b/internal/database/slowquery.go @@ -19,29 +19,21 @@ func logSlowQuery(logger *logger.Logger, query string, start time.Time) { } // queryRow wraps QueryRow with slow query logging -// nolint:unused // kept for consistency with other query wrappers -func (d *Database) queryRow(query string, args ...interface{}) *sql.Row { +// +//nolint:unused // kept for consistency with other query wrappers +func (d *Database) queryRow(query string, args ...any) *sql.Row { start := time.Now() defer logSlowQuery(d.logger, query, start) - return d.db.QueryRow(query, args...) -} - -// query wraps Query with slow query logging -// nolint:unused // kept for future use to ensure all queries go through slow query logging -func (d *Database) query(query string, args ...interface{}) (*sql.Rows, error) { - start := time.Now() - defer logSlowQuery(d.logger, query, start) - - return d.db.Query(query, args...) + return d.db.QueryRowContext(context.Background(), query, args...) } // exec wraps Exec with slow query logging -func (d *Database) exec(query string, args ...interface{}) error { +func (d *Database) exec(query string, args ...any) error { start := time.Now() defer logSlowQuery(d.logger, query, start) - _, err := d.db.Exec(query, args...) + _, err := d.db.ExecContext(context.Background(), query, args...) return err } @@ -49,35 +41,22 @@ func (d *Database) exec(query string, args ...interface{}) error { // loggingTx wraps sql.Tx to log slow queries type loggingTx struct { *sql.Tx + logger *logger.Logger } // QueryRow wraps sql.Tx.QueryRow to log slow queries -func (tx *loggingTx) QueryRow(query string, args ...interface{}) *sql.Row { +func (tx *loggingTx) QueryRow(query string, args ...any) *sql.Row { start := time.Now() defer logSlowQuery(tx.logger, query, start) - return tx.Tx.QueryRow(query, args...) -} - -// Query wraps sql.Tx.Query to log slow queries -func (tx *loggingTx) Query(query string, args ...interface{}) (*sql.Rows, error) { - start := time.Now() - defer logSlowQuery(tx.logger, query, start) - - return tx.Tx.Query(query, args...) -} - -// QueryContext wraps sql.Tx.QueryContext to log slow queries -func (tx *loggingTx) QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error) { - start := time.Now() - defer logSlowQuery(tx.logger, query, start) - - return tx.Tx.QueryContext(ctx, query, args...) + return tx.Tx.QueryRowContext(context.Background(), query, args...) } // QueryRowContext wraps sql.Tx.QueryRowContext to log slow queries -func (tx *loggingTx) QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row { +func (tx *loggingTx) QueryRowContext( + ctx context.Context, query string, args ...any, +) *sql.Row { start := time.Now() defer logSlowQuery(tx.logger, query, start) @@ -85,15 +64,17 @@ func (tx *loggingTx) QueryRowContext(ctx context.Context, query string, args ... } // Exec wraps sql.Tx.Exec to log slow queries -func (tx *loggingTx) Exec(query string, args ...interface{}) (sql.Result, error) { +func (tx *loggingTx) Exec(query string, args ...any) (sql.Result, error) { start := time.Now() defer logSlowQuery(tx.logger, query, start) - return tx.Tx.Exec(query, args...) + return tx.Tx.ExecContext(context.Background(), query, args...) } // ExecContext wraps sql.Tx.ExecContext to log slow queries -func (tx *loggingTx) ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error) { +func (tx *loggingTx) ExecContext( + ctx context.Context, query string, args ...any, +) (sql.Result, error) { start := time.Now() defer logSlowQuery(tx.logger, query, start) diff --git a/internal/database/utils.go b/internal/database/utils.go index 8094c78..4ba0b0d 100644 --- a/internal/database/utils.go +++ b/internal/database/utils.go @@ -27,6 +27,7 @@ func prefixMaskLength(prefix string) (int, error) { if err != nil { return 0, fmt.Errorf("invalid prefix %s: %w", prefix, err) } + maskLength, _ := network.Mask.Size() return maskLength, nil diff --git a/internal/logger/logger.go b/internal/logger/logger.go index 3e90ad8..d46d523 100644 --- a/internal/logger/logger.go +++ b/internal/logger/logger.go @@ -74,6 +74,7 @@ func getSourceAttrs() []slog.Attr { // Get the function name fn := runtime.FuncForPC(pc) + var funcName string if fn != nil { funcName = filepath.Base(fn.Name()) diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 25aef45..ad04309 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -64,6 +64,7 @@ func New() *Tracker { // first counts one reconnect. func (t *Tracker) SetConnected(connected bool) { t.isConnected.Store(connected) + if !connected { return } @@ -120,9 +121,11 @@ func (t *Tracker) GetStreamMetrics() StreamMetrics { if msgCount >= 0 { totalMessages = uint64(msgCount) } + if byteCount >= 0 { totalBytes = uint64(byteCount) } + if wireByteCount >= 0 { totalWireBytes = uint64(wireByteCount) } @@ -166,7 +169,7 @@ func (t *Tracker) RecordWithdrawal() { func (t *Tracker) SetBGPPeerCount(count int) { // BGP peer count is always small (< 1000), so int32 is safe if count > 0 && count < 1<<31 { - t.bgpPeerCount.Store(int32(count)) //nolint:gosec // count is validated + t.bgpPeerCount.Store(int32(count)) } } @@ -214,7 +217,8 @@ type StreamMetrics struct { TotalMessages uint64 // TotalBytes is the total number of decompressed bytes received since startup TotalBytes uint64 - // TotalWireBytes is the total number of bytes received on the wire (before decompression) + // TotalWireBytes is the total number of bytes received on the wire (before + // decompression) TotalWireBytes uint64 // ConnectedSince is the time when the current connection was established ConnectedSince time.Time @@ -222,9 +226,11 @@ type StreamMetrics struct { Connected bool // MessagesPerSec is the rate of messages received per second (1-minute average) MessagesPerSec float64 - // BitsPerSec is the rate of decompressed bits received per second (1-minute average) + // BitsPerSec is the rate of decompressed bits received per second (1-minute + // average) BitsPerSec float64 - // WireBitsPerSec is the rate of bits received on the wire per second (1-minute average) + // WireBitsPerSec is the rate of bits received on the wire per second + // (1-minute average) WireBitsPerSec float64 // ReconnectCount is the number of reconnections since startup ReconnectCount uint64 diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go index e8776fd..c301122 100644 --- a/internal/metrics/metrics_test.go +++ b/internal/metrics/metrics_test.go @@ -1,12 +1,18 @@ -package metrics +package metrics_test -import "testing" +import ( + "testing" + + "git.eeqj.de/sneak/routewatch/internal/metrics" +) // TestReconnectCount connects three times with a disconnect between each, the // way the streamer does, and expects every connection after the first to count // one reconnect. func TestReconnectCount(t *testing.T) { - tracker := New() + t.Parallel() + + tracker := metrics.New() tracker.SetConnected(true) tracker.SetConnected(false) diff --git a/internal/ristypes/ris.go b/internal/ristypes/ris.go index a6831a4..1ad2072 100644 --- a/internal/ristypes/ris.go +++ b/internal/ristypes/ris.go @@ -1,4 +1,5 @@ -// Package ristypes defines the data structures for RIS Live BGP messages and announcements. +// Package ristypes defines the data structures for RIS Live BGP messages and +// announcements. package ristypes import ( @@ -17,25 +18,30 @@ type ASPath []int func (p *ASPath) UnmarshalJSON(data []byte) error { // First try to unmarshal as a simple array of integers var simple []int - if err := json.Unmarshal(data, &simple); err == nil { + + err := json.Unmarshal(data, &simple) + if err == nil { *p = ASPath(simple) return nil } // If that fails, unmarshal as array of interfaces and flatten - var raw []interface{} - if err := json.Unmarshal(data, &raw); err != nil { + var raw []any + + err = json.Unmarshal(data, &raw) + if err != nil { return err } // Flatten the array result := make([]int, 0) + for _, item := range raw { switch v := item.(type) { case float64: result = append(result, int(v)) - case []interface{}: + case []any: // Nested array - flatten it for _, nested := range v { if num, ok := nested.(float64); ok { @@ -62,6 +68,8 @@ type RISLiveMessage struct { // It contains metadata about the BGP session (peer, ASN, host) along with // the actual BGP update data including AS path, communities, announcements, // and withdrawals. +// +//nolint:tagliatelle // field names are the RIS Live feed's type RISMessage struct { Type string `json:"type"` Timestamp float64 `json:"timestamp"` @@ -90,6 +98,8 @@ type RISMessage struct { // RISAnnouncement represents a BGP route announcement within a RIS message. // It contains the next hop IP address and the list of prefixes being announced // via that next hop. +// +//nolint:tagliatelle // field names are the RIS Live feed's type RISAnnouncement struct { NextHop string `json:"next_hop"` Prefixes []string `json:"prefixes"` diff --git a/internal/ristypes/ris_test.go b/internal/ristypes/ris_test.go index 6a3b7b0..2b3f7d2 100644 --- a/internal/ristypes/ris_test.go +++ b/internal/ristypes/ris_test.go @@ -1,10 +1,12 @@ -package ristypes +package ristypes_test import ( "bufio" "encoding/json" "os" "testing" + + "git.eeqj.de/sneak/routewatch/internal/ristypes" ) // messageExamplesPath is the captured RIS Live feed used as decode fixtures, @@ -15,32 +17,43 @@ const messageExamplesPath = "../../docs/message-examples.json" // streamer does and checks that the fields no handler reads (Community, Raw) // stay empty while the fields handlers use (Path, Announcements) still decode. func TestDecodeDropsCommunityAndRaw(t *testing.T) { + t.Parallel() + f, err := os.Open(messageExamplesPath) if err != nil { t.Fatalf("open fixtures: %v", err) } - defer f.Close() + + defer func() { + _ = f.Close() + }() var messages, withPath, withAnnouncements int scanner := bufio.NewScanner(f) scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024) + for scanner.Scan() { line := scanner.Bytes() if len(line) == 0 { continue } - var wrapper RISLiveMessage - if err := json.Unmarshal(line, &wrapper); err != nil { + var wrapper ristypes.RISLiveMessage + + err := json.Unmarshal(line, &wrapper) + if err != nil { t.Fatalf("unmarshal message %d: %v", messages+1, err) } + messages++ msg := wrapper.Data if msg.Community != nil { - t.Errorf("message %d: Community decoded, want empty: %v", messages, msg.Community) + t.Errorf("message %d: Community decoded, want empty: %v", + messages, msg.Community) } + if msg.Raw != "" { t.Errorf("message %d: Raw decoded, want empty", messages) } @@ -48,11 +61,14 @@ func TestDecodeDropsCommunityAndRaw(t *testing.T) { if len(msg.Path) > 0 { withPath++ } + if len(msg.Announcements) > 0 { withAnnouncements++ } } - if err := scanner.Err(); err != nil { + + err = scanner.Err() + if err != nil { t.Fatalf("scan fixtures: %v", err) } @@ -62,6 +78,7 @@ func TestDecodeDropsCommunityAndRaw(t *testing.T) { // The fixtures include announcement messages; a used field must still decode, // otherwise an empty Community/Raw would prove nothing. if withPath == 0 || withAnnouncements == 0 { - t.Fatalf("used fields did not decode: withPath=%d withAnnouncements=%d", withPath, withAnnouncements) + t.Fatalf("used fields did not decode: withPath=%d withAnnouncements=%d", + withPath, withAnnouncements) } } diff --git a/internal/routewatch/app.go b/internal/routewatch/app.go index 0c29d75..c38a8cd 100644 --- a/internal/routewatch/app.go +++ b/internal/routewatch/app.go @@ -1,10 +1,11 @@ -// Package routewatch contains the primary RouteWatch type that represents a running instance -// of the application and contains pointers to its core dependencies, and is responsible for initialization. +// Package routewatch contains the primary RouteWatch type that represents a +// running instance of the application and contains pointers to its core +// dependencies, and is responsible for initialization. package routewatch import ( "context" - "fmt" + "errors" "sync" "time" @@ -18,6 +19,10 @@ import ( "go.uber.org/fx" ) +// errNonBatchedNotImplemented is returned by Run when batched database writes +// are disabled, since only the batched handlers exist. +var errNonBatchedNotImplemented = errors.New("non-batched handlers not implemented") + // Dependencies contains all dependencies for RouteWatch type Dependencies struct { fx.In @@ -68,8 +73,10 @@ func (rw *RouteWatch) Run(ctx context.Context) error { // Apply runtime limit if specified if rw.maxRuntime > 0 { var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, rw.maxRuntime) defer cancel() + rw.logger.Info("Running with time limit", "max_runtime", rw.maxRuntime) } @@ -96,29 +103,31 @@ func (rw *RouteWatch) Run(ctx context.Context) error { // Non-batched handlers not implemented yet rw.logger.Error("Non-batched handlers not implemented") - return fmt.Errorf("non-batched handlers not implemented") + return errNonBatchedNotImplemented } // No longer need routing table handler - PrefixHandler maintains live_routes table // Start streaming - if err := rw.streamer.Start(); err != nil { + err := rw.streamer.Start(ctx) + if err != nil { return err } // Start HTTP server - if err := rw.server.Start(); err != nil { + err = rw.server.Start(ctx) + if err != nil { return err } // Start ASN WHOIS fetcher for background updates rw.asnFetcher = NewASNFetcher(rw.db, rw.logger.Logger) - rw.asnFetcher.Start() + rw.asnFetcher.Start(ctx) rw.server.SetASNFetcher(rw.asnFetcher) // Start database maintenance goroutine rw.dbMaintainer = NewDBMaintainer(rw.db, rw.logger.Logger) - rw.dbMaintainer.Start() + rw.dbMaintainer.Start(ctx) // Wait for context cancellation <-ctx.Done() @@ -127,13 +136,14 @@ func (rw *RouteWatch) Run(ctx context.Context) error { } // Shutdown performs graceful shutdown of all services -func (rw *RouteWatch) Shutdown() { +func (rw *RouteWatch) Shutdown(ctx context.Context) { rw.mu.Lock() if rw.shutdown { rw.mu.Unlock() return } + rw.shutdown = true rw.mu.Unlock() @@ -142,14 +152,17 @@ func (rw *RouteWatch) Shutdown() { rw.logger.Info("Flushing database handler") rw.dbHandler.Stop() } + if rw.peerHandler != nil { rw.logger.Info("Flushing peer handler") rw.peerHandler.Stop() } + if rw.prefixHandler != nil { rw.logger.Info("Flushing prefix handler") rw.prefixHandler.Stop() } + if rw.peeringHandler != nil { rw.logger.Info("Flushing peering handler") rw.peeringHandler.Stop() @@ -168,11 +181,15 @@ func (rw *RouteWatch) Shutdown() { // Stop services rw.streamer.Stop() - // Stop HTTP server with a timeout + // Stop HTTP server with a timeout of its own, however much of ctx is left const serverStopTimeout = 5 * time.Second - stopCtx, cancel := context.WithTimeout(context.Background(), serverStopTimeout) + + stopCtx, cancel := context.WithTimeout( + context.WithoutCancel(ctx), serverStopTimeout) defer cancel() - if err := rw.server.Stop(stopCtx); err != nil { + + err := rw.server.Stop(stopCtx) + if err != nil { rw.logger.Error("Failed to stop HTTP server gracefully", "error", err) } @@ -185,12 +202,11 @@ func (rw *RouteWatch) Shutdown() { "bits_per_sec", metrics.BitsPerSec, "duration", time.Since(metrics.ConnectedSince), ) - } // getModule provides all fx dependencies -func getModule() fx.Option { - return fx.Options( +func getModule() []fx.Option { + return []fx.Option{ fx.Provide( logger.New, config.New, @@ -203,5 +219,5 @@ func getModule() fx.Option { server.New, New, ), - ) + } } diff --git a/internal/routewatch/app_integration_test.go b/internal/routewatch/app_integration_internal_test.go similarity index 78% rename from internal/routewatch/app_integration_test.go rename to internal/routewatch/app_integration_internal_test.go index a359d03..36b2dea 100644 --- a/internal/routewatch/app_integration_test.go +++ b/internal/routewatch/app_integration_internal_test.go @@ -50,7 +50,9 @@ func newMockStore() *mockStore { } // GetOrCreateASN mock implementation -func (m *mockStore) GetOrCreateASN(number int, timestamp time.Time) (*database.ASN, error) { +func (m *mockStore) GetOrCreateASN( + number int, timestamp time.Time, +) (*database.ASN, error) { m.mu.Lock() defer m.mu.Unlock() @@ -99,11 +101,14 @@ func (m *mockStore) UpdatePrefixesBatch(prefixes map[string]time.Time) error { } } } + return nil } // GetOrCreatePrefix mock implementation -func (m *mockStore) GetOrCreatePrefix(prefix string, timestamp time.Time) (*database.Prefix, error) { +func (m *mockStore) GetOrCreatePrefix( + prefix string, timestamp time.Time, +) (*database.Prefix, error) { m.mu.Lock() defer m.mu.Unlock() @@ -168,7 +173,7 @@ func (m *mockStore) RecordPeering(asA, asB int, _ time.Time) error { } // UpdatePeer mock implementation -func (m *mockStore) UpdatePeer(peerIP string, peerASN int, messageType string, timestamp time.Time) error { +func (m *mockStore) UpdatePeer(_ string, _ int, _ string, _ time.Time) error { // Simple mock - just return nil return nil } @@ -194,41 +199,45 @@ func (m *mockStore) GetStats() (database.Stats, error) { } // GetStatsContext returns statistics about the mock store with context support -func (m *mockStore) GetStatsContext(ctx context.Context) (database.Stats, error) { +func (m *mockStore) GetStatsContext(_ context.Context) (database.Stats, error) { return m.GetStats() } // UpsertLiveRoute mock implementation -func (m *mockStore) UpsertLiveRoute(route *database.LiveRoute) error { +func (m *mockStore) UpsertLiveRoute(_ *database.LiveRoute) error { // Simple mock - just return nil return nil } // DeleteLiveRoute mock implementation -func (m *mockStore) DeleteLiveRoute(prefix string, originASN int, peerIP string) error { +func (m *mockStore) DeleteLiveRoute(_ string, _ int, _ string) error { // Simple mock - just return nil return nil } // GetPrefixDistribution mock implementation -func (m *mockStore) GetPrefixDistribution() (ipv4 []database.PrefixDistribution, ipv6 []database.PrefixDistribution, err error) { +func (m *mockStore) GetPrefixDistribution() ( + []database.PrefixDistribution, []database.PrefixDistribution, error, +) { // Return empty distributions for now return nil, nil, nil } // GetPrefixDistributionContext mock implementation with context support -func (m *mockStore) GetPrefixDistributionContext(ctx context.Context) (ipv4 []database.PrefixDistribution, ipv6 []database.PrefixDistribution, err error) { +func (m *mockStore) GetPrefixDistributionContext(_ context.Context) ( + []database.PrefixDistribution, []database.PrefixDistribution, error, +) { return m.GetPrefixDistribution() } // GetLiveRouteCounts mock implementation -func (m *mockStore) GetLiveRouteCounts() (ipv4Count, ipv6Count int, err error) { +func (m *mockStore) GetLiveRouteCounts() (int, int, error) { // Return mock counts return m.RouteCount / 2, m.RouteCount / 2, nil } // GetLiveRouteCountsContext mock implementation with context support -func (m *mockStore) GetLiveRouteCountsContext(ctx context.Context) (ipv4Count, ipv6Count int, err error) { +func (m *mockStore) GetLiveRouteCountsContext(_ context.Context) (int, int, error) { return m.GetLiveRouteCounts() } @@ -247,41 +256,51 @@ func (m *mockStore) GetASDetails(asn int) (*database.ASN, []database.LiveRoute, } // GetASDetailsContext mock implementation with context support -func (m *mockStore) GetASDetailsContext(ctx context.Context, asn int) (*database.ASN, []database.LiveRoute, error) { +func (m *mockStore) GetASDetailsContext( + _ context.Context, asn int, +) (*database.ASN, []database.LiveRoute, error) { return m.GetASDetails(asn) } // GetPrefixDetails mock implementation -func (m *mockStore) GetPrefixDetails(prefix string) ([]database.LiveRoute, map[int]database.ASN, error) { +func (m *mockStore) GetPrefixDetails( + _ string, +) ([]database.LiveRoute, map[int]database.ASN, error) { // Return empty routes for now return []database.LiveRoute{}, map[int]database.ASN{}, nil } // GetPrefixDetailsContext mock implementation with context support func (m *mockStore) GetPrefixDetailsContext( - ctx context.Context, prefix string, + _ context.Context, prefix string, ) ([]database.LiveRoute, map[int]database.ASN, error) { return m.GetPrefixDetails(prefix) } -func (m *mockStore) GetRandomPrefixesByLength(maskLength, ipVersion, limit int) ([]database.LiveRoute, error) { +func (m *mockStore) GetRandomPrefixesByLength( + _, _, _ int, +) ([]database.LiveRoute, error) { // Return empty routes for now return []database.LiveRoute{}, nil } // GetRandomPrefixesByLengthContext mock implementation with context support -func (m *mockStore) GetRandomPrefixesByLengthContext(ctx context.Context, maskLength, ipVersion, limit int) ([]database.LiveRoute, error) { +func (m *mockStore) GetRandomPrefixesByLengthContext( + _ context.Context, maskLength, ipVersion, limit int, +) ([]database.LiveRoute, error) { return m.GetRandomPrefixesByLength(maskLength, ipVersion, limit) } // GetASPeers mock implementation -func (m *mockStore) GetASPeers(asn int) ([]database.ASPeer, error) { +func (m *mockStore) GetASPeers(_ int) ([]database.ASPeer, error) { // Return empty peers for now return []database.ASPeer{}, nil } // GetASPeersContext mock implementation with context support -func (m *mockStore) GetASPeersContext(ctx context.Context, asn int) ([]database.ASPeer, error) { +func (m *mockStore) GetASPeersContext( + _ context.Context, asn int, +) ([]database.ASPeer, error) { return m.GetASPeers(asn) } @@ -291,8 +310,11 @@ func (m *mockStore) GetIPInfo(ip string) (*database.IPInfo, error) { } // GetIPInfoContext mock implementation with context support -func (m *mockStore) GetIPInfoContext(ctx context.Context, ip string) (*database.IPInfo, error) { +func (m *mockStore) GetIPInfoContext( + _ context.Context, ip string, +) (*database.IPInfo, error) { now := time.Now() + return &database.IPInfo{ IP: ip, Netblock: "8.8.8.0/24", @@ -309,17 +331,21 @@ func (m *mockStore) GetIPInfoContext(ctx context.Context, ip string) (*database. } // GetNextStaleASN mock implementation -func (m *mockStore) GetNextStaleASN(ctx context.Context, staleThreshold time.Duration) (int, error) { +func (m *mockStore) GetNextStaleASN(_ context.Context, _ time.Duration) (int, error) { return 0, database.ErrNoStaleASN } // UpdateASNWHOIS mock implementation -func (m *mockStore) UpdateASNWHOIS(ctx context.Context, update *database.ASNWHOISUpdate) error { +func (m *mockStore) UpdateASNWHOIS( + _ context.Context, _ *database.ASNWHOISUpdate, +) error { return nil } // GetWHOISStats mock implementation -func (m *mockStore) GetWHOISStats(ctx context.Context, staleThreshold time.Duration) (*database.WHOISStats, error) { +func (m *mockStore) GetWHOISStats( + _ context.Context, _ time.Duration, +) (*database.WHOISStats, error) { m.mu.Lock() defer m.mu.Unlock() @@ -347,19 +373,22 @@ func (m *mockStore) UpsertLiveRouteBatch(routes []*database.LiveRoute) error { LastSeen: route.LastUpdated, } m.PrefixCount++ + if route.IPVersion == 4 { m.IPv4Prefixes++ } else { m.IPv6Prefixes++ } } + m.RouteCount++ } + return nil } // DeleteLiveRouteBatch mock implementation -func (m *mockStore) DeleteLiveRouteBatch(deletions []database.LiveRouteDeletion) error { +func (m *mockStore) DeleteLiveRouteBatch(_ []database.LiveRouteDeletion) error { // Simple mock - just return nil return nil } @@ -379,43 +408,53 @@ func (m *mockStore) GetOrCreateASNBatch(asns map[int]time.Time) error { m.ASNCount++ } } + return nil } // UpdatePeerBatch mock implementation -func (m *mockStore) UpdatePeerBatch(peers map[string]database.PeerUpdate) error { +func (m *mockStore) UpdatePeerBatch(_ map[string]database.PeerUpdate) error { // Simple mock - just return nil return nil } // Vacuum mock implementation -func (m *mockStore) Vacuum(ctx context.Context) error { +func (m *mockStore) Vacuum(_ context.Context) error { return nil } // Analyze mock implementation -func (m *mockStore) Analyze(ctx context.Context) error { +func (m *mockStore) Analyze(_ context.Context) error { return nil } // Checkpoint mock implementation -func (m *mockStore) Checkpoint(ctx context.Context) error { +func (m *mockStore) Checkpoint(_ context.Context) error { return nil } // Ping mock implementation -func (m *mockStore) Ping(ctx context.Context) error { +func (m *mockStore) Ping(_ context.Context) error { return nil } func TestRouteWatchLiveFeed(t *testing.T) { + t.Parallel() + if testing.Short() { - t.Skip("skipping live RIPE RIS network feed test in short mode; run without -short to include it") + t.Skip("skipping live RIPE RIS network feed test in short mode; " + + "run without -short to include it") } // Create mock database mockDB := newMockStore() - defer mockDB.Close() + + defer func() { + err := mockDB.Close() + if err != nil { + t.Errorf("failed to close mock database: %v", err) + } + }() logger := logger.New() @@ -468,16 +507,19 @@ func TestRouteWatchLiveFeed(t *testing.T) { if stats.ASNs == 0 { t.Error("Expected to receive some ASNs from live feed") } + t.Logf("Received %d unique ASNs in 5 seconds", stats.ASNs) if stats.Prefixes == 0 { t.Error("Expected to receive some prefixes from live feed") } - t.Logf("Received %d unique prefixes (%d IPv4, %d IPv6) in 5 seconds", stats.Prefixes, stats.IPv4Prefixes, stats.IPv6Prefixes) + + t.Logf("Received %d unique prefixes (%d IPv4, %d IPv6) in 5 seconds", + stats.Prefixes, stats.IPv4Prefixes, stats.IPv6Prefixes) if stats.Peerings == 0 { t.Error("Expected to receive some peerings from live feed") } - t.Logf("Recorded %d AS peering relationships in 5 seconds", stats.Peerings) + t.Logf("Recorded %d AS peering relationships in 5 seconds", stats.Peerings) } diff --git a/internal/routewatch/app_test.go b/internal/routewatch/app_test.go index 6adfec8..e5f11c8 100644 --- a/internal/routewatch/app_test.go +++ b/internal/routewatch/app_test.go @@ -1,3 +1,3 @@ -package routewatch +package routewatch_test // Tests for routewatch package are in app_integration_test.go diff --git a/internal/routewatch/ashandler.go b/internal/routewatch/ashandler.go index 3eae6f3..a8b93dc 100644 --- a/internal/routewatch/ashandler.go +++ b/internal/routewatch/ashandler.go @@ -71,7 +71,7 @@ func NewASHandler(db database.Store, logger *logger.Logger) *ASHandler { // the AS path information needed to track autonomous systems. func (h *ASHandler) WantsMessage(messageType string) bool { // We only care about UPDATE messages for the database - return messageType == "UPDATE" + return messageType == updateMessageType } // QueueCapacity returns the recommended message queue size for this handler. @@ -124,9 +124,20 @@ func (h *ASHandler) HandleMessage(msg *ristypes.RISMessage) { } } +// Stop gracefully shuts down the ASHandler by signaling the background flush +// goroutine to terminate and waiting for it to complete. Any pending ASN +// operations in the current batch are flushed to the database before Stop +// returns. This method should be called during application shutdown to ensure +// no data is lost. +func (h *ASHandler) Stop() { + close(h.stopCh) + h.wg.Wait() +} + // flushLoop runs in a goroutine and periodically flushes batches func (h *ASHandler) flushLoop() { defer h.wg.Done() + ticker := time.NewTicker(asnBatchTimeout) defer ticker.Stop() @@ -149,7 +160,8 @@ func (h *ASHandler) flushLoop() { } } -// flushBatchLocked flushes the ASN batch to the database (must be called with mutex held) +// flushBatchLocked flushes the ASN batch to the database (must be called with +// mutex held) func (h *ASHandler) flushBatchLocked() { if len(h.batch) == 0 { return @@ -164,21 +176,13 @@ func (h *ASHandler) flushBatchLocked() { } // Process all ASNs in a single batch transaction - if err := h.db.GetOrCreateASNBatch(asnMap); err != nil { - h.logger.Error("Failed to process ASN batch", "error", err, "count", len(asnMap)) + err := h.db.GetOrCreateASNBatch(asnMap) + if err != nil { + h.logger.Error("Failed to process ASN batch", + "error", err, "count", len(asnMap)) } // Clear batch h.batch = h.batch[:0] h.lastFlush = time.Now() } - -// Stop gracefully shuts down the ASHandler by signaling the background flush -// goroutine to terminate and waiting for it to complete. Any pending ASN -// operations in the current batch are flushed to the database before Stop -// returns. This method should be called during application shutdown to ensure -// no data is lost. -func (h *ASHandler) Stop() { - close(h.stopCh) - h.wg.Wait() -} diff --git a/internal/routewatch/asnfetcher.go b/internal/routewatch/asnfetcher.go index 7a0b246..7320276 100644 --- a/internal/routewatch/asnfetcher.go +++ b/internal/routewatch/asnfetcher.go @@ -1,8 +1,8 @@ -// Package routewatch contains the ASN WHOIS fetcher for background updates. package routewatch import ( "context" + "errors" "log/slog" "sync" "time" @@ -73,10 +73,13 @@ func NewASNFetcher(db database.Store, logger *slog.Logger) *ASNFetcher { } } -// Start begins the background ASN fetcher goroutine. -func (f *ASNFetcher) Start() { +// Start begins the background ASN fetcher goroutine. It runs until Stop: +// the end of ctx does not stop it. +func (f *ASNFetcher) Start(ctx context.Context) { f.wg.Add(1) - go f.run() + + go f.run(context.WithoutCancel(ctx)) + f.logger.Info("ASN fetcher started", "base_interval", baseInterval, "min_interval", minInterval, @@ -151,10 +154,7 @@ func (f *ASNFetcher) recordSuccess() { f.consecutiveFails = 0 // Decrease interval by half, but not below minimum - newInterval := f.currentInterval / backoffMultiplier - if newInterval < minInterval { - newInterval = minInterval - } + newInterval := max(f.currentInterval/backoffMultiplier, minInterval) if newInterval != f.currentInterval { f.logger.Debug("Decreased fetch interval", @@ -177,10 +177,7 @@ func (f *ASNFetcher) recordFailure() { f.consecutiveFails++ // Exponential backoff: multiply by 2, capped at max - newInterval := f.currentInterval * backoffMultiplier - if newInterval > maxInterval { - newInterval = maxInterval - } + newInterval := min(f.currentInterval*backoffMultiplier, maxInterval) if newInterval != f.currentInterval { f.logger.Debug("Increased fetch interval due to failure", @@ -199,7 +196,7 @@ func (f *ASNFetcher) recordFailure() { } // run is the main background loop. -func (f *ASNFetcher) run() { +func (f *ASNFetcher) run(ctx context.Context) { defer f.wg.Done() timer := time.NewTimer(f.getInterval()) @@ -212,13 +209,13 @@ func (f *ASNFetcher) run() { case asn := <-f.immediateQueue: // Process immediate request (respects lock) - f.tryFetch(asn) + f.tryFetch(ctx, asn) // Reset timer after immediate fetch timer.Reset(f.getInterval()) case <-timer.C: // Background fetch of stale/missing ASN - f.fetchNextStale() + f.fetchNextStale(ctx) // Reset timer with potentially updated interval timer.Reset(f.getInterval()) } @@ -227,7 +224,7 @@ func (f *ASNFetcher) run() { // tryFetch attempts to fetch and update an ASN, respecting the fetch lock. // Returns true if fetch was successful. -func (f *ASNFetcher) tryFetch(asn int) bool { +func (f *ASNFetcher) tryFetch(ctx context.Context, asn int) bool { // Try to acquire lock, skip if another fetch is running if !f.fetchMu.TryLock() { f.logger.Debug("Skipping fetch, another fetch in progress", "asn", asn) @@ -236,11 +233,11 @@ func (f *ASNFetcher) tryFetch(asn int) bool { } defer f.fetchMu.Unlock() - return f.fetchAndUpdate(asn) + return f.fetchAndUpdate(ctx, asn) } // fetchNextStale finds and fetches the next ASN needing WHOIS data. -func (f *ASNFetcher) fetchNextStale() { +func (f *ASNFetcher) fetchNextStale(ctx context.Context) { // Try to acquire lock, skip if another fetch is running if !f.fetchMu.TryLock() { f.logger.Debug("Skipping stale fetch, another fetch in progress") @@ -249,12 +246,12 @@ func (f *ASNFetcher) fetchNextStale() { } defer f.fetchMu.Unlock() - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + lookupCtx, cancel := context.WithTimeout(ctx, time.Minute) defer cancel() - asn, err := f.db.GetNextStaleASN(ctx, whoisStaleThreshold) + asn, err := f.db.GetNextStaleASN(lookupCtx, whoisStaleThreshold) if err != nil { - if err != database.ErrNoStaleASN { + if !errors.Is(err, database.ErrNoStaleASN) { f.logger.Error("Failed to get stale ASN", "error", err) f.recordFailure() } @@ -263,13 +260,13 @@ func (f *ASNFetcher) fetchNextStale() { return } - f.fetchAndUpdate(asn) + f.fetchAndUpdate(ctx, asn) } // fetchAndUpdate performs a WHOIS lookup and updates the database. // Returns true if successful. -func (f *ASNFetcher) fetchAndUpdate(asn int) bool { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) +func (f *ASNFetcher) fetchAndUpdate(ctx context.Context, asn int) bool { + ctx, cancel := context.WithTimeout(ctx, time.Minute) defer cancel() f.logger.Info("Fetching WHOIS data", "asn", asn) diff --git a/internal/routewatch/cli.go b/internal/routewatch/cli.go index 6c2bd6c..ea7d475 100644 --- a/internal/routewatch/cli.go +++ b/internal/routewatch/cli.go @@ -57,14 +57,19 @@ func logDebugStats(logger *logger.Logger) { // CLIEntry is the main entry point for the routewatch command-line interface. // It initializes the application using the fx dependency injection framework, -// sets up signal handling for graceful shutdown, and starts the RouteWatch service. -// This function blocks until the application receives a shutdown signal or encounters -// a fatal error. +// sets up signal handling for graceful shutdown, and starts the RouteWatch +// service. This function blocks until the application receives a shutdown +// signal or encounters a fatal error. func CLIEntry() { app := fx.New( - getModule(), + fx.Options(getModule()...), fx.StopTimeout(shutdownTimeout), // Allow 60 seconds for graceful shutdown - fx.Invoke(func(lc fx.Lifecycle, rw *RouteWatch, logger *logger.Logger, shutdowner fx.Shutdowner) { + fx.Invoke(func( + lc fx.Lifecycle, + rw *RouteWatch, + logger *logger.Logger, + shutdowner fx.Shutdowner, + ) { lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { // Start debug stats logging @@ -77,25 +82,31 @@ func CLIEntry() { go func() { <-sigCh logger.Info("Received shutdown signal") - if err := shutdowner.Shutdown(); err != nil { + + err := shutdowner.Shutdown() + if err != nil { logger.Error("Failed to shutdown gracefully", "error", err) } }() go func() { - if err := rw.Run(ctx); err != nil { + err := rw.Run(ctx) + if err != nil { logger.Error("RouteWatch error", "error", err) - if err := shutdowner.Shutdown(fx.ExitCode(failureExitCode)); err != nil { - logger.Error("Failed to shutdown gracefully", "error", err) + + err = shutdowner.Shutdown(fx.ExitCode(failureExitCode)) + if err != nil { + logger.Error("Failed to shutdown gracefully", + "error", err) } } }() return nil }, - OnStop: func(_ context.Context) error { + OnStop: func(ctx context.Context) error { logger.Info("Shutting down RouteWatch") - rw.Shutdown() + rw.Shutdown(ctx) return nil }, diff --git a/internal/routewatch/dbmaintainer.go b/internal/routewatch/dbmaintainer.go index 993d616..8859a46 100644 --- a/internal/routewatch/dbmaintainer.go +++ b/internal/routewatch/dbmaintainer.go @@ -1,4 +1,3 @@ -// Package routewatch contains the database maintainer for background maintenance tasks. package routewatch import ( @@ -64,10 +63,13 @@ func NewDBMaintainer(db database.Store, logger *slog.Logger) *DBMaintainer { } } -// Start begins the background maintenance goroutine. -func (m *DBMaintainer) Start() { +// Start begins the background maintenance goroutine. It runs until Stop: +// the end of ctx does not stop it. +func (m *DBMaintainer) Start(ctx context.Context) { m.wg.Add(1) - go m.run() + + go m.run(context.WithoutCancel(ctx)) + m.logger.Info("Database maintainer started", "checkpoint_interval", checkpointInterval, "vacuum_interval", vacuumInterval, @@ -83,15 +85,17 @@ func (m *DBMaintainer) Stop() { } // run is the main background loop. -func (m *DBMaintainer) run() { +func (m *DBMaintainer) run(ctx context.Context) { defer m.wg.Done() // Use different timers for each task checkpointTimer := time.NewTimer(checkpointInterval) - vacuumTimer := time.NewTimer(vacuumInterval) - analyzeTimer := time.NewTimer(analyzeInterval) defer checkpointTimer.Stop() + + vacuumTimer := time.NewTimer(vacuumInterval) defer vacuumTimer.Stop() + + analyzeTimer := time.NewTimer(analyzeInterval) defer analyzeTimer.Stop() for { @@ -100,90 +104,88 @@ func (m *DBMaintainer) run() { return case <-checkpointTimer.C: - m.runCheckpoint() + m.runCheckpoint(ctx) checkpointTimer.Reset(checkpointInterval) case <-vacuumTimer.C: - m.runVacuum() + m.runVacuum(ctx) vacuumTimer.Reset(vacuumInterval) case <-analyzeTimer.C: - m.runAnalyze() + m.runAnalyze(ctx) analyzeTimer.Reset(analyzeInterval) } } } // runCheckpoint performs a WAL checkpoint to keep the WAL file small. -func (m *DBMaintainer) runCheckpoint() { - ctx, cancel := context.WithTimeout(context.Background(), checkpointTimeout) - defer cancel() - - startTime := time.Now() - - err := m.db.Checkpoint(ctx) - - m.statsMu.Lock() - m.lastCheckpoint = time.Now() - m.lastCheckpointError = err - if err == nil { - m.checkpointCount++ - } - m.statsMu.Unlock() +func (m *DBMaintainer) runCheckpoint(ctx context.Context) { + duration, err := runWithTimeout(ctx, checkpointTimeout, m.db.Checkpoint) + m.recordResult(&m.lastCheckpoint, &m.lastCheckpointError, &m.checkpointCount, err) if err != nil { - m.logger.Error("WAL checkpoint failed", "error", err, "duration", time.Since(startTime)) + m.logger.Error("WAL checkpoint failed", "error", err, "duration", duration) } else { - m.logger.Debug("WAL checkpoint completed", "duration", time.Since(startTime)) + m.logger.Debug("WAL checkpoint completed", "duration", duration) } } // runVacuum performs an incremental vacuum operation on the database. -func (m *DBMaintainer) runVacuum() { - ctx, cancel := context.WithTimeout(context.Background(), vacuumTimeout) - defer cancel() - +func (m *DBMaintainer) runVacuum(ctx context.Context) { m.logger.Debug("Running incremental vacuum") - startTime := time.Now() - err := m.db.Vacuum(ctx) - - m.statsMu.Lock() - m.lastVacuum = time.Now() - m.lastVacuumError = err - if err == nil { - m.vacuumCount++ - } - m.statsMu.Unlock() + duration, err := runWithTimeout(ctx, vacuumTimeout, m.db.Vacuum) + m.recordResult(&m.lastVacuum, &m.lastVacuumError, &m.vacuumCount, err) if err != nil { - m.logger.Error("Incremental vacuum failed", "error", err, "duration", time.Since(startTime)) + m.logger.Error("Incremental vacuum failed", "error", err, "duration", duration) } else { - m.logger.Debug("Incremental vacuum completed", "duration", time.Since(startTime)) + m.logger.Debug("Incremental vacuum completed", "duration", duration) } } // runAnalyze performs an ANALYZE operation on the database. -func (m *DBMaintainer) runAnalyze() { - ctx, cancel := context.WithTimeout(context.Background(), analyzeTimeout) - defer cancel() - +func (m *DBMaintainer) runAnalyze(ctx context.Context) { m.logger.Info("Starting database ANALYZE") - startTime := time.Now() - err := m.db.Analyze(ctx) - - m.statsMu.Lock() - m.lastAnalyze = time.Now() - m.lastAnalyzeError = err - if err == nil { - m.analyzeCount++ - } - m.statsMu.Unlock() + duration, err := runWithTimeout(ctx, analyzeTimeout, m.db.Analyze) + m.recordResult(&m.lastAnalyze, &m.lastAnalyzeError, &m.analyzeCount, err) if err != nil { - m.logger.Error("ANALYZE failed", "error", err, "duration", time.Since(startTime)) + m.logger.Error("ANALYZE failed", "error", err, "duration", duration) } else { - m.logger.Info("ANALYZE completed", "duration", time.Since(startTime)) + m.logger.Info("ANALYZE completed", "duration", duration) + } +} + +// runWithTimeout runs operation with a context that ends after timeout and +// returns how long it took and its error. +func runWithTimeout( + ctx context.Context, + timeout time.Duration, + operation func(context.Context) error, +) (time.Duration, error) { + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + startTime := time.Now() + err := operation(ctx) + + return time.Since(startTime), err +} + +// recordResult stores when a maintenance task last ran and its error, and +// counts it when it succeeded. +func (m *DBMaintainer) recordResult( + lastRun *time.Time, lastErr *error, successCount *int, err error, +) { + m.statsMu.Lock() + defer m.statsMu.Unlock() + + *lastRun = time.Now() + *lastErr = err + + if err == nil { + *successCount++ } } diff --git a/internal/routewatch/handler.go b/internal/routewatch/handler.go index 328b7ff..70332f7 100644 --- a/internal/routewatch/handler.go +++ b/internal/routewatch/handler.go @@ -1,10 +1,16 @@ package routewatch import ( + "slices" + "git.eeqj.de/sneak/routewatch/internal/logger" "git.eeqj.de/sneak/routewatch/internal/ristypes" ) +// updateMessageType is the RIS message type of BGP UPDATE messages, the only +// type the database handlers process. +const updateMessageType = "UPDATE" + // SimpleHandler is a basic implementation of streamer.MessageHandler that // filters messages by type and delegates processing to a callback function. // It provides a simple way to handle specific RIS message types without @@ -15,10 +21,11 @@ type SimpleHandler struct { callback func(*ristypes.RISMessage) } -// NewSimpleHandler creates a new SimpleHandler that accepts specific message types. -// The messageTypes parameter specifies which RIS message types this handler will process. -// If messageTypes is empty, the handler will accept all message types. -// The callback function is invoked for each message that passes the type filter. +// NewSimpleHandler creates a new SimpleHandler that accepts specific message +// types. The messageTypes parameter specifies which RIS message types this +// handler will process. If messageTypes is empty, the handler will accept all +// message types. The callback function is invoked for each message that +// passes the type filter. func NewSimpleHandler( logger *logger.Logger, messageTypes []string, @@ -31,22 +38,17 @@ func NewSimpleHandler( } } -// WantsMessage returns true if this handler wants to process messages of the given type. -// It checks whether messageType is in the handler's configured list of accepted types. -// If no specific types were configured (empty messageTypes slice), it returns true for all types. +// WantsMessage returns true if this handler wants to process messages of the +// given type. It checks whether messageType is in the handler's configured +// list of accepted types. If no specific types were configured (empty +// messageTypes slice), it returns true for all types. func (h *SimpleHandler) WantsMessage(messageType string) bool { // If no specific types are set, accept all messages if len(h.messageTypes) == 0 { return true } - for _, t := range h.messageTypes { - if t == messageType { - return true - } - } - - return false + return slices.Contains(h.messageTypes, messageType) } // HandleMessage processes a RIS message by invoking the configured callback function. diff --git a/internal/routewatch/peerhandler.go b/internal/routewatch/peerhandler.go index 2f7bf70..54bec6d 100644 --- a/internal/routewatch/peerhandler.go +++ b/internal/routewatch/peerhandler.go @@ -1,7 +1,8 @@ package routewatch -// peerhandler.go provides batched peer tracking functionality for BGP route monitoring. -// It tracks BGP peers from all incoming RIS messages and maintains peer state in the database. +// peerhandler.go provides batched peer tracking functionality for BGP route +// monitoring. It tracks BGP peers from all incoming RIS messages and +// maintains peer state in the database. import ( "strconv" @@ -92,8 +93,10 @@ func (h *PeerHandler) QueueCapacity() int { func (h *PeerHandler) HandleMessage(msg *ristypes.RISMessage) { // Parse peer ASN from string peerASN := 0 + if msg.PeerASN != "" { - if asn, err := strconv.Atoi(msg.PeerASN); err == nil { + asn, err := strconv.Atoi(msg.PeerASN) + if err == nil { peerASN = asn } } @@ -115,9 +118,16 @@ func (h *PeerHandler) HandleMessage(msg *ristypes.RISMessage) { } } +// Stop gracefully stops the handler and flushes remaining batches +func (h *PeerHandler) Stop() { + close(h.stopCh) + h.wg.Wait() +} + // flushLoop runs in a goroutine and periodically flushes batches func (h *PeerHandler) flushLoop() { defer h.wg.Done() + ticker := time.NewTicker(peerBatchTimeout) defer ticker.Stop() @@ -140,7 +150,8 @@ func (h *PeerHandler) flushLoop() { } } -// flushBatchLocked flushes the peer batch to the database (must be called with mutex held) +// flushBatchLocked flushes the peer batch to the database (must be called +// with mutex held) func (h *PeerHandler) flushBatchLocked() { if len(h.peerBatch) == 0 { return @@ -149,7 +160,8 @@ func (h *PeerHandler) flushBatchLocked() { // Deduplicate by peer IP, keeping the latest update for each peer peerMap := make(map[string]peerUpdate) for _, update := range h.peerBatch { - if existing, ok := peerMap[update.peerIP]; !ok || update.timestamp.After(existing.timestamp) { + existing, ok := peerMap[update.peerIP] + if !ok || update.timestamp.After(existing.timestamp) { peerMap[update.peerIP] = update } } @@ -166,17 +178,13 @@ func (h *PeerHandler) flushBatchLocked() { } // Process all peers in a single batch transaction - if err := h.db.UpdatePeerBatch(dbPeerMap); err != nil { - h.logger.Error("Failed to process peer batch", "error", err, "count", len(dbPeerMap)) + err := h.db.UpdatePeerBatch(dbPeerMap) + if err != nil { + h.logger.Error("Failed to process peer batch", + "error", err, "count", len(dbPeerMap)) } // Clear batch h.peerBatch = h.peerBatch[:0] h.lastFlush = time.Now() } - -// Stop gracefully stops the handler and flushes remaining batches -func (h *PeerHandler) Stop() { - close(h.stopCh) - h.wg.Wait() -} diff --git a/internal/routewatch/peeringhandler.go b/internal/routewatch/peeringhandler.go index b4f3277..6e71421 100644 --- a/internal/routewatch/peeringhandler.go +++ b/internal/routewatch/peeringhandler.go @@ -72,7 +72,7 @@ func NewPeeringHandler(db database.Store, logger *logger.Logger) *PeeringHandler // the AS path information needed to extract peering relationships. func (h *PeeringHandler) WantsMessage(messageType string) bool { // We only care about UPDATE messages that have AS paths - return messageType == "UPDATE" + return messageType == updateMessageType } // QueueCapacity returns the buffer size for the handler's message queue. @@ -101,6 +101,7 @@ func (h *PeeringHandler) HandleMessage(msg *ristypes.RISMessage) { return } + key := string(pathJSON) h.mu.Lock() @@ -116,6 +117,23 @@ func (h *PeeringHandler) HandleMessage(msg *ristypes.RISMessage) { h.mu.Unlock() } +// ProcessPeeringsNow triggers immediate processing of all accumulated AS +// paths into peering records. This bypasses the normal periodic processing +// schedule and is primarily intended for testing purposes. +func (h *PeeringHandler) ProcessPeeringsNow() { + h.processPeerings() +} + +// Stop gracefully shuts down the handler by signaling the background +// goroutine to stop and performing a final synchronous processing of +// any remaining AS paths. This ensures no peering data is lost during +// shutdown. +func (h *PeeringHandler) Stop() { + close(h.stopCh) + // Process any remaining peerings synchronously + h.processPeerings() +} + // processLoop runs periodically to process AS paths into peerings func (h *PeeringHandler) processLoop() { ticker := time.NewTicker(peeringProcessInterval) @@ -134,13 +152,6 @@ func (h *PeeringHandler) processLoop() { } } -// ProcessPeeringsNow triggers immediate processing of all accumulated AS -// paths into peering records. This bypasses the normal periodic processing -// schedule and is primarily intended for testing purposes. -func (h *PeeringHandler) ProcessPeeringsNow() { - h.processPeerings() -} - // processPeerings extracts peerings from AS paths and writes to database func (h *PeeringHandler) processPeerings() { // Take the accumulated paths and replace the map with a fresh empty one @@ -160,11 +171,14 @@ func (h *PeeringHandler) processPeerings() { type peeringKey struct { low, high int } + peerings := make(map[peeringKey]time.Time) for pathJSON, timestamp := range paths { var path []int - if err := json.Unmarshal([]byte(pathJSON), &path); err != nil { + + err := json.Unmarshal([]byte(pathJSON), &path) + if err != nil { h.logger.Error("Failed to decode AS path", "error", err) continue @@ -197,6 +211,7 @@ func (h *PeeringHandler) processPeerings() { // Record peerings in database start := time.Now() successCount := 0 + for key, ts := range peerings { err := h.db.RecordPeering(key.low, key.high, ts) if err != nil { @@ -218,13 +233,3 @@ func (h *PeeringHandler) processPeerings() { "duration", time.Since(start), ) } - -// Stop gracefully shuts down the handler by signaling the background -// goroutine to stop and performing a final synchronous processing of -// any remaining AS paths. This ensures no peering data is lost during -// shutdown. -func (h *PeeringHandler) Stop() { - close(h.stopCh) - // Process any remaining peerings synchronously - h.processPeerings() -} diff --git a/internal/routewatch/peeringhandler_test.go b/internal/routewatch/peeringhandler_internal_test.go similarity index 95% rename from internal/routewatch/peeringhandler_test.go rename to internal/routewatch/peeringhandler_internal_test.go index 364fa46..de4f5dd 100644 --- a/internal/routewatch/peeringhandler_test.go +++ b/internal/routewatch/peeringhandler_internal_test.go @@ -53,7 +53,8 @@ func newTestHandler(db database.Store) *PeeringHandler { } } -func snapshot(h *PeeringHandler) (tracked, dropped int) { +// snapshot returns how many paths the handler tracks and how many it dropped. +func snapshot(h *PeeringHandler) (int, int) { h.mu.Lock() defer h.mu.Unlock() @@ -82,6 +83,8 @@ func handle(h *PeeringHandler, ts time.Time, asns ...int) { // empties the path map (the swap) and that a path seen again after a run is // recorded in the next run too. func TestPeeringHandlerProcessesEachRunAndEmpties(t *testing.T) { + t.Parallel() + store := &recordingStore{mockStore: newMockStore()} h := newTestHandler(store) @@ -94,6 +97,7 @@ func TestPeeringHandlerProcessesEachRunAndEmpties(t *testing.T) { if tracked, _ := snapshot(h); tracked != 0 { t.Fatalf("map not empty after first run: %d paths remain", tracked) } + if got := store.callCount(); got != 1 { t.Fatalf("want 1 RecordPeering call after first run, got %d", got) } @@ -105,6 +109,7 @@ func TestPeeringHandlerProcessesEachRunAndEmpties(t *testing.T) { if tracked, _ := snapshot(h); tracked != 0 { t.Fatalf("map not empty after second run: %d paths remain", tracked) } + if got := store.callCount(); got != 2 { t.Fatalf("want 2 RecordPeering calls after second run, got %d", got) } @@ -114,6 +119,8 @@ func TestPeeringHandlerProcessesEachRunAndEmpties(t *testing.T) { // and counts them, while a path already tracked is refreshed rather than // dropped. func TestPeeringHandlerCapDropsAndCounts(t *testing.T) { + t.Parallel() + store := &recordingStore{mockStore: newMockStore()} h := newTestHandler(store) @@ -126,6 +133,7 @@ func TestPeeringHandlerCapDropsAndCounts(t *testing.T) { h.mu.Lock() h.asPaths[existing] = now + for i := 0; len(h.asPaths) < maxTrackedPaths; i++ { h.asPaths[strconv.Itoa(i)] = now } @@ -138,6 +146,7 @@ func TestPeeringHandlerCapDropsAndCounts(t *testing.T) { if tracked != maxTrackedPaths { t.Fatalf("want map size %d after drop, got %d", maxTrackedPaths, tracked) } + if dropped != 1 { t.Fatalf("want dropped count 1, got %d", dropped) } @@ -150,6 +159,7 @@ func TestPeeringHandlerCapDropsAndCounts(t *testing.T) { if tracked != maxTrackedPaths { t.Fatalf("want map size %d after refresh, got %d", maxTrackedPaths, tracked) } + if dropped != 1 { t.Fatalf("want dropped count still 1 after refresh, got %d", dropped) } @@ -157,7 +167,9 @@ func TestPeeringHandlerCapDropsAndCounts(t *testing.T) { h.mu.Lock() gotTS := h.asPaths[existing] h.mu.Unlock() + if !gotTS.Equal(refreshed) { - t.Fatalf("existing path timestamp not refreshed: want %v, got %v", refreshed, gotTS) + t.Fatalf("existing path timestamp not refreshed: want %v, got %v", + refreshed, gotTS) } } diff --git a/internal/routewatch/prefixhandler.go b/internal/routewatch/prefixhandler.go index 59f3991..1f26bc0 100644 --- a/internal/routewatch/prefixhandler.go +++ b/internal/routewatch/prefixhandler.go @@ -33,8 +33,8 @@ const ( ipv6Version = 6 ) -// PrefixHandler tracks BGP prefixes and maintains a live routing table in the database. -// Routes are added on announcement and deleted on withdrawal. +// PrefixHandler tracks BGP prefixes and maintains a live routing table in the +// database. Routes are added on announcement and deleted on withdrawal. type PrefixHandler struct { db database.Store logger *logger.Logger @@ -79,10 +79,11 @@ func (h *PrefixHandler) SetMetricsTracker(metrics *metrics.Tracker) { h.metrics = metrics } -// WantsMessage returns true if this handler wants to process messages of the given type +// WantsMessage returns true if this handler wants to process messages of the +// given type func (h *PrefixHandler) WantsMessage(messageType string) bool { // We only care about UPDATE messages for the routing table - return messageType == "UPDATE" + return messageType == updateMessageType } // QueueCapacity returns the desired queue capacity for this handler @@ -146,9 +147,16 @@ func (h *PrefixHandler) HandleMessage(msg *ristypes.RISMessage) { } } +// Stop gracefully stops the handler and flushes remaining batches +func (h *PrefixHandler) Stop() { + close(h.stopCh) + h.wg.Wait() +} + // flushLoop runs in a goroutine and periodically flushes batches func (h *PrefixHandler) flushLoop() { defer h.wg.Done() + ticker := time.NewTicker(prefixBatchTimeout) defer ticker.Stop() @@ -171,7 +179,8 @@ func (h *PrefixHandler) flushLoop() { } } -// flushBatchLocked flushes the prefix batch to the database (must be called with mutex held) +// flushBatchLocked flushes the prefix batch to the database (must be called +// with mutex held) func (h *PrefixHandler) flushBatchLocked() { if len(h.batch) == 0 { return @@ -185,21 +194,26 @@ func (h *PrefixHandler) flushBatchLocked() { // order decides, not the timestamp: timestamps are whole seconds, so // updates within one second carry the same one. type prefixAndPeer struct{ prefix, peer string } + lastUpdates := make(map[prefixAndPeer]prefixUpdate) + for _, update := range h.batch { lastUpdates[prefixAndPeer{update.prefix, update.peer}] = update } // Collect routes to upsert and delete - var routesToUpsert []*database.LiveRoute - var routesToDelete []database.LiveRouteDeletion + var ( + routesToUpsert []*database.LiveRoute + routesToDelete []database.LiveRouteDeletion + ) // Collect unique prefixes to update prefixesToUpdate := make(map[string]time.Time) for _, update := range lastUpdates { // Track prefix for both announcements and withdrawals - if _, exists := prefixesToUpdate[update.prefix]; !exists || update.timestamp.After(prefixesToUpdate[update.prefix]) { + seen, exists := prefixesToUpdate[update.prefix] + if !exists || update.timestamp.After(seen) { prefixesToUpdate[update.prefix] = update.timestamp } @@ -213,7 +227,8 @@ func (h *PrefixHandler) flushBatchLocked() { // Parse CIDR to get IP version _, ipVersion, err := parseCIDR(update.prefix) if err != nil { - h.logger.Error("Failed to parse CIDR for withdrawal", "prefix", update.prefix, "error", err) + h.logger.Error("Failed to parse CIDR for withdrawal", + "prefix", update.prefix, "error", err) continue } @@ -228,30 +243,7 @@ func (h *PrefixHandler) flushBatchLocked() { } } - // Process batch operations - successCount := 0 - if len(routesToUpsert) > 0 { - if err := h.db.UpsertLiveRouteBatch(routesToUpsert); err != nil { - h.logger.Error("Failed to upsert route batch", "error", err, "count", len(routesToUpsert)) - } else { - successCount += len(routesToUpsert) - } - } - - if len(routesToDelete) > 0 { - if err := h.db.DeleteLiveRouteBatch(routesToDelete); err != nil { - h.logger.Error("Failed to delete route batch", "error", err, "count", len(routesToDelete)) - } else { - successCount += len(routesToDelete) - } - } - - // Update prefix tables - if len(prefixesToUpdate) > 0 { - if err := h.db.UpdatePrefixesBatch(prefixesToUpdate); err != nil { - h.logger.Error("Failed to update prefix batch", "error", err, "count", len(prefixesToUpdate)) - } - } + successCount := h.writeBatch(routesToUpsert, routesToDelete, prefixesToUpdate) elapsed := time.Since(startTime) h.logger.Debug("Flushed prefix batch", @@ -266,6 +258,47 @@ func (h *PrefixHandler) flushBatchLocked() { h.lastFlush = time.Now() } +// writeBatch writes the collected route upserts, route deletions and prefix +// updates to the database and returns how many routes it wrote. +func (h *PrefixHandler) writeBatch( + routesToUpsert []*database.LiveRoute, + routesToDelete []database.LiveRouteDeletion, + prefixesToUpdate map[string]time.Time, +) int { + successCount := 0 + + if len(routesToUpsert) > 0 { + err := h.db.UpsertLiveRouteBatch(routesToUpsert) + if err != nil { + h.logger.Error("Failed to upsert route batch", + "error", err, "count", len(routesToUpsert)) + } else { + successCount += len(routesToUpsert) + } + } + + if len(routesToDelete) > 0 { + err := h.db.DeleteLiveRouteBatch(routesToDelete) + if err != nil { + h.logger.Error("Failed to delete route batch", + "error", err, "count", len(routesToDelete)) + } else { + successCount += len(routesToDelete) + } + } + + // Update prefix tables + if len(prefixesToUpdate) > 0 { + err := h.db.UpdatePrefixesBatch(prefixesToUpdate) + if err != nil { + h.logger.Error("Failed to update prefix batch", + "error", err, "count", len(prefixesToUpdate)) + } + } + + return successCount +} + // canonicalPrefix returns prefix in the text form net/netip prints, the form // the IP lookup builds when it looks a prefix up. The feed sends IPv6 // announcements compressed ("2001:db8::/32") but IPv6 withdrawals uncompressed @@ -281,8 +314,8 @@ func canonicalPrefix(prefix string) string { return p.Masked().String() } -// parseCIDR extracts the mask length and IP version from a prefix string -func parseCIDR(prefix string) (maskLength int, ipVersion int, err error) { +// parseCIDR returns the mask length and the IP version of a prefix string +func parseCIDR(prefix string) (int, int, error) { _, ipNet, err := net.ParseCIDR(prefix) if err != nil { return 0, 0, err @@ -296,50 +329,6 @@ func parseCIDR(prefix string) (maskLength int, ipVersion int, err error) { return ones, ipv4Version, nil } -// processAnnouncement handles storing an announcement in the database -// nolint:unused // kept for potential future use -func (h *PrefixHandler) processAnnouncement(_ *database.Prefix, update prefixUpdate) { - // Parse CIDR to get mask length - maskLength, ipVersion, err := parseCIDR(update.prefix) - if err != nil { - h.logger.Error("Failed to parse CIDR", - "prefix", update.prefix, - "error", err, - ) - - return - } - - // Track route update metrics - if h.metrics != nil { - if ipVersion == ipv4Version { - h.metrics.RecordIPv4Update() - } else { - h.metrics.RecordIPv6Update() - } - } - - // Create live route record - liveRoute := &database.LiveRoute{ - ID: uuid.New(), - Prefix: update.prefix, - MaskLength: maskLength, - IPVersion: ipVersion, - OriginASN: update.originASN, - PeerIP: update.peer, - ASPath: update.path, - NextHop: update.peer, // Using peer as next hop - LastUpdated: update.timestamp, - } - - if err := h.db.UpsertLiveRoute(liveRoute); err != nil { - h.logger.Error("Failed to upsert live route", - "prefix", update.prefix, - "error", err, - ) - } -} - // createLiveRoute creates a LiveRoute from a prefix update func (h *PrefixHandler) createLiveRoute(update prefixUpdate) *database.LiveRoute { // Parse CIDR to get mask length @@ -377,104 +366,3 @@ func (h *PrefixHandler) createLiveRoute(update prefixUpdate) *database.LiveRoute return liveRoute } - -// processAnnouncementDirect handles storing an announcement directly without prefix table -// nolint:unused // kept for potential future use -func (h *PrefixHandler) processAnnouncementDirect(update prefixUpdate) { - // Parse CIDR to get mask length - maskLength, ipVersion, err := parseCIDR(update.prefix) - if err != nil { - h.logger.Error("Failed to parse CIDR", - "prefix", update.prefix, - "error", err, - ) - - return - } - - // Track route update metrics - if h.metrics != nil { - if ipVersion == ipv4Version { - h.metrics.RecordIPv4Update() - } else { - h.metrics.RecordIPv6Update() - } - } - - // Create live route record - liveRoute := &database.LiveRoute{ - ID: uuid.New(), - Prefix: update.prefix, - MaskLength: maskLength, - IPVersion: ipVersion, - OriginASN: update.originASN, - PeerIP: update.peer, - ASPath: update.path, - NextHop: update.peer, // Using peer as next hop - LastUpdated: update.timestamp, - } - - if err := h.db.UpsertLiveRoute(liveRoute); err != nil { - h.logger.Error("Failed to upsert live route", - "prefix", update.prefix, - "error", err, - ) - } -} - -// processWithdrawalDirect handles removing a route directly without prefix table -// nolint:unused // kept for potential future use -func (h *PrefixHandler) processWithdrawalDirect(update prefixUpdate) { - // For withdrawals, we need to delete the route from live_routes - if update.originASN > 0 { - if err := h.db.DeleteLiveRoute(update.prefix, update.originASN, update.peer); err != nil { - h.logger.Error("Failed to delete live route", - "prefix", update.prefix, - "origin_asn", update.originASN, - "peer", update.peer, - "error", err, - ) - } - } else { - // If no origin ASN, just delete all routes for this prefix from this peer - if err := h.db.DeleteLiveRoute(update.prefix, 0, update.peer); err != nil { - h.logger.Error("Failed to delete live route (no origin ASN)", - "prefix", update.prefix, - "peer", update.peer, - "error", err, - ) - } - } -} - -// processWithdrawal handles removing a route from the live routing table -// nolint:unused // kept for potential future use -func (h *PrefixHandler) processWithdrawal(_ *database.Prefix, update prefixUpdate) { - // For withdrawals, we need to delete the route from live_routes - // Since we have the origin ASN from the update, we can delete the specific route - if update.originASN > 0 { - if err := h.db.DeleteLiveRoute(update.prefix, update.originASN, update.peer); err != nil { - h.logger.Error("Failed to delete live route", - "prefix", update.prefix, - "origin_asn", update.originASN, - "peer", update.peer, - "error", err, - ) - } - } else { - // If no origin ASN, just delete all routes for this prefix from this peer - if err := h.db.DeleteLiveRoute(update.prefix, 0, update.peer); err != nil { - h.logger.Error("Failed to delete live route (no origin ASN)", - "prefix", update.prefix, - "peer", update.peer, - "error", err, - ) - } - } -} - -// Stop gracefully stops the handler and flushes remaining batches -func (h *PrefixHandler) Stop() { - close(h.stopCh) - h.wg.Wait() -} diff --git a/internal/routewatch/prefixhandler_test.go b/internal/routewatch/prefixhandler_internal_test.go similarity index 56% rename from internal/routewatch/prefixhandler_test.go rename to internal/routewatch/prefixhandler_internal_test.go index fcfa9a3..482ff9e 100644 --- a/internal/routewatch/prefixhandler_test.go +++ b/internal/routewatch/prefixhandler_internal_test.go @@ -13,7 +13,10 @@ import ( "git.eeqj.de/sneak/routewatch/internal/ristypes" ) -const testPeerIP = "2001:db8:ffff::1" +const ( + testPeerIP = "2001:db8:ffff::1" + testV4Prefix = "192.0.2.0/24" +) // TestPrefixHandlerStoresPrefixesTheIPLookupFinds runs announcements and // withdrawals through the prefix handler into a real database and looks the @@ -22,6 +25,8 @@ const testPeerIP = "2001:db8:ffff::1" // remove the route the announcement stored, and the IP lookup must find the // stored prefix. func TestPrefixHandlerStoresPrefixesTheIPLookupFinds(t *testing.T) { + t.Parallel() + db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New()) if err != nil { t.Fatalf("failed to create database: %v", err) @@ -33,6 +38,7 @@ func TestPrefixHandlerStoresPrefixesTheIPLookupFinds(t *testing.T) { flush := func() { h.mu.Lock() defer h.mu.Unlock() + h.flushBatchLocked() } @@ -43,34 +49,38 @@ func TestPrefixHandlerStoresPrefixesTheIPLookupFinds(t *testing.T) { ParsedTimestamp: ts, Announcements: []ristypes.RISAnnouncement{{ NextHop: testPeerIP, - Prefixes: []string{"2001:db8:1::/48", "192.0.2.0/24"}, + Prefixes: []string{"2001:db8:1::/48", testV4Prefix}, }}, }) flush() for ip, want := range map[string]string{ "2001:db8:1::1": "2001:db8:1::/48", - "192.0.2.1": "192.0.2.0/24", + "192.0.2.1": testV4Prefix, } { info, err := db.GetIPInfo(ip) if err != nil { t.Fatalf("GetIPInfo(%s) after announcement: %v", ip, err) } + if info.Netblock != want || info.ASN != testASNB { - t.Errorf("GetIPInfo(%s) = %s AS%d, want %s AS%d", ip, info.Netblock, info.ASN, want, testASNB) + t.Errorf("GetIPInfo(%s) = %s AS%d, want %s AS%d", + ip, info.Netblock, info.ASN, want, testASNB) } } h.HandleMessage(&ristypes.RISMessage{ Peer: testPeerIP, ParsedTimestamp: ts.Add(time.Minute), - Withdrawals: []string{"2001:db8:1:0:0:0:0:0/48", "192.0.2.0/24"}, + Withdrawals: []string{"2001:db8:1:0:0:0:0:0/48", testV4Prefix}, }) flush() for _, ip := range []string{"2001:db8:1::1", "192.0.2.1"} { - if info, err := db.GetIPInfo(ip); !errors.Is(err, database.ErrNoRoute) { - t.Errorf("GetIPInfo(%s) after withdrawal = %+v, %v; want ErrNoRoute", ip, info, err) + info, err := db.GetIPInfo(ip) + if !errors.Is(err, database.ErrNoRoute) { + t.Errorf("GetIPInfo(%s) after withdrawal = %+v, %v; want ErrNoRoute", + ip, info, err) } } } @@ -82,6 +92,8 @@ func TestPrefixHandlerStoresPrefixesTheIPLookupFinds(t *testing.T) { // live route for it, and updates from different peers are all applied. The // in-memory route count and prefix distribution must match the route table. func TestPrefixHandlerAppliesEachPeersLastUpdateInABatch(t *testing.T) { + t.Parallel() + const ( prefix = "198.51.100.0/24" prefixMaskLength = 24 @@ -94,11 +106,15 @@ func TestPrefixHandlerAppliesEachPeersLastUpdateInABatch(t *testing.T) { Peer: peer, Path: ristypes.ASPath{testASNA, testASNB}, ParsedTimestamp: ts, - Announcements: []ristypes.RISAnnouncement{{NextHop: peer, Prefixes: []string{prefix}}}, + Announcements: []ristypes.RISAnnouncement{ + {NextHop: peer, Prefixes: []string{prefix}}, + }, } } withdraw := func(peer string) *ristypes.RISMessage { - return &ristypes.RISMessage{Peer: peer, ParsedTimestamp: ts, Withdrawals: []string{prefix}} + return &ristypes.RISMessage{ + Peer: peer, ParsedTimestamp: ts, Withdrawals: []string{prefix}, + } } announceAndWithdraw := func(peer string) *ristypes.RISMessage { msg := announce(peer) @@ -152,62 +168,101 @@ func TestPrefixHandlerAppliesEachPeersLastUpdateInABatch(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New()) - if err != nil { - t.Fatalf("failed to create database: %v", err) - } - defer func() { _ = db.Close() }() + t.Parallel() - // Built without NewPrefixHandler's flush timer, so the test flushes itself. - h := &PrefixHandler{db: db, logger: logger.New()} - for _, batch := range tt.batches { - for _, msg := range batch { - h.HandleMessage(msg) - } - h.mu.Lock() - h.flushBatchLocked() - h.mu.Unlock() - } - - ctx := context.Background() - routes, _, err := db.GetPrefixDetailsContext(ctx, prefix) - if err != nil && !errors.Is(err, database.ErrNoRoute) { - t.Fatalf("GetPrefixDetailsContext: %v", err) - } - var gotPeers []string - for _, route := range routes { - gotPeers = append(gotPeers, route.PeerIP) - } - if !slices.Equal(gotPeers, tt.wantPeers) { - t.Errorf("peers with a live route for %s = %v, want %v", prefix, gotPeers, tt.wantPeers) - } - - stats, err := db.GetStatsContext(ctx) - if err != nil { - t.Fatalf("GetStatsContext: %v", err) - } - tableRoutesV4, _, err := db.GetLiveRouteCountsContext(ctx) - if err != nil { - t.Fatalf("GetLiveRouteCountsContext: %v", err) - } - if stats.IPv4Routes != len(tt.wantPeers) || tableRoutesV4 != len(tt.wantPeers) { - t.Errorf("IPv4 live routes: %d in memory, %d in the table, want %d", - stats.IPv4Routes, tableRoutesV4, len(tt.wantPeers)) - } - - var wantDistribution []database.PrefixDistribution - if len(tt.wantPeers) > 0 { - wantDistribution = []database.PrefixDistribution{{MaskLength: prefixMaskLength, Count: 1}} - } - queryDistributionV4, _, err := db.GetPrefixDistributionContext(ctx) - if err != nil { - t.Fatalf("GetPrefixDistributionContext: %v", err) - } - if !slices.Equal(stats.IPv4PrefixDistribution, wantDistribution) || - !slices.Equal(queryDistributionV4, wantDistribution) { - t.Errorf("IPv4 prefix distribution: %v in memory, %v from the table, want %v", - stats.IPv4PrefixDistribution, queryDistributionV4, wantDistribution) - } + db := handleBatches(t, tt.batches) + checkPrefixRoutes(t, db, prefix, prefixMaskLength, tt.wantPeers) }) } } + +// handleBatches runs each batch of messages through a prefix handler on a new +// database, flushing after each batch, and returns the database. The database +// is closed when the test ends. +func handleBatches( + t *testing.T, batches [][]*ristypes.RISMessage, +) *database.Database { + t.Helper() + + db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New()) + if err != nil { + t.Fatalf("failed to create database: %v", err) + } + + t.Cleanup(func() { _ = db.Close() }) + + // Built without NewPrefixHandler's flush timer, so the test flushes itself. + h := &PrefixHandler{db: db, logger: logger.New()} + + for _, batch := range batches { + for _, msg := range batch { + h.HandleMessage(msg) + } + + h.mu.Lock() + h.flushBatchLocked() + h.mu.Unlock() + } + + return db +} + +// checkPrefixRoutes checks that exactly wantPeers have a live route for the +// IPv4 prefix, and that the in-memory route count and prefix distribution +// match the route table. +func checkPrefixRoutes( + t *testing.T, db *database.Database, prefix string, maskLength int, + wantPeers []string, +) { + t.Helper() + + ctx := context.Background() + + routes, _, err := db.GetPrefixDetailsContext(ctx, prefix) + if err != nil && !errors.Is(err, database.ErrNoRoute) { + t.Fatalf("GetPrefixDetailsContext: %v", err) + } + + gotPeers := make([]string, 0, len(routes)) + for _, route := range routes { + gotPeers = append(gotPeers, route.PeerIP) + } + + if !slices.Equal(gotPeers, wantPeers) { + t.Errorf("peers with a live route for %s = %v, want %v", + prefix, gotPeers, wantPeers) + } + + stats, err := db.GetStatsContext(ctx) + if err != nil { + t.Fatalf("GetStatsContext: %v", err) + } + + tableRoutesV4, _, err := db.GetLiveRouteCountsContext(ctx) + if err != nil { + t.Fatalf("GetLiveRouteCountsContext: %v", err) + } + + if stats.IPv4Routes != len(wantPeers) || tableRoutesV4 != len(wantPeers) { + t.Errorf("IPv4 live routes: %d in memory, %d in the table, want %d", + stats.IPv4Routes, tableRoutesV4, len(wantPeers)) + } + + var wantDistribution []database.PrefixDistribution + if len(wantPeers) > 0 { + wantDistribution = []database.PrefixDistribution{ + {MaskLength: maskLength, Count: 1}, + } + } + + queryDistributionV4, _, err := db.GetPrefixDistributionContext(ctx) + if err != nil { + t.Fatalf("GetPrefixDistributionContext: %v", err) + } + + if !slices.Equal(stats.IPv4PrefixDistribution, wantDistribution) || + !slices.Equal(queryDistributionV4, wantDistribution) { + t.Errorf("IPv4 prefix distribution: %v in memory, %v from the table, want %v", + stats.IPv4PrefixDistribution, queryDistributionV4, wantDistribution) + } +} diff --git a/internal/server/handlers.go b/internal/server/handlers.go index 0856bd2..b5cdffa 100644 --- a/internal/server/handlers.go +++ b/internal/server/handlers.go @@ -15,6 +15,7 @@ import ( "time" "git.eeqj.de/sneak/routewatch/internal/database" + "git.eeqj.de/sneak/routewatch/internal/streamer" "git.eeqj.de/sneak/routewatch/internal/templates" asinfo "git.eeqj.de/sneak/routewatch/pkg/asinfo" "github.com/dustin/go-humanize" @@ -29,6 +30,22 @@ const ( healthCheckTimeout = 2 * time.Second ) +// Keys of the JSON envelope the handlers and middleware write, and the status +// value of an error answer. +const ( + jsonKeyStatus = "status" + jsonKeyError = "error" + jsonKeyMsg = "msg" + jsonKeyCode = "code" + jsonStatusError = "error" +) + +// IP versions as database.LiveRoute.IPVersion records them. +const ( + ipVersionV4 = 4 + ipVersionV6 = 6 +) + // HealthCheckResponse represents the health check response. type HealthCheckResponse struct { Status string `json:"status"` @@ -68,7 +85,7 @@ func (s *Server) handleHealthCheck() http.HandlerFunc { // Build response status := "ok" if !healthy { - status = "error" + status = jsonStatusError } response := HealthCheckResponse{ @@ -81,7 +98,8 @@ func (s *Server) handleHealthCheck() http.HandlerFunc { w.WriteHeader(http.StatusServiceUnavailable) } - if err := writeJSONSuccess(w, response); err != nil { + err = writeJSONSuccess(w, response) + if err != nil { s.logger.Error("Failed to encode health check response", "error", err) } } @@ -93,7 +111,9 @@ func (s *Server) handleIndex() http.HandlerFunc { w.Header().Set("Content-Type", "text/html; charset=utf-8") tmpl := templates.IndexTemplate() - if err := tmpl.Execute(w, nil); err != nil { + + err := tmpl.Execute(w, nil) + if err != nil { s.logger.Error("Failed to render index template", "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) } @@ -102,30 +122,36 @@ func (s *Server) handleIndex() http.HandlerFunc { // writeJSONError writes a standardized JSON error response with the given // status code and error message. -func writeJSONError(w http.ResponseWriter, statusCode int, message string) { +func (s *Server) writeJSONError(w http.ResponseWriter, statusCode int, message string) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(statusCode) - _ = json.NewEncoder(w).Encode(map[string]interface{}{ - "status": "error", - "error": map[string]interface{}{ - "msg": message, - "code": statusCode, + + err := json.NewEncoder(w).Encode(map[string]any{ + jsonKeyStatus: jsonStatusError, + jsonKeyError: map[string]any{ + jsonKeyMsg: message, + jsonKeyCode: statusCode, }, }) + if err != nil { + s.logger.Error("Failed to encode error response", "error", err) + } } // writeJSONSuccess writes a standardized JSON success response containing // the provided data wrapped in a status envelope. -func writeJSONSuccess(w http.ResponseWriter, data interface{}) error { +func writeJSONSuccess(w http.ResponseWriter, data any) error { w.Header().Set("Content-Type", "application/json") - return json.NewEncoder(w).Encode(map[string]interface{}{ - "status": "ok", - "data": data, + return json.NewEncoder(w).Encode(map[string]any{ + jsonKeyStatus: "ok", + "data": data, }) } // WHOISStatsInfo contains WHOIS fetcher statistics for the status page. +// +//nolint:tagliatelle // field names are the existing JSON API type WHOISStatsInfo struct { TotalASNs int `json:"total_asns"` FreshASNs int `json:"fresh_asns"` @@ -138,41 +164,75 @@ type WHOISStatsInfo struct { FreshPercent float64 `json:"fresh_percent"` } +// Stats is the /status.json answer: uptime, message counts, database stats, +// and route information. +// +//nolint:tagliatelle // field names are the existing JSON API +type Stats struct { + Uptime string `json:"uptime"` + TotalMessages uint64 `json:"total_messages"` + TotalBytes uint64 `json:"total_bytes"` + TotalWireBytes uint64 `json:"total_wire_bytes"` + MessagesPerSec float64 `json:"messages_per_sec"` + MbitsPerSec float64 `json:"mbits_per_sec"` + WireMbitsPerSec float64 `json:"wire_mbits_per_sec"` + Connected bool `json:"connected"` + GoVersion string `json:"go_version"` + Goroutines int `json:"goroutines"` + MemoryUsage string `json:"memory_usage"` + ASNs int `json:"asns"` + Prefixes int `json:"prefixes"` + IPv4Prefixes int `json:"ipv4_prefixes"` + IPv6Prefixes int `json:"ipv6_prefixes"` + Peerings int `json:"peerings"` + Peers int `json:"peers"` + DatabaseSizeBytes int64 `json:"database_size_bytes"` + LiveRoutes int `json:"live_routes"` + IPv4Routes int `json:"ipv4_routes"` + IPv6Routes int `json:"ipv6_routes"` + OldestRoute *time.Time `json:"oldest_route,omitempty"` + NewestRoute *time.Time `json:"newest_route,omitempty"` + IPv4UpdatesPerSec float64 `json:"ipv4_updates_per_sec"` + IPv6UpdatesPerSec float64 `json:"ipv6_updates_per_sec"` + IPv4PrefixDistribution []database.PrefixDistribution `json:"ipv4_prefix_distribution"` + IPv6PrefixDistribution []database.PrefixDistribution `json:"ipv6_prefix_distribution"` + WHOISStats *WHOISStatsInfo `json:"whois_stats,omitempty"` +} + +// databaseStats gets the database stats, giving up when ctx is done; it then +// returns ctx's error. The channels are buffered so the goroutine's send never +// blocks if the timeout wins and nothing here receives; otherwise it would +// block forever and leak. +func (s *Server) databaseStats(ctx context.Context) (database.Stats, error) { + statsChan := make(chan database.Stats, 1) + errChan := make(chan error, 1) + + go func() { + dbStats, err := s.db.GetStatsContext(ctx) + if err != nil { + s.logger.Debug("Database stats query failed", "error", err) + + errChan <- err + + return + } + + statsChan <- dbStats + }() + + select { + case <-ctx.Done(): + return database.Stats{}, ctx.Err() + case err := <-errChan: + return database.Stats{}, err + case dbStats := <-statsChan: + return dbStats, nil + } +} + // handleStatusJSON returns a handler that serves JSON statistics including // uptime, message counts, database stats, and route information. func (s *Server) handleStatusJSON() http.HandlerFunc { - // Stats represents the statistics response - type Stats struct { - Uptime string `json:"uptime"` - TotalMessages uint64 `json:"total_messages"` - TotalBytes uint64 `json:"total_bytes"` - TotalWireBytes uint64 `json:"total_wire_bytes"` - MessagesPerSec float64 `json:"messages_per_sec"` - MbitsPerSec float64 `json:"mbits_per_sec"` - WireMbitsPerSec float64 `json:"wire_mbits_per_sec"` - Connected bool `json:"connected"` - GoVersion string `json:"go_version"` - Goroutines int `json:"goroutines"` - MemoryUsage string `json:"memory_usage"` - ASNs int `json:"asns"` - Prefixes int `json:"prefixes"` - IPv4Prefixes int `json:"ipv4_prefixes"` - IPv6Prefixes int `json:"ipv6_prefixes"` - Peerings int `json:"peerings"` - Peers int `json:"peers"` - DatabaseSizeBytes int64 `json:"database_size_bytes"` - LiveRoutes int `json:"live_routes"` - IPv4Routes int `json:"ipv4_routes"` - IPv6Routes int `json:"ipv6_routes"` - OldestRoute *time.Time `json:"oldest_route,omitempty"` - NewestRoute *time.Time `json:"newest_route,omitempty"` - IPv4UpdatesPerSec float64 `json:"ipv4_updates_per_sec"` - IPv6UpdatesPerSec float64 `json:"ipv6_updates_per_sec"` - IPv4PrefixDistribution []database.PrefixDistribution `json:"ipv4_prefix_distribution"` - IPv6PrefixDistribution []database.PrefixDistribution `json:"ipv6_prefix_distribution"` - WHOISStats *WHOISStatsInfo `json:"whois_stats,omitempty"` - } - return func(w http.ResponseWriter, r *http.Request) { // Create a 4 second timeout context for this request ctx, cancel := context.WithTimeout(r.Context(), statsContextTimeout) @@ -180,42 +240,19 @@ func (s *Server) handleStatusJSON() http.HandlerFunc { metrics := s.streamer.GetMetrics() - // Get database stats with timeout. The channels are buffered so the - // goroutine's send never blocks if the timeout wins and nothing here - // receives; otherwise it would block forever and leak. - statsChan := make(chan database.Stats, 1) - errChan := make(chan error, 1) - - go func() { - dbStats, err := s.db.GetStatsContext(ctx) - if err != nil { - s.logger.Debug("Database stats query failed", "error", err) - errChan <- err - - return - } - statsChan <- dbStats - }() - - var dbStats database.Stats - select { - case <-ctx.Done(): + dbStats, err := s.databaseStats(ctx) + if err != nil && ctx.Err() != nil { s.logger.Error("Database stats timeout in status.json") - writeJSONError(w, http.StatusRequestTimeout, "Database timeout") + s.writeJSONError(w, http.StatusRequestTimeout, "Database timeout") return - case err := <-errChan: - s.logger.Error("Failed to get database stats", "error", err) - writeJSONError(w, http.StatusInternalServerError, err.Error()) - - return - case dbStats = <-statsChan: - // Success } - uptime := time.Since(metrics.ConnectedSince).Truncate(time.Second).String() - if metrics.ConnectedSince.IsZero() { - uptime = "0s" + if err != nil { + s.logger.Error("Failed to get database stats", "error", err) + s.writeJSONError(w, http.StatusInternalServerError, err.Error()) + + return } const bitsPerMegabit = 1000000.0 @@ -228,13 +265,10 @@ func (s *Server) handleStatusJSON() http.HandlerFunc { runtime.ReadMemStats(&memStats) // Get WHOIS stats if fetcher is available - var whoisStats *WHOISStatsInfo - if s.asnFetcher != nil { - whoisStats = s.getWHOISStats(ctx) - } + whoisStats := s.getWHOISStats(ctx) stats := Stats{ - Uptime: uptime, + Uptime: streamUptime(metrics.ConnectedSince), TotalMessages: metrics.TotalMessages, TotalBytes: metrics.TotalBytes, TotalWireBytes: metrics.TotalWireBytes, @@ -264,14 +298,20 @@ func (s *Server) handleStatusJSON() http.HandlerFunc { WHOISStats: whoisStats, } - if err := writeJSONSuccess(w, stats); err != nil { + err = writeJSONSuccess(w, stats) + if err != nil { s.logger.Error("Failed to encode stats", "error", err) } } } -// getWHOISStats builds WHOIS statistics from database and fetcher. +// getWHOISStats builds WHOIS statistics from database and fetcher, or returns +// nil when there is no fetcher. func (s *Server) getWHOISStats(ctx context.Context) *WHOISStatsInfo { + if s.asnFetcher == nil { + return nil + } + // Get database WHOIS stats dbStats, err := s.db.GetWHOISStats(ctx, whoisStaleThreshold) if err != nil { @@ -286,7 +326,8 @@ func (s *Server) getWHOISStats(ctx context.Context) *WHOISStatsInfo { // Calculate fresh percentage var freshPercent float64 if dbStats.TotalASNs > 0 { - freshPercent = float64(dbStats.FreshASNs) / float64(dbStats.TotalASNs) * percentMultiplier + freshPercent = float64(dbStats.FreshASNs) / float64(dbStats.TotalASNs) * + percentMultiplier } return &WHOISStatsInfo{ @@ -302,82 +343,174 @@ func (s *Server) getWHOISStats(ctx context.Context) *WHOISStatsInfo { } } +// streamUptime is how long the stream has been connected, to the second, or +// "0s" if it has never connected. +func streamUptime(connectedSince time.Time) string { + if connectedSince.IsZero() { + return "0s" + } + + return time.Since(connectedSince).Truncate(time.Second).String() +} + +// connectionDuration is how long the stream has been connected, to the +// second, or "disconnected". +func connectionDuration(connected bool, connectedSince time.Time) string { + if !connected || connectedSince.IsZero() { + return "disconnected" + } + + return time.Since(connectedSince).Truncate(time.Second).String() +} + // whoisStaleThreshold matches the fetcher's threshold for consistency. const whoisStaleThreshold = 30 * 24 * time.Hour // percentMultiplier converts a ratio to a percentage. const percentMultiplier = 100 +// HandlerStatsInfo represents handler statistics in the API response +// +//nolint:tagliatelle // field names are the existing JSON API +type HandlerStatsInfo struct { + Name string `json:"name"` + QueueLength int `json:"queue_length"` + QueueCapacity int `json:"queue_capacity"` + QueueHighWaterMark int `json:"queue_high_water_mark"` + ProcessedCount uint64 `json:"processed_count"` + DroppedCount uint64 `json:"dropped_count"` + AvgProcessTimeMs float64 `json:"avg_process_time_ms"` + MinProcessTimeMs float64 `json:"min_process_time_ms"` + MaxProcessTimeMs float64 `json:"max_process_time_ms"` +} + +// GCStats represents garbage collection statistics +// +//nolint:tagliatelle // field names are the existing JSON API +type GCStats struct { + NumGC uint32 `json:"num_gc"` + TotalPauseMs uint64 `json:"total_pause_ms"` + LastPauseMs float64 `json:"last_pause_ms"` + HeapAllocBytes uint64 `json:"heap_alloc_bytes"` + HeapSysBytes uint64 `json:"heap_sys_bytes"` +} + +// StreamStats represents stream statistics including announcements/withdrawals +// +//nolint:tagliatelle // field names are the existing JSON API +type StreamStats struct { + Announcements uint64 `json:"announcements"` + Withdrawals uint64 `json:"withdrawals"` + RouteChurnPerSec float64 `json:"route_churn_per_sec"` + BGPPeerCount int `json:"bgp_peer_count"` +} + +// StatsResponse represents the API statistics response +// +//nolint:tagliatelle // field names are the existing JSON API +type StatsResponse struct { + Uptime string `json:"uptime"` + TotalMessages uint64 `json:"total_messages"` + TotalBytes uint64 `json:"total_bytes"` + TotalWireBytes uint64 `json:"total_wire_bytes"` + MessagesPerSec float64 `json:"messages_per_sec"` + MbitsPerSec float64 `json:"mbits_per_sec"` + WireMbitsPerSec float64 `json:"wire_mbits_per_sec"` + Connected bool `json:"connected"` + ConnectionDuration string `json:"connection_duration"` + ReconnectCount uint64 `json:"reconnect_count"` + GoVersion string `json:"go_version"` + Goroutines int `json:"goroutines"` + MemoryUsage string `json:"memory_usage"` + GC GCStats `json:"gc"` + Stream StreamStats `json:"stream"` + ASNs int `json:"asns"` + Prefixes int `json:"prefixes"` + IPv4Prefixes int `json:"ipv4_prefixes"` + IPv6Prefixes int `json:"ipv6_prefixes"` + Peerings int `json:"peerings"` + Peers int `json:"peers"` + DatabaseSizeBytes int64 `json:"database_size_bytes"` + LiveRoutes int `json:"live_routes"` + IPv4Routes int `json:"ipv4_routes"` + IPv6Routes int `json:"ipv6_routes"` + OldestRoute *time.Time `json:"oldest_route,omitempty"` + NewestRoute *time.Time `json:"newest_route,omitempty"` + IPv4UpdatesPerSec float64 `json:"ipv4_updates_per_sec"` + IPv6UpdatesPerSec float64 `json:"ipv6_updates_per_sec"` + HandlerStats []HandlerStatsInfo `json:"handler_stats"` + IPv4PrefixDistribution []database.PrefixDistribution `json:"ipv4_prefix_distribution"` + IPv6PrefixDistribution []database.PrefixDistribution `json:"ipv6_prefix_distribution"` + WHOISStats *WHOISStatsInfo `json:"whois_stats,omitempty"` +} + +// handlerStatsInfo converts the streamer's handler statistics for the API +// response. +func handlerStatsInfo(handlerStats []streamer.HandlerStats) []HandlerStatsInfo { + const microsecondsPerMillisecond = 1000.0 + + handlerStatsInfo := make([]HandlerStatsInfo, 0, len(handlerStats)) + for _, hs := range handlerStats { + handlerStatsInfo = append(handlerStatsInfo, HandlerStatsInfo{ + Name: hs.Name, + QueueLength: hs.QueueLength, + QueueCapacity: hs.QueueCapacity, + QueueHighWaterMark: hs.QueueHighWaterMark, + ProcessedCount: hs.ProcessedCount, + DroppedCount: hs.DroppedCount, + AvgProcessTimeMs: float64(hs.AvgProcessTime.Microseconds()) / + microsecondsPerMillisecond, + MinProcessTimeMs: float64(hs.MinProcessTime.Microseconds()) / + microsecondsPerMillisecond, + MaxProcessTimeMs: float64(hs.MaxProcessTime.Microseconds()) / + microsecondsPerMillisecond, + }) + } + + return handlerStatsInfo +} + +// gcStats reports the garbage collector's figures from memStats. +func gcStats(memStats *runtime.MemStats) GCStats { + const ( + nanosecondsPerMillisecond = 1e6 + gcPauseHistorySize = 256 // Size of runtime.MemStats.PauseNs circular buffer + ) + + // Calculate last GC pause + var lastPauseMs float64 + + if memStats.NumGC > 0 { + // PauseNs is a circular buffer, get the most recent pause + lastPauseIdx := (memStats.NumGC + gcPauseHistorySize - 1) % gcPauseHistorySize + lastPauseMs = float64(memStats.PauseNs[lastPauseIdx]) / nanosecondsPerMillisecond + } + + return GCStats{ + NumGC: memStats.NumGC, + TotalPauseMs: memStats.PauseTotalNs / uint64(nanosecondsPerMillisecond), + LastPauseMs: lastPauseMs, + HeapAllocBytes: memStats.HeapAlloc, + HeapSysBytes: memStats.HeapSys, + } +} + +// streamStats reports the announcement, withdrawal and BGP peer figures of the +// streamer's metrics tracker. +func (s *Server) streamStats() StreamStats { + metricsTracker := s.streamer.GetMetricsTracker() + + return StreamStats{ + Announcements: metricsTracker.GetAnnouncementCount(), + Withdrawals: metricsTracker.GetWithdrawalCount(), + RouteChurnPerSec: metricsTracker.GetChurnRate(), + BGPPeerCount: metricsTracker.GetBGPPeerCount(), + } +} + // handleStats returns a handler that serves API v1 statistics including // detailed handler queue statistics and performance metrics. func (s *Server) handleStats() http.HandlerFunc { - // HandlerStatsInfo represents handler statistics in the API response - type HandlerStatsInfo struct { - Name string `json:"name"` - QueueLength int `json:"queue_length"` - QueueCapacity int `json:"queue_capacity"` - QueueHighWaterMark int `json:"queue_high_water_mark"` - ProcessedCount uint64 `json:"processed_count"` - DroppedCount uint64 `json:"dropped_count"` - AvgProcessTimeMs float64 `json:"avg_process_time_ms"` - MinProcessTimeMs float64 `json:"min_process_time_ms"` - MaxProcessTimeMs float64 `json:"max_process_time_ms"` - } - - // GCStats represents garbage collection statistics - type GCStats struct { - NumGC uint32 `json:"num_gc"` - TotalPauseMs uint64 `json:"total_pause_ms"` - LastPauseMs float64 `json:"last_pause_ms"` - HeapAllocBytes uint64 `json:"heap_alloc_bytes"` - HeapSysBytes uint64 `json:"heap_sys_bytes"` - } - - // StreamStats represents stream statistics including announcements/withdrawals - type StreamStats struct { - Announcements uint64 `json:"announcements"` - Withdrawals uint64 `json:"withdrawals"` - RouteChurnPerSec float64 `json:"route_churn_per_sec"` - BGPPeerCount int `json:"bgp_peer_count"` - } - - // StatsResponse represents the API statistics response - type StatsResponse struct { - Uptime string `json:"uptime"` - TotalMessages uint64 `json:"total_messages"` - TotalBytes uint64 `json:"total_bytes"` - TotalWireBytes uint64 `json:"total_wire_bytes"` - MessagesPerSec float64 `json:"messages_per_sec"` - MbitsPerSec float64 `json:"mbits_per_sec"` - WireMbitsPerSec float64 `json:"wire_mbits_per_sec"` - Connected bool `json:"connected"` - ConnectionDuration string `json:"connection_duration"` - ReconnectCount uint64 `json:"reconnect_count"` - GoVersion string `json:"go_version"` - Goroutines int `json:"goroutines"` - MemoryUsage string `json:"memory_usage"` - GC GCStats `json:"gc"` - Stream StreamStats `json:"stream"` - ASNs int `json:"asns"` - Prefixes int `json:"prefixes"` - IPv4Prefixes int `json:"ipv4_prefixes"` - IPv6Prefixes int `json:"ipv6_prefixes"` - Peerings int `json:"peerings"` - Peers int `json:"peers"` - DatabaseSizeBytes int64 `json:"database_size_bytes"` - LiveRoutes int `json:"live_routes"` - IPv4Routes int `json:"ipv4_routes"` - IPv6Routes int `json:"ipv6_routes"` - OldestRoute *time.Time `json:"oldest_route,omitempty"` - NewestRoute *time.Time `json:"newest_route,omitempty"` - IPv4UpdatesPerSec float64 `json:"ipv4_updates_per_sec"` - IPv6UpdatesPerSec float64 `json:"ipv6_updates_per_sec"` - HandlerStats []HandlerStatsInfo `json:"handler_stats"` - IPv4PrefixDistribution []database.PrefixDistribution `json:"ipv4_prefix_distribution"` - IPv6PrefixDistribution []database.PrefixDistribution `json:"ipv6_prefix_distribution"` - WHOISStats *WHOISStatsInfo `json:"whois_stats,omitempty"` - } - return func(w http.ResponseWriter, r *http.Request) { // Create a 4 second timeout context for this request ctx, cancel := context.WithTimeout(r.Context(), statsContextTimeout) @@ -394,41 +527,19 @@ func (s *Server) handleStats() http.HandlerFunc { metrics := s.streamer.GetMetrics() - // Get database stats with timeout. The channels are buffered so the - // goroutine's send never blocks if the timeout wins and nothing here - // receives; otherwise it would block forever and leak. - statsChan := make(chan database.Stats, 1) - errChan := make(chan error, 1) - - go func() { - dbStats, err := s.db.GetStatsContext(ctx) - if err != nil { - s.logger.Debug("Database stats query failed", "error", err) - errChan <- err - - return - } - statsChan <- dbStats - }() - - var dbStats database.Stats - select { - case <-ctx.Done(): + dbStats, err := s.databaseStats(ctx) + if err != nil && ctx.Err() != nil { s.logger.Error("Database stats timeout") + // Don't write response here - timeout middleware already handles it return - case err := <-errChan: - s.logger.Error("Failed to get database stats", "error", err) - writeJSONError(w, http.StatusInternalServerError, err.Error()) - - return - case dbStats = <-statsChan: - // Success } - uptime := time.Since(metrics.ConnectedSince).Truncate(time.Second).String() - if metrics.ConnectedSince.IsZero() { - uptime = "0s" + if err != nil { + s.logger.Error("Failed to get database stats", "error", err) + s.writeJSONError(w, http.StatusInternalServerError, err.Error()) + + return } const bitsPerMegabit = 1000000.0 @@ -436,86 +547,30 @@ func (s *Server) handleStats() http.HandlerFunc { // Get route update metrics routeMetrics := s.streamer.GetMetricsTracker().GetRouteMetrics() - // Get handler stats - handlerStats := s.streamer.GetHandlerStats() - handlerStatsInfo := make([]HandlerStatsInfo, 0, len(handlerStats)) - const microsecondsPerMillisecond = 1000.0 - for _, hs := range handlerStats { - handlerStatsInfo = append(handlerStatsInfo, HandlerStatsInfo{ - Name: hs.Name, - QueueLength: hs.QueueLength, - QueueCapacity: hs.QueueCapacity, - QueueHighWaterMark: hs.QueueHighWaterMark, - ProcessedCount: hs.ProcessedCount, - DroppedCount: hs.DroppedCount, - AvgProcessTimeMs: float64(hs.AvgProcessTime.Microseconds()) / microsecondsPerMillisecond, - MinProcessTimeMs: float64(hs.MinProcessTime.Microseconds()) / microsecondsPerMillisecond, - MaxProcessTimeMs: float64(hs.MaxProcessTime.Microseconds()) / microsecondsPerMillisecond, - }) - } - // Get memory stats var memStats runtime.MemStats runtime.ReadMemStats(&memStats) // Get WHOIS stats if fetcher is available - var whoisStats *WHOISStatsInfo - if s.asnFetcher != nil { - whoisStats = s.getWHOISStats(ctx) - } - - // Calculate connection duration - connectionDuration := "disconnected" - if metrics.Connected && !metrics.ConnectedSince.IsZero() { - connectionDuration = time.Since(metrics.ConnectedSince).Truncate(time.Second).String() - } - - // Get announcement/withdrawal stats from metrics tracker - metricsTracker := s.streamer.GetMetricsTracker() - announcements := metricsTracker.GetAnnouncementCount() - withdrawals := metricsTracker.GetWithdrawalCount() - churnRate := metricsTracker.GetChurnRate() - bgpPeerCount := metricsTracker.GetBGPPeerCount() - - // Calculate last GC pause - const ( - nanosecondsPerMillisecond = 1e6 - gcPauseHistorySize = 256 // Size of runtime.MemStats.PauseNs circular buffer - ) - var lastPauseMs float64 - if memStats.NumGC > 0 { - // PauseNs is a circular buffer, get the most recent pause - lastPauseIdx := (memStats.NumGC + gcPauseHistorySize - 1) % gcPauseHistorySize - lastPauseMs = float64(memStats.PauseNs[lastPauseIdx]) / nanosecondsPerMillisecond - } + whoisStats := s.getWHOISStats(ctx) stats := StatsResponse{ - Uptime: uptime, - TotalMessages: metrics.TotalMessages, - TotalBytes: metrics.TotalBytes, - TotalWireBytes: metrics.TotalWireBytes, - MessagesPerSec: metrics.MessagesPerSec, - MbitsPerSec: metrics.BitsPerSec / bitsPerMegabit, - WireMbitsPerSec: metrics.WireBitsPerSec / bitsPerMegabit, - Connected: metrics.Connected, - ConnectionDuration: connectionDuration, - ReconnectCount: metrics.ReconnectCount, - GoVersion: runtime.Version(), - Goroutines: runtime.NumGoroutine(), - MemoryUsage: humanize.Bytes(memStats.Alloc), - GC: GCStats{ - NumGC: memStats.NumGC, - TotalPauseMs: memStats.PauseTotalNs / uint64(nanosecondsPerMillisecond), - LastPauseMs: lastPauseMs, - HeapAllocBytes: memStats.HeapAlloc, - HeapSysBytes: memStats.HeapSys, - }, - Stream: StreamStats{ - Announcements: announcements, - Withdrawals: withdrawals, - RouteChurnPerSec: churnRate, - BGPPeerCount: bgpPeerCount, - }, + Uptime: streamUptime(metrics.ConnectedSince), + TotalMessages: metrics.TotalMessages, + TotalBytes: metrics.TotalBytes, + TotalWireBytes: metrics.TotalWireBytes, + MessagesPerSec: metrics.MessagesPerSec, + MbitsPerSec: metrics.BitsPerSec / bitsPerMegabit, + WireMbitsPerSec: metrics.WireBitsPerSec / bitsPerMegabit, + Connected: metrics.Connected, + ConnectionDuration: connectionDuration( + metrics.Connected, metrics.ConnectedSince), + ReconnectCount: metrics.ReconnectCount, + GoVersion: runtime.Version(), + Goroutines: runtime.NumGoroutine(), + MemoryUsage: humanize.Bytes(memStats.Alloc), + GC: gcStats(&memStats), + Stream: s.streamStats(), ASNs: dbStats.ASNs, Prefixes: dbStats.Prefixes, IPv4Prefixes: dbStats.IPv4Prefixes, @@ -530,13 +585,14 @@ func (s *Server) handleStats() http.HandlerFunc { NewestRoute: dbStats.NewestRoute, IPv4UpdatesPerSec: routeMetrics.IPv4UpdatesPerSec, IPv6UpdatesPerSec: routeMetrics.IPv6UpdatesPerSec, - HandlerStats: handlerStatsInfo, + HandlerStats: handlerStatsInfo(s.streamer.GetHandlerStats()), IPv4PrefixDistribution: dbStats.IPv4PrefixDistribution, IPv6PrefixDistribution: dbStats.IPv6PrefixDistribution, WHOISStats: whoisStats, } - if err := writeJSONSuccess(w, stats); err != nil { + err = writeJSONSuccess(w, stats) + if err != nil { s.logger.Error("Failed to encode stats", "error", err) } } @@ -549,7 +605,9 @@ func (s *Server) handleStatusHTML() http.HandlerFunc { w.Header().Set("Content-Type", "text/html; charset=utf-8") tmpl := templates.StatusTemplate() - if err := tmpl.Execute(w, nil); err != nil { + + err := tmpl.Execute(w, nil) + if err != nil { s.logger.Error("Failed to render template", "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) } @@ -579,13 +637,14 @@ func (s *Server) handleIPInfo() http.HandlerFunc { if target == "" { target = chi.URLParam(r, "addr") } + if target == "" { // Use client IP (RealIP middleware has already processed this) target = extractClientIP(r) } if target == "" { - writeJSONError(w, http.StatusBadRequest, "Could not determine IP address") + s.writeJSONError(w, http.StatusBadRequest, "Could not determine IP address") return } @@ -597,33 +656,11 @@ func (s *Server) handleIPInfo() http.HandlerFunc { } // Collect all IPs to look up - var ipsToLookup []string - - // Check if target contains commas (multiple IPs) - targets := strings.Split(target, ",") - for _, t := range targets { - t = strings.TrimSpace(t) - if t == "" { - continue - } - - // Check if this target is an IP address - if parsedIP := net.ParseIP(t); parsedIP != nil { - ipsToLookup = append(ipsToLookup, t) - } else { - // It's a hostname - resolve it - resolved, err := net.DefaultResolver.LookupHost(ctx, t) - if err != nil { - response.Errors = append(response.Errors, t+": "+err.Error()) - - continue - } - ipsToLookup = append(ipsToLookup, resolved...) - } - } + ipsToLookup, resolveErrors := resolveLookupTargets(ctx, target) + response.Errors = append(response.Errors, resolveErrors...) if len(ipsToLookup) == 0 { - writeJSONError(w, http.StatusBadRequest, "No valid IPs or hostnames provided") + s.writeJSONError(w, http.StatusBadRequest, "No valid IPs or hostnames provided") return } @@ -641,12 +678,8 @@ func (s *Server) handleIPInfo() http.HandlerFunc { } // Do PTR lookup for this IP - ptrs, err := net.DefaultResolver.LookupAddr(ctx, ip) - if err == nil && len(ptrs) > 0 { - // Remove trailing dots from PTR records - for i, ptr := range ptrs { - ptrs[i] = strings.TrimSuffix(ptr, ".") - } + ptrs := lookupPTRNames(ctx, ip) + if len(ptrs) > 0 { ipInfo.PTR = ptrs } @@ -666,17 +699,67 @@ func (s *Server) handleIPInfo() http.HandlerFunc { // Return response (even if no results, include errors) if len(response.Results) == 0 && len(response.Errors) > 0 { - writeJSONError(w, http.StatusNotFound, "No routes found: "+response.Errors[0]) + s.writeJSONError(w, http.StatusNotFound, "No routes found: "+response.Errors[0]) return } - if err := writeJSONSuccess(w, response); err != nil { + err := writeJSONSuccess(w, response) + if err != nil { s.logger.Error("Failed to encode IP lookup response", "error", err) } } } +// resolveLookupTargets splits a comma-separated list of IP addresses and +// hostnames and returns the IP addresses to look up, resolving each hostname. +// A hostname that does not resolve is reported in the returned errors instead. +func resolveLookupTargets(ctx context.Context, target string) ([]string, []string) { + var ipsToLookup, resolveErrors []string + + for t := range strings.SplitSeq(target, ",") { + t = strings.TrimSpace(t) + if t == "" { + continue + } + + // Check if this target is an IP address + if net.ParseIP(t) != nil { + ipsToLookup = append(ipsToLookup, t) + + continue + } + + // It's a hostname - resolve it + resolved, err := net.DefaultResolver.LookupHost(ctx, t) + if err != nil { + resolveErrors = append(resolveErrors, t+": "+err.Error()) + + continue + } + + ipsToLookup = append(ipsToLookup, resolved...) + } + + return ipsToLookup, resolveErrors +} + +// lookupPTRNames returns the PTR records of ip without their trailing dots, or +// nil when the lookup fails. +func lookupPTRNames(ctx context.Context, ip string) []string { + ptrs, err := net.DefaultResolver.LookupAddr(ctx, ip) + if err != nil { + return nil + } + + // Remove trailing dots from PTR records + for i, ptr := range ptrs { + ptrs[i] = strings.TrimSuffix(ptr, ".") + } + + return ptrs +} + // extractClientIP extracts the client IP from the request. // Works with chi's RealIP middleware which sets RemoteAddr. func extractClientIP(r *http.Request) string { @@ -694,9 +777,10 @@ func extractClientIP(r *http.Request) string { func (s *Server) handleASDetailJSON() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { asnStr := chi.URLParam(r, "asn") + asn, err := strconv.Atoi(asnStr) if err != nil { - writeJSONError(w, http.StatusBadRequest, "Invalid ASN") + s.writeJSONError(w, http.StatusBadRequest, "Invalid ASN") return } @@ -704,38 +788,69 @@ func (s *Server) handleASDetailJSON() http.HandlerFunc { asInfo, prefixes, err := s.db.GetASDetailsContext(r.Context(), asn) if err != nil { if errors.Is(err, database.ErrNoRoute) { - writeJSONError(w, http.StatusNotFound, err.Error()) + s.writeJSONError(w, http.StatusNotFound, err.Error()) } else { - writeJSONError(w, http.StatusInternalServerError, err.Error()) + s.writeJSONError(w, http.StatusInternalServerError, err.Error()) } return } // Group prefixes by IP version - const ipVersionV4 = 4 - var ipv4Prefixes, ipv6Prefixes []database.LiveRoute - for _, p := range prefixes { - if p.IPVersion == ipVersionV4 { - ipv4Prefixes = append(ipv4Prefixes, p) - } else { - ipv6Prefixes = append(ipv6Prefixes, p) - } - } + ipv4Prefixes, ipv6Prefixes := splitRoutesByIPVersion(prefixes) - response := map[string]interface{}{ + response := map[string]any{ "asn": asInfo, "ipv4_prefixes": ipv4Prefixes, "ipv6_prefixes": ipv6Prefixes, "total_count": len(prefixes), } - if err := writeJSONSuccess(w, response); err != nil { + err = writeJSONSuccess(w, response) + if err != nil { s.logger.Error("Failed to encode AS details", "error", err) } } } +// splitRoutesByIPVersion groups routes into the IPv4 and the IPv6 ones. +func splitRoutesByIPVersion( + routes []database.LiveRoute, +) ([]database.LiveRoute, []database.LiveRoute) { + var ipv4Routes, ipv6Routes []database.LiveRoute + + for _, route := range routes { + if route.IPVersion == ipVersionV4 { + ipv4Routes = append(ipv4Routes, route) + } else { + ipv6Routes = append(ipv6Routes, route) + } + } + + return ipv4Routes, ipv6Routes +} + +// sortRoutesByNetwork sorts routes by network address, then by mask length. +func sortRoutesByNetwork(routes []database.LiveRoute) { + sort.Slice(routes, func(i, j int) bool { + // Parse the prefixes to compare network addresses + ipI, netI, _ := net.ParseCIDR(routes[i].Prefix) + ipJ, netJ, _ := net.ParseCIDR(routes[j].Prefix) + + // Compare by network address first + cmp := bytes.Compare(ipI.To16(), ipJ.To16()) + if cmp != 0 { + return cmp < 0 + } + + // If network addresses are equal, compare by mask length + onesI, _ := netI.Mask.Size() + onesJ, _ := netJ.Mask.Size() + + return onesI < onesJ + }) +} + // handlePrefixDetailJSON returns prefix details as JSON func (s *Server) handlePrefixDetailJSON() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { @@ -744,7 +859,8 @@ func (s *Server) handlePrefixDetailJSON() http.HandlerFunc { lenParam := chi.URLParam(r, "len") if prefixParam == "" || lenParam == "" { - writeJSONError(w, http.StatusBadRequest, "Prefix and length parameters are required") + s.writeJSONError(w, http.StatusBadRequest, + "Prefix and length parameters are required") return } @@ -753,18 +869,19 @@ func (s *Server) handlePrefixDetailJSON() http.HandlerFunc { // tables store, so that any valid way of writing it finds the route. parsed, err := netip.ParsePrefix(prefixParam + "/" + lenParam) if err != nil { - writeJSONError(w, http.StatusBadRequest, err.Error()) + s.writeJSONError(w, http.StatusBadRequest, err.Error()) return } + prefix := parsed.Masked().String() routes, _, err := s.db.GetPrefixDetailsContext(r.Context(), prefix) if err != nil { if errors.Is(err, database.ErrNoRoute) { - writeJSONError(w, http.StatusNotFound, err.Error()) + s.writeJSONError(w, http.StatusNotFound, err.Error()) } else { - writeJSONError(w, http.StatusInternalServerError, err.Error()) + s.writeJSONError(w, http.StatusInternalServerError, err.Error()) } return @@ -776,7 +893,7 @@ func (s *Server) handlePrefixDetailJSON() http.HandlerFunc { originMap[route.OriginASN] = append(originMap[route.OriginASN], route) } - response := map[string]interface{}{ + response := map[string]any{ "prefix": prefix, "routes": routes, "origins": originMap, @@ -784,7 +901,8 @@ func (s *Server) handlePrefixDetailJSON() http.HandlerFunc { "origin_count": len(originMap), } - if err := writeJSONSuccess(w, response); err != nil { + err = writeJSONSuccess(w, response) + if err != nil { s.logger.Error("Failed to encode prefix details", "error", err) } } @@ -794,6 +912,7 @@ func (s *Server) handlePrefixDetailJSON() http.HandlerFunc { func (s *Server) handleASDetail() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { asnStr := chi.URLParam(r, "asn") + asn, err := strconv.Atoi(asnStr) if err != nil { http.Error(w, "Invalid ASN", http.StatusBadRequest) @@ -822,52 +941,11 @@ func (s *Server) handleASDetail() http.HandlerFunc { } // Group prefixes by IP version - const ipVersionV4 = 4 - var ipv4Prefixes, ipv6Prefixes []database.LiveRoute - for _, p := range prefixes { - if p.IPVersion == ipVersionV4 { - ipv4Prefixes = append(ipv4Prefixes, p) - } else { - ipv6Prefixes = append(ipv6Prefixes, p) - } - } + ipv4Prefixes, ipv6Prefixes := splitRoutesByIPVersion(prefixes) // Sort prefixes by network address - sort.Slice(ipv4Prefixes, func(i, j int) bool { - // Parse the prefixes to compare network addresses - ipI, netI, _ := net.ParseCIDR(ipv4Prefixes[i].Prefix) - ipJ, netJ, _ := net.ParseCIDR(ipv4Prefixes[j].Prefix) - - // Compare by network address first - cmp := bytes.Compare(ipI.To4(), ipJ.To4()) - if cmp != 0 { - return cmp < 0 - } - - // If network addresses are equal, compare by mask length - onesI, _ := netI.Mask.Size() - onesJ, _ := netJ.Mask.Size() - - return onesI < onesJ - }) - - sort.Slice(ipv6Prefixes, func(i, j int) bool { - // Parse the prefixes to compare network addresses - ipI, netI, _ := net.ParseCIDR(ipv6Prefixes[i].Prefix) - ipJ, netJ, _ := net.ParseCIDR(ipv6Prefixes[j].Prefix) - - // Compare by network address first - cmp := bytes.Compare(ipI.To16(), ipJ.To16()) - if cmp != 0 { - return cmp < 0 - } - - // If network addresses are equal, compare by mask length - onesI, _ := netI.Mask.Size() - onesJ, _ := netJ.Mask.Size() - - return onesI < onesJ - }) + sortRoutesByNetwork(ipv4Prefixes) + sortRoutesByNetwork(ipv6Prefixes) // Prepare template data data := struct { @@ -899,8 +977,11 @@ func (s *Server) handleASDetail() http.HandlerFunc { } w.Header().Set("Content-Type", "text/html; charset=utf-8") + tmpl := templates.ASDetailTemplate() - if err := tmpl.Execute(w, data); err != nil { + + err = tmpl.Execute(w, data) + if err != nil { s.logger.Error("Failed to render AS detail template", "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) } @@ -928,9 +1009,10 @@ func (s *Server) handlePrefixDetail() http.HandlerFunc { return } + prefix := parsed.Masked().String() - routes, originASNs, err := s.db.GetPrefixDetailsContext(r.Context(), prefix) + routes, originNames, err := s.db.GetPrefixDetailsContext(r.Context(), prefix) if err != nil { if errors.Is(err, database.ErrNoRoute) { http.Error(w, "Prefix not found", http.StatusNotFound) @@ -943,39 +1025,12 @@ func (s *Server) handlePrefixDetail() http.HandlerFunc { } // Group by origin AS and collect unique AS info - type ASNInfo struct { - ASN int - Handle string - Description string - PeerCount int - } - originMap := make(map[int]*ASNInfo) - for _, route := range routes { - if _, exists := originMap[route.OriginASN]; !exists { - originMap[route.OriginASN] = &ASNInfo{ - ASN: route.OriginASN, - Handle: originASNs[route.OriginASN].Handle, - Description: originASNs[route.OriginASN].Description, - PeerCount: 0, - } - } - originMap[route.OriginASN].PeerCount++ - } + originMap := originASNs(routes, originNames) // Get the first route to extract some common info var maskLength, ipVersion int if len(routes) > 0 { - // Parse CIDR to get mask length and IP version - _, ipNet, err := net.ParseCIDR(prefix) - if err == nil { - ones, _ := ipNet.Mask.Size() - maskLength = ones - if ipNet.IP.To4() != nil { - ipVersion = 4 - } else { - ipVersion = 6 - } - } + maskLength, ipVersion = prefixMaskLengthAndIPVersion(prefix) } // Convert origin map to sorted slice @@ -985,33 +1040,7 @@ func (s *Server) handlePrefixDetail() http.HandlerFunc { } // Create enhanced routes with AS path handles - type ASPathEntry struct { - ASN int - Handle string - } - type EnhancedRoute struct { - database.LiveRoute - ASPathWithHandle []ASPathEntry - } - - enhancedRoutes := make([]EnhancedRoute, len(routes)) - for i, route := range routes { - enhancedRoute := EnhancedRoute{ - LiveRoute: route, - ASPathWithHandle: make([]ASPathEntry, len(route.ASPath)), - } - - // Look up handle for each AS in the path - for j, asn := range route.ASPath { - handle := asinfo.GetHandle(asn) - enhancedRoute.ASPathWithHandle[j] = ASPathEntry{ - ASN: asn, - Handle: handle, - } - } - - enhancedRoutes[i] = enhancedRoute - } + enhancedRoutes := routesWithASPathHandles(routes) // Prepare template data data := struct { @@ -1041,16 +1070,118 @@ func (s *Server) handlePrefixDetail() http.HandlerFunc { } w.Header().Set("Content-Type", "text/html; charset=utf-8") + tmpl := templates.PrefixDetailTemplate() - if err := tmpl.Execute(w, data); err != nil { + + err = tmpl.Execute(w, data) + if err != nil { s.logger.Error("Failed to render prefix detail template", "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) } } } -// handlePrefixLength shows a random sample of IPv4 prefixes with the specified mask length +// ASNInfo is one origin AS of a prefix on the prefix detail page. +type ASNInfo struct { + ASN int + Handle string + Description string + PeerCount int +} + +// originASNs groups routes by origin AS, with each AS's handle and description +// from names, as the prefix query returned them, and the number of routes it +// originates. +func originASNs( + routes []database.LiveRoute, + names map[int]database.ASN, +) map[int]*ASNInfo { + originMap := make(map[int]*ASNInfo) + + for _, route := range routes { + if _, exists := originMap[route.OriginASN]; !exists { + originMap[route.OriginASN] = &ASNInfo{ + ASN: route.OriginASN, + Handle: names[route.OriginASN].Handle, + Description: names[route.OriginASN].Description, + PeerCount: 0, + } + } + + originMap[route.OriginASN].PeerCount++ + } + + return originMap +} + +// ASPathEntry is one AS of a route's AS path, with its handle. +type ASPathEntry struct { + ASN int + Handle string +} + +// EnhancedRoute is a live route with the handle of each AS in its path. +type EnhancedRoute struct { + database.LiveRoute + + ASPathWithHandle []ASPathEntry +} + +// routesWithASPathHandles looks up the handle of each AS in the path of each +// route. +func routesWithASPathHandles(routes []database.LiveRoute) []EnhancedRoute { + enhancedRoutes := make([]EnhancedRoute, len(routes)) + + for i, route := range routes { + enhancedRoute := EnhancedRoute{ + LiveRoute: route, + ASPathWithHandle: make([]ASPathEntry, len(route.ASPath)), + } + + // Look up handle for each AS in the path + for j, asn := range route.ASPath { + handle := asinfo.GetHandle(asn) + enhancedRoute.ASPathWithHandle[j] = ASPathEntry{ + ASN: asn, + Handle: handle, + } + } + + enhancedRoutes[i] = enhancedRoute + } + + return enhancedRoutes +} + +// prefixMaskLengthAndIPVersion returns the mask length and IP version of a +// prefix in CIDR notation, or zeros if it does not parse. +func prefixMaskLengthAndIPVersion(prefix string) (int, int) { + _, ipNet, err := net.ParseCIDR(prefix) + if err != nil { + return 0, 0 + } + + maskLength, _ := ipNet.Mask.Size() + + if ipNet.IP.To4() != nil { + return maskLength, ipVersionV4 + } + + return maskLength, ipVersionV6 +} + +// handlePrefixLength shows a random sample of IPv4 prefixes with the +// specified mask length func (s *Server) handlePrefixLength() http.HandlerFunc { + const maxIPv4MaskLength = 32 + + return s.prefixLengthHandler(ipVersionV4, maxIPv4MaskLength) +} + +// prefixLengthHandler returns a handler that shows a random sample of the +// prefixes of one IP version with the specified mask length, which may be at +// most maxMaskLength. +func (s *Server) prefixLengthHandler(ipVersion, maxMaskLength int) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { lengthStr := chi.URLParam(r, "length") if lengthStr == "" { @@ -1066,19 +1197,19 @@ func (s *Server) handlePrefixLength() http.HandlerFunc { return } - // Validate IPv4 mask length - const maxIPv4MaskLength = 32 - if maskLength < 0 || maskLength > maxIPv4MaskLength { - http.Error(w, "Invalid IPv4 mask length", http.StatusBadRequest) + // Validate mask length + if maskLength < 0 || maskLength > maxMaskLength { + http.Error(w, "Invalid IPv"+strconv.Itoa(ipVersion)+" mask length", + http.StatusBadRequest) return } - const ipVersion = 4 - // Get random sample of prefixes const maxPrefixes = 500 - prefixes, err := s.db.GetRandomPrefixesByLengthContext(r.Context(), maskLength, ipVersion, maxPrefixes) + + prefixes, err := s.db.GetRandomPrefixesByLengthContext( + r.Context(), maskLength, ipVersion, maxPrefixes) if err != nil { s.logger.Error("Failed to get prefixes by length", "error", err) http.Error(w, "Internal server error", http.StatusInternalServerError) @@ -1099,6 +1230,7 @@ func (s *Server) handlePrefixLength() http.HandlerFunc { // Create enhanced prefixes with AS descriptions type EnhancedPrefix struct { database.LiveRoute + OriginASDescription string Age string } @@ -1117,7 +1249,7 @@ func (s *Server) handlePrefixLength() http.HandlerFunc { } // Render template - data := map[string]interface{}{ + data := map[string]any{ "MaskLength": maskLength, "IPVersion": ipVersion, "Prefixes": enhancedPrefixes, @@ -1133,102 +1265,21 @@ func (s *Server) handlePrefixLength() http.HandlerFunc { } tmpl := templates.PrefixLengthTemplate() - if err := tmpl.Execute(w, data); err != nil { + + err = tmpl.Execute(w, data) + if err != nil { s.logger.Error("Failed to render prefix length template", "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) } } } -// handlePrefixLength6 shows a random sample of IPv6 prefixes with the specified mask length +// handlePrefixLength6 shows a random sample of IPv6 prefixes with the +// specified mask length func (s *Server) handlePrefixLength6() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - lengthStr := chi.URLParam(r, "length") - if lengthStr == "" { - http.Error(w, "Length parameter is required", http.StatusBadRequest) + const maxIPv6MaskLength = 128 - return - } - - maskLength, err := strconv.Atoi(lengthStr) - if err != nil { - http.Error(w, "Invalid mask length", http.StatusBadRequest) - - return - } - - // Validate IPv6 mask length - const maxIPv6MaskLength = 128 - if maskLength < 0 || maskLength > maxIPv6MaskLength { - http.Error(w, "Invalid IPv6 mask length", http.StatusBadRequest) - - return - } - - const ipVersion = 6 - - // Get random sample of prefixes - const maxPrefixes = 500 - prefixes, err := s.db.GetRandomPrefixesByLengthContext(r.Context(), maskLength, ipVersion, maxPrefixes) - if err != nil { - s.logger.Error("Failed to get prefixes by length", "error", err) - http.Error(w, "Internal server error", http.StatusInternalServerError) - - return - } - - // Sort prefixes for display - sort.Slice(prefixes, func(i, j int) bool { - // First compare by IP version - if prefixes[i].IPVersion != prefixes[j].IPVersion { - return prefixes[i].IPVersion < prefixes[j].IPVersion - } - // Then by prefix - return prefixes[i].Prefix < prefixes[j].Prefix - }) - - // Create enhanced prefixes with AS descriptions - type EnhancedPrefix struct { - database.LiveRoute - OriginASDescription string - Age string - } - - enhancedPrefixes := make([]EnhancedPrefix, len(prefixes)) - for i, prefix := range prefixes { - enhancedPrefixes[i] = EnhancedPrefix{ - LiveRoute: prefix, - Age: formatAge(prefix.LastUpdated), - } - - // Get AS description - if asInfo, ok := asinfo.Get(prefix.OriginASN); ok { - enhancedPrefixes[i].OriginASDescription = asInfo.Description - } - } - - // Render template - data := map[string]interface{}{ - "MaskLength": maskLength, - "IPVersion": ipVersion, - "Prefixes": enhancedPrefixes, - "Count": len(prefixes), - } - - // Check if context is still valid before writing response - select { - case <-r.Context().Done(): - // Request was cancelled, don't write response - return - default: - } - - tmpl := templates.PrefixLengthTemplate() - if err := tmpl.Execute(w, data); err != nil { - s.logger.Error("Failed to render prefix length template", "error", err) - http.Error(w, "Internal Server Error", http.StatusInternalServerError) - } - } + return s.prefixLengthHandler(ipVersionV6, maxIPv6MaskLength) } // formatAge returns a human-readable age string @@ -1237,13 +1288,14 @@ func formatAge(timestamp time.Time) string { const hoursPerDay = 24 - if age < time.Minute { + switch { + case age < time.Minute: return "< 1m" - } else if age < time.Hour { + case age < time.Hour: minutes := int(age.Minutes()) return strconv.Itoa(minutes) + "m" - } else if age < hoursPerDay*time.Hour { + case age < hoursPerDay*time.Hour: hours := int(age.Hours()) return strconv.Itoa(hours) + "h" diff --git a/internal/server/handlers_test.go b/internal/server/handlers_internal_test.go similarity index 68% rename from internal/server/handlers_test.go rename to internal/server/handlers_internal_test.go index 3821654..a22fba0 100644 --- a/internal/server/handlers_test.go +++ b/internal/server/handlers_internal_test.go @@ -22,12 +22,20 @@ import ( "github.com/google/uuid" ) +// The prefix and peer addresses the prefix page tests store routes for. +const ( + testPrefixV4 = "192.0.2.0/24" + testPeerV4 = "198.51.100.1" + testPeerV6 = "2001:db8::1" +) + // blockingStatsDB embeds database.Store (left nil) and overrides only // GetStatsContext, which blocks until release is closed. The stats handlers // call it in a goroutine; every other Store method is unused on the timeout // path and would panic if called. type blockingStatsDB struct { database.Store + release chan struct{} } @@ -43,6 +51,8 @@ func (d blockingStatsDB) GetStatsContext(_ context.Context) (database.Stats, err // value. Before the fix the per-request goroutine sent on an unbuffered channel // that nothing received once the timeout won, so it blocked forever and every // poll leaked one goroutine. +// +//nolint:paralleltest // counts goroutines, so no other test may run alongside func TestStatsHandlersDoNotLeakOnTimeout(t *testing.T) { release := make(chan struct{}) db := blockingStatsDB{release: release} @@ -62,7 +72,7 @@ func TestStatsHandlersDoNotLeakOnTimeout(t *testing.T) { for _, handler := range handlers { for range iterations { ctx, cancel := context.WithTimeout(context.Background(), requestTimeout) - req := httptest.NewRequest(http.MethodGet, "/", nil).WithContext(ctx) + req := httptest.NewRequestWithContext(ctx, http.MethodGet, "/", nil) handler(httptest.NewRecorder(), req) cancel() } @@ -87,27 +97,33 @@ func TestStatsHandlersDoNotLeakOnTimeout(t *testing.T) { // The oldest and newest route times still come from one-row lookups at the ends // of an index; with the database closed they are left out of the answer. func TestStatsHandlersAnswerFromMemory(t *testing.T) { + t.Parallel() + db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New()) if err != nil { t.Fatalf("database.New: %v", err) } ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) - if err := db.UpsertLiveRouteBatch([]*database.LiveRoute{ + + err = db.UpsertLiveRouteBatch([]*database.LiveRoute{ { ID: uuid.New(), Prefix: "198.51.100.0/24", MaskLength: 24, IPVersion: 4, - OriginASN: 64500, PeerIP: "192.0.2.1", ASPath: []int{64500}, NextHop: "192.0.2.1", - LastUpdated: ts, + OriginASN: 64500, PeerIP: "192.0.2.1", ASPath: []int{64500}, + NextHop: "192.0.2.1", LastUpdated: ts, }, { ID: uuid.New(), Prefix: "2001:db8::/32", MaskLength: 32, IPVersion: 6, - OriginASN: 64501, PeerIP: "2001:db8::1", ASPath: []int{64501}, NextHop: "2001:db8::1", - LastUpdated: ts, + OriginASN: 64501, PeerIP: testPeerV6, ASPath: []int{64501}, + NextHop: testPeerV6, LastUpdated: ts, }, - }); err != nil { + }) + if err != nil { t.Fatalf("UpsertLiveRouteBatch: %v", err) } - if err := db.Close(); err != nil { + + err = db.Close() + if err != nil { t.Fatalf("Close: %v", err) } @@ -119,35 +135,45 @@ func TestStatsHandlersAnswerFromMemory(t *testing.T) { for name, handler := range handlers { rec := httptest.NewRecorder() - handler(rec, httptest.NewRequest(http.MethodGet, "/", nil)) + handler(rec, httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil)) + if rec.Code != http.StatusOK { - t.Errorf("%s: status %d, want %d; body %s", name, rec.Code, http.StatusOK, rec.Body) + t.Errorf("%s: status %d, want %d; body %s", + name, rec.Code, http.StatusOK, rec.Body) continue } + //nolint:tagliatelle // field names are the existing JSON API var body struct { Data struct { - IPv4Routes int `json:"ipv4_routes"` - IPv6Routes int `json:"ipv6_routes"` - IPv4PrefixDistribution []database.PrefixDistribution `json:"ipv4_prefix_distribution"` - IPv6PrefixDistribution []database.PrefixDistribution `json:"ipv6_prefix_distribution"` + IPv4Routes int `json:"ipv4_routes"` + IPv6Routes int `json:"ipv6_routes"` + IPv4Distribution []database.PrefixDistribution `json:"ipv4_prefix_distribution"` + IPv6Distribution []database.PrefixDistribution `json:"ipv6_prefix_distribution"` } `json:"data"` } - if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + + err := json.Unmarshal(rec.Body.Bytes(), &body) + if err != nil { t.Fatalf("%s: decoding the answer: %v", name, err) } if body.Data.IPv4Routes != 1 || body.Data.IPv6Routes != 1 { - t.Errorf("%s: routes = (v4 %d, v6 %d), want (1, 1)", name, body.Data.IPv4Routes, body.Data.IPv6Routes) + t.Errorf("%s: routes = (v4 %d, v6 %d), want (1, 1)", + name, body.Data.IPv4Routes, body.Data.IPv6Routes) } + wantV4 := []database.PrefixDistribution{{MaskLength: 24, Count: 1}} - if !slices.Equal(body.Data.IPv4PrefixDistribution, wantV4) { - t.Errorf("%s: IPv4 distribution = %v, want %v", name, body.Data.IPv4PrefixDistribution, wantV4) + if !slices.Equal(body.Data.IPv4Distribution, wantV4) { + t.Errorf("%s: IPv4 distribution = %v, want %v", + name, body.Data.IPv4Distribution, wantV4) } + wantV6 := []database.PrefixDistribution{{MaskLength: 32, Count: 1}} - if !slices.Equal(body.Data.IPv6PrefixDistribution, wantV6) { - t.Errorf("%s: IPv6 distribution = %v, want %v", name, body.Data.IPv6PrefixDistribution, wantV6) + if !slices.Equal(body.Data.IPv6Distribution, wantV6) { + t.Errorf("%s: IPv6 distribution = %v, want %v", + name, body.Data.IPv6Distribution, wantV6) } } } @@ -159,25 +185,30 @@ func TestStatsHandlersAnswerFromMemory(t *testing.T) { // "2001:DB8:0:0::/32" was not found although "2001:db8::/32" was stored, and // "foo/24" answered 500 (https://git.eeqj.de/sneak/routewatch/issues/71). func TestPrefixDetailHandlersParsePrefix(t *testing.T) { + t.Parallel() + db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New()) if err != nil { t.Fatalf("database.New: %v", err) } + t.Cleanup(func() { _ = db.Close() }) ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) - if err := db.UpsertLiveRouteBatch([]*database.LiveRoute{ + + err = db.UpsertLiveRouteBatch([]*database.LiveRoute{ { - ID: uuid.New(), Prefix: "192.0.2.0/24", MaskLength: 24, IPVersion: 4, - OriginASN: 64500, PeerIP: "198.51.100.1", ASPath: []int{64500}, NextHop: "198.51.100.1", - LastUpdated: ts, + ID: uuid.New(), Prefix: testPrefixV4, MaskLength: 24, IPVersion: 4, + OriginASN: 64500, PeerIP: testPeerV4, ASPath: []int{64500}, + NextHop: testPeerV4, LastUpdated: ts, }, { ID: uuid.New(), Prefix: "2001:db8::/32", MaskLength: 32, IPVersion: 6, - OriginASN: 64501, PeerIP: "2001:db8::1", ASPath: []int{64501}, NextHop: "2001:db8::1", - LastUpdated: ts, + OriginASN: 64501, PeerIP: testPeerV6, ASPath: []int{64501}, + NextHop: testPeerV6, LastUpdated: ts, }, - }); err != nil { + }) + if err != nil { t.Fatalf("UpsertLiveRouteBatch: %v", err) } @@ -197,20 +228,29 @@ func TestPrefixDetailHandlersParsePrefix(t *testing.T) { for _, tt := range tests { rec := httptest.NewRecorder() - s.router.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, tt.path, nil)) + s.router.ServeHTTP(rec, httptest.NewRequestWithContext( + t.Context(), http.MethodGet, tt.path, nil)) + if rec.Code != tt.want { - t.Errorf("%s: status %d, want %d; body %s", tt.path, rec.Code, tt.want, rec.Body) + t.Errorf("%s: status %d, want %d; body %s", + tt.path, rec.Code, tt.want, rec.Body) } } } +// errASDetailsLookupFailed is what failingASDetailsDB answers every AS details +// lookup with. +var errASDetailsLookupFailed = errors.New("AS details lookup failed") + // failingASDetailsDB wraps a real store and fails every AS details lookup. type failingASDetailsDB struct { database.Store } -func (failingASDetailsDB) GetASDetailsContext(_ context.Context, _ int) (*database.ASN, []database.LiveRoute, error) { - return nil, nil, errors.New("AS details lookup failed") +func (failingASDetailsDB) GetASDetailsContext( + _ context.Context, _ int, +) (*database.ASN, []database.LiveRoute, error) { + return nil, nil, errASDetailsLookupFailed } // TestPrefixDetailShowsOriginNames checks that the prefix page shows each @@ -220,51 +260,66 @@ func (failingASDetailsDB) GetASDetailsContext(_ context.Context, _ int) (*databa // (https://git.eeqj.de/sneak/routewatch/issues/67). The AS path column shows // handles too, so the test looks for the origin list's "(handle)" form. func TestPrefixDetailShowsOriginNames(t *testing.T) { + t.Parallel() + db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New()) if err != nil { t.Fatalf("database.New: %v", err) } + t.Cleanup(func() { _ = db.Close() }) ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) origins := []int{15169, 13335} + for _, asn := range origins { - if _, err := db.GetOrCreateASN(asn, ts); err != nil { + _, err := db.GetOrCreateASN(asn, ts) + if err != nil { t.Fatalf("GetOrCreateASN(%d): %v", asn, err) } } - if err := db.UpsertLiveRouteBatch([]*database.LiveRoute{ + + err = db.UpsertLiveRouteBatch([]*database.LiveRoute{ { - ID: uuid.New(), Prefix: "192.0.2.0/24", MaskLength: 24, IPVersion: 4, - OriginASN: origins[0], PeerIP: "198.51.100.1", ASPath: []int{origins[0]}, NextHop: "198.51.100.1", - LastUpdated: ts, + ID: uuid.New(), Prefix: testPrefixV4, MaskLength: 24, IPVersion: 4, + OriginASN: origins[0], PeerIP: testPeerV4, + ASPath: []int{origins[0]}, NextHop: testPeerV4, LastUpdated: ts, }, { - ID: uuid.New(), Prefix: "192.0.2.0/24", MaskLength: 24, IPVersion: 4, - OriginASN: origins[1], PeerIP: "198.51.100.2", ASPath: []int{origins[1]}, NextHop: "198.51.100.2", - LastUpdated: ts, + ID: uuid.New(), Prefix: testPrefixV4, MaskLength: 24, IPVersion: 4, + OriginASN: origins[1], PeerIP: "198.51.100.2", + ASPath: []int{origins[1]}, NextHop: "198.51.100.2", LastUpdated: ts, }, - }); err != nil { + }) + if err != nil { t.Fatalf("UpsertLiveRouteBatch: %v", err) } - s := New(failingASDetailsDB{db}, streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{}) + s := New(failingASDetailsDB{db}, streamer.New(logger.New(), metrics.New()), + logger.New(), &config.Config{}) rec := httptest.NewRecorder() - s.router.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/prefix/192.0.2.0/24", nil)) + s.router.ServeHTTP(rec, httptest.NewRequestWithContext( + t.Context(), http.MethodGet, "/prefix/"+testPrefixV4, nil)) + if rec.Code != http.StatusOK { t.Fatalf("status %d, want %d; body %s", rec.Code, http.StatusOK, rec.Body) } page := rec.Body.String() + for _, asn := range origins { info, ok := asinfo.Get(asn) if !ok { t.Fatalf("AS%d is not in the AS info data", asn) } - if want := "(" + html.EscapeString(info.Handle) + ")"; !strings.Contains(page, want) { + + want := "(" + html.EscapeString(info.Handle) + ")" + if !strings.Contains(page, want) { t.Errorf("page does not show AS%d's handle %q", asn, want) } - if want := html.EscapeString(info.Description); !strings.Contains(page, want) { + + want = html.EscapeString(info.Description) + if !strings.Contains(page, want) { t.Errorf("page does not show AS%d's description %q", asn, want) } } @@ -274,12 +329,15 @@ func TestPrefixDetailShowsOriginNames(t *testing.T) { // current count. func settledGoroutineCount() int { prev := runtime.NumGoroutine() + for range 20 { time.Sleep(10 * time.Millisecond) + cur := runtime.NumGoroutine() if cur == prev { return cur } + prev = cur } @@ -292,6 +350,7 @@ func waitForGoroutines(target int) bool { if runtime.NumGoroutine() <= target { return true } + time.Sleep(10 * time.Millisecond) } diff --git a/internal/server/middleware.go b/internal/server/middleware.go index dd61ee8..43e5ee9 100644 --- a/internal/server/middleware.go +++ b/internal/server/middleware.go @@ -14,6 +14,7 @@ import ( // responseWriter wraps http.ResponseWriter to capture the response type responseWriter struct { http.ResponseWriter + body *bytes.Buffer statusCode int written bool @@ -86,8 +87,10 @@ func JSONResponseMiddleware(next http.Handler) http.Handler { } // Parse the original response - var originalResponse map[string]interface{} - if err := json.Unmarshal(rw.body.Bytes(), &originalResponse); err != nil { + var originalResponse map[string]any + + err := json.Unmarshal(rw.body.Bytes(), &originalResponse) + if err != nil { // If we can't parse it, just write original w.WriteHeader(rw.statusCode) _, _ = w.Write(rw.body.Bytes()) @@ -96,7 +99,7 @@ func JSONResponseMiddleware(next http.Handler) http.Handler { } // Add @meta field - originalResponse["@meta"] = map[string]interface{}{ + originalResponse["@meta"] = map[string]any{ "time_zone": "UTC", "api_version": 1, "execution_time": fmt.Sprintf("%d ms", execTime.Milliseconds()), @@ -112,6 +115,7 @@ func JSONResponseMiddleware(next http.Handler) http.Handler { // timeoutWriter wraps ResponseWriter to prevent concurrent writes after timeout type timeoutWriter struct { http.ResponseWriter + mu sync.Mutex written bool header http.Header // cached header to prevent concurrent access @@ -158,6 +162,7 @@ func (tw *timeoutWriter) Header() http.Header { func (tw *timeoutWriter) markWritten() { tw.mu.Lock() defer tw.mu.Unlock() + tw.written = true } @@ -168,7 +173,10 @@ func (tw *timeoutWriter) markWritten() { // The timeout parameter specifies the maximum duration allowed for request processing. // The returned middleware handles panics from the wrapped handler by re-panicking // after cleanup, and prevents concurrent writes to the response after timeout occurs. -func TimeoutMiddleware(timeout time.Duration, logger *slog.Logger) func(http.Handler) http.Handler { +func TimeoutMiddleware( + timeout time.Duration, + logger *slog.Logger, +) func(http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { startTime := time.Now() @@ -182,7 +190,7 @@ func TimeoutMiddleware(timeout time.Duration, logger *slog.Logger) func(http.Han } done := make(chan struct{}) - panicChan := make(chan interface{}, 1) + panicChan := make(chan any, 1) go func() { defer func() { @@ -202,6 +210,7 @@ func TimeoutMiddleware(timeout time.Duration, logger *slog.Logger) func(http.Han return case <-ctx.Done(): tw.markWritten() // Prevent the handler from writing after timeout + execTime := time.Since(startTime) // Log the timeout as a warning @@ -219,13 +228,13 @@ func TimeoutMiddleware(timeout time.Duration, logger *slog.Logger) func(http.Han w.WriteHeader(http.StatusRequestTimeout) tw.mu.Unlock() - response := map[string]interface{}{ - "status": "error", - "error": map[string]interface{}{ - "msg": "Request timeout", - "code": http.StatusRequestTimeout, + response := map[string]any{ + jsonKeyStatus: jsonStatusError, + jsonKeyError: map[string]any{ + jsonKeyMsg: "Request timeout", + jsonKeyCode: http.StatusRequestTimeout, }, - "@meta": map[string]interface{}{ + "@meta": map[string]any{ "time_zone": "UTC", "api_version": 1, "execution_time": fmt.Sprintf("%d ms", execTime.Milliseconds()), @@ -273,8 +282,10 @@ func JSONValidationMiddleware(next http.Handler) http.Handler { // For JSON responses, validate the JSON if rw.body.Len() > 0 { - var testParse interface{} - if err := json.Unmarshal(rw.body.Bytes(), &testParse); err == nil { + var testParse any + + err := json.Unmarshal(rw.body.Bytes(), &testParse) + if err == nil { // Valid JSON, write it out w.Header().Set("Content-Type", "application/json") w.WriteHeader(rw.statusCode) @@ -293,6 +304,7 @@ func JSONValidationMiddleware(next http.Handler) http.Handler { if statusCode == http.StatusOK { statusCode = http.StatusInternalServerError } + w.WriteHeader(statusCode) errorMsg := "Internal server error" @@ -300,11 +312,11 @@ func JSONValidationMiddleware(next http.Handler) http.Handler { errorMsg = "Request timeout" } - response := map[string]interface{}{ - "status": "error", - "error": map[string]interface{}{ - "msg": errorMsg, - "code": statusCode, + response := map[string]any{ + jsonKeyStatus: jsonStatusError, + jsonKeyError: map[string]any{ + jsonKeyMsg: errorMsg, + jsonKeyCode: statusCode, }, } _ = json.NewEncoder(w).Encode(response) @@ -314,6 +326,7 @@ func JSONValidationMiddleware(next http.Handler) http.Handler { // statusWriter wraps http.ResponseWriter to capture the status code type statusWriter struct { http.ResponseWriter + statusCode int written bool } @@ -323,6 +336,7 @@ func (sw *statusWriter) WriteHeader(statusCode int) { sw.statusCode = statusCode sw.written = true } + sw.ResponseWriter.WriteHeader(statusCode) } @@ -362,11 +376,12 @@ func RequestLoggerMiddleware(logger *slog.Logger) func(http.Handler) http.Handle // Slow query threshold (1 second) const slowQueryThreshold = 1 * time.Second - if sw.statusCode >= http.StatusInternalServerError { + switch { + case sw.statusCode >= http.StatusInternalServerError: logLevel = slog.LevelError - } else if sw.statusCode >= http.StatusBadRequest { + case sw.statusCode >= http.StatusBadRequest: logLevel = slog.LevelWarn - } else if duration >= slowQueryThreshold { + case duration >= slowQueryThreshold: logLevel = slog.LevelWarn } diff --git a/internal/server/routes.go b/internal/server/routes.go index cde3293..82ce950 100644 --- a/internal/server/routes.go +++ b/internal/server/routes.go @@ -16,7 +16,9 @@ func (s *Server) setupRoutes() { r.Use(middleware.RealIP) r.Use(RequestLoggerMiddleware(s.logger.Logger)) // Structured request logging r.Use(middleware.Recoverer) + const requestTimeout = 30 * time.Second // Increased from 8s for slow queries + r.Use(TimeoutMiddleware(requestTimeout, s.logger.Logger)) r.Use(JSONResponseMiddleware) @@ -24,7 +26,10 @@ func (s *Server) setupRoutes() { r.Get("/", s.handleIndex()) r.Get("/status", s.handleStatusHTML()) r.Get("/status.json", JSONValidationMiddleware(s.handleStatusJSON()).ServeHTTP) - r.Get("/.well-known/healthcheck.json", JSONValidationMiddleware(s.handleHealthCheck()).ServeHTTP) + r.Get( + "/.well-known/healthcheck.json", + JSONValidationMiddleware(s.handleHealthCheck()).ServeHTTP, + ) // AS and prefix detail pages r.Get("/as/{asn}", s.handleASDetail()) diff --git a/internal/server/server.go b/internal/server/server.go index 16ba2b3..d709b7a 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -3,6 +3,7 @@ package server import ( "context" + "errors" "fmt" "net" "net/http" @@ -42,7 +43,12 @@ type Server struct { } // New creates a new HTTP server -func New(db database.Store, streamer *streamer.Streamer, logger *logger.Logger, cfg *config.Config) *Server { +func New( + db database.Store, + streamer *streamer.Streamer, + logger *logger.Logger, + cfg *config.Config, +) *Server { s := &Server{ db: db, streamer: streamer, @@ -57,8 +63,8 @@ func New(db database.Store, streamer *streamer.Streamer, logger *logger.Logger, // Start listens on the configured port and serves HTTP in the background. // It returns an error if it cannot listen, for example because the port is -// already in use. -func (s *Server) Start() error { +// already in use. ctx bounds only opening the port, not serving. +func (s *Server) Start(ctx context.Context) error { const ( network = "tcp" readHeaderTimeout = 40 * time.Second @@ -78,7 +84,9 @@ func (s *Server) Start() error { s.logger.Info("Starting HTTP server", "port", s.port, "addr", s.srv.Addr) - listener, err := net.Listen(network, s.srv.Addr) + var listenConfig net.ListenConfig + + listener, err := listenConfig.Listen(ctx, network, s.srv.Addr) if err != nil { return fmt.Errorf("starting HTTP server: %w", err) } @@ -86,7 +94,8 @@ func (s *Server) Start() error { s.logger.Info("HTTP server listening", "addr", listener.Addr().String()) go func() { - if err := s.srv.Serve(listener); err != nil && err != http.ErrServerClosed { + err := s.srv.Serve(listener) + if err != nil && !errors.Is(err, http.ErrServerClosed) { s.logger.Error("HTTP server error", "error", err) } }() diff --git a/internal/server/server_test.go b/internal/server/server_test.go index ff7c7d2..1f6964b 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -1,4 +1,4 @@ -package server +package server_test import ( "context" @@ -8,6 +8,7 @@ import ( "git.eeqj.de/sneak/routewatch/internal/config" "git.eeqj.de/sneak/routewatch/internal/logger" "git.eeqj.de/sneak/routewatch/internal/metrics" + "git.eeqj.de/sneak/routewatch/internal/server" "git.eeqj.de/sneak/routewatch/internal/streamer" ) @@ -16,18 +17,31 @@ import ( // listened in a goroutine, only logged the failure and returned nil, so the // daemon kept running without its web server. func TestStartFailsWhenPortIsHeld(t *testing.T) { - held, err := net.Listen("tcp", ":0") + t.Parallel() + + var listenConfig net.ListenConfig + + held, err := listenConfig.Listen(t.Context(), "tcp", ":0") if err != nil { t.Fatalf("failed to open the test listener: %v", err) } defer func() { _ = held.Close() }() - port := held.Addr().(*net.TCPAddr).Port + addr, ok := held.Addr().(*net.TCPAddr) + if !ok { + t.Fatalf("test listener address %v is not a TCP address", held.Addr()) + } + + port := addr.Port log := logger.New() - s := New(nil, streamer.New(log, metrics.New()), log, &config.Config{Port: port}) + s := server.New(nil, streamer.New(log, metrics.New()), log, + &config.Config{Port: port}) + defer func() { _ = s.Stop(context.Background()) }() - if err := s.Start(); err == nil { - t.Fatalf("Start returned nil while port %d was held by another listener", port) + err = s.Start(t.Context()) + if err == nil { + t.Fatalf("Start returned nil while port %d was held by another listener", + port) } } diff --git a/internal/streamer/streamer.go b/internal/streamer/streamer.go index 04e4429..1a119af 100644 --- a/internal/streamer/streamer.go +++ b/internal/streamer/streamer.go @@ -1,5 +1,6 @@ -// Package streamer implements an HTTP client that connects to the RIPE RIS Live streaming API, -// parses BGP UPDATE messages from the JSON stream, and dispatches them to registered handlers. +// Package streamer implements an HTTP client that connects to the RIPE RIS Live +// streaming API, parses BGP UPDATE messages from the JSON stream, and +// dispatches them to registered handlers. package streamer import ( @@ -25,31 +26,32 @@ import ( // countingReader wraps an io.Reader and counts bytes read type countingReader struct { reader io.Reader - count int64 + count atomic.Int64 } // Read implements io.Reader and counts bytes func (c *countingReader) Read(p []byte) (int, error) { n, err := c.reader.Read(p) - atomic.AddInt64(&c.count, int64(n)) + c.count.Add(int64(n)) return n, err } // Count returns the total bytes read func (c *countingReader) Count() int64 { - return atomic.LoadInt64(&c.count) + return c.count.Load() } // Configuration constants for the RIS Live streamer. const ( risLiveURL = "https://ris-live.ripe.net/v1/stream/?format=json&" + "client=https%3A%2F%2Fgit.eeqj.de%2Fsneak%2Froutewatch" - metricsWindowSize = 60 // seconds for rolling average - metricsUpdateRate = time.Second - minBackoffDelay = 5 * time.Second - maxBackoffDelay = 320 * time.Second - streamIdleTimeout = 60 * time.Second // Reconnect when no line arrives for this long + metricsWindowSize = 60 // seconds for rolling average + metricsUpdateRate = time.Second + minBackoffDelay = 5 * time.Second + maxBackoffDelay = 320 * time.Second + // Reconnect when no line arrives for this long + streamIdleTimeout = 60 * time.Second metricsLogInterval = 10 * time.Second bytesPerKB = 1024 bytesPerMB = 1024 * 1024 @@ -60,14 +62,22 @@ const ( backpressureSlope = 2.0 // Slope for linear drop probability increase ) -// errStreamEnded is returned by stream when RIS Live ends the response. -var errStreamEnded = errors.New("RIS Live ended the stream") +var ( + errAlreadyRunning = errors.New("streamer already running") + errUnexpectedStatus = errors.New("unexpected status code") + // errStreamEnded is returned by stream when RIS Live ends the response. + errStreamEnded = errors.New("RIS Live ended the stream") + // errStreamIdle is returned by stream when no line arrived for the idle + // timeout. + errStreamIdle = errors.New("no line received") +) // MessageHandler defines the interface for processing RIS messages. // Implementations must specify which message types they want to receive, // how to process messages, and their desired queue capacity. type MessageHandler interface { - // WantsMessage returns true if this handler wants to process messages of the given type. + // WantsMessage returns true if this handler wants to process messages of + // the given type. WantsMessage(messageType string) bool // HandleMessage processes a RIS message. This method is called from a dedicated @@ -109,18 +119,20 @@ type handlerInfo struct { // backoff, dispatches messages to registered handlers via per-handler queues, and // implements backpressure to prevent queue overflow during high traffic periods. type Streamer struct { - logger *logger.Logger - client *http.Client - url string - idleTimeout time.Duration // Drop the connection when no line arrives for this long - minBackoff time.Duration // First wait before reconnecting; doubles up to maxBackoffDelay + logger *logger.Logger + client *http.Client + url string + // Drop the connection when no line arrives for this long + idleTimeout time.Duration + // First wait before reconnecting; doubles up to maxBackoffDelay + minBackoff time.Duration handlers []*handlerInfo rawHandler RawMessageHandler mu sync.RWMutex cancel context.CancelFunc running bool metrics *metrics.Tracker - totalDropped uint64 // Total dropped messages across all handlers + totalDropped atomic.Uint64 // Total dropped messages across all handlers random *rand.Rand // Random number generator for backpressure drops bgpPeers map[string]bool // Track active BGP peers by peer IP bgpPeersMu sync.RWMutex // Protects bgpPeers map @@ -128,7 +140,8 @@ type Streamer struct { // New creates a new Streamer instance configured to connect to the RIS Live API. // The logger is used for structured logging of connection events and errors. -// The metrics tracker is used to record message counts, bytes received, and connection status. +// The metrics tracker is used to record message counts, bytes received, and +// connection status. func New(logger *logger.Logger, metrics *metrics.Tracker) *Streamer { return &Streamer{ logger: logger, @@ -162,7 +175,8 @@ func (s *Streamer) RegisterHandler(handler MessageHandler) { handler: handler, queue: make(chan *ristypes.RISMessage, handler.QueueCapacity()), metrics: handlerMetrics{ - minTime: time.Duration(math.MaxInt64), // Initialize to max so first value sets the floor + // Initialize to max so first value sets the floor + minTime: time.Duration(math.MaxInt64), }, } @@ -180,22 +194,24 @@ func (s *Streamer) RegisterHandler(handler MessageHandler) { func (s *Streamer) RegisterRawHandler(handler RawMessageHandler) { s.mu.Lock() defer s.mu.Unlock() + s.rawHandler = handler } // Start begins streaming BGP updates from the RIS Live API in a background goroutine. // It starts worker goroutines for each registered handler and manages automatic // reconnection with exponential backoff on connection failures. -// Returns an error if the streamer is already running. -func (s *Streamer) Start() error { +// Returns an error if the streamer is already running. It streams until Stop: +// the end of ctx does not stop it. +func (s *Streamer) Start(ctx context.Context) error { s.mu.Lock() defer s.mu.Unlock() if s.running { - return fmt.Errorf("streamer already running") + return errAlreadyRunning } - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(context.WithoutCancel(ctx)) s.cancel = cancel s.running = true @@ -225,44 +241,21 @@ func (s *Streamer) Stop() { return } + s.cancel() s.cancel = nil // Close all handler queues to signal workers to stop for _, info := range s.handlers { close(info.queue) } + s.running = false s.mu.Unlock() s.metrics.SetConnected(false) } -// runHandlerWorker processes messages for a specific handler -func (s *Streamer) runHandlerWorker(info *handlerInfo) { - for msg := range info.queue { - start := time.Now() - info.handler.HandleMessage(msg) - elapsed := time.Since(start) - - // Update metrics - info.metrics.mu.Lock() - info.metrics.processedCount++ - info.metrics.totalTime += elapsed - - // Update min time - if elapsed < info.metrics.minTime { - info.metrics.minTime = elapsed - } - - // Update max time - if elapsed > info.metrics.maxTime { - info.metrics.maxTime = elapsed - } - info.metrics.mu.Unlock() - } -} - -// IsRunning reports whether the streamer is currently connected and processing messages. -// This is safe to call concurrently from multiple goroutines. +// IsRunning reports whether the streamer is currently connected and processing +// messages. This is safe to call concurrently from multiple goroutines. func (s *Streamer) IsRunning() bool { s.mu.RLock() defer s.mu.RUnlock() @@ -323,12 +316,15 @@ func (s *Streamer) GetHandlerStats() []HandlerStats { // Calculate average time if info.metrics.processedCount > 0 { processedCount := info.metrics.processedCount + const maxInt64 = 1<<63 - 1 + if processedCount > maxInt64 { processedCount = maxInt64 } - //nolint:gosec // processedCount is explicitly bounded above - hs.AvgProcessTime = info.metrics.totalTime / time.Duration(processedCount) + + hs.AvgProcessTime = info.metrics.totalTime / + time.Duration(processedCount) } info.metrics.mu.Unlock() @@ -339,11 +335,38 @@ func (s *Streamer) GetHandlerStats() []HandlerStats { return stats } -// GetDroppedMessages returns the total number of messages dropped across all handlers -// due to queue overflow or backpressure. This counter is monotonically increasing -// and is safe to call concurrently. +// GetDroppedMessages returns the total number of messages dropped across all +// handlers due to queue overflow or backpressure. This counter is monotonically +// increasing and is safe to call concurrently. func (s *Streamer) GetDroppedMessages() uint64 { - return atomic.LoadUint64(&s.totalDropped) + return s.totalDropped.Load() +} + +// runHandlerWorker processes messages for a specific handler +func (s *Streamer) runHandlerWorker(info *handlerInfo) { + for msg := range info.queue { + start := time.Now() + + info.handler.HandleMessage(msg) + + elapsed := time.Since(start) + + // Update metrics + info.metrics.mu.Lock() + info.metrics.processedCount++ + info.metrics.totalTime += elapsed + + // Update min time + if elapsed < info.metrics.minTime { + info.metrics.minTime = elapsed + } + + // Update max time + if elapsed > info.metrics.maxTime { + info.metrics.maxTime = elapsed + } + info.metrics.mu.Unlock() + } } // logMetrics logs the current streaming statistics @@ -352,7 +375,8 @@ func (s *Streamer) logMetrics() { uptime := time.Since(metrics.ConnectedSince) const bitsPerMegabit = 1000000 - totalDropped := atomic.LoadUint64(&s.totalDropped) + + totalDropped := s.totalDropped.Load() s.logger.Info( "Stream statistics", @@ -378,16 +402,19 @@ func (s *Streamer) logMetrics() { // Log per-handler statistics s.mu.RLock() + for i, info := range s.handlers { info.metrics.mu.Lock() if info.metrics.processedCount > 0 { // Safe conversion: processedCount is bounded by maxInt64 processedCount := info.metrics.processedCount + const maxInt64 = 1<<63 - 1 + if processedCount > maxInt64 { processedCount = maxInt64 } - //nolint:gosec // processedCount is explicitly bounded above + avgTime := info.metrics.totalTime / time.Duration(processedCount) s.logger.Info( "Handler statistics", @@ -404,6 +431,7 @@ func (s *Streamer) logMetrics() { } info.metrics.mu.Unlock() } + s.mu.RUnlock() } @@ -412,7 +440,8 @@ func (s *Streamer) updateMetrics(messageBytes int) { s.metrics.RecordMessage(int64(messageBytes)) } -// streamWithReconnect handles streaming with automatic reconnection and exponential backoff +// streamWithReconnect handles streaming with automatic reconnection and +// exponential backoff func (s *Streamer) streamWithReconnect(ctx context.Context) { backoffDelay := s.minBackoff consecutiveFailures := 0 @@ -450,6 +479,7 @@ func (s *Streamer) streamWithReconnect(ctx context.Context) { if streamDuration > 30*time.Second { s.logger.Info("Resetting backoff delay due to successful connection", "stream_duration", streamDuration) + backoffDelay = s.minBackoff consecutiveFailures = 0 } else { @@ -489,7 +519,7 @@ func (s *Streamer) stream(ctx context.Context) error { idleTimer := time.AfterFunc(s.idleTimeout, connCancel) defer idleTimer.Stop() - req, err := http.NewRequestWithContext(connCtx, "GET", s.url, nil) + req, err := http.NewRequestWithContext(connCtx, http.MethodGet, s.url, nil) if err != nil { return fmt.Errorf("failed to create request: %w", err) } @@ -502,13 +532,14 @@ func (s *Streamer) stream(ctx context.Context) error { return fmt.Errorf("failed to connect to RIS Live: %w", err) } defer func() { - if err := resp.Body.Close(); err != nil { + err := resp.Body.Close() + if err != nil { s.logger.Error("Failed to close response body", "error", err) } }() if resp.StatusCode != http.StatusOK { - return fmt.Errorf("unexpected status code: %d", resp.StatusCode) + return fmt.Errorf("%w: %d", errUnexpectedStatus, resp.StatusCode) } // Wrap body with counting reader to track actual wire bytes @@ -522,7 +553,9 @@ func (s *Streamer) stream(ctx context.Context) error { return fmt.Errorf("failed to create gzip reader: %w", err) } defer func() { _ = gzReader.Close() }() + reader = gzReader + s.logger.Info("Connected to RIS Live stream", "compression", "gzip") } else { s.logger.Info("Connected to RIS Live stream", "compression", "none") @@ -530,9 +563,6 @@ func (s *Streamer) stream(ctx context.Context) error { s.metrics.SetConnected(true) - // Track wire bytes for metrics updates - var lastWireBytes int64 - // Start metrics logging goroutine metricsTicker := time.NewTicker(metricsLogInterval) defer metricsTicker.Stop() @@ -549,25 +579,44 @@ func (s *Streamer) stream(ctx context.Context) error { }() // Wire byte update ticker - update metrics with actual wire bytes periodically + go s.recordWireBytes(connCtx, wireCounter) + + return s.readStream(ctx, connCtx, reader, idleTimer) +} + +// recordWireBytes adds the bytes wireCounter has counted to the metrics once a +// second until ctx is cancelled. +func (s *Streamer) recordWireBytes( + ctx context.Context, wireCounter *countingReader, +) { wireUpdateTicker := time.NewTicker(time.Second) defer wireUpdateTicker.Stop() - go func() { - for { - select { - case <-wireUpdateTicker.C: - currentBytes := wireCounter.Count() - delta := currentBytes - lastWireBytes - if delta > 0 { - s.metrics.RecordWireBytes(delta) - lastWireBytes = currentBytes - } - case <-connCtx.Done(): - return - } - } - }() + var lastWireBytes int64 + for { + select { + case <-wireUpdateTicker.C: + currentBytes := wireCounter.Count() + + delta := currentBytes - lastWireBytes + if delta > 0 { + s.metrics.RecordWireBytes(delta) + + lastWireBytes = currentBytes + } + case <-ctx.Done(): + return + } + } +} + +// readStream hands each line of the stream to processLine until the stream +// ends, a line fails, or ctx is cancelled. Each line restarts idleTimer, which +// cancels connCtx when it fires. +func (s *Streamer) readStream( + ctx, connCtx context.Context, reader io.Reader, idleTimer *time.Timer, +) error { scanner := bufio.NewScanner(reader) for scanner.Scan() { @@ -586,139 +635,10 @@ func (s *Streamer) stream(ctx context.Context) error { continue } - // Update metrics with decompressed message size - s.updateMetrics(len(line)) - - // Call raw handler if registered - s.mu.RLock() - rawHandler := s.rawHandler - s.mu.RUnlock() - - if rawHandler != nil { - // Call raw handler synchronously to preserve order - rawHandler(string(line)) + err := s.processLine(ctx, line) + if err != nil { + return err } - - // Parse the message first - var wrapper ristypes.RISLiveMessage - if err := json.Unmarshal(line, &wrapper); err != nil { - // Log the error and return to trigger reconnection - s.logger.Error("Failed to parse JSON", - "error", err, - "line", string(line), - "line_length", len(line)) - - return fmt.Errorf("JSON parse error: %w", err) - } - - // Check if it's a ris_message wrapper - if wrapper.Type != "ris_message" { - s.logger.Error("Unexpected wrapper type", - "type", wrapper.Type, - "line", string(line), - ) - - continue - } - - // Get the actual message - msg := wrapper.Data - - // Parse the timestamp - msg.ParsedTimestamp = time.Unix(int64(msg.Timestamp), 0).UTC() - - // Process based on message type - switch msg.Type { - case "UPDATE": - // Process BGP UPDATE messages - // Will be dispatched to handlers - case "RIS_PEER_STATE": - // RIS peer state messages - silently ignore - continue - case "KEEPALIVE": - // BGP keepalive messages - silently process - continue - case "OPEN": - // BGP open messages - track peer as active - s.bgpPeersMu.Lock() - s.bgpPeers[msg.Peer] = true - peerCount := len(s.bgpPeers) - s.bgpPeersMu.Unlock() - s.metrics.SetBGPPeerCount(peerCount) - - s.logger.Info("BGP session opened", - "peer", msg.Peer, - "peer_asn", msg.PeerASN, - "total_peers", peerCount, - ) - - continue - case "NOTIFICATION": - // BGP notification messages (session closed) - s.bgpPeersMu.Lock() - delete(s.bgpPeers, msg.Peer) - peerCount := len(s.bgpPeers) - s.bgpPeersMu.Unlock() - s.metrics.SetBGPPeerCount(peerCount) - - s.logger.Warn("BGP notification", - "peer", msg.Peer, - "peer_asn", msg.PeerASN, - "total_peers", peerCount, - ) - - continue - case "STATE": - // Peer state changes - silently ignore - continue - default: - s.logger.Warn("Unknown message type, skipping", - "type", msg.Type, - ) - - continue - } - - // Dispatch to interested handlers. Stop cancels ctx and closes the - // queues under the write lock, so if ctx is cancelled here, under the - // read lock, the queues are closed and must not be sent to. - s.mu.RLock() - if ctx.Err() != nil { - s.mu.RUnlock() - - return ctx.Err() - } - for _, info := range s.handlers { - if !info.handler.WantsMessage(msg.Type) { - continue - } - - // Check if we should drop due to backpressure - if s.shouldDropForBackpressure(info) { - atomic.AddUint64(&info.metrics.droppedCount, 1) - atomic.AddUint64(&s.totalDropped, 1) - - continue - } - - // Try to queue the message - select { - case info.queue <- &msg: - // Message queued successfully - // Update high water mark if needed - queueLen := len(info.queue) - info.metrics.mu.Lock() - if queueLen > info.metrics.queueHighWaterMark { - info.metrics.queueHighWaterMark = queueLen - } - info.metrics.mu.Unlock() - default: - // Queue is full, drop the message - atomic.AddUint64(&info.metrics.droppedCount, 1) - atomic.AddUint64(&s.totalDropped, 1) - } - } - s.mu.RUnlock() } if ctx.Err() != nil { @@ -726,17 +646,173 @@ func (s *Streamer) stream(ctx context.Context) error { } if connCtx.Err() != nil { - return fmt.Errorf("no line received for %s", s.idleTimeout) + return fmt.Errorf("%w for %s", errStreamIdle, s.idleTimeout) } - if err := scanner.Err(); err != nil { + err := scanner.Err() + if err != nil { return fmt.Errorf("scanner error: %w", err) } return errStreamEnded } -// shouldDropForBackpressure determines if a message should be dropped based on queue utilization +// processLine passes one line of the stream to the raw handler, parses it, +// and dispatches an UPDATE message to the handlers. An error ends the +// connection. +func (s *Streamer) processLine(ctx context.Context, line []byte) error { + // Update metrics with decompressed message size + s.updateMetrics(len(line)) + + // Call raw handler if registered + s.mu.RLock() + rawHandler := s.rawHandler + s.mu.RUnlock() + + if rawHandler != nil { + // Call raw handler synchronously to preserve order + rawHandler(string(line)) + } + + // Parse the message first + var wrapper ristypes.RISLiveMessage + + err := json.Unmarshal(line, &wrapper) + if err != nil { + // Log the error and return to trigger reconnection + s.logger.Error("Failed to parse JSON", + "error", err, + "line", string(line), + "line_length", len(line)) + + return fmt.Errorf("JSON parse error: %w", err) + } + + // Check if it's a ris_message wrapper + if wrapper.Type != "ris_message" { + s.logger.Error("Unexpected wrapper type", + "type", wrapper.Type, + "line", string(line), + ) + + return nil + } + + // Get the actual message + msg := wrapper.Data + + // Parse the timestamp + msg.ParsedTimestamp = time.Unix(int64(msg.Timestamp), 0).UTC() + + // Process based on message type + switch msg.Type { + case "UPDATE": + // Process BGP UPDATE messages + // Will be dispatched to handlers + case "RIS_PEER_STATE": + // RIS peer state messages - silently ignore + return nil + case "KEEPALIVE": + // BGP keepalive messages - silently process + return nil + case "OPEN": + // BGP open messages - track peer as active + s.bgpPeersMu.Lock() + s.bgpPeers[msg.Peer] = true + peerCount := len(s.bgpPeers) + s.bgpPeersMu.Unlock() + s.metrics.SetBGPPeerCount(peerCount) + + s.logger.Info("BGP session opened", + "peer", msg.Peer, + "peer_asn", msg.PeerASN, + "total_peers", peerCount, + ) + + return nil + case "NOTIFICATION": + // BGP notification messages (session closed) + s.bgpPeersMu.Lock() + delete(s.bgpPeers, msg.Peer) + peerCount := len(s.bgpPeers) + s.bgpPeersMu.Unlock() + s.metrics.SetBGPPeerCount(peerCount) + + s.logger.Warn("BGP notification", + "peer", msg.Peer, + "peer_asn", msg.PeerASN, + "total_peers", peerCount, + ) + + return nil + case "STATE": + // Peer state changes - silently ignore + return nil + default: + s.logger.Warn("Unknown message type, skipping", + "type", msg.Type, + ) + + return nil + } + + return s.dispatchMessage(ctx, &msg) +} + +// dispatchMessage queues msg for each handler that wants it, dropping it for +// a handler whose queue is full or under backpressure. +func (s *Streamer) dispatchMessage( + ctx context.Context, msg *ristypes.RISMessage, +) error { + // Stop cancels ctx and closes the queues under the write lock, so if ctx + // is cancelled here, under the read lock, the queues are closed and must + // not be sent to. + s.mu.RLock() + + if ctx.Err() != nil { + s.mu.RUnlock() + + return ctx.Err() + } + + for _, info := range s.handlers { + if !info.handler.WantsMessage(msg.Type) { + continue + } + + // Check if we should drop due to backpressure + if s.shouldDropForBackpressure(info) { + atomic.AddUint64(&info.metrics.droppedCount, 1) + s.totalDropped.Add(1) + + continue + } + + // Try to queue the message + select { + case info.queue <- msg: + // Message queued successfully + // Update high water mark if needed + queueLen := len(info.queue) + info.metrics.mu.Lock() + if queueLen > info.metrics.queueHighWaterMark { + info.metrics.queueHighWaterMark = queueLen + } + info.metrics.mu.Unlock() + default: + // Queue is full, drop the message + atomic.AddUint64(&info.metrics.droppedCount, 1) + s.totalDropped.Add(1) + } + } + + s.mu.RUnlock() + + return nil +} + +// shouldDropForBackpressure determines if a message should be dropped based on +// queue utilization func (s *Streamer) shouldDropForBackpressure(info *handlerInfo) bool { // Calculate queue utilization queueLen := len(info.queue) diff --git a/internal/streamer/streamer_test.go b/internal/streamer/streamer_internal_test.go similarity index 78% rename from internal/streamer/streamer_test.go rename to internal/streamer/streamer_internal_test.go index d2da82c..d5061cd 100644 --- a/internal/streamer/streamer_test.go +++ b/internal/streamer/streamer_internal_test.go @@ -17,6 +17,8 @@ import ( ) func TestNewStreamer(t *testing.T) { + t.Parallel() + logger := logger.New() metricsTracker := metrics.New() s := New(logger, metricsTracker) @@ -55,10 +57,13 @@ func TestNewStreamer(t *testing.T) { // goroutine count returns to its starting value. Each connection starts two // ticker goroutines; before the fix they lived until the streamer's lifetime // context was cancelled, so every reconnect leaked two. +// +//nolint:paralleltest // counts goroutines, so no other test may run alongside func TestStreamDoesNotLeakTickersAcrossReconnects(t *testing.T) { // The handler returns immediately, so the response body is empty and each // stream call ends at once, standing in for a dropped connection. - srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {})) + srv := httptest.NewServer(http.HandlerFunc( + func(_ http.ResponseWriter, _ *http.Request) {})) defer srv.Close() s := New(logger.New(), metrics.New()) @@ -66,19 +71,23 @@ func TestStreamDoesNotLeakTickersAcrossReconnects(t *testing.T) { // One warm-up connection so any persistent HTTP transport goroutine exists // before we take the baseline. - if err := s.stream(context.Background()); !errors.Is(err, errStreamEnded) { + err := s.stream(context.Background()) + if !errors.Is(err, errStreamEnded) { t.Fatalf("warm-up stream returned %v, want %v", err, errStreamEnded) } + s.client.CloseIdleConnections() baseline := settledGoroutineCount() const reconnects = 20 for range reconnects { - if err := s.stream(context.Background()); !errors.Is(err, errStreamEnded) { + err := s.stream(context.Background()) + if !errors.Is(err, errStreamEnded) { t.Fatalf("stream returned %v, want %v", err, errStreamEnded) } } + s.client.CloseIdleConnections() if !waitForGoroutines(baseline) { @@ -88,8 +97,8 @@ func TestStreamDoesNotLeakTickersAcrossReconnects(t *testing.T) { } // updateLine is one UPDATE message as RIS Live sends it. -const updateLine = `{"type":"ris_message","data":{"type":"UPDATE","peer":"192.0.2.1",` + - `"peer_asn":"64496","timestamp":1700000000}}` + "\n" +const updateLine = `{"type":"ris_message","data":{"type":"UPDATE",` + + `"peer":"192.0.2.1","peer_asn":"64496","timestamp":1700000000}}` + "\n" // updateHandler wants UPDATE messages. For each one it handles it sends on // handled if there is room, and otherwise does nothing; a nil handled is @@ -98,8 +107,11 @@ type updateHandler struct { handled chan struct{} } -func (updateHandler) WantsMessage(messageType string) bool { return messageType == "UPDATE" } -func (updateHandler) QueueCapacity() int { return 10 } +func (updateHandler) WantsMessage(messageType string) bool { + return messageType == "UPDATE" +} + +func (updateHandler) QueueCapacity() int { return 10 } func (h updateHandler) HandleMessage(*ristypes.RISMessage) { select { @@ -108,15 +120,27 @@ func (h updateHandler) HandleMessage(*ristypes.RISMessage) { } } +// flush sends what w has buffered to the client. The writers httptest servers +// hand to handlers can always do that. +func flush(w http.ResponseWriter) { + flusher, ok := w.(http.Flusher) + if ok { + flusher.Flush() + } +} + // TestStopBeforeMessageReachesQueues stops the streamer after the read loop // has checked for cancellation but before it hands the message to the handler // queues. That is the gap a stop from another goroutine can land in, and it // used to end in "send on closed channel". The raw handler runs in that gap on // the read loop itself, so calling Stop from it hits the gap every time. func TestStopBeforeMessageReachesQueues(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = io.WriteString(w, updateLine) - })) + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, updateLine) + })) defer srv.Close() s := New(logger.New(), metrics.New()) @@ -130,7 +154,8 @@ func TestStopBeforeMessageReachesQueues(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) s.cancel = cancel - if err := s.stream(ctx); !errors.Is(err, context.Canceled) { + err := s.stream(ctx) + if !errors.Is(err, context.Canceled) { t.Fatalf("stream returned %v, want %v", err, context.Canceled) } @@ -142,6 +167,8 @@ func TestStopBeforeMessageReachesQueues(t *testing.T) { // normally, as when RIS Live restarts. The streamer must treat that as a // disconnect and connect again. func TestReconnectWhenStreamEnds(t *testing.T) { + t.Parallel() + endResponse := func(http.ResponseWriter, *http.Request) {} checkReconnect(t, endResponse, streamIdleTimeout) } @@ -150,9 +177,12 @@ func TestReconnectWhenStreamEnds(t *testing.T) { // open without sending anything. The streamer must drop it once the idle // timeout passes and connect again. func TestReconnectWhenStreamGoesIdle(t *testing.T) { + t.Parallel() + const idleTimeout = 50 * time.Millisecond + holdResponse := func(w http.ResponseWriter, r *http.Request) { - w.(http.Flusher).Flush() + flush(w) <-r.Context().Done() } checkReconnect(t, holdResponse, idleTimeout) @@ -162,7 +192,10 @@ func TestReconnectWhenStreamGoesIdle(t *testing.T) { // request and never answer it, not even with headers. The streamer must give // up on it once the idle timeout passes and connect again. func TestReconnectWhenServerNeverAnswers(t *testing.T) { + t.Parallel() + const idleTimeout = 50 * time.Millisecond + neverAnswer := func(_ http.ResponseWriter, r *http.Request) { <-r.Context().Done() } @@ -174,26 +207,32 @@ func TestReconnectWhenServerNeverAnswers(t *testing.T) { // line must start the idle timeout over, so the stream lasts until the server // ends it instead of being dropped as idle. func TestStreamNotDroppedWhileLinesArrive(t *testing.T) { + t.Parallel() + const ( idleTimeout = 100 * time.Millisecond lineInterval = idleTimeout / 10 sendFor = 5 * idleTimeout ) - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - deadline := time.Now().Add(sendFor) - for time.Now().Before(deadline) { - _, _ = io.WriteString(w, updateLine) - w.(http.Flusher).Flush() - time.Sleep(lineInterval) - } - })) + + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + deadline := time.Now().Add(sendFor) + for time.Now().Before(deadline) { + _, _ = io.WriteString(w, updateLine) + + flush(w) + time.Sleep(lineInterval) + } + })) defer srv.Close() s := New(logger.New(), metrics.New()) s.url = srv.URL s.idleTimeout = idleTimeout - if err := s.stream(context.Background()); !errors.Is(err, errStreamEnded) { + err := s.stream(context.Background()) + if !errors.Is(err, errStreamEnded) { t.Fatalf("stream returned %v, want %v", err, errStreamEnded) } } @@ -203,28 +242,36 @@ func TestStreamNotDroppedWhileLinesArrive(t *testing.T) { // line and then stays open. The streamer must come back for a second // response, must not be marked connected when it does, and must handle the // UPDATE in it. -func checkReconnect(t *testing.T, firstResponse http.HandlerFunc, idleTimeout time.Duration) { +func checkReconnect( + t *testing.T, firstResponse http.HandlerFunc, idleTimeout time.Duration, +) { t.Helper() const maxWait = 5 * time.Second tracker := metrics.New() connectedAtReconnect := make(chan bool, 1) - var requests atomic.Int32 - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if requests.Add(1) == 1 { - firstResponse(w, r) - return - } - select { - case connectedAtReconnect <- tracker.IsConnected(): - default: - } - _, _ = io.WriteString(w, updateLine) - w.(http.Flusher).Flush() - <-r.Context().Done() - })) + var requests atomic.Int32 + + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + if requests.Add(1) == 1 { + firstResponse(w, r) + + return + } + + select { + case connectedAtReconnect <- tracker.IsConnected(): + default: + } + + _, _ = io.WriteString(w, updateLine) + + flush(w) + <-r.Context().Done() + })) defer srv.Close() s := New(logger.New(), tracker) @@ -233,7 +280,9 @@ func checkReconnect(t *testing.T, firstResponse http.HandlerFunc, idleTimeout ti s.minBackoff = time.Millisecond handler := updateHandler{handled: make(chan struct{}, 1)} s.RegisterHandler(handler) - if err := s.Start(); err != nil { + + err := s.Start(t.Context()) + if err != nil { t.Fatalf("Start returned error: %v", err) } defer s.Stop() @@ -245,7 +294,8 @@ func checkReconnect(t *testing.T, firstResponse http.HandlerFunc, idleTimeout ti } if <-connectedAtReconnect { - t.Error("streamer was still marked connected when it came back for a second response") + t.Error("streamer was still marked connected when it came back " + + "for a second response") } } @@ -253,12 +303,15 @@ func checkReconnect(t *testing.T, firstResponse http.HandlerFunc, idleTimeout ti // current count. func settledGoroutineCount() int { prev := runtime.NumGoroutine() + for range 20 { time.Sleep(10 * time.Millisecond) + cur := runtime.NumGoroutine() if cur == prev { return cur } + prev = cur } @@ -271,6 +324,7 @@ func waitForGoroutines(target int) bool { if runtime.NumGoroutine() <= target { return true } + time.Sleep(10 * time.Millisecond) } diff --git a/internal/templates/templates.go b/internal/templates/templates.go index b388343..16d6737 100644 --- a/internal/templates/templates.go +++ b/internal/templates/templates.go @@ -60,6 +60,7 @@ func timeSince(t time.Time) string { if duration < time.Minute { return "just now" } + if duration < time.Hour { minutes := int(duration.Minutes()) if minutes == 1 { @@ -68,6 +69,7 @@ func timeSince(t time.Time) string { return duration.Truncate(time.Minute).String() + " ago" } + if duration < hoursPerDay*time.Hour { hours := int(duration.Hours()) if hours == 1 { @@ -76,10 +78,12 @@ func timeSince(t time.Time) string { return duration.Truncate(time.Hour).String() + " ago" } + days := int(duration.Hours() / hoursPerDay) if days == 1 { return "1 day ago" } + if days < daysPerMonth { return duration.Truncate(hoursPerDay*time.Hour).String() + " ago" } @@ -118,7 +122,7 @@ func initTemplates() { "appLicense": func() string { return version.License }, "appRepoURL": func() string { return version.RepoURL }, "appGitRevision": func() string { return version.GitRevisionShort }, - "appGitCommitURL": func() string { return version.CommitURL() }, + "appGitCommitURL": version.CommitURL, } // Parse index template @@ -134,19 +138,22 @@ func initTemplates() { } // Parse AS detail template - defaultTemplates.ASDetail, err = template.New("asDetail").Funcs(funcs).Parse(asDetailHTML) + defaultTemplates.ASDetail, err = template.New("asDetail"). + Funcs(funcs).Parse(asDetailHTML) if err != nil { panic("failed to parse AS detail template: " + err.Error()) } // Parse prefix detail template - defaultTemplates.PrefixDetail, err = template.New("prefixDetail").Funcs(funcs).Parse(prefixDetailHTML) + defaultTemplates.PrefixDetail, err = template.New("prefixDetail"). + Funcs(funcs).Parse(prefixDetailHTML) if err != nil { panic("failed to parse prefix detail template: " + err.Error()) } // Parse prefix length template - defaultTemplates.PrefixLength, err = template.New("prefixLength").Funcs(funcs).Parse(prefixLengthHTML) + defaultTemplates.PrefixLength, err = template.New("prefixLength"). + Funcs(funcs).Parse(prefixLengthHTML) if err != nil { panic("failed to parse prefix length template: " + err.Error()) } diff --git a/internal/version/version.go b/internal/version/version.go index b32b2d8..20c44d5 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -3,7 +3,7 @@ package version // Build-time variables set via ldflags // -//nolint:gochecknoglobals // These must be variables to allow ldflags injection at build time +//nolint:gochecknoglobals // variables so that ldflags can set them at build time var ( // GitRevision is the git commit hash GitRevision = "unknown" diff --git a/internal/whois/whois.go b/internal/whois/whois.go index 1c4e55b..3713c9f 100644 --- a/internal/whois/whois.go +++ b/internal/whois/whois.go @@ -98,7 +98,8 @@ func (c *Client) LookupASN(ctx context.Context, asn int) (*ASNInfo, error) { if err != nil { // Return partial data from IANA if RIR query fails info := c.parseResponse(asn, rir, ianaResp) - info.RawResponse = ianaResp + "\n--- RIR query failed: " + err.Error() + " ---\n" + info.RawResponse = fmt.Sprintf("%s\n--- RIR query failed: %v ---\n", + ianaResp, err) return info, nil } @@ -120,28 +121,33 @@ func (c *Client) query(ctx context.Context, server, query string) (string, error defer func() { _ = conn.Close() }() // Set deadlines - if err := conn.SetWriteDeadline(time.Now().Add(writeTimeout)); err != nil { + err = conn.SetWriteDeadline(time.Now().Add(writeTimeout)) + if err != nil { return "", fmt.Errorf("set write deadline: %w", err) } // Send query - if _, err := fmt.Fprintf(conn, "%s\r\n", query); err != nil { + _, err = fmt.Fprintf(conn, "%s\r\n", query) + if err != nil { return "", fmt.Errorf("write query: %w", err) } // Read response - if err := conn.SetReadDeadline(time.Now().Add(readTimeout)); err != nil { + err = conn.SetReadDeadline(time.Now().Add(readTimeout)) + if err != nil { return "", fmt.Errorf("set read deadline: %w", err) } var sb strings.Builder + scanner := bufio.NewScanner(conn) for scanner.Scan() { sb.WriteString(scanner.Text()) sb.WriteString("\n") } - if err := scanner.Err(); err != nil { + err = scanner.Err() + if err != nil { return sb.String(), fmt.Errorf("read response: %w", err) } @@ -149,10 +155,10 @@ func (c *Client) query(ctx context.Context, server, query string) (string, error } // parseIANAReferral extracts the RIR and WHOIS server from an IANA response. -func (c *Client) parseIANAReferral(response string) (rir, whoisServer string) { - lines := strings.Split(response, "\n") +func (c *Client) parseIANAReferral(response string) (string, string) { + var rir string - for _, line := range lines { + for line := range strings.SplitSeq(response, "\n") { line = strings.TrimSpace(line) // Look for "refer:" line @@ -183,18 +189,9 @@ func (c *Client) parseIANAReferral(response string) (rir, whoisServer string) { // Also check organisation line for RIR hints if strings.HasPrefix(strings.ToLower(line), "organisation:") { - org := strings.ToLower(line) - switch { - case strings.Contains(org, "arin"): - rir = RIRARIN - case strings.Contains(org, "ripe"): - rir = RIRRIPE - case strings.Contains(org, "apnic"): - rir = RIRAPNIC - case strings.Contains(org, "lacnic"): - rir = RIRLACNIC - case strings.Contains(org, "afrinic"): - rir = RIRAFRNIC + orgRIR := rirInOrganisationLine(strings.ToLower(line)) + if orgRIR != "" { + rir = orgRIR } } } @@ -202,6 +199,25 @@ func (c *Client) parseIANAReferral(response string) (rir, whoisServer string) { return rir, "" } +// rirInOrganisationLine returns the RIR a lowercased IANA "organisation:" +// line names, or "" when it names none. +func rirInOrganisationLine(org string) string { + switch { + case strings.Contains(org, "arin"): + return RIRARIN + case strings.Contains(org, "ripe"): + return RIRRIPE + case strings.Contains(org, "apnic"): + return RIRAPNIC + case strings.Contains(org, "lacnic"): + return RIRLACNIC + case strings.Contains(org, "afrinic"): + return RIRAFRNIC + default: + return "" + } +} + // parseResponse extracts ASN information from a WHOIS response. func (c *Client) parseResponse(asn int, rir, response string) *ASNInfo { info := &ASNInfo{ @@ -211,11 +227,13 @@ func (c *Client) parseResponse(asn int, rir, response string) *ASNInfo { } lines := strings.Split(response, "\n") + var addressLines []string for _, line := range lines { line = strings.TrimSpace(line) - if line == "" || strings.HasPrefix(line, "%") || strings.HasPrefix(line, "#") { + if line == "" || strings.HasPrefix(line, "%") || + strings.HasPrefix(line, "#") { continue } @@ -232,63 +250,14 @@ func (c *Client) parseResponse(asn int, rir, response string) *ASNInfo { continue } - switch key { - // AS Name (varies by RIR) - case "asname", "as-name": - if info.ASName == "" { - info.ASName = value - } - - // Organization - case "orgname", "org-name", "owner": - if info.OrgName == "" { - info.OrgName = value - } - case "orgid", "org-id", "org": - if info.OrgID == "" { - info.OrgID = value - } - // Address (collect multiple lines) - case "address": + if key == "address" { addressLines = append(addressLines, value) - // Country - case "country": - if info.CountryCode == "" && len(value) == 2 { - info.CountryCode = strings.ToUpper(value) - } - - // Abuse contact - case "orgabuseemail", "abuse-mailbox": - if info.AbuseEmail == "" { - info.AbuseEmail = value - } - case "orgabusephone": - if info.AbusePhone == "" { - info.AbusePhone = value - } - - // Tech contact - case "orgtechemail": - if info.TechEmail == "" { - info.TechEmail = value - } - case "orgtechphone": - if info.TechPhone == "" { - info.TechPhone = value - } - - // Registration dates - case "regdate", "created": - if info.RegDate == nil { - info.RegDate = c.parseDate(value) - } - case "updated", "last-modified", "changed": - if info.LastMod == nil { - info.LastMod = c.parseDate(value) - } + continue } + + c.setResponseField(info, key, value) } // Combine address lines @@ -304,6 +273,65 @@ func (c *Client) parseResponse(asn int, rir, response string) *ASNInfo { return info } +// setResponseField stores the value of one "key: value" line of a WHOIS +// response in info, keeping the first value seen for each field. +func (c *Client) setResponseField(info *ASNInfo, key, value string) { + switch key { + // Country + case "country": + if info.CountryCode == "" && len(value) == 2 { + info.CountryCode = strings.ToUpper(value) + } + + // Registration dates + case "regdate", "created": + if info.RegDate == nil { + info.RegDate = c.parseDate(value) + } + case "updated", "last-modified", "changed": + if info.LastMod == nil { + info.LastMod = c.parseDate(value) + } + + default: + field := textFieldForKey(info, key) + if field != nil && *field == "" { + *field = value + } + } +} + +// textFieldForKey returns the text field of info that a WHOIS response key +// fills, or nil when the key fills none. +func textFieldForKey(info *ASNInfo, key string) *string { + switch key { + // AS Name (varies by RIR) + case "asname", "as-name": + return &info.ASName + + // Organization + case "orgname", "org-name", "owner": + return &info.OrgName + case "orgid", "org-id", "org": + return &info.OrgID + + // Abuse contact + case "orgabuseemail", "abuse-mailbox": + return &info.AbuseEmail + case "orgabusephone": + return &info.AbusePhone + + // Tech contact + case "orgtechemail": + return &info.TechEmail + case "orgtechphone": + return &info.TechPhone + + default: + return nil + } +} + // parseDate attempts to parse various date formats used in WHOIS responses. func (c *Client) parseDate(value string) *time.Time { // Common formats @@ -319,15 +347,15 @@ func (c *Client) parseDate(value string) *time.Time { value = strings.TrimSpace(value) // Handle "YYYYMMDD" format from LACNIC if len(value) == lacnicDateFormatLen { - if _, err := time.Parse("20060102", value); err == nil { - t, _ := time.Parse("20060102", value) - + t, err := time.Parse("20060102", value) + if err == nil { return &t } } for _, format := range formats { - if t, err := time.Parse(format, value); err == nil { + t, err := time.Parse(format, value) + if err == nil { return &t } } diff --git a/pkg/asinfo/asinfo.go b/pkg/asinfo/asinfo.go index 500ec20..372f2e1 100644 --- a/pkg/asinfo/asinfo.go +++ b/pkg/asinfo/asinfo.go @@ -1,4 +1,3 @@ -// Package asinfo provides information about Autonomous Systems (AS) package asinfo import ( @@ -54,7 +53,9 @@ func initRegistry() { } var asInfos []Info - if err := json.Unmarshal(jsonData, &asInfos); err != nil { + + err = json.Unmarshal(jsonData, &asInfos) + if err != nil { panic("failed to unmarshal embedded AS data: " + err.Error()) } @@ -137,6 +138,7 @@ func Search(query string) []Info { defer defaultRegistry.mu.RUnlock() var results []Info + for _, info := range defaultRegistry.byASN { if contains(info.Handle, query) || contains(info.Description, query) { results = append(results, *info) diff --git a/pkg/asinfo/asinfo_test.go b/pkg/asinfo/asinfo_test.go index 5d8c205..3dabfa6 100644 --- a/pkg/asinfo/asinfo_test.go +++ b/pkg/asinfo/asinfo_test.go @@ -1,10 +1,15 @@ -package asinfo +package asinfo_test import ( + "strings" "testing" + + "git.eeqj.de/sneak/routewatch/pkg/asinfo" ) func TestGet(t *testing.T) { + t.Parallel() + tests := []struct { name string asn int @@ -38,20 +43,26 @@ func TestGet(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - info, ok := Get(tt.asn) + t.Parallel() + + info, ok := asinfo.Get(tt.asn) if ok != tt.wantOK { t.Errorf("Get(%d) ok = %v, want %v", tt.asn, ok, tt.wantOK) + return } if ok && info.Description != tt.wantDesc { - t.Errorf("Get(%d) description = %q, want %q", tt.asn, info.Description, tt.wantDesc) + t.Errorf("Get(%d) description = %q, want %q", + tt.asn, info.Description, tt.wantDesc) } }) } } func TestGetDescription(t *testing.T) { + t.Parallel() + tests := []struct { asn int want string @@ -62,7 +73,10 @@ func TestGetDescription(t *testing.T) { for _, tt := range tests { t.Run("", func(t *testing.T) { - if got := GetDescription(tt.asn); got != tt.want { + t.Parallel() + + got := asinfo.GetDescription(tt.asn) + if got != tt.want { t.Errorf("GetDescription(%d) = %q, want %q", tt.asn, got, tt.want) } }) @@ -70,6 +84,8 @@ func TestGetDescription(t *testing.T) { } func TestGetHandle(t *testing.T) { + t.Parallel() + tests := []struct { asn int want string @@ -81,7 +97,10 @@ func TestGetHandle(t *testing.T) { for _, tt := range tests { t.Run("", func(t *testing.T) { - if got := GetHandle(tt.asn); got != tt.want { + t.Parallel() + + got := asinfo.GetHandle(tt.asn) + if got != tt.want { t.Errorf("GetHandle(%d) = %q, want %q", tt.asn, got, tt.want) } }) @@ -89,18 +108,23 @@ func TestGetHandle(t *testing.T) { } func TestTotal(t *testing.T) { - total := Total() + t.Parallel() + + total := asinfo.Total() if total < 100000 { t.Errorf("Total() = %d, expected > 100000", total) } // Verify it's consistent - if total2 := Total(); total2 != total { + total2 := asinfo.Total() + if total2 != total { t.Errorf("Total() returned different values: %d vs %d", total, total2) } } func TestSearch(t *testing.T) { + t.Parallel() + tests := []struct { name string query string @@ -130,16 +154,21 @@ func TestSearch(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - results := Search(tt.query) + t.Parallel() + + results := asinfo.Search(tt.query) if len(results) < tt.wantMin { - t.Errorf("Search(%q) returned %d results, want at least %d", tt.query, len(results), tt.wantMin) + t.Errorf("Search(%q) returned %d results, want at least %d", + tt.query, len(results), tt.wantMin) } // Verify all results contain the query if tt.query != "" { for _, r := range results { - if !contains(r.Handle, tt.query) && !contains(r.Description, tt.query) { - t.Errorf("Search result ASN %d doesn't contain query %q", r.ASN, tt.query) + if !strings.Contains(r.Handle, tt.query) && + !strings.Contains(r.Description, tt.query) { + t.Errorf("Search result ASN %d doesn't contain query %q", + r.ASN, tt.query) } } } @@ -148,13 +177,17 @@ func TestSearch(t *testing.T) { } func TestDataIntegrity(t *testing.T) { + t.Parallel() + // Verify no duplicate ASNs - all := All() + all := asinfo.All() seen := make(map[int]bool) + for _, info := range all { if seen[info.ASN] { t.Errorf("Duplicate ASN found: %d", info.ASN) } + seen[info.ASN] = true } @@ -163,6 +196,7 @@ func TestDataIntegrity(t *testing.T) { if info.Handle == "" && info.ASN != 0 { t.Errorf("ASN %d has empty handle", info.ASN) } + if info.Description == "" { t.Errorf("ASN %d has empty description", info.ASN) } @@ -174,8 +208,9 @@ func BenchmarkGet(b *testing.B) { asns := []int{1, 15169, 13335, 32934, 8075, 16509} b.ResetTimer() - for i := 0; i < b.N; i++ { - Get(asns[i%len(asns)]) + + for i := range b.N { + asinfo.Get(asns[i%len(asns)]) } } @@ -183,7 +218,8 @@ func BenchmarkSearch(b *testing.B) { queries := []string{"Google", "Amazon", "Microsoft", "University"} b.ResetTimer() - for i := 0; i < b.N; i++ { - Search(queries[i%len(queries)]) + + for i := range b.N { + asinfo.Search(queries[i%len(queries)]) } } diff --git a/script/bootstrap b/script/bootstrap index 3a156e5..2b4d229 100755 --- a/script/bootstrap +++ b/script/bootstrap @@ -3,7 +3,8 @@ # this repo. Idempotent: every install is guarded by a check so already # installed tools are skipped. Base tooling comes from nix, apt, brew, # or apk (detected in that order); assumes NOTHING is present (not git, -# make, or go). +# make, or go). The linter and the tests run only in Docker, as phases of +# the Dockerfile, so nothing installs golangci-lint here. set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" @@ -58,11 +59,10 @@ main() { # Go toolchain if missing go; then pkg_install go golang go go; fi - # golangci-lint: packaged in nix, brew, and apk. There is no apt - # package; on apt systems install it manually from a hash-verified - # GitHub release archive (never curl | sh). - if missing golangci-lint; then - pkg_install golangci-lint golangci-lint golangci-lint golangci-lint + # Warn, don't fail: make build works without it. + if missing docker; then + echo "bootstrap: WARNING: docker not found; install it to run" \ + "make test, make lint, make check and make docker." >&2 fi go mod download diff --git a/script/check b/script/check index cc046f7..92875f7 100755 --- a/script/check +++ b/script/check @@ -1,7 +1,8 @@ #!/bin/sh # script/check: run all checks (test, lint, fmt-check). Our own -# extension to scripts-to-rule-them-all. Must not modify any files. -# Generic: usually needs no adaptation. +# extension to scripts-to-rule-them-all. test and lint are Docker +# phases; fmt-check is native, because a formatter writes the working +# tree. Must not modify any files. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" diff --git a/script/cibuild b/script/cibuild index 3da5857..d8d3200 100755 --- a/script/cibuild +++ b/script/cibuild @@ -1,14 +1,28 @@ #!/bin/sh -# script/cibuild: run the CI build. The Dockerfile runs script/check -# (via make check), so a successful build implies all checks pass. -# Generic: needs no adaptation. The Gitea workflow runs this on push. +# script/cibuild: run the CI build. It bootstraps first: a CI runner +# checks out and runs this and nothing else, and script/fmt-check runs +# the formatter on the host, which a pristine checkout cannot do. +# --no-cache for the same reason as script/docker: the gate phases the +# final stage depends on are RUN steps, and a cached one is a check that +# did not run. set -eu -ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" - docker build . + "$SCRIPT_DIR/bootstrap" + "$SCRIPT_DIR/check" + # Own line: a failing command substitution inside an argument does + # not trip `set -e`, so the inline form degrades silently to an + # empty constant. The VERSION build argument takes precedence over + # the version a build stage derives from the .git in the context. + version="$(git describe --tags --always --dirty 2>/dev/null || true)" + [ -n "$version" ] || version="unknown" + docker build --no-cache \ + --build-arg VERSION="$version" \ + -t "$("$SCRIPT_DIR/projectname")" . } main "$@" diff --git a/script/lint b/script/lint index a675833..2d8b075 100755 --- a/script/lint +++ b/script/lint @@ -1,13 +1,23 @@ #!/bin/sh -# script/lint: run the linters. +# script/lint: run the linter. Linting is a phase of the Dockerfile and +# this builds that phase alone; the linter is never installed or run on +# a developer host, where a shared result cache and a host-global lock +# make its answer untrustworthy. +# +# The phase is not the last stage in the file, so it is built only when +# --target names it. --no-cache because a cached lint layer is a lint +# that did not run. The tag makes each build replace the previous image +# instead of leaving a dangling one behind. set -eu -ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" - go vet ./... - golangci-lint run + docker build --no-cache \ + --target lint \ + -t "$("$SCRIPT_DIR/projectname")-lint" . } main "$@" diff --git a/script/test b/script/test index bffb59e..cd239f2 100755 --- a/script/test +++ b/script/test @@ -1,17 +1,19 @@ #!/bin/sh -# script/test: run the test suite. On failure, rerun verbosely so the -# failing tests are visible in the output. +# script/test: run the test suite. Testing is a phase of the Dockerfile +# and this builds that phase alone, on the same terms as script/lint: +# --target because a phase that is not the last stage is built only when +# named, --no-cache because a cached test layer is a test that did not +# run, and a tag so each build replaces the previous image. set -eu -ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" - go test -short -timeout 30s -race -cover ./... || { - echo "--- Rerunning with -v for details ---" - go test -short -timeout 30s -race -v ./... - exit 1 - } + docker build --no-cache \ + --target test \ + -t "$("$SCRIPT_DIR/projectname")-test" . } main "$@"