diff --git a/.dockerignore b/.dockerignore index 89e43ce..fdde48f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,15 +1,83 @@ -# Docker does not read .gitignore, and a pattern here matches from the root of -# the build context only: a pattern meant for every directory needs `**/`. +# .dockerignore does NOT use .gitignore semantics. Docker matches with +# moby/patternmatcher: filepath.Match plus `**`, so `*` does not cross +# `/` and an unprefixed pattern is anchored at the context root. Every +# depth-independent pattern therefore needs `**/`, or `config/.env` and +# `certs/server.key` still ship while this file reads as solved. Only +# genuinely root-anchored entries go unprefixed. Never transplant these +# into .gitignore, where `**/` is wrong. +# +# Matching is case-sensitive, so secrets use character ranges rather +# than an ALL-CAPS twin, which would still miss `Server.Key`. +# +# Extend with this repo's own host-built artifacts, written anchored: +# `/myapp`, never `**/myapp`, which also matches `cmd/myapp/` and +# deletes the package directory from the context. # .git is sent without its config. Without a VERSION build argument the # stage that compiles runs `git describe --tags --always` on .git, which # does not need .git/config; that file can hold a credential, such as a # password in a remote URL or the token the CI checkout step stores there. -.git/config +# Each submodule keeps a config with the same exposure in its git directory +# under .git/modules/, nested again for a submodule's own submodules, or in +# its own .git directory when it keeps one. +# KNOWN GAP: a submodule whose name has a `config` segment (`config`, +# `deploy/config`, `config/lib`) loses its whole git directory, because +# `**/.git/modules/**/config` also matches that segment's directory +# under .git/modules/. Go's version stamping then fails the build; +# nothing leaks. Name such a submodule without that segment: +# `git submodule add --name`. +**/.git/config +**/.git/modules/**/config -# Local build and debug output: `make build`, `make run`, `make asupdate`, test -# binaries, coverage profiles and source archives. +# Agent scratch: one full checkout of the repo per in-flight agent. +# Anchored because it occurs once where agents run at the repo root. +# KNOWN GAP: a repo running agents in subdirectories still ships +# `services/api/.claude/` and must add its own anchored entry. +.claude + +# Environment files. `*.env` covers bare `.env` and the `prod.env` +# convention. Re-include a committed template with a negation if the +# build needs one: `!docs/example.env`. +**/*.[eE][nN][vV] +**/.[eE][nN][vV].* +**/.[eE][nN][vV][rR][cC] + +# Private keys and the bundles carrying them. Public certificates +# (*.crt, *.cer) are deliberately absent: they are legitimate inputs. +**/*.[pP][eE][mM] +**/*.[kK][eE][yY] +**/*.[pP]12 +**/*.[pP][fF][xX] +**/[iI][dD]_[rR][sS][aA] +**/[iI][dD]_[dD][sS][aA] +**/[iI][dD]_[eE][cC][dD][sS][aA] +**/[iI][dD]_[eE][cC][dD][sS][aA]_[sS][kK] +**/[iI][dD]_[eE][dD]25519 +**/[iI][dD]_[eE][dD]25519_[sS][kK] + +# Dependencies: restored inside the image, never copied in. +**/node_modules + +# OS metadata. +**/.DS_Store +**/Thumbs.db + +# Editor state: never a build input, and it churns COPY. +**/*.swp +**/*.swo +**/*~ +**/*.bak +**/.idea +**/.vscode +**/*.sublime-* + +# This repository's host-built artifacts: `make build` writes bin/, `go +# build` in the root the binaries, `make run` log.txt, `make asupdate` its +# .tmp file; test binaries, coverage profiles and source archives. /bin +/routewatch +/asinfo-gen +/streamdumper /log.txt /out /pkg/asinfo/asdata.json @@ -18,15 +86,12 @@ **/*.out **/*.tmp -# Local databases and secrets. The image carries a source archive of the whole -# build context, so these would otherwise ship inside it. +# Local databases. The image carries a source archive of the whole build +# context, so these would otherwise ship inside it. **/*.db **/*.db-journal **/*.db-wal -**/.env # A local Go workspace points at directories outside the build context. /go.work /go.work.sum - -**/.DS_Store diff --git a/.editorconfig b/.editorconfig index 2fe0ce0..de84fae 100644 --- a/.editorconfig +++ b/.editorconfig @@ -10,3 +10,9 @@ insert_final_newline = true [Makefile] indent_style = tab + +[*.go] +indent_style = tab + +# This repository's own sections, such as one for another language it +# uses, go below this comment, and a re-vendor keeps them. diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index a55bc55..2e96965 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -1,9 +1,20 @@ name: check on: [push] +# Free the shared runner: a new push cancels only the same branch's older run. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: check: runs-on: ubuntu-latest + # Free the shared runner from a hung build. + timeout-minutes: 20 steps: - # actions/checkout v4.2.2, 2026-02-28 + # actions/checkout v4.2.2, 2026-02-22 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + # script/cibuild needs no token, so none is left in .git/config. + with: + persist-credentials: false + # All history and tags, so git describe finds the version tag. + fetch-depth: 0 - run: script/cibuild diff --git a/.gitignore b/.gitignore index 4951af1..6407e7a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,41 +1,79 @@ -# Binaries for programs and plugins +# OS +.DS_Store +Thumbs.db + +# Editors +*.swp +*.swo +*~ +*.bak +.idea/ +.vscode/ +*.sublime-* + +# Agent scratch (worktrees of this repo, created and destroyed by +# in-flight tooling). Unanchored: .gitignore patterns already match at +# every depth, so no prefix is wanted here. This is not a .dockerignore +# entry and must not be given a `**/` prefix on the way into one. +.claude/ + +# Node +node_modules/ + +# Secrets. Unanchored like every entry above, so each matches at every +# depth. Matching is case-sensitive on Linux, so names use character +# ranges rather than a lowercase form that misses `Server.Key`. + +# Environment files. `*.env` covers bare `.env` and the `prod.env` +# convention. Only the templates `example.env` and `sample.env` are +# re-included below. A repository that commits any other template adds +# its own negation at the end of this file, for example `!.env.example`. +*.[eE][nN][vV] +.[eE][nN][vV].* +.[eE][nN][vV][rR][cC] +!example.env +!sample.env + +# Private keys and the bundles carrying them. +*.[pP][eE][mM] +*.[kK][eE][yY] +*.[pP]12 +*.[pP][fF][xX] +[iI][dD]_[rR][sS][aA] +[iI][dD]_[dD][sS][aA] +[iI][dD]_[eE][cC][dD][sS][aA] +[iI][dD]_[eE][cC][dD][sS][aA]_[sS][kK] +[iI][dD]_[eE][dD]25519 +[iI][dD]_[eE][dD]25519_[sS][kK] + +# This repository's own entries, such as its build outputs, go below +# this comment, and a re-vendor keeps them. Anchor a binary built at the +# root: `/myapp`, never `myapp`, which also ignores `cmd/myapp/`. + +# Go: logs, test binaries, coverage profiles, and the binaries `make build` +# and `go build` in the repository root write. +*.log +*.out +*.test *.exe *.exe~ *.dll *.so *.dylib /bin/ - -.DS_Store -log.txt - -# Test binary, built with `go test -c` -*.test - -# Output of the go coverage tool, specifically when used with LiteIDE -*.out - -# Dependency directories (remove the comment below to include it) -# vendor/ - -# Go workspace file +/routewatch +/asinfo-gen +/streamdumper go.work go.work.sum -# env file -.env - -# Database files +# Local databases, temporary files and debug output. *.db *.db-journal *.db-wal - -# Temporary files *.tmp - -# Raw AS data (we only commit the gzipped version) -pkg/asinfo/asdata.json - -# Debug output files -out log.txt +out + +# Raw AS data (only the gzipped version is committed). +pkg/asinfo/asdata.json diff --git a/.golangci.yml b/.golangci.yml index 9d7f245..32db9fe 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,92 +1,101 @@ version: "2" +# Config schema uses the golangci-lint v2 layout (settings live under +# linters.settings, not top-level linters-settings) so that the +# thresholds below are actually applied by golangci-lint >= v2. + run: - go: "1.24" - tests: false + timeout: 5m + modules-download-mode: readonly linters: + default: all enable: - # Additional linters requested - - testifylint # Checks usage of github.com/stretchr/testify - - usetesting # usetesting is an analyzer that detects using os.Setenv instead of t.Setenv since Go 1.17 - # - tagliatelle # Disabled: we need snake_case for external API compatibility - - nlreturn # nlreturn checks for a new line before return and branch statements - - nilnil # Checks that there is no simultaneous return of nil error and an invalid value - - nestif # Reports deeply nested if statements - - mnd # An analyzer to detect magic numbers - - lll # Reports long lines - - intrange # intrange is a linter to find places where for loops could make use of an integer range - - gochecknoglobals # Check that no global variables exist - - # Default/existing linters that are commonly useful - - govet - - errcheck - - staticcheck - - unused - - ineffassign - - misspell - - revive - - gosec - - unconvert - - unparam - -linters-settings: - lll: - line-length: 120 - - nestif: - min-complexity: 4 - - nlreturn: - block-size: 2 - - revive: - rules: - - name: var-naming - arguments: - - [] - - [] - - "upperCaseConst=true" - - tagliatelle: - case: + # Successor to the deprecated gomodguard. Named explicitly, rather than + # left to `default: all`, because it carries the module policy below. + - gomodguard_v2 + disable: + # Genuinely incompatible with project patterns + - exhaustruct # Requires all struct fields + - exhaustruct_v5 # Requires all struct fields (successor to exhaustruct) + - godot # Requires comments to end with periods + - wrapcheck # Too verbose for internal packages + - varnamelen # Short names like db, id are idiomatic Go + # Deprecated: the warning is attached to the old name, so it is + # silenced by disabling that name, not by enabling the successor. + - wsl # Deprecated, replaced by wsl_v5 + - gomodguard # Deprecated, replaced by gomodguard_v2 + # Misses findings at random in v2.14.0; back once a pinned release fixes it + - canonicalheader + settings: + lll: + line-length: 88 + funlen: + lines: 80 + statements: 50 + cyclop: + max-complexity: 15 + dupl: + threshold: 100 + depguard: + # Test-support code must not be compiled into the shipped binary. A + # test-support package exists to hand a test privileges the program + # itself must never have, so a file that is not a test must not import + # one. Test files, and the files inside a package whose directory name + # ends in `test`, are where that code belongs, and are exempt. + # + # The deny list below is the one part of this file a repository is + # expected to extend, and the only part it may. depguard matches an + # import path against a list of prefixes, so it cannot be told "any path + # whose last segment ends in test"; a repository's own test-support + # packages have to be named here one at a time, by full import path, + # under a module path that differs from repository to repository. Add + # them; change nothing else. rules: - json: snake - yaml: snake - xml: snake - bson: snake - - testifylint: - enable-all: true - - usetesting: {} + test-support: + list-mode: lax + files: + - "$all" + - "!$test" + - "!**/*test/**" + deny: + - pkg: net/http/httptest + desc: >- + Test-support code belongs in test files and in packages whose + directory name ends in test, not in the shipped binary. + # Only decisions already recorded in the Go package defaults are + # listed here. Every entry matches the module path exactly. + gomodguard_v2: + blocked: + - module: github.com/rs/zerolog + recommendations: + - log/slog + reason: "Structured logging is stdlib log/slog." + # One entry per pre-fork module path, because the later releases + # are separate paths. A prefix match would be shorter but would + # also reach github.com/go-redis/redismock, the test double for + # the successor these entries recommend. + - module: github.com/go-redis/redis + recommendations: + - github.com/redis/go-redis/v9 + reason: "Pre-fork module; use the maintained go-redis v9." + - module: github.com/go-redis/redis/v7 + recommendations: + - github.com/redis/go-redis/v9 + reason: "Pre-fork module; use the maintained go-redis v9." + - module: github.com/go-redis/redis/v8 + recommendations: + - github.com/redis/go-redis/v9 + reason: "Pre-fork module; use the maintained go-redis v9." + - module: github.com/sergi/go-diff + recommendations: + - github.com/aymanbagabas/go-udiff + reason: "No unified diff output; use go-udiff." + - module: github.com/hexops/gotextdiff + recommendations: + - github.com/aymanbagabas/go-udiff + reason: "Unmaintained fork; use go-udiff." issues: max-issues-per-linter: 0 max-same-issues: 0 - exclude-rules: - # Exclude unused parameter warnings for cobra command signatures - - text: "parameter '(args|cmd)' seems to be unused" - linters: - - revive - - # Allow ALL_CAPS constant names - - text: "don't use ALL_CAPS in Go names" - linters: - - revive - - # Allow snake_case JSON tags for external API compatibility - - path: "internal/types/ris.go" - linters: - - tagliatelle - - # Allow snake_case JSON tags for database models - - path: "internal/database/models.go" - linters: - - tagliatelle - - # Allow generic package name for types that define data structures - - path: "internal/types/" - text: "avoid meaningless package names" - linters: - - revive diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..b7ebc67 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,95 @@ +# IMPORTANT RULES + +- Claude is an inanimate tool. The spam that Claude attempts to insert into + commit messages (which it erroneously refers to as "attribution") is not + attribution, as I am the sole author of code created using Claude. It is + corporate advertising for Anthropic and is therefore completely unacceptable + in commit messages. + +- Tests should always be run before committing code. No commits should be made + that do not pass tests. + +- Code should always be formatted before committing. Do not commit unformatted + code. + +- Code should always be linted and linter errors fixed before committing. NEVER + commit code that does not pass the linter. DO NOT modify the linter config + unless specifically instructed. + +- The test suite is fast and local. When running tests, NEVER run individual + parts of the test suite, always run the whole thing by running "make test". + +- Do not stop working on a task until you have reached the definition of done + provided to you in the initial instruction. Don't do part or most of the work, + do all of the work until the criteria for done are met. + +- When you complete each task, if the tests are passing and the code is + formatted and there are no linter errors, always commit and push your work. + Use a good commit message and don't mention any author or co-author + attribution. + +- Do not create additional files in the root directory of the project without + asking permission first. Configuration files, documentation, and build files + are acceptable in the root, but source code and other files should be + organized in appropriate subdirectories. + +- Do not use bare strings or numbers in code, especially if they appear anywhere + more than once. Always define a constant (usually at the top of the file) and + give it a descriptive name, then use that constant in the code instead of the + bare string or number. + +- If you are fixing a bug, write a test first that reproduces the bug and fails, + and then fix the bug in the code, using the test to verify that the fix + worked. + +- When implementing new features, be aware of potential side-effects (such as + state files on disk, data in the database, etc.) and ensure that it is + possible to mock or stub these side-effects in tests when designing an API. + +- When dealing with dates and times or timestamps, always use, display, and + store UTC. Set the local timezone to UTC on startup. If the user needs to see + the time in a different timezone, store the user's timezone in a separate + field and convert the UTC time to the user's timezone when displaying it. For + internal use and internal applications and administrative purposes, always + display UTC. + +- When implementing programs, put the main.go in ./cmd//main.go + and put the program's code in ./internal//. This allows for + multiple programs to be implemented in the same repository without cluttering + the root directory. main.go should simply import and call + .CLIEntry(). The full implementation should be in + ./internal//. + +- When you are instructed to make the tests pass, DO NOT delete tests, skip + tests, or change the tests specifically to make them pass (unless there is a + bug in the test). This is cheating, and it is bad. You should only be + modifying the test if it is incorrect or if the test is no longer relevant. In + almost all cases, you should be fixing the code that is being tested, or + updating the tests to match a refactored implementation. + +- Always write a `Makefile` with the default target being `test`, and with a + `fmt` target that formats the code. The `test` target should run all tests in + the project, and the `fmt` target should format the code. `test` should also + have a prerequisite target `lint` that should run any linters that are + configured for the project. + +- After each completed bugfix or feature, the code must be committed. Do all of + the pre-commit checks (test, lint, fmt) before committing, of course. After + each commit, push to the remote. + +- Always write tests, even if they are extremely simple and just check for + correct syntax (ability to compile/import). If you are writing a new feature, + write a test for it. You don't need to target complete coverage, but you + should at least test any new functionality you add. + +- Always use structured logging. Log any relevant state/context with the + messages (but do not log secrets). If stdout is not a terminal, output the + structured logs in jsonl format. Use go's log/slog. + +- You do not need to summarize your changes in the chat after making them. + Making the changes and committing them is sufficient. If anything out of the + ordinary happened, please explain it, but in the normal case where you found + and fixed the bug, or implemented the feature, there is no need for the + end-of-change summary. + +- When testing daemons, use a 15 second timeout always. diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index 3124a8a..0000000 --- a/CLAUDE.md +++ /dev/null @@ -1,97 +0,0 @@ -# IMPORTANT RULES - -* Claude is an inanimate tool. The spam that Claude attempts to insert into - commit messages (which it erroneously refers to as "attribution") is not - attribution, as I am the sole author of code created using Claude. It is - corporate advertising for Anthropic and is therefore completely - unacceptable in commit messages. - -* Tests should always be run before committing code. No commits should be - made that do not pass tests. - -* Code should always be formatted before committing. Do not commit - unformatted code. - -* Code should always be linted and linter errors fixed before committing. - NEVER commit code that does not pass the linter. DO NOT modify the linter - config unless specifically instructed. - -* The test suite is fast and local. When running tests, NEVER run - individual parts of the test suite, always run the whole thing by running - "make test". - -* Do not stop working on a task until you have reached the definition of - done provided to you in the initial instruction. Don't do part or most of - the work, do all of the work until the criteria for done are met. - -* When you complete each task, if the tests are passing and the code is - formatted and there are no linter errors, always commit and push your - work. Use a good commit message and don't mention any author or co-author - attribution. - -* Do not create additional files in the root directory of the project - without asking permission first. Configuration files, documentation, and - build files are acceptable in the root, but source code and other files - should be organized in appropriate subdirectories. - -* Do not use bare strings or numbers in code, especially if they appear - anywhere more than once. Always define a constant (usually at the top of - the file) and give it a descriptive name, then use that constant in the - code instead of the bare string or number. - -* If you are fixing a bug, write a test first that reproduces the bug and - fails, and then fix the bug in the code, using the test to verify that the - fix worked. - -* When implementing new features, be aware of potential side-effects (such - as state files on disk, data in the database, etc.) and ensure that it is - possible to mock or stub these side-effects in tests when designing an - API. - -* When dealing with dates and times or timestamps, always use, display, and - store UTC. Set the local timezone to UTC on startup. If the user needs - to see the time in a different timezone, store the user's timezone in a - separate field and convert the UTC time to the user's timezone when - displaying it. For internal use and internal applications and - administrative purposes, always display UTC. - -* When implementing programs, put the main.go in - ./cmd//main.go and put the program's code in - ./internal//. This allows for multiple programs to be - implemented in the same repository without cluttering the root directory. - main.go should simply import and call .CLIEntry(). The - full implementation should be in ./internal//. - -* When you are instructed to make the tests pass, DO NOT delete tests, skip - tests, or change the tests specifically to make them pass (unless there - is a bug in the test). This is cheating, and it is bad. You should only - be modifying the test if it is incorrect or if the test is no longer - relevant. In almost all cases, you should be fixing the code that is - being tested, or updating the tests to match a refactored implementation. - -* Always write a `Makefile` with the default target being `test`, and with a - `fmt` target that formats the code. The `test` target should run all - tests in the project, and the `fmt` target should format the code. `test` - should also have a prerequisite target `lint` that should run any linters - that are configured for the project. - -* After each completed bugfix or feature, the code must be committed. Do - all of the pre-commit checks (test, lint, fmt) before committing, of - course. After each commit, push to the remote. - -* Always write tests, even if they are extremely simple and just check for - correct syntax (ability to compile/import). If you are writing a new - feature, write a test for it. You don't need to target complete coverage, - but you should at least test any new functionality you add. - -* Always use structured logging. Log any relevant state/context with the - messages (but do not log secrets). If stdout is not a terminal, output - the structured logs in jsonl format. Use go's log/slog. - -* You do not need to summarize your changes in the chat after making them. - Making the changes and committing them is sufficient. If anything out of - the ordinary happened, please explain it, but in the normal case where you - found and fixed the bug, or implemented the feature, there is no need for - the end-of-change summary. - -* When testing daemons, use a 15 second timeout always. diff --git a/Dockerfile b/Dockerfile index dfd9d5a..a8cfb8c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,44 @@ -# Lint stage — fast feedback on formatting and lint issues. -# The golangci-lint image bundles Go, gcc and make, so it can run go vet on -# the CGO sqlite package and golangci-lint without extra installs. -# golangci/golangci-lint:v2.7.2 (Go 1.25.5), 2026-09-21 -FROM golangci/golangci-lint@sha256:5d6d5c70a61f1356adfd9dd6316ce286799fefc9d743421356ff1b00842368ba AS lint +# In every stage ARG CACHEBUST comes right after the steps that install +# system packages and download Go's modules, or right after FROM in a +# stage with neither, and every step that lints, tests or compiles +# comes after it. script/ passes a new value on every build, which +# reruns every RUN step after it. Docker keeps the layers above it +# until the base image, their own lines or go.mod and go.sum change. +# The steps that compile mount Go's build cache, and golangci-lint run +# also mounts golangci-lint's; both survive from build to build. +# Lint phase. The image is Debian-based and has gcc, so it can check the +# cgo SQLite package without extra installs. +# golangci/golangci-lint:v2.14.0 (Go 1.27.0), 2026-10-19 +FROM golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint WORKDIR /src - COPY go.mod go.sum ./ RUN go mod download - +ARG CACHEBUST COPY . . +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/root/.cache/golangci-lint \ + golangci-lint run --config .golangci.yml ./... -RUN make fmt-check -RUN make lint +# Test phase. -race needs cgo and so a C compiler, which the Debian Go +# image ships and the alpine one does not. -count=1 because the mounted +# cache also holds test results, which go test would otherwise replay. +# -short skips the live RIS Live feed test, so the suite needs no network. +# golang:1.24-bookworm, 2026-09-21 +FROM golang@sha256:1a6d4452c65dea36aac2e2d606b01b4a029ec90cc1ae53890540ce6173ea77ac AS test +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +ARG CACHEBUST +COPY . . +RUN --mount=type=cache,target=/root/.cache/go-build \ + go test -short -count=1 -timeout 90s -race -cover ./... || \ + { echo "--- Rerunning with -v for details ---"; \ + go test -short -count=1 -timeout 90s -race -v ./...; exit 1; } -# Build stage +# Build stage. Nothing is wanted from either phase above; the copies +# are what make BuildKit build them first, so nothing after them can +# run unless lint and test passed. # golang:1.24-bookworm, 2026-09-21 FROM golang@sha256:1a6d4452c65dea36aac2e2d606b01b4a029ec90cc1ae53890540ce6173ea77ac AS builder @@ -26,36 +50,38 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* WORKDIR /src - -# Force BuildKit to run the lint stage before compiling or testing. +COPY go.mod go.sum ./ +RUN go mod download +ARG CACHEBUST COPY --from=lint /src/go.sum /dev/null - -# Copy everything +COPY --from=test /src/go.sum /dev/null +# A tar-stream context keeps the sender's file owners, which git refuses. +RUN git config --system --add safe.directory /src COPY . . -# Vendor dependencies (must be after copying source) -RUN go mod download && go mod vendor - -# Run the test suite in the build stage: -race needs cgo and the C compiler -# installed above. The suite is offline (the live-feed test is opt-in). -RUN make test +# Vendor dependencies for the source archive (must be after copying source) +RUN go mod vendor # Build the binary with CGO enabled (required for sqlite3). The version the -# page footer shows is the VERSION build argument when one is given, otherwise -# `git describe --tags --always` of the .git in the build context (git comes -# with this image): the tag on a tagged commit, tag-N-gHASH after one, the -# short commit when no tag is reachable. A context that carries .git and still -# yields no version fails the build. The footer links to the full commit. +# page footer shows is the VERSION build arg when one is given, otherwise +# `git describe --tags --always` on the .git in the build context: the tag +# on a tagged commit, tag-N-gHASH after one, the short commit when no tag is +# reachable. With .git present, a version that is still empty, dev or +# unknown fails the build: git is missing or could not read the checkout. +# Without .git the footer says unknown. The footer links to the full commit. ARG VERSION -RUN version="${VERSION:-$(git describe --tags --always || echo unknown)}"; \ - if [ -e .git ] && { [ -z "$version" ] || [ "$version" = dev ] || \ - [ "$version" = unknown ]; }; then \ - echo "no version could be derived although the build context carries .git" >&2; \ - exit 1; \ +RUN --mount=type=cache,target=/root/.cache/go-build \ + VERSION="${VERSION:-$(git describe --tags --always)}"; \ + if [ -e .git ]; then \ + case "$VERSION" in ""|dev|unknown) \ + echo "version is '$VERSION' although .git is present" >&2; \ + exit 1 ;; \ + esac; \ fi; \ + REVISION="$(git rev-parse --verify HEAD || echo unknown)"; \ CGO_ENABLED=1 GOOS=linux go build -o /routewatch -ldflags "\ - -X git.eeqj.de/sneak/routewatch/internal/version.GitRevision=$(git rev-parse --verify HEAD || echo unknown) \ - -X git.eeqj.de/sneak/routewatch/internal/version.GitRevisionShort=$version" \ + -X git.eeqj.de/sneak/routewatch/internal/version.GitRevision=$REVISION \ + -X git.eeqj.de/sneak/routewatch/internal/version.GitRevisionShort=${VERSION:-unknown}" \ ./cmd/routewatch # Create source archive with vendored dependencies @@ -64,7 +90,7 @@ RUN tar --zstd -cf /routewatch-source.tar.zst \ --exclude='*.tar.zst' \ . -# Runtime stage +# Runtime stage, and the last one # debian:bookworm-slim, 2026-09-21 FROM debian@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251 @@ -76,6 +102,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ && rm -rf /var/lib/apt/lists/* +ARG CACHEBUST + # Create non-root user RUN useradd -r -u 1000 -m routewatch diff --git a/README.md b/README.md index 0c2adac..dc40d61 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,6 @@ table in real time: it streams BGP UPDATE messages from the RIPE RIS Live service, maintains a live routing table in SQLite, and provides HTTP APIs for querying routing information. - ## Features - Real-time streaming of BGP updates from RIPE RIS Live @@ -38,6 +37,7 @@ DEBUG=routewatch ./routewatch ## HTTP Endpoints ### Web Interface + - `GET /` - Redirects to /status - `GET /status` - HTML status dashboard - `GET /status.json` - JSON statistics @@ -48,6 +48,7 @@ DEBUG=routewatch ./routewatch - `GET /ip/{ip}` - Redirects to prefix containing the IP ### API v1 + - `GET /api/v1/stats` - Detailed statistics with handler metrics - `GET /api/v1/ip/{ip}` - Look up AS information for an IP address - `GET /api/v1/as/{asn}` - Get prefixes announced by an AS @@ -107,52 +108,57 @@ routewatch/ ### Execution Flow 1. **Startup** (`cmd/routewatch/main.go` → `internal/routewatch/cli.go`) - - Uber fx dependency injection initializes all components - - Signal handlers registered for graceful shutdown + - Uber fx dependency injection initializes all components + - Signal handlers registered for graceful shutdown 2. **Initialization** (`internal/routewatch/app.go`) - - Database created with SQLite schema (WAL mode, 3GB cache) - - Message handlers registered with the streamer - - HTTP server started on configured port + - Database created with SQLite schema (WAL mode, 3GB cache) + - Message handlers registered with the streamer + - HTTP server started on configured port 3. **Message Processing Pipeline** - ``` - RIS Live Stream → JSON Parser → Message Dispatcher → Handler Queues → Batch Writers → SQLite - ``` - - Streamer connects to `ris-live.ripe.net` via HTTP - - Parses BGP UPDATE messages from JSON stream - - Dispatches to registered handlers based on message type - - Each handler has its own queue with backpressure handling - - Handlers batch writes for efficiency (25K-30K ops, 1-2s timeout) + + ``` + RIS Live Stream → JSON Parser → Message Dispatcher → Handler Queues → Batch Writers → SQLite + ``` + + - Streamer connects to `ris-live.ripe.net` via HTTP + - Parses BGP UPDATE messages from JSON stream + - Dispatches to registered handlers based on message type + - Each handler has its own queue with backpressure handling + - Handlers batch writes for efficiency (25K-30K ops, 1-2s timeout) 4. **Handler Details** - - **ASHandler**: Tracks all ASNs seen in AS paths - - **PeerHandler**: Records BGP peer information - - **PrefixHandler**: Maintains live routing table (upserts on announcement, deletes on withdrawal, - deletes all of a peer's routes when its BGP session goes down). Routes are kept per peer IP, so - when one peer IP has sessions on two collectors, a session going down on one also removes the - routes the other still carries, until they are announced again. - - **PeeringHandler**: Extracts AS peering relationships from AS paths + - **ASHandler**: Tracks all ASNs seen in AS paths + - **PeerHandler**: Records BGP peer information + - **PrefixHandler**: Maintains live routing table (upserts on announcement, + deletes on withdrawal, deletes all of a peer's routes when its BGP session + goes down). Routes are kept per peer IP, so when one peer IP has sessions + on two collectors, a session going down on one also removes the routes the + other still carries, until they are announced again. + - **PeeringHandler**: Extracts AS peering relationships from AS paths 5. **HTTP Request Flow** - ``` - Request → Chi Router → Middleware (timeout, logging) → Handler → Database Query → Response - ``` + ``` + Request → Chi Router → Middleware (timeout, logging) → Handler → Database Query → Response + ``` ### Key Design Patterns - **Batched Writes**: All database operations are batched for performance -- **Backpressure**: Probabilistic message dropping when queues exceed 50% capacity +- **Backpressure**: Probabilistic message dropping when queues exceed 50% + capacity - **Graceful Shutdown**: 60-second timeout, flushes all pending batches -- **Reconnection**: Any end of the stream other than shutdown, including RIS Live - ending the response normally, marks the feed disconnected and reconnects with - exponential backoff (5s-320s). The backoff resets after a stream that lasted - more than 30s. A stream with no line for 60s is dropped and reconnected. +- **Reconnection**: Any end of the stream other than shutdown, including RIS + Live ending the response normally, marks the feed disconnected and reconnects + with exponential backoff (5s-320s). The backoff resets after a stream that + lasted more than 30s. A stream with no line for 60s is dropped and + reconnected. - **IP Lookup**: the most specific live route is found by looking up the - address's prefix at each mask length, longest first, in the prefix index - (at most 25 lookups for IPv4, 113 for IPv6); prefixes are stored in the - text form Go's `net/netip` prints. Routes shorter than /8 for IPv4 or /16 - for IPv6, such as a default route, are not used. + address's prefix at each mask length, longest first, in the prefix index (at + most 25 lookups for IPv4, 113 for IPv6); prefixes are stored in the text form + Go's `net/netip` prints. Routes shorter than /8 for IPv4 or /16 for IPv6, such + as a default route, are not used. ### Database Schema @@ -177,23 +183,24 @@ bgp_peers(id, peer_ip, peer_asn, last_message_type, last_seen) Configuration is handled via environment variables and OS-specific paths: -| Variable | Default | Description | -|----------|----------|-------------| -| `PORT` | `8080` | HTTP server port, a whole number from 1 to 65535 | -| `DEBUG` | (empty) | Set to `routewatch` for debug logging | -| `XDG_DATA_HOME` | `/var/lib` (in the Docker image) | Base of the state directory; must be an absolute path | -| `GOMEMLIMIT` | `1536MiB` (in the Docker image) | Go soft memory limit; see Memory | -| `MALLOC_ARENA_MAX` | `2` (in the Docker image) | glibc malloc arena cap, a positive whole number; see Memory | +| Variable | Default | Description | +| ------------------ | -------------------------------- | ----------------------------------------------------------- | +| `PORT` | `8080` | HTTP server port, a whole number from 1 to 65535 | +| `DEBUG` | (empty) | Set to `routewatch` for debug logging | +| `XDG_DATA_HOME` | `/var/lib` (in the Docker image) | Base of the state directory; must be an absolute path | +| `GOMEMLIMIT` | `1536MiB` (in the Docker image) | Go soft memory limit; see Memory | +| `MALLOC_ARENA_MAX` | `2` (in the Docker image) | glibc malloc arena cap, a positive whole number; see Memory | A variable that is set to an invalid value stops the start with an error and a non-zero exit. An empty variable counts as unset. State directory (database location): + - macOS: `~/Library/Application Support/routewatch/` -- Linux: `/var/lib/berlin.sneak.app.routewatch/` when running as root, - otherwise `$XDG_DATA_HOME/berlin.sneak.app.routewatch/` (with `XDG_DATA_HOME` - unset, `~/.local/share/berlin.sneak.app.routewatch/`). In the Docker image - this is `/var/lib/berlin.sneak.app.routewatch/`. +- Linux: `/var/lib/berlin.sneak.app.routewatch/` when running as root, otherwise + `$XDG_DATA_HOME/berlin.sneak.app.routewatch/` (with `XDG_DATA_HOME` unset, + `~/.local/share/berlin.sneak.app.routewatch/`). In the Docker image this is + `/var/lib/berlin.sneak.app.routewatch/`. ## Memory @@ -201,6 +208,7 @@ The daemon holds a live routing table, so its memory grows with the size of the data it tracks. The image sets ceilings that keep it inside a 5 GiB container. Budget: + - Go heap: a 1.5 GiB soft limit (`GOMEMLIMIT=1536MiB`, set in the image). - SQLite: at most 640 MiB of page cache across the connection pool (64 MiB per connection, 10 connections) and a 1.5 GiB hard heap limit for the C library. @@ -223,11 +231,12 @@ or the equivalent in your deployment tool. This leaves headroom above the ceilings for spikes and the kernel page cache. Override the Go soft limit by setting `GOMEMLIMIT` in the environment (for -example `-e GOMEMLIMIT=1GiB`); this replaces the image default. `MALLOC_ARENA_MAX` -can be overridden the same way, but raising it lets RSS climb again on a -many-core host. +example `-e GOMEMLIMIT=1GiB`); this replaces the image default. +`MALLOC_ARENA_MAX` can be overridden the same way, but raising it lets RSS climb +again on a many-core host. What happens at each limit: + - Go soft limit: as the heap approaches `GOMEMLIMIT`, the runtime runs garbage collection more aggressively rather than growing further. - SQLite: at a 1 GiB soft heap limit it recycles its page cache instead of @@ -264,8 +273,8 @@ What the [upaas](https://git.eeqj.de/sneak/upaas) app needs: limit, so on a host with swap Docker allows the same amount of swap again. - Health check: the image's `HEALTHCHECK` requests `/.well-known/healthcheck.json` on the container port. upaas reads the - container's health 60 seconds after a deploy and fails the deploy unless it - is `healthy`. + container's health 60 seconds after a deploy and fails the deploy unless it is + `healthy`. ## Development @@ -288,35 +297,40 @@ make This repository adheres to the [Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all) standard: normalized scripts in `script/` are the entrypoints for the -development workflow, and the Makefile targets are thin shims that call -them. We provide: +development workflow, and the Makefile targets are thin shims that call them. We +provide: -- `script/bootstrap` — install all development dependencies (go, - golangci-lint, Go module download) +- `script/bootstrap` — install all development dependencies (git, make, go, the + pinned node and yarn, prettier from `yarn.lock`, Go module download) - `script/setup` — make a fresh clone ready for development: runs `script/bootstrap`, then `script/install-precommit` -- `script/projectname` — print the project name (used for the Docker - image tag) -- `script/test` — run the test suite - (`go test -short -timeout 30s -race -cover ./...`, verbose rerun on - failure). The `-short` flag skips the live-network integration test so the - default run is deterministic and offline. -- `script/lint` — run `go vet ./...` and `golangci-lint run` -- `script/fmt` — format all code (writes) +- `script/projectname` — print the project name (used for the Docker image tag) +- `script/test` — build the `test` phase of the `Dockerfile` alone, which runs + `go test -short -count=1 -timeout 90s -race -cover ./...` (verbose rerun on + failure). The `-short` flag skips the live-network integration test so the run + is deterministic and offline. +- `script/lint` — build the `lint` phase of the `Dockerfile` alone, which runs + golangci-lint v2.14.0 with `.golangci.yml` +- `script/fmt` — format the Go code with `go fmt` and the markdown with prettier + (writes) - `script/fmt-check` — check formatting (read-only) -- `script/check` — run `script/test`, `script/lint`, and - `script/fmt-check` -- `script/docker` — build the Docker image tagged via - `script/projectname` -- `script/cibuild` — CI entrypoint: `docker build .` -- `script/precommit` — pre-commit gate: `go mod tidy` + `go fmt` (must - not change files), then `script/check` -- `script/install-precommit` — install the git pre-commit hook that - runs `script/precommit` +- `script/check` — run `script/test`, `script/lint`, and `script/fmt-check` +- `script/docker` — build the Docker image tagged via `script/projectname` +- `script/cibuild` — CI entrypoint: runs `script/bootstrap` and `script/check`, + then builds the image +- `script/precommit` — pre-commit gate: `go mod tidy` + `go fmt` (must not + change files), then `script/check` +- `script/install-precommit` — install the git pre-commit hook that runs + `script/precommit` -The live-network integration test `TestRouteWatchLiveFeed` streams the RIPE -RIS feed for a few seconds and is skipped in short mode. To run it on demand, -invoke `go test` directly without `-short`: +Tests and the linter run only in Docker, as the `test` and `lint` phases of the +`Dockerfile`; the image cannot be built unless both pass. Every `docker build` +in `script/` passes a new `CACHEBUST` value, so each run executes every check, +while Go's build cache and golangci-lint's cache are kept between builds. + +The live-network integration test `TestRouteWatchLiveFeed` streams the RIPE RIS +feed for a few seconds and is skipped in short mode. To run it on demand, invoke +`go test` directly without `-short`: `go test -run TestRouteWatchLiveFeed ./internal/routewatch/`. ## License diff --git a/REPO_POLICIES.md b/REPO_POLICIES.md index bc2f161..1067698 100644 --- a/REPO_POLICIES.md +++ b/REPO_POLICIES.md @@ -1,6 +1,6 @@ --- title: Repository Policies -last_modified: 2026-07-06 +last_modified: 2026-10-18 --- This document covers repository structure, tooling, and workflow standards. Code @@ -54,21 +54,37 @@ style conventions are in separate documents: `cibuild`. `script/bootstrap` installs all dependencies idempotently and assumes nothing is present: base tools come from nix, apt, brew, or apk (detected in that order; apt runs noninteractive). For node it uses the - installed node if present; otherwise it installs a PINNED node version via - nvm, first installing nvm itself if missing — from a hash-verified GitHub - release archive (never `curl | sh`), with bash installed as an explicit - prerequisite since nvm requires bash. yarn is then pinned via - `corepack prepare yarn@ --activate`. Never install "latest" or "lts"; - always exact versions. `script/cibuild` runs the CI build: it changes to the - repo root and runs `docker build .`; the Gitea workflow calls it. Four further - scripts are our own extensions to the standard: `script/check` runs - `script/test`, `script/lint`, and `script/fmt-check`; `script/precommit` is - what the git pre-commit hook runs, and it calls `script/check`; - `script/install-precommit` installs the git pre-commit hook (the `make hooks` - target shims to it); and `script/projectname` (literally that filename) simply - outputs the project's name. Scripts that need the name call - `script/projectname` — e.g. `script/docker` assembles its image tag from it — - so those scripts stay byte-identical across all repos. Repo-type-specific + installed node only when its major version is the pinned one; otherwise (node + missing, or another major, such as the node 24 the CI runner image ships) it + installs the PINNED node version via nvm and installs yarn and the packages + under it. nvm itself is installed first if missing, from a hash-verified + GitHub release archive (never `curl | sh`), with bash installed as an explicit + prerequisite since nvm requires bash. Only the major version is compared + because the `Dockerfile` stages start from a node image whose exact version is + not the pin, and nvm cannot install a prebuilt node on alpine. yarn is pinned + via `corepack prepare yarn@ --activate`. Never install "latest" or + "lts"; always exact versions. `script/cibuild` runs the CI build: it changes + to the repo root, runs `script/bootstrap`, runs `script/check`, and builds the + image with the version; the Gitea workflow calls it. **`script/cibuild` runs + `script/bootstrap` first**, because the workflow checks out the repo and runs + nothing else, while `script/fmt-check` runs the formatter on the host: on a + pristine checkout with nothing installed the run dies there, after the + containerised gates have passed. **The bootstrap alone is not enough**: when + `script/bootstrap` installs node and yarn under nvm it leaves neither on the + `PATH` of the shell that called it, so a bare `yarn` either exits 127 or runs + under another node. The host entrypoints that need yarn — `script/fmt` and + `script/fmt-check` — therefore use the same test as `script/bootstrap`: when + the node on `PATH` has the pinned major version they run the `yarn` on `PATH`, + and otherwise they load nvm through `$HOME/.nvm/nvm.sh` and run yarn under the + pinned node. A runner carrying nothing but docker and git then gets through + `script/check`. Four further scripts are our own extensions to the standard: + `script/check` runs `script/test`, `script/lint` and `script/fmt-check`; + `script/precommit` is what the git pre-commit hook runs, and it calls + `script/check`; `script/install-precommit` installs the git pre-commit hook + (the `make hooks` target shims to it); and `script/projectname` (literally + that filename) simply outputs the project's name. Scripts that need the name + call `script/projectname` — e.g. `script/docker` assembles its image tag from + it — so those scripts stay byte-identical across all repos. Repo-type-specific pre-commit extras (e.g. `go mod tidy` verification in Go repos) belong in `script/precommit`, not in the hook itself. Model scripts are at `https://git.eeqj.de/sneak/prompts/raw/branch/main/script/`. The README @@ -89,87 +105,290 @@ style conventions are in separate documents: contributor should be able to understand the entire development workflow by reading the Makefile. -- Every repo should have a `Dockerfile`. All Dockerfiles must run `make check` - as a build step so the build fails if the branch is not green. For non-server - repos, the Dockerfile should bring up a development environment and run - `make check`. For server repos, `make check` should run as an early build - stage before the final image is assembled. Dockerfiles install development - prerequisites by running `script/bootstrap` rather than duplicating installs - inline; COPY `script/` and the dependency manifests (`package.json` + - `yarn.lock`, `go.mod` + `go.sum`, etc.) before running it so the bootstrap - layer stays cached until dependencies change. +- Every repo should have a `Dockerfile`, and it carries the repo's gates: a + `lint` phase and a `test` phase, with the final stage depending on both so the + image cannot be built unless they pass. For non-server repos the final stage + brings up a development environment; for server repos it is the runtime image. + The gate phases and the build stage start from their pinned base images and + install what those images lack either inline, as the canonical Go `Dockerfile` + below does for `git`, or by running `script/bootstrap`, as the `prompts` + repo's own `Dockerfile` does for its yarn packages. The development + environment stage installs development prerequisites by running + `script/bootstrap` rather than duplicating its installs inline. A stage that + runs `script/bootstrap` COPYs `script/` and the dependency manifests + (`package.json` + `yarn.lock`, `go.mod` + `go.sum`, etc.) before running it. -- **Dockerfiles must use a separate lint stage for fail-fast feedback.** Go - repos use a multistage build where linting runs in an independent stage based - on the `golangci/golangci-lint` image (pinned by hash). This stage runs - `make fmt-check` and `make lint` before the full build begins. The build stage - then declares an explicit dependency on the lint stage via - `COPY --from=lint /src/go.sum /dev/null`, which forces BuildKit to complete - linting before proceeding to compilation and tests. This ensures lint failures - surface in seconds rather than minutes, without blocking on dependency - download or compilation in the build stage. +- **Linting and testing run in Docker, as phases of the `Dockerfile`.** There is + no separate lint file. `script/lint` and `script/test` each build one phase + and nothing else: - The standard pattern for a Go repo Dockerfile is: + ```sh + docker build --build-arg CACHEBUST="$(date +%s)-$$" \ + --target lint --output type=cacheonly . + docker build --build-arg CACHEBUST="$(date +%s)-$$" \ + --target test --output type=cacheonly . + ``` + + **A stage that is not the last one in the file is built only when the final + stage's chain depends on it, or when `--target` names it.** That is why the + two gates are always invoked by name here, and why the final stage carries a + `COPY --from=` of a harmless file from each of them: without that edge a + plain `docker build .` builds the last stage alone and exits 0 having linted + and tested nothing. + + **The gate builds write no image.** With `--output type=cacheonly` the phase + runs and a failing step fails the build, but the result is not exported. + Nothing uses those images, and writing one out is slow: a Go test phase's + image holds the toolchain. A build given neither `--output` nor `-t` writes + an untagged image and leaves it dangling, on every developer host and every + CI runner. `script/cibuild` and `script/docker` build the image that ships + and tag it, so each build replaces the previous image; each assigns the tag + on its own line before the build, so `set -e` stops it where + `script/projectname` fails. + + Inside a phase the tool is invoked directly — `golangci-lint`, `go test`, + `eslint`, `prettier` — never through `make lint` or `script/test`, which are + themselves a `docker build` and would recurse into a daemon that does not + exist in a build step. Formatting is the exception and stays on the host: + `script/fmt` writes the working tree, and `script/fmt-check` is its + read-only twin. + + **No lint verdict may come from a host invocation of the linter.** On a + shared host golangci-lint reads a result cache keyed on file content rather + than location, so a second checkout of the same content is served the first + one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs + exit non-zero with `parallel golangci-lint is running` — a status a caller + cannot tell from real findings. Both have produced wrong verdicts in this + org, in both directions. A container has its own `TMPDIR`, so the lock is + out of reach. The lint phase does keep golangci-lint's cache between check + builds (below), but it lints every checkout at `/src`, and a finding is + taken from that cache only for the same files, settings and golangci-lint + version. + +- **Any build that runs checks is given a new `CACHEBUST` value.** Docker + invalidates a `COPY` layer only when the copied content changes, so on an + unchanged tree the check `RUN` is served from cache, nothing executes, and the + build still exits 0. Every `docker build` in `script/` therefore passes + `--build-arg CACHEBUST="$(date +%s)-$$"`, the time and the script's process + id, so no two builds share a value, not even the three in one + `script/cibuild`. `script/lint`, `script/test`, `script/cibuild` and + `script/docker` are the four, and there is no fifth — `script/check` runs the + two gate phases and `script/fmt-check`, and builds no image of its own. In + every stage of the `Dockerfile`, `ARG CACHEBUST` comes right after the steps + that install system packages and download Go's modules, or right after `FROM` + in a stage with neither, and every step that lints, tests or compiles comes + after it. A step that installs anything else as well, such as a + `script/bootstrap` that also installs node packages, comes after it too. A new + value reruns every `RUN` step after that line; `COPY` steps are compared by + content and need nothing. Docker keeps the layers above the line until the + base image, their own lines or the files they copy change. A stage without the + `ARG` line is served from cache on an unchanged tree, so a repository + re-vendors the scripts and adds the `ARG` lines in one commit. The scripts do + not pass `--no-cache`: it reruns every step too, but it also builds the layers + above the line again and empties every cache mount, so Go's build cache and + golangci-lint's, which the canonical Go `Dockerfile` below mounts so that they + survive between check builds, would start empty every time. A bare + `docker build .` is not evidence that anything ran: a sub-second build + reporting success is a cache hit, not a result. Never invalidate by pruning — + `docker builder prune` and friends destroy a build cache shared with every + other build on the host. When a check is added or changed, prove it works by + planting a defect it must catch and watching the run fail on it, then revert + the defect. A green run alone shows neither that the check ran nor that it + covers what it should. + +- **The gate phases are separate stages, and the build stage depends on both.** + The lint phase is based on the `golangci/golangci-lint` image (pinned by + hash), so lint failures surface in seconds rather than after a full compile, + and the test phase is based on the Debian Go image. The canonical Go repo + `Dockerfile`: ```dockerfile - # Lint stage — fast feedback on formatting and lint issues + # In every stage ARG CACHEBUST comes right after the steps that install + # system packages and download Go's modules, or right after FROM in a + # stage with neither, and every step that lints, tests or compiles + # comes after it. script/ passes a new value on every build, which + # reruns every RUN step after it. Docker keeps the layers above it + # until the base image, their own lines or go.mod and go.sum change. + # The steps that compile mount Go's build cache, and golangci-lint run + # also mounts golangci-lint's; both survive from build to build. + + # Lint phase # golangci/golangci-lint:v2.x.x, YYYY-MM-DD FROM golangci/golangci-lint@sha256:... AS lint WORKDIR /src COPY go.mod go.sum ./ RUN go mod download + ARG CACHEBUST COPY . . - RUN make fmt-check - RUN make lint + RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/root/.cache/golangci-lint \ + golangci-lint run --config .golangci.yml ./... - # Build stage - # golang:1.x-alpine, YYYY-MM-DD - FROM golang@sha256:... AS builder + # Test phase. -race needs cgo and so a C compiler, which the Debian Go + # image ships and the alpine one does not. -count=1 because the mounted + # cache also holds test results, which go test would otherwise replay. + # golang:1.x, YYYY-MM-DD + FROM golang@sha256:... AS test WORKDIR /src - - # Force BuildKit to run the lint stage before proceeding - COPY --from=lint /src/go.sum /dev/null - COPY go.mod go.sum ./ RUN go mod download + ARG CACHEBUST COPY . . - RUN make test + RUN --mount=type=cache,target=/root/.cache/go-build \ + go test -count=1 -timeout 90s -race -cover ./... || \ + { echo "--- Rerunning with -v for details ---"; \ + go test -count=1 -timeout 90s -race -v ./...; exit 1; } - ARG VERSION=dev - RUN CGO_ENABLED=0 go build -trimpath \ - -ldflags="-s -w -X main.Version=${VERSION}" \ - -o /app ./cmd/app/ + # Build stage. Nothing is wanted from either phase above; the copies + # are what make BuildKit build them first, so nothing after them can + # run unless lint and test passed. + # golang:1.x-alpine, YYYY-MM-DD + FROM golang@sha256:... AS builder + RUN apk add --no-cache git + WORKDIR /src + COPY go.mod go.sum ./ + RUN go mod download + ARG CACHEBUST + COPY --from=lint /src/go.sum /dev/null + COPY --from=test /src/go.sum /dev/null + # A tar-stream context keeps the sender's file owners, which git refuses. + RUN git config --system --add safe.directory /src + COPY . . - # Runtime stage + # The VERSION build arg when one is given, otherwise + # `git describe --tags --always` on the .git in the build context. With + # .git present, a version that is still empty, dev or unknown fails the + # build: git is missing or could not read the checkout. + ARG VERSION + RUN --mount=type=cache,target=/root/.cache/go-build \ + VERSION="${VERSION:-$(git describe --tags --always)}"; \ + if [ -e .git ]; then \ + case "$VERSION" in ""|dev|unknown) \ + echo "version is '$VERSION' although .git is present" >&2; \ + exit 1 ;; \ + esac; \ + fi; \ + CGO_ENABLED=0 go build -trimpath \ + -ldflags="-s -w -X main.Version=${VERSION}" \ + -o /app ./cmd/app/ + + # Runtime stage, and the last one FROM alpine@sha256:... + ARG CACHEBUST COPY --from=builder /app /usr/local/bin/app ENTRYPOINT ["app"] ``` Key points: - - The lint stage uses the `golangci/golangci-lint` image directly (it - includes both Go and the linter), so there is no need to install the - linter separately. - - `COPY --from=lint /src/go.sum /dev/null` is a no-op file copy that creates - a stage dependency. BuildKit runs stages in parallel by default; without - this line, the build stage would not wait for lint to finish and a lint - failure might not fail the overall build. + - The lint phase uses the `golangci/golangci-lint` image directly (it has + both Go and the linter), so nothing needs installing. + - `COPY --from= /src/go.sum /dev/null` is a no-op copy whose only + purpose is the ordering edge. BuildKit runs stages in parallel by default, + and a stage nothing depends on is not built at all, so without these two + lines a red gate would not fail the build. + - Keep the runtime stage last, and if you add a stage after it, give it the + same two copies. A plain `docker build .` builds the last stage's chain + and nothing else. + - Check builds keep these, and none of them can hand back a stale result: + - The layers above `ARG CACHEBUST`: the system packages and Go's + modules. Docker installs the packages again when the base image or the + install line changes, and downloads the modules again when `go.mod` or + `go.sum` changes. Go checks every module against `go.sum` as it + downloads it, and no later build can change a kept layer. The modules + are kept in a layer rather than a cache mount because a cache mount is + shared by every build on the host, and Go builds from whatever files + it holds without checking them again (only `go mod verify` does), so a + file changed there would go unnoticed. + - Go's build cache (`/root/.cache/go-build`, where Go keeps it for + root), mounted on the steps that compile: `golangci-lint run`, both + `go test` runs and `go build`. Go reuses a compiled package only when + all its inputs are unchanged. The cache also holds test results, which + is why both `go test` runs carry `-count=1`. + - golangci-lint's cache (`/root/.cache/golangci-lint`, where the pinned + image keeps it), mounted on `golangci-lint run`. golangci-lint reuses + a package's findings only when the package's files and those of every + package it imports, `go.mod`, the linters and their settings in + `.golangci.yml`, and the golangci-lint version are all unchanged. - If the project uses `//go:embed` directives that reference build artifacts - (e.g. a web frontend compiled in a separate stage), the lint stage must + (e.g. a web frontend compiled in a separate stage), the lint phase must create placeholder files so the embed directives resolve. Example: `RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`. - The lint stage should not depend on the actual build output — it exists to - fail fast. - - If the project requires CGO or system libraries for linting (e.g. - `vips-dev`), install them in the lint stage with `apk add`. - - The build stage runs `make test` after compilation setup. Tests run in the - build stage, not the lint stage, because they may require compiled - artifacts or heavier dependencies. + - If the project requires CGO or system libraries for linting, install them + in the lint phase. The `golangci/golangci-lint` image is Debian-based and + has no `apk`, so install with `apt-get` under the Debian package name + (`libvips-dev`, where alpine says `vips-dev`), and delete the package + lists in the same `RUN`, so the layer does not keep them. The test phase, + on the Debian Go image, installs the libraries its tests need the same + way. Either install goes right after `FROM`, above `ARG CACHEBUST`, so + Docker keeps it until the base image or the install line changes: + + ```dockerfile + FROM golangci/golangci-lint@sha256:... AS lint + RUN apt-get update \ + && apt-get install -y --no-install-recommends libvips-dev \ + && rm -rf /var/lib/apt/lists/* + WORKDIR /src + COPY go.mod go.sum ./ + RUN go mod download + ARG CACHEBUST + ``` + + - `.dockerignore` lets `.git` into the build context. It keeps out every git + `config` at any depth (`**/.git/config`, `**/.git/modules/**/config`): the + repository's own, each submodule's under `.git/modules/`, and that of a + submodule keeping its own `.git` directory. `git describe` does not need + them, and each can hold a credential: a password in a remote URL, or the + token the CI checkout step stores there. A submodule whose name has a + `config` segment (`config`, `deploy/config`, `config/lib`) loses its whole + git directory to `**/.git/modules/**/config`, and Go's version stamping + then fails the build: give it a name without that segment + (`git submodule add --name`). The stage that compiles has `git` (the + Debian Go image has it; an alpine one needs `apk add --no-cache git`) and + takes the version from the `VERSION` build argument when one is given, + otherwise from `git describe --tags --always`. That gives the tag on a + tagged commit; on a later commit, the tag, the number of commits since it + and the short commit (`v1.2.3-4-gabc1234`); and the short commit when no + tag is reachable. The stage that compiles also marks its working directory + safe for git (`git config --system --add safe.directory /src`): a context + sent as a tar stream keeps the sender's file owners, and git refuses a + checkout owned by another user, so the version would come out empty. + `ARG VERSION` has no default, and the build fails if the context carries + `.git` and the version still comes out empty, `dev` or `unknown`. A plain + `docker build .` with no build arguments must succeed; a Dockerfile that + refuses an empty build argument drops that refusal and keeps the argument. + A checkout whose `.git` is a file (a linked worktree, or a repository + checked out as a submodule) is the exception: that file points to a git + directory outside the build context, so the build cannot read the version + and a plain `docker build .` fails; pass the version with + `--build-arg VERSION=...`, as `script/docker` and `script/cibuild` already + do. - Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that - runs `script/cibuild` (which runs `docker build .`) on push. Since the - Dockerfile already runs `make check`, a successful build implies all checks - pass. + runs `script/cibuild` on push, and checks out the repo as its only other step, + with `persist-credentials: false`: `script/cibuild` needs no token, and + without it the checkout leaves the job's token in `.git/config` for every + later step. The checkout step also sets `fetch-depth: 0`, which fetches the + tags `git describe` needs: by default it clones shallow with no tags, and a + tagged repository's CI build would stamp a bare short commit id. The + workflow's `concurrency` block groups runs by workflow and branch + (`${{ github.workflow }}-${{ github.ref }}`) with `cancel-in-progress: true`, + so a new push cancels the older run on the same branch, queued or running, and + no other: runs for replaced commits do not hold up the shared runner. + `script/cibuild` bootstraps, runs the gate phases, and then builds the image, + so a successful run means every check passed; a bare `docker build .` does not + carry the same guarantee, because its gate phases may come from the cache. The + image build gets a `CACHEBUST` value of its own and so runs the gate phases a + second time. That is the price of the rule above, and it is worth paying: the + image that ships is built from a run of its own gates rather than from a cache + entry. The `check` job sets `timeout-minutes: 20`, so a hung build frees the + shared runner after 20 minutes. That allows for the three Docker builds + described above (the test phase, the lint phase, then the image), each held to + the 5-minute Docker build limit below, plus the bootstrap. A separate workflow + limited to `main` by a `branches` list under `on: push` cannot be checked by + review: to try a change to it, add the feature branch to that list and push, + then remove the branch from the list again before merging. Keep any job in it + that publishes behind `if: github.ref_name == 'main'`, so the run from the + feature branch publishes nothing. - Use platform-standard formatters: `black` for Python, `prettier` for JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with @@ -189,14 +408,21 @@ style conventions are in separate documents: module under test to verify it compiles/parses. There is no excuse for `make test` to be a no-op. -- `make test` must complete in under 20 seconds. Add a 30-second timeout in the - Makefile. +- `make test` must complete in under 60 seconds. That is the hard cap, and a + suite that exceeds it fails. Under 20 seconds is the target. A suite between + 20 and 60 seconds is still green, but the overage must be filed as an + improvement bug against that repo. Add a 90-second timeout to the test + invocation (`go test -timeout 90s`). The backstop deliberately sits above the + hard cap so that it catches a genuinely hung test rather than a merely slow + one. -- **`make test` should use the conditional verbose rerun pattern.** Run tests - without `-v` (verbose) first. If tests fail, automatically rerun with `-v` to - show full output. This keeps CI logs and `docker build` output clean on - success (just package/suite summaries) while providing full diagnostic detail - on failure (every test case, every assertion). The general shell pattern: +- **The test command should use the conditional verbose rerun pattern.** Run + tests without `-v` (verbose) first. If tests fail, automatically rerun with + `-v` to show full output. This keeps CI logs and `docker build` output clean + on success (just package/suite summaries) while providing full diagnostic + detail on failure (every test case, every assertion). The command lives in the + `test` phase of the `Dockerfile`, since `script/test` builds that phase; the + Makefile form below is the same pattern for any repo-local invocation: ```makefile test: @@ -209,11 +435,25 @@ style conventions are in separate documents: ```makefile test: - @go test -timeout 30s -race -cover ./... || \ + @go test -count=1 -timeout 90s -race -cover ./... || \ { echo "--- Rerunning with -v for details ---"; \ - go test -timeout 30s -race -v ./...; exit 1; } + go test -count=1 -timeout 90s -race -v ./...; exit 1; } ``` + `-count=1` is required on both invocations: it defeats Go's test _result_ + cache, so neither run can report a stored pass in place of running the + tests. It leaves the build cache alone, so it costs the runtime of the suite + and no recompilation. + + That cache is Go's own, separate from Docker's layer cache. Go stores a + passing result in its cache directory (`GOCACHE`), and when the same tests + run again on unchanged code it prints that result, marked `(cached)`, + without running them. The directory lasts from one run to the next on a + developer's machine, where this target runs, and in the `test` phase of the + `Dockerfile`, which mounts it to keep compiled packages between check + builds, so the phase's `go test` runs carry `-count=1` too. A new + `CACHEBUST` value (above) is what makes that step run on an unchanged tree. + Python example: ```makefile @@ -239,10 +479,89 @@ style conventions are in separate documents: must be in `.gitignore`. No exceptions. - `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`), - editor files (`.swp`, `*~`), language build artifacts, and `node_modules/`. - Fetch the standard `.gitignore` from + editor files (`.swp`, `*~`), in-repo agent scratch directories (`.claude/`), + `node_modules/`, and the repo's own build outputs. Fetch the standard + `.gitignore` from `https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when setting up - a new repo. + a new repo. A repo's `.gitignore` is the standard file followed by the repo's + own entries, such as its binaries; a re-vendor replaces the standard part and + keeps those entries. These patterns are written to `.gitignore`'s own + semantics, in which an unanchored pattern already matches at every depth; they + are not a `.dockerignore` and must not be transplanted into one unmodified. + +- **`.dockerignore` does not use `.gitignore` semantics, and copying patterns + across unmodified leaves secrets in the build context.** Docker matches with + `moby/patternmatcher`: `filepath.Match` semantics plus a `**` extension, so + `*` does not cross `/` and a pattern without a leading `**/` is anchored at + the build-context root. A `.dockerignore` listing `.env`, `*.pem` and `*.key` + therefore excludes only the copies at the repository root, while `config/.env` + and `certs/server.key` still reach the context and can land in an image layer + — which is more dangerous than a short file with no secret patterns at all, + because it reads as solved and stops anyone looking. Give every + depth-independent pattern the `**/` prefix and leave only genuinely + root-anchored entries unprefixed: `.claude`, and the repo's own host-built + binary, written `/myapp` and never `**/myapp`, which would also match + `cmd/myapp/` and delete the package directory from the context. Matching is + case-sensitive, and an ALL-CAPS twin per pattern still misses `Server.Key`, so + secret names use character ranges — `**/*.[kK][eE][yY]`, `**/*.[pP][eE][mM]`, + and likewise for `.envrc` and the extensionless SSH keys. Where such a pattern + also catches something the build needs, re-include it with a negation + (`!docs/example.env`); deleting the pattern reopens the exposure for every + other file it covers. Fetch the standard `.dockerignore` from + `https://git.eeqj.de/sneak/prompts/raw/branch/main/.dockerignore` and extend + it with the repo's own artifacts. + +- **In-repo agent scratch belongs in both files, written to each file's own + semantics.** `.claude/` holds one worktree per in-flight agent — an entire + additional checkout of the repo — so under `COPY . .` the build context + inflates by a multiple of the repo and another session's unreviewed work can + be copied into an image layer. In `.gitignore` the entry is `.claude/`, + unanchored. In `.dockerignore` it is `.claude`, anchored and with **no** `**/` + prefix, because the prefixed form would also delete any nested directory of + that name from the build. Anchoring carries a known gap that the canonical + `.dockerignore` states in its own comment, since consuming repos receive the + file and not the tracker: the directory is created in the agent's working + directory, so a repo running agents in subdirectories still ships + `services/api/.claude/` and must add its own anchored entry there. + +- **A plain `docker build .` of a clone stamps the version that + `git describe --tags --always` gives**, derived from the `.git` in the build + context as the canonical `Dockerfile` above shows. Without its failure check, + a missing `git` or an unreadable checkout would leave `-X main.Version=` empty + and the build would still exit 0. `script/docker` and `script/cibuild` pass + the version they compute on the host; it takes precedence. They do this + byte-identically across repos: + + ```sh + # The version and the tag each get their own line: a failing command + # substitution inside an argument does not trip `set -e`, so the inline + # form degrades to an empty constant. + version="$(git describe --tags --always --dirty 2>/dev/null || true)" + [ -n "$version" ] || version="unknown" + tag="$(script/projectname)" + docker build --build-arg CACHEBUST="$(date +%s)-$$" \ + --build-arg VERSION="$version" \ + -t "$tag" . + ``` + + `--always` makes an untagged repo yield an abbreviated commit hash rather + than failing, and the `[ -n "$version" ]` line is the single place the + fallback is applied — a live check that fires on a build from an export with + no `.git` and on a repository with no commits yet. Do not fold it into the + substitution as `|| echo unknown`, which makes the guard unreachable. The + Dockerfile's side is `ARG VERSION` in the stage that compiles, declared + there because `ARG` is stage-scoped; passing `VERSION` to a repo whose + Dockerfile declares no such `ARG` is ignored and costs nothing, which is why + the scripts stay byte-identical. One consequence for CI: the standard + checkout action clones shallow and fetches no tags, so the canonical + `.gitea/workflows/check.yml` sets `fetch-depth: 0` on its checkout step. + +- **Verify `.dockerignore` by enumerating the image, not by reading the + patterns.** Plant files at the root _and_ at least two directories deep, build + a probe image that does `COPY . .`, and list what actually landed + (`docker run --rm --entrypoint find IMAGE /app`). The `transferring context` + size is not a substitute: a nested secret is a few bytes, and BuildKit + transfers only the delta from the previous build. - **No build artifacts in version control.** Code-derived data (compiled bundles, minified output, generated assets) must never be committed to the @@ -258,9 +577,61 @@ style conventions are in separate documents: - Make all changes on a feature branch. You can do whatever you want on a feature branch. -- `.golangci.yml` is standardized and must _NEVER_ be modified by an agent, only - manually by the user. Fetch from - `https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml`. +- `.golangci.yml` is standardized. The vendored copy in a consuming repo must + _NEVER_ be modified by an agent: fetch it from + `https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml` and keep it + byte-identical, so that no repo can quietly loosen its own linting. Linter + configuration changes are made to the canonical copy in the `prompts` repo and + reach consuming repos by re-vendoring; an agent may open a PR against + canonical, which only the user merges. One list is exempt from byte-identity, + because it cannot be written once for every repo: the `deny` list of the + `test-support` depguard rule, where a repo names its own test-support packages + by full import path. A repo adds entries there and changes nothing else, and a + re-vendor carries its entries forward. The canonical golangci-lint version is + v2.14.0 (released 2026-09-24), pinned as the digest of the lint phase's base + image + (`golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f`, + which reports `2.14.0 built with go1.27.0 from 114493f9`). A module's `go` + directive must not name a newer Go minor version than the one golangci-lint + was built with, or golangci-lint refuses to lint it: this release lints + `go 1.27.1` but not `go 1.28`. That digest is the only pin, since no repo + installs golangci-lint on the host. A repo sets the lint phase digest to the + one named here and re-vendors `.golangci.yml` in the same commit, whichever of + the two prompted the change: the canonical copy can name linters that an older + golangci-lint rejects, and a newer golangci-lint can add linters that + `default: all` switches on until the canonical copy disables them. + +- **`script/bootstrap` installs a pinned tool by comparing versions, never by + testing presence.** An `if ! command -v ; then install; fi` guard tests + `PATH` only, so on an already-provisioned machine the pin is inert and a + version bump is a silent no-op — while the Dockerfile, installing into a clean + image, gets the pinned version, so a local `make check` and `make docker` can + disagree about what the tool even is. The canonical form: + - compares the installed version against the pin over the **whole** version + token; a parser that stops at the first `-` reports `2.12.2` for a host + running `2.12.2-rc1` and skips the install; + - treats absent, non-zero, empty or unrecognised `--version` output as a + mismatch, so the failure direction is a redundant install and never a + skipped one; + - after installing, re-resolves the binary the way callers do — `hash -r`, + then through `PATH`, not through the directory the installer wrote to — + and fails naming the resolved path, since an install that a shadowing + binary hides succeeds while changing nothing any caller sees; + - is actually called, and prints the version on both success paths: a + function defined and never invoked has the same exit status and the same + empty output as one that worked. + + Node is the exception to the whole-token comparison: the canonical + `script/bootstrap` compares only its major version, because the node 22 + alpine image the `Dockerfile` stages start from is not the exact pin, and + nvm cannot install a prebuilt node on alpine. + + Keep it POSIX sh: no arrays, no `[[`, no `grep -P`. + + A Go tool a repo needs on the host is installed with `go install` pinned to + a commit hash (`go install @`). It is never tracked as + a `go.mod` tool dependency or through a `tools.go` file, either of which + pulls the tool's own dependencies into the repo's `go.mod` and `go.sum`. - When pinning images or packages by hash, add a comment above the reference with the version and date (YYYY-MM-DD). @@ -371,15 +742,21 @@ style conventions are in separate documents: Never edit existing migrations after release. - All repos should have an `.editorconfig` enforcing the project's indentation - settings. + settings: the standard file from + `https://git.eeqj.de/sneak/prompts/raw/branch/main/.editorconfig`, which sets + tabs for `Makefile` and Go files, followed by the repo's own sections, such as + one for another language it uses. A re-vendor replaces the standard part and + keeps those sections. - Avoid putting files in the repo root unless necessary. Root should contain - only project-level config files (`README.md`, `Makefile`, `Dockerfile`, - `LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, and - language-specific config). Everything else goes in a subdirectory. Canonical - subdirectory names: + only project-level config files (`README.md`, `AGENTS.md`, `Makefile`, + `Dockerfile`, `LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, + and language-specific config). Everything else goes in a subdirectory. + Canonical subdirectory names: - `bin/` — executable scripts and tools - - `cmd/` — Go command entrypoints + - `cmd/` — Go command entrypoints; thin only: one `main.go` per binary whose + body is a single call into `internal/` or `pkg/`, no project logic in + `cmd/` - `configs/` — configuration templates and examples - `deploy/` — deployment manifests (k8s, compose, terraform) - `docs/` — documentation and markdown (README.md stays in root) @@ -406,3 +783,7 @@ style conventions are in separate documents: - Go: `go.mod`, `go.sum`, `.golangci.yml` - JS: `package.json`, `yarn.lock`, `.prettierrc`, `.prettierignore` - Python: `pyproject.toml` + +- Guidance for coding agents lives in one `AGENTS.md` at the repository root. It + is never committed under a file or directory named after one agent tool, such + as `CLAUDE.md` or `.claude/`, and never split into separate memory files. diff --git a/TODO.md b/TODO.md index d0b7920..ade4f7b 100644 --- a/TODO.md +++ b/TODO.md @@ -1,122 +1,121 @@ # Workflow -* branch (from `main`) -* do the work in Next Step -* move Next Step to the top of Completed Steps -* move the top item of Future Steps into Next Step -* commit (`TODO.md` changes in the same commit as the work) -* merge to `main` if the branch is not protected, otherwise open a PR -* push +- branch (from `main`) +- do the work in Next Step +- move Next Step to the top of Completed Steps +- move the top item of Future Steps into Next Step +- commit (`TODO.md` changes in the same commit as the work) +- merge to `main` if the branch is not protected, otherwise open a PR +- push # Status -pre-1.0. No git tags. The Docker build runs the format check, the linter -and the tests, and the Gitea workflow runs that build on every push. The -image sets memory ceilings for a 5 GiB container (README "Memory") and the -README says how to run it under upaas (README "Running under upaas"). A -35-hour run of `3898daa` on the live feed peaked at about 1 GiB, without a -container memory limit. +pre-1.0. No git tags. The linter and the tests are phases of the Docker build, +and the Gitea workflow runs `script/cibuild` (the format check of the Go code +and the markdown, both phases, then the image) on every push. The image sets +memory ceilings for a 5 GiB container (README "Memory") and the README says how +to run it under upaas (README "Running under upaas"). A 35-hour run of `3898daa` +on the live feed peaked at about 1 GiB, without a container memory limit. # Next Step `next` waits for sneak to merge it to `main` through -https://git.eeqj.de/sneak/routewatch/pulls/6. After that, setting -routewatch up under upaas on fsn1app1 and deploying it are his -(https://git.eeqj.de/sneak/routewatch/issues/31), and so is the run under a -real 5 GiB limit (https://git.eeqj.de/sneak/routewatch/issues/3). +https://git.eeqj.de/sneak/routewatch/pulls/6. After that, setting routewatch up +under upaas on fsn1app1 and deploying it are his +(https://git.eeqj.de/sneak/routewatch/issues/31), and so is the run under a real +5 GiB limit (https://git.eeqj.de/sneak/routewatch/issues/3). # Completed Steps -- 2026-10-03: `/api/v1/stats` serves the prefix distribution from memory, - seeded at startup and adjusted on every live-route write, so a request no - longer reads every live route (closes #30) -- 2026-10-03: a new database is created with `auto_vacuum` set to - incremental, through the connection string so it is set before the file - is first written, and each periodic incremental vacuum now returns up to - 1000 free pages; the late `PRAGMA auto_vacuum` in `Initialize`, which - SQLite ignored, is gone (closes #43) -- 2026-10-03: looking up an IP address no longer reads every IPv6 route: - both families find the most specific live route with at most 33 or 129 - lookups on the prefix index, and the IPv4 range columns are gone. Prefixes - from the feed are stored in one text form, so an IPv6 withdrawal, which the - feed sends uncompressed, now removes its route (closes #48) -- 2026-10-02: a plain `docker build .` stamps the commit's tag or short - commit (`git describe --tags --always`) into the page footer instead of - `unknown`: `.dockerignore` sends `.git` without `.git/config`, a `VERSION` - build argument takes precedence, and `make build` stamps the same value - (closes #46) +- 2026-10-19: the shared files are re-vendored from `sneak/prompts` at + `0653579`: golangci-lint v2.14.0 and its configuration, lint and test as + `Dockerfile` phases that keep Go's and golangci-lint's caches, markdown + formatted with prettier, agent guidance in `AGENTS.md`; the code is fixed for + what the new linter configuration finds (closes #52) +- 2026-10-03: `/api/v1/stats` serves the prefix distribution from memory, seeded + at startup and adjusted on every live-route write, so a request no longer + reads every live route (closes #30) +- 2026-10-03: a new database is created with `auto_vacuum` set to incremental, + through the connection string so it is set before the file is first written, + and each periodic incremental vacuum now returns up to 1000 free pages; the + late `PRAGMA auto_vacuum` in `Initialize`, which SQLite ignored, is gone + (closes #43) +- 2026-10-03: looking up an IP address no longer reads every IPv6 route: both + families find the most specific live route with at most 33 or 129 lookups on + the prefix index, and the IPv4 range columns are gone. Prefixes from the feed + are stored in one text form, so an IPv6 withdrawal, which the feed sends + uncompressed, now removes its route (closes #48) +- 2026-10-02: a plain `docker build .` stamps the commit's tag or short commit + (`git describe --tags --always`) into the page footer instead of `unknown`: + `.dockerignore` sends `.git` without `.git/config`, a `VERSION` build argument + takes precedence, and `make build` stamps the same value (closes #46) - 2026-09-29: the entrypoint creates the data directory if it is missing and - stops the start if a step fails; README "Running under upaas" no longer - asks for the host directory to be created first (closes #42) -- 2026-09-29: `.dockerignore` keeps `.git`, local build output, local - databases and `.env` out of the Docker build context, and so out of the - source archive in the image (closes #39) -- 2026-09-29: README first line names the MIT license and the author; - the License section now says MIT and links `LICENSE`, and an Author - section was added; this file brought up to date (closes #38) + stops the start if a step fails; README "Running under upaas" no longer asks + for the host directory to be created first (closes #42) +- 2026-09-29: `.dockerignore` keeps `.git`, local build output, local databases + and `.env` out of the Docker build context, and so out of the source archive + in the image (closes #39) +- 2026-09-29: README first line names the MIT license and the author; the + License section now says MIT and links `LICENSE`, and an Author section was + added; this file brought up to date (closes #38) - 2026-09-29: MIT `LICENSE` (closes #1) -- 2026-09-28: stopping the daemon while the feed is flowing no longer - panics with "send on closed channel": the read loop checks for a stop - just before handing a message to the handler queues, and a second - `Stop` no longer closes the queues again (closes #34) -- 2026-09-28: `docker stop` no longer kills the daemon 2 seconds after the - stop signal: the entrypoint switches to the `routewatch` user with - `setpriv` instead of `runuser`, so the daemon receives the signal itself - and gets the whole wait `docker stop` allows, up to its own 60-second - limit (closes #33) +- 2026-09-28: stopping the daemon while the feed is flowing no longer panics + with "send on closed channel": the read loop checks for a stop just before + handing a message to the handler queues, and a second `Stop` no longer closes + the queues again (closes #34) +- 2026-09-28: `docker stop` no longer kills the daemon 2 seconds after the stop + signal: the entrypoint switches to the `routewatch` user with `setpriv` + instead of `runuser`, so the daemon receives the signal itself and gets the + whole wait `docker stop` allows, up to its own 60-second limit (closes #33) - 2026-09-28: ready to run under upaas: a set but invalid `PORT`, - `XDG_DATA_HOME` or `MALLOC_ARENA_MAX` stops the start, the health - check follows `PORT`, README "Running under upaas" section (closes - #31) -- 2026-09-22: realtime in-memory database statistics: counts seeded at - startup and adjusted on every write, oldest/newest route timestamps via - index-end lookups; `/api/v1/stats` no longer scans the tables (closes - #27) -- 2026-09-21: batch writes take the write lock when their transaction - begins (`_txlock=immediate`), so they wait out a WAL checkpoint instead - of failing with "database is locked" (closes #25) -- 2026-09-21: `MALLOC_ARENA_MAX=2` in the image caps glibc malloc arenas, - so memory outside the Go runtime no longer grows with the core count - (closes #23) -- 2026-09-21: `GOMEMLIMIT=1536MiB` in the image; README Memory section - with the memory budget and the 5 GiB container limit (closes #13) -- 2026-09-21: the four handler queues hold at most 20,000 messages each, - down from 100,000 (closes #11) -- 2026-09-21: two goroutine leaks fixed: the stats handlers after a - timeout and the streamer's tickers on every reconnect (closes #12) -- 2026-09-21: parsed RIS messages no longer keep the unused `Community` - and `Raw` fields (closes #9) -- 2026-09-21: `.editorconfig`, and a Gitea workflow that runs - `script/cibuild` on every push (closes #14) -- 2026-09-21: the peering handler's AS-path map holds at most 500,000 - paths and is swapped for an empty one every 30 seconds instead of copied - (closes #10) -- 2026-09-21: SQLite memory bounded across the whole connection pool: a - 64 MiB page cache on each connection, 1 GiB soft and 1.5 GiB hard heap - limits (closes #8) -- 2026-09-21: the Docker build runs the format check, the linter and the - tests, linting in a separate stage on a golangci-lint image pinned by - digest (closes #5) + `XDG_DATA_HOME` or `MALLOC_ARENA_MAX` stops the start, the health check + follows `PORT`, README "Running under upaas" section (closes #31) +- 2026-09-22: realtime in-memory database statistics: counts seeded at startup + and adjusted on every write, oldest/newest route timestamps via index-end + lookups; `/api/v1/stats` no longer scans the tables (closes #27) +- 2026-09-21: batch writes take the write lock when their transaction begins + (`_txlock=immediate`), so they wait out a WAL checkpoint instead of failing + with "database is locked" (closes #25) +- 2026-09-21: `MALLOC_ARENA_MAX=2` in the image caps glibc malloc arenas, so + memory outside the Go runtime no longer grows with the core count (closes #23) +- 2026-09-21: `GOMEMLIMIT=1536MiB` in the image; README Memory section with the + memory budget and the 5 GiB container limit (closes #13) +- 2026-09-21: the four handler queues hold at most 20,000 messages each, down + from 100,000 (closes #11) +- 2026-09-21: two goroutine leaks fixed: the stats handlers after a timeout and + the streamer's tickers on every reconnect (closes #12) +- 2026-09-21: parsed RIS messages no longer keep the unused `Community` and + `Raw` fields (closes #9) +- 2026-09-21: `.editorconfig`, and a Gitea workflow that runs `script/cibuild` + on every push (closes #14) +- 2026-09-21: the peering handler's AS-path map holds at most 500,000 paths and + is swapped for an empty one every 30 seconds instead of copied (closes #10) +- 2026-09-21: SQLite memory bounded across the whole connection pool: a 64 MiB + page cache on each connection, 1 GiB soft and 1.5 GiB hard heap limits (closes + #8) +- 2026-09-21: the Docker build runs the format check, the linter and the tests, + linting in a separate stage on a golangci-lint image pinned by digest (closes + #5) - 2026-09-21: `make test` skips the live-network feed test (`-short`), so `make check` no longer depends on the network (closes #2) -- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints, - Makefile shims, README Entrypoints section -- 2026-02-22: repo policy compliance: required policy files, .gitignore - update, Makefile fmt-check/check/docker/hooks targets, gofmt pass +- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints, Makefile + shims, README Entrypoints section +- 2026-02-22: repo policy compliance: required policy files, .gitignore update, + Makefile fmt-check/check/docker/hooks targets, gofmt pass (repo-policies-compliance, unmerged) - 2026-01-01: WAL checkpointing: periodic (5s), on startup with logging, TRUNCATE mode; fixed slow queries -- 2025-12-31: status page: navbar, home page with search, oldest/newest - route timestamps, NULL handling in WHOIS stats; container runs as - routewatch user with proper state dir -- 2025-12-30: structured HTTP request logging, increased timeouts, - CIDR prefix URL routing fixes, status page metrics and footer +- 2025-12-31: status page: navbar, home page with search, oldest/newest route + timestamps, NULL handling in WHOIS stats; container runs as routewatch user + with proper state dir +- 2025-12-30: structured HTTP request logging, increased timeouts, CIDR prefix + URL routing fixes, status page metrics and footer - 2025-12-29: Dockerfile multi-stage build with source archive; SQLite incremental vacuum for non-blocking space reclamation - 2025-07-27: initial RouteWatch BGP stream monitor # Future Steps -- Production memory under 5 GiB: whether to test under a real 5 GiB - container limit on fsn1app1 is open for sneak +- Production memory under 5 GiB: whether to test under a real 5 GiB container + limit on fsn1app1 is open for sneak (https://git.eeqj.de/sneak/routewatch/issues/3) diff --git a/cmd/asinfo-gen/main.go b/cmd/asinfo-gen/main.go index 1d895b9..fbc0a50 100644 --- a/cmd/asinfo-gen/main.go +++ b/cmd/asinfo-gen/main.go @@ -2,8 +2,10 @@ package main import ( + "context" "encoding/csv" "encoding/json" + "errors" "io" "log" "net/http" @@ -27,13 +29,17 @@ func main() { // Fetch the CSV data log.Println("Fetching AS CSV data...") - resp, err := http.Get(asCsvURL) + + req, err := http.NewRequestWithContext( + context.Background(), http.MethodGet, asCsvURL, nil) + if err != nil { + log.Fatalf("Failed to fetch CSV: %v", err) + } + + resp, err := http.DefaultClient.Do(req) if err != nil { log.Fatalf("Failed to fetch CSV: %v", err) } - defer func() { - _ = resp.Body.Close() - }() if resp.StatusCode != http.StatusOK { log.Fatalf("Failed to fetch CSV: HTTP %d", resp.StatusCode) @@ -48,17 +54,20 @@ func main() { log.Fatalf("Failed to read CSV header: %v", err) } - if len(header) != 3 || header[0] != "asn" || header[1] != "handle" || header[2] != "description" { + if len(header) != 3 || header[0] != "asn" || header[1] != "handle" || + header[2] != "description" { log.Fatalf("Unexpected CSV header: %v", header) } // Read all records var asInfos []ASInfo + for { record, err := reader.Read() - if err == io.EOF { + if errors.Is(err, io.EOF) { break } + if err != nil { log.Printf("Error reading CSV record: %v", err) @@ -86,12 +95,18 @@ func main() { }) } + // Closed here rather than deferred: log.Fatalf exits without running + // deferred calls. + _ = resp.Body.Close() + log.Printf("Parsed %d AS records", len(asInfos)) // Convert to JSON and output to stdout encoder := json.NewEncoder(os.Stdout) encoder.SetIndent("", "\t") - if err := encoder.Encode(asInfos); err != nil { + + err = encoder.Encode(asInfos) + if err != nil { log.Fatalf("Failed to encode JSON: %v", err) } } diff --git a/cmd/streamdumper/main.go b/cmd/streamdumper/main.go index 7754d7a..95a0722 100644 --- a/cmd/streamdumper/main.go +++ b/cmd/streamdumper/main.go @@ -26,9 +26,17 @@ func main() { // Register raw message handler that prints to stdout s.RegisterRawHandler(func(line string) { - fmt.Println(line) + _, _ = fmt.Fprintln(os.Stdout, line) }) + // Start streaming before any defer: log.Fatal exits without running + // deferred calls. + err := s.Start(context.Background()) + if err != nil { + log.Fatal("Failed to start streamer:", err) + } + defer s.Stop() + // Set up context with cancellation ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -36,18 +44,13 @@ func main() { // Handle signals sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) + go func() { <-sigChan log.Println("Received shutdown signal") cancel() }() - // Start streaming - if err := s.Start(); err != nil { - log.Fatal("Failed to start streamer:", err) - } - defer s.Stop() - // Wait for context cancellation <-ctx.Done() } diff --git a/internal/config/config.go b/internal/config/config.go index 87a6ce3..efb290f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -2,6 +2,7 @@ package config import ( + "errors" "fmt" "os" "path/filepath" @@ -27,6 +28,12 @@ const ( maxPort = 65535 ) +var ( + errRelativeXDGDataHome = errors.New("XDG_DATA_HOME must be an absolute path") + errUnsupportedOS = errors.New("unsupported operating system") + errInvalidPort = errors.New("PORT must be a whole number") +) + // Config holds configuration for the entire application type Config struct { // StateDir is the directory for all application state (database, snapshots) @@ -38,11 +45,14 @@ type Config struct { // MaxRuntime is the maximum runtime (0 = run forever) MaxRuntime time.Duration - // EnableBatchedDatabaseWrites enables batched database operations for better performance + // EnableBatchedDatabaseWrites enables batched database operations for + // better performance EnableBatchedDatabaseWrites bool - // RouteExpirationTimeout is how long a route can go without being refreshed before expiring - // Default is 2 hours which is conservative for BGP (typical BGP hold time is 90-180 seconds) + // RouteExpirationTimeout is how long a route can go without being + // refreshed before expiring + // Default is 2 hours which is conservative for BGP (typical BGP hold + // time is 90-180 seconds) RouteExpirationTimeout time.Duration } @@ -59,11 +69,14 @@ func New() (*Config, error) { } return &Config{ - StateDir: stateDir, - Port: port, - MaxRuntime: 0, // Run forever by default - EnableBatchedDatabaseWrites: true, // Enable batching by default - RouteExpirationTimeout: defaultRouteExpirationMinutes * time.Minute, // For active route monitoring + StateDir: stateDir, + Port: port, + // Run forever by default + MaxRuntime: 0, + // Enable batching by default + EnableBatchedDatabaseWrites: true, + // For active route monitoring + RouteExpirationTimeout: defaultRouteExpirationMinutes * time.Minute, }, nil } @@ -89,7 +102,7 @@ func getStateDirectory() (string, error) { // the database somewhere unexpected. xdgData := os.Getenv("XDG_DATA_HOME") if xdgData != "" && !filepath.IsAbs(xdgData) { - return "", fmt.Errorf("XDG_DATA_HOME must be an absolute path, got %q", xdgData) + return "", fmt.Errorf("%w, got %q", errRelativeXDGDataHome, xdgData) } // Unix-like: /var/lib/berlin.sneak.app.routewatch if root, else XDG_DATA_HOME @@ -111,7 +124,7 @@ func getStateDirectory() (string, error) { return filepath.Join(home, ".local", "share", AppIdentifier), nil default: - return "", fmt.Errorf("unsupported operating system: %s", runtime.GOOS) + return "", fmt.Errorf("%w: %s", errUnsupportedOS, runtime.GOOS) } } @@ -125,7 +138,8 @@ func getPort() (int, error) { // ParseUint, unlike Atoi, refuses a sign: the health check URL cannot use "+9090" port, err := strconv.ParseUint(value, 10, 0) if err != nil || port < 1 || port > maxPort { - return 0, fmt.Errorf("PORT must be a whole number from 1 to %d, got %q", maxPort, value) + return 0, fmt.Errorf("%w from 1 to %d, got %q", + errInvalidPort, maxPort, value) } return int(port), nil @@ -134,7 +148,8 @@ func getPort() (int, error) { // EnsureDirectories creates all necessary directories if they don't exist func (c *Config) EnsureDirectories() error { // Ensure state directory exists - if err := os.MkdirAll(c.StateDir, dirPermissions); err != nil { + err := os.MkdirAll(c.StateDir, dirPermissions) + if err != nil { return fmt.Errorf("failed to create state directory: %w", err) } diff --git a/internal/config/config_test.go b/internal/config/config_internal_test.go similarity index 91% rename from internal/config/config_test.go rename to internal/config/config_internal_test.go index 84f9414..c9f10bd 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_internal_test.go @@ -36,7 +36,8 @@ func TestNewRefusesInvalidPort(t *testing.T) { t.Setenv("PORT", value) t.Setenv("XDG_DATA_HOME", "") - if _, err := New(); err == nil { + _, err := New() + if err == nil { t.Errorf("New() with PORT=%q returned no error", value) } }) @@ -51,12 +52,16 @@ func TestNewRefusesRelativeXDGDataHome(t *testing.T) { t.Setenv("PORT", "") t.Setenv("XDG_DATA_HOME", "relative/path") - if _, err := New(); err == nil { + + _, err := New() + if err == nil { t.Error("New() with a relative XDG_DATA_HOME returned no error") } t.Setenv("XDG_DATA_HOME", "/var/lib") - if _, err := New(); err != nil { + + _, err = New() + if err != nil { t.Errorf("New() with XDG_DATA_HOME=/var/lib: %v", err) } } diff --git a/internal/database/counts.go b/internal/database/counts.go index 71ce82f..5c8d4a5 100644 --- a/internal/database/counts.go +++ b/internal/database/counts.go @@ -33,8 +33,10 @@ type liveCounts struct { // seed sets every count to the value read from the tables at startup. It runs // before any writer, so it needs no coordination with the adjust methods. -func (c *liveCounts) seed(asns, prefixesV4, prefixesV6, peerings, peers, routesV4, routesV6 int, - distributionV4, distributionV6 []PrefixDistribution) { +func (c *liveCounts) seed( + asns, prefixesV4, prefixesV6, peerings, peers, routesV4, routesV6 int, + distributionV4, distributionV6 []PrefixDistribution, +) { c.mu.Lock() defer c.mu.Unlock() @@ -45,9 +47,11 @@ func (c *liveCounts) seed(asns, prefixesV4, prefixesV6, peerings, peers, routesV c.peers = peers c.routesV4 = routesV4 c.routesV6 = routesV6 + for _, entry := range distributionV4 { addAtMaskLength(c.distributionV4[:], entry.MaskLength, entry.Count) } + for _, entry := range distributionV6 { addAtMaskLength(c.distributionV6[:], entry.MaskLength, entry.Count) } @@ -102,6 +106,7 @@ func (c *liveCounts) addToDistribution(maskLengthsV4, maskLengthsV6 []int, n int for _, maskLength := range maskLengthsV4 { addAtMaskLength(c.distributionV4[:], maskLength, n) } + for _, maskLength := range maskLengthsV6 { addAtMaskLength(c.distributionV6[:], maskLength, n) } @@ -138,9 +143,11 @@ func (c *liveCounts) fill(s *Stats) { // ascending order, the way the distribution query returns them. func distributionList(counts []int) []PrefixDistribution { var list []PrefixDistribution + for maskLength, count := range counts { if count > 0 { - list = append(list, PrefixDistribution{MaskLength: maskLength, Count: count}) + list = append(list, + PrefixDistribution{MaskLength: maskLength, Count: count}) } } @@ -152,7 +159,8 @@ func distributionList(counts []int) []PrefixDistribution { func (d *Database) countRows(ctx context.Context, table string) (int, error) { var n int // table is one of a fixed set of literals below, never external input. - if err := d.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM "+table).Scan(&n); err != nil { + err := d.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM "+table).Scan(&n) + if err != nil { return 0, fmt.Errorf("failed to count %s: %w", table, err) } @@ -167,30 +175,37 @@ func (d *Database) seedCounts(ctx context.Context) error { if err != nil { return err } + prefixesV4, err := d.countRows(ctx, "prefixes_v4") if err != nil { return err } + prefixesV6, err := d.countRows(ctx, "prefixes_v6") if err != nil { return err } + peerings, err := d.countRows(ctx, "peerings") if err != nil { return err } + peers, err := d.countRows(ctx, "bgp_peers") if err != nil { return err } - routesV4, err := d.countRows(ctx, "live_routes_v4") + + routesV4, err := d.countRows(ctx, tableLiveRoutesV4) if err != nil { return err } - routesV6, err := d.countRows(ctx, "live_routes_v6") + + routesV6, err := d.countRows(ctx, tableLiveRoutesV6) if err != nil { return err } + distributionV4, distributionV6, err := d.GetPrefixDistributionContext(ctx) if err != nil { return err diff --git a/internal/database/counts_test.go b/internal/database/counts_internal_test.go similarity index 80% rename from internal/database/counts_test.go rename to internal/database/counts_internal_test.go index 5cd62b1..c5decf7 100644 --- a/internal/database/counts_test.go +++ b/internal/database/counts_internal_test.go @@ -12,6 +12,12 @@ import ( "github.com/google/uuid" ) +// testV4PeerIP is the peer the IPv4 routes in these tests are learned from. +const testV4PeerIP = "192.0.2.1" + +// testV6Prefix is the prefix of the IPv6 routes in these tests. +const testV6Prefix = "2001:db8::/32" + // mkV4Route builds an IPv4 live route with its mask length taken from the // prefix. func mkV4Route(t *testing.T, prefix string, asn int, ts time.Time) *LiveRoute { @@ -28,7 +34,7 @@ func mkV4Route(t *testing.T, prefix string, asn int, ts time.Time) *LiveRoute { MaskLength: maskLength, IPVersion: ipVersionV4, OriginASN: asn, - PeerIP: "192.0.2.1", + PeerIP: testV4PeerIP, ASPath: []int{asn}, NextHop: "192.0.2.254", LastUpdated: ts, @@ -56,6 +62,8 @@ func mkV6Route(prefix string, asn int, ts time.Time) *LiveRoute { // deleted. These counts are what /api/v1/stats reports; before this change the // endpoint recomputed them with a COUNT(*) over each table on every request. func TestLiveCountsTrackWritesInRealtime(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -70,6 +78,7 @@ func TestLiveCountsTrackWritesInRealtime(t *testing.T) { if err != nil { t.Fatalf("GetStatsContext on empty database: %v", err) } + if empty.ASNs != 0 || empty.Prefixes != 0 || empty.Peerings != 0 || empty.Peers != 0 || empty.LiveRoutes != 0 { t.Fatalf("empty database counts nonzero: %+v", empty) @@ -77,30 +86,16 @@ func TestLiveCountsTrackWritesInRealtime(t *testing.T) { ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) - if err := db.GetOrCreateASNBatch(map[int]time.Time{64500: ts, 64501: ts}); err != nil { - t.Fatalf("GetOrCreateASNBatch: %v", err) - } - if err := db.UpdatePrefixesBatch(map[string]time.Time{ - "198.51.100.0/24": ts, - "2001:db8::/32": ts, - }); err != nil { - t.Fatalf("UpdatePrefixesBatch: %v", err) - } - if err := db.UpdatePeerBatch(map[string]PeerUpdate{ - "192.0.2.1": {PeerIP: "192.0.2.1", PeerASN: 64500, MessageType: "UPDATE", Timestamp: ts}, - }); err != nil { - t.Fatalf("UpdatePeerBatch: %v", err) - } - if err := db.RecordPeering(64500, 64501, ts); err != nil { - t.Fatalf("RecordPeering: %v", err) - } + writeASNsPrefixesAndPeers(t, db, ts) routes := []*LiveRoute{ mkV4Route(t, "198.51.100.0/24", 64500, ts), mkV4Route(t, "203.0.113.0/24", 64501, ts.Add(time.Minute)), - mkV6Route("2001:db8::/32", 64502, ts.Add(2*time.Minute)), + mkV6Route(testV6Prefix, 64502, ts.Add(2*time.Minute)), } - if err := db.UpsertLiveRouteBatch(routes); err != nil { + + err = db.UpsertLiveRouteBatch(routes) + if err != nil { t.Fatalf("UpsertLiveRouteBatch: %v", err) } @@ -108,44 +103,87 @@ func TestLiveCountsTrackWritesInRealtime(t *testing.T) { if err != nil { t.Fatalf("GetStatsContext: %v", err) } + assertCounts(t, "after inserts", stats, wantCounts{ asns: 2, prefixes: 2, peerings: 1, peers: 1, ipv4Routes: 2, ipv6Routes: 1, liveRoutes: 3, }) // Re-announcing the same routes is an update, not an insert: counts hold. - if err := db.UpsertLiveRouteBatch(routes); err != nil { + err = db.UpsertLiveRouteBatch(routes) + if err != nil { t.Fatalf("UpsertLiveRouteBatch (re-announce): %v", err) } + stats, err = db.GetStatsContext(ctx) if err != nil { t.Fatalf("GetStatsContext: %v", err) } + assertCounts(t, "after re-announce", stats, wantCounts{ asns: 2, prefixes: 2, peerings: 1, peers: 1, ipv4Routes: 2, ipv6Routes: 1, liveRoutes: 3, }) // A withdrawal removes one route. - if err := db.DeleteLiveRouteBatch([]LiveRouteDeletion{ - {Prefix: "203.0.113.0/24", PeerIP: "192.0.2.1", IPVersion: ipVersionV4}, - }); err != nil { + err = db.DeleteLiveRouteBatch([]LiveRouteDeletion{ + {Prefix: "203.0.113.0/24", PeerIP: testV4PeerIP, IPVersion: ipVersionV4}, + }) + if err != nil { t.Fatalf("DeleteLiveRouteBatch: %v", err) } + stats, err = db.GetStatsContext(ctx) if err != nil { t.Fatalf("GetStatsContext: %v", err) } + assertCounts(t, "after delete", stats, wantCounts{ asns: 2, prefixes: 2, peerings: 1, peers: 1, ipv4Routes: 1, ipv6Routes: 1, liveRoutes: 2, }) } +// writeASNsPrefixesAndPeers writes two ASNs, two prefixes, one peer and one +// peering: the rows besides live routes that the stats count. +func writeASNsPrefixesAndPeers(t *testing.T, db *Database, ts time.Time) { + t.Helper() + + err := db.GetOrCreateASNBatch(map[int]time.Time{64500: ts, 64501: ts}) + if err != nil { + t.Fatalf("GetOrCreateASNBatch: %v", err) + } + + err = db.UpdatePrefixesBatch(map[string]time.Time{ + "198.51.100.0/24": ts, + testV6Prefix: ts, + }) + if err != nil { + t.Fatalf("UpdatePrefixesBatch: %v", err) + } + + err = db.UpdatePeerBatch(map[string]PeerUpdate{ + testV4PeerIP: { + PeerIP: testV4PeerIP, PeerASN: 64500, + MessageType: "UPDATE", Timestamp: ts, + }, + }) + if err != nil { + t.Fatalf("UpdatePeerBatch: %v", err) + } + + err = db.RecordPeering(64500, 64501, ts) + if err != nil { + t.Fatalf("RecordPeering: %v", err) + } +} + // TestLiveCountsSeededFromDatabaseAtStartup writes rows, reopens the same // database file, and checks the counts and the prefix distribution come back // from the seed scan rather than starting at zero. func TestLiveCountsSeededFromDatabaseAtStartup(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -154,16 +192,22 @@ func TestLiveCountsSeededFromDatabaseAtStartup(t *testing.T) { } ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) - if err := db.GetOrCreateASNBatch(map[int]time.Time{64500: ts, 64501: ts, 64502: ts}); err != nil { + + err = db.GetOrCreateASNBatch(map[int]time.Time{64500: ts, 64501: ts, 64502: ts}) + if err != nil { t.Fatalf("GetOrCreateASNBatch: %v", err) } - if err := db.UpsertLiveRouteBatch([]*LiveRoute{ + + err = db.UpsertLiveRouteBatch([]*LiveRoute{ mkV4Route(t, "198.51.100.0/24", 64500, ts), - mkV6Route("2001:db8::/32", 64502, ts), - }); err != nil { + mkV6Route(testV6Prefix, 64502, ts), + }) + if err != nil { t.Fatalf("UpsertLiveRouteBatch: %v", err) } - if err := db.Close(); err != nil { + + err = db.Close() + if err != nil { t.Fatalf("Close: %v", err) } @@ -177,13 +221,16 @@ func TestLiveCountsSeededFromDatabaseAtStartup(t *testing.T) { if err != nil { t.Fatalf("GetStatsContext after reopen: %v", err) } + if stats.ASNs != 3 { t.Errorf("seeded ASNs = %d, want 3", stats.ASNs) } + if stats.IPv4Routes != 1 || stats.IPv6Routes != 1 || stats.LiveRoutes != 2 { t.Errorf("seeded routes = (v4 %d, v6 %d, total %d), want (1, 1, 2)", stats.IPv4Routes, stats.IPv6Routes, stats.LiveRoutes) } + assertDistribution(t, "seeded IPv4 distribution", stats.IPv4PrefixDistribution, []PrefixDistribution{{MaskLength: 24, Count: 1}}) assertDistribution(t, "seeded IPv6 distribution", stats.IPv6PrefixDistribution, @@ -196,26 +243,91 @@ func TestLiveCountsSeededFromDatabaseAtStartup(t *testing.T) { // tables. The steps run once through the batch methods the prefix handler uses // and once through the single-route methods. func TestPrefixDistributionTracksWrites(t *testing.T) { + t.Parallel() + + steps := prefixDistributionSteps(t) + + for _, batch := range []bool{true, false} { + name := "single-route writes" + if batch { + name = "batch writes" + } + + t.Run(name, func(t *testing.T) { + t.Parallel() + + db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New()) + if err != nil { + t.Fatalf("failed to create database: %v", err) + } + defer func() { _ = db.Close() }() + + ctx := context.Background() + + for _, step := range steps { + err := announceRoutes(db, batch, step.announce) + if err != nil { + t.Fatalf("%s: announce: %v", step.name, err) + } + + err = withdrawRoutes(db, batch, step.withdraw) + if err != nil { + t.Fatalf("%s: withdraw: %v", step.name, err) + } + + stats, err := db.GetStatsContext(ctx) + if err != nil { + t.Fatalf("%s: GetStatsContext: %v", step.name, err) + } + + queryV4, queryV6, err := db.GetPrefixDistributionContext(ctx) + if err != nil { + t.Fatalf("%s: GetPrefixDistributionContext: %v", step.name, err) + } + + assertDistribution(t, step.name+": IPv4 distribution", + stats.IPv4PrefixDistribution, step.wantV4) + assertDistribution(t, step.name+": IPv6 distribution", + stats.IPv6PrefixDistribution, step.wantV6) + assertDistribution(t, step.name+": IPv4 distribution query", + queryV4, step.wantV4) + assertDistribution(t, step.name+": IPv6 distribution query", + queryV6, step.wantV6) + } + }) + } +} + +// prefixDistributionStep is one step of TestPrefixDistributionTracksWrites: +// the routes it announces, then the routes it withdraws, then the +// distributions both the stats and the distribution query must report. +type prefixDistributionStep struct { + name string + announce []*LiveRoute + withdraw []*LiveRoute + wantV4 []PrefixDistribution + wantV6 []PrefixDistribution +} + +// prefixDistributionSteps returns the steps of +// TestPrefixDistributionTracksWrites, in the order they run. +func prefixDistributionSteps(t *testing.T) []prefixDistributionStep { + t.Helper() + ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) shared := mkV4Route(t, "198.51.100.0/24", 64500, ts) sharedSecondPeer := mkV4Route(t, "198.51.100.0/24", 64500, ts) sharedSecondPeer.PeerIP = "192.0.2.2" other := mkV4Route(t, "203.0.113.0/24", 64501, ts) wide := mkV4Route(t, "172.16.0.0/16", 64502, ts) - v6 := mkV6Route("2001:db8::/32", 64503, ts) - v6SecondPeer := mkV6Route("2001:db8::/32", 64503, ts) + v6 := mkV6Route(testV6Prefix, 64503, ts) + v6SecondPeer := mkV6Route(testV6Prefix, 64503, ts) v6SecondPeer.PeerIP = "2001:db8::2" all := []PrefixDistribution{{MaskLength: 16, Count: 1}, {MaskLength: 24, Count: 2}} v6Only := []PrefixDistribution{{MaskLength: 32, Count: 1}} - steps := []struct { - name string - announce []*LiveRoute - withdraw []*LiveRoute - wantV4 []PrefixDistribution - wantV6 []PrefixDistribution - }{ + return []prefixDistributionStep{ { name: "new routes", announce: []*LiveRoute{shared, other, wide, v6}, @@ -239,7 +351,9 @@ func TestPrefixDistributionTracksWrites(t *testing.T) { { name: "withdrawal of the last route for a prefix", withdraw: []*LiveRoute{sharedSecondPeer, v6SecondPeer}, - wantV4: []PrefixDistribution{{MaskLength: 16, Count: 1}, {MaskLength: 24, Count: 1}}, + wantV4: []PrefixDistribution{ + {MaskLength: 16, Count: 1}, {MaskLength: 24, Count: 1}, + }, }, { name: "withdrawal of every remaining route", @@ -270,45 +384,6 @@ func TestPrefixDistributionTracksWrites(t *testing.T) { wantV6: v6Only, }, } - - for _, batch := range []bool{true, false} { - name := "single-route writes" - if batch { - name = "batch writes" - } - - t.Run(name, func(t *testing.T) { - db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New()) - if err != nil { - t.Fatalf("failed to create database: %v", err) - } - defer func() { _ = db.Close() }() - - ctx := context.Background() - for _, step := range steps { - if err := announceRoutes(db, batch, step.announce); err != nil { - t.Fatalf("%s: announce: %v", step.name, err) - } - if err := withdrawRoutes(db, batch, step.withdraw); err != nil { - t.Fatalf("%s: withdraw: %v", step.name, err) - } - - stats, err := db.GetStatsContext(ctx) - if err != nil { - t.Fatalf("%s: GetStatsContext: %v", step.name, err) - } - queryV4, queryV6, err := db.GetPrefixDistributionContext(ctx) - if err != nil { - t.Fatalf("%s: GetPrefixDistributionContext: %v", step.name, err) - } - - assertDistribution(t, step.name+": IPv4 distribution", stats.IPv4PrefixDistribution, step.wantV4) - assertDistribution(t, step.name+": IPv6 distribution", stats.IPv6PrefixDistribution, step.wantV6) - assertDistribution(t, step.name+": IPv4 distribution query", queryV4, step.wantV4) - assertDistribution(t, step.name+": IPv6 distribution query", queryV6, step.wantV6) - } - }) - } } // announceRoutes writes routes in one UpsertLiveRouteBatch, or with one @@ -317,8 +392,10 @@ func announceRoutes(db *Database, batch bool, routes []*LiveRoute) error { if batch { return db.UpsertLiveRouteBatch(routes) } + for _, route := range routes { - if err := db.UpsertLiveRoute(route); err != nil { + err := db.UpsertLiveRoute(route) + if err != nil { return err } } @@ -331,7 +408,8 @@ func announceRoutes(db *Database, batch bool, routes []*LiveRoute) error { func withdrawRoutes(db *Database, batch bool, routes []*LiveRoute) error { if !batch { for _, route := range routes { - if err := db.DeleteLiveRoute(route.Prefix, route.PeerIP); err != nil { + err := db.DeleteLiveRoute(route.Prefix, route.PeerIP) + if err != nil { return err } } @@ -354,6 +432,8 @@ func withdrawRoutes(db *Database, batch bool, routes []*LiveRoute) error { // routes stay, and the route counts and prefix distribution held in memory // equal what is read from the route tables. func TestDeleteLiveRoutesOfPeer(t *testing.T) { + t.Parallel() + const ( peerIP = "192.0.2.1" otherPeerIP = "192.0.2.2" @@ -375,14 +455,16 @@ func TestDeleteLiveRoutesOfPeer(t *testing.T) { } onlyV6 := mkV6Route("2001:db8:1::/48", 64503, ts) onlyV6.MaskLength = 48 - if err := db.UpsertLiveRouteBatch([]*LiveRoute{ + + err = db.UpsertLiveRouteBatch([]*LiveRoute{ ofPeer(peerIP, mkV4Route(t, sharedV4, 64500, ts)), ofPeer(otherPeerIP, mkV4Route(t, sharedV4, 64500, ts)), ofPeer(peerIP, mkV4Route(t, "203.0.113.0/24", 64501, ts)), ofPeer(peerIP, mkV6Route(sharedV6, 64502, ts)), ofPeer(otherPeerIP, mkV6Route(sharedV6, 64502, ts)), ofPeer(peerIP, onlyV6), - }); err != nil { + }) + if err != nil { t.Fatalf("UpsertLiveRouteBatch: %v", err) } @@ -392,40 +474,61 @@ func TestDeleteLiveRoutesOfPeer(t *testing.T) { if err != nil { t.Fatalf("DeleteLiveRoutesOfPeer: %v", err) } + if deleted != want { - t.Errorf("DeleteLiveRoutesOfPeer deleted %d routes, want %d", deleted, want) + t.Errorf("DeleteLiveRoutesOfPeer deleted %d routes, want %d", + deleted, want) } } - ctx := context.Background() for _, prefix := range []string{sharedV4, sharedV6} { - routes, _, err := db.GetPrefixDetailsContext(ctx, prefix) + routes, _, err := db.GetPrefixDetailsContext(context.Background(), prefix) if err != nil { t.Fatalf("GetPrefixDetailsContext(%s): %v", prefix, err) } + if len(routes) != 1 || routes[0].PeerIP != otherPeerIP { - t.Errorf("live routes for %s = %+v, want only the one from %s", prefix, routes, otherPeerIP) + t.Errorf("live routes for %s = %+v, want only the one from %s", + prefix, routes, otherPeerIP) } } + checkOneLiveRoutePerIPVersion(t, db) +} + +// checkOneLiveRoutePerIPVersion checks that the database has one IPv4 live +// route, for a /24, and one IPv6 live route, for a /32, both as counted in +// memory and as read from the route tables. +func checkOneLiveRoutePerIPVersion(t *testing.T, db *Database) { + t.Helper() + + ctx := context.Background() + stats, err := db.GetStatsContext(ctx) if err != nil { t.Fatalf("GetStatsContext: %v", err) } + tableV4, tableV6, err := db.GetLiveRouteCountsContext(ctx) if err != nil { t.Fatalf("GetLiveRouteCountsContext: %v", err) } - if stats.IPv4Routes != 1 || tableV4 != 1 || stats.IPv6Routes != 1 || tableV6 != 1 { - t.Errorf("live routes (IPv4, IPv6): (%d, %d) in memory, (%d, %d) in the tables, want (1, 1)", + + if stats.IPv4Routes != 1 || tableV4 != 1 || stats.IPv6Routes != 1 || + tableV6 != 1 { + t.Errorf("live routes (IPv4, IPv6): (%d, %d) in memory, (%d, %d) in "+ + "the tables, want (1, 1)", stats.IPv4Routes, stats.IPv6Routes, tableV4, tableV6) } + queryV4, queryV6, err := db.GetPrefixDistributionContext(ctx) if err != nil { t.Fatalf("GetPrefixDistributionContext: %v", err) } + wantV4 := []PrefixDistribution{{MaskLength: 24, Count: 1}} wantV6 := []PrefixDistribution{{MaskLength: 32, Count: 1}} + assertDistribution(t, "IPv4 distribution", stats.IPv4PrefixDistribution, wantV4) assertDistribution(t, "IPv6 distribution", stats.IPv6PrefixDistribution, wantV6) assertDistribution(t, "IPv4 distribution query", queryV4, wantV4) @@ -437,6 +540,8 @@ func TestDeleteLiveRoutesOfPeer(t *testing.T) { // MIN/MAX union query read its result into *time.Time, which the driver could // not parse, so it logged a warning every call and left both timestamps nil. func TestStatsRouteTimestamps(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -451,6 +556,7 @@ func TestStatsRouteTimestamps(t *testing.T) { if err != nil { t.Fatalf("GetStatsContext on empty database: %v", err) } + if empty.OldestRoute != nil || empty.NewestRoute != nil { t.Fatalf("empty database timestamps = (%v, %v), want (nil, nil)", empty.OldestRoute, empty.NewestRoute) @@ -459,11 +565,13 @@ func TestStatsRouteTimestamps(t *testing.T) { base := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) oldest := base newest := base.Add(2 * time.Minute) - if err := db.UpsertLiveRouteBatch([]*LiveRoute{ + + err = db.UpsertLiveRouteBatch([]*LiveRoute{ mkV4Route(t, "198.51.100.0/24", 64500, base.Add(time.Minute)), mkV4Route(t, "203.0.113.0/24", 64501, oldest), - mkV6Route("2001:db8::/32", 64502, newest), - }); err != nil { + mkV6Route(testV6Prefix, 64502, newest), + }) + if err != nil { t.Fatalf("UpsertLiveRouteBatch: %v", err) } @@ -471,9 +579,11 @@ func TestStatsRouteTimestamps(t *testing.T) { if err != nil { t.Fatalf("GetStatsContext: %v", err) } + if stats.OldestRoute == nil || !stats.OldestRoute.Equal(oldest) { t.Errorf("OldestRoute = %v, want %v", stats.OldestRoute, oldest) } + if stats.NewestRoute == nil || !stats.NewestRoute.Equal(newest) { t.Errorf("NewestRoute = %v, want %v", stats.NewestRoute, newest) } @@ -482,6 +592,8 @@ func TestStatsRouteTimestamps(t *testing.T) { // TestLiveCountsConcurrentReadWrite runs writers and stats readers at once so // the race detector proves the counters are safe under concurrent use. func TestLiveCountsConcurrentReadWrite(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -493,17 +605,21 @@ func TestLiveCountsConcurrentReadWrite(t *testing.T) { ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) const writers = 4 - var wg sync.WaitGroup + var wg sync.WaitGroup wg.Add(writers) + for w := range writers { go func(base int) { defer wg.Done() + for i := range 25 { asn := 65000 + base*100 + i - route := mkV6Route("2001:db8::/32", asn, ts) + route := mkV6Route(testV6Prefix, asn, ts) route.PeerIP = "2001:db8::" + uuid.NewString() - if err := db.UpsertLiveRoute(route); err != nil { + + err := db.UpsertLiveRoute(route) + if err != nil { t.Errorf("UpsertLiveRoute: %v", err) return @@ -514,15 +630,19 @@ func TestLiveCountsConcurrentReadWrite(t *testing.T) { var readerWG sync.WaitGroup readerWG.Add(1) + stop := make(chan struct{}) + go func() { defer readerWG.Done() + for { select { case <-stop: return default: - if _, err := db.GetStatsContext(context.Background()); err != nil { + _, err := db.GetStatsContext(context.Background()) + if err != nil { t.Errorf("GetStatsContext: %v", err) return @@ -539,6 +659,7 @@ func TestLiveCountsConcurrentReadWrite(t *testing.T) { if err != nil { t.Fatalf("GetStatsContext: %v", err) } + if want := writers * 25; stats.IPv6Routes != want { t.Errorf("IPv6Routes = %d, want %d", stats.IPv6Routes, want) } @@ -563,21 +684,27 @@ func assertCounts(t *testing.T, when string, got Stats, want wantCounts) { if got.ASNs != want.asns { t.Errorf("%s: ASNs = %d, want %d", when, got.ASNs, want.asns) } + if got.Prefixes != want.prefixes { t.Errorf("%s: Prefixes = %d, want %d", when, got.Prefixes, want.prefixes) } + if got.Peerings != want.peerings { t.Errorf("%s: Peerings = %d, want %d", when, got.Peerings, want.peerings) } + if got.Peers != want.peers { t.Errorf("%s: Peers = %d, want %d", when, got.Peers, want.peers) } + if got.IPv4Routes != want.ipv4Routes { t.Errorf("%s: IPv4Routes = %d, want %d", when, got.IPv4Routes, want.ipv4Routes) } + if got.IPv6Routes != want.ipv6Routes { t.Errorf("%s: IPv6Routes = %d, want %d", when, got.IPv6Routes, want.ipv6Routes) } + if got.LiveRoutes != want.liveRoutes { t.Errorf("%s: LiveRoutes = %d, want %d", when, got.LiveRoutes, want.liveRoutes) } diff --git a/internal/database/database.go b/internal/database/database.go index 7005042..d17a505 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -1,4 +1,3 @@ -// Package database provides SQLite storage for BGP routing data including ASNs, prefixes, announcements and peerings. package database import ( @@ -36,6 +35,13 @@ const ( ipVersionV6 = 6 ipv4Bits = 32 ipv6Bits = 128 + + tableLiveRoutesV4 = "live_routes_v4" + tableLiveRoutesV6 = "live_routes_v6" + + // Names of the IP versions in error and log messages. + ipv4Family = "IPv4" + ipv6Family = "IPv6" ) // The IP lookup does not use routes shorter than these mask lengths. A default @@ -81,6 +87,9 @@ var ( ErrNoRoute = errors.New("no route found") // ErrNoStaleASN is returned when no ASN needs WHOIS refresh ErrNoStaleASN = errors.New("no stale ASN found") + + errInvalidASN = errors.New("invalid ASN") + errSameASNInPeering = errors.New("cannot create peering with same ASN") ) // Database manages the SQLite database connection and operations. @@ -103,7 +112,9 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) { // Ensure directory exists dir := filepath.Dir(dbPath) - if err := os.MkdirAll(dir, dirPermissions); err != nil { + + err := os.MkdirAll(dir, dirPermissions) + if err != nil { return nil, fmt.Errorf("failed to create database directory: %w", err) } @@ -124,12 +135,14 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) { sqliteCacheSizeKiB, sqliteBusyTimeoutMs, ) + db, err := sql.Open("sqlite3", dsn) if err != nil { return nil, fmt.Errorf("failed to open database: %w", err) } - if err := db.Ping(); err != nil { + err = db.PingContext(context.Background()) + if err != nil { return nil, fmt.Errorf("failed to ping database: %w", err) } @@ -140,16 +153,20 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) { db.SetMaxIdleConns(maxConns) db.SetConnMaxLifetime(0) - database := &Database{db: db, logger: logger, path: dbPath, counts: &liveCounts{}} + database := &Database{ + db: db, logger: logger, path: dbPath, counts: &liveCounts{}, + } - if err := database.Initialize(); err != nil { + err = database.Initialize() + if err != nil { return nil, fmt.Errorf("failed to initialize database: %w", err) } // Seed the in-memory statistics counters from the tables once, before the // streamer starts writing. From here on every write keeps them current, so // the stats endpoints never scan the tables to report counts. - if err := database.seedCounts(context.Background()); err != nil { + err = database.seedCounts(context.Background()) + if err != nil { return nil, fmt.Errorf("failed to seed statistics counters: %w", err) } @@ -171,14 +188,18 @@ func (d *Database) Initialize() error { } for _, pragma := range pragmas { - if err := d.exec(pragma); err != nil { + err := d.exec(pragma) + if err != nil { d.logger.Warn("Failed to set pragma", "pragma", pragma, "error", err) } } // Run WAL checkpoint on startup to consolidate any existing WAL var walPages, checkpointedPages, movedPages int - err := d.db.QueryRow("PRAGMA wal_checkpoint(TRUNCATE)").Scan(&walPages, &checkpointedPages, &movedPages) + + err := d.db.QueryRowContext(context.Background(), + "PRAGMA wal_checkpoint(TRUNCATE)", + ).Scan(&walPages, &checkpointedPages, &movedPages) if err != nil { d.logger.Warn("Failed to checkpoint WAL on startup", "error", err) } else { @@ -202,43 +223,6 @@ func (d *Database) Close() error { return d.db.Close() } -// lock acquires the database mutex and logs debug information -func (d *Database) lock(operation string) { - // Get caller information - _, file, line, _ := runtime.Caller(1) - caller := fmt.Sprintf("%s:%d", filepath.Base(file), line) - - d.logger.Debug("Acquiring database lock", "operation", operation, "caller", caller) - - d.mu.Lock() - d.lockedAt = time.Now() - d.lockedBy = fmt.Sprintf("%s (%s)", operation, caller) - - d.logger.Debug("Database lock acquired", "operation", operation, "caller", caller) -} - -// unlock releases the database mutex and logs debug information including hold duration -func (d *Database) unlock() { - holdDuration := time.Since(d.lockedAt) - lockedBy := d.lockedBy - - d.lockedAt = time.Time{} - d.lockedBy = "" - d.mu.Unlock() - - d.logger.Debug("Database lock released", "held_by", lockedBy, "duration_ms", holdDuration.Milliseconds()) -} - -// beginTx starts a new transaction with logging -func (d *Database) beginTx() (*loggingTx, error) { - tx, err := d.db.Begin() - if err != nil { - return nil, err - } - - return &loggingTx{Tx: tx, logger: d.logger}, nil -} - // A live-route upsert is an UPDATE followed, only when no row matched, by an // INSERT. A live route is keyed by prefix and peer, so an announcement replaces // the peer's route for the prefix whatever its origin AS. The UPDATE's @@ -248,52 +232,73 @@ func (d *Database) beginTx() (*loggingTx, error) { // two statements. The id column is set only on INSERT, so an updated route // keeps its original id, exactly as the previous ON CONFLICT upsert did. const ( - updateLiveRouteV4SQL = `UPDATE live_routes_v4 SET mask_length = ?, origin_asn = ?, as_path = ?, - next_hop = ?, last_updated = ? WHERE prefix = ? AND peer_ip = ?` + updateLiveRouteV4SQL = `UPDATE live_routes_v4 + SET mask_length = ?, origin_asn = ?, as_path = ?, next_hop = ?, + last_updated = ? WHERE prefix = ? AND peer_ip = ?` - insertLiveRouteV4SQL = `INSERT INTO live_routes_v4 (id, prefix, mask_length, origin_asn, peer_ip, - as_path, next_hop, last_updated) VALUES (?, ?, ?, ?, ?, ?, ?, ?)` + insertLiveRouteV4SQL = `INSERT INTO live_routes_v4 + (id, prefix, mask_length, origin_asn, peer_ip, as_path, next_hop, + last_updated) VALUES (?, ?, ?, ?, ?, ?, ?, ?)` - updateLiveRouteV6SQL = `UPDATE live_routes_v6 SET mask_length = ?, origin_asn = ?, as_path = ?, - next_hop = ?, last_updated = ? WHERE prefix = ? AND peer_ip = ?` + updateLiveRouteV6SQL = `UPDATE live_routes_v6 + SET mask_length = ?, origin_asn = ?, as_path = ?, next_hop = ?, + last_updated = ? WHERE prefix = ? AND peer_ip = ?` - insertLiveRouteV6SQL = `INSERT INTO live_routes_v6 (id, prefix, mask_length, origin_asn, peer_ip, - as_path, next_hop, last_updated) VALUES (?, ?, ?, ?, ?, ?, ?, ?)` + insertLiveRouteV6SQL = `INSERT INTO live_routes_v6 + (id, prefix, mask_length, origin_asn, peer_ip, as_path, next_hop, + last_updated) VALUES (?, ?, ?, ?, ?, ?, ?, ?)` ) // Before a new route is inserted and after a route is deleted, the write looks // up whether any live route has that prefix, to keep the in-memory prefix // distribution exact. The lookup reads one entry of the prefix index. const ( - prefixHasLiveRouteV4SQL = `SELECT EXISTS (SELECT 1 FROM live_routes_v4 WHERE prefix = ?)` - prefixHasLiveRouteV6SQL = `SELECT EXISTS (SELECT 1 FROM live_routes_v6 WHERE prefix = ?)` + prefixHasLiveRouteV4SQL = `SELECT EXISTS + (SELECT 1 FROM live_routes_v4 WHERE prefix = ?)` + prefixHasLiveRouteV6SQL = `SELECT EXISTS + (SELECT 1 FROM live_routes_v6 WHERE prefix = ?)` ) -// upsertRouteRowV4 updates an IPv4 live route, inserting it when no row matched. -// It reports whether a new row was inserted and whether that row is the first -// live route for its prefix. -func upsertRouteRowV4(upd, ins, has *sql.Stmt, route *LiveRoute, pathJSON string) ( - inserted, newPrefix bool, err error) { - res, err := upd.Exec(route.MaskLength, route.OriginASN, pathJSON, route.NextHop, route.LastUpdated, - route.Prefix, route.PeerIP) +// A deletion removes the live route the peer has for the prefix, if any. +const ( + deleteLiveRouteV4SQL = `DELETE FROM live_routes_v4 + WHERE prefix = ? AND peer_ip = ?` + deleteLiveRouteV6SQL = `DELETE FROM live_routes_v6 + WHERE prefix = ? AND peer_ip = ?` +) + +// upsertRouteRow updates a live route with the update statement of its IP +// version's table, inserting it when no row matched. It reports whether a new +// row was inserted and whether that row is the first live route for its prefix. +func upsertRouteRow( + ctx context.Context, upd, ins, has *sql.Stmt, route *LiveRoute, + pathJSON string, +) (bool, bool, error) { + res, err := upd.ExecContext(ctx, route.MaskLength, route.OriginASN, pathJSON, + route.NextHop, route.LastUpdated, route.Prefix, route.PeerIP) if err != nil { return false, false, err } + affected, err := res.RowsAffected() if err != nil { return false, false, err } + if affected > 0 { return false, false, nil } var prefixHadRoute bool - if err := has.QueryRow(route.Prefix).Scan(&prefixHadRoute); err != nil { + + err = has.QueryRowContext(ctx, route.Prefix).Scan(&prefixHadRoute) + if err != nil { return false, false, err } - _, err = ins.Exec(route.ID.String(), route.Prefix, route.MaskLength, route.OriginASN, - route.PeerIP, pathJSON, route.NextHop, route.LastUpdated) + _, err = ins.ExecContext(ctx, route.ID.String(), route.Prefix, + route.MaskLength, route.OriginASN, route.PeerIP, pathJSON, + route.NextHop, route.LastUpdated) if err != nil { return false, false, err } @@ -301,39 +306,75 @@ func upsertRouteRowV4(upd, ins, has *sql.Stmt, route *LiveRoute, pathJSON string return true, !prefixHadRoute, nil } -// upsertRouteRowV6 updates an IPv6 live route, inserting it when no row matched. -// It reports whether a new row was inserted and whether that row is the first -// live route for its prefix. -func upsertRouteRowV6(upd, ins, has *sql.Stmt, route *LiveRoute, pathJSON string) ( - inserted, newPrefix bool, err error) { - res, err := upd.Exec(route.MaskLength, route.OriginASN, pathJSON, route.NextHop, route.LastUpdated, - route.Prefix, route.PeerIP) - if err != nil { - return false, false, err - } - affected, err := res.RowsAffected() - if err != nil { - return false, false, err - } - if affected > 0 { - return false, false, nil +// upsertRoutesInTable upserts routes, all of one IP version, in the live route +// table of that version. It returns how many new rows it inserted and the mask +// lengths of the prefixes that got their first live route. +func upsertRoutesInTable( + ctx context.Context, tx *loggingTx, routes []*LiveRoute, ipVersion int, +) (int, []int, error) { + if len(routes) == 0 { + return 0, nil, nil } - var prefixHadRoute bool - if err := has.QueryRow(route.Prefix).Scan(&prefixHadRoute); err != nil { - return false, false, err + family, updateSQL, insertSQL, lookupSQL := ipv4Family, updateLiveRouteV4SQL, + insertLiveRouteV4SQL, prefixHasLiveRouteV4SQL + if ipVersion == ipVersionV6 { + family, updateSQL, insertSQL, lookupSQL = ipv6Family, + updateLiveRouteV6SQL, insertLiveRouteV6SQL, prefixHasLiveRouteV6SQL } - _, err = ins.Exec(route.ID.String(), route.Prefix, route.MaskLength, route.OriginASN, - route.PeerIP, pathJSON, route.NextHop, route.LastUpdated) + upd, err := tx.PrepareContext(ctx, updateSQL) if err != nil { - return false, false, err + return 0, nil, fmt.Errorf("failed to prepare %s update statement: %w", + family, err) + } + defer func() { _ = upd.Close() }() + + ins, err := tx.PrepareContext(ctx, insertSQL) + if err != nil { + return 0, nil, fmt.Errorf("failed to prepare %s insert statement: %w", + family, err) + } + defer func() { _ = ins.Close() }() + + has, err := tx.PrepareContext(ctx, lookupSQL) + if err != nil { + return 0, nil, fmt.Errorf( + "failed to prepare %s prefix lookup statement: %w", family, err) + } + defer func() { _ = has.Close() }() + + var newRows int + // Mask lengths of the prefixes that get their first live route. + var newPrefixMaskLengths []int + + for _, route := range routes { + pathJSON, err := json.Marshal(route.ASPath) + if err != nil { + return 0, nil, fmt.Errorf("failed to encode AS path: %w", err) + } + + inserted, newPrefix, err := upsertRouteRow(ctx, upd, ins, has, route, + string(pathJSON)) + if err != nil { + return 0, nil, fmt.Errorf("failed to upsert route %s: %w", + route.Prefix, err) + } + + if inserted { + newRows++ + } + + if newPrefix { + newPrefixMaskLengths = append(newPrefixMaskLengths, route.MaskLength) + } } - return true, !prefixHadRoute, nil + return newRows, newPrefixMaskLengths, nil } -// UpsertLiveRouteBatch inserts or updates multiple live routes in a single transaction +// UpsertLiveRouteBatch inserts or updates multiple live routes in a single +// transaction, the IPv4 routes first and then the IPv6 routes. func (d *Database) UpsertLiveRouteBatch(routes []*LiveRoute) error { if len(routes) == 0 { return nil @@ -346,86 +387,34 @@ func (d *Database) UpsertLiveRouteBatch(routes []*LiveRoute) error { if err != nil { return fmt.Errorf("failed to begin transaction: %w", err) } - defer func() { - if err := tx.Rollback(); err != nil && err != sql.ErrTxDone { - d.logger.Error("Failed to rollback transaction", "error", err) - } - }() + defer d.rollback(tx) - // Prepare the update and insert statements for both tables. - updV4, err := tx.Prepare(updateLiveRouteV4SQL) - if err != nil { - return fmt.Errorf("failed to prepare IPv4 update statement: %w", err) - } - defer func() { _ = updV4.Close() }() + var routesV4, routesV6 []*LiveRoute - insV4, err := tx.Prepare(insertLiveRouteV4SQL) - if err != nil { - return fmt.Errorf("failed to prepare IPv4 insert statement: %w", err) - } - defer func() { _ = insV4.Close() }() - - updV6, err := tx.Prepare(updateLiveRouteV6SQL) - if err != nil { - return fmt.Errorf("failed to prepare IPv6 update statement: %w", err) - } - defer func() { _ = updV6.Close() }() - - insV6, err := tx.Prepare(insertLiveRouteV6SQL) - if err != nil { - return fmt.Errorf("failed to prepare IPv6 insert statement: %w", err) - } - defer func() { _ = insV6.Close() }() - - hasV4, err := tx.Prepare(prefixHasLiveRouteV4SQL) - if err != nil { - return fmt.Errorf("failed to prepare IPv4 prefix lookup statement: %w", err) - } - defer func() { _ = hasV4.Close() }() - - hasV6, err := tx.Prepare(prefixHasLiveRouteV6SQL) - if err != nil { - return fmt.Errorf("failed to prepare IPv6 prefix lookup statement: %w", err) - } - defer func() { _ = hasV6.Close() }() - - var newV4, newV6 int - // Mask lengths of the prefixes that get their first live route in this batch. - var newPrefixMaskLengthsV4, newPrefixMaskLengthsV6 []int for _, route := range routes { - pathJSON, err := json.Marshal(route.ASPath) - if err != nil { - return fmt.Errorf("failed to encode AS path: %w", err) - } - if route.IPVersion == ipVersionV4 { - inserted, newPrefix, err := upsertRouteRowV4(updV4, insV4, hasV4, route, string(pathJSON)) - if err != nil { - return fmt.Errorf("failed to upsert route %s: %w", route.Prefix, err) - } - if inserted { - newV4++ - } - if newPrefix { - newPrefixMaskLengthsV4 = append(newPrefixMaskLengthsV4, route.MaskLength) - } - - continue - } - - inserted, newPrefix, err := upsertRouteRowV6(updV6, insV6, hasV6, route, string(pathJSON)) - if err != nil { - return fmt.Errorf("failed to upsert route %s: %w", route.Prefix, err) - } - if inserted { - newV6++ - } - if newPrefix { - newPrefixMaskLengthsV6 = append(newPrefixMaskLengthsV6, route.MaskLength) + routesV4 = append(routesV4, route) + } else { + routesV6 = append(routesV6, route) } } - if err = tx.Commit(); err != nil { + ctx := context.Background() + + newV4, newPrefixMaskLengthsV4, err := upsertRoutesInTable(ctx, tx, + routesV4, ipVersionV4) + if err != nil { + return err + } + + newV6, newPrefixMaskLengthsV6, err := upsertRoutesInTable(ctx, tx, + routesV6, ipVersionV6) + if err != nil { + return err + } + + err = tx.Commit() + if err != nil { return fmt.Errorf("failed to commit transaction: %w", err) } @@ -435,7 +424,103 @@ func (d *Database) UpsertLiveRouteBatch(routes []*LiveRoute) error { return nil } -// DeleteLiveRouteBatch deletes multiple live routes in a single transaction +// deleteRouteRow runs one deletion with the delete statement of its IP +// version's table. It returns how many rows it removed and whether that left +// the prefix with no live route. +func deleteRouteRow( + ctx context.Context, deleteStmt, has *sql.Stmt, del LiveRouteDeletion, +) (int64, bool, error) { + res, err := deleteStmt.ExecContext(ctx, del.Prefix, del.PeerIP) + if err != nil { + return 0, false, fmt.Errorf("failed to delete route %s: %w", + del.Prefix, err) + } + + // The feed often withdraws a route that is not live, which deletes + // nothing, so the in-memory route counters use the rows-affected count. + affected, err := res.RowsAffected() + if err != nil { + return 0, false, fmt.Errorf("failed to count deleted route %s: %w", + del.Prefix, err) + } + + if affected == 0 { + return 0, false, nil + } + + // The prefix leaves the distribution when no live route has it any more. + var prefixHasRoute bool + + err = has.QueryRowContext(ctx, del.Prefix).Scan(&prefixHasRoute) + if err != nil { + return 0, false, fmt.Errorf("failed to look up prefix %s: %w", + del.Prefix, err) + } + + return affected, !prefixHasRoute, nil +} + +// deleteRoutesInTable runs deletions, all of one IP version, on the live route +// table of that version. It returns how many rows it removed and the mask +// lengths of the prefixes it left with no live route. +func deleteRoutesInTable( + ctx context.Context, tx *loggingTx, deletions []LiveRouteDeletion, + ipVersion int, +) (int64, []int, error) { + if len(deletions) == 0 { + return 0, nil, nil + } + + family, deleteSQL, lookupSQL := ipv4Family, deleteLiveRouteV4SQL, + prefixHasLiveRouteV4SQL + if ipVersion == ipVersionV6 { + family, deleteSQL, lookupSQL = ipv6Family, deleteLiveRouteV6SQL, + prefixHasLiveRouteV6SQL + } + + deleteStmt, err := tx.PrepareContext(ctx, deleteSQL) + if err != nil { + return 0, nil, fmt.Errorf("failed to prepare %s delete statement: %w", + family, err) + } + defer func() { _ = deleteStmt.Close() }() + + has, err := tx.PrepareContext(ctx, lookupSQL) + if err != nil { + return 0, nil, fmt.Errorf( + "failed to prepare %s prefix lookup statement: %w", family, err) + } + defer func() { _ = has.Close() }() + + var deleted int64 + // Mask lengths of the prefixes left with no live route. + var gonePrefixMaskLengths []int + + for _, del := range deletions { + affected, prefixGone, err := deleteRouteRow(ctx, deleteStmt, has, del) + if err != nil { + return 0, nil, err + } + + deleted += affected + + if !prefixGone { + continue + } + + maskLength, err := prefixMaskLength(del.Prefix) + if err != nil { + return 0, nil, err + } + + gonePrefixMaskLengths = append(gonePrefixMaskLengths, maskLength) + } + + return deleted, gonePrefixMaskLengths, nil +} + +// DeleteLiveRouteBatch deletes multiple live routes in a single transaction, +// the IPv4 routes first and then the IPv6 routes. func (d *Database) DeleteLiveRouteBatch(deletions []LiveRouteDeletion) error { if len(deletions) == 0 { return nil @@ -448,91 +533,34 @@ func (d *Database) DeleteLiveRouteBatch(deletions []LiveRouteDeletion) error { if err != nil { return fmt.Errorf("failed to begin transaction: %w", err) } - defer func() { - if err := tx.Rollback(); err != nil && err != sql.ErrTxDone { - d.logger.Error("Failed to rollback transaction", "error", err) - } - }() + defer d.rollback(tx) - // Prepare statements for both IPv4 and IPv6 tables - deleteV4, err := tx.Prepare(`DELETE FROM live_routes_v4 WHERE prefix = ? AND peer_ip = ?`) - if err != nil { - return fmt.Errorf("failed to prepare IPv4 delete statement: %w", err) - } - defer func() { _ = deleteV4.Close() }() + var deletionsV4, deletionsV6 []LiveRouteDeletion - deleteV6, err := tx.Prepare(`DELETE FROM live_routes_v6 WHERE prefix = ? AND peer_ip = ?`) - if err != nil { - return fmt.Errorf("failed to prepare IPv6 delete statement: %w", err) - } - defer func() { _ = deleteV6.Close() }() - - hasV4, err := tx.Prepare(prefixHasLiveRouteV4SQL) - if err != nil { - return fmt.Errorf("failed to prepare IPv4 prefix lookup statement: %w", err) - } - defer func() { _ = hasV4.Close() }() - - hasV6, err := tx.Prepare(prefixHasLiveRouteV6SQL) - if err != nil { - return fmt.Errorf("failed to prepare IPv6 prefix lookup statement: %w", err) - } - defer func() { _ = hasV6.Close() }() - - // Process deletions - var deletedV4, deletedV6 int64 - // Mask lengths of the prefixes this batch leaves with no live route. - var gonePrefixMaskLengthsV4, gonePrefixMaskLengthsV6 []int for _, del := range deletions { - stmt := deleteV4 - if del.IPVersion != ipVersionV4 { - stmt = deleteV6 - } - - res, err := stmt.Exec(del.Prefix, del.PeerIP) - if err != nil { - return fmt.Errorf("failed to delete route %s: %w", del.Prefix, err) - } - - // The feed often withdraws a route that is not live, which deletes - // nothing, so the in-memory route counters use the rows-affected count. - affected, err := res.RowsAffected() - if err != nil { - return fmt.Errorf("failed to count deleted route %s: %w", del.Prefix, err) - } if del.IPVersion == ipVersionV4 { - deletedV4 += affected + deletionsV4 = append(deletionsV4, del) } else { - deletedV6 += affected - } - if affected == 0 { - continue - } - - // The prefix leaves the distribution when no live route has it any more. - has := hasV4 - if del.IPVersion != ipVersionV4 { - has = hasV6 - } - var prefixHasRoute bool - if err := has.QueryRow(del.Prefix).Scan(&prefixHasRoute); err != nil { - return fmt.Errorf("failed to look up prefix %s: %w", del.Prefix, err) - } - if prefixHasRoute { - continue - } - maskLength, err := prefixMaskLength(del.Prefix) - if err != nil { - return err - } - if del.IPVersion == ipVersionV4 { - gonePrefixMaskLengthsV4 = append(gonePrefixMaskLengthsV4, maskLength) - } else { - gonePrefixMaskLengthsV6 = append(gonePrefixMaskLengthsV6, maskLength) + deletionsV6 = append(deletionsV6, del) } } - if err = tx.Commit(); err != nil { + ctx := context.Background() + + deletedV4, gonePrefixMaskLengthsV4, err := deleteRoutesInTable(ctx, tx, + deletionsV4, ipVersionV4) + if err != nil { + return err + } + + deletedV6, gonePrefixMaskLengthsV6, err := deleteRoutesInTable(ctx, tx, + deletionsV6, ipVersionV6) + if err != nil { + return err + } + + err = tx.Commit() + if err != nil { return fmt.Errorf("failed to commit transaction: %w", err) } @@ -547,41 +575,56 @@ func (d *Database) DeleteLiveRouteBatch(deletions []LiveRouteDeletion) error { // live route for: the delete takes those prefixes out of the prefix // distribution. const ( - peerOnlyRouteMaskLengthsV4SQL = `SELECT r.mask_length FROM live_routes_v4 AS r WHERE r.peer_ip = ? - AND NOT EXISTS (SELECT 1 FROM live_routes_v4 AS o WHERE o.prefix = r.prefix AND o.peer_ip != r.peer_ip)` + peerOnlyRouteMaskLengthsV4SQL = `SELECT r.mask_length + FROM live_routes_v4 AS r WHERE r.peer_ip = ? + AND NOT EXISTS (SELECT 1 FROM live_routes_v4 AS o + WHERE o.prefix = r.prefix AND o.peer_ip != r.peer_ip)` deletePeerLiveRoutesV4SQL = `DELETE FROM live_routes_v4 WHERE peer_ip = ?` - peerOnlyRouteMaskLengthsV6SQL = `SELECT r.mask_length FROM live_routes_v6 AS r WHERE r.peer_ip = ? - AND NOT EXISTS (SELECT 1 FROM live_routes_v6 AS o WHERE o.prefix = r.prefix AND o.peer_ip != r.peer_ip)` + peerOnlyRouteMaskLengthsV6SQL = `SELECT r.mask_length + FROM live_routes_v6 AS r WHERE r.peer_ip = ? + AND NOT EXISTS (SELECT 1 FROM live_routes_v6 AS o + WHERE o.prefix = r.prefix AND o.peer_ip != r.peer_ip)` deletePeerLiveRoutesV6SQL = `DELETE FROM live_routes_v6 WHERE peer_ip = ?` ) // deletePeerRoutes deletes all live routes of peerIP from one table, given that // table's two statements above. It returns how many routes it deleted and the // mask length of each prefix it left with no live route. -func deletePeerRoutes(tx *loggingTx, onlyRouteMaskLengthsSQL, deleteSQL, peerIP string) ( - deleted int64, goneMaskLengths []int, err error) { - rows, err := tx.Query(onlyRouteMaskLengthsSQL, peerIP) +func deletePeerRoutes( + ctx context.Context, tx *loggingTx, + onlyRouteMaskLengthsSQL, deleteSQL, peerIP string, +) (int64, []int, error) { + rows, err := tx.QueryContext(ctx, onlyRouteMaskLengthsSQL, peerIP) if err != nil { return 0, nil, err } defer func() { _ = rows.Close() }() + + var goneMaskLengths []int + for rows.Next() { var maskLength int - if err := rows.Scan(&maskLength); err != nil { + + err := rows.Scan(&maskLength) + if err != nil { return 0, nil, err } + goneMaskLengths = append(goneMaskLengths, maskLength) } - if err := rows.Err(); err != nil { - return 0, nil, err - } - res, err := tx.Exec(deleteSQL, peerIP) + err = rows.Err() if err != nil { return 0, nil, err } - deleted, err = res.RowsAffected() + + res, err := tx.ExecContext(ctx, deleteSQL, peerIP) + if err != nil { + return 0, nil, err + } + + deleted, err := res.RowsAffected() if err != nil { return 0, nil, err } @@ -600,24 +643,26 @@ func (d *Database) DeleteLiveRoutesOfPeer(peerIP string) (int, error) { if err != nil { return 0, fmt.Errorf("failed to begin transaction: %w", err) } - defer func() { - if err := tx.Rollback(); err != nil && err != sql.ErrTxDone { - d.logger.Error("Failed to rollback transaction", "error", err) - } - }() + defer d.rollback(tx) - deletedV4, goneMaskLengthsV4, err := deletePeerRoutes(tx, + ctx := context.Background() + + deletedV4, goneMaskLengthsV4, err := deletePeerRoutes(ctx, tx, peerOnlyRouteMaskLengthsV4SQL, deletePeerLiveRoutesV4SQL, peerIP) if err != nil { - return 0, fmt.Errorf("failed to delete the IPv4 routes of peer %s: %w", peerIP, err) - } - deletedV6, goneMaskLengthsV6, err := deletePeerRoutes(tx, - peerOnlyRouteMaskLengthsV6SQL, deletePeerLiveRoutesV6SQL, peerIP) - if err != nil { - return 0, fmt.Errorf("failed to delete the IPv6 routes of peer %s: %w", peerIP, err) + return 0, fmt.Errorf("failed to delete the IPv4 routes of peer %s: %w", + peerIP, err) } - if err := tx.Commit(); err != nil { + deletedV6, goneMaskLengthsV6, err := deletePeerRoutes(ctx, tx, + peerOnlyRouteMaskLengthsV6SQL, deletePeerLiveRoutesV6SQL, peerIP) + if err != nil { + return 0, fmt.Errorf("failed to delete the IPv6 routes of peer %s: %w", + peerIP, err) + } + + err = tx.Commit() + if err != nil { return 0, fmt.Errorf("failed to commit transaction: %w", err) } @@ -627,7 +672,107 @@ func (d *Database) DeleteLiveRoutesOfPeer(peerIP string) (int, error) { return int(deletedV4 + deletedV6), nil } -// UpdatePrefixesBatch updates the last_seen time for multiple prefixes in a single transaction +// upsertPrefix updates the last_seen time of prefix with the statements of its +// IP version's table, inserting the prefix when it is new. It reports whether +// it inserted it. +func upsertPrefix( + ctx context.Context, selectStmt, updateStmt, insertStmt *sql.Stmt, + prefix string, timestamp time.Time, +) (bool, error) { + var id string + + err := selectStmt.QueryRowContext(ctx, prefix).Scan(&id) + + switch { + case err == nil: + // Prefix exists, update last_seen + _, err = updateStmt.ExecContext(ctx, timestamp, prefix) + if err != nil { + return false, fmt.Errorf("failed to update prefix %s: %w", + prefix, err) + } + + return false, nil + case errors.Is(err, sql.ErrNoRows): + // Prefix doesn't exist, create it + _, err = insertStmt.ExecContext(ctx, generateUUID().String(), prefix, + timestamp, timestamp) + if err != nil { + return false, fmt.Errorf("failed to insert prefix %s: %w", + prefix, err) + } + + return true, nil + default: + return false, fmt.Errorf("failed to query prefix %s: %w", prefix, err) + } +} + +// upsertPrefixesInTable updates the last_seen time of prefixes, all of one IP +// version, in the prefix table of that version, inserting the ones that are +// new. It returns how many it inserted. +func upsertPrefixesInTable( + ctx context.Context, tx *loggingTx, prefixes map[string]time.Time, + ipVersion int, +) (int, error) { + if len(prefixes) == 0 { + return 0, nil + } + + family := ipv4Family + selectSQL := "SELECT id FROM prefixes_v4 WHERE prefix = ?" + updateSQL := "UPDATE prefixes_v4 SET last_seen = ? WHERE prefix = ?" + insertSQL := "INSERT INTO prefixes_v4 (id, prefix, first_seen, last_seen) " + + "VALUES (?, ?, ?, ?)" + + if ipVersion == ipVersionV6 { + family = ipv6Family + selectSQL = "SELECT id FROM prefixes_v6 WHERE prefix = ?" + updateSQL = "UPDATE prefixes_v6 SET last_seen = ? WHERE prefix = ?" + insertSQL = "INSERT INTO prefixes_v6 (id, prefix, first_seen, last_seen) " + + "VALUES (?, ?, ?, ?)" + } + + selectStmt, err := tx.PrepareContext(ctx, selectSQL) + if err != nil { + return 0, fmt.Errorf("failed to prepare %s select statement: %w", + family, err) + } + defer func() { _ = selectStmt.Close() }() + + updateStmt, err := tx.PrepareContext(ctx, updateSQL) + if err != nil { + return 0, fmt.Errorf("failed to prepare %s update statement: %w", + family, err) + } + defer func() { _ = updateStmt.Close() }() + + insertStmt, err := tx.PrepareContext(ctx, insertSQL) + if err != nil { + return 0, fmt.Errorf("failed to prepare %s insert statement: %w", + family, err) + } + defer func() { _ = insertStmt.Close() }() + + var newPrefixes int + + for prefix, timestamp := range prefixes { + inserted, err := upsertPrefix(ctx, selectStmt, updateStmt, insertStmt, + prefix, timestamp) + if err != nil { + return 0, err + } + + if inserted { + newPrefixes++ + } + } + + return newPrefixes, nil +} + +// UpdatePrefixesBatch updates the last_seen time for multiple prefixes in a +// single transaction, the IPv4 prefixes first and then the IPv6 prefixes. func (d *Database) UpdatePrefixesBatch(prefixes map[string]time.Time) error { if len(prefixes) == 0 { return nil @@ -640,87 +785,33 @@ func (d *Database) UpdatePrefixesBatch(prefixes map[string]time.Time) error { if err != nil { return fmt.Errorf("failed to begin transaction: %w", err) } - defer func() { - if err := tx.Rollback(); err != nil && err != sql.ErrTxDone { - d.logger.Error("Failed to rollback transaction", "error", err) - } - }() + defer d.rollback(tx) - // Prepare statements for both IPv4 and IPv6 tables - selectV4Stmt, err := tx.Prepare("SELECT id FROM prefixes_v4 WHERE prefix = ?") - if err != nil { - return fmt.Errorf("failed to prepare IPv4 select statement: %w", err) - } - defer func() { _ = selectV4Stmt.Close() }() + prefixesV4 := make(map[string]time.Time) + prefixesV6 := make(map[string]time.Time) - updateV4Stmt, err := tx.Prepare("UPDATE prefixes_v4 SET last_seen = ? WHERE prefix = ?") - if err != nil { - return fmt.Errorf("failed to prepare IPv4 update statement: %w", err) - } - defer func() { _ = updateV4Stmt.Close() }() - - insertV4Stmt, err := tx.Prepare("INSERT INTO prefixes_v4 (id, prefix, first_seen, last_seen) VALUES (?, ?, ?, ?)") - if err != nil { - return fmt.Errorf("failed to prepare IPv4 insert statement: %w", err) - } - defer func() { _ = insertV4Stmt.Close() }() - - selectV6Stmt, err := tx.Prepare("SELECT id FROM prefixes_v6 WHERE prefix = ?") - if err != nil { - return fmt.Errorf("failed to prepare IPv6 select statement: %w", err) - } - defer func() { _ = selectV6Stmt.Close() }() - - updateV6Stmt, err := tx.Prepare("UPDATE prefixes_v6 SET last_seen = ? WHERE prefix = ?") - if err != nil { - return fmt.Errorf("failed to prepare IPv6 update statement: %w", err) - } - defer func() { _ = updateV6Stmt.Close() }() - - insertV6Stmt, err := tx.Prepare("INSERT INTO prefixes_v6 (id, prefix, first_seen, last_seen) VALUES (?, ?, ?, ?)") - if err != nil { - return fmt.Errorf("failed to prepare IPv6 insert statement: %w", err) - } - defer func() { _ = insertV6Stmt.Close() }() - - var newV4, newV6 int for prefix, timestamp := range prefixes { - ipVersion := detectIPVersion(prefix) - - var selectStmt, updateStmt, insertStmt *sql.Stmt - if ipVersion == ipVersionV4 { - selectStmt, updateStmt, insertStmt = selectV4Stmt, updateV4Stmt, insertV4Stmt + if detectIPVersion(prefix) == ipVersionV4 { + prefixesV4[prefix] = timestamp } else { - selectStmt, updateStmt, insertStmt = selectV6Stmt, updateV6Stmt, insertV6Stmt - } - - var id string - err = selectStmt.QueryRow(prefix).Scan(&id) - - switch err { - case nil: - // Prefix exists, update last_seen - _, err = updateStmt.Exec(timestamp, prefix) - if err != nil { - return fmt.Errorf("failed to update prefix %s: %w", prefix, err) - } - case sql.ErrNoRows: - // Prefix doesn't exist, create it - _, err = insertStmt.Exec(generateUUID().String(), prefix, timestamp, timestamp) - if err != nil { - return fmt.Errorf("failed to insert prefix %s: %w", prefix, err) - } - if ipVersion == ipVersionV4 { - newV4++ - } else { - newV6++ - } - default: - return fmt.Errorf("failed to query prefix %s: %w", prefix, err) + prefixesV6[prefix] = timestamp } } - if err = tx.Commit(); err != nil { + ctx := context.Background() + + newV4, err := upsertPrefixesInTable(ctx, tx, prefixesV4, ipVersionV4) + if err != nil { + return err + } + + newV6, err := upsertPrefixesInTable(ctx, tx, prefixesV6, ipVersionV6) + if err != nil { + return err + } + + err = tx.Commit() + if err != nil { return fmt.Errorf("failed to commit transaction: %w", err) } @@ -729,145 +820,10 @@ func (d *Database) UpdatePrefixesBatch(prefixes map[string]time.Time) error { return nil } -// GetOrCreateASNBatch creates or updates multiple ASNs in a single transaction -func (d *Database) GetOrCreateASNBatch(asns map[int]time.Time) error { - if len(asns) == 0 { - return nil - } - - d.lock("GetOrCreateASNBatch") - defer d.unlock() - - tx, err := d.beginTx() - if err != nil { - return fmt.Errorf("failed to begin transaction: %w", err) - } - defer func() { - if err := tx.Rollback(); err != nil && err != sql.ErrTxDone { - d.logger.Error("Failed to rollback transaction", "error", err) - } - }() - - // Prepare statements - selectStmt, err := tx.Prepare( - "SELECT asn, handle, description, first_seen, last_seen FROM asns WHERE asn = ?") - if err != nil { - return fmt.Errorf("failed to prepare select statement: %w", err) - } - defer func() { _ = selectStmt.Close() }() - - updateStmt, err := tx.Prepare("UPDATE asns SET last_seen = ? WHERE asn = ?") - if err != nil { - return fmt.Errorf("failed to prepare update statement: %w", err) - } - defer func() { _ = updateStmt.Close() }() - - insertStmt, err := tx.Prepare( - "INSERT INTO asns (asn, handle, description, first_seen, last_seen) VALUES (?, ?, ?, ?, ?)") - if err != nil { - return fmt.Errorf("failed to prepare insert statement: %w", err) - } - defer func() { _ = insertStmt.Close() }() - - var newASNs int - for number, timestamp := range asns { - var asn ASN - var handle, description sql.NullString - - err = selectStmt.QueryRow(number).Scan(&asn.ASN, &handle, &description, &asn.FirstSeen, &asn.LastSeen) - - if err == nil { - // ASN exists, update last_seen - _, err = updateStmt.Exec(timestamp, number) - if err != nil { - return fmt.Errorf("failed to update ASN %d: %w", number, err) - } - - continue - } - - if err == sql.ErrNoRows { - // ASN doesn't exist, create it - asn = ASN{ - ASN: number, - FirstSeen: timestamp, - LastSeen: timestamp, - } - - // Look up ASN info - if info, ok := asinfo.Get(number); ok { - asn.Handle = info.Handle - asn.Description = info.Description - } - - _, err = insertStmt.Exec(asn.ASN, asn.Handle, asn.Description, asn.FirstSeen, asn.LastSeen) - if err != nil { - return fmt.Errorf("failed to insert ASN %d: %w", number, err) - } - newASNs++ - - continue - } - - if err != nil { - return fmt.Errorf("failed to query ASN %d: %w", number, err) - } - } - - if err = tx.Commit(); err != nil { - return fmt.Errorf("failed to commit transaction: %w", err) - } - - d.counts.addASNs(newASNs) - - return nil -} - -// GetOrCreateASN retrieves an existing ASN or creates a new one if it doesn't exist. -func (d *Database) GetOrCreateASN(number int, timestamp time.Time) (*ASN, error) { - d.lock("GetOrCreateASN") - defer d.unlock() - - tx, err := d.beginTx() - if err != nil { - return nil, err - } - defer func() { - if err := tx.Rollback(); err != nil && err != sql.ErrTxDone { - d.logger.Error("Failed to rollback transaction", "error", err) - } - }() - - var asn ASN - var handle, description sql.NullString - err = tx.QueryRow("SELECT asn, handle, description, first_seen, last_seen FROM asns WHERE asn = ?", number). - Scan(&asn.ASN, &handle, &description, &asn.FirstSeen, &asn.LastSeen) - - if err == nil { - // ASN exists, update last_seen - asn.Handle = handle.String - asn.Description = description.String - _, err = tx.Exec("UPDATE asns SET last_seen = ? WHERE asn = ?", timestamp, number) - if err != nil { - return nil, err - } - asn.LastSeen = timestamp - - if err = tx.Commit(); err != nil { - d.logger.Error("Failed to commit transaction for ASN update", "asn", number, "error", err) - - return nil, err - } - - return &asn, nil - } - - if err != sql.ErrNoRows { - return nil, err - } - - // ASN doesn't exist, create it with ASN info lookup - asn = ASN{ +// newASN returns a new ASN record first and last seen at timestamp, with the +// handle and description from the AS info lookup when it knows the number. +func newASN(number int, timestamp time.Time) ASN { + asn := ASN{ ASN: number, FirstSeen: timestamp, LastSeen: timestamp, @@ -879,14 +835,177 @@ func (d *Database) GetOrCreateASN(number int, timestamp time.Time) (*ASN, error) asn.Description = info.Description } - _, err = tx.Exec("INSERT INTO asns (asn, handle, description, first_seen, last_seen) VALUES (?, ?, ?, ?, ?)", + return asn +} + +// upsertASN updates the last_seen time of ASN number with the given +// statements, inserting the ASN when it is new. It reports whether it inserted +// it. +func upsertASN( + ctx context.Context, selectStmt, updateStmt, insertStmt *sql.Stmt, + number int, timestamp time.Time, +) (bool, error) { + var ( + asn ASN + handle, description sql.NullString + ) + + err := selectStmt.QueryRowContext(ctx, number).Scan(&asn.ASN, &handle, + &description, &asn.FirstSeen, &asn.LastSeen) + if err == nil { + // ASN exists, update last_seen + _, err = updateStmt.ExecContext(ctx, timestamp, number) + if err != nil { + return false, fmt.Errorf("failed to update ASN %d: %w", number, err) + } + + return false, nil + } + + if !errors.Is(err, sql.ErrNoRows) { + return false, fmt.Errorf("failed to query ASN %d: %w", number, err) + } + + // ASN doesn't exist, create it + asn = newASN(number, timestamp) + + _, err = insertStmt.ExecContext(ctx, asn.ASN, asn.Handle, + asn.Description, asn.FirstSeen, asn.LastSeen) + if err != nil { + return false, fmt.Errorf("failed to insert ASN %d: %w", number, err) + } + + return true, nil +} + +// GetOrCreateASNBatch creates or updates multiple ASNs in a single transaction +func (d *Database) GetOrCreateASNBatch(asns map[int]time.Time) error { + if len(asns) == 0 { + return nil + } + + d.lock("GetOrCreateASNBatch") + defer d.unlock() + + ctx := context.Background() + + tx, err := d.beginTx() + if err != nil { + return fmt.Errorf("failed to begin transaction: %w", err) + } + defer d.rollback(tx) + + // Prepare statements + selectStmt, err := tx.PrepareContext(ctx, + "SELECT asn, handle, description, first_seen, last_seen "+ + "FROM asns WHERE asn = ?") + if err != nil { + return fmt.Errorf("failed to prepare select statement: %w", err) + } + defer func() { _ = selectStmt.Close() }() + + updateStmt, err := tx.PrepareContext(ctx, + "UPDATE asns SET last_seen = ? WHERE asn = ?") + if err != nil { + return fmt.Errorf("failed to prepare update statement: %w", err) + } + defer func() { _ = updateStmt.Close() }() + + insertStmt, err := tx.PrepareContext(ctx, + "INSERT INTO asns (asn, handle, description, first_seen, last_seen) "+ + "VALUES (?, ?, ?, ?, ?)") + if err != nil { + return fmt.Errorf("failed to prepare insert statement: %w", err) + } + defer func() { _ = insertStmt.Close() }() + + var newASNs int + + for number, timestamp := range asns { + inserted, err := upsertASN(ctx, selectStmt, updateStmt, insertStmt, + number, timestamp) + if err != nil { + return err + } + + if inserted { + newASNs++ + } + } + + err = tx.Commit() + if err != nil { + return fmt.Errorf("failed to commit transaction: %w", err) + } + + d.counts.addASNs(newASNs) + + return nil +} + +// GetOrCreateASN retrieves an existing ASN or creates a new one if it doesn't +// exist. +func (d *Database) GetOrCreateASN(number int, timestamp time.Time) (*ASN, error) { + d.lock("GetOrCreateASN") + defer d.unlock() + + tx, err := d.beginTx() + if err != nil { + return nil, err + } + defer d.rollback(tx) + + var ( + asn ASN + handle, description sql.NullString + ) + + err = tx.QueryRow("SELECT asn, handle, description, first_seen, last_seen "+ + "FROM asns WHERE asn = ?", number). + Scan(&asn.ASN, &handle, &description, &asn.FirstSeen, &asn.LastSeen) + if err == nil { + // ASN exists, update last_seen + asn.Handle = handle.String + asn.Description = description.String + + _, err = tx.Exec("UPDATE asns SET last_seen = ? WHERE asn = ?", + timestamp, number) + if err != nil { + return nil, err + } + + asn.LastSeen = timestamp + + err = tx.Commit() + if err != nil { + d.logger.Error("Failed to commit transaction for ASN update", + "asn", number, "error", err) + + return nil, err + } + + return &asn, nil + } + + if !errors.Is(err, sql.ErrNoRows) { + return nil, err + } + + // ASN doesn't exist, create it with ASN info lookup + asn = newASN(number, timestamp) + + _, err = tx.Exec("INSERT INTO asns "+ + "(asn, handle, description, first_seen, last_seen) "+ + "VALUES (?, ?, ?, ?, ?)", asn.ASN, asn.Handle, asn.Description, asn.FirstSeen, asn.LastSeen) if err != nil { return nil, err } - if err = tx.Commit(); err != nil { - d.logger.Error("Failed to commit transaction for ASN creation", "asn", number, "error", err) + err = tx.Commit() + if err != nil { + d.logger.Error("Failed to commit transaction for ASN creation", + "asn", number, "error", err) return nil, err } @@ -896,8 +1015,10 @@ func (d *Database) GetOrCreateASN(number int, timestamp time.Time) (*ASN, error) return &asn, nil } -// GetOrCreatePrefix retrieves an existing prefix or creates a new one if it doesn't exist. -func (d *Database) GetOrCreatePrefix(prefix string, timestamp time.Time) (*Prefix, error) { +// GetOrCreatePrefix retrieves an existing prefix or creates a new one if it +// doesn't exist. +func (d *Database) GetOrCreatePrefix( + prefix string, timestamp time.Time) (*Prefix, error) { d.lock("GetOrCreatePrefix") defer d.unlock() @@ -905,38 +1026,42 @@ func (d *Database) GetOrCreatePrefix(prefix string, timestamp time.Time) (*Prefi if err != nil { return nil, err } - defer func() { - if err := tx.Rollback(); err != nil && err != sql.ErrTxDone { - d.logger.Error("Failed to rollback transaction", "error", err) - } - }() + defer d.rollback(tx) // Determine table based on IP version ipVersion := detectIPVersion(prefix) + tableName := "prefixes_v4" if ipVersion == ipVersionV6 { tableName = "prefixes_v6" } - var p Prefix - var idStr string - query := fmt.Sprintf("SELECT id, prefix, first_seen, last_seen FROM %s WHERE prefix = ?", tableName) - err = tx.QueryRow(query, prefix). - Scan(&idStr, &p.Prefix, &p.FirstSeen, &p.LastSeen) + var ( + p Prefix + idStr string + ) + query := fmt.Sprintf("SELECT id, prefix, first_seen, last_seen "+ + "FROM %s WHERE prefix = ?", tableName) + + err = tx.QueryRow(query, prefix).Scan(&idStr, &p.Prefix, &p.FirstSeen, &p.LastSeen) if err == nil { // Prefix exists, update last_seen p.ID, _ = uuid.Parse(idStr) p.IPVersion = ipVersion updateQuery := fmt.Sprintf("UPDATE %s SET last_seen = ? WHERE id = ?", tableName) + _, err = tx.Exec(updateQuery, timestamp, p.ID.String()) if err != nil { return nil, err } + p.LastSeen = timestamp - if err = tx.Commit(); err != nil { - d.logger.Error("Failed to commit transaction for prefix update", "prefix", prefix, "error", err) + err = tx.Commit() + if err != nil { + d.logger.Error("Failed to commit transaction for prefix update", + "prefix", prefix, "error", err) return nil, err } @@ -944,26 +1069,25 @@ func (d *Database) GetOrCreatePrefix(prefix string, timestamp time.Time) (*Prefi return &p, nil } - if err != sql.ErrNoRows { + if !errors.Is(err, sql.ErrNoRows) { return nil, err } // Prefix doesn't exist, create it - p = Prefix{ - ID: generateUUID(), - Prefix: prefix, - IPVersion: ipVersion, - FirstSeen: timestamp, - LastSeen: timestamp, - } - insertQuery := fmt.Sprintf("INSERT INTO %s (id, prefix, first_seen, last_seen) VALUES (?, ?, ?, ?)", tableName) + p = Prefix{ID: generateUUID(), Prefix: prefix, IPVersion: ipVersion, + FirstSeen: timestamp, LastSeen: timestamp} + insertQuery := fmt.Sprintf("INSERT INTO %s "+ + "(id, prefix, first_seen, last_seen) VALUES (?, ?, ?, ?)", tableName) + _, err = tx.Exec(insertQuery, p.ID.String(), p.Prefix, p.FirstSeen, p.LastSeen) if err != nil { return nil, err } - if err = tx.Commit(); err != nil { - d.logger.Error("Failed to commit transaction for prefix creation", "prefix", prefix, "error", err) + err = tx.Commit() + if err != nil { + d.logger.Error("Failed to commit transaction for prefix creation", + "prefix", prefix, "error", err) return nil, err } @@ -977,17 +1101,20 @@ func (d *Database) GetOrCreatePrefix(prefix string, timestamp time.Time) (*Prefi return &p, nil } -// RecordAnnouncement inserts a new BGP announcement or withdrawal into the database. +// RecordAnnouncement inserts a new BGP announcement or withdrawal into the +// database. func (d *Database) RecordAnnouncement(announcement *Announcement) error { d.lock("RecordAnnouncement") defer d.unlock() err := d.exec(` - INSERT INTO announcements (id, prefix_id, peer_asn, origin_asn, path, next_hop, timestamp, is_withdrawal) + INSERT INTO announcements (id, prefix_id, peer_asn, origin_asn, path, + next_hop, timestamp, is_withdrawal) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, announcement.ID.String(), announcement.PrefixID.String(), announcement.PeerASN, announcement.OriginASN, - announcement.Path, announcement.NextHop, announcement.Timestamp, announcement.IsWithdrawal) + announcement.Path, announcement.NextHop, announcement.Timestamp, + announcement.IsWithdrawal) return err } @@ -996,10 +1123,11 @@ func (d *Database) RecordAnnouncement(announcement *Announcement) error { func (d *Database) RecordPeering(asA, asB int, timestamp time.Time) error { // Validate ASNs if asA <= 0 || asB <= 0 { - return fmt.Errorf("invalid ASN: asA=%d, asB=%d", asA, asB) + return fmt.Errorf("%w: asA=%d, asB=%d", errInvalidASN, asA, asB) } + if asA == asB { - return fmt.Errorf("cannot create peering with same ASN: %d", asA) + return fmt.Errorf("%w: %d", errSameASNInPeering, asA) } // Normalize: ensure asA < asB @@ -1014,21 +1142,20 @@ func (d *Database) RecordPeering(asA, asB int, timestamp time.Time) error { if err != nil { return err } - defer func() { - if err := tx.Rollback(); err != nil && err != sql.ErrTxDone { - d.logger.Error("Failed to rollback transaction", "error", err) - } - }() + defer d.rollback(tx) var exists bool - err = tx.QueryRow("SELECT EXISTS(SELECT 1 FROM peerings WHERE as_a = ? AND as_b = ?)", + + err = tx.QueryRow( + "SELECT EXISTS(SELECT 1 FROM peerings WHERE as_a = ? AND as_b = ?)", asA, asB).Scan(&exists) if err != nil { return err } if exists { - _, err = tx.Exec("UPDATE peerings SET last_seen = ? WHERE as_a = ? AND as_b = ?", + _, err = tx.Exec( + "UPDATE peerings SET last_seen = ? WHERE as_a = ? AND as_b = ?", timestamp, asA, asB) } else { _, err = tx.Exec(` @@ -1036,11 +1163,13 @@ func (d *Database) RecordPeering(asA, asB int, timestamp time.Time) error { VALUES (?, ?, ?, ?, ?)`, generateUUID().String(), asA, asB, timestamp, timestamp) } + if err != nil { return err } - if err = tx.Commit(); err != nil { + err = tx.Commit() + if err != nil { d.logger.Error("Failed to commit transaction for peering", "as_a", asA, "as_b", asB, @@ -1066,61 +1195,68 @@ func (d *Database) UpdatePeerBatch(peers map[string]PeerUpdate) error { d.lock("UpdatePeerBatch") defer d.unlock() + ctx := context.Background() + tx, err := d.beginTx() if err != nil { return fmt.Errorf("failed to begin transaction: %w", err) } - defer func() { - if err := tx.Rollback(); err != nil && err != sql.ErrTxDone { - d.logger.Error("Failed to rollback transaction", "error", err) - } - }() + defer d.rollback(tx) // Prepare statements - checkStmt, err := tx.Prepare("SELECT EXISTS(SELECT 1 FROM bgp_peers WHERE peer_ip = ?)") + checkStmt, err := tx.PrepareContext(ctx, + "SELECT EXISTS(SELECT 1 FROM bgp_peers WHERE peer_ip = ?)") if err != nil { return fmt.Errorf("failed to prepare check statement: %w", err) } defer func() { _ = checkStmt.Close() }() - updateStmt, err := tx.Prepare( - "UPDATE bgp_peers SET peer_asn = ?, last_seen = ?, last_message_type = ? WHERE peer_ip = ?") + updateStmt, err := tx.PrepareContext(ctx, + "UPDATE bgp_peers SET peer_asn = ?, last_seen = ?, "+ + "last_message_type = ? WHERE peer_ip = ?") if err != nil { return fmt.Errorf("failed to prepare update statement: %w", err) } defer func() { _ = updateStmt.Close() }() - insertStmt, err := tx.Prepare( - "INSERT INTO bgp_peers (id, peer_ip, peer_asn, first_seen, last_seen, last_message_type) VALUES (?, ?, ?, ?, ?, ?)") + insertStmt, err := tx.PrepareContext(ctx, + "INSERT INTO bgp_peers (id, peer_ip, peer_asn, first_seen, "+ + "last_seen, last_message_type) VALUES (?, ?, ?, ?, ?, ?)") if err != nil { return fmt.Errorf("failed to prepare insert statement: %w", err) } defer func() { _ = insertStmt.Close() }() var newPeers int + for _, update := range peers { var exists bool - err = checkStmt.QueryRow(update.PeerIP).Scan(&exists) + + err = checkStmt.QueryRowContext(ctx, update.PeerIP).Scan(&exists) if err != nil { return fmt.Errorf("failed to check peer %s: %w", update.PeerIP, err) } if exists { - _, err = updateStmt.Exec(update.PeerASN, update.Timestamp, update.MessageType, update.PeerIP) + _, err = updateStmt.ExecContext(ctx, update.PeerASN, + update.Timestamp, update.MessageType, update.PeerIP) } else { - _, err = insertStmt.Exec( + _, err = insertStmt.ExecContext(ctx, generateUUID().String(), update.PeerIP, update.PeerASN, update.Timestamp, update.Timestamp, update.MessageType) } + if err != nil { return fmt.Errorf("failed to update peer %s: %w", update.PeerIP, err) } + if !exists { newPeers++ } } - if err = tx.Commit(); err != nil { + err = tx.Commit() + if err != nil { return fmt.Errorf("failed to commit transaction: %w", err) } @@ -1130,7 +1266,9 @@ func (d *Database) UpdatePeerBatch(peers map[string]PeerUpdate) error { } // UpdatePeer updates or creates a BGP peer record -func (d *Database) UpdatePeer(peerIP string, peerASN int, messageType string, timestamp time.Time) error { +func (d *Database) UpdatePeer( + peerIP string, peerASN int, messageType string, timestamp time.Time, +) error { d.lock("UpdatePeer") defer d.unlock() @@ -1138,34 +1276,38 @@ func (d *Database) UpdatePeer(peerIP string, peerASN int, messageType string, ti if err != nil { return err } - defer func() { - if err := tx.Rollback(); err != nil && err != sql.ErrTxDone { - d.logger.Error("Failed to rollback transaction", "error", err) - } - }() + defer d.rollback(tx) var exists bool - err = tx.QueryRow("SELECT EXISTS(SELECT 1 FROM bgp_peers WHERE peer_ip = ?)", peerIP).Scan(&exists) + + err = tx.QueryRow( + "SELECT EXISTS(SELECT 1 FROM bgp_peers WHERE peer_ip = ?)", peerIP, + ).Scan(&exists) if err != nil { return err } if exists { _, err = tx.Exec( - "UPDATE bgp_peers SET peer_asn = ?, last_seen = ?, last_message_type = ? WHERE peer_ip = ?", + "UPDATE bgp_peers SET peer_asn = ?, last_seen = ?, "+ + "last_message_type = ? WHERE peer_ip = ?", peerASN, timestamp, messageType, peerIP, ) } else { _, err = tx.Exec( - "INSERT INTO bgp_peers (id, peer_ip, peer_asn, first_seen, last_seen, last_message_type) VALUES (?, ?, ?, ?, ?, ?)", - generateUUID().String(), peerIP, peerASN, timestamp, timestamp, messageType, + "INSERT INTO bgp_peers (id, peer_ip, peer_asn, first_seen, "+ + "last_seen, last_message_type) VALUES (?, ?, ?, ?, ?, ?)", + generateUUID().String(), peerIP, peerASN, timestamp, timestamp, + messageType, ) } + if err != nil { return err } - if err = tx.Commit(); err != nil { + err = tx.Commit() + if err != nil { d.logger.Error("Failed to commit transaction for peer update", "peer_ip", peerIP, "peer_asn", peerASN, @@ -1202,7 +1344,8 @@ func (d *Database) GetStatsContext(ctx context.Context) (Stats, error) { d.counts.fill(&stats) // Database file size is a cheap stat() on the file. - if fileInfo, err := os.Stat(d.path); err != nil { + fileInfo, err := os.Stat(d.path) + if err != nil { d.logger.Warn("Failed to get database file size", "error", err) } else { stats.FileSizeBytes = fileInfo.Size() @@ -1225,64 +1368,6 @@ func (d *Database) GetStatsContext(ctx context.Context) (Stats, error) { return stats, nil } -// routeTimestampRange returns the earliest and latest last_updated across both -// live route tables, or nil values when both tables are empty. Each query reads -// one row from an end of the last_updated index rather than scanning the tables. -func (d *Database) routeTimestampRange(ctx context.Context) (oldest, newest *time.Time, err error) { - oldestV4, ok, err := d.scanRouteTimestamp(ctx, - "SELECT last_updated FROM live_routes_v4 ORDER BY last_updated ASC LIMIT 1") - if err != nil { - return nil, nil, err - } - if ok { - oldest = &oldestV4 - } - - oldestV6, ok, err := d.scanRouteTimestamp(ctx, - "SELECT last_updated FROM live_routes_v6 ORDER BY last_updated ASC LIMIT 1") - if err != nil { - return nil, nil, err - } - if ok && (oldest == nil || oldestV6.Before(*oldest)) { - oldest = &oldestV6 - } - - newestV4, ok, err := d.scanRouteTimestamp(ctx, - "SELECT last_updated FROM live_routes_v4 ORDER BY last_updated DESC LIMIT 1") - if err != nil { - return nil, nil, err - } - if ok { - newest = &newestV4 - } - - newestV6, ok, err := d.scanRouteTimestamp(ctx, - "SELECT last_updated FROM live_routes_v6 ORDER BY last_updated DESC LIMIT 1") - if err != nil { - return nil, nil, err - } - if ok && (newest == nil || newestV6.After(*newest)) { - newest = &newestV6 - } - - return oldest, newest, nil -} - -// scanRouteTimestamp runs a single-row timestamp query. ok is false when the -// table is empty. The query selects the last_updated column directly so the -// driver parses the DATETIME value into a time.Time. -func (d *Database) scanRouteTimestamp(ctx context.Context, query string) (ts time.Time, ok bool, err error) { - err = d.db.QueryRowContext(ctx, query).Scan(&ts) - switch { - case errors.Is(err, sql.ErrNoRows): - return time.Time{}, false, nil - case err != nil: - return time.Time{}, false, err - default: - return ts, true, nil - } -} - // UpsertLiveRoute inserts or updates a live route func (d *Database) UpsertLiveRoute(route *LiveRoute) error { d.lock("UpsertLiveRoute") @@ -1293,37 +1378,37 @@ func (d *Database) UpsertLiveRoute(route *LiveRoute) error { return fmt.Errorf("failed to encode AS path: %w", err) } - updateSQL, insertSQL, lookupSQL := updateLiveRouteV4SQL, insertLiveRouteV4SQL, prefixHasLiveRouteV4SQL + updateSQL, insertSQL, lookupSQL := updateLiveRouteV4SQL, + insertLiveRouteV4SQL, prefixHasLiveRouteV4SQL if route.IPVersion == ipVersionV6 { - updateSQL, insertSQL, lookupSQL = updateLiveRouteV6SQL, insertLiveRouteV6SQL, prefixHasLiveRouteV6SQL + updateSQL, insertSQL, lookupSQL = updateLiveRouteV6SQL, + insertLiveRouteV6SQL, prefixHasLiveRouteV6SQL } + ctx := context.Background() + // The write lock is held, so no other writer can insert this key between the // update and the insert even though they are separate autocommit statements. - upd, err := d.db.Prepare(updateSQL) + upd, err := d.db.PrepareContext(ctx, updateSQL) if err != nil { return fmt.Errorf("failed to prepare update statement: %w", err) } defer func() { _ = upd.Close() }() - ins, err := d.db.Prepare(insertSQL) + ins, err := d.db.PrepareContext(ctx, insertSQL) if err != nil { return fmt.Errorf("failed to prepare insert statement: %w", err) } defer func() { _ = ins.Close() }() - has, err := d.db.Prepare(lookupSQL) + has, err := d.db.PrepareContext(ctx, lookupSQL) if err != nil { return fmt.Errorf("failed to prepare prefix lookup statement: %w", err) } defer func() { _ = has.Close() }() - var inserted, newPrefix bool - if route.IPVersion == ipVersionV4 { - inserted, newPrefix, err = upsertRouteRowV4(upd, ins, has, route, string(pathJSON)) - } else { - inserted, newPrefix, err = upsertRouteRowV6(upd, ins, has, route, string(pathJSON)) - } + inserted, newPrefix, err := upsertRouteRow(ctx, upd, ins, has, route, + string(pathJSON)) if err != nil { return fmt.Errorf("failed to upsert route %s: %w", route.Prefix, err) } @@ -1335,6 +1420,7 @@ func (d *Database) UpsertLiveRoute(route *LiveRoute) error { d.counts.addRoutes(0, 1) } } + if newPrefix { if route.IPVersion == ipVersionV4 { d.counts.addToDistribution([]int{route.MaskLength}, nil, 1) @@ -1365,20 +1451,16 @@ func (d *Database) DeleteLiveRoute(prefix, peerIP string) error { if err != nil { return fmt.Errorf("failed to begin transaction: %w", err) } - defer func() { - if err := tx.Rollback(); err != nil && err != sql.ErrTxDone { - d.logger.Error("Failed to rollback transaction", "error", err) - } - }() + defer d.rollback(tx) // Literal per-table queries (rather than one formatted with the table name) // so the delete carries no dynamically built SQL. - var res sql.Result + deleteSQL := deleteLiveRouteV6SQL if isV4 { - res, err = tx.Exec(`DELETE FROM live_routes_v4 WHERE prefix = ? AND peer_ip = ?`, prefix, peerIP) - } else { - res, err = tx.Exec(`DELETE FROM live_routes_v6 WHERE prefix = ? AND peer_ip = ?`, prefix, peerIP) + deleteSQL = deleteLiveRouteV4SQL } + + res, err := tx.Exec(deleteSQL, prefix, peerIP) if err != nil { return err } @@ -1387,6 +1469,7 @@ func (d *Database) DeleteLiveRoute(prefix, peerIP string) error { if err != nil { return err } + if affected == 0 { return nil } @@ -1396,12 +1479,16 @@ func (d *Database) DeleteLiveRoute(prefix, peerIP string) error { if isV4 { lookupSQL = prefixHasLiveRouteV4SQL } + var prefixHasRoute bool - if err := tx.QueryRow(lookupSQL, prefix).Scan(&prefixHasRoute); err != nil { + + err = tx.QueryRow(lookupSQL, prefix).Scan(&prefixHasRoute) + if err != nil { return err } - if err := tx.Commit(); err != nil { + err = tx.Commit() + if err != nil { return fmt.Errorf("failed to commit transaction: %w", err) } @@ -1410,6 +1497,7 @@ func (d *Database) DeleteLiveRoute(prefix, peerIP string) error { } else { d.counts.addRoutes(0, -int(affected)) } + if !prefixHasRoute { maskLength, _ := ipnet.Mask.Size() if isV4 { @@ -1422,16 +1510,22 @@ func (d *Database) DeleteLiveRoute(prefix, peerIP string) error { return nil } -// GetPrefixDistribution returns the distribution of unique prefixes by mask length -func (d *Database) GetPrefixDistribution() (ipv4 []PrefixDistribution, ipv6 []PrefixDistribution, err error) { +// GetPrefixDistribution returns the distribution of unique prefixes by mask +// length +func (d *Database) GetPrefixDistribution() ( + []PrefixDistribution, []PrefixDistribution, error, +) { return d.GetPrefixDistributionContext(context.Background()) } -// GetPrefixDistributionContext returns the distribution of unique prefixes by mask length with context support. -// It reads every live route, so the stats read does not call it; it seeds the -// in-memory distribution once at startup. +// GetPrefixDistributionContext returns the distribution of unique prefixes by +// mask length with context support. It reads every live route, so the stats +// read does not call it; it seeds the in-memory distribution once at startup. func (d *Database) GetPrefixDistributionContext(ctx context.Context) ( - ipv4 []PrefixDistribution, ipv6 []PrefixDistribution, err error) { + []PrefixDistribution, []PrefixDistribution, error, +) { + var ipv4, ipv6 []PrefixDistribution + // IPv4 distribution - count unique prefixes from v4 table query := ` SELECT mask_length, COUNT(DISTINCT prefix) as count @@ -1439,6 +1533,7 @@ func (d *Database) GetPrefixDistributionContext(ctx context.Context) ( GROUP BY mask_length ORDER BY mask_length ` + rows4, err := d.db.QueryContext(ctx, query) if err != nil { return nil, nil, fmt.Errorf("failed to query IPv4 distribution: %w", err) @@ -1451,13 +1546,19 @@ func (d *Database) GetPrefixDistributionContext(ctx context.Context) ( for rows4.Next() { var dist PrefixDistribution - if err := rows4.Scan(&dist.MaskLength, &dist.Count); err != nil { - return nil, nil, fmt.Errorf("failed to scan IPv4 distribution: %w", err) + + err := rows4.Scan(&dist.MaskLength, &dist.Count) + if err != nil { + return nil, nil, fmt.Errorf("failed to scan IPv4 distribution: %w", + err) } + ipv4 = append(ipv4, dist) } + // A read that stops partway ends the loop without an error of its own. - if err := rows4.Err(); err != nil { + err = rows4.Err() + if err != nil { return nil, nil, fmt.Errorf("failed to read IPv4 distribution: %w", err) } @@ -1468,6 +1569,7 @@ func (d *Database) GetPrefixDistributionContext(ctx context.Context) ( GROUP BY mask_length ORDER BY mask_length ` + rows6, err := d.db.QueryContext(ctx, query) if err != nil { return nil, nil, fmt.Errorf("failed to query IPv6 distribution: %w", err) @@ -1480,12 +1582,18 @@ func (d *Database) GetPrefixDistributionContext(ctx context.Context) ( for rows6.Next() { var dist PrefixDistribution - if err := rows6.Scan(&dist.MaskLength, &dist.Count); err != nil { - return nil, nil, fmt.Errorf("failed to scan IPv6 distribution: %w", err) + + err := rows6.Scan(&dist.MaskLength, &dist.Count) + if err != nil { + return nil, nil, fmt.Errorf("failed to scan IPv6 distribution: %w", + err) } + ipv6 = append(ipv6, dist) } - if err := rows6.Err(); err != nil { + + err = rows6.Err() + if err != nil { return nil, nil, fmt.Errorf("failed to read IPv6 distribution: %w", err) } @@ -1493,20 +1601,27 @@ func (d *Database) GetPrefixDistributionContext(ctx context.Context) ( } // GetLiveRouteCounts returns the count of IPv4 and IPv6 routes -func (d *Database) GetLiveRouteCounts() (ipv4Count, ipv6Count int, err error) { +func (d *Database) GetLiveRouteCounts() (int, int, error) { return d.GetLiveRouteCountsContext(context.Background()) } -// GetLiveRouteCountsContext returns the count of IPv4 and IPv6 routes with context support -func (d *Database) GetLiveRouteCountsContext(ctx context.Context) (ipv4Count, ipv6Count int, err error) { +// GetLiveRouteCountsContext returns the count of IPv4 and IPv6 routes with +// context support +func (d *Database) GetLiveRouteCountsContext( + ctx context.Context, +) (int, int, error) { + var ipv4Count, ipv6Count int + // Get IPv4 count from dedicated table - err = d.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM live_routes_v4").Scan(&ipv4Count) + err := d.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM live_routes_v4"). + Scan(&ipv4Count) if err != nil { return 0, 0, fmt.Errorf("failed to count IPv4 routes: %w", err) } // Get IPv6 count from dedicated table - err = d.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM live_routes_v6").Scan(&ipv6Count) + err = d.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM live_routes_v6"). + Scan(&ipv6Count) if err != nil { return 0, 0, fmt.Errorf("failed to count IPv6 routes: %w", err) } @@ -1519,18 +1634,25 @@ func (d *Database) GetASDetails(asn int) (*ASN, []LiveRoute, error) { return d.GetASDetailsContext(context.Background(), asn) } -// GetASDetailsContext returns detailed information about an AS including prefixes with context support -func (d *Database) GetASDetailsContext(ctx context.Context, asn int) (*ASN, []LiveRoute, error) { +// GetASDetailsContext returns detailed information about an AS including +// prefixes with context support +func (d *Database) GetASDetailsContext( + ctx context.Context, asn int, +) (*ASN, []LiveRoute, error) { // Get AS information - var asnInfo ASN - var handle, description sql.NullString - err := d.db.QueryRowContext(ctx, - "SELECT asn, handle, description, first_seen, last_seen FROM asns WHERE asn = ?", - asn, - ).Scan(&asnInfo.ASN, &handle, &description, &asnInfo.FirstSeen, &asnInfo.LastSeen) + var ( + asnInfo ASN + handle, description sql.NullString + ) + err := d.db.QueryRowContext(ctx, + "SELECT asn, handle, description, first_seen, last_seen "+ + "FROM asns WHERE asn = ?", + asn, + ).Scan(&asnInfo.ASN, &handle, &description, &asnInfo.FirstSeen, + &asnInfo.LastSeen) if err != nil { - if err == sql.ErrNoRows { + if errors.Is(err, sql.ErrNoRows) { return nil, nil, fmt.Errorf("%w: AS%d", ErrNoRoute, asn) } @@ -1541,9 +1663,6 @@ func (d *Database) GetASDetailsContext(ctx context.Context, asn int) (*ASN, []Li asnInfo.Description = description.String // Get prefixes announced by this AS from both tables - var allPrefixes []LiveRoute - - // Query IPv4 prefixes queryV4 := ` SELECT prefix, mask_length, MAX(last_updated) as last_updated FROM live_routes_v4 @@ -1551,38 +1670,11 @@ func (d *Database) GetASDetailsContext(ctx context.Context, asn int) (*ASN, []Li GROUP BY prefix, mask_length ` - rows4, err := d.db.QueryContext(ctx, queryV4, asn) + allPrefixes, err := d.originPrefixes(ctx, queryV4, asn, ipVersionV4) if err != nil { - return &asnInfo, nil, fmt.Errorf("failed to query IPv4 prefixes: %w", err) - } - defer func() { _ = rows4.Close() }() - - for rows4.Next() { - var route LiveRoute - var lastUpdatedStr string - err := rows4.Scan(&route.Prefix, &route.MaskLength, &lastUpdatedStr) - if err != nil { - d.logger.Error("Failed to scan IPv4 prefix row", "error", err, "asn", asn) - - continue - } - // Parse the timestamp string - route.LastUpdated, err = time.Parse("2006-01-02 15:04:05-07:00", lastUpdatedStr) - if err != nil { - // Try without timezone - route.LastUpdated, err = time.Parse("2006-01-02 15:04:05", lastUpdatedStr) - if err != nil { - d.logger.Error("Failed to parse timestamp", "error", err, "timestamp", lastUpdatedStr) - - continue - } - } - route.OriginASN = asn - route.IPVersion = ipVersionV4 - allPrefixes = append(allPrefixes, route) + return &asnInfo, nil, err } - // Query IPv6 prefixes queryV6 := ` SELECT prefix, mask_length, MAX(last_updated) as last_updated FROM live_routes_v6 @@ -1590,41 +1682,18 @@ func (d *Database) GetASDetailsContext(ctx context.Context, asn int) (*ASN, []Li GROUP BY prefix, mask_length ` - rows6, err := d.db.QueryContext(ctx, queryV6, asn) + prefixesV6, err := d.originPrefixes(ctx, queryV6, asn, ipVersionV6) if err != nil { - return &asnInfo, allPrefixes, fmt.Errorf("failed to query IPv6 prefixes: %w", err) - } - defer func() { _ = rows6.Close() }() - - for rows6.Next() { - var route LiveRoute - var lastUpdatedStr string - err := rows6.Scan(&route.Prefix, &route.MaskLength, &lastUpdatedStr) - if err != nil { - d.logger.Error("Failed to scan IPv6 prefix row", "error", err, "asn", asn) - - continue - } - // Parse the timestamp string - route.LastUpdated, err = time.Parse("2006-01-02 15:04:05-07:00", lastUpdatedStr) - if err != nil { - // Try without timezone - route.LastUpdated, err = time.Parse("2006-01-02 15:04:05", lastUpdatedStr) - if err != nil { - d.logger.Error("Failed to parse timestamp", "error", err, "timestamp", lastUpdatedStr) - - continue - } - } - route.OriginASN = asn - route.IPVersion = ipVersionV6 - allPrefixes = append(allPrefixes, route) + return &asnInfo, allPrefixes, err } - return &asnInfo, allPrefixes, nil + return &asnInfo, append(allPrefixes, prefixesV6...), nil } -// ASPeer represents a peering relationship with another AS including handle, description, and timestamps. +// ASPeer represents a peering relationship with another AS including handle, +// description, and timestamps. +// +//nolint:tagliatelle // field names are the existing JSON API type ASPeer struct { ASN int `json:"asn"` Handle string `json:"handle"` @@ -1638,8 +1707,11 @@ func (d *Database) GetASPeers(asn int) ([]ASPeer, error) { return d.GetASPeersContext(context.Background(), asn) } -// GetASPeersContext returns all ASes that peer with the given AS with context support -func (d *Database) GetASPeersContext(ctx context.Context, asn int) ([]ASPeer, error) { +// GetASPeersContext returns all ASes that peer with the given AS with context +// support +func (d *Database) GetASPeersContext( + ctx context.Context, asn int, +) ([]ASPeer, error) { query := ` SELECT CASE @@ -1666,18 +1738,23 @@ func (d *Database) GetASPeersContext(ctx context.Context, asn int) ([]ASPeer, er defer func() { _ = rows.Close() }() var peers []ASPeer + for rows.Next() { var peer ASPeer - err := rows.Scan(&peer.ASN, &peer.Handle, &peer.Description, &peer.FirstSeen, &peer.LastSeen) + + err := rows.Scan(&peer.ASN, &peer.Handle, &peer.Description, + &peer.FirstSeen, &peer.LastSeen) if err != nil { d.logger.Error("Failed to scan peer row", "error", err, "asn", asn) continue } + peers = append(peers, peer) } - if err := rows.Err(); err != nil { + err = rows.Err() + if err != nil { return nil, fmt.Errorf("error iterating AS peers: %w", err) } @@ -1685,30 +1762,34 @@ func (d *Database) GetASPeersContext(ctx context.Context, asn int) ([]ASPeer, er } // GetPrefixDetails returns detailed information about a prefix -func (d *Database) GetPrefixDetails(prefix string) ([]LiveRoute, map[int]ASN, error) { +func (d *Database) GetPrefixDetails( + prefix string, +) ([]LiveRoute, map[int]ASN, error) { return d.GetPrefixDetailsContext(context.Background(), prefix) } // GetPrefixDetailsContext returns the live routes for a prefix and, keyed by // AS number, the handle and description of each AS that originates it. -func (d *Database) GetPrefixDetailsContext(ctx context.Context, prefix string) ([]LiveRoute, map[int]ASN, error) { +func (d *Database) GetPrefixDetailsContext( + ctx context.Context, prefix string, +) ([]LiveRoute, map[int]ASN, error) { // Determine if it's IPv4 or IPv6 by parsing the prefix _, ipnet, err := net.ParseCIDR(prefix) if err != nil { return nil, nil, fmt.Errorf("invalid prefix format: %w", err) } - tableName := "live_routes_v4" + tableName := tableLiveRoutesV4 ipVersion := ipVersionV4 + if ipnet.IP.To4() == nil { - tableName = "live_routes_v6" + tableName = tableLiveRoutesV6 ipVersion = ipVersionV6 } - //nolint:gosec // Table name is hardcoded based on IP version query := fmt.Sprintf(` - SELECT lr.origin_asn, lr.peer_ip, lr.as_path, lr.next_hop, lr.last_updated, - a.handle, a.description + SELECT lr.origin_asn, lr.peer_ip, lr.as_path, lr.next_hop, + lr.last_updated, a.handle, a.description FROM %s lr LEFT JOIN asns a ON a.asn = lr.origin_asn WHERE lr.prefix = ? @@ -1722,11 +1803,15 @@ func (d *Database) GetPrefixDetailsContext(ctx context.Context, prefix string) ( defer func() { _ = rows.Close() }() var routes []LiveRoute + origins := make(map[int]ASN) + for rows.Next() { - var route LiveRoute - var pathJSON string - var handle, description sql.NullString + var ( + route LiveRoute + pathJSON string + handle, description sql.NullString + ) err := rows.Scan( &route.OriginASN, &route.PeerIP, &pathJSON, &route.NextHop, @@ -1737,7 +1822,8 @@ func (d *Database) GetPrefixDetailsContext(ctx context.Context, prefix string) ( } // Decode AS path - if err := json.Unmarshal([]byte(pathJSON), &route.ASPath); err != nil { + err = json.Unmarshal([]byte(pathJSON), &route.ASPath) + if err != nil { route.ASPath = []int{} } @@ -1752,6 +1838,11 @@ func (d *Database) GetPrefixDetailsContext(ctx context.Context, prefix string) ( } } + err = rows.Err() + if err != nil { + return nil, nil, fmt.Errorf("failed to read prefix details: %w", err) + } + if len(routes) == 0 { return nil, nil, fmt.Errorf("%w: %s", ErrNoRoute, prefix) } @@ -1759,9 +1850,13 @@ func (d *Database) GetPrefixDetailsContext(ctx context.Context, prefix string) ( return routes, origins, nil } -// GetRandomPrefixesByLength returns a random sample of prefixes with the specified mask length -func (d *Database) GetRandomPrefixesByLength(maskLength, ipVersion, limit int) ([]LiveRoute, error) { - return d.GetRandomPrefixesByLengthContext(context.Background(), maskLength, ipVersion, limit) +// GetRandomPrefixesByLength returns a random sample of prefixes with the +// specified mask length +func (d *Database) GetRandomPrefixesByLength( + maskLength, ipVersion, limit int, +) ([]LiveRoute, error) { + return d.GetRandomPrefixesByLengthContext(context.Background(), + maskLength, ipVersion, limit) } // GetRandomPrefixesByLengthContext returns limit prefixes with the given mask @@ -1773,205 +1868,41 @@ func (d *Database) GetRandomPrefixesByLength(maskLength, ipVersion, limit int) ( // index search for each listed prefix's newest route. func (d *Database) GetRandomPrefixesByLengthContext( ctx context.Context, maskLength, ipVersion, limit int) ([]LiveRoute, error) { - tableName := "live_routes_v4" + tableName := tableLiveRoutesV4 if ipVersion == ipVersionV6 { - tableName = "live_routes_v6" + tableName = tableLiveRoutesV6 } // Reading one prefix more than maxPrefixesToShuffle tells whether these // are all the prefixes with this mask length. - prefixes, err := d.firstPrefixesByLength(ctx, tableName, maskLength, maxPrefixesToShuffle+1) + prefixes, err := d.firstPrefixesByLength(ctx, tableName, maskLength, + maxPrefixesToShuffle+1) if err != nil { return nil, err } + + //nolint:gosec // The sample needs variety, not unpredictability. rand.Shuffle(len(prefixes), func(i, j int) { prefixes[i], prefixes[j] = prefixes[j], prefixes[i] }) + if len(prefixes) > maxPrefixesToShuffle { - prefixes, err = d.samplePrefixesByLength(ctx, tableName, maskLength, limit, prefixes) + prefixes, err = d.samplePrefixesByLength(ctx, tableName, maskLength, + limit, prefixes) if err != nil { return nil, err } } + prefixes = prefixes[:min(limit, len(prefixes))] return d.newestRouteOfEachPrefix(ctx, tableName, ipVersion, prefixes) } -// firstPrefixesByLength returns the first n prefixes, in text order, with the -// given mask length. Each prefix is one search of the mask length and prefix -// index for the prefix after the previous one, so the read does not grow with -// the number of peers that carry a prefix, as a SELECT DISTINCT would. -func (d *Database) firstPrefixesByLength( - ctx context.Context, tableName string, maskLength, n int) ([]string, error) { - //nolint:gosec // Table name is hardcoded based on IP version - query := fmt.Sprintf(` - SELECT prefix FROM %s - WHERE mask_length = ? AND prefix > ? - ORDER BY prefix - LIMIT 1 - `, tableName) - - stmt, err := d.db.PrepareContext(ctx, query) - if err != nil { - return nil, fmt.Errorf("failed to prepare prefix query: %w", err) - } - defer func() { _ = stmt.Close() }() - - var prefixes []string - previous := "" // every prefix sorts after the empty string - for len(prefixes) < n { - var prefix string - err := stmt.QueryRowContext(ctx, maskLength, previous).Scan(&prefix) - if errors.Is(err, sql.ErrNoRows) { - break - } - if err != nil { - return nil, fmt.Errorf("failed to list prefixes: %w", err) - } - prefixes = append(prefixes, prefix) - previous = prefix - } - - return prefixes, nil -} - -// samplePrefixesByLength picks limit distinct prefixes with the given mask -// length at random, in at most sampleLookupsPerPrefix lookups for each prefix -// asked for. ORDER BY RANDOM() would read every route with that mask length. -// Instead each lookup draws a row id between those of the first and last route -// with that mask length and takes the prefix of the first such route at or -// after it. SQLite keeps the row id at the end of every index entry, so the -// mask length index is ordered by mask length, then row id, and the lookup is -// one search of it. -// -// The sample is uneven, as the page needs a varied sample, not an even one: a -// route is picked in proportion to the row ids between it and the previous -// route with that mask length, including the row ids of routes with other mask -// lengths and of withdrawn routes. A route that follows a long run of -// withdrawn routes is picked far more often than one that directly follows -// another route with that mask length. -// -// Should the lookups keep landing on prefixes already picked, the rest are -// taken from shuffled, prefixes of that length in random order. -func (d *Database) samplePrefixesByLength( - ctx context.Context, tableName string, maskLength, limit int, shuffled []string) ([]string, error) { - //nolint:gosec // Table name is hardcoded based on IP version - rangeQuery := fmt.Sprintf(` - SELECT - (SELECT MIN(rowid) FROM %[1]s WHERE mask_length = ?), - (SELECT MAX(rowid) FROM %[1]s WHERE mask_length = ?) - `, tableName) - - var first, last sql.NullInt64 - if err := d.db.QueryRowContext(ctx, rangeQuery, maskLength, maskLength).Scan(&first, &last); err != nil { - return nil, fmt.Errorf("failed to read row id range: %w", err) - } - if !first.Valid { - // Every route with this mask length was withdrawn since they were listed. - return nil, nil - } - - //nolint:gosec // Table name is hardcoded based on IP version - lookupQuery := fmt.Sprintf(` - SELECT prefix FROM %s - WHERE mask_length = ? AND rowid >= ? - ORDER BY rowid - LIMIT 1 - `, tableName) - - stmt, err := d.db.PrepareContext(ctx, lookupQuery) - if err != nil { - return nil, fmt.Errorf("failed to prepare prefix lookup: %w", err) - } - defer func() { _ = stmt.Close() }() - - picked := make(map[string]bool, limit) - prefixes := make([]string, 0, limit) - pick := func(prefix string) { - if !picked[prefix] { - picked[prefix] = true - prefixes = append(prefixes, prefix) - } - } - - for lookups := 0; lookups < limit*sampleLookupsPerPrefix && len(prefixes) < limit; lookups++ { - //nolint:gosec // The sample needs variety, not unpredictability. - rowID := first.Int64 + rand.Int64N(last.Int64-first.Int64+1) - - var prefix string - err := stmt.QueryRowContext(ctx, maskLength, rowID).Scan(&prefix) - if errors.Is(err, sql.ErrNoRows) { - // The routes from rowID on were withdrawn since the range was read. - continue - } - if err != nil { - return nil, fmt.Errorf("failed to look up prefix: %w", err) - } - pick(prefix) - } - - for _, prefix := range shuffled { - if len(prefixes) == limit { - break - } - pick(prefix) - } - - return prefixes, nil -} - -// newestRouteOfEachPrefix returns the most recently updated live route of each -// prefix, one search of the prefix and last_updated index per prefix however -// many peers carry it. A prefix whose routes were all withdrawn since it was -// picked is left out. -func (d *Database) newestRouteOfEachPrefix( - ctx context.Context, tableName string, ipVersion int, prefixes []string) ([]LiveRoute, error) { - //nolint:gosec // Table name is hardcoded based on IP version - query := fmt.Sprintf(` - SELECT prefix, mask_length, origin_asn, as_path, peer_ip, last_updated - FROM %s - WHERE prefix = ? - ORDER BY last_updated DESC - LIMIT 1 - `, tableName) - - stmt, err := d.db.PrepareContext(ctx, query) - if err != nil { - return nil, fmt.Errorf("failed to prepare route query: %w", err) - } - defer func() { _ = stmt.Close() }() - - routes := make([]LiveRoute, 0, len(prefixes)) - for _, prefix := range prefixes { - route := LiveRoute{IPVersion: ipVersion} - var pathJSON string - err := stmt.QueryRowContext(ctx, prefix).Scan( - &route.Prefix, - &route.MaskLength, - &route.OriginASN, - &pathJSON, - &route.PeerIP, - &route.LastUpdated, - ) - if errors.Is(err, sql.ErrNoRows) { - continue - } - if err != nil { - return nil, fmt.Errorf("failed to query route of %s: %w", prefix, err) - } - - if err := json.Unmarshal([]byte(pathJSON), &route.ASPath); err != nil { - route.ASPath = []int{} - } - routes = append(routes, route) - } - - return routes, nil -} - // GetNextStaleASN returns a random ASN that needs WHOIS data refresh. -func (d *Database) GetNextStaleASN(ctx context.Context, staleThreshold time.Duration) (int, error) { +func (d *Database) GetNextStaleASN( + ctx context.Context, staleThreshold time.Duration, +) (int, error) { cutoff := time.Now().Add(-staleThreshold) // Select a random stale ASN using ORDER BY RANDOM() @@ -1984,9 +1915,10 @@ func (d *Database) GetNextStaleASN(ctx context.Context, staleThreshold time.Dura ` var asn int + err := d.db.QueryRowContext(ctx, query, cutoff).Scan(&asn) if err != nil { - if err == sql.ErrNoRows { + if errors.Is(err, sql.ErrNoRows) { return 0, ErrNoStaleASN } @@ -1997,6 +1929,8 @@ func (d *Database) GetNextStaleASN(ctx context.Context, staleThreshold time.Dura } // WHOISStats contains statistics about WHOIS data freshness. +// +//nolint:tagliatelle // field names are the existing JSON API type WHOISStats struct { TotalASNs int `json:"total_asns"` StaleASNs int `json:"stale_asns"` @@ -2005,19 +1939,25 @@ type WHOISStats struct { } // GetWHOISStats returns statistics about WHOIS data freshness. -func (d *Database) GetWHOISStats(ctx context.Context, staleThreshold time.Duration) (*WHOISStats, error) { +func (d *Database) GetWHOISStats( + ctx context.Context, staleThreshold time.Duration, +) (*WHOISStats, error) { cutoff := time.Now().Add(-staleThreshold) query := ` SELECT COUNT(*) as total, - COALESCE(SUM(CASE WHEN whois_updated_at IS NULL THEN 1 ELSE 0 END), 0) as never_fetched, - COALESCE(SUM(CASE WHEN whois_updated_at IS NOT NULL AND whois_updated_at < ? THEN 1 ELSE 0 END), 0) as stale, - COALESCE(SUM(CASE WHEN whois_updated_at IS NOT NULL AND whois_updated_at >= ? THEN 1 ELSE 0 END), 0) as fresh + COALESCE(SUM(CASE WHEN whois_updated_at IS NULL + THEN 1 ELSE 0 END), 0) as never_fetched, + COALESCE(SUM(CASE WHEN whois_updated_at IS NOT NULL + AND whois_updated_at < ? THEN 1 ELSE 0 END), 0) as stale, + COALESCE(SUM(CASE WHEN whois_updated_at IS NOT NULL + AND whois_updated_at >= ? THEN 1 ELSE 0 END), 0) as fresh FROM asns ` var stats WHOISStats + err := d.db.QueryRowContext(ctx, query, cutoff, cutoff).Scan( &stats.TotalASNs, &stats.NeverFetched, @@ -2032,7 +1972,9 @@ func (d *Database) GetWHOISStats(ctx context.Context, staleThreshold time.Durati } // UpdateASNWHOIS updates an ASN record with WHOIS data. -func (d *Database) UpdateASNWHOIS(ctx context.Context, update *ASNWHOISUpdate) error { +func (d *Database) UpdateASNWHOIS( + ctx context.Context, update *ASNWHOISUpdate, +) error { d.lock("UpdateASNWHOIS") defer d.unlock() @@ -2084,41 +2026,17 @@ func (d *Database) GetIPInfo(ip string) (*IPInfo, error) { return d.GetIPInfoContext(context.Background(), ip) } -// GetIPInfoContext returns comprehensive IP information with context support. -// -// It finds the most specific live route for the address with point lookups on -// the prefix index: for each mask length from the longest (32 or 128) down to -// minLookupMaskLengthV4 or minLookupMaskLengthV6 it looks up the address's -// network prefix at that length, and the first length with a live route wins. -// That is at most 25 or 113 indexed lookups, however many routes are live. The -// prefix is looked up in the text form net/netip prints, which is the form the -// prefix handler stores. -func (d *Database) GetIPInfoContext(ctx context.Context, ip string) (*IPInfo, error) { - addr, err := netip.ParseAddr(ip) - if err != nil { - return nil, fmt.Errorf("%w: %s", ErrInvalidIP, ip) - } - // An IPv4 address written in IPv6 form (::ffff:192.0.2.1) is an IPv4 lookup. - addr = addr.Unmap() - - info := &IPInfo{IP: ip, IPVersion: ipVersionV6} - routesTable, prefixesTable := "live_routes_v6", "prefixes_v6" - minMaskLength := minLookupMaskLengthV6 - if addr.Is4() { - info.IPVersion = ipVersionV4 - routesTable, prefixesTable = "live_routes_v4", "prefixes_v4" - minMaskLength = minLookupMaskLengthV4 - } - - // Route info with peer count and prefix first_seen - //nolint:gosec // Table names are hardcoded based on IP version - query := fmt.Sprintf(` +// ipInfoQuery reads the live route for one prefix with its peer count, the +// prefix's first_seen and the origin AS's details. %[1]s is the live route +// table and %[2]s the prefix table of the address's IP version. +const ipInfoQuery = ` SELECT lr.prefix, lr.mask_length, lr.origin_asn, lr.last_updated, - (SELECT COUNT(DISTINCT peer_ip) FROM %[1]s WHERE prefix = lr.prefix) as num_peers, + (SELECT COUNT(DISTINCT peer_ip) FROM %[1]s + WHERE prefix = lr.prefix) as num_peers, p.first_seen, a.handle, a.description, @@ -2135,7 +2053,39 @@ func (d *Database) GetIPInfoContext(ctx context.Context, ip string) (*IPInfo, er LEFT JOIN asns a ON a.asn = lr.origin_asn WHERE lr.prefix = ? LIMIT 1 - `, routesTable, prefixesTable) + ` + +// GetIPInfoContext returns comprehensive IP information with context support. +// +// It finds the most specific live route for the address with point lookups on +// the prefix index: for each mask length from the longest (32 or 128) down to +// minLookupMaskLengthV4 or minLookupMaskLengthV6 it looks up the address's +// network prefix at that length, and the first length with a live route wins. +// That is at most 25 or 113 indexed lookups, however many routes are live. The +// prefix is looked up in the text form net/netip prints, which is the form the +// prefix handler stores. +func (d *Database) GetIPInfoContext( + ctx context.Context, ip string, +) (*IPInfo, error) { + addr, err := netip.ParseAddr(ip) + if err != nil { + return nil, fmt.Errorf("%w: %s", ErrInvalidIP, ip) + } + // An IPv4 address written in IPv6 form (::ffff:192.0.2.1) is an IPv4 lookup. + addr = addr.Unmap() + + info := &IPInfo{IP: ip, IPVersion: ipVersionV6} + minMaskLength := minLookupMaskLengthV6 + + routesTable, prefixesTable := tableLiveRoutesV6, "prefixes_v6" + if addr.Is4() { + routesTable, prefixesTable = tableLiveRoutesV4, "prefixes_v4" + minMaskLength = minLookupMaskLengthV4 + info.IPVersion = ipVersionV4 + } + + // Route info with peer count and prefix first_seen + query := fmt.Sprintf(ipInfoQuery, routesTable, prefixesTable) stmt, err := d.db.PrepareContext(ctx, query) if err != nil { @@ -2143,41 +2093,36 @@ func (d *Database) GetIPInfoContext(ctx context.Context, ip string) (*IPInfo, er } defer func() { _ = stmt.Close() }() - var handle, description, asName, orgName, orgID, address, countryCode, abuseEmail, rir sql.NullString - var prefixFirstSeen sql.NullTime - var whoisUpdatedAt sql.NullTime + var ( + handle, description, asName, orgName, orgID sql.NullString + address, countryCode, abuseEmail, rir sql.NullString + prefixFirstSeen, whoisUpdatedAt sql.NullTime + ) found := false + for bits := addr.BitLen(); bits >= minMaskLength; bits-- { prefix := netip.PrefixFrom(addr, bits).Masked().String() + err := stmt.QueryRowContext(ctx, prefix).Scan( - &info.Netblock, - &info.MaskLength, - &info.ASN, - &info.LastSeen, - &info.NumPeers, - &prefixFirstSeen, - &handle, - &description, - &asName, - &orgName, - &orgID, - &address, - &countryCode, - &abuseEmail, - &rir, - &whoisUpdatedAt, + &info.Netblock, &info.MaskLength, &info.ASN, &info.LastSeen, + &info.NumPeers, &prefixFirstSeen, &handle, &description, + &asName, &orgName, &orgID, &address, &countryCode, &abuseEmail, + &rir, &whoisUpdatedAt, ) if errors.Is(err, sql.ErrNoRows) { continue } + if err != nil { return nil, fmt.Errorf("failed to query routes: %w", err) } + found = true break } + if !found { return nil, fmt.Errorf("%w for IP %s", ErrNoRoute, ip) } @@ -2198,18 +2143,23 @@ func (d *Database) GetIPInfoContext(ctx context.Context, ip string) (*IPInfo, er // Check if WHOIS data needs refresh (never fetched or older than 30 days) const staleThreshold = 30 * 24 * time.Hour - info.NeedsWHOISRefresh = !whoisUpdatedAt.Valid || time.Since(whoisUpdatedAt.Time) > staleThreshold + + info.NeedsWHOISRefresh = !whoisUpdatedAt.Valid || + time.Since(whoisUpdatedAt.Time) > staleThreshold return info, nil } -// Vacuum runs incremental vacuum to reclaim unused pages without blocking writes. -// It frees up to the specified number of pages per call (0 = all freeable pages). +// Vacuum runs incremental vacuum to reclaim unused pages without blocking +// writes. It frees up to the specified number of pages per call (0 = all +// freeable pages). func (d *Database) Vacuum(ctx context.Context) error { // Free up to 1000 pages per call (~4MB with default 4KB page size) // This keeps each vacuum operation quick and non-blocking const pagesToFree = 1000 - rows, err := d.db.QueryContext(ctx, fmt.Sprintf("PRAGMA incremental_vacuum(%d)", pagesToFree)) + + rows, err := d.db.QueryContext(ctx, + fmt.Sprintf("PRAGMA incremental_vacuum(%d)", pagesToFree)) if err != nil { return fmt.Errorf("failed to run incremental vacuum: %w", err) } @@ -2220,7 +2170,9 @@ func (d *Database) Vacuum(ctx context.Context) error { // than one page. for rows.Next() { } - if err := rows.Err(); err != nil { + + err = rows.Err() + if err != nil { return fmt.Errorf("failed to run incremental vacuum: %w", err) } @@ -2252,6 +2204,7 @@ func (d *Database) Checkpoint(ctx context.Context) error { // Ping checks if the database connection is alive with a lightweight query. func (d *Database) Ping(ctx context.Context) error { var result int + err := d.db.QueryRowContext(ctx, "SELECT 1").Scan(&result) if err != nil { return fmt.Errorf("database ping failed: %w", err) @@ -2259,3 +2212,380 @@ func (d *Database) Ping(ctx context.Context) error { return nil } + +// lock acquires the database mutex and logs debug information +func (d *Database) lock(operation string) { + // Get caller information + _, file, line, _ := runtime.Caller(1) + caller := fmt.Sprintf("%s:%d", filepath.Base(file), line) + + d.logger.Debug("Acquiring database lock", + "operation", operation, "caller", caller) + + d.mu.Lock() + d.lockedAt = time.Now() + d.lockedBy = fmt.Sprintf("%s (%s)", operation, caller) + + d.logger.Debug("Database lock acquired", + "operation", operation, "caller", caller) +} + +// unlock releases the database mutex and logs debug information including +// hold duration +func (d *Database) unlock() { + holdDuration := time.Since(d.lockedAt) + lockedBy := d.lockedBy + + d.lockedAt = time.Time{} + d.lockedBy = "" + d.mu.Unlock() + + d.logger.Debug("Database lock released", + "held_by", lockedBy, "duration_ms", holdDuration.Milliseconds()) +} + +// beginTx starts a new transaction with logging +func (d *Database) beginTx() (*loggingTx, error) { + tx, err := d.db.BeginTx(context.Background(), nil) + if err != nil { + return nil, err + } + + return &loggingTx{Tx: tx, logger: d.logger}, nil +} + +// rollback rolls back tx unless it has already been committed, and logs a +// failure. Every write defers it right after beginning its transaction. +func (d *Database) rollback(tx *loggingTx) { + err := tx.Rollback() + if err != nil && !errors.Is(err, sql.ErrTxDone) { + d.logger.Error("Failed to rollback transaction", "error", err) + } +} + +// routeTimestampRange returns the earliest and latest last_updated across both +// live route tables, or nil values when both tables are empty. Each query reads +// one row from an end of the last_updated index rather than scanning the tables. +func (d *Database) routeTimestampRange( + ctx context.Context, +) (*time.Time, *time.Time, error) { + var oldest, newest *time.Time + + oldestV4, ok, err := d.scanRouteTimestamp(ctx, "SELECT last_updated "+ + "FROM live_routes_v4 ORDER BY last_updated ASC LIMIT 1") + if err != nil { + return nil, nil, err + } + + if ok { + oldest = &oldestV4 + } + + oldestV6, ok, err := d.scanRouteTimestamp(ctx, "SELECT last_updated "+ + "FROM live_routes_v6 ORDER BY last_updated ASC LIMIT 1") + if err != nil { + return nil, nil, err + } + + if ok && (oldest == nil || oldestV6.Before(*oldest)) { + oldest = &oldestV6 + } + + newestV4, ok, err := d.scanRouteTimestamp(ctx, "SELECT last_updated "+ + "FROM live_routes_v4 ORDER BY last_updated DESC LIMIT 1") + if err != nil { + return nil, nil, err + } + + if ok { + newest = &newestV4 + } + + newestV6, ok, err := d.scanRouteTimestamp(ctx, "SELECT last_updated "+ + "FROM live_routes_v6 ORDER BY last_updated DESC LIMIT 1") + if err != nil { + return nil, nil, err + } + + if ok && (newest == nil || newestV6.After(*newest)) { + newest = &newestV6 + } + + return oldest, newest, nil +} + +// scanRouteTimestamp runs a single-row timestamp query. The bool is false when +// the table is empty. The query selects the last_updated column directly so the +// driver parses the DATETIME value into a time.Time. +func (d *Database) scanRouteTimestamp( + ctx context.Context, query string, +) (time.Time, bool, error) { + var ts time.Time + + err := d.db.QueryRowContext(ctx, query).Scan(&ts) + switch { + case errors.Is(err, sql.ErrNoRows): + return time.Time{}, false, nil + case err != nil: + return time.Time{}, false, err + default: + return ts, true, nil + } +} + +// originPrefixes runs query, which selects the prefix, mask length and latest +// last_updated of every prefix asn originates in the live route table of +// ipVersion. Rows that fail to scan or whose timestamp does not parse are +// logged and skipped. +func (d *Database) originPrefixes( + ctx context.Context, query string, asn, ipVersion int, +) ([]LiveRoute, error) { + family := ipv4Family + if ipVersion == ipVersionV6 { + family = ipv6Family + } + + rows, err := d.db.QueryContext(ctx, query, asn) + if err != nil { + return nil, fmt.Errorf("failed to query %s prefixes: %w", family, err) + } + defer func() { _ = rows.Close() }() + + var routes []LiveRoute + + for rows.Next() { + var ( + route LiveRoute + lastUpdatedStr string + ) + + err := rows.Scan(&route.Prefix, &route.MaskLength, &lastUpdatedStr) + if err != nil { + d.logger.Error("Failed to scan "+family+" prefix row", + "error", err, "asn", asn) + + continue + } + + // Parse the timestamp string + route.LastUpdated, err = time.Parse("2006-01-02 15:04:05-07:00", + lastUpdatedStr) + if err != nil { + // Try without timezone + route.LastUpdated, err = time.Parse("2006-01-02 15:04:05", + lastUpdatedStr) + if err != nil { + d.logger.Error("Failed to parse timestamp", + "error", err, "timestamp", lastUpdatedStr) + + continue + } + } + + route.OriginASN = asn + route.IPVersion = ipVersion + routes = append(routes, route) + } + + err = rows.Err() + if err != nil { + return nil, fmt.Errorf("failed to read %s prefixes: %w", family, err) + } + + return routes, nil +} + +// firstPrefixesByLength returns the first n prefixes, in text order, with the +// given mask length. Each prefix is one search of the mask length and prefix +// index for the prefix after the previous one, so the read does not grow with +// the number of peers that carry a prefix, as a SELECT DISTINCT would. +func (d *Database) firstPrefixesByLength( + ctx context.Context, tableName string, maskLength, n int) ([]string, error) { + //nolint:gosec // Table name is hardcoded based on IP version + query := fmt.Sprintf(` + SELECT prefix FROM %s + WHERE mask_length = ? AND prefix > ? + ORDER BY prefix + LIMIT 1 + `, tableName) + + stmt, err := d.db.PrepareContext(ctx, query) + if err != nil { + return nil, fmt.Errorf("failed to prepare prefix query: %w", err) + } + defer func() { _ = stmt.Close() }() + + var prefixes []string + + previous := "" // every prefix sorts after the empty string + + for len(prefixes) < n { + var prefix string + + err := stmt.QueryRowContext(ctx, maskLength, previous).Scan(&prefix) + if errors.Is(err, sql.ErrNoRows) { + break + } + + if err != nil { + return nil, fmt.Errorf("failed to list prefixes: %w", err) + } + + prefixes = append(prefixes, prefix) + previous = prefix + } + + return prefixes, nil +} + +// samplePrefixesByLength picks limit distinct prefixes with the given mask +// length at random, in at most sampleLookupsPerPrefix lookups for each prefix +// asked for. ORDER BY RANDOM() would read every route with that mask length. +// Instead each lookup draws a row id between those of the first and last route +// with that mask length and takes the prefix of the first such route at or +// after it. SQLite keeps the row id at the end of every index entry, so the +// mask length index is ordered by mask length, then row id, and the lookup is +// one search of it. +// +// The sample is uneven, as the page needs a varied sample, not an even one: a +// route is picked in proportion to the row ids between it and the previous +// route with that mask length, including the row ids of routes with other mask +// lengths and of withdrawn routes. A route that follows a long run of +// withdrawn routes is picked far more often than one that directly follows +// another route with that mask length. +// +// Should the lookups keep landing on prefixes already picked, the rest are +// taken from shuffled, prefixes of that length in random order. +func (d *Database) samplePrefixesByLength( + ctx context.Context, tableName string, maskLength, limit int, + shuffled []string, +) ([]string, error) { + //nolint:gosec // Table name is hardcoded based on IP version + rangeQuery := fmt.Sprintf(` + SELECT + (SELECT MIN(rowid) FROM %[1]s WHERE mask_length = ?), + (SELECT MAX(rowid) FROM %[1]s WHERE mask_length = ?) + `, tableName) + + var first, last sql.NullInt64 + + err := d.db.QueryRowContext(ctx, rangeQuery, maskLength, maskLength). + Scan(&first, &last) + if err != nil { + return nil, fmt.Errorf("failed to read row id range: %w", err) + } + + if !first.Valid { + // Every route with this mask length was withdrawn since they were listed. + return nil, nil + } + + //nolint:gosec // Table name is hardcoded based on IP version + lookupQuery := fmt.Sprintf(` + SELECT prefix FROM %s + WHERE mask_length = ? AND rowid >= ? + ORDER BY rowid + LIMIT 1 + `, tableName) + + stmt, err := d.db.PrepareContext(ctx, lookupQuery) + if err != nil { + return nil, fmt.Errorf("failed to prepare prefix lookup: %w", err) + } + defer func() { _ = stmt.Close() }() + + picked := make(map[string]bool, limit) + prefixes := make([]string, 0, limit) + pick := func(prefix string) { + if !picked[prefix] { + picked[prefix] = true + prefixes = append(prefixes, prefix) + } + } + + maxLookups := limit * sampleLookupsPerPrefix + for lookups := 0; lookups < maxLookups && len(prefixes) < limit; lookups++ { + //nolint:gosec // The sample needs variety, not unpredictability. + rowID := first.Int64 + rand.Int64N(last.Int64-first.Int64+1) + + var prefix string + + err := stmt.QueryRowContext(ctx, maskLength, rowID).Scan(&prefix) + if errors.Is(err, sql.ErrNoRows) { + // The routes from rowID on were withdrawn since the range was read. + continue + } + + if err != nil { + return nil, fmt.Errorf("failed to look up prefix: %w", err) + } + + pick(prefix) + } + + for _, prefix := range shuffled { + if len(prefixes) == limit { + break + } + + pick(prefix) + } + + return prefixes, nil +} + +// newestRouteOfEachPrefix returns the most recently updated live route of each +// prefix, one search of the prefix and last_updated index per prefix however +// many peers carry it. A prefix whose routes were all withdrawn since it was +// picked is left out. +func (d *Database) newestRouteOfEachPrefix( + ctx context.Context, tableName string, ipVersion int, prefixes []string, +) ([]LiveRoute, error) { + //nolint:gosec // Table name is hardcoded based on IP version + query := fmt.Sprintf(` + SELECT prefix, mask_length, origin_asn, as_path, peer_ip, last_updated + FROM %s + WHERE prefix = ? + ORDER BY last_updated DESC + LIMIT 1 + `, tableName) + + stmt, err := d.db.PrepareContext(ctx, query) + if err != nil { + return nil, fmt.Errorf("failed to prepare route query: %w", err) + } + defer func() { _ = stmt.Close() }() + + routes := make([]LiveRoute, 0, len(prefixes)) + + for _, prefix := range prefixes { + route := LiveRoute{IPVersion: ipVersion} + + var pathJSON string + + err := stmt.QueryRowContext(ctx, prefix).Scan( + &route.Prefix, + &route.MaskLength, + &route.OriginASN, + &pathJSON, + &route.PeerIP, + &route.LastUpdated, + ) + if errors.Is(err, sql.ErrNoRows) { + continue + } + + if err != nil { + return nil, fmt.Errorf("failed to query route of %s: %w", prefix, err) + } + + err = json.Unmarshal([]byte(pathJSON), &route.ASPath) + if err != nil { + route.ASPath = []int{} + } + + routes = append(routes, route) + } + + return routes, nil +} diff --git a/internal/database/database_test.go b/internal/database/database_internal_test.go similarity index 80% rename from internal/database/database_test.go rename to internal/database/database_internal_test.go index 3628feb..50f3f77 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_internal_test.go @@ -78,6 +78,8 @@ const ( // address, ErrNoRoute when none covers it, and the next less specific prefix // once the only route of the most specific one is withdrawn. func TestGetIPInfoFindsMostSpecificLiveRoute(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -95,24 +97,7 @@ func TestGetIPInfoFindsMostSpecificLiveRoute(t *testing.T) { "2001:db8:1::/48": 64501, "2001:db8:1:2::/64": 64502, } - ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) - routes := make([]*LiveRoute, 0, len(origins)) - for prefix, asn := range origins { - routes = append(routes, &LiveRoute{ - ID: uuid.New(), - Prefix: prefix, - MaskLength: netip.MustParsePrefix(prefix).Bits(), - IPVersion: detectIPVersion(prefix), - OriginASN: asn, - PeerIP: testPeerIP, - ASPath: []int{asn}, - NextHop: testPeerIP, - LastUpdated: ts, - }) - } - if err := db.UpsertLiveRouteBatch(routes); err != nil { - t.Fatalf("UpsertLiveRouteBatch: %v", err) - } + writeOriginRoutes(t, db, origins) // lookup checks that ip resolves to the live prefix want, or to ErrNoRoute // when want is empty. @@ -127,11 +112,13 @@ func TestGetIPInfoFindsMostSpecificLiveRoute(t *testing.T) { return } + if err != nil { t.Errorf("GetIPInfo(%s): %v", ip, err) return } + if info.Netblock != want || info.MaskLength != netip.MustParsePrefix(want).Bits() || info.ASN != origins[want] { t.Errorf("GetIPInfo(%s) = %s (mask %d) AS%d, want %s AS%d", @@ -166,6 +153,8 @@ func TestGetIPInfoFindsMostSpecificLiveRoute(t *testing.T) { // route for an address, and that a route of exactly /8 or /16, or longer, still // is. The prefixes are written out so that a wrong limit fails the test. func TestGetIPInfoIgnoresShortRoutes(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -174,24 +163,11 @@ func TestGetIPInfoIgnoresShortRoutes(t *testing.T) { } defer func() { _ = db.Close() }() - ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + // announce writes one live route for prefix, originated by asn. announce := func(prefix string, asn int) { t.Helper() - err := db.UpsertLiveRouteBatch([]*LiveRoute{{ - ID: uuid.New(), - Prefix: prefix, - MaskLength: netip.MustParsePrefix(prefix).Bits(), - IPVersion: detectIPVersion(prefix), - OriginASN: asn, - PeerIP: testPeerIP, - ASPath: []int{asn}, - NextHop: testPeerIP, - LastUpdated: ts, - }}) - if err != nil { - t.Fatalf("UpsertLiveRouteBatch(%s): %v", prefix, err) - } + writeOriginRoutes(t, db, map[string]int{prefix: asn}) } // lookup checks that ip resolves to the live prefix netblock from asn. @@ -204,8 +180,10 @@ func TestGetIPInfoIgnoresShortRoutes(t *testing.T) { return } + if info.Netblock != netblock || info.ASN != asn { - t.Errorf("GetIPInfo(%s) = %s AS%d, want %s AS%d", ip, info.Netblock, info.ASN, netblock, asn) + t.Errorf("GetIPInfo(%s) = %s AS%d, want %s AS%d", + ip, info.Netblock, info.ASN, netblock, asn) } } @@ -216,8 +194,10 @@ func TestGetIPInfoIgnoresShortRoutes(t *testing.T) { announce("192.0.0.0/7", 64500) for _, ip := range []string{"2001:db8::1", "192.0.2.1"} { - if info, err := db.GetIPInfo(ip); !errors.Is(err, ErrNoRoute) { - t.Errorf("GetIPInfo(%s) with only short routes = %+v, %v; want ErrNoRoute", ip, info, err) + info, err := db.GetIPInfo(ip) + if !errors.Is(err, ErrNoRoute) { + t.Errorf("GetIPInfo(%s) with only short routes = %+v, %v; "+ + "want ErrNoRoute", ip, info, err) } } @@ -238,6 +218,8 @@ func TestGetIPInfoIgnoresShortRoutes(t *testing.T) { // origin AS. An upsert updates the peer's route before it would insert one, so // only an insert straight into the table shows what the table allows. func TestLiveRouteTablesHoldOneRoutePerPrefixAndPeer(t *testing.T) { + t.Parallel() + db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New()) if err != nil { t.Fatalf("failed to create database: %v", err) @@ -245,33 +227,76 @@ func TestLiveRouteTablesHoldOneRoutePerPrefixAndPeer(t *testing.T) { defer func() { _ = db.Close() }() const otherOriginASN = 64501 + ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + for _, tt := range []struct { route *LiveRoute insertSQL string }{ - {route: mkV4Route(t, "198.51.100.0/24", 64500, ts), insertSQL: insertLiveRouteV4SQL}, - {route: mkV6Route("2001:db8::/32", 64500, ts), insertSQL: insertLiveRouteV6SQL}, + { + route: mkV4Route(t, "198.51.100.0/24", 64500, ts), + insertSQL: insertLiveRouteV4SQL, + }, + { + route: mkV6Route(testV6Prefix, 64500, ts), + insertSQL: insertLiveRouteV6SQL, + }, } { - if err := db.UpsertLiveRoute(tt.route); err != nil { + err := db.UpsertLiveRoute(tt.route) + if err != nil { t.Fatalf("UpsertLiveRoute(%s): %v", tt.route.Prefix, err) } - _, err := db.db.Exec(tt.insertSQL, uuid.NewString(), tt.route.Prefix, tt.route.MaskLength, - otherOriginASN, tt.route.PeerIP, "[]", tt.route.NextHop, ts) + _, err = db.db.ExecContext(t.Context(), tt.insertSQL, uuid.NewString(), + tt.route.Prefix, tt.route.MaskLength, otherOriginASN, + tt.route.PeerIP, "[]", tt.route.NextHop, ts) + var sqliteErr sqlite3.Error - if !errors.As(err, &sqliteErr) || sqliteErr.ExtendedCode != sqlite3.ErrConstraintUnique { - t.Errorf("second route from peer %s for %s: error %v, want a UNIQUE constraint error", + if !errors.As(err, &sqliteErr) || + sqliteErr.ExtendedCode != sqlite3.ErrConstraintUnique { + t.Errorf("second route from peer %s for %s: error %v, "+ + "want a UNIQUE constraint error", tt.route.PeerIP, tt.route.Prefix, err) } } } +// writeOriginRoutes writes one live route from testPeerIP for each prefix in +// origins, originated by the AS it maps to. +func writeOriginRoutes(t *testing.T, db *Database, origins map[string]int) { + t.Helper() + + ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + routes := make([]*LiveRoute, 0, len(origins)) + + for prefix, asn := range origins { + routes = append(routes, &LiveRoute{ + ID: uuid.New(), + Prefix: prefix, + MaskLength: netip.MustParsePrefix(prefix).Bits(), + IPVersion: detectIPVersion(prefix), + OriginASN: asn, + PeerIP: testPeerIP, + ASPath: []int{asn}, + NextHop: testPeerIP, + LastUpdated: ts, + }) + } + + err := db.UpsertLiveRouteBatch(routes) + if err != nil { + t.Fatalf("UpsertLiveRouteBatch: %v", err) + } +} + // TestConnectionPoolPragmas holds several pooled connections open at once and // checks each one carries the per-connection settings from the DSN, plus the // process-wide hard heap limit, and sees the new file with auto_vacuum // incremental. func TestConnectionPoolPragmas(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -291,57 +316,64 @@ func TestConnectionPoolPragmas(t *testing.T) { } }() - for i := 0; i < heldConnections; i++ { + for i := range heldConnections { c, err := db.db.Conn(ctx) if err != nil { t.Fatalf("failed to open connection %d: %v", i, err) } + conns = append(conns, c) } for i, c := range conns { - var cacheSize int - if err := c.QueryRowContext(ctx, "PRAGMA cache_size").Scan(&cacheSize); err != nil { - t.Fatalf("conn %d: failed to read cache_size: %v", i, err) - } + cacheSize := readPragma(ctx, t, c, i, "cache_size") if cacheSize != sqliteCacheSizeKiB { - t.Errorf("conn %d: cache_size = %d, want %d", i, cacheSize, sqliteCacheSizeKiB) + t.Errorf("conn %d: cache_size = %d, want %d", + i, cacheSize, sqliteCacheSizeKiB) } - var busyTimeout int - if err := c.QueryRowContext(ctx, "PRAGMA busy_timeout").Scan(&busyTimeout); err != nil { - t.Fatalf("conn %d: failed to read busy_timeout: %v", i, err) - } + busyTimeout := readPragma(ctx, t, c, i, "busy_timeout") if busyTimeout != sqliteBusyTimeoutMs { - t.Errorf("conn %d: busy_timeout = %d, want %d", i, busyTimeout, sqliteBusyTimeoutMs) + t.Errorf("conn %d: busy_timeout = %d, want %d", + i, busyTimeout, sqliteBusyTimeoutMs) } - var tempStore int - if err := c.QueryRowContext(ctx, "PRAGMA temp_store").Scan(&tempStore); err != nil { - t.Fatalf("conn %d: failed to read temp_store: %v", i, err) - } + tempStore := readPragma(ctx, t, c, i, "temp_store") if tempStore == tempStoreMemory { - t.Errorf("conn %d: temp_store = %d, want anything but %d (MEMORY)", i, tempStore, tempStoreMemory) + t.Errorf("conn %d: temp_store = %d, want anything but %d (MEMORY)", + i, tempStore, tempStoreMemory) } - var hardHeapLimit int64 - if err := c.QueryRowContext(ctx, "PRAGMA hard_heap_limit").Scan(&hardHeapLimit); err != nil { - t.Fatalf("conn %d: failed to read hard_heap_limit: %v", i, err) - } + hardHeapLimit := readPragma(ctx, t, c, i, "hard_heap_limit") if hardHeapLimit != sqliteHardHeapLimitBytes { - t.Errorf("conn %d: hard_heap_limit = %d, want %d", i, hardHeapLimit, sqliteHardHeapLimitBytes) + t.Errorf("conn %d: hard_heap_limit = %d, want %d", + i, hardHeapLimit, sqliteHardHeapLimitBytes) } - var autoVacuum int - if err := c.QueryRowContext(ctx, "PRAGMA auto_vacuum").Scan(&autoVacuum); err != nil { - t.Fatalf("conn %d: failed to read auto_vacuum: %v", i, err) - } + autoVacuum := readPragma(ctx, t, c, i, "auto_vacuum") if autoVacuum != autoVacuumIncremental { - t.Errorf("conn %d: auto_vacuum = %d, want %d (incremental)", i, autoVacuum, autoVacuumIncremental) + t.Errorf("conn %d: auto_vacuum = %d, want %d (incremental)", + i, autoVacuum, autoVacuumIncremental) } } } +// readPragma reads the integer value of PRAGMA name on connection i, c. +func readPragma( + ctx context.Context, t *testing.T, c *sql.Conn, i int, name string, +) int64 { + t.Helper() + + var value int64 + + err := c.QueryRowContext(ctx, "PRAGMA "+name).Scan(&value) + if err != nil { + t.Fatalf("conn %d: failed to read %s: %v", i, name, err) + } + + return value +} + // TestVacuumReturnsFreePages checks that after routes are deleted from a new // database, one Vacuum call returns every page they used. The deletes leave // fewer free pages than the 1000 Vacuum frees per call, so none may remain. @@ -349,6 +381,8 @@ func TestConnectionPoolPragmas(t *testing.T) { // the free pages stayed in the file, and with the PRAGMA run by ExecContext // Vacuum freed only one page per call. func TestVacuumReturnsFreePages(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -359,10 +393,12 @@ func TestVacuumReturnsFreePages(t *testing.T) { ctx := context.Background() ts := time.Now().UTC() + const asn = 64500 routes := make([]*LiveRoute, 0, vacuumTestRoutes) deletions := make([]LiveRouteDeletion, 0, vacuumTestRoutes) + for i := range vacuumTestRoutes { route := mkV4Route(t, fmt.Sprintf("10.%d.%d.0/24", i/256, i%256), asn, ts) routes = append(routes, route) @@ -373,29 +409,39 @@ func TestVacuumReturnsFreePages(t *testing.T) { }) } - if err := db.UpsertLiveRouteBatch(routes); err != nil { + err = db.UpsertLiveRouteBatch(routes) + if err != nil { t.Fatalf("failed to write routes: %v", err) } - if err := db.DeleteLiveRouteBatch(deletions); err != nil { + + err = db.DeleteLiveRouteBatch(deletions) + if err != nil { t.Fatalf("failed to delete routes: %v", err) } var before int - if err := db.db.QueryRowContext(ctx, "PRAGMA freelist_count").Scan(&before); err != nil { + + err = db.db.QueryRowContext(ctx, "PRAGMA freelist_count").Scan(&before) + if err != nil { t.Fatalf("failed to read freelist_count: %v", err) } + if before == 0 { t.Fatalf("no free pages after deleting %d routes", vacuumTestRoutes) } - if err := db.Vacuum(ctx); err != nil { + err = db.Vacuum(ctx) + if err != nil { t.Fatalf("Vacuum failed: %v", err) } var after int - if err := db.db.QueryRowContext(ctx, "PRAGMA freelist_count").Scan(&after); err != nil { + + err = db.db.QueryRowContext(ctx, "PRAGMA freelist_count").Scan(&after) + if err != nil { t.Fatalf("failed to read freelist_count: %v", err) } + if after != 0 { t.Errorf("free pages after Vacuum = %d of %d, want 0", after, before) } @@ -410,6 +456,8 @@ func TestVacuumReturnsFreePages(t *testing.T) { // no batch is dropped. The checkpoint runs without the Database mutex, exactly // as the background maintainer does in production. func TestBatchWriteDuringCheckpoint(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -422,8 +470,10 @@ func TestBatchWriteDuringCheckpoint(t *testing.T) { var wg sync.WaitGroup wg.Add(1) + go func() { defer wg.Done() + for { select { case <-ctx.Done(): @@ -435,12 +485,14 @@ func TestBatchWriteDuringCheckpoint(t *testing.T) { }() ts := time.Now().UTC() - for i := 0; i < contentionIterations; i++ { + for i := range contentionIterations { asns := map[int]time.Time{ i % contendedASNCount: ts, (i % contendedASNCount) + asnSecondBand: ts, } - if err := db.GetOrCreateASNBatch(asns); err != nil { + + err := db.GetOrCreateASNBatch(asns) + if err != nil { cancel() wg.Wait() t.Fatalf("batch write failed under checkpoint contention: %v", err) @@ -455,6 +507,8 @@ func TestBatchWriteDuringCheckpoint(t *testing.T) { // family learned from two peers in the same second and checks that the // prefix-length sample lists each of them once. func TestGetRandomPrefixesByLengthListsPrefixOnce(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -467,18 +521,24 @@ func TestGetRandomPrefixesByLengthListsPrefixOnce(t *testing.T) { v4First := mkV4Route(t, "198.51.100.0/24", 64500, ts) v4Second := mkV4Route(t, "198.51.100.0/24", 64500, ts) v4Second.PeerIP = secondV4PeerIP - v6First := mkV6Route("2001:db8::/32", 64500, ts) - v6Second := mkV6Route("2001:db8::/32", 64500, ts) + v6First := mkV6Route(testV6Prefix, 64500, ts) + v6Second := mkV6Route(testV6Prefix, 64500, ts) v6Second.PeerIP = secondV6PeerIP - if err := db.UpsertLiveRouteBatch([]*LiveRoute{v4First, v4Second, v6First, v6Second}); err != nil { + + err = db.UpsertLiveRouteBatch( + []*LiveRoute{v4First, v4Second, v6First, v6Second}) + if err != nil { t.Fatalf("UpsertLiveRouteBatch: %v", err) } for _, want := range []*LiveRoute{v4First, v6First} { - routes, err := db.GetRandomPrefixesByLength(want.MaskLength, want.IPVersion, sampleLimit) + routes, err := db.GetRandomPrefixesByLength(want.MaskLength, + want.IPVersion, sampleLimit) if err != nil { - t.Fatalf("GetRandomPrefixesByLength(%d, %d): %v", want.MaskLength, want.IPVersion, err) + t.Fatalf("GetRandomPrefixesByLength(%d, %d): %v", + want.MaskLength, want.IPVersion, err) } + if len(routes) != 1 || routes[0].Prefix != want.Prefix { t.Errorf("GetRandomPrefixesByLength(%d, %d) = %+v, want %s listed once", want.MaskLength, want.IPVersion, routes, want.Prefix) @@ -490,6 +550,8 @@ func TestGetRandomPrefixesByLengthListsPrefixOnce(t *testing.T) { // two peers at different times, the older route first, and checks that the // prefix-length sample lists the newer route. func TestGetRandomPrefixesByLengthListsNewestRoute(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -502,17 +564,21 @@ func TestGetRandomPrefixesByLengthListsNewestRoute(t *testing.T) { olderRoute := mkV4Route(t, "198.51.100.0/24", 64500, older) newerRoute := mkV4Route(t, "198.51.100.0/24", 64500, older.Add(time.Minute)) newerRoute.PeerIP = secondV4PeerIP - if err := db.UpsertLiveRouteBatch([]*LiveRoute{olderRoute, newerRoute}); err != nil { + + err = db.UpsertLiveRouteBatch([]*LiveRoute{olderRoute, newerRoute}) + if err != nil { t.Fatalf("UpsertLiveRouteBatch: %v", err) } - routes, err := db.GetRandomPrefixesByLength(newerRoute.MaskLength, ipVersionV4, sampleLimit) + routes, err := db.GetRandomPrefixesByLength(newerRoute.MaskLength, + ipVersionV4, sampleLimit) if err != nil { t.Fatalf("GetRandomPrefixesByLength: %v", err) } + if len(routes) != 1 || routes[0].PeerIP != newerRoute.PeerIP { - t.Errorf("GetRandomPrefixesByLength = %+v, want only the newer route, from %s", - routes, newerRoute.PeerIP) + t.Errorf("GetRandomPrefixesByLength = %+v, want only the newer route, "+ + "from %s", routes, newerRoute.PeerIP) } } @@ -523,6 +589,8 @@ func TestGetRandomPrefixesByLengthListsNewestRoute(t *testing.T) { // shuffled prefixes, and the random lookups used for a mask length with more // than maxPrefixesToShuffle prefixes. func TestGetRandomPrefixesByLengthSamplesManyPrefixes(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -532,8 +600,11 @@ func TestGetRandomPrefixesByLengthSamplesManyPrefixes(t *testing.T) { defer func() { _ = db.Close() }() ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + var routes []*LiveRoute - var stored []string + + stored := make([]string, 0, sampledPrefixCount) + for i := range sampledPrefixCount { prefix := fmt.Sprintf("10.0.%d.0/24", i) for _, p := range []string{prefix, fmt.Sprintf("172.%d.0.0/16", i)} { @@ -542,22 +613,29 @@ func TestGetRandomPrefixesByLengthSamplesManyPrefixes(t *testing.T) { second.PeerIP = secondV4PeerIP routes = append(routes, first, second) } + stored = append(stored, prefix) } - if err := db.UpsertLiveRouteBatch(routes); err != nil { + + err = db.UpsertLiveRouteBatch(routes) + if err != nil { t.Fatalf("UpsertLiveRouteBatch: %v", err) } + slices.Sort(stored) maskLength := routes[0].MaskLength + sample, err := db.GetRandomPrefixesByLength(maskLength, ipVersionV4, sampleLimit) if err != nil { t.Fatalf("GetRandomPrefixesByLength: %v", err) } - var listed []string + + listed := make([]string, 0, len(sample)) for _, route := range sample { listed = append(listed, route.Prefix) } + checkPrefixSample(t, "GetRandomPrefixesByLength", listed, stored) looked, err := db.samplePrefixesByLength( @@ -565,6 +643,7 @@ func TestGetRandomPrefixesByLengthSamplesManyPrefixes(t *testing.T) { if err != nil { t.Fatalf("samplePrefixesByLength: %v", err) } + checkPrefixSample(t, "samplePrefixesByLength", looked, stored) } @@ -577,15 +656,20 @@ func checkPrefixSample(t *testing.T, name string, sample, stored []string) { if len(sample) != sampleLimit { t.Errorf("%s listed %d prefixes, want %d", name, len(sample), sampleLimit) } + listed := make(map[string]bool) + for _, prefix := range sample { if listed[prefix] || !slices.Contains(stored, prefix) { t.Errorf("%s lists %s twice or it is not one of the /24s", name, prefix) } + listed[prefix] = true } + if slices.Equal(slices.Sorted(slices.Values(sample)), stored[:sampleLimit]) { - t.Errorf("%s lists just the first %d prefixes in text order", name, sampleLimit) + t.Errorf("%s lists just the first %d prefixes in text order", + name, sampleLimit) } } @@ -595,6 +679,8 @@ func checkPrefixSample(t *testing.T, name string, sample, stored []string) { // order, and that the sample lists the prefixes asked for, each once, some of // them after those first ones, which only the random lookups reach. func TestGetRandomPrefixesByLengthSamplesPastFirstPrefixes(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -604,51 +690,64 @@ func TestGetRandomPrefixesByLengthSamplesPastFirstPrefixes(t *testing.T) { defer func() { _ = db.Close() }() ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) - var routes []*LiveRoute - var stored []string + routes := make([]*LiveRoute, 0, largeSamplePrefixCount) + stored := make([]string, 0, largeSamplePrefixCount) + for i := range largeSamplePrefixCount { route := mkV6Route(fmt.Sprintf("2001:%x::/32", i), 64500, ts) routes = append(routes, route) stored = append(stored, route.Prefix) } - if err := db.UpsertLiveRouteBatch(routes); err != nil { + + err = db.UpsertLiveRouteBatch(routes) + if err != nil { t.Fatalf("UpsertLiveRouteBatch: %v", err) } + slices.Sort(stored) firstPrefixes := stored[:maxPrefixesToShuffle+1] - maskLength := routes[0].MaskLength - read, err := db.firstPrefixesByLength( - context.Background(), liveRoutesV6Table, maskLength, maxPrefixesToShuffle+1) + + read, err := db.firstPrefixesByLength(context.Background(), + liveRoutesV6Table, maskLength, maxPrefixesToShuffle+1) if err != nil { t.Fatalf("firstPrefixesByLength: %v", err) } + if !slices.Equal(read, firstPrefixes) { - t.Errorf("firstPrefixesByLength read %d prefixes, want the first %d in text order", - len(read), len(firstPrefixes)) + t.Errorf("firstPrefixesByLength read %d prefixes, want the first %d "+ + "in text order", len(read), len(firstPrefixes)) } - sample, err := db.GetRandomPrefixesByLength(maskLength, ipVersionV6, largeSampleLimit) + sample, err := db.GetRandomPrefixesByLength(maskLength, ipVersionV6, + largeSampleLimit) if err != nil { t.Fatalf("GetRandomPrefixesByLength: %v", err) } + if len(sample) != largeSampleLimit { - t.Errorf("GetRandomPrefixesByLength listed %d prefixes, want %d", len(sample), largeSampleLimit) + t.Errorf("GetRandomPrefixesByLength listed %d prefixes, want %d", + len(sample), largeSampleLimit) } + listed := make(map[string]bool) pastFirstPrefixes := 0 + for _, route := range sample { if listed[route.Prefix] { t.Errorf("GetRandomPrefixesByLength lists %s twice", route.Prefix) } + listed[route.Prefix] = true + if !slices.Contains(firstPrefixes, route.Prefix) { pastFirstPrefixes++ } } + if pastFirstPrefixes == 0 { - t.Errorf("GetRandomPrefixesByLength lists only prefixes among the first %d in text order", - len(firstPrefixes)) + t.Errorf("GetRandomPrefixesByLength lists only prefixes among the first "+ + "%d in text order", len(firstPrefixes)) } } @@ -657,6 +756,8 @@ func TestGetRandomPrefixesByLengthSamplesPastFirstPrefixes(t *testing.T) { // sampleLimit prefixes: the stored one, then the shuffled ones in order. The // stored prefix is the second shuffled one, so the fill-up must skip it. func TestSamplePrefixesByLengthFillsUpFromShuffled(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -665,20 +766,25 @@ func TestSamplePrefixesByLengthFillsUpFromShuffled(t *testing.T) { } defer func() { _ = db.Close() }() - var shuffled []string + shuffled := make([]string, 0, sampledPrefixCount) for i := range sampledPrefixCount { shuffled = append(shuffled, fmt.Sprintf("10.0.%d.0/24", i)) } - route := mkV4Route(t, shuffled[1], 64500, time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)) - if err := db.UpsertLiveRouteBatch([]*LiveRoute{route}); err != nil { + + ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + route := mkV4Route(t, shuffled[1], 64500, ts) + + err = db.UpsertLiveRouteBatch([]*LiveRoute{route}) + if err != nil { t.Fatalf("UpsertLiveRouteBatch: %v", err) } - sample, err := db.samplePrefixesByLength( - context.Background(), liveRoutesV4Table, route.MaskLength, sampleLimit, shuffled) + sample, err := db.samplePrefixesByLength(context.Background(), + liveRoutesV4Table, route.MaskLength, sampleLimit, shuffled) if err != nil { t.Fatalf("samplePrefixesByLength: %v", err) } + want := append([]string{shuffled[1], shuffled[0]}, shuffled[2:sampleLimit]...) if !slices.Equal(sample, want) { t.Errorf("samplePrefixesByLength = %v, want %v", sample, want) @@ -691,6 +797,8 @@ func TestSamplePrefixesByLengthFillsUpFromShuffled(t *testing.T) { // left, and that only the prefixes that still have a route get their newest // route listed. func TestGetRandomPrefixesByLengthSkipsWithdrawnRoutes(t *testing.T) { + t.Parallel() + cfg := &config.Config{StateDir: t.TempDir()} db, err := New(cfg, logger.New()) @@ -702,29 +810,38 @@ func TestGetRandomPrefixesByLengthSkipsWithdrawnRoutes(t *testing.T) { ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) kept := mkV4Route(t, "198.51.100.0/24", 64500, ts) withdrawn := mkV4Route(t, "172.16.0.0/16", 64500, ts) - if err := db.UpsertLiveRouteBatch([]*LiveRoute{kept, withdrawn}); err != nil { + + err = db.UpsertLiveRouteBatch([]*LiveRoute{kept, withdrawn}) + if err != nil { t.Fatalf("UpsertLiveRouteBatch: %v", err) } - if err := db.DeleteLiveRoute(withdrawn.Prefix, withdrawn.PeerIP); err != nil { + + err = db.DeleteLiveRoute(withdrawn.Prefix, withdrawn.PeerIP) + if err != nil { t.Fatalf("DeleteLiveRoute: %v", err) } ctx := context.Background() - sample, err := db.samplePrefixesByLength( - ctx, liveRoutesV4Table, withdrawn.MaskLength, sampleLimit, []string{withdrawn.Prefix}) + + sample, err := db.samplePrefixesByLength(ctx, liveRoutesV4Table, + withdrawn.MaskLength, sampleLimit, []string{withdrawn.Prefix}) if err != nil { t.Fatalf("samplePrefixesByLength: %v", err) } + if len(sample) != 0 { - t.Errorf("samplePrefixesByLength = %v, want none for a mask length with no route left", sample) + t.Errorf("samplePrefixesByLength = %v, want none for a mask length "+ + "with no route left", sample) } - routes, err := db.newestRouteOfEachPrefix( - ctx, liveRoutesV4Table, ipVersionV4, []string{withdrawn.Prefix, kept.Prefix}) + routes, err := db.newestRouteOfEachPrefix(ctx, liveRoutesV4Table, + ipVersionV4, []string{withdrawn.Prefix, kept.Prefix}) if err != nil { t.Fatalf("newestRouteOfEachPrefix: %v", err) } + if len(routes) != 1 || routes[0].Prefix != kept.Prefix { - t.Errorf("newestRouteOfEachPrefix = %+v, want only the route of %s", routes, kept.Prefix) + t.Errorf("newestRouteOfEachPrefix = %+v, want only the route of %s", + routes, kept.Prefix) } } diff --git a/internal/database/interface.go b/internal/database/interface.go index 0311f05..2f5f8ac 100644 --- a/internal/database/interface.go +++ b/internal/database/interface.go @@ -29,6 +29,8 @@ type Stats struct { // Store defines the interface for database operations. It provides methods for // managing ASNs, prefixes, announcements, peerings, BGP peers, and live routes. // Implementations must be safe for concurrent use. +// +//nolint:interfacebloat // the one seam handlers and tests depend on type Store interface { // ASN operations GetOrCreateASN(number int, timestamp time.Time) (*ASN, error) @@ -58,8 +60,10 @@ type Store interface { DeleteLiveRoute(prefix, peerIP string) error DeleteLiveRouteBatch(deletions []LiveRouteDeletion) error DeleteLiveRoutesOfPeer(peerIP string) (deleted int, err error) - GetPrefixDistribution() (ipv4 []PrefixDistribution, ipv6 []PrefixDistribution, err error) - GetPrefixDistributionContext(ctx context.Context) (ipv4 []PrefixDistribution, ipv6 []PrefixDistribution, err error) + GetPrefixDistribution() ( + ipv4 []PrefixDistribution, ipv6 []PrefixDistribution, err error) + GetPrefixDistributionContext(ctx context.Context) ( + ipv4 []PrefixDistribution, ipv6 []PrefixDistribution, err error) GetLiveRouteCounts() (ipv4Count, ipv6Count int, err error) GetLiveRouteCountsContext(ctx context.Context) (ipv4Count, ipv6Count int, err error) @@ -78,9 +82,13 @@ type Store interface { GetASPeers(asn int) ([]ASPeer, error) GetASPeersContext(ctx context.Context, asn int) ([]ASPeer, error) GetPrefixDetails(prefix string) ([]LiveRoute, map[int]ASN, error) - GetPrefixDetailsContext(ctx context.Context, prefix string) ([]LiveRoute, map[int]ASN, error) + GetPrefixDetailsContext( + ctx context.Context, prefix string, + ) ([]LiveRoute, map[int]ASN, error) GetRandomPrefixesByLength(maskLength, ipVersion, limit int) ([]LiveRoute, error) - GetRandomPrefixesByLengthContext(ctx context.Context, maskLength, ipVersion, limit int) ([]LiveRoute, error) + GetRandomPrefixesByLengthContext( + ctx context.Context, maskLength, ipVersion, limit int, + ) ([]LiveRoute, error) // Lifecycle Close() error diff --git a/internal/database/models.go b/internal/database/models.go index 54c54f8..3872f95 100644 --- a/internal/database/models.go +++ b/internal/database/models.go @@ -1,4 +1,3 @@ -// Package database provides SQLite storage for BGP routing data. package database import ( @@ -9,6 +8,8 @@ import ( // ASN represents an Autonomous System Number with its metadata including // handle, description, WHOIS data, and first/last seen timestamps. +// +//nolint:tagliatelle // field names are the existing JSON API type ASN struct { ASN int `json:"asn"` Handle string `json:"handle"` @@ -36,37 +37,39 @@ type ASN struct { // Prefix represents an IP prefix (CIDR block) with its IP version (4 or 6) // and first/last seen timestamps. type Prefix struct { - ID uuid.UUID `json:"id"` - Prefix string `json:"prefix"` - IPVersion int `json:"ip_version"` // 4 or 6 - FirstSeen time.Time `json:"first_seen"` - LastSeen time.Time `json:"last_seen"` + ID uuid.UUID + Prefix string + IPVersion int // 4 or 6 + FirstSeen time.Time + LastSeen time.Time } // Announcement represents a BGP announcement or withdrawal event, // containing the prefix, AS path, origin ASN, peer ASN, next hop, and timestamp. type Announcement struct { - ID uuid.UUID `json:"id"` - PrefixID uuid.UUID `json:"prefix_id"` - PeerASN int `json:"peer_asn"` - OriginASN int `json:"origin_asn"` - Path string `json:"path"` // JSON-encoded AS path - NextHop string `json:"next_hop"` - Timestamp time.Time `json:"timestamp"` - IsWithdrawal bool `json:"is_withdrawal"` + ID uuid.UUID + PrefixID uuid.UUID + PeerASN int + OriginASN int + Path string // JSON-encoded AS path + NextHop string + Timestamp time.Time + IsWithdrawal bool } // ASNPeering represents a peering relationship between two ASNs, // stored with the lower ASN as ASA and the higher as ASB. type ASNPeering struct { - ID uuid.UUID `json:"id"` - ASA int `json:"as_a"` - ASB int `json:"as_b"` - FirstSeen time.Time `json:"first_seen"` - LastSeen time.Time `json:"last_seen"` + ID uuid.UUID + ASA int + ASB int + FirstSeen time.Time + LastSeen time.Time } // LiveRoute represents a route in the live routing table +// +//nolint:tagliatelle // field names are the existing JSON API type LiveRoute struct { ID uuid.UUID `json:"id"` Prefix string `json:"prefix"` @@ -80,12 +83,16 @@ type LiveRoute struct { } // PrefixDistribution represents the distribution of prefixes by mask length +// +//nolint:tagliatelle // field names are the existing JSON API type PrefixDistribution struct { MaskLength int `json:"mask_length"` Count int `json:"count"` } // IPInfo represents comprehensive IP information for the /ip endpoint +// +//nolint:tagliatelle // field names are the existing JSON API type IPInfo struct { IP string `json:"ip"` PTR []string `json:"ptr,omitempty"` diff --git a/internal/database/slowquery.go b/internal/database/slowquery.go index 1ac302a..d027211 100644 --- a/internal/database/slowquery.go +++ b/internal/database/slowquery.go @@ -18,30 +18,12 @@ func logSlowQuery(logger *logger.Logger, query string, start time.Time) { } } -// queryRow wraps QueryRow with slow query logging -// nolint:unused // kept for consistency with other query wrappers -func (d *Database) queryRow(query string, args ...interface{}) *sql.Row { - start := time.Now() - defer logSlowQuery(d.logger, query, start) - - return d.db.QueryRow(query, args...) -} - -// query wraps Query with slow query logging -// nolint:unused // kept for future use to ensure all queries go through slow query logging -func (d *Database) query(query string, args ...interface{}) (*sql.Rows, error) { - start := time.Now() - defer logSlowQuery(d.logger, query, start) - - return d.db.Query(query, args...) -} - // exec wraps Exec with slow query logging -func (d *Database) exec(query string, args ...interface{}) error { +func (d *Database) exec(query string, args ...any) error { start := time.Now() defer logSlowQuery(d.logger, query, start) - _, err := d.db.Exec(query, args...) + _, err := d.db.ExecContext(context.Background(), query, args...) return err } @@ -49,51 +31,51 @@ func (d *Database) exec(query string, args ...interface{}) error { // loggingTx wraps sql.Tx to log slow queries type loggingTx struct { *sql.Tx + logger *logger.Logger } // QueryRow wraps sql.Tx.QueryRow to log slow queries -func (tx *loggingTx) QueryRow(query string, args ...interface{}) *sql.Row { +func (tx *loggingTx) QueryRow(query string, args ...any) *sql.Row { start := time.Now() defer logSlowQuery(tx.logger, query, start) - return tx.Tx.QueryRow(query, args...) -} - -// Query wraps sql.Tx.Query to log slow queries -func (tx *loggingTx) Query(query string, args ...interface{}) (*sql.Rows, error) { - start := time.Now() - defer logSlowQuery(tx.logger, query, start) - - return tx.Tx.Query(query, args...) -} - -// QueryContext wraps sql.Tx.QueryContext to log slow queries -func (tx *loggingTx) QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error) { - start := time.Now() - defer logSlowQuery(tx.logger, query, start) - - return tx.Tx.QueryContext(ctx, query, args...) + return tx.Tx.QueryRowContext(context.Background(), query, args...) } // QueryRowContext wraps sql.Tx.QueryRowContext to log slow queries -func (tx *loggingTx) QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row { +func (tx *loggingTx) QueryRowContext( + ctx context.Context, query string, args ...any, +) *sql.Row { start := time.Now() defer logSlowQuery(tx.logger, query, start) return tx.Tx.QueryRowContext(ctx, query, args...) } -// Exec wraps sql.Tx.Exec to log slow queries -func (tx *loggingTx) Exec(query string, args ...interface{}) (sql.Result, error) { +// QueryContext wraps sql.Tx.QueryContext to log slow queries +func (tx *loggingTx) QueryContext( + ctx context.Context, query string, args ...any, +) (*sql.Rows, error) { start := time.Now() defer logSlowQuery(tx.logger, query, start) - return tx.Tx.Exec(query, args...) + //nolint:sqlclosecheck // returns the rows, which the caller closes + return tx.Tx.QueryContext(ctx, query, args...) +} + +// Exec wraps sql.Tx.Exec to log slow queries +func (tx *loggingTx) Exec(query string, args ...any) (sql.Result, error) { + start := time.Now() + defer logSlowQuery(tx.logger, query, start) + + return tx.Tx.ExecContext(context.Background(), query, args...) } // ExecContext wraps sql.Tx.ExecContext to log slow queries -func (tx *loggingTx) ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error) { +func (tx *loggingTx) ExecContext( + ctx context.Context, query string, args ...any, +) (sql.Result, error) { start := time.Now() defer logSlowQuery(tx.logger, query, start) diff --git a/internal/database/utils.go b/internal/database/utils.go index 8094c78..4ba0b0d 100644 --- a/internal/database/utils.go +++ b/internal/database/utils.go @@ -27,6 +27,7 @@ func prefixMaskLength(prefix string) (int, error) { if err != nil { return 0, fmt.Errorf("invalid prefix %s: %w", prefix, err) } + maskLength, _ := network.Mask.Size() return maskLength, nil diff --git a/internal/logger/logger.go b/internal/logger/logger.go index 3e90ad8..d46d523 100644 --- a/internal/logger/logger.go +++ b/internal/logger/logger.go @@ -74,6 +74,7 @@ func getSourceAttrs() []slog.Attr { // Get the function name fn := runtime.FuncForPC(pc) + var funcName string if fn != nil { funcName = filepath.Base(fn.Name()) diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 25aef45..ad04309 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -64,6 +64,7 @@ func New() *Tracker { // first counts one reconnect. func (t *Tracker) SetConnected(connected bool) { t.isConnected.Store(connected) + if !connected { return } @@ -120,9 +121,11 @@ func (t *Tracker) GetStreamMetrics() StreamMetrics { if msgCount >= 0 { totalMessages = uint64(msgCount) } + if byteCount >= 0 { totalBytes = uint64(byteCount) } + if wireByteCount >= 0 { totalWireBytes = uint64(wireByteCount) } @@ -166,7 +169,7 @@ func (t *Tracker) RecordWithdrawal() { func (t *Tracker) SetBGPPeerCount(count int) { // BGP peer count is always small (< 1000), so int32 is safe if count > 0 && count < 1<<31 { - t.bgpPeerCount.Store(int32(count)) //nolint:gosec // count is validated + t.bgpPeerCount.Store(int32(count)) } } @@ -214,7 +217,8 @@ type StreamMetrics struct { TotalMessages uint64 // TotalBytes is the total number of decompressed bytes received since startup TotalBytes uint64 - // TotalWireBytes is the total number of bytes received on the wire (before decompression) + // TotalWireBytes is the total number of bytes received on the wire (before + // decompression) TotalWireBytes uint64 // ConnectedSince is the time when the current connection was established ConnectedSince time.Time @@ -222,9 +226,11 @@ type StreamMetrics struct { Connected bool // MessagesPerSec is the rate of messages received per second (1-minute average) MessagesPerSec float64 - // BitsPerSec is the rate of decompressed bits received per second (1-minute average) + // BitsPerSec is the rate of decompressed bits received per second (1-minute + // average) BitsPerSec float64 - // WireBitsPerSec is the rate of bits received on the wire per second (1-minute average) + // WireBitsPerSec is the rate of bits received on the wire per second + // (1-minute average) WireBitsPerSec float64 // ReconnectCount is the number of reconnections since startup ReconnectCount uint64 diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go index e8776fd..c301122 100644 --- a/internal/metrics/metrics_test.go +++ b/internal/metrics/metrics_test.go @@ -1,12 +1,18 @@ -package metrics +package metrics_test -import "testing" +import ( + "testing" + + "git.eeqj.de/sneak/routewatch/internal/metrics" +) // TestReconnectCount connects three times with a disconnect between each, the // way the streamer does, and expects every connection after the first to count // one reconnect. func TestReconnectCount(t *testing.T) { - tracker := New() + t.Parallel() + + tracker := metrics.New() tracker.SetConnected(true) tracker.SetConnected(false) diff --git a/internal/ristypes/ris.go b/internal/ristypes/ris.go index e8130b6..4b67ee5 100644 --- a/internal/ristypes/ris.go +++ b/internal/ristypes/ris.go @@ -1,4 +1,5 @@ -// Package ristypes defines the data structures for RIS Live BGP messages and announcements. +// Package ristypes defines the data structures for RIS Live BGP messages and +// announcements. package ristypes import ( @@ -33,25 +34,30 @@ type ASPath []int func (p *ASPath) UnmarshalJSON(data []byte) error { // First try to unmarshal as a simple array of integers var simple []int - if err := json.Unmarshal(data, &simple); err == nil { + + err := json.Unmarshal(data, &simple) + if err == nil { *p = ASPath(simple) return nil } // If that fails, unmarshal as array of interfaces and flatten - var raw []interface{} - if err := json.Unmarshal(data, &raw); err != nil { + var raw []any + + err = json.Unmarshal(data, &raw) + if err != nil { return err } // Flatten the array result := make([]int, 0) + for _, item := range raw { switch v := item.(type) { case float64: result = append(result, int(v)) - case []interface{}: + case []any: // Nested array - flatten it for _, nested := range v { if num, ok := nested.(float64); ok { @@ -78,6 +84,8 @@ type RISLiveMessage struct { // It contains metadata about the BGP session (peer, ASN, host) along with // the actual BGP update data including AS path, communities, announcements, // and withdrawals. +// +//nolint:tagliatelle // field names are the RIS Live feed's type RISMessage struct { Type string `json:"type"` Timestamp float64 `json:"timestamp"` @@ -108,6 +116,8 @@ type RISMessage struct { // RISAnnouncement represents a BGP route announcement within a RIS message. // It contains the next hop IP address and the list of prefixes being announced // via that next hop. +// +//nolint:tagliatelle // field names are the RIS Live feed's type RISAnnouncement struct { NextHop string `json:"next_hop"` Prefixes []string `json:"prefixes"` diff --git a/internal/ristypes/ris_test.go b/internal/ristypes/ris_test.go index 6a3b7b0..2b3f7d2 100644 --- a/internal/ristypes/ris_test.go +++ b/internal/ristypes/ris_test.go @@ -1,10 +1,12 @@ -package ristypes +package ristypes_test import ( "bufio" "encoding/json" "os" "testing" + + "git.eeqj.de/sneak/routewatch/internal/ristypes" ) // messageExamplesPath is the captured RIS Live feed used as decode fixtures, @@ -15,32 +17,43 @@ const messageExamplesPath = "../../docs/message-examples.json" // streamer does and checks that the fields no handler reads (Community, Raw) // stay empty while the fields handlers use (Path, Announcements) still decode. func TestDecodeDropsCommunityAndRaw(t *testing.T) { + t.Parallel() + f, err := os.Open(messageExamplesPath) if err != nil { t.Fatalf("open fixtures: %v", err) } - defer f.Close() + + defer func() { + _ = f.Close() + }() var messages, withPath, withAnnouncements int scanner := bufio.NewScanner(f) scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024) + for scanner.Scan() { line := scanner.Bytes() if len(line) == 0 { continue } - var wrapper RISLiveMessage - if err := json.Unmarshal(line, &wrapper); err != nil { + var wrapper ristypes.RISLiveMessage + + err := json.Unmarshal(line, &wrapper) + if err != nil { t.Fatalf("unmarshal message %d: %v", messages+1, err) } + messages++ msg := wrapper.Data if msg.Community != nil { - t.Errorf("message %d: Community decoded, want empty: %v", messages, msg.Community) + t.Errorf("message %d: Community decoded, want empty: %v", + messages, msg.Community) } + if msg.Raw != "" { t.Errorf("message %d: Raw decoded, want empty", messages) } @@ -48,11 +61,14 @@ func TestDecodeDropsCommunityAndRaw(t *testing.T) { if len(msg.Path) > 0 { withPath++ } + if len(msg.Announcements) > 0 { withAnnouncements++ } } - if err := scanner.Err(); err != nil { + + err = scanner.Err() + if err != nil { t.Fatalf("scan fixtures: %v", err) } @@ -62,6 +78,7 @@ func TestDecodeDropsCommunityAndRaw(t *testing.T) { // The fixtures include announcement messages; a used field must still decode, // otherwise an empty Community/Raw would prove nothing. if withPath == 0 || withAnnouncements == 0 { - t.Fatalf("used fields did not decode: withPath=%d withAnnouncements=%d", withPath, withAnnouncements) + t.Fatalf("used fields did not decode: withPath=%d withAnnouncements=%d", + withPath, withAnnouncements) } } diff --git a/internal/routewatch/app.go b/internal/routewatch/app.go index 0c29d75..9e79f11 100644 --- a/internal/routewatch/app.go +++ b/internal/routewatch/app.go @@ -1,10 +1,13 @@ -// Package routewatch contains the primary RouteWatch type that represents a running instance -// of the application and contains pointers to its core dependencies, and is responsible for initialization. +// Package routewatch contains the primary RouteWatch type that represents a +// running instance of the application and contains pointers to its core +// dependencies, and is responsible for initialization. It also contains the +// ASN WHOIS fetcher for background updates and the database maintainer for +// background maintenance tasks. package routewatch import ( "context" - "fmt" + "errors" "sync" "time" @@ -18,6 +21,10 @@ import ( "go.uber.org/fx" ) +// errNonBatchedNotImplemented is returned by Run when batched database writes +// are disabled, since only the batched handlers exist. +var errNonBatchedNotImplemented = errors.New("non-batched handlers not implemented") + // Dependencies contains all dependencies for RouteWatch type Dependencies struct { fx.In @@ -68,8 +75,10 @@ func (rw *RouteWatch) Run(ctx context.Context) error { // Apply runtime limit if specified if rw.maxRuntime > 0 { var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, rw.maxRuntime) defer cancel() + rw.logger.Info("Running with time limit", "max_runtime", rw.maxRuntime) } @@ -96,29 +105,31 @@ func (rw *RouteWatch) Run(ctx context.Context) error { // Non-batched handlers not implemented yet rw.logger.Error("Non-batched handlers not implemented") - return fmt.Errorf("non-batched handlers not implemented") + return errNonBatchedNotImplemented } // No longer need routing table handler - PrefixHandler maintains live_routes table // Start streaming - if err := rw.streamer.Start(); err != nil { + err := rw.streamer.Start(ctx) + if err != nil { return err } // Start HTTP server - if err := rw.server.Start(); err != nil { + err = rw.server.Start(ctx) + if err != nil { return err } // Start ASN WHOIS fetcher for background updates rw.asnFetcher = NewASNFetcher(rw.db, rw.logger.Logger) - rw.asnFetcher.Start() + rw.asnFetcher.Start(ctx) rw.server.SetASNFetcher(rw.asnFetcher) // Start database maintenance goroutine rw.dbMaintainer = NewDBMaintainer(rw.db, rw.logger.Logger) - rw.dbMaintainer.Start() + rw.dbMaintainer.Start(ctx) // Wait for context cancellation <-ctx.Done() @@ -127,13 +138,14 @@ func (rw *RouteWatch) Run(ctx context.Context) error { } // Shutdown performs graceful shutdown of all services -func (rw *RouteWatch) Shutdown() { +func (rw *RouteWatch) Shutdown(ctx context.Context) { rw.mu.Lock() if rw.shutdown { rw.mu.Unlock() return } + rw.shutdown = true rw.mu.Unlock() @@ -142,14 +154,17 @@ func (rw *RouteWatch) Shutdown() { rw.logger.Info("Flushing database handler") rw.dbHandler.Stop() } + if rw.peerHandler != nil { rw.logger.Info("Flushing peer handler") rw.peerHandler.Stop() } + if rw.prefixHandler != nil { rw.logger.Info("Flushing prefix handler") rw.prefixHandler.Stop() } + if rw.peeringHandler != nil { rw.logger.Info("Flushing peering handler") rw.peeringHandler.Stop() @@ -168,11 +183,15 @@ func (rw *RouteWatch) Shutdown() { // Stop services rw.streamer.Stop() - // Stop HTTP server with a timeout + // Stop HTTP server with a timeout of its own, however much of ctx is left const serverStopTimeout = 5 * time.Second - stopCtx, cancel := context.WithTimeout(context.Background(), serverStopTimeout) + + stopCtx, cancel := context.WithTimeout( + context.WithoutCancel(ctx), serverStopTimeout) defer cancel() - if err := rw.server.Stop(stopCtx); err != nil { + + err := rw.server.Stop(stopCtx) + if err != nil { rw.logger.Error("Failed to stop HTTP server gracefully", "error", err) } @@ -185,12 +204,11 @@ func (rw *RouteWatch) Shutdown() { "bits_per_sec", metrics.BitsPerSec, "duration", time.Since(metrics.ConnectedSince), ) - } // getModule provides all fx dependencies -func getModule() fx.Option { - return fx.Options( +func getModule() []fx.Option { + return []fx.Option{ fx.Provide( logger.New, config.New, @@ -203,5 +221,5 @@ func getModule() fx.Option { server.New, New, ), - ) + } } diff --git a/internal/routewatch/app_integration_test.go b/internal/routewatch/app_integration_internal_test.go similarity index 78% rename from internal/routewatch/app_integration_test.go rename to internal/routewatch/app_integration_internal_test.go index a23d39a..13bb7aa 100644 --- a/internal/routewatch/app_integration_test.go +++ b/internal/routewatch/app_integration_internal_test.go @@ -50,7 +50,9 @@ func newMockStore() *mockStore { } // GetOrCreateASN mock implementation -func (m *mockStore) GetOrCreateASN(number int, timestamp time.Time) (*database.ASN, error) { +func (m *mockStore) GetOrCreateASN( + number int, timestamp time.Time, +) (*database.ASN, error) { m.mu.Lock() defer m.mu.Unlock() @@ -99,11 +101,14 @@ func (m *mockStore) UpdatePrefixesBatch(prefixes map[string]time.Time) error { } } } + return nil } // GetOrCreatePrefix mock implementation -func (m *mockStore) GetOrCreatePrefix(prefix string, timestamp time.Time) (*database.Prefix, error) { +func (m *mockStore) GetOrCreatePrefix( + prefix string, timestamp time.Time, +) (*database.Prefix, error) { m.mu.Lock() defer m.mu.Unlock() @@ -168,7 +173,7 @@ func (m *mockStore) RecordPeering(asA, asB int, _ time.Time) error { } // UpdatePeer mock implementation -func (m *mockStore) UpdatePeer(peerIP string, peerASN int, messageType string, timestamp time.Time) error { +func (m *mockStore) UpdatePeer(_ string, _ int, _ string, _ time.Time) error { // Simple mock - just return nil return nil } @@ -194,41 +199,45 @@ func (m *mockStore) GetStats() (database.Stats, error) { } // GetStatsContext returns statistics about the mock store with context support -func (m *mockStore) GetStatsContext(ctx context.Context) (database.Stats, error) { +func (m *mockStore) GetStatsContext(_ context.Context) (database.Stats, error) { return m.GetStats() } // UpsertLiveRoute mock implementation -func (m *mockStore) UpsertLiveRoute(route *database.LiveRoute) error { +func (m *mockStore) UpsertLiveRoute(_ *database.LiveRoute) error { // Simple mock - just return nil return nil } // DeleteLiveRoute mock implementation -func (m *mockStore) DeleteLiveRoute(prefix, peerIP string) error { +func (m *mockStore) DeleteLiveRoute(_, _ string) error { // Simple mock - just return nil return nil } // GetPrefixDistribution mock implementation -func (m *mockStore) GetPrefixDistribution() (ipv4 []database.PrefixDistribution, ipv6 []database.PrefixDistribution, err error) { +func (m *mockStore) GetPrefixDistribution() ( + []database.PrefixDistribution, []database.PrefixDistribution, error, +) { // Return empty distributions for now return nil, nil, nil } // GetPrefixDistributionContext mock implementation with context support -func (m *mockStore) GetPrefixDistributionContext(ctx context.Context) (ipv4 []database.PrefixDistribution, ipv6 []database.PrefixDistribution, err error) { +func (m *mockStore) GetPrefixDistributionContext(_ context.Context) ( + []database.PrefixDistribution, []database.PrefixDistribution, error, +) { return m.GetPrefixDistribution() } // GetLiveRouteCounts mock implementation -func (m *mockStore) GetLiveRouteCounts() (ipv4Count, ipv6Count int, err error) { +func (m *mockStore) GetLiveRouteCounts() (int, int, error) { // Return mock counts return m.RouteCount / 2, m.RouteCount / 2, nil } // GetLiveRouteCountsContext mock implementation with context support -func (m *mockStore) GetLiveRouteCountsContext(ctx context.Context) (ipv4Count, ipv6Count int, err error) { +func (m *mockStore) GetLiveRouteCountsContext(_ context.Context) (int, int, error) { return m.GetLiveRouteCounts() } @@ -247,41 +256,51 @@ func (m *mockStore) GetASDetails(asn int) (*database.ASN, []database.LiveRoute, } // GetASDetailsContext mock implementation with context support -func (m *mockStore) GetASDetailsContext(ctx context.Context, asn int) (*database.ASN, []database.LiveRoute, error) { +func (m *mockStore) GetASDetailsContext( + _ context.Context, asn int, +) (*database.ASN, []database.LiveRoute, error) { return m.GetASDetails(asn) } // GetPrefixDetails mock implementation -func (m *mockStore) GetPrefixDetails(prefix string) ([]database.LiveRoute, map[int]database.ASN, error) { +func (m *mockStore) GetPrefixDetails( + _ string, +) ([]database.LiveRoute, map[int]database.ASN, error) { // Return empty routes for now return []database.LiveRoute{}, map[int]database.ASN{}, nil } // GetPrefixDetailsContext mock implementation with context support func (m *mockStore) GetPrefixDetailsContext( - ctx context.Context, prefix string, + _ context.Context, prefix string, ) ([]database.LiveRoute, map[int]database.ASN, error) { return m.GetPrefixDetails(prefix) } -func (m *mockStore) GetRandomPrefixesByLength(maskLength, ipVersion, limit int) ([]database.LiveRoute, error) { +func (m *mockStore) GetRandomPrefixesByLength( + _, _, _ int, +) ([]database.LiveRoute, error) { // Return empty routes for now return []database.LiveRoute{}, nil } // GetRandomPrefixesByLengthContext mock implementation with context support -func (m *mockStore) GetRandomPrefixesByLengthContext(ctx context.Context, maskLength, ipVersion, limit int) ([]database.LiveRoute, error) { +func (m *mockStore) GetRandomPrefixesByLengthContext( + _ context.Context, maskLength, ipVersion, limit int, +) ([]database.LiveRoute, error) { return m.GetRandomPrefixesByLength(maskLength, ipVersion, limit) } // GetASPeers mock implementation -func (m *mockStore) GetASPeers(asn int) ([]database.ASPeer, error) { +func (m *mockStore) GetASPeers(_ int) ([]database.ASPeer, error) { // Return empty peers for now return []database.ASPeer{}, nil } // GetASPeersContext mock implementation with context support -func (m *mockStore) GetASPeersContext(ctx context.Context, asn int) ([]database.ASPeer, error) { +func (m *mockStore) GetASPeersContext( + _ context.Context, asn int, +) ([]database.ASPeer, error) { return m.GetASPeers(asn) } @@ -291,8 +310,11 @@ func (m *mockStore) GetIPInfo(ip string) (*database.IPInfo, error) { } // GetIPInfoContext mock implementation with context support -func (m *mockStore) GetIPInfoContext(ctx context.Context, ip string) (*database.IPInfo, error) { +func (m *mockStore) GetIPInfoContext( + _ context.Context, ip string, +) (*database.IPInfo, error) { now := time.Now() + return &database.IPInfo{ IP: ip, Netblock: "8.8.8.0/24", @@ -309,17 +331,21 @@ func (m *mockStore) GetIPInfoContext(ctx context.Context, ip string) (*database. } // GetNextStaleASN mock implementation -func (m *mockStore) GetNextStaleASN(ctx context.Context, staleThreshold time.Duration) (int, error) { +func (m *mockStore) GetNextStaleASN(_ context.Context, _ time.Duration) (int, error) { return 0, database.ErrNoStaleASN } // UpdateASNWHOIS mock implementation -func (m *mockStore) UpdateASNWHOIS(ctx context.Context, update *database.ASNWHOISUpdate) error { +func (m *mockStore) UpdateASNWHOIS( + _ context.Context, _ *database.ASNWHOISUpdate, +) error { return nil } // GetWHOISStats mock implementation -func (m *mockStore) GetWHOISStats(ctx context.Context, staleThreshold time.Duration) (*database.WHOISStats, error) { +func (m *mockStore) GetWHOISStats( + _ context.Context, _ time.Duration, +) (*database.WHOISStats, error) { m.mu.Lock() defer m.mu.Unlock() @@ -347,25 +373,28 @@ func (m *mockStore) UpsertLiveRouteBatch(routes []*database.LiveRoute) error { LastSeen: route.LastUpdated, } m.PrefixCount++ + if route.IPVersion == 4 { m.IPv4Prefixes++ } else { m.IPv6Prefixes++ } } + m.RouteCount++ } + return nil } // DeleteLiveRouteBatch mock implementation -func (m *mockStore) DeleteLiveRouteBatch(deletions []database.LiveRouteDeletion) error { +func (m *mockStore) DeleteLiveRouteBatch(_ []database.LiveRouteDeletion) error { // Simple mock - just return nil return nil } // DeleteLiveRoutesOfPeer mock implementation -func (m *mockStore) DeleteLiveRoutesOfPeer(peerIP string) (int, error) { +func (m *mockStore) DeleteLiveRoutesOfPeer(_ string) (int, error) { // Simple mock - deletes nothing return 0, nil } @@ -385,43 +414,53 @@ func (m *mockStore) GetOrCreateASNBatch(asns map[int]time.Time) error { m.ASNCount++ } } + return nil } // UpdatePeerBatch mock implementation -func (m *mockStore) UpdatePeerBatch(peers map[string]database.PeerUpdate) error { +func (m *mockStore) UpdatePeerBatch(_ map[string]database.PeerUpdate) error { // Simple mock - just return nil return nil } // Vacuum mock implementation -func (m *mockStore) Vacuum(ctx context.Context) error { +func (m *mockStore) Vacuum(_ context.Context) error { return nil } // Analyze mock implementation -func (m *mockStore) Analyze(ctx context.Context) error { +func (m *mockStore) Analyze(_ context.Context) error { return nil } // Checkpoint mock implementation -func (m *mockStore) Checkpoint(ctx context.Context) error { +func (m *mockStore) Checkpoint(_ context.Context) error { return nil } // Ping mock implementation -func (m *mockStore) Ping(ctx context.Context) error { +func (m *mockStore) Ping(_ context.Context) error { return nil } func TestRouteWatchLiveFeed(t *testing.T) { + t.Parallel() + if testing.Short() { - t.Skip("skipping live RIPE RIS network feed test in short mode; run without -short to include it") + t.Skip("skipping live RIPE RIS network feed test in short mode; " + + "run without -short to include it") } // Create mock database mockDB := newMockStore() - defer mockDB.Close() + + defer func() { + err := mockDB.Close() + if err != nil { + t.Errorf("failed to close mock database: %v", err) + } + }() logger := logger.New() @@ -474,16 +513,19 @@ func TestRouteWatchLiveFeed(t *testing.T) { if stats.ASNs == 0 { t.Error("Expected to receive some ASNs from live feed") } + t.Logf("Received %d unique ASNs in 5 seconds", stats.ASNs) if stats.Prefixes == 0 { t.Error("Expected to receive some prefixes from live feed") } - t.Logf("Received %d unique prefixes (%d IPv4, %d IPv6) in 5 seconds", stats.Prefixes, stats.IPv4Prefixes, stats.IPv6Prefixes) + + t.Logf("Received %d unique prefixes (%d IPv4, %d IPv6) in 5 seconds", + stats.Prefixes, stats.IPv4Prefixes, stats.IPv6Prefixes) if stats.Peerings == 0 { t.Error("Expected to receive some peerings from live feed") } - t.Logf("Recorded %d AS peering relationships in 5 seconds", stats.Peerings) + t.Logf("Recorded %d AS peering relationships in 5 seconds", stats.Peerings) } diff --git a/internal/routewatch/app_test.go b/internal/routewatch/app_test.go index 6adfec8..9e52502 100644 --- a/internal/routewatch/app_test.go +++ b/internal/routewatch/app_test.go @@ -1,3 +1,3 @@ -package routewatch +package routewatch_test -// Tests for routewatch package are in app_integration_test.go +// Tests for routewatch package are in app_integration_internal_test.go diff --git a/internal/routewatch/ashandler.go b/internal/routewatch/ashandler.go index 4ff8920..27a003f 100644 --- a/internal/routewatch/ashandler.go +++ b/internal/routewatch/ashandler.go @@ -41,9 +41,11 @@ type ASHandler struct { // Batching. mu guards batch only: the flush goroutine takes the batch // under mu and writes it after releasing mu, so HandleMessage waits for // a database write only while batch is at asnBatchMaxSize. - mu sync.Mutex - batch []asnOp - batchTaken sync.Cond // signalled when the flush goroutine takes batch; its L is &mu + mu sync.Mutex + batch []asnOp + // batchTaken is signalled when the flush goroutine takes batch; its L is + // &mu. + batchTaken sync.Cond batchFullCh chan struct{} // wakes the flush goroutine when batch is full lastFlush time.Time // used only by the flush goroutine stopCh chan struct{} @@ -83,7 +85,7 @@ func NewASHandler(db database.Store, logger *logger.Logger) *ASHandler { // the AS path information needed to track autonomous systems. func (h *ASHandler) WantsMessage(messageType string) bool { // We only care about UPDATE messages for the database - return messageType == "UPDATE" + return messageType == ristypes.MessageTypeUpdate } // QueueCapacity returns the recommended message queue size for this handler. @@ -148,10 +150,21 @@ func (h *ASHandler) HandleMessage(msg *ristypes.RISMessage) { } } +// Stop gracefully shuts down the ASHandler by signaling the background flush +// goroutine to terminate and waiting for it to complete. Any pending ASN +// operations in the current batch are flushed to the database before Stop +// returns. This method should be called during application shutdown to ensure +// no data is lost. +func (h *ASHandler) Stop() { + close(h.stopCh) + h.wg.Wait() +} + // flushLoop runs in a goroutine, the only one that writes the batch: it // flushes when the batch is full, periodically, and once more on Stop. func (h *ASHandler) flushLoop() { defer h.wg.Done() + ticker := time.NewTicker(asnBatchTimeout) defer ticker.Stop() @@ -181,6 +194,7 @@ func (h *ASHandler) takeBatch() []asnOp { if len(h.batch) == 0 { return nil } + batch := h.batch h.batch = make([]asnOp, 0, asnBatchSize) h.batchTaken.Broadcast() @@ -205,19 +219,11 @@ func (h *ASHandler) flushBatch() { } // Process all ASNs in a single batch transaction - if err := h.db.GetOrCreateASNBatch(asnMap); err != nil { - h.logger.Error("Failed to process ASN batch", "error", err, "count", len(asnMap)) + err := h.db.GetOrCreateASNBatch(asnMap) + if err != nil { + h.logger.Error("Failed to process ASN batch", + "error", err, "count", len(asnMap)) } h.lastFlush = time.Now() } - -// Stop gracefully shuts down the ASHandler by signaling the background flush -// goroutine to terminate and waiting for it to complete. Any pending ASN -// operations in the current batch are flushed to the database before Stop -// returns. This method should be called during application shutdown to ensure -// no data is lost. -func (h *ASHandler) Stop() { - close(h.stopCh) - h.wg.Wait() -} diff --git a/internal/routewatch/asnfetcher.go b/internal/routewatch/asnfetcher.go index 7a0b246..7320276 100644 --- a/internal/routewatch/asnfetcher.go +++ b/internal/routewatch/asnfetcher.go @@ -1,8 +1,8 @@ -// Package routewatch contains the ASN WHOIS fetcher for background updates. package routewatch import ( "context" + "errors" "log/slog" "sync" "time" @@ -73,10 +73,13 @@ func NewASNFetcher(db database.Store, logger *slog.Logger) *ASNFetcher { } } -// Start begins the background ASN fetcher goroutine. -func (f *ASNFetcher) Start() { +// Start begins the background ASN fetcher goroutine. It runs until Stop: +// the end of ctx does not stop it. +func (f *ASNFetcher) Start(ctx context.Context) { f.wg.Add(1) - go f.run() + + go f.run(context.WithoutCancel(ctx)) + f.logger.Info("ASN fetcher started", "base_interval", baseInterval, "min_interval", minInterval, @@ -151,10 +154,7 @@ func (f *ASNFetcher) recordSuccess() { f.consecutiveFails = 0 // Decrease interval by half, but not below minimum - newInterval := f.currentInterval / backoffMultiplier - if newInterval < minInterval { - newInterval = minInterval - } + newInterval := max(f.currentInterval/backoffMultiplier, minInterval) if newInterval != f.currentInterval { f.logger.Debug("Decreased fetch interval", @@ -177,10 +177,7 @@ func (f *ASNFetcher) recordFailure() { f.consecutiveFails++ // Exponential backoff: multiply by 2, capped at max - newInterval := f.currentInterval * backoffMultiplier - if newInterval > maxInterval { - newInterval = maxInterval - } + newInterval := min(f.currentInterval*backoffMultiplier, maxInterval) if newInterval != f.currentInterval { f.logger.Debug("Increased fetch interval due to failure", @@ -199,7 +196,7 @@ func (f *ASNFetcher) recordFailure() { } // run is the main background loop. -func (f *ASNFetcher) run() { +func (f *ASNFetcher) run(ctx context.Context) { defer f.wg.Done() timer := time.NewTimer(f.getInterval()) @@ -212,13 +209,13 @@ func (f *ASNFetcher) run() { case asn := <-f.immediateQueue: // Process immediate request (respects lock) - f.tryFetch(asn) + f.tryFetch(ctx, asn) // Reset timer after immediate fetch timer.Reset(f.getInterval()) case <-timer.C: // Background fetch of stale/missing ASN - f.fetchNextStale() + f.fetchNextStale(ctx) // Reset timer with potentially updated interval timer.Reset(f.getInterval()) } @@ -227,7 +224,7 @@ func (f *ASNFetcher) run() { // tryFetch attempts to fetch and update an ASN, respecting the fetch lock. // Returns true if fetch was successful. -func (f *ASNFetcher) tryFetch(asn int) bool { +func (f *ASNFetcher) tryFetch(ctx context.Context, asn int) bool { // Try to acquire lock, skip if another fetch is running if !f.fetchMu.TryLock() { f.logger.Debug("Skipping fetch, another fetch in progress", "asn", asn) @@ -236,11 +233,11 @@ func (f *ASNFetcher) tryFetch(asn int) bool { } defer f.fetchMu.Unlock() - return f.fetchAndUpdate(asn) + return f.fetchAndUpdate(ctx, asn) } // fetchNextStale finds and fetches the next ASN needing WHOIS data. -func (f *ASNFetcher) fetchNextStale() { +func (f *ASNFetcher) fetchNextStale(ctx context.Context) { // Try to acquire lock, skip if another fetch is running if !f.fetchMu.TryLock() { f.logger.Debug("Skipping stale fetch, another fetch in progress") @@ -249,12 +246,12 @@ func (f *ASNFetcher) fetchNextStale() { } defer f.fetchMu.Unlock() - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + lookupCtx, cancel := context.WithTimeout(ctx, time.Minute) defer cancel() - asn, err := f.db.GetNextStaleASN(ctx, whoisStaleThreshold) + asn, err := f.db.GetNextStaleASN(lookupCtx, whoisStaleThreshold) if err != nil { - if err != database.ErrNoStaleASN { + if !errors.Is(err, database.ErrNoStaleASN) { f.logger.Error("Failed to get stale ASN", "error", err) f.recordFailure() } @@ -263,13 +260,13 @@ func (f *ASNFetcher) fetchNextStale() { return } - f.fetchAndUpdate(asn) + f.fetchAndUpdate(ctx, asn) } // fetchAndUpdate performs a WHOIS lookup and updates the database. // Returns true if successful. -func (f *ASNFetcher) fetchAndUpdate(asn int) bool { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) +func (f *ASNFetcher) fetchAndUpdate(ctx context.Context, asn int) bool { + ctx, cancel := context.WithTimeout(ctx, time.Minute) defer cancel() f.logger.Info("Fetching WHOIS data", "asn", asn) diff --git a/internal/routewatch/batchwrite_test.go b/internal/routewatch/batchwrite_internal_test.go similarity index 82% rename from internal/routewatch/batchwrite_test.go rename to internal/routewatch/batchwrite_internal_test.go index 4551eae..d6e6d1d 100644 --- a/internal/routewatch/batchwrite_test.go +++ b/internal/routewatch/batchwrite_internal_test.go @@ -49,21 +49,11 @@ func newBlockingStore() *blockingStore { } } -func (s *blockingStore) waitForRelease() { - s.startOnce.Do(func() { close(s.writeStarted) }) - <-s.release -} - -// releaseWrites lets the held writes, and all later ones, finish. A second call -// does nothing. -func (s *blockingStore) releaseWrites() { - s.releaseOnce.Do(func() { close(s.release) }) -} - func (s *blockingStore) UpsertLiveRouteBatch(routes []*database.LiveRoute) error { s.waitForRelease() s.mu.Lock() defer s.mu.Unlock() + for _, route := range routes { s.prefixes[route.Prefix] = true } @@ -75,6 +65,7 @@ func (s *blockingStore) GetOrCreateASNBatch(asns map[int]time.Time) error { s.waitForRelease() s.mu.Lock() defer s.mu.Unlock() + for asn := range asns { s.asns[asn] = true } @@ -86,6 +77,7 @@ func (s *blockingStore) UpdatePeerBatch(peers map[string]database.PeerUpdate) er s.waitForRelease() s.mu.Lock() defer s.mu.Unlock() + for peerIP := range peers { s.peers[peerIP] = true } @@ -93,9 +85,21 @@ func (s *blockingStore) UpdatePeerBatch(peers map[string]database.PeerUpdate) er return nil } +func (s *blockingStore) waitForRelease() { + s.startOnce.Do(func() { close(s.writeStarted) }) + <-s.release +} + +// releaseWrites lets the held writes, and all later ones, finish. A second call +// does nothing. +func (s *blockingStore) releaseWrites() { + s.releaseOnce.Do(func() { close(s.release) }) +} + // returnsInTime reports whether f returns within handleMessageTimeout. func returnsInTime(f func()) bool { done := make(chan struct{}) + go func() { f() close(done) @@ -115,7 +119,9 @@ func returnsInTime(f func()) bool { func startBlockedWrite( t *testing.T, store *blockingStore, - h interface{ HandleMessage(*ristypes.RISMessage) }, + h interface { + HandleMessage(msg *ristypes.RISMessage) + }, fill []*ristypes.RISMessage, ) { t.Helper() @@ -125,7 +131,8 @@ func startBlockedWrite( h.HandleMessage(msg) } }) { - t.Fatal("HandleMessage did not return after filling the batch: it waits for the database write") + t.Fatal("HandleMessage did not return after filling the batch: " + + "it waits for the database write") } select { @@ -143,13 +150,14 @@ func handleWhileAWriteIsBlocked( t *testing.T, store *blockingStore, h interface { - HandleMessage(*ristypes.RISMessage) + HandleMessage(msg *ristypes.RISMessage) Stop() }, fill []*ristypes.RISMessage, extra *ristypes.RISMessage, ) { t.Helper() + defer h.Stop() defer store.releaseWrites() @@ -169,22 +177,25 @@ func handleAtMaxBatchSize( t *testing.T, store *blockingStore, h interface { - HandleMessage(*ristypes.RISMessage) + HandleMessage(msg *ristypes.RISMessage) Stop() }, fill, toMax []*ristypes.RISMessage, extra *ristypes.RISMessage, ) { t.Helper() + defer h.Stop() defer store.releaseWrites() startBlockedWrite(t, store, h, fill) + for _, msg := range toMax { h.HandleMessage(msg) } returned := make(chan struct{}) + go func() { h.HandleMessage(extra) close(returned) @@ -192,11 +203,13 @@ func handleAtMaxBatchSize( select { case <-returned: - t.Fatal("HandleMessage returned while the batch was at its maximum size and a write was blocked") + t.Fatal("HandleMessage returned while the batch was at its maximum " + + "size and a write was blocked") case <-time.After(stillWaitingAfter): } store.releaseWrites() + select { case <-returned: case <-time.After(handleMessageTimeout): @@ -208,46 +221,57 @@ func handleAtMaxBatchSize( // with origin AS testASNA. func prefixAnnouncement(prefixes []string) *ristypes.RISMessage { return &ristypes.RISMessage{ - Peer: testPeerIP, - Path: ristypes.ASPath{testASNA}, - Announcements: []ristypes.RISAnnouncement{{NextHop: testPeerIP, Prefixes: prefixes}}, + Peer: testPeerIP, + Path: ristypes.ASPath{testASNA}, + Announcements: []ristypes.RISAnnouncement{ + {NextHop: testPeerIP, Prefixes: prefixes}, + }, } } func TestPrefixHandlerHandlesMessagesWhileAWriteIsBlocked(t *testing.T) { - const ( - fillPrefix = "192.0.2.0/24" - extraPrefix = "198.51.100.0/24" - ) + t.Parallel() + + const extraPrefix = "198.51.100.0/24" store := newBlockingStore() h := NewPrefixHandler(store, logger.New()) + fill := slices.Repeat([]string{testV4Prefix}, prefixBatchSize) handleWhileAWriteIsBlocked(t, store, h, - []*ristypes.RISMessage{prefixAnnouncement(slices.Repeat([]string{fillPrefix}, prefixBatchSize))}, + []*ristypes.RISMessage{prefixAnnouncement(fill)}, prefixAnnouncement([]string{extraPrefix})) store.mu.Lock() defer store.mu.Unlock() + if !store.prefixes[extraPrefix] { - t.Errorf("Stop did not write %s, handled while a write was blocked", extraPrefix) + t.Errorf("Stop did not write %s, handled while a write was blocked", + extraPrefix) } } func TestASHandlerHandlesMessagesWhileAWriteIsBlocked(t *testing.T) { + t.Parallel() + store := newBlockingStore() h := NewASHandler(store, logger.New()) + fill := slices.Repeat(ristypes.ASPath{testASNA}, asnBatchSize) handleWhileAWriteIsBlocked(t, store, h, - []*ristypes.RISMessage{{Path: slices.Repeat(ristypes.ASPath{testASNA}, asnBatchSize)}}, + []*ristypes.RISMessage{{Path: fill}}, &ristypes.RISMessage{Path: ristypes.ASPath{testASNB}}) store.mu.Lock() defer store.mu.Unlock() + if !store.asns[testASNB] { - t.Errorf("Stop did not write AS%d, handled while a write was blocked", testASNB) + t.Errorf("Stop did not write AS%d, handled while a write was blocked", + testASNB) } } func TestPeerHandlerHandlesMessagesWhileAWriteIsBlocked(t *testing.T) { + t.Parallel() + const otherPeerIP = "2001:db8:ffff::2" store := newBlockingStore() @@ -258,60 +282,76 @@ func TestPeerHandlerHandlesMessagesWhileAWriteIsBlocked(t *testing.T) { store.mu.Lock() defer store.mu.Unlock() + if !store.peers[otherPeerIP] { - t.Errorf("Stop did not write peer %s, handled while a write was blocked", otherPeerIP) + t.Errorf("Stop did not write peer %s, handled while a write was blocked", + otherPeerIP) } } func TestPrefixHandlerWaitsAtTheMaximumBatchSize(t *testing.T) { - const ( - fillPrefix = "192.0.2.0/24" - extraPrefix = "198.51.100.0/24" - ) + t.Parallel() + + const extraPrefix = "198.51.100.0/24" store := newBlockingStore() h := NewPrefixHandler(store, logger.New()) + fill := slices.Repeat([]string{testV4Prefix}, prefixBatchSize) + toMax := slices.Repeat([]string{testV4Prefix}, prefixBatchMaxSize) handleAtMaxBatchSize(t, store, h, - []*ristypes.RISMessage{prefixAnnouncement(slices.Repeat([]string{fillPrefix}, prefixBatchSize))}, - []*ristypes.RISMessage{prefixAnnouncement(slices.Repeat([]string{fillPrefix}, prefixBatchMaxSize))}, + []*ristypes.RISMessage{prefixAnnouncement(fill)}, + []*ristypes.RISMessage{prefixAnnouncement(toMax)}, prefixAnnouncement([]string{extraPrefix})) store.mu.Lock() defer store.mu.Unlock() + if !store.prefixes[extraPrefix] { - t.Errorf("Stop did not write %s, handled after the batch reached its maximum size", extraPrefix) + t.Errorf("Stop did not write %s, handled after the batch reached its "+ + "maximum size", extraPrefix) } } func TestASHandlerWaitsAtTheMaximumBatchSize(t *testing.T) { + t.Parallel() + store := newBlockingStore() h := NewASHandler(store, logger.New()) + fill := slices.Repeat(ristypes.ASPath{testASNA}, asnBatchSize) + toMax := slices.Repeat(ristypes.ASPath{testASNA}, asnBatchMaxSize) handleAtMaxBatchSize(t, store, h, - []*ristypes.RISMessage{{Path: slices.Repeat(ristypes.ASPath{testASNA}, asnBatchSize)}}, - []*ristypes.RISMessage{{Path: slices.Repeat(ristypes.ASPath{testASNA}, asnBatchMaxSize)}}, + []*ristypes.RISMessage{{Path: fill}}, + []*ristypes.RISMessage{{Path: toMax}}, &ristypes.RISMessage{Path: ristypes.ASPath{testASNB}}) store.mu.Lock() defer store.mu.Unlock() + if !store.asns[testASNB] { - t.Errorf("Stop did not write AS%d, handled after the batch reached its maximum size", testASNB) + t.Errorf("Stop did not write AS%d, handled after the batch reached its "+ + "maximum size", testASNB) } } func TestPeerHandlerWaitsAtTheMaximumBatchSize(t *testing.T) { + t.Parallel() + const otherPeerIP = "2001:db8:ffff::2" store := newBlockingStore() h := NewPeerHandler(store, logger.New()) + message := []*ristypes.RISMessage{{Peer: testPeerIP}} handleAtMaxBatchSize(t, store, h, - slices.Repeat([]*ristypes.RISMessage{{Peer: testPeerIP}}, peerBatchSize), - slices.Repeat([]*ristypes.RISMessage{{Peer: testPeerIP}}, peerBatchMaxSize), + slices.Repeat(message, peerBatchSize), + slices.Repeat(message, peerBatchMaxSize), &ristypes.RISMessage{Peer: otherPeerIP}) store.mu.Lock() defer store.mu.Unlock() + if !store.peers[otherPeerIP] { - t.Errorf("Stop did not write peer %s, handled after the batch reached its maximum size", otherPeerIP) + t.Errorf("Stop did not write peer %s, handled after the batch reached "+ + "its maximum size", otherPeerIP) } } @@ -320,6 +360,8 @@ func TestPeerHandlerWaitsAtTheMaximumBatchSize(t *testing.T) { // of a full batch, and a second one exactly a full batch. Only the full batch // leaves a wake-up in batchFullCh. func TestHandlersWakeTheFlushGoroutineWhenTheBatchIsFull(t *testing.T) { + t.Parallel() + tests := []struct { name string batchSize int @@ -331,7 +373,8 @@ func TestHandlersWakeTheFlushGoroutineWhenTheBatchIsFull(t *testing.T) { batchSize: prefixBatchSize, handle: func(n int) chan struct{} { h := &PrefixHandler{batchFullCh: make(chan struct{}, 1)} - h.HandleMessage(prefixAnnouncement(slices.Repeat([]string{"192.0.2.0/24"}, n))) + h.HandleMessage(prefixAnnouncement( + slices.Repeat([]string{testV4Prefix}, n))) return h.batchFullCh }, @@ -342,7 +385,9 @@ func TestHandlersWakeTheFlushGoroutineWhenTheBatchIsFull(t *testing.T) { handle: func(n int) chan struct{} { h := &ASHandler{batchFullCh: make(chan struct{}, 1)} // The origin AS and each AS in the path are one update each. - h.HandleMessage(&ristypes.RISMessage{Path: slices.Repeat(ristypes.ASPath{testASNA}, n-1)}) + h.HandleMessage(&ristypes.RISMessage{ + Path: slices.Repeat(ristypes.ASPath{testASNA}, n-1), + }) return h.batchFullCh }, @@ -363,10 +408,16 @@ func TestHandlersWakeTheFlushGoroutineWhenTheBatchIsFull(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := len(tt.handle(tt.batchSize - 1)); got != 0 { - t.Errorf("one update short of a full batch left %d wake-ups, want none", got) + t.Parallel() + + got := len(tt.handle(tt.batchSize - 1)) + if got != 0 { + t.Errorf("one update short of a full batch left %d wake-ups, "+ + "want none", got) } - if got := len(tt.handle(tt.batchSize)); got != 1 { + + got = len(tt.handle(tt.batchSize)) + if got != 1 { t.Errorf("a full batch left %d wake-ups, want 1", got) } }) diff --git a/internal/routewatch/cli.go b/internal/routewatch/cli.go index 6c2bd6c..ea7d475 100644 --- a/internal/routewatch/cli.go +++ b/internal/routewatch/cli.go @@ -57,14 +57,19 @@ func logDebugStats(logger *logger.Logger) { // CLIEntry is the main entry point for the routewatch command-line interface. // It initializes the application using the fx dependency injection framework, -// sets up signal handling for graceful shutdown, and starts the RouteWatch service. -// This function blocks until the application receives a shutdown signal or encounters -// a fatal error. +// sets up signal handling for graceful shutdown, and starts the RouteWatch +// service. This function blocks until the application receives a shutdown +// signal or encounters a fatal error. func CLIEntry() { app := fx.New( - getModule(), + fx.Options(getModule()...), fx.StopTimeout(shutdownTimeout), // Allow 60 seconds for graceful shutdown - fx.Invoke(func(lc fx.Lifecycle, rw *RouteWatch, logger *logger.Logger, shutdowner fx.Shutdowner) { + fx.Invoke(func( + lc fx.Lifecycle, + rw *RouteWatch, + logger *logger.Logger, + shutdowner fx.Shutdowner, + ) { lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { // Start debug stats logging @@ -77,25 +82,31 @@ func CLIEntry() { go func() { <-sigCh logger.Info("Received shutdown signal") - if err := shutdowner.Shutdown(); err != nil { + + err := shutdowner.Shutdown() + if err != nil { logger.Error("Failed to shutdown gracefully", "error", err) } }() go func() { - if err := rw.Run(ctx); err != nil { + err := rw.Run(ctx) + if err != nil { logger.Error("RouteWatch error", "error", err) - if err := shutdowner.Shutdown(fx.ExitCode(failureExitCode)); err != nil { - logger.Error("Failed to shutdown gracefully", "error", err) + + err = shutdowner.Shutdown(fx.ExitCode(failureExitCode)) + if err != nil { + logger.Error("Failed to shutdown gracefully", + "error", err) } } }() return nil }, - OnStop: func(_ context.Context) error { + OnStop: func(ctx context.Context) error { logger.Info("Shutting down RouteWatch") - rw.Shutdown() + rw.Shutdown(ctx) return nil }, diff --git a/internal/routewatch/dbmaintainer.go b/internal/routewatch/dbmaintainer.go index 993d616..8859a46 100644 --- a/internal/routewatch/dbmaintainer.go +++ b/internal/routewatch/dbmaintainer.go @@ -1,4 +1,3 @@ -// Package routewatch contains the database maintainer for background maintenance tasks. package routewatch import ( @@ -64,10 +63,13 @@ func NewDBMaintainer(db database.Store, logger *slog.Logger) *DBMaintainer { } } -// Start begins the background maintenance goroutine. -func (m *DBMaintainer) Start() { +// Start begins the background maintenance goroutine. It runs until Stop: +// the end of ctx does not stop it. +func (m *DBMaintainer) Start(ctx context.Context) { m.wg.Add(1) - go m.run() + + go m.run(context.WithoutCancel(ctx)) + m.logger.Info("Database maintainer started", "checkpoint_interval", checkpointInterval, "vacuum_interval", vacuumInterval, @@ -83,15 +85,17 @@ func (m *DBMaintainer) Stop() { } // run is the main background loop. -func (m *DBMaintainer) run() { +func (m *DBMaintainer) run(ctx context.Context) { defer m.wg.Done() // Use different timers for each task checkpointTimer := time.NewTimer(checkpointInterval) - vacuumTimer := time.NewTimer(vacuumInterval) - analyzeTimer := time.NewTimer(analyzeInterval) defer checkpointTimer.Stop() + + vacuumTimer := time.NewTimer(vacuumInterval) defer vacuumTimer.Stop() + + analyzeTimer := time.NewTimer(analyzeInterval) defer analyzeTimer.Stop() for { @@ -100,90 +104,88 @@ func (m *DBMaintainer) run() { return case <-checkpointTimer.C: - m.runCheckpoint() + m.runCheckpoint(ctx) checkpointTimer.Reset(checkpointInterval) case <-vacuumTimer.C: - m.runVacuum() + m.runVacuum(ctx) vacuumTimer.Reset(vacuumInterval) case <-analyzeTimer.C: - m.runAnalyze() + m.runAnalyze(ctx) analyzeTimer.Reset(analyzeInterval) } } } // runCheckpoint performs a WAL checkpoint to keep the WAL file small. -func (m *DBMaintainer) runCheckpoint() { - ctx, cancel := context.WithTimeout(context.Background(), checkpointTimeout) - defer cancel() - - startTime := time.Now() - - err := m.db.Checkpoint(ctx) - - m.statsMu.Lock() - m.lastCheckpoint = time.Now() - m.lastCheckpointError = err - if err == nil { - m.checkpointCount++ - } - m.statsMu.Unlock() +func (m *DBMaintainer) runCheckpoint(ctx context.Context) { + duration, err := runWithTimeout(ctx, checkpointTimeout, m.db.Checkpoint) + m.recordResult(&m.lastCheckpoint, &m.lastCheckpointError, &m.checkpointCount, err) if err != nil { - m.logger.Error("WAL checkpoint failed", "error", err, "duration", time.Since(startTime)) + m.logger.Error("WAL checkpoint failed", "error", err, "duration", duration) } else { - m.logger.Debug("WAL checkpoint completed", "duration", time.Since(startTime)) + m.logger.Debug("WAL checkpoint completed", "duration", duration) } } // runVacuum performs an incremental vacuum operation on the database. -func (m *DBMaintainer) runVacuum() { - ctx, cancel := context.WithTimeout(context.Background(), vacuumTimeout) - defer cancel() - +func (m *DBMaintainer) runVacuum(ctx context.Context) { m.logger.Debug("Running incremental vacuum") - startTime := time.Now() - err := m.db.Vacuum(ctx) - - m.statsMu.Lock() - m.lastVacuum = time.Now() - m.lastVacuumError = err - if err == nil { - m.vacuumCount++ - } - m.statsMu.Unlock() + duration, err := runWithTimeout(ctx, vacuumTimeout, m.db.Vacuum) + m.recordResult(&m.lastVacuum, &m.lastVacuumError, &m.vacuumCount, err) if err != nil { - m.logger.Error("Incremental vacuum failed", "error", err, "duration", time.Since(startTime)) + m.logger.Error("Incremental vacuum failed", "error", err, "duration", duration) } else { - m.logger.Debug("Incremental vacuum completed", "duration", time.Since(startTime)) + m.logger.Debug("Incremental vacuum completed", "duration", duration) } } // runAnalyze performs an ANALYZE operation on the database. -func (m *DBMaintainer) runAnalyze() { - ctx, cancel := context.WithTimeout(context.Background(), analyzeTimeout) - defer cancel() - +func (m *DBMaintainer) runAnalyze(ctx context.Context) { m.logger.Info("Starting database ANALYZE") - startTime := time.Now() - err := m.db.Analyze(ctx) - - m.statsMu.Lock() - m.lastAnalyze = time.Now() - m.lastAnalyzeError = err - if err == nil { - m.analyzeCount++ - } - m.statsMu.Unlock() + duration, err := runWithTimeout(ctx, analyzeTimeout, m.db.Analyze) + m.recordResult(&m.lastAnalyze, &m.lastAnalyzeError, &m.analyzeCount, err) if err != nil { - m.logger.Error("ANALYZE failed", "error", err, "duration", time.Since(startTime)) + m.logger.Error("ANALYZE failed", "error", err, "duration", duration) } else { - m.logger.Info("ANALYZE completed", "duration", time.Since(startTime)) + m.logger.Info("ANALYZE completed", "duration", duration) + } +} + +// runWithTimeout runs operation with a context that ends after timeout and +// returns how long it took and its error. +func runWithTimeout( + ctx context.Context, + timeout time.Duration, + operation func(context.Context) error, +) (time.Duration, error) { + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + startTime := time.Now() + err := operation(ctx) + + return time.Since(startTime), err +} + +// recordResult stores when a maintenance task last ran and its error, and +// counts it when it succeeded. +func (m *DBMaintainer) recordResult( + lastRun *time.Time, lastErr *error, successCount *int, err error, +) { + m.statsMu.Lock() + defer m.statsMu.Unlock() + + *lastRun = time.Now() + *lastErr = err + + if err == nil { + *successCount++ } } diff --git a/internal/routewatch/handler.go b/internal/routewatch/handler.go index 328b7ff..b4a7441 100644 --- a/internal/routewatch/handler.go +++ b/internal/routewatch/handler.go @@ -1,6 +1,8 @@ package routewatch import ( + "slices" + "git.eeqj.de/sneak/routewatch/internal/logger" "git.eeqj.de/sneak/routewatch/internal/ristypes" ) @@ -15,10 +17,11 @@ type SimpleHandler struct { callback func(*ristypes.RISMessage) } -// NewSimpleHandler creates a new SimpleHandler that accepts specific message types. -// The messageTypes parameter specifies which RIS message types this handler will process. -// If messageTypes is empty, the handler will accept all message types. -// The callback function is invoked for each message that passes the type filter. +// NewSimpleHandler creates a new SimpleHandler that accepts specific message +// types. The messageTypes parameter specifies which RIS message types this +// handler will process. If messageTypes is empty, the handler will accept all +// message types. The callback function is invoked for each message that +// passes the type filter. func NewSimpleHandler( logger *logger.Logger, messageTypes []string, @@ -31,22 +34,17 @@ func NewSimpleHandler( } } -// WantsMessage returns true if this handler wants to process messages of the given type. -// It checks whether messageType is in the handler's configured list of accepted types. -// If no specific types were configured (empty messageTypes slice), it returns true for all types. +// WantsMessage returns true if this handler wants to process messages of the +// given type. It checks whether messageType is in the handler's configured +// list of accepted types. If no specific types were configured (empty +// messageTypes slice), it returns true for all types. func (h *SimpleHandler) WantsMessage(messageType string) bool { // If no specific types are set, accept all messages if len(h.messageTypes) == 0 { return true } - for _, t := range h.messageTypes { - if t == messageType { - return true - } - } - - return false + return slices.Contains(h.messageTypes, messageType) } // HandleMessage processes a RIS message by invoking the configured callback function. diff --git a/internal/routewatch/peerhandler.go b/internal/routewatch/peerhandler.go index 61510f0..dcb1bc4 100644 --- a/internal/routewatch/peerhandler.go +++ b/internal/routewatch/peerhandler.go @@ -1,7 +1,8 @@ package routewatch -// peerhandler.go provides batched peer tracking functionality for BGP route monitoring. -// It tracks BGP peers from incoming UPDATE messages and maintains peer state in the database. +// peerhandler.go provides batched peer tracking functionality for BGP route +// monitoring. It tracks BGP peers from incoming UPDATE messages and maintains +// peer state in the database. import ( "strconv" @@ -44,9 +45,11 @@ type PeerHandler struct { // Batching. mu guards peerBatch only: the flush goroutine takes the batch // under mu and writes it after releasing mu, so HandleMessage waits for // a database write only while peerBatch is at peerBatchMaxSize. - mu sync.Mutex - peerBatch []peerUpdate - batchTaken sync.Cond // signalled when the flush goroutine takes peerBatch; its L is &mu + mu sync.Mutex + peerBatch []peerUpdate + // batchTaken is signalled when the flush goroutine takes peerBatch; its L + // is &mu. + batchTaken sync.Cond batchFullCh chan struct{} // wakes the flush goroutine when peerBatch is full lastFlush time.Time // used only by the flush goroutine stopCh chan struct{} @@ -104,8 +107,10 @@ func (h *PeerHandler) QueueCapacity() int { func (h *PeerHandler) HandleMessage(msg *ristypes.RISMessage) { // Parse peer ASN from string peerASN := 0 + if msg.PeerASN != "" { - if asn, err := strconv.Atoi(msg.PeerASN); err == nil { + asn, err := strconv.Atoi(msg.PeerASN) + if err == nil { peerASN = asn } } @@ -139,10 +144,17 @@ func (h *PeerHandler) HandleMessage(msg *ristypes.RISMessage) { } } +// Stop gracefully stops the handler and flushes remaining batches +func (h *PeerHandler) Stop() { + close(h.stopCh) + h.wg.Wait() +} + // flushLoop runs in a goroutine, the only one that writes the batch: it // flushes when the batch is full, periodically, and once more on Stop. func (h *PeerHandler) flushLoop() { defer h.wg.Done() + ticker := time.NewTicker(peerBatchTimeout) defer ticker.Stop() @@ -172,6 +184,7 @@ func (h *PeerHandler) takeBatch() []peerUpdate { if len(h.peerBatch) == 0 { return nil } + batch := h.peerBatch h.peerBatch = make([]peerUpdate, 0, peerBatchSize) h.batchTaken.Broadcast() @@ -190,7 +203,8 @@ func (h *PeerHandler) flushBatch() { // Deduplicate by peer IP, keeping the latest update for each peer peerMap := make(map[string]peerUpdate) for _, update := range batch { - if existing, ok := peerMap[update.peerIP]; !ok || update.timestamp.After(existing.timestamp) { + existing, ok := peerMap[update.peerIP] + if !ok || update.timestamp.After(existing.timestamp) { peerMap[update.peerIP] = update } } @@ -207,15 +221,11 @@ func (h *PeerHandler) flushBatch() { } // Process all peers in a single batch transaction - if err := h.db.UpdatePeerBatch(dbPeerMap); err != nil { - h.logger.Error("Failed to process peer batch", "error", err, "count", len(dbPeerMap)) + err := h.db.UpdatePeerBatch(dbPeerMap) + if err != nil { + h.logger.Error("Failed to process peer batch", + "error", err, "count", len(dbPeerMap)) } h.lastFlush = time.Now() } - -// Stop gracefully stops the handler and flushes remaining batches -func (h *PeerHandler) Stop() { - close(h.stopCh) - h.wg.Wait() -} diff --git a/internal/routewatch/peeringhandler.go b/internal/routewatch/peeringhandler.go index b4f3277..25df1d7 100644 --- a/internal/routewatch/peeringhandler.go +++ b/internal/routewatch/peeringhandler.go @@ -72,7 +72,7 @@ func NewPeeringHandler(db database.Store, logger *logger.Logger) *PeeringHandler // the AS path information needed to extract peering relationships. func (h *PeeringHandler) WantsMessage(messageType string) bool { // We only care about UPDATE messages that have AS paths - return messageType == "UPDATE" + return messageType == ristypes.MessageTypeUpdate } // QueueCapacity returns the buffer size for the handler's message queue. @@ -101,6 +101,7 @@ func (h *PeeringHandler) HandleMessage(msg *ristypes.RISMessage) { return } + key := string(pathJSON) h.mu.Lock() @@ -116,6 +117,23 @@ func (h *PeeringHandler) HandleMessage(msg *ristypes.RISMessage) { h.mu.Unlock() } +// ProcessPeeringsNow triggers immediate processing of all accumulated AS +// paths into peering records. This bypasses the normal periodic processing +// schedule and is primarily intended for testing purposes. +func (h *PeeringHandler) ProcessPeeringsNow() { + h.processPeerings() +} + +// Stop gracefully shuts down the handler by signaling the background +// goroutine to stop and performing a final synchronous processing of +// any remaining AS paths. This ensures no peering data is lost during +// shutdown. +func (h *PeeringHandler) Stop() { + close(h.stopCh) + // Process any remaining peerings synchronously + h.processPeerings() +} + // processLoop runs periodically to process AS paths into peerings func (h *PeeringHandler) processLoop() { ticker := time.NewTicker(peeringProcessInterval) @@ -134,13 +152,6 @@ func (h *PeeringHandler) processLoop() { } } -// ProcessPeeringsNow triggers immediate processing of all accumulated AS -// paths into peering records. This bypasses the normal periodic processing -// schedule and is primarily intended for testing purposes. -func (h *PeeringHandler) ProcessPeeringsNow() { - h.processPeerings() -} - // processPeerings extracts peerings from AS paths and writes to database func (h *PeeringHandler) processPeerings() { // Take the accumulated paths and replace the map with a fresh empty one @@ -160,11 +171,14 @@ func (h *PeeringHandler) processPeerings() { type peeringKey struct { low, high int } + peerings := make(map[peeringKey]time.Time) for pathJSON, timestamp := range paths { var path []int - if err := json.Unmarshal([]byte(pathJSON), &path); err != nil { + + err := json.Unmarshal([]byte(pathJSON), &path) + if err != nil { h.logger.Error("Failed to decode AS path", "error", err) continue @@ -197,6 +211,7 @@ func (h *PeeringHandler) processPeerings() { // Record peerings in database start := time.Now() successCount := 0 + for key, ts := range peerings { err := h.db.RecordPeering(key.low, key.high, ts) if err != nil { @@ -218,13 +233,3 @@ func (h *PeeringHandler) processPeerings() { "duration", time.Since(start), ) } - -// Stop gracefully shuts down the handler by signaling the background -// goroutine to stop and performing a final synchronous processing of -// any remaining AS paths. This ensures no peering data is lost during -// shutdown. -func (h *PeeringHandler) Stop() { - close(h.stopCh) - // Process any remaining peerings synchronously - h.processPeerings() -} diff --git a/internal/routewatch/peeringhandler_test.go b/internal/routewatch/peeringhandler_internal_test.go similarity index 95% rename from internal/routewatch/peeringhandler_test.go rename to internal/routewatch/peeringhandler_internal_test.go index 364fa46..de4f5dd 100644 --- a/internal/routewatch/peeringhandler_test.go +++ b/internal/routewatch/peeringhandler_internal_test.go @@ -53,7 +53,8 @@ func newTestHandler(db database.Store) *PeeringHandler { } } -func snapshot(h *PeeringHandler) (tracked, dropped int) { +// snapshot returns how many paths the handler tracks and how many it dropped. +func snapshot(h *PeeringHandler) (int, int) { h.mu.Lock() defer h.mu.Unlock() @@ -82,6 +83,8 @@ func handle(h *PeeringHandler, ts time.Time, asns ...int) { // empties the path map (the swap) and that a path seen again after a run is // recorded in the next run too. func TestPeeringHandlerProcessesEachRunAndEmpties(t *testing.T) { + t.Parallel() + store := &recordingStore{mockStore: newMockStore()} h := newTestHandler(store) @@ -94,6 +97,7 @@ func TestPeeringHandlerProcessesEachRunAndEmpties(t *testing.T) { if tracked, _ := snapshot(h); tracked != 0 { t.Fatalf("map not empty after first run: %d paths remain", tracked) } + if got := store.callCount(); got != 1 { t.Fatalf("want 1 RecordPeering call after first run, got %d", got) } @@ -105,6 +109,7 @@ func TestPeeringHandlerProcessesEachRunAndEmpties(t *testing.T) { if tracked, _ := snapshot(h); tracked != 0 { t.Fatalf("map not empty after second run: %d paths remain", tracked) } + if got := store.callCount(); got != 2 { t.Fatalf("want 2 RecordPeering calls after second run, got %d", got) } @@ -114,6 +119,8 @@ func TestPeeringHandlerProcessesEachRunAndEmpties(t *testing.T) { // and counts them, while a path already tracked is refreshed rather than // dropped. func TestPeeringHandlerCapDropsAndCounts(t *testing.T) { + t.Parallel() + store := &recordingStore{mockStore: newMockStore()} h := newTestHandler(store) @@ -126,6 +133,7 @@ func TestPeeringHandlerCapDropsAndCounts(t *testing.T) { h.mu.Lock() h.asPaths[existing] = now + for i := 0; len(h.asPaths) < maxTrackedPaths; i++ { h.asPaths[strconv.Itoa(i)] = now } @@ -138,6 +146,7 @@ func TestPeeringHandlerCapDropsAndCounts(t *testing.T) { if tracked != maxTrackedPaths { t.Fatalf("want map size %d after drop, got %d", maxTrackedPaths, tracked) } + if dropped != 1 { t.Fatalf("want dropped count 1, got %d", dropped) } @@ -150,6 +159,7 @@ func TestPeeringHandlerCapDropsAndCounts(t *testing.T) { if tracked != maxTrackedPaths { t.Fatalf("want map size %d after refresh, got %d", maxTrackedPaths, tracked) } + if dropped != 1 { t.Fatalf("want dropped count still 1 after refresh, got %d", dropped) } @@ -157,7 +167,9 @@ func TestPeeringHandlerCapDropsAndCounts(t *testing.T) { h.mu.Lock() gotTS := h.asPaths[existing] h.mu.Unlock() + if !gotTS.Equal(refreshed) { - t.Fatalf("existing path timestamp not refreshed: want %v, got %v", refreshed, gotTS) + t.Fatalf("existing path timestamp not refreshed: want %v, got %v", + refreshed, gotTS) } } diff --git a/internal/routewatch/prefixhandler.go b/internal/routewatch/prefixhandler.go index 4aec6ca..a79238c 100644 --- a/internal/routewatch/prefixhandler.go +++ b/internal/routewatch/prefixhandler.go @@ -41,9 +41,9 @@ const ( ipv6Version = 6 ) -// PrefixHandler tracks BGP prefixes and maintains a live routing table in the database. -// Routes are added on announcement and deleted on withdrawal, and all routes of -// a peer are deleted when its BGP session goes down. +// PrefixHandler tracks BGP prefixes and maintains a live routing table in the +// database. Routes are added on announcement and deleted on withdrawal, and all +// routes of a peer are deleted when its BGP session goes down. type PrefixHandler struct { db database.Store logger *logger.Logger @@ -52,10 +52,14 @@ type PrefixHandler struct { // Batching. mu guards batch and peersDown only: the flush goroutine takes // both under mu and writes them after releasing mu, so HandleMessage // waits for a database write only while batch is at prefixBatchMaxSize. - mu sync.Mutex - batch []prefixUpdate - peersDown []string // peers whose BGP session went down since the batch was taken - batchTaken sync.Cond // signalled when the flush goroutine takes batch; its L is &mu + mu sync.Mutex + batch []prefixUpdate + // peersDown holds the peers whose BGP session went down since the batch + // was taken. + peersDown []string + // batchTaken is signalled when the flush goroutine takes batch; its L is + // &mu. + batchTaken sync.Cond batchFullCh chan struct{} // wakes the flush goroutine when batch is full lastFlush time.Time // used only by the flush goroutine stopCh chan struct{} @@ -95,11 +99,13 @@ func (h *PrefixHandler) SetMetricsTracker(metrics *metrics.Tracker) { h.metrics = metrics } -// WantsMessage returns true if this handler wants to process messages of the given type +// WantsMessage returns true if this handler wants to process messages of the +// given type func (h *PrefixHandler) WantsMessage(messageType string) bool { // UPDATE messages change routes; a STATE message can say a peer's // session went down, which removes all of its routes. - return messageType == ristypes.MessageTypeUpdate || messageType == ristypes.MessageTypeState + return messageType == ristypes.MessageTypeUpdate || + messageType == ristypes.MessageTypeState } // QueueCapacity returns the desired queue capacity for this handler @@ -183,6 +189,12 @@ func (h *PrefixHandler) HandleMessage(msg *ristypes.RISMessage) { } } +// Stop gracefully stops the handler and flushes remaining batches +func (h *PrefixHandler) Stop() { + close(h.stopCh) + h.wg.Wait() +} + // peerSessionDown notes that the peer's BGP session went down, so that the next // flush deletes all of the peer's live routes before it writes the batch. The // peer's updates still in the batch came before the down, which replaces them, @@ -192,7 +204,9 @@ func (h *PrefixHandler) peerSessionDown(peer string) { h.mu.Lock() defer h.mu.Unlock() - h.batch = slices.DeleteFunc(h.batch, func(update prefixUpdate) bool { return update.peer == peer }) + h.batch = slices.DeleteFunc(h.batch, func(update prefixUpdate) bool { + return update.peer == peer + }) h.peersDown = append(h.peersDown, peer) } @@ -200,6 +214,7 @@ func (h *PrefixHandler) peerSessionDown(peer string) { // flushes when the batch is full, periodically, and once more on Stop. func (h *PrefixHandler) flushLoop() { defer h.wg.Done() + ticker := time.NewTicker(prefixBatchTimeout) defer ticker.Stop() @@ -230,6 +245,7 @@ func (h *PrefixHandler) takeBatch() ([]prefixUpdate, []string) { if len(h.batch) == 0 && len(h.peersDown) == 0 { return nil, nil } + batch, peersDown := h.batch, h.peersDown h.batch = make([]prefixUpdate, 0, prefixBatchSize) h.peersDown = nil @@ -250,43 +266,34 @@ func (h *PrefixHandler) flushBatch() { batchSize := len(batch) // The routes of a peer whose session went down are deleted first: the - // batch holds only the updates the peer sent after the down. Most downs - // come from peers that have no routes, so only a delete that removed - // routes is logged. - for _, peer := range peersDown { - deleted, err := h.db.DeleteLiveRoutesOfPeer(peer) - if err != nil { - h.logger.Error("Failed to delete the live routes of a peer whose session went down", - "peer", peer, "error", err) - - continue - } - if deleted > 0 { - h.logger.Info("Deleted the live routes of a peer whose session went down", - "peer", peer, "routes", deleted) - } - } + // batch holds only the updates the peer sent after the down. + h.deleteLiveRoutesOfPeers(peersDown) // For each prefix and peer, keep only the last update the feed delivered. // Updates from different peers for the same prefix are all kept. Arrival // order decides, not the timestamp: timestamps are whole seconds, so // updates within one second carry the same one. type prefixAndPeer struct{ prefix, peer string } + lastUpdates := make(map[prefixAndPeer]prefixUpdate) + for _, update := range batch { lastUpdates[prefixAndPeer{update.prefix, update.peer}] = update } // Collect routes to upsert and delete - var routesToUpsert []*database.LiveRoute - var routesToDelete []database.LiveRouteDeletion + var ( + routesToUpsert []*database.LiveRoute + routesToDelete []database.LiveRouteDeletion + ) // Collect unique prefixes to update prefixesToUpdate := make(map[string]time.Time) for _, update := range lastUpdates { // Track prefix for both announcements and withdrawals - if _, exists := prefixesToUpdate[update.prefix]; !exists || update.timestamp.After(prefixesToUpdate[update.prefix]) { + seen, exists := prefixesToUpdate[update.prefix] + if !exists || update.timestamp.After(seen) { prefixesToUpdate[update.prefix] = update.timestamp } @@ -300,7 +307,8 @@ func (h *PrefixHandler) flushBatch() { // Parse CIDR to get IP version _, ipVersion, err := parseCIDR(update.prefix) if err != nil { - h.logger.Error("Failed to parse CIDR for withdrawal", "prefix", update.prefix, "error", err) + h.logger.Error("Failed to parse CIDR for withdrawal", + "prefix", update.prefix, "error", err) continue } @@ -314,30 +322,7 @@ func (h *PrefixHandler) flushBatch() { } } - // Process batch operations - successCount := 0 - if len(routesToUpsert) > 0 { - if err := h.db.UpsertLiveRouteBatch(routesToUpsert); err != nil { - h.logger.Error("Failed to upsert route batch", "error", err, "count", len(routesToUpsert)) - } else { - successCount += len(routesToUpsert) - } - } - - if len(routesToDelete) > 0 { - if err := h.db.DeleteLiveRouteBatch(routesToDelete); err != nil { - h.logger.Error("Failed to delete route batch", "error", err, "count", len(routesToDelete)) - } else { - successCount += len(routesToDelete) - } - } - - // Update prefix tables - if len(prefixesToUpdate) > 0 { - if err := h.db.UpdatePrefixesBatch(prefixesToUpdate); err != nil { - h.logger.Error("Failed to update prefix batch", "error", err, "count", len(prefixesToUpdate)) - } - } + successCount := h.writeBatch(routesToUpsert, routesToDelete, prefixesToUpdate) elapsed := time.Since(startTime) h.logger.Debug("Flushed prefix batch", @@ -351,6 +336,69 @@ func (h *PrefixHandler) flushBatch() { h.lastFlush = time.Now() } +// deleteLiveRoutesOfPeers deletes all live routes of each of peers, the peers +// whose BGP session went down. Most downs come from peers that have no routes, +// so only a delete that removed routes is logged. +func (h *PrefixHandler) deleteLiveRoutesOfPeers(peers []string) { + for _, peer := range peers { + deleted, err := h.db.DeleteLiveRoutesOfPeer(peer) + if err != nil { + h.logger.Error( + "Failed to delete the live routes of a peer whose session went down", + "peer", peer, "error", err) + + continue + } + + if deleted > 0 { + h.logger.Info( + "Deleted the live routes of a peer whose session went down", + "peer", peer, "routes", deleted) + } + } +} + +// writeBatch writes the collected route upserts, route deletions and prefix +// updates to the database and returns how many routes it wrote. +func (h *PrefixHandler) writeBatch( + routesToUpsert []*database.LiveRoute, + routesToDelete []database.LiveRouteDeletion, + prefixesToUpdate map[string]time.Time, +) int { + successCount := 0 + + if len(routesToUpsert) > 0 { + err := h.db.UpsertLiveRouteBatch(routesToUpsert) + if err != nil { + h.logger.Error("Failed to upsert route batch", + "error", err, "count", len(routesToUpsert)) + } else { + successCount += len(routesToUpsert) + } + } + + if len(routesToDelete) > 0 { + err := h.db.DeleteLiveRouteBatch(routesToDelete) + if err != nil { + h.logger.Error("Failed to delete route batch", + "error", err, "count", len(routesToDelete)) + } else { + successCount += len(routesToDelete) + } + } + + // Update prefix tables + if len(prefixesToUpdate) > 0 { + err := h.db.UpdatePrefixesBatch(prefixesToUpdate) + if err != nil { + h.logger.Error("Failed to update prefix batch", + "error", err, "count", len(prefixesToUpdate)) + } + } + + return successCount +} + // canonicalPrefix returns prefix in the text form net/netip prints, the form // the IP lookup builds when it looks a prefix up. The feed sends IPv6 // announcements compressed ("2001:db8::/32") but IPv6 withdrawals uncompressed @@ -366,8 +414,8 @@ func canonicalPrefix(prefix string) string { return p.Masked().String() } -// parseCIDR extracts the mask length and IP version from a prefix string -func parseCIDR(prefix string) (maskLength int, ipVersion int, err error) { +// parseCIDR returns the mask length and the IP version of a prefix string +func parseCIDR(prefix string) (int, int, error) { _, ipNet, err := net.ParseCIDR(prefix) if err != nil { return 0, 0, err @@ -381,50 +429,6 @@ func parseCIDR(prefix string) (maskLength int, ipVersion int, err error) { return ones, ipv4Version, nil } -// processAnnouncement handles storing an announcement in the database -// nolint:unused // kept for potential future use -func (h *PrefixHandler) processAnnouncement(_ *database.Prefix, update prefixUpdate) { - // Parse CIDR to get mask length - maskLength, ipVersion, err := parseCIDR(update.prefix) - if err != nil { - h.logger.Error("Failed to parse CIDR", - "prefix", update.prefix, - "error", err, - ) - - return - } - - // Track route update metrics - if h.metrics != nil { - if ipVersion == ipv4Version { - h.metrics.RecordIPv4Update() - } else { - h.metrics.RecordIPv6Update() - } - } - - // Create live route record - liveRoute := &database.LiveRoute{ - ID: uuid.New(), - Prefix: update.prefix, - MaskLength: maskLength, - IPVersion: ipVersion, - OriginASN: update.originASN, - PeerIP: update.peer, - ASPath: update.path, - NextHop: update.peer, // Using peer as next hop - LastUpdated: update.timestamp, - } - - if err := h.db.UpsertLiveRoute(liveRoute); err != nil { - h.logger.Error("Failed to upsert live route", - "prefix", update.prefix, - "error", err, - ) - } -} - // createLiveRoute creates a LiveRoute from a prefix update func (h *PrefixHandler) createLiveRoute(update prefixUpdate) *database.LiveRoute { // Parse CIDR to get mask length @@ -462,77 +466,3 @@ func (h *PrefixHandler) createLiveRoute(update prefixUpdate) *database.LiveRoute return liveRoute } - -// processAnnouncementDirect handles storing an announcement directly without prefix table -// nolint:unused // kept for potential future use -func (h *PrefixHandler) processAnnouncementDirect(update prefixUpdate) { - // Parse CIDR to get mask length - maskLength, ipVersion, err := parseCIDR(update.prefix) - if err != nil { - h.logger.Error("Failed to parse CIDR", - "prefix", update.prefix, - "error", err, - ) - - return - } - - // Track route update metrics - if h.metrics != nil { - if ipVersion == ipv4Version { - h.metrics.RecordIPv4Update() - } else { - h.metrics.RecordIPv6Update() - } - } - - // Create live route record - liveRoute := &database.LiveRoute{ - ID: uuid.New(), - Prefix: update.prefix, - MaskLength: maskLength, - IPVersion: ipVersion, - OriginASN: update.originASN, - PeerIP: update.peer, - ASPath: update.path, - NextHop: update.peer, // Using peer as next hop - LastUpdated: update.timestamp, - } - - if err := h.db.UpsertLiveRoute(liveRoute); err != nil { - h.logger.Error("Failed to upsert live route", - "prefix", update.prefix, - "error", err, - ) - } -} - -// processWithdrawalDirect handles removing a route directly without prefix table -// nolint:unused // kept for potential future use -func (h *PrefixHandler) processWithdrawalDirect(update prefixUpdate) { - if err := h.db.DeleteLiveRoute(update.prefix, update.peer); err != nil { - h.logger.Error("Failed to delete live route", - "prefix", update.prefix, - "peer", update.peer, - "error", err, - ) - } -} - -// processWithdrawal handles removing a route from the live routing table -// nolint:unused // kept for potential future use -func (h *PrefixHandler) processWithdrawal(_ *database.Prefix, update prefixUpdate) { - if err := h.db.DeleteLiveRoute(update.prefix, update.peer); err != nil { - h.logger.Error("Failed to delete live route", - "prefix", update.prefix, - "peer", update.peer, - "error", err, - ) - } -} - -// Stop gracefully stops the handler and flushes remaining batches -func (h *PrefixHandler) Stop() { - close(h.stopCh) - h.wg.Wait() -} diff --git a/internal/routewatch/prefixhandler_test.go b/internal/routewatch/prefixhandler_internal_test.go similarity index 66% rename from internal/routewatch/prefixhandler_test.go rename to internal/routewatch/prefixhandler_internal_test.go index 2575a49..82c425f 100644 --- a/internal/routewatch/prefixhandler_test.go +++ b/internal/routewatch/prefixhandler_internal_test.go @@ -17,6 +17,12 @@ import ( const testPeerIP = "2001:db8:ffff::1" +// An IPv4 and an IPv6 prefix the tests announce. +const ( + testV4Prefix = "192.0.2.0/24" + testV6Prefix = "2001:db8:1::/48" +) + // TestPrefixHandlerStoresPrefixesTheIPLookupFinds runs announcements and // withdrawals through the prefix handler into a real database and looks the // addresses up. The prefixes are written the way the feed sends them: IPv6 @@ -24,11 +30,9 @@ const testPeerIP = "2001:db8:ffff::1" // remove the route the announcement stored, and the IP lookup must find the // stored prefix. func TestPrefixHandlerStoresPrefixesTheIPLookupFinds(t *testing.T) { - db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New()) - if err != nil { - t.Fatalf("failed to create database: %v", err) - } - defer func() { _ = db.Close() }() + t.Parallel() + + db := newTestDatabase(t) // Built without NewPrefixHandler's flush timer, so the test flushes itself. h := &PrefixHandler{db: db, logger: logger.New()} @@ -40,34 +44,38 @@ func TestPrefixHandlerStoresPrefixesTheIPLookupFinds(t *testing.T) { ParsedTimestamp: ts, Announcements: []ristypes.RISAnnouncement{{ NextHop: testPeerIP, - Prefixes: []string{"2001:db8:1::/48", "192.0.2.0/24"}, + Prefixes: []string{testV6Prefix, testV4Prefix}, }}, }) h.flushBatch() for ip, want := range map[string]string{ - "2001:db8:1::1": "2001:db8:1::/48", - "192.0.2.1": "192.0.2.0/24", + "2001:db8:1::1": testV6Prefix, + "192.0.2.1": testV4Prefix, } { info, err := db.GetIPInfo(ip) if err != nil { t.Fatalf("GetIPInfo(%s) after announcement: %v", ip, err) } + if info.Netblock != want || info.ASN != testASNB { - t.Errorf("GetIPInfo(%s) = %s AS%d, want %s AS%d", ip, info.Netblock, info.ASN, want, testASNB) + t.Errorf("GetIPInfo(%s) = %s AS%d, want %s AS%d", + ip, info.Netblock, info.ASN, want, testASNB) } } h.HandleMessage(&ristypes.RISMessage{ Peer: testPeerIP, ParsedTimestamp: ts.Add(time.Minute), - Withdrawals: []string{"2001:db8:1:0:0:0:0:0/48", "192.0.2.0/24"}, + Withdrawals: []string{"2001:db8:1:0:0:0:0:0/48", testV4Prefix}, }) h.flushBatch() for _, ip := range []string{"2001:db8:1::1", "192.0.2.1"} { - if info, err := db.GetIPInfo(ip); !errors.Is(err, database.ErrNoRoute) { - t.Errorf("GetIPInfo(%s) after withdrawal = %+v, %v; want ErrNoRoute", ip, info, err) + info, err := db.GetIPInfo(ip) + if !errors.Is(err, database.ErrNoRoute) { + t.Errorf("GetIPInfo(%s) after withdrawal = %+v, %v; want ErrNoRoute", + ip, info, err) } } } @@ -79,6 +87,8 @@ func TestPrefixHandlerStoresPrefixesTheIPLookupFinds(t *testing.T) { // live route for it, and updates from different peers are all applied. The // in-memory route count and prefix distribution must match the route table. func TestPrefixHandlerAppliesEachPeersLastUpdateInABatch(t *testing.T) { + t.Parallel() + const ( prefix = "198.51.100.0/24" prefixMaskLength = 24 @@ -91,11 +101,14 @@ func TestPrefixHandlerAppliesEachPeersLastUpdateInABatch(t *testing.T) { Peer: peer, Path: ristypes.ASPath{testASNA, testASNB}, ParsedTimestamp: ts, - Announcements: []ristypes.RISAnnouncement{{NextHop: peer, Prefixes: []string{prefix}}}, + Announcements: []ristypes.RISAnnouncement{ + {NextHop: peer, Prefixes: []string{prefix}}, + }, } } withdraw := func(peer string) *ristypes.RISMessage { - return &ristypes.RISMessage{Peer: peer, ParsedTimestamp: ts, Withdrawals: []string{prefix}} + return &ristypes.RISMessage{Peer: peer, ParsedTimestamp: ts, + Withdrawals: []string{prefix}} } announceAndWithdraw := func(peer string) *ristypes.RISMessage { msg := announce(peer) @@ -112,8 +125,10 @@ func TestPrefixHandlerAppliesEachPeersLastUpdateInABatch(t *testing.T) { wantPeers []string }{ { - name: "two peers announce the prefix", - batches: [][]*ristypes.RISMessage{{announce(testPeerIP), announce(otherPeerIP)}}, + name: "two peers announce the prefix", + batches: [][]*ristypes.RISMessage{ + {announce(testPeerIP), announce(otherPeerIP)}, + }, wantPeers: []string{testPeerIP, otherPeerIP}, }, { @@ -149,39 +164,10 @@ func TestPrefixHandlerAppliesEachPeersLastUpdateInABatch(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New()) - if err != nil { - t.Fatalf("failed to create database: %v", err) - } - defer func() { _ = db.Close() }() + t.Parallel() - // Built without NewPrefixHandler's flush timer, so the test flushes itself. - h := &PrefixHandler{db: db, logger: logger.New()} - for _, batch := range tt.batches { - for _, msg := range batch { - h.HandleMessage(msg) - } - h.flushBatch() - } - - ctx := context.Background() - routes, _, err := db.GetPrefixDetailsContext(ctx, prefix) - if err != nil && !errors.Is(err, database.ErrNoRoute) { - t.Fatalf("GetPrefixDetailsContext: %v", err) - } - var gotPeers []string - for _, route := range routes { - gotPeers = append(gotPeers, route.PeerIP) - } - if !slices.Equal(gotPeers, tt.wantPeers) { - t.Errorf("peers with a live route for %s = %v, want %v", prefix, gotPeers, tt.wantPeers) - } - - var wantDistribution []database.PrefixDistribution - if len(tt.wantPeers) > 0 { - wantDistribution = []database.PrefixDistribution{{MaskLength: prefixMaskLength, Count: 1}} - } - assertCounts(t, db, ipv4Version, len(tt.wantPeers), wantDistribution) + db := handleBatches(t, tt.batches) + checkLivePeers(t, db, prefix, prefixMaskLength, tt.wantPeers) }) } } @@ -194,14 +180,22 @@ func TestPrefixHandlerAppliesEachPeersLastUpdateInABatch(t *testing.T) { // in. The in-memory route count and prefix distribution must match the route // table. func TestPrefixHandlerKeepsOneRoutePerPrefixAndPeer(t *testing.T) { + t.Parallel() + families := []struct { ipVersion int prefix, otherPrefix string // The mask length of both prefixes. maskLength int }{ - {ipVersion: ipv4Version, prefix: "198.51.100.0/24", otherPrefix: "203.0.113.0/24", maskLength: 24}, - {ipVersion: ipv6Version, prefix: "2001:db8:1::/48", otherPrefix: "2001:db8:2::/48", maskLength: 48}, + { + ipVersion: ipv4Version, maskLength: 24, + prefix: "198.51.100.0/24", otherPrefix: "203.0.113.0/24", + }, + { + ipVersion: ipv6Version, maskLength: 48, + prefix: testV6Prefix, otherPrefix: "2001:db8:2::/48", + }, } ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) @@ -210,7 +204,9 @@ func TestPrefixHandlerKeepsOneRoutePerPrefixAndPeer(t *testing.T) { Peer: testPeerIP, Path: path, ParsedTimestamp: ts, - Announcements: []ristypes.RISAnnouncement{{NextHop: testPeerIP, Prefixes: []string{announced}}}, + Announcements: []ristypes.RISAnnouncement{ + {NextHop: testPeerIP, Prefixes: []string{announced}}, + }, } } @@ -219,71 +215,50 @@ func TestPrefixHandlerKeepsOneRoutePerPrefixAndPeer(t *testing.T) { // One message that announces otherPrefix with origin AS testASNB and // withdraws prefix. - announceOtherAndWithdraw := announce(ristypes.ASPath{testASNA, testASNB}, otherPrefix) + announceOtherAndWithdraw := announce( + ristypes.ASPath{testASNA, testASNB}, otherPrefix) announceOtherAndWithdraw.Withdrawals = []string{prefix} tests := []struct { - name string - messages []*ristypes.RISMessage + name string + // Each message is handled and then flushed as its own batch. + batches [][]*ristypes.RISMessage // The origin AS of the peer's live route for each prefix. A prefix // left out must have no live route. wantOrigins map[string]int }{ { name: "the peer announces the prefix with a new origin AS", - messages: []*ristypes.RISMessage{ - announce(ristypes.ASPath{testASNA}, prefix), - announce(ristypes.ASPath{testASNA, testASNB}, prefix), + batches: [][]*ristypes.RISMessage{ + {announce(ristypes.ASPath{testASNA}, prefix)}, + {announce(ristypes.ASPath{testASNA, testASNB}, prefix)}, }, wantOrigins: map[string]int{prefix: testASNB}, }, { - name: "one message announces a prefix with another origin AS and withdraws the prefix", - messages: []*ristypes.RISMessage{ - announce(ristypes.ASPath{testASNA}, prefix), - announceOtherAndWithdraw, + name: "one message announces a prefix with another origin AS " + + "and withdraws the prefix", + batches: [][]*ristypes.RISMessage{ + {announce(ristypes.ASPath{testASNA}, prefix)}, + {announceOtherAndWithdraw}, }, wantOrigins: map[string]int{otherPrefix: testASNB}, }, } for _, tt := range tests { - t.Run(fmt.Sprintf("IPv%d %s", family.ipVersion, tt.name), func(t *testing.T) { - db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New()) - if err != nil { - t.Fatalf("failed to create database: %v", err) - } - defer func() { _ = db.Close() }() + name := fmt.Sprintf("IPv%d %s", family.ipVersion, tt.name) + t.Run(name, func(t *testing.T) { + t.Parallel() - // Built without NewPrefixHandler's flush timer, so the test flushes itself. - h := &PrefixHandler{db: db, logger: logger.New()} - for _, msg := range tt.messages { - h.HandleMessage(msg) - h.flushBatch() - } - - for _, p := range []string{prefix, otherPrefix} { - routes, _, err := db.GetPrefixDetailsContext(context.Background(), p) - if err != nil && !errors.Is(err, database.ErrNoRoute) { - t.Fatalf("GetPrefixDetailsContext(%s): %v", p, err) - } - var gotOrigins []int - for _, route := range routes { - gotOrigins = append(gotOrigins, route.OriginASN) - } - var wantOrigins []int - if origin, ok := tt.wantOrigins[p]; ok { - wantOrigins = []int{origin} - } - if !slices.Equal(gotOrigins, wantOrigins) { - t.Errorf("origin ASes of the live routes for %s = %v, want %v", p, gotOrigins, wantOrigins) - } - } + db := handleBatches(t, tt.batches) + checkLiveOrigins(t, db, tt.wantOrigins, prefix, otherPrefix) wantDistribution := []database.PrefixDistribution{ {MaskLength: family.maskLength, Count: len(tt.wantOrigins)}, } - assertCounts(t, db, family.ipVersion, len(tt.wantOrigins), wantDistribution) + assertCounts(t, db, family.ipVersion, len(tt.wantOrigins), + wantDistribution) }) } } @@ -293,12 +268,16 @@ func TestPrefixHandlerKeepsOneRoutePerPrefixAndPeer(t *testing.T) { // messages go to the prefix handler, which acts on a down, and not to the peer // handler, which would record a peer whose session never comes up as active. func TestStateMessagesGoToThePrefixHandlerNotThePeerHandler(t *testing.T) { + t.Parallel() + if !(&PrefixHandler{}).WantsMessage(ristypes.MessageTypeState) { t.Error("the prefix handler does not want STATE messages") } + if (&PeerHandler{}).WantsMessage(ristypes.MessageTypeState) { t.Error("the peer handler wants STATE messages") } + if !(&PeerHandler{}).WantsMessage(ristypes.MessageTypeUpdate) { t.Error("the peer handler does not want UPDATE messages") } @@ -313,6 +292,8 @@ func TestStateMessagesGoToThePrefixHandlerNotThePeerHandler(t *testing.T) { // in the same batch, is kept. The in-memory route counts and prefix // distribution must match the route tables. func TestPrefixHandlerDeletesAPeersRoutesWhenItsSessionGoesDown(t *testing.T) { + t.Parallel() + const ( otherPeerIP = "2001:db8:ffff::2" // Both peers announce the shared prefixes, only testPeerIP the others. @@ -335,14 +316,17 @@ func TestPrefixHandlerDeletesAPeersRoutesWhenItsSessionGoesDown(t *testing.T) { wantAfterDown bool }{ { - name: "the down ends the batch", - batch: []*ristypes.RISMessage{peerAnnouncement(testPeerIP, beforeDown), sessionDown(testPeerIP)}, + name: "the down ends the batch", + batch: []*ristypes.RISMessage{ + peerAnnouncement(testPeerIP, beforeDown), + sessionDown(), + }, }, { name: "the peer announces a prefix after the down", batch: []*ristypes.RISMessage{ peerAnnouncement(testPeerIP, beforeDown), - sessionDown(testPeerIP), + sessionDown(), peerAnnouncement(testPeerIP, afterDown), }, wantAfterDown: true, @@ -351,21 +335,16 @@ func TestPrefixHandlerDeletesAPeersRoutesWhenItsSessionGoesDown(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New()) - if err != nil { - t.Fatalf("failed to create database: %v", err) - } - defer func() { _ = db.Close() }() + t.Parallel() - // Built without NewPrefixHandler's flush timer, so the test flushes itself. - h := &PrefixHandler{db: db, logger: logger.New()} - h.HandleMessage(peerAnnouncement(testPeerIP, sharedV4, sharedV6, onlyV4, onlyV6)) - h.HandleMessage(peerAnnouncement(otherPeerIP, sharedV4, sharedV6)) - h.flushBatch() - for _, msg := range tt.batch { - h.HandleMessage(msg) - } - h.flushBatch() + db := handleBatches(t, [][]*ristypes.RISMessage{ + { + peerAnnouncement(testPeerIP, + sharedV4, sharedV6, onlyV4, onlyV6), + peerAnnouncement(otherPeerIP, sharedV4, sharedV6), + }, + tt.batch, + }) // The peers left with a live route for each prefix. wantPeers := map[string][]string{ @@ -377,16 +356,20 @@ func TestPrefixHandlerDeletesAPeersRoutesWhenItsSessionGoesDown(t *testing.T) { afterDown: nil, } wantRoutesV6 := 1 + if tt.wantAfterDown { wantPeers[afterDown] = []string{testPeerIP} wantRoutesV6++ } assertPeers(t, db, wantPeers) - assertCounts(t, db, ipv4Version, 1, - []database.PrefixDistribution{{MaskLength: maskLengthV4, Count: 1}}) + assertCounts(t, db, ipv4Version, 1, []database.PrefixDistribution{ + {MaskLength: maskLengthV4, Count: 1}, + }) assertCounts(t, db, ipv6Version, wantRoutesV6, - []database.PrefixDistribution{{MaskLength: maskLengthV6, Count: wantRoutesV6}}) + []database.PrefixDistribution{ + {MaskLength: maskLengthV6, Count: wantRoutesV6}, + }) }) } } @@ -398,6 +381,8 @@ func TestPrefixHandlerDeletesAPeersRoutesWhenItsSessionGoesDown(t *testing.T) { // that handles it and on no later flush, so the route the peer announced after // it is kept. func TestPrefixHandlerKeepsWhatAPeerAnnouncesAfterTheFlushOfItsDown(t *testing.T) { + t.Parallel() + const ( otherPeerIP = "2001:db8:ffff::2" beforeDown = "192.0.2.0/24" @@ -406,30 +391,22 @@ func TestPrefixHandlerKeepsWhatAPeerAnnouncesAfterTheFlushOfItsDown(t *testing.T maskLength = 24 ) - db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New()) - if err != nil { - t.Fatalf("failed to create database: %v", err) - } - defer func() { _ = db.Close() }() - - // Built without NewPrefixHandler's flush timer, so the test flushes itself. - h := &PrefixHandler{db: db, logger: logger.New()} - for _, msg := range []*ristypes.RISMessage{ - peerAnnouncement(testPeerIP, beforeDown), - sessionDown(testPeerIP), - peerAnnouncement(testPeerIP, afterDown), - peerAnnouncement(otherPeerIP, otherPeers), - } { - h.HandleMessage(msg) - h.flushBatch() - } + // Each message is handled and then flushed as its own batch. + db := handleBatches(t, [][]*ristypes.RISMessage{ + {peerAnnouncement(testPeerIP, beforeDown)}, + {sessionDown()}, + {peerAnnouncement(testPeerIP, afterDown)}, + {peerAnnouncement(otherPeerIP, otherPeers)}, + }) assertPeers(t, db, map[string][]string{ beforeDown: nil, afterDown: {testPeerIP}, otherPeers: {otherPeerIP}, }) - assertCounts(t, db, ipv4Version, 2, []database.PrefixDistribution{{MaskLength: maskLength, Count: 2}}) + assertCounts(t, db, ipv4Version, 2, []database.PrefixDistribution{ + {MaskLength: maskLength, Count: 2}, + }) } // firstWriteHeldStore is a real database whose first live route batch write @@ -442,7 +419,11 @@ type firstWriteHeldStore struct { once sync.Once } -func (s *firstWriteHeldStore) UpsertLiveRouteBatch(routes []*database.LiveRoute) error { +// UpsertLiveRouteBatch holds the first call until release is closed, then +// writes routes to the database. +func (s *firstWriteHeldStore) UpsertLiveRouteBatch( + routes []*database.LiveRoute, +) error { s.once.Do(func() { close(s.writeStarted) <-s.release @@ -457,27 +438,32 @@ func (s *firstWriteHeldStore) UpsertLiveRouteBatch(routes []*database.LiveRoute) // must delete the route the held write stored and keep the one announced after // the down. func TestPrefixHandlerHandlesADownWhileAFlushWrites(t *testing.T) { + t.Parallel() + const ( beforeDown = "192.0.2.0/24" afterDown = "198.51.100.0/24" maskLength = 24 ) - db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New()) - if err != nil { - t.Fatalf("failed to create database: %v", err) + db := newTestDatabase(t) + store := &firstWriteHeldStore{ + Database: db, + writeStarted: make(chan struct{}), + release: make(chan struct{}), } - defer func() { _ = db.Close() }() - store := &firstWriteHeldStore{Database: db, writeStarted: make(chan struct{}), release: make(chan struct{})} // Built without NewPrefixHandler's flush timer, so the test flushes itself. h := &PrefixHandler{db: store, logger: logger.New()} h.HandleMessage(peerAnnouncement(testPeerIP, beforeDown)) + flushed := make(chan struct{}) + go func() { h.flushBatch() close(flushed) }() + select { case <-store.writeStarted: case <-time.After(handleMessageTimeout): @@ -485,21 +471,67 @@ func TestPrefixHandlerHandlesADownWhileAFlushWrites(t *testing.T) { } if !returnsInTime(func() { - h.HandleMessage(sessionDown(testPeerIP)) + h.HandleMessage(sessionDown()) h.HandleMessage(peerAnnouncement(testPeerIP, afterDown)) }) { t.Fatal("HandleMessage did not return while a database write was held") } + close(store.release) + select { case <-flushed: case <-time.After(handleMessageTimeout): t.Fatal("the flush did not finish after its write was let go") } + h.flushBatch() - assertPeers(t, db, map[string][]string{beforeDown: nil, afterDown: {testPeerIP}}) - assertCounts(t, db, ipv4Version, 1, []database.PrefixDistribution{{MaskLength: maskLength, Count: 1}}) + assertPeers(t, db, map[string][]string{ + beforeDown: nil, + afterDown: {testPeerIP}, + }) + assertCounts(t, db, ipv4Version, 1, []database.PrefixDistribution{ + {MaskLength: maskLength, Count: 1}, + }) +} + +// newTestDatabase opens a database in a new temporary directory. It is closed +// when the test ends. +func newTestDatabase(t *testing.T) *database.Database { + t.Helper() + + db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New()) + if err != nil { + t.Fatalf("failed to create database: %v", err) + } + + t.Cleanup(func() { _ = db.Close() }) + + return db +} + +// handleBatches runs each batch of messages through a prefix handler into a +// new database, flushing after each batch, and returns the database. +func handleBatches( + t *testing.T, batches [][]*ristypes.RISMessage, +) *database.Database { + t.Helper() + + db := newTestDatabase(t) + + // Built without NewPrefixHandler's flush timer, so the test flushes itself. + h := &PrefixHandler{db: db, logger: logger.New()} + + for _, batch := range batches { + for _, msg := range batch { + h.HandleMessage(msg) + } + + h.flushBatch() + } + + return db } // peerAnnouncement returns an UPDATE message in which peer announces prefixes @@ -510,18 +542,48 @@ func peerAnnouncement(peer string, prefixes ...string) *ristypes.RISMessage { Peer: peer, Path: ristypes.ASPath{testASNA, testASNB}, ParsedTimestamp: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC), - Announcements: []ristypes.RISAnnouncement{{NextHop: peer, Prefixes: prefixes}}, + Announcements: []ristypes.RISAnnouncement{ + {NextHop: peer, Prefixes: prefixes}, + }, } } -// sessionDown returns the STATE message that says peer's BGP session went down. -func sessionDown(peer string) *ristypes.RISMessage { - return &ristypes.RISMessage{Type: ristypes.MessageTypeState, Peer: peer, State: ristypes.StateDown} +// sessionDown returns the STATE message that says the BGP session of the peer +// testPeerIP went down. +func sessionDown() *ristypes.RISMessage { + return &ristypes.RISMessage{ + Type: ristypes.MessageTypeState, + Peer: testPeerIP, + State: ristypes.StateDown, + } +} + +// checkLivePeers checks that exactly wantPeers have a live route for the IPv4 +// prefix, and that the in-memory route count and prefix distribution match +// the route table. +func checkLivePeers( + t *testing.T, db *database.Database, prefix string, maskLength int, + wantPeers []string, +) { + t.Helper() + + assertPeers(t, db, map[string][]string{prefix: wantPeers}) + + var wantDistribution []database.PrefixDistribution + if len(wantPeers) > 0 { + wantDistribution = []database.PrefixDistribution{ + {MaskLength: maskLength, Count: 1}, + } + } + + assertCounts(t, db, ipv4Version, len(wantPeers), wantDistribution) } // assertPeers checks, for each prefix in wantPeers, that the peers with a live // route for it are the ones listed, in peer IP order; nil means none. -func assertPeers(t *testing.T, db *database.Database, wantPeers map[string][]string) { +func assertPeers( + t *testing.T, db *database.Database, wantPeers map[string][]string, +) { t.Helper() for prefix, want := range wantPeers { @@ -529,12 +591,46 @@ func assertPeers(t *testing.T, db *database.Database, wantPeers map[string][]str if err != nil && !errors.Is(err, database.ErrNoRoute) { t.Fatalf("GetPrefixDetailsContext(%s): %v", prefix, err) } - var got []string + + got := make([]string, 0, len(routes)) for _, route := range routes { got = append(got, route.PeerIP) } + if !slices.Equal(got, want) { - t.Errorf("peers with a live route for %s = %v, want %v", prefix, got, want) + t.Errorf("peers with a live route for %s = %v, want %v", + prefix, got, want) + } + } +} + +// checkLiveOrigins checks that each of prefixes has one live route, with the +// origin AS wantOrigins gives for it, or none when wantOrigins leaves it out. +func checkLiveOrigins( + t *testing.T, db *database.Database, wantOrigins map[string]int, + prefixes ...string, +) { + t.Helper() + + for _, prefix := range prefixes { + routes, _, err := db.GetPrefixDetailsContext(context.Background(), prefix) + if err != nil && !errors.Is(err, database.ErrNoRoute) { + t.Fatalf("GetPrefixDetailsContext(%s): %v", prefix, err) + } + + gotOrigins := make([]int, 0, len(routes)) + for _, route := range routes { + gotOrigins = append(gotOrigins, route.OriginASN) + } + + var want []int + if origin, ok := wantOrigins[prefix]; ok { + want = []int{origin} + } + + if !slices.Equal(gotOrigins, want) { + t.Errorf("origin ASes of the live routes for %s = %v, want %v", + prefix, gotOrigins, want) } } } @@ -542,38 +638,49 @@ func assertPeers(t *testing.T, db *database.Database, wantPeers map[string][]str // assertCounts checks that the live route count and prefix distribution of one // IP version are wantRoutes and wantDistribution, both as held in memory and as // read from the route table. -func assertCounts(t *testing.T, db *database.Database, ipVersion, wantRoutes int, - wantDistribution []database.PrefixDistribution) { +func assertCounts( + t *testing.T, db *database.Database, ipVersion, wantRoutes int, + wantDistribution []database.PrefixDistribution, +) { t.Helper() ctx := context.Background() + stats, err := db.GetStatsContext(ctx) if err != nil { t.Fatalf("GetStatsContext: %v", err) } + tableRoutesV4, tableRoutesV6, err := db.GetLiveRouteCountsContext(ctx) if err != nil { t.Fatalf("GetLiveRouteCountsContext: %v", err) } - queryDistributionV4, queryDistributionV6, err := db.GetPrefixDistributionContext(ctx) + + queryDistributionV4, queryDistributionV6, err := + db.GetPrefixDistributionContext(ctx) if err != nil { t.Fatalf("GetPrefixDistributionContext: %v", err) } memoryRoutes, tableRoutes := stats.IPv4Routes, tableRoutesV4 - memoryDistribution, queryDistribution := stats.IPv4PrefixDistribution, queryDistributionV4 + memoryDistribution := stats.IPv4PrefixDistribution + queryDistribution := queryDistributionV4 + if ipVersion == ipv6Version { memoryRoutes, tableRoutes = stats.IPv6Routes, tableRoutesV6 - memoryDistribution, queryDistribution = stats.IPv6PrefixDistribution, queryDistributionV6 + memoryDistribution = stats.IPv6PrefixDistribution + queryDistribution = queryDistributionV6 } if memoryRoutes != wantRoutes || tableRoutes != wantRoutes { t.Errorf("IPv%d live routes: %d in memory, %d in the table, want %d", ipVersion, memoryRoutes, tableRoutes, wantRoutes) } + if !slices.Equal(memoryDistribution, wantDistribution) || !slices.Equal(queryDistribution, wantDistribution) { - t.Errorf("IPv%d prefix distribution: %v in memory, %v from the table, want %v", - ipVersion, memoryDistribution, queryDistribution, wantDistribution) + t.Errorf("IPv%d prefix distribution: %v in memory, %v from the table, "+ + "want %v", ipVersion, memoryDistribution, queryDistribution, + wantDistribution) } } diff --git a/internal/server/handlers.go b/internal/server/handlers.go index 37c5c5b..2c436ae 100644 --- a/internal/server/handlers.go +++ b/internal/server/handlers.go @@ -15,6 +15,8 @@ import ( "time" "git.eeqj.de/sneak/routewatch/internal/database" + "git.eeqj.de/sneak/routewatch/internal/metrics" + "git.eeqj.de/sneak/routewatch/internal/streamer" "git.eeqj.de/sneak/routewatch/internal/templates" asinfo "git.eeqj.de/sneak/routewatch/pkg/asinfo" "github.com/dustin/go-humanize" @@ -29,6 +31,22 @@ const ( healthCheckTimeout = 2 * time.Second ) +// Keys of the JSON envelope the handlers and middleware write, and the status +// value of an error answer. +const ( + jsonKeyStatus = "status" + jsonKeyError = "error" + jsonKeyMsg = "msg" + jsonKeyCode = "code" + jsonStatusError = "error" +) + +// IP versions as database.LiveRoute.IPVersion records them. +const ( + ipVersionV4 = 4 + ipVersionV6 = 6 +) + // HealthCheckResponse represents the health check response. type HealthCheckResponse struct { Status string `json:"status"` @@ -68,7 +86,7 @@ func (s *Server) handleHealthCheck() http.HandlerFunc { // Build response status := "ok" if !healthy { - status = "error" + status = jsonStatusError } response := HealthCheckResponse{ @@ -81,7 +99,8 @@ func (s *Server) handleHealthCheck() http.HandlerFunc { w.WriteHeader(http.StatusServiceUnavailable) } - if err := writeJSONSuccess(w, response); err != nil { + err = writeJSONSuccess(w, response) + if err != nil { s.logger.Error("Failed to encode health check response", "error", err) } } @@ -93,7 +112,9 @@ func (s *Server) handleIndex() http.HandlerFunc { w.Header().Set("Content-Type", "text/html; charset=utf-8") tmpl := templates.IndexTemplate() - if err := tmpl.Execute(w, nil); err != nil { + + err := tmpl.Execute(w, nil) + if err != nil { s.logger.Error("Failed to render index template", "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) } @@ -102,30 +123,36 @@ func (s *Server) handleIndex() http.HandlerFunc { // writeJSONError writes a standardized JSON error response with the given // status code and error message. -func writeJSONError(w http.ResponseWriter, statusCode int, message string) { +func (s *Server) writeJSONError(w http.ResponseWriter, statusCode int, message string) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(statusCode) - _ = json.NewEncoder(w).Encode(map[string]interface{}{ - "status": "error", - "error": map[string]interface{}{ - "msg": message, - "code": statusCode, + + err := json.NewEncoder(w).Encode(map[string]any{ + jsonKeyStatus: jsonStatusError, + jsonKeyError: map[string]any{ + jsonKeyMsg: message, + jsonKeyCode: statusCode, }, }) + if err != nil { + s.logger.Error("Failed to encode error response", "error", err) + } } // writeJSONSuccess writes a standardized JSON success response containing // the provided data wrapped in a status envelope. -func writeJSONSuccess(w http.ResponseWriter, data interface{}) error { +func writeJSONSuccess(w http.ResponseWriter, data any) error { w.Header().Set("Content-Type", "application/json") - return json.NewEncoder(w).Encode(map[string]interface{}{ - "status": "ok", - "data": data, + return json.NewEncoder(w).Encode(map[string]any{ + jsonKeyStatus: "ok", + "data": data, }) } // WHOISStatsInfo contains WHOIS fetcher statistics for the status page. +// +//nolint:tagliatelle // field names are the existing JSON API type WHOISStatsInfo struct { TotalASNs int `json:"total_asns"` FreshASNs int `json:"fresh_asns"` @@ -138,41 +165,75 @@ type WHOISStatsInfo struct { FreshPercent float64 `json:"fresh_percent"` } +// Stats is the /status.json answer: uptime, message counts, database stats, +// and route information. +// +//nolint:tagliatelle // field names are the existing JSON API +type Stats struct { + Uptime string `json:"uptime"` + TotalMessages uint64 `json:"total_messages"` + TotalBytes uint64 `json:"total_bytes"` + TotalWireBytes uint64 `json:"total_wire_bytes"` + MessagesPerSec float64 `json:"messages_per_sec"` + MbitsPerSec float64 `json:"mbits_per_sec"` + WireMbitsPerSec float64 `json:"wire_mbits_per_sec"` + Connected bool `json:"connected"` + GoVersion string `json:"go_version"` + Goroutines int `json:"goroutines"` + MemoryUsage string `json:"memory_usage"` + ASNs int `json:"asns"` + Prefixes int `json:"prefixes"` + IPv4Prefixes int `json:"ipv4_prefixes"` + IPv6Prefixes int `json:"ipv6_prefixes"` + Peerings int `json:"peerings"` + Peers int `json:"peers"` + DatabaseSizeBytes int64 `json:"database_size_bytes"` + LiveRoutes int `json:"live_routes"` + IPv4Routes int `json:"ipv4_routes"` + IPv6Routes int `json:"ipv6_routes"` + OldestRoute *time.Time `json:"oldest_route,omitempty"` + NewestRoute *time.Time `json:"newest_route,omitempty"` + IPv4UpdatesPerSec float64 `json:"ipv4_updates_per_sec"` + IPv6UpdatesPerSec float64 `json:"ipv6_updates_per_sec"` + IPv4PrefixDistribution []database.PrefixDistribution `json:"ipv4_prefix_distribution"` + IPv6PrefixDistribution []database.PrefixDistribution `json:"ipv6_prefix_distribution"` + WHOISStats *WHOISStatsInfo `json:"whois_stats,omitempty"` +} + +// databaseStats gets the database stats, giving up when ctx is done; it then +// returns ctx's error. The channels are buffered so the goroutine's send never +// blocks if the timeout wins and nothing here receives; otherwise it would +// block forever and leak. +func (s *Server) databaseStats(ctx context.Context) (database.Stats, error) { + statsChan := make(chan database.Stats, 1) + errChan := make(chan error, 1) + + go func() { + dbStats, err := s.db.GetStatsContext(ctx) + if err != nil { + s.logger.Debug("Database stats query failed", "error", err) + + errChan <- err + + return + } + + statsChan <- dbStats + }() + + select { + case <-ctx.Done(): + return database.Stats{}, ctx.Err() + case err := <-errChan: + return database.Stats{}, err + case dbStats := <-statsChan: + return dbStats, nil + } +} + // handleStatusJSON returns a handler that serves JSON statistics including // uptime, message counts, database stats, and route information. func (s *Server) handleStatusJSON() http.HandlerFunc { - // Stats represents the statistics response - type Stats struct { - Uptime string `json:"uptime"` - TotalMessages uint64 `json:"total_messages"` - TotalBytes uint64 `json:"total_bytes"` - TotalWireBytes uint64 `json:"total_wire_bytes"` - MessagesPerSec float64 `json:"messages_per_sec"` - MbitsPerSec float64 `json:"mbits_per_sec"` - WireMbitsPerSec float64 `json:"wire_mbits_per_sec"` - Connected bool `json:"connected"` - GoVersion string `json:"go_version"` - Goroutines int `json:"goroutines"` - MemoryUsage string `json:"memory_usage"` - ASNs int `json:"asns"` - Prefixes int `json:"prefixes"` - IPv4Prefixes int `json:"ipv4_prefixes"` - IPv6Prefixes int `json:"ipv6_prefixes"` - Peerings int `json:"peerings"` - Peers int `json:"peers"` - DatabaseSizeBytes int64 `json:"database_size_bytes"` - LiveRoutes int `json:"live_routes"` - IPv4Routes int `json:"ipv4_routes"` - IPv6Routes int `json:"ipv6_routes"` - OldestRoute *time.Time `json:"oldest_route,omitempty"` - NewestRoute *time.Time `json:"newest_route,omitempty"` - IPv4UpdatesPerSec float64 `json:"ipv4_updates_per_sec"` - IPv6UpdatesPerSec float64 `json:"ipv6_updates_per_sec"` - IPv4PrefixDistribution []database.PrefixDistribution `json:"ipv4_prefix_distribution"` - IPv6PrefixDistribution []database.PrefixDistribution `json:"ipv6_prefix_distribution"` - WHOISStats *WHOISStatsInfo `json:"whois_stats,omitempty"` - } - return func(w http.ResponseWriter, r *http.Request) { // Create a 4 second timeout context for this request ctx, cancel := context.WithTimeout(r.Context(), statsContextTimeout) @@ -180,49 +241,21 @@ func (s *Server) handleStatusJSON() http.HandlerFunc { metrics := s.streamer.GetMetrics() - // Get database stats with timeout. The channels are buffered so the - // goroutine's send never blocks if the timeout wins and nothing here - // receives; otherwise it would block forever and leak. - statsChan := make(chan database.Stats, 1) - errChan := make(chan error, 1) - - go func() { - dbStats, err := s.db.GetStatsContext(ctx) - if err != nil { - s.logger.Debug("Database stats query failed", "error", err) - errChan <- err - - return - } - statsChan <- dbStats - }() - - var dbStats database.Stats - var err error - select { - case <-ctx.Done(): - err = ctx.Err() - case err = <-errChan: - case dbStats = <-statsChan: - } + dbStats, err := s.databaseStats(ctx) if deadlinePassed(ctx) { s.logger.Error("Database stats timeout in status.json") - writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) + s.writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) return } + if err != nil { s.logger.Error("Failed to get database stats", "error", err) - writeJSONError(w, http.StatusInternalServerError, err.Error()) + s.writeJSONError(w, http.StatusInternalServerError, err.Error()) return } - uptime := time.Since(metrics.ConnectedSince).Truncate(time.Second).String() - if metrics.ConnectedSince.IsZero() { - uptime = "0s" - } - const bitsPerMegabit = 1000000.0 // Get route update metrics @@ -237,14 +270,14 @@ func (s *Server) handleStatusJSON() http.HandlerFunc { if s.asnFetcher != nil { whoisStats = s.getWHOISStats(ctx) if deadlinePassed(ctx) { - writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) + s.writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) return } } stats := Stats{ - Uptime: uptime, + Uptime: streamUptime(metrics.ConnectedSince), TotalMessages: metrics.TotalMessages, TotalBytes: metrics.TotalBytes, TotalWireBytes: metrics.TotalWireBytes, @@ -274,7 +307,8 @@ func (s *Server) handleStatusJSON() http.HandlerFunc { WHOISStats: whoisStats, } - if err := writeJSONSuccess(w, stats); err != nil { + err = writeJSONSuccess(w, stats) + if err != nil { s.logger.Error("Failed to encode stats", "error", err) } } @@ -296,7 +330,8 @@ func (s *Server) getWHOISStats(ctx context.Context) *WHOISStatsInfo { // Calculate fresh percentage var freshPercent float64 if dbStats.TotalASNs > 0 { - freshPercent = float64(dbStats.FreshASNs) / float64(dbStats.TotalASNs) * percentMultiplier + freshPercent = float64(dbStats.FreshASNs) / float64(dbStats.TotalASNs) * + percentMultiplier } return &WHOISStatsInfo{ @@ -312,82 +347,174 @@ func (s *Server) getWHOISStats(ctx context.Context) *WHOISStatsInfo { } } +// streamUptime is how long the stream has been connected, to the second, or +// "0s" if it has never connected. +func streamUptime(connectedSince time.Time) string { + if connectedSince.IsZero() { + return "0s" + } + + return time.Since(connectedSince).Truncate(time.Second).String() +} + +// connectionDuration is how long the stream has been connected, to the +// second, or "disconnected". +func connectionDuration(connected bool, connectedSince time.Time) string { + if !connected || connectedSince.IsZero() { + return "disconnected" + } + + return time.Since(connectedSince).Truncate(time.Second).String() +} + // whoisStaleThreshold matches the fetcher's threshold for consistency. const whoisStaleThreshold = 30 * 24 * time.Hour // percentMultiplier converts a ratio to a percentage. const percentMultiplier = 100 +// HandlerStatsInfo represents handler statistics in the API response +// +//nolint:tagliatelle // field names are the existing JSON API +type HandlerStatsInfo struct { + Name string `json:"name"` + QueueLength int `json:"queue_length"` + QueueCapacity int `json:"queue_capacity"` + QueueHighWaterMark int `json:"queue_high_water_mark"` + ProcessedCount uint64 `json:"processed_count"` + DroppedCount uint64 `json:"dropped_count"` + AvgProcessTimeMs float64 `json:"avg_process_time_ms"` + MinProcessTimeMs float64 `json:"min_process_time_ms"` + MaxProcessTimeMs float64 `json:"max_process_time_ms"` +} + +// GCStats represents garbage collection statistics +// +//nolint:tagliatelle // field names are the existing JSON API +type GCStats struct { + NumGC uint32 `json:"num_gc"` + TotalPauseMs uint64 `json:"total_pause_ms"` + LastPauseMs float64 `json:"last_pause_ms"` + HeapAllocBytes uint64 `json:"heap_alloc_bytes"` + HeapSysBytes uint64 `json:"heap_sys_bytes"` +} + +// StreamStats represents stream statistics including announcements/withdrawals +// +//nolint:tagliatelle // field names are the existing JSON API +type StreamStats struct { + Announcements uint64 `json:"announcements"` + Withdrawals uint64 `json:"withdrawals"` + RouteChurnPerSec float64 `json:"route_churn_per_sec"` + BGPPeerCount int `json:"bgp_peer_count"` +} + +// StatsResponse represents the API statistics response +// +//nolint:tagliatelle // field names are the existing JSON API +type StatsResponse struct { + Uptime string `json:"uptime"` + TotalMessages uint64 `json:"total_messages"` + TotalBytes uint64 `json:"total_bytes"` + TotalWireBytes uint64 `json:"total_wire_bytes"` + MessagesPerSec float64 `json:"messages_per_sec"` + MbitsPerSec float64 `json:"mbits_per_sec"` + WireMbitsPerSec float64 `json:"wire_mbits_per_sec"` + Connected bool `json:"connected"` + ConnectionDuration string `json:"connection_duration"` + ReconnectCount uint64 `json:"reconnect_count"` + GoVersion string `json:"go_version"` + Goroutines int `json:"goroutines"` + MemoryUsage string `json:"memory_usage"` + GC GCStats `json:"gc"` + Stream StreamStats `json:"stream"` + ASNs int `json:"asns"` + Prefixes int `json:"prefixes"` + IPv4Prefixes int `json:"ipv4_prefixes"` + IPv6Prefixes int `json:"ipv6_prefixes"` + Peerings int `json:"peerings"` + Peers int `json:"peers"` + DatabaseSizeBytes int64 `json:"database_size_bytes"` + LiveRoutes int `json:"live_routes"` + IPv4Routes int `json:"ipv4_routes"` + IPv6Routes int `json:"ipv6_routes"` + OldestRoute *time.Time `json:"oldest_route,omitempty"` + NewestRoute *time.Time `json:"newest_route,omitempty"` + IPv4UpdatesPerSec float64 `json:"ipv4_updates_per_sec"` + IPv6UpdatesPerSec float64 `json:"ipv6_updates_per_sec"` + HandlerStats []HandlerStatsInfo `json:"handler_stats"` + IPv4PrefixDistribution []database.PrefixDistribution `json:"ipv4_prefix_distribution"` + IPv6PrefixDistribution []database.PrefixDistribution `json:"ipv6_prefix_distribution"` + WHOISStats *WHOISStatsInfo `json:"whois_stats,omitempty"` +} + +// handlerStatsInfo converts the streamer's handler statistics for the API +// response. +func handlerStatsInfo(handlerStats []streamer.HandlerStats) []HandlerStatsInfo { + const microsecondsPerMillisecond = 1000.0 + + handlerStatsInfo := make([]HandlerStatsInfo, 0, len(handlerStats)) + for _, hs := range handlerStats { + handlerStatsInfo = append(handlerStatsInfo, HandlerStatsInfo{ + Name: hs.Name, + QueueLength: hs.QueueLength, + QueueCapacity: hs.QueueCapacity, + QueueHighWaterMark: hs.QueueHighWaterMark, + ProcessedCount: hs.ProcessedCount, + DroppedCount: hs.DroppedCount, + AvgProcessTimeMs: float64(hs.AvgProcessTime.Microseconds()) / + microsecondsPerMillisecond, + MinProcessTimeMs: float64(hs.MinProcessTime.Microseconds()) / + microsecondsPerMillisecond, + MaxProcessTimeMs: float64(hs.MaxProcessTime.Microseconds()) / + microsecondsPerMillisecond, + }) + } + + return handlerStatsInfo +} + +// gcStats reports the garbage collector's figures from memStats. +func gcStats(memStats *runtime.MemStats) GCStats { + const ( + nanosecondsPerMillisecond = 1e6 + gcPauseHistorySize = 256 // Size of runtime.MemStats.PauseNs circular buffer + ) + + // Calculate last GC pause + var lastPauseMs float64 + + if memStats.NumGC > 0 { + // PauseNs is a circular buffer, get the most recent pause + lastPauseIdx := (memStats.NumGC + gcPauseHistorySize - 1) % gcPauseHistorySize + lastPauseMs = float64(memStats.PauseNs[lastPauseIdx]) / nanosecondsPerMillisecond + } + + return GCStats{ + NumGC: memStats.NumGC, + TotalPauseMs: memStats.PauseTotalNs / uint64(nanosecondsPerMillisecond), + LastPauseMs: lastPauseMs, + HeapAllocBytes: memStats.HeapAlloc, + HeapSysBytes: memStats.HeapSys, + } +} + +// streamStats reports the announcement, withdrawal and BGP peer figures of the +// streamer's metrics tracker. +func (s *Server) streamStats() StreamStats { + metricsTracker := s.streamer.GetMetricsTracker() + + return StreamStats{ + Announcements: metricsTracker.GetAnnouncementCount(), + Withdrawals: metricsTracker.GetWithdrawalCount(), + RouteChurnPerSec: metricsTracker.GetChurnRate(), + BGPPeerCount: metricsTracker.GetBGPPeerCount(), + } +} + // handleStats returns a handler that serves API v1 statistics including // detailed handler queue statistics and performance metrics. func (s *Server) handleStats() http.HandlerFunc { - // HandlerStatsInfo represents handler statistics in the API response - type HandlerStatsInfo struct { - Name string `json:"name"` - QueueLength int `json:"queue_length"` - QueueCapacity int `json:"queue_capacity"` - QueueHighWaterMark int `json:"queue_high_water_mark"` - ProcessedCount uint64 `json:"processed_count"` - DroppedCount uint64 `json:"dropped_count"` - AvgProcessTimeMs float64 `json:"avg_process_time_ms"` - MinProcessTimeMs float64 `json:"min_process_time_ms"` - MaxProcessTimeMs float64 `json:"max_process_time_ms"` - } - - // GCStats represents garbage collection statistics - type GCStats struct { - NumGC uint32 `json:"num_gc"` - TotalPauseMs uint64 `json:"total_pause_ms"` - LastPauseMs float64 `json:"last_pause_ms"` - HeapAllocBytes uint64 `json:"heap_alloc_bytes"` - HeapSysBytes uint64 `json:"heap_sys_bytes"` - } - - // StreamStats represents stream statistics including announcements/withdrawals - type StreamStats struct { - Announcements uint64 `json:"announcements"` - Withdrawals uint64 `json:"withdrawals"` - RouteChurnPerSec float64 `json:"route_churn_per_sec"` - BGPPeerCount int `json:"bgp_peer_count"` - } - - // StatsResponse represents the API statistics response - type StatsResponse struct { - Uptime string `json:"uptime"` - TotalMessages uint64 `json:"total_messages"` - TotalBytes uint64 `json:"total_bytes"` - TotalWireBytes uint64 `json:"total_wire_bytes"` - MessagesPerSec float64 `json:"messages_per_sec"` - MbitsPerSec float64 `json:"mbits_per_sec"` - WireMbitsPerSec float64 `json:"wire_mbits_per_sec"` - Connected bool `json:"connected"` - ConnectionDuration string `json:"connection_duration"` - ReconnectCount uint64 `json:"reconnect_count"` - GoVersion string `json:"go_version"` - Goroutines int `json:"goroutines"` - MemoryUsage string `json:"memory_usage"` - GC GCStats `json:"gc"` - Stream StreamStats `json:"stream"` - ASNs int `json:"asns"` - Prefixes int `json:"prefixes"` - IPv4Prefixes int `json:"ipv4_prefixes"` - IPv6Prefixes int `json:"ipv6_prefixes"` - Peerings int `json:"peerings"` - Peers int `json:"peers"` - DatabaseSizeBytes int64 `json:"database_size_bytes"` - LiveRoutes int `json:"live_routes"` - IPv4Routes int `json:"ipv4_routes"` - IPv6Routes int `json:"ipv6_routes"` - OldestRoute *time.Time `json:"oldest_route,omitempty"` - NewestRoute *time.Time `json:"newest_route,omitempty"` - IPv4UpdatesPerSec float64 `json:"ipv4_updates_per_sec"` - IPv6UpdatesPerSec float64 `json:"ipv6_updates_per_sec"` - HandlerStats []HandlerStatsInfo `json:"handler_stats"` - IPv4PrefixDistribution []database.PrefixDistribution `json:"ipv4_prefix_distribution"` - IPv6PrefixDistribution []database.PrefixDistribution `json:"ipv6_prefix_distribution"` - WHOISStats *WHOISStatsInfo `json:"whois_stats,omitempty"` - } - return func(w http.ResponseWriter, r *http.Request) { // Create a 4 second timeout context for this request ctx, cancel := context.WithTimeout(r.Context(), statsContextTimeout) @@ -396,7 +523,7 @@ func (s *Server) handleStats() http.HandlerFunc { // Check if context is already cancelled select { case <-ctx.Done(): - writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) + s.writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) return default: @@ -404,165 +531,94 @@ func (s *Server) handleStats() http.HandlerFunc { metrics := s.streamer.GetMetrics() - // Get database stats with timeout. The channels are buffered so the - // goroutine's send never blocks if the timeout wins and nothing here - // receives; otherwise it would block forever and leak. - statsChan := make(chan database.Stats, 1) - errChan := make(chan error, 1) - - go func() { - dbStats, err := s.db.GetStatsContext(ctx) - if err != nil { - s.logger.Debug("Database stats query failed", "error", err) - errChan <- err - - return - } - statsChan <- dbStats - }() - - var dbStats database.Stats - var err error - select { - case <-ctx.Done(): - err = ctx.Err() - case err = <-errChan: - case dbStats = <-statsChan: - } + dbStats, err := s.databaseStats(ctx) if deadlinePassed(ctx) { s.logger.Error("Database stats timeout") - writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) + s.writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) return } + if err != nil { s.logger.Error("Failed to get database stats", "error", err) - writeJSONError(w, http.StatusInternalServerError, err.Error()) + s.writeJSONError(w, http.StatusInternalServerError, err.Error()) return } - uptime := time.Since(metrics.ConnectedSince).Truncate(time.Second).String() - if metrics.ConnectedSince.IsZero() { - uptime = "0s" - } - - const bitsPerMegabit = 1000000.0 - - // Get route update metrics - routeMetrics := s.streamer.GetMetricsTracker().GetRouteMetrics() - - // Get handler stats - handlerStats := s.streamer.GetHandlerStats() - handlerStatsInfo := make([]HandlerStatsInfo, 0, len(handlerStats)) - const microsecondsPerMillisecond = 1000.0 - for _, hs := range handlerStats { - handlerStatsInfo = append(handlerStatsInfo, HandlerStatsInfo{ - Name: hs.Name, - QueueLength: hs.QueueLength, - QueueCapacity: hs.QueueCapacity, - QueueHighWaterMark: hs.QueueHighWaterMark, - ProcessedCount: hs.ProcessedCount, - DroppedCount: hs.DroppedCount, - AvgProcessTimeMs: float64(hs.AvgProcessTime.Microseconds()) / microsecondsPerMillisecond, - MinProcessTimeMs: float64(hs.MinProcessTime.Microseconds()) / microsecondsPerMillisecond, - MaxProcessTimeMs: float64(hs.MaxProcessTime.Microseconds()) / microsecondsPerMillisecond, - }) - } - - // Get memory stats - var memStats runtime.MemStats - runtime.ReadMemStats(&memStats) - // Get WHOIS stats if fetcher is available var whoisStats *WHOISStatsInfo if s.asnFetcher != nil { whoisStats = s.getWHOISStats(ctx) if deadlinePassed(ctx) { - writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) + s.writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) return } } - // Calculate connection duration - connectionDuration := "disconnected" - if metrics.Connected && !metrics.ConnectedSince.IsZero() { - connectionDuration = time.Since(metrics.ConnectedSince).Truncate(time.Second).String() - } - - // Get announcement/withdrawal stats from metrics tracker - metricsTracker := s.streamer.GetMetricsTracker() - announcements := metricsTracker.GetAnnouncementCount() - withdrawals := metricsTracker.GetWithdrawalCount() - churnRate := metricsTracker.GetChurnRate() - bgpPeerCount := metricsTracker.GetBGPPeerCount() - - // Calculate last GC pause - const ( - nanosecondsPerMillisecond = 1e6 - gcPauseHistorySize = 256 // Size of runtime.MemStats.PauseNs circular buffer - ) - var lastPauseMs float64 - if memStats.NumGC > 0 { - // PauseNs is a circular buffer, get the most recent pause - lastPauseIdx := (memStats.NumGC + gcPauseHistorySize - 1) % gcPauseHistorySize - lastPauseMs = float64(memStats.PauseNs[lastPauseIdx]) / nanosecondsPerMillisecond - } - - stats := StatsResponse{ - Uptime: uptime, - TotalMessages: metrics.TotalMessages, - TotalBytes: metrics.TotalBytes, - TotalWireBytes: metrics.TotalWireBytes, - MessagesPerSec: metrics.MessagesPerSec, - MbitsPerSec: metrics.BitsPerSec / bitsPerMegabit, - WireMbitsPerSec: metrics.WireBitsPerSec / bitsPerMegabit, - Connected: metrics.Connected, - ConnectionDuration: connectionDuration, - ReconnectCount: metrics.ReconnectCount, - GoVersion: runtime.Version(), - Goroutines: runtime.NumGoroutine(), - MemoryUsage: humanize.Bytes(memStats.Alloc), - GC: GCStats{ - NumGC: memStats.NumGC, - TotalPauseMs: memStats.PauseTotalNs / uint64(nanosecondsPerMillisecond), - LastPauseMs: lastPauseMs, - HeapAllocBytes: memStats.HeapAlloc, - HeapSysBytes: memStats.HeapSys, - }, - Stream: StreamStats{ - Announcements: announcements, - Withdrawals: withdrawals, - RouteChurnPerSec: churnRate, - BGPPeerCount: bgpPeerCount, - }, - ASNs: dbStats.ASNs, - Prefixes: dbStats.Prefixes, - IPv4Prefixes: dbStats.IPv4Prefixes, - IPv6Prefixes: dbStats.IPv6Prefixes, - Peerings: dbStats.Peerings, - Peers: dbStats.Peers, - DatabaseSizeBytes: dbStats.FileSizeBytes, - LiveRoutes: dbStats.LiveRoutes, - IPv4Routes: dbStats.IPv4Routes, - IPv6Routes: dbStats.IPv6Routes, - OldestRoute: dbStats.OldestRoute, - NewestRoute: dbStats.NewestRoute, - IPv4UpdatesPerSec: routeMetrics.IPv4UpdatesPerSec, - IPv6UpdatesPerSec: routeMetrics.IPv6UpdatesPerSec, - HandlerStats: handlerStatsInfo, - IPv4PrefixDistribution: dbStats.IPv4PrefixDistribution, - IPv6PrefixDistribution: dbStats.IPv6PrefixDistribution, - WHOISStats: whoisStats, - } - - if err := writeJSONSuccess(w, stats); err != nil { + err = writeJSONSuccess(w, s.statsResponse(metrics, dbStats, whoisStats)) + if err != nil { s.logger.Error("Failed to encode stats", "error", err) } } } +// statsResponse builds the /api/v1/stats answer from the stream metrics, the +// database stats and the WHOIS stats, with the route update rates, memory use +// and handler queues as they are now. +func (s *Server) statsResponse( + streamMetrics metrics.StreamMetrics, + dbStats database.Stats, + whoisStats *WHOISStatsInfo, +) StatsResponse { + const bitsPerMegabit = 1000000.0 + + // Get route update metrics + routeMetrics := s.streamer.GetMetricsTracker().GetRouteMetrics() + + // Get memory stats + var memStats runtime.MemStats + runtime.ReadMemStats(&memStats) + + return StatsResponse{ + Uptime: streamUptime(streamMetrics.ConnectedSince), + TotalMessages: streamMetrics.TotalMessages, + TotalBytes: streamMetrics.TotalBytes, + TotalWireBytes: streamMetrics.TotalWireBytes, + MessagesPerSec: streamMetrics.MessagesPerSec, + MbitsPerSec: streamMetrics.BitsPerSec / bitsPerMegabit, + WireMbitsPerSec: streamMetrics.WireBitsPerSec / bitsPerMegabit, + Connected: streamMetrics.Connected, + ConnectionDuration: connectionDuration( + streamMetrics.Connected, streamMetrics.ConnectedSince), + ReconnectCount: streamMetrics.ReconnectCount, + GoVersion: runtime.Version(), + Goroutines: runtime.NumGoroutine(), + MemoryUsage: humanize.Bytes(memStats.Alloc), + GC: gcStats(&memStats), + Stream: s.streamStats(), + ASNs: dbStats.ASNs, + Prefixes: dbStats.Prefixes, + IPv4Prefixes: dbStats.IPv4Prefixes, + IPv6Prefixes: dbStats.IPv6Prefixes, + Peerings: dbStats.Peerings, + Peers: dbStats.Peers, + DatabaseSizeBytes: dbStats.FileSizeBytes, + LiveRoutes: dbStats.LiveRoutes, + IPv4Routes: dbStats.IPv4Routes, + IPv6Routes: dbStats.IPv6Routes, + OldestRoute: dbStats.OldestRoute, + NewestRoute: dbStats.NewestRoute, + IPv4UpdatesPerSec: routeMetrics.IPv4UpdatesPerSec, + IPv6UpdatesPerSec: routeMetrics.IPv6UpdatesPerSec, + HandlerStats: handlerStatsInfo(s.streamer.GetHandlerStats()), + IPv4PrefixDistribution: dbStats.IPv4PrefixDistribution, + IPv6PrefixDistribution: dbStats.IPv6PrefixDistribution, + WHOISStats: whoisStats, + } +} + // handleStatusHTML returns a handler that serves the HTML status page, // which displays real-time statistics fetched via JavaScript. func (s *Server) handleStatusHTML() http.HandlerFunc { @@ -570,7 +626,9 @@ func (s *Server) handleStatusHTML() http.HandlerFunc { w.Header().Set("Content-Type", "text/html; charset=utf-8") tmpl := templates.StatusTemplate() - if err := tmpl.Execute(w, nil); err != nil { + + err := tmpl.Execute(w, nil) + if err != nil { s.logger.Error("Failed to render template", "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) } @@ -595,18 +653,9 @@ type IPLookupResponse struct { // Always returns the same response structure with PTR records for each IP. func (s *Server) handleIPInfo() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - // Get IP/hostname from URL param, falling back to client IP - target := chi.URLParam(r, "ip") + target := lookupTarget(r) if target == "" { - target = chi.URLParam(r, "addr") - } - if target == "" { - // Use client IP (RealIP middleware has already processed this) - target = extractClientIP(r) - } - - if target == "" { - writeJSONError(w, http.StatusBadRequest, "Could not determine IP address") + s.writeJSONError(w, http.StatusBadRequest, "Could not determine IP address") return } @@ -618,38 +667,17 @@ func (s *Server) handleIPInfo() http.HandlerFunc { } // Collect all IPs to look up - var ipsToLookup []string + ipsToLookup, resolveErrors := resolveLookupTargets(ctx, target) + if deadlinePassed(ctx) { + s.writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) - // Check if target contains commas (multiple IPs) - targets := strings.Split(target, ",") - for _, t := range targets { - t = strings.TrimSpace(t) - if t == "" { - continue - } - - // Check if this target is an IP address - if parsedIP := net.ParseIP(t); parsedIP != nil { - ipsToLookup = append(ipsToLookup, t) - } else { - // It's a hostname - resolve it - resolved, err := net.DefaultResolver.LookupHost(ctx, t) - if deadlinePassed(ctx) { - writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) - - return - } - if err != nil { - response.Errors = append(response.Errors, t+": "+err.Error()) - - continue - } - ipsToLookup = append(ipsToLookup, resolved...) - } + return } + response.Errors = append(response.Errors, resolveErrors...) + if len(ipsToLookup) == 0 { - writeJSONError(w, http.StatusBadRequest, "No valid IPs or hostnames provided") + s.writeJSONError(w, http.StatusBadRequest, "No valid IPs or hostnames provided") return } @@ -661,10 +689,11 @@ func (s *Server) handleIPInfo() http.HandlerFunc { for _, ip := range ipsToLookup { ipInfo, err := s.db.GetIPInfoContext(ctx, ip) if deadlinePassed(ctx) { - writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) + s.writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) return } + if err != nil { response.Errors = append(response.Errors, ip+": "+err.Error()) @@ -672,17 +701,14 @@ func (s *Server) handleIPInfo() http.HandlerFunc { } // Do PTR lookup for this IP - ptrs, err := net.DefaultResolver.LookupAddr(ctx, ip) + ptrs := lookupPTRNames(ctx, ip) if deadlinePassed(ctx) { - writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) + s.writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) return } - if err == nil && len(ptrs) > 0 { - // Remove trailing dots from PTR records - for i, ptr := range ptrs { - ptrs[i] = strings.TrimSuffix(ptr, ".") - } + + if len(ptrs) > 0 { ipInfo.PTR = ptrs } @@ -694,25 +720,104 @@ func (s *Server) handleIPInfo() http.HandlerFunc { } // Queue WHOIS refresh for stale ASNs (non-blocking) - if s.asnFetcher != nil { - for asn := range refreshASNs { - s.asnFetcher.QueueImmediate(asn) - } - } + s.queueWHOISRefresh(refreshASNs) // Return response (even if no results, include errors) if len(response.Results) == 0 && len(response.Errors) > 0 { - writeJSONError(w, http.StatusNotFound, "No routes found: "+response.Errors[0]) + s.writeJSONError(w, http.StatusNotFound, "No routes found: "+response.Errors[0]) return } - if err := writeJSONSuccess(w, response); err != nil { + err := writeJSONSuccess(w, response) + if err != nil { s.logger.Error("Failed to encode IP lookup response", "error", err) } } } +// lookupTarget returns what an IP lookup request asks for: the ip or addr URL +// parameter, or else the client's own address. +func lookupTarget(r *http.Request) string { + target := chi.URLParam(r, "ip") + if target == "" { + target = chi.URLParam(r, "addr") + } + + if target == "" { + // Use client IP (RealIP middleware has already processed this) + target = extractClientIP(r) + } + + return target +} + +// queueWHOISRefresh asks the WHOIS fetcher, when there is one, to refresh +// each of asns now. +func (s *Server) queueWHOISRefresh(asns map[int]bool) { + if s.asnFetcher == nil { + return + } + + for asn := range asns { + s.asnFetcher.QueueImmediate(asn) + } +} + +// resolveLookupTargets splits a comma-separated list of IP addresses and +// hostnames and returns the IP addresses to look up, resolving each hostname. +// A hostname that does not resolve is reported in the returned errors instead. +// It stops once ctx's deadline has passed; the caller then answers 504. +func resolveLookupTargets(ctx context.Context, target string) ([]string, []string) { + var ipsToLookup, resolveErrors []string + + for t := range strings.SplitSeq(target, ",") { + t = strings.TrimSpace(t) + if t == "" { + continue + } + + // Check if this target is an IP address + if net.ParseIP(t) != nil { + ipsToLookup = append(ipsToLookup, t) + + continue + } + + // It's a hostname - resolve it + resolved, err := net.DefaultResolver.LookupHost(ctx, t) + if deadlinePassed(ctx) { + return nil, nil + } + + if err != nil { + resolveErrors = append(resolveErrors, t+": "+err.Error()) + + continue + } + + ipsToLookup = append(ipsToLookup, resolved...) + } + + return ipsToLookup, resolveErrors +} + +// lookupPTRNames returns the PTR records of ip without their trailing dots, or +// nil when the lookup fails. +func lookupPTRNames(ctx context.Context, ip string) []string { + ptrs, err := net.DefaultResolver.LookupAddr(ctx, ip) + if err != nil { + return nil + } + + // Remove trailing dots from PTR records + for i, ptr := range ptrs { + ptrs[i] = strings.TrimSuffix(ptr, ".") + } + + return ptrs +} + // extractClientIP extracts the client IP from the request. // Works with chi's RealIP middleware which sets RemoteAddr. func extractClientIP(r *http.Request) string { @@ -730,53 +835,86 @@ func extractClientIP(r *http.Request) string { func (s *Server) handleASDetailJSON() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { asnStr := chi.URLParam(r, "asn") + asn, err := strconv.Atoi(asnStr) if err != nil { - writeJSONError(w, http.StatusBadRequest, "Invalid ASN") + s.writeJSONError(w, http.StatusBadRequest, "Invalid ASN") return } asInfo, prefixes, err := s.db.GetASDetailsContext(r.Context(), asn) if deadlinePassed(r.Context()) { - writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) + s.writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) return } + if err != nil { if errors.Is(err, database.ErrNoRoute) { - writeJSONError(w, http.StatusNotFound, err.Error()) + s.writeJSONError(w, http.StatusNotFound, err.Error()) } else { - writeJSONError(w, http.StatusInternalServerError, err.Error()) + s.writeJSONError(w, http.StatusInternalServerError, err.Error()) } return } // Group prefixes by IP version - const ipVersionV4 = 4 - var ipv4Prefixes, ipv6Prefixes []database.LiveRoute - for _, p := range prefixes { - if p.IPVersion == ipVersionV4 { - ipv4Prefixes = append(ipv4Prefixes, p) - } else { - ipv6Prefixes = append(ipv6Prefixes, p) - } - } + ipv4Prefixes, ipv6Prefixes := splitRoutesByIPVersion(prefixes) - response := map[string]interface{}{ + response := map[string]any{ "asn": asInfo, "ipv4_prefixes": ipv4Prefixes, "ipv6_prefixes": ipv6Prefixes, "total_count": len(prefixes), } - if err := writeJSONSuccess(w, response); err != nil { + err = writeJSONSuccess(w, response) + if err != nil { s.logger.Error("Failed to encode AS details", "error", err) } } } +// splitRoutesByIPVersion groups routes into the IPv4 and the IPv6 ones. +func splitRoutesByIPVersion( + routes []database.LiveRoute, +) ([]database.LiveRoute, []database.LiveRoute) { + var ipv4Routes, ipv6Routes []database.LiveRoute + + for _, route := range routes { + if route.IPVersion == ipVersionV4 { + ipv4Routes = append(ipv4Routes, route) + } else { + ipv6Routes = append(ipv6Routes, route) + } + } + + return ipv4Routes, ipv6Routes +} + +// sortRoutesByNetwork sorts routes by network address, then by mask length. +func sortRoutesByNetwork(routes []database.LiveRoute) { + sort.Slice(routes, func(i, j int) bool { + // Parse the prefixes to compare network addresses + ipI, netI, _ := net.ParseCIDR(routes[i].Prefix) + ipJ, netJ, _ := net.ParseCIDR(routes[j].Prefix) + + // Compare by network address first + cmp := bytes.Compare(ipI.To16(), ipJ.To16()) + if cmp != 0 { + return cmp < 0 + } + + // If network addresses are equal, compare by mask length + onesI, _ := netI.Mask.Size() + onesJ, _ := netJ.Mask.Size() + + return onesI < onesJ + }) +} + // handlePrefixDetailJSON returns prefix details as JSON func (s *Server) handlePrefixDetailJSON() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { @@ -785,7 +923,8 @@ func (s *Server) handlePrefixDetailJSON() http.HandlerFunc { lenParam := chi.URLParam(r, "len") if prefixParam == "" || lenParam == "" { - writeJSONError(w, http.StatusBadRequest, "Prefix and length parameters are required") + s.writeJSONError(w, http.StatusBadRequest, + "Prefix and length parameters are required") return } @@ -794,23 +933,25 @@ func (s *Server) handlePrefixDetailJSON() http.HandlerFunc { // tables store, so that any valid way of writing it finds the route. parsed, err := netip.ParsePrefix(prefixParam + "/" + lenParam) if err != nil { - writeJSONError(w, http.StatusBadRequest, err.Error()) + s.writeJSONError(w, http.StatusBadRequest, err.Error()) return } + prefix := parsed.Masked().String() routes, _, err := s.db.GetPrefixDetailsContext(r.Context(), prefix) if deadlinePassed(r.Context()) { - writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) + s.writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage) return } + if err != nil { if errors.Is(err, database.ErrNoRoute) { - writeJSONError(w, http.StatusNotFound, err.Error()) + s.writeJSONError(w, http.StatusNotFound, err.Error()) } else { - writeJSONError(w, http.StatusInternalServerError, err.Error()) + s.writeJSONError(w, http.StatusInternalServerError, err.Error()) } return @@ -822,7 +963,7 @@ func (s *Server) handlePrefixDetailJSON() http.HandlerFunc { originMap[route.OriginASN] = append(originMap[route.OriginASN], route) } - response := map[string]interface{}{ + response := map[string]any{ "prefix": prefix, "routes": routes, "origins": originMap, @@ -830,7 +971,8 @@ func (s *Server) handlePrefixDetailJSON() http.HandlerFunc { "origin_count": len(originMap), } - if err := writeJSONSuccess(w, response); err != nil { + err = writeJSONSuccess(w, response) + if err != nil { s.logger.Error("Failed to encode prefix details", "error", err) } } @@ -840,6 +982,7 @@ func (s *Server) handlePrefixDetailJSON() http.HandlerFunc { func (s *Server) handleASDetail() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { asnStr := chi.URLParam(r, "asn") + asn, err := strconv.Atoi(asnStr) if err != nil { http.Error(w, "Invalid ASN", http.StatusBadRequest) @@ -853,6 +996,7 @@ func (s *Server) handleASDetail() http.HandlerFunc { return } + if err != nil { if errors.Is(err, database.ErrNoRoute) { http.Error(w, "AS not found", http.StatusNotFound) @@ -871,6 +1015,7 @@ func (s *Server) handleASDetail() http.HandlerFunc { return } + if err != nil { s.logger.Error("Failed to get AS peers", "error", err) // Continue without peers rather than failing the whole request @@ -878,52 +1023,11 @@ func (s *Server) handleASDetail() http.HandlerFunc { } // Group prefixes by IP version - const ipVersionV4 = 4 - var ipv4Prefixes, ipv6Prefixes []database.LiveRoute - for _, p := range prefixes { - if p.IPVersion == ipVersionV4 { - ipv4Prefixes = append(ipv4Prefixes, p) - } else { - ipv6Prefixes = append(ipv6Prefixes, p) - } - } + ipv4Prefixes, ipv6Prefixes := splitRoutesByIPVersion(prefixes) // Sort prefixes by network address - sort.Slice(ipv4Prefixes, func(i, j int) bool { - // Parse the prefixes to compare network addresses - ipI, netI, _ := net.ParseCIDR(ipv4Prefixes[i].Prefix) - ipJ, netJ, _ := net.ParseCIDR(ipv4Prefixes[j].Prefix) - - // Compare by network address first - cmp := bytes.Compare(ipI.To4(), ipJ.To4()) - if cmp != 0 { - return cmp < 0 - } - - // If network addresses are equal, compare by mask length - onesI, _ := netI.Mask.Size() - onesJ, _ := netJ.Mask.Size() - - return onesI < onesJ - }) - - sort.Slice(ipv6Prefixes, func(i, j int) bool { - // Parse the prefixes to compare network addresses - ipI, netI, _ := net.ParseCIDR(ipv6Prefixes[i].Prefix) - ipJ, netJ, _ := net.ParseCIDR(ipv6Prefixes[j].Prefix) - - // Compare by network address first - cmp := bytes.Compare(ipI.To16(), ipJ.To16()) - if cmp != 0 { - return cmp < 0 - } - - // If network addresses are equal, compare by mask length - onesI, _ := netI.Mask.Size() - onesJ, _ := netJ.Mask.Size() - - return onesI < onesJ - }) + sortRoutesByNetwork(ipv4Prefixes) + sortRoutesByNetwork(ipv6Prefixes) // Prepare template data data := struct { @@ -947,16 +1051,17 @@ func (s *Server) handleASDetail() http.HandlerFunc { } // Check if context is still valid before writing response - select { - case <-r.Context().Done(): + if r.Context().Err() != nil { // Request was cancelled, don't write response return - default: } w.Header().Set("Content-Type", "text/html; charset=utf-8") + tmpl := templates.ASDetailTemplate() - if err := tmpl.Execute(w, data); err != nil { + + err = tmpl.Execute(w, data) + if err != nil { s.logger.Error("Failed to render AS detail template", "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) } @@ -984,14 +1089,16 @@ func (s *Server) handlePrefixDetail() http.HandlerFunc { return } + prefix := parsed.Masked().String() - routes, originASNs, err := s.db.GetPrefixDetailsContext(r.Context(), prefix) + routes, originNames, err := s.db.GetPrefixDetailsContext(r.Context(), prefix) if deadlinePassed(r.Context()) { http.Error(w, requestTimeoutMessage, http.StatusGatewayTimeout) return } + if err != nil { if errors.Is(err, database.ErrNoRoute) { http.Error(w, "Prefix not found", http.StatusNotFound) @@ -1004,75 +1111,16 @@ func (s *Server) handlePrefixDetail() http.HandlerFunc { } // Group by origin AS and collect unique AS info - type ASNInfo struct { - ASN int - Handle string - Description string - PeerCount int - } - originMap := make(map[int]*ASNInfo) - for _, route := range routes { - if _, exists := originMap[route.OriginASN]; !exists { - originMap[route.OriginASN] = &ASNInfo{ - ASN: route.OriginASN, - Handle: originASNs[route.OriginASN].Handle, - Description: originASNs[route.OriginASN].Description, - PeerCount: 0, - } - } - originMap[route.OriginASN].PeerCount++ - } + origins := originASNs(routes, originNames) // Get the first route to extract some common info var maskLength, ipVersion int if len(routes) > 0 { - // Parse CIDR to get mask length and IP version - _, ipNet, err := net.ParseCIDR(prefix) - if err == nil { - ones, _ := ipNet.Mask.Size() - maskLength = ones - if ipNet.IP.To4() != nil { - ipVersion = 4 - } else { - ipVersion = 6 - } - } - } - - // Convert origin map to sorted slice - var origins []*ASNInfo - for _, origin := range originMap { - origins = append(origins, origin) + maskLength, ipVersion = prefixMaskLengthAndIPVersion(prefix) } // Create enhanced routes with AS path handles - type ASPathEntry struct { - ASN int - Handle string - } - type EnhancedRoute struct { - database.LiveRoute - ASPathWithHandle []ASPathEntry - } - - enhancedRoutes := make([]EnhancedRoute, len(routes)) - for i, route := range routes { - enhancedRoute := EnhancedRoute{ - LiveRoute: route, - ASPathWithHandle: make([]ASPathEntry, len(route.ASPath)), - } - - // Look up handle for each AS in the path - for j, asn := range route.ASPath { - handle := asinfo.GetHandle(asn) - enhancedRoute.ASPathWithHandle[j] = ASPathEntry{ - ASN: asn, - Handle: handle, - } - } - - enhancedRoutes[i] = enhancedRoute - } + enhancedRoutes := routesWithASPathHandles(routes) // Prepare template data data := struct { @@ -1090,7 +1138,7 @@ func (s *Server) handlePrefixDetail() http.HandlerFunc { Routes: enhancedRoutes, Origins: origins, PeerCount: len(routes), - OriginCount: len(originMap), + OriginCount: len(origins), } // Check if context is still valid before writing response @@ -1102,16 +1150,123 @@ func (s *Server) handlePrefixDetail() http.HandlerFunc { } w.Header().Set("Content-Type", "text/html; charset=utf-8") + tmpl := templates.PrefixDetailTemplate() - if err := tmpl.Execute(w, data); err != nil { + + err = tmpl.Execute(w, data) + if err != nil { s.logger.Error("Failed to render prefix detail template", "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) } } } -// handlePrefixLength shows a random sample of IPv4 prefixes with the specified mask length +// ASNInfo is one origin AS of a prefix on the prefix detail page. +type ASNInfo struct { + ASN int + Handle string + Description string + PeerCount int +} + +// originASNs groups routes by origin AS, with each AS's handle and description +// from names, as the prefix query returned them, and the number of routes it +// originates. The origin ASes come in no particular order. +func originASNs( + routes []database.LiveRoute, + names map[int]database.ASN, +) []*ASNInfo { + originMap := make(map[int]*ASNInfo) + + for _, route := range routes { + if _, exists := originMap[route.OriginASN]; !exists { + originMap[route.OriginASN] = &ASNInfo{ + ASN: route.OriginASN, + Handle: names[route.OriginASN].Handle, + Description: names[route.OriginASN].Description, + PeerCount: 0, + } + } + + originMap[route.OriginASN].PeerCount++ + } + + origins := make([]*ASNInfo, 0, len(originMap)) + for _, origin := range originMap { + origins = append(origins, origin) + } + + return origins +} + +// ASPathEntry is one AS of a route's AS path, with its handle. +type ASPathEntry struct { + ASN int + Handle string +} + +// EnhancedRoute is a live route with the handle of each AS in its path. +type EnhancedRoute struct { + database.LiveRoute + + ASPathWithHandle []ASPathEntry +} + +// routesWithASPathHandles looks up the handle of each AS in the path of each +// route. +func routesWithASPathHandles(routes []database.LiveRoute) []EnhancedRoute { + enhancedRoutes := make([]EnhancedRoute, len(routes)) + + for i, route := range routes { + enhancedRoute := EnhancedRoute{ + LiveRoute: route, + ASPathWithHandle: make([]ASPathEntry, len(route.ASPath)), + } + + // Look up handle for each AS in the path + for j, asn := range route.ASPath { + handle := asinfo.GetHandle(asn) + enhancedRoute.ASPathWithHandle[j] = ASPathEntry{ + ASN: asn, + Handle: handle, + } + } + + enhancedRoutes[i] = enhancedRoute + } + + return enhancedRoutes +} + +// prefixMaskLengthAndIPVersion returns the mask length and IP version of a +// prefix in CIDR notation, or zeros if it does not parse. +func prefixMaskLengthAndIPVersion(prefix string) (int, int) { + _, ipNet, err := net.ParseCIDR(prefix) + if err != nil { + return 0, 0 + } + + maskLength, _ := ipNet.Mask.Size() + + if ipNet.IP.To4() != nil { + return maskLength, ipVersionV4 + } + + return maskLength, ipVersionV6 +} + +// handlePrefixLength shows a random sample of IPv4 prefixes with the +// specified mask length func (s *Server) handlePrefixLength() http.HandlerFunc { + const maxIPv4MaskLength = 32 + + return s.prefixLengthHandler(ipVersionV4, maxIPv4MaskLength) +} + +// prefixLengthHandler returns a handler that shows a random sample of the +// prefixes of one IP version with the specified mask length, which may be at +// most maxMaskLength. +func (s *Server) prefixLengthHandler(ipVersion, maxMaskLength int) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { lengthStr := chi.URLParam(r, "length") if lengthStr == "" { @@ -1127,24 +1282,25 @@ func (s *Server) handlePrefixLength() http.HandlerFunc { return } - // Validate IPv4 mask length - const maxIPv4MaskLength = 32 - if maskLength < 0 || maskLength > maxIPv4MaskLength { - http.Error(w, "Invalid IPv4 mask length", http.StatusBadRequest) + // Validate mask length + if maskLength < 0 || maskLength > maxMaskLength { + http.Error(w, "Invalid IPv"+strconv.Itoa(ipVersion)+" mask length", + http.StatusBadRequest) return } - const ipVersion = 4 - // Get random sample of prefixes const maxPrefixes = 500 - prefixes, err := s.db.GetRandomPrefixesByLengthContext(r.Context(), maskLength, ipVersion, maxPrefixes) + + prefixes, err := s.db.GetRandomPrefixesByLengthContext( + r.Context(), maskLength, ipVersion, maxPrefixes) if deadlinePassed(r.Context()) { http.Error(w, requestTimeoutMessage, http.StatusGatewayTimeout) return } + if err != nil { s.logger.Error("Failed to get prefixes by length", "error", err) http.Error(w, "Internal server error", http.StatusInternalServerError) @@ -1162,31 +1318,11 @@ func (s *Server) handlePrefixLength() http.HandlerFunc { return prefixes[i].Prefix < prefixes[j].Prefix }) - // Create enhanced prefixes with AS descriptions - type EnhancedPrefix struct { - database.LiveRoute - OriginASDescription string - Age string - } - - enhancedPrefixes := make([]EnhancedPrefix, len(prefixes)) - for i, prefix := range prefixes { - enhancedPrefixes[i] = EnhancedPrefix{ - LiveRoute: prefix, - Age: formatAge(prefix.LastUpdated), - } - - // Get AS description - if asInfo, ok := asinfo.Get(prefix.OriginASN); ok { - enhancedPrefixes[i].OriginASDescription = asInfo.Description - } - } - // Render template - data := map[string]interface{}{ + data := map[string]any{ "MaskLength": maskLength, "IPVersion": ipVersion, - "Prefixes": enhancedPrefixes, + "Prefixes": prefixesWithASDescriptions(prefixes), "Count": len(prefixes), } @@ -1199,107 +1335,49 @@ func (s *Server) handlePrefixLength() http.HandlerFunc { } tmpl := templates.PrefixLengthTemplate() - if err := tmpl.Execute(w, data); err != nil { + + err = tmpl.Execute(w, data) + if err != nil { s.logger.Error("Failed to render prefix length template", "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) } } } -// handlePrefixLength6 shows a random sample of IPv6 prefixes with the specified mask length -func (s *Server) handlePrefixLength6() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - lengthStr := chi.URLParam(r, "length") - if lengthStr == "" { - http.Error(w, "Length parameter is required", http.StatusBadRequest) +// EnhancedPrefix is a live route on the prefix length page, with the +// description of its origin AS and its age. +type EnhancedPrefix struct { + database.LiveRoute - return + OriginASDescription string + Age string +} + +// prefixesWithASDescriptions looks up the description of the origin AS of +// each prefix and gives each one's age. +func prefixesWithASDescriptions(prefixes []database.LiveRoute) []EnhancedPrefix { + enhancedPrefixes := make([]EnhancedPrefix, len(prefixes)) + for i, prefix := range prefixes { + enhancedPrefixes[i] = EnhancedPrefix{ + LiveRoute: prefix, + Age: formatAge(prefix.LastUpdated), } - maskLength, err := strconv.Atoi(lengthStr) - if err != nil { - http.Error(w, "Invalid mask length", http.StatusBadRequest) - - return - } - - // Validate IPv6 mask length - const maxIPv6MaskLength = 128 - if maskLength < 0 || maskLength > maxIPv6MaskLength { - http.Error(w, "Invalid IPv6 mask length", http.StatusBadRequest) - - return - } - - const ipVersion = 6 - - // Get random sample of prefixes - const maxPrefixes = 500 - prefixes, err := s.db.GetRandomPrefixesByLengthContext(r.Context(), maskLength, ipVersion, maxPrefixes) - if deadlinePassed(r.Context()) { - http.Error(w, requestTimeoutMessage, http.StatusGatewayTimeout) - - return - } - if err != nil { - s.logger.Error("Failed to get prefixes by length", "error", err) - http.Error(w, "Internal server error", http.StatusInternalServerError) - - return - } - - // Sort prefixes for display - sort.Slice(prefixes, func(i, j int) bool { - // First compare by IP version - if prefixes[i].IPVersion != prefixes[j].IPVersion { - return prefixes[i].IPVersion < prefixes[j].IPVersion - } - // Then by prefix - return prefixes[i].Prefix < prefixes[j].Prefix - }) - - // Create enhanced prefixes with AS descriptions - type EnhancedPrefix struct { - database.LiveRoute - OriginASDescription string - Age string - } - - enhancedPrefixes := make([]EnhancedPrefix, len(prefixes)) - for i, prefix := range prefixes { - enhancedPrefixes[i] = EnhancedPrefix{ - LiveRoute: prefix, - Age: formatAge(prefix.LastUpdated), - } - - // Get AS description - if asInfo, ok := asinfo.Get(prefix.OriginASN); ok { - enhancedPrefixes[i].OriginASDescription = asInfo.Description - } - } - - // Render template - data := map[string]interface{}{ - "MaskLength": maskLength, - "IPVersion": ipVersion, - "Prefixes": enhancedPrefixes, - "Count": len(prefixes), - } - - // Check if context is still valid before writing response - select { - case <-r.Context().Done(): - // Request was cancelled, don't write response - return - default: - } - - tmpl := templates.PrefixLengthTemplate() - if err := tmpl.Execute(w, data); err != nil { - s.logger.Error("Failed to render prefix length template", "error", err) - http.Error(w, "Internal Server Error", http.StatusInternalServerError) + // Get AS description + if asInfo, ok := asinfo.Get(prefix.OriginASN); ok { + enhancedPrefixes[i].OriginASDescription = asInfo.Description } } + + return enhancedPrefixes +} + +// handlePrefixLength6 shows a random sample of IPv6 prefixes with the +// specified mask length +func (s *Server) handlePrefixLength6() http.HandlerFunc { + const maxIPv6MaskLength = 128 + + return s.prefixLengthHandler(ipVersionV6, maxIPv6MaskLength) } // formatAge returns a human-readable age string @@ -1308,13 +1386,14 @@ func formatAge(timestamp time.Time) string { const hoursPerDay = 24 - if age < time.Minute { + switch { + case age < time.Minute: return "< 1m" - } else if age < time.Hour { + case age < time.Hour: minutes := int(age.Minutes()) return strconv.Itoa(minutes) + "m" - } else if age < hoursPerDay*time.Hour { + case age < hoursPerDay*time.Hour: hours := int(age.Hours()) return strconv.Itoa(hours) + "h" diff --git a/internal/server/handlers_test.go b/internal/server/handlers_internal_test.go similarity index 70% rename from internal/server/handlers_test.go rename to internal/server/handlers_internal_test.go index 1691c0f..7932601 100644 --- a/internal/server/handlers_test.go +++ b/internal/server/handlers_internal_test.go @@ -22,12 +22,26 @@ import ( "github.com/google/uuid" ) +// The prefix and peer addresses the prefix page tests store routes for. +const ( + testPrefixV4 = "192.0.2.0/24" + testPeerV4 = "198.51.100.1" + testPeerV6 = "2001:db8::1" +) + +// The names the stats handler tests report the two stats handlers by. +const ( + statusJSONHandlerName = "status.json" + statsHandlerName = "stats" +) + // blockingStatsDB embeds database.Store (left nil) and overrides only // GetStatsContext, which blocks until release is closed. The stats handlers // call it in a goroutine; every other Store method is unused on the timeout // path and would panic if called. type blockingStatsDB struct { database.Store + release chan struct{} } @@ -43,14 +57,16 @@ func (d blockingStatsDB) GetStatsContext(_ context.Context) (database.Stats, err // value. Before the fix the per-request goroutine sent on an unbuffered channel // that nothing received once the timeout won, so it blocked forever and every // poll leaked one goroutine. +// +//nolint:paralleltest // counts goroutines, so no other test may run alongside func TestStatsHandlersDoNotLeakOnTimeout(t *testing.T) { release := make(chan struct{}) db := blockingStatsDB{release: release} s := New(db, streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{}) handlers := map[string]http.HandlerFunc{ - "status.json": s.handleStatusJSON(), - "stats": s.handleStats(), + statusJSONHandlerName: s.handleStatusJSON(), + statsHandlerName: s.handleStats(), } baseline := settledGoroutineCount() @@ -62,7 +78,7 @@ func TestStatsHandlersDoNotLeakOnTimeout(t *testing.T) { for _, handler := range handlers { for range iterations { ctx, cancel := context.WithTimeout(context.Background(), requestTimeout) - req := httptest.NewRequest(http.MethodGet, "/", nil).WithContext(ctx) + req := httptest.NewRequestWithContext(ctx, http.MethodGet, "/", nil) handler(httptest.NewRecorder(), req) cancel() } @@ -87,67 +103,83 @@ func TestStatsHandlersDoNotLeakOnTimeout(t *testing.T) { // The oldest and newest route times still come from one-row lookups at the ends // of an index; with the database closed they are left out of the answer. func TestStatsHandlersAnswerFromMemory(t *testing.T) { + t.Parallel() + db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New()) if err != nil { t.Fatalf("database.New: %v", err) } ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) - if err := db.UpsertLiveRouteBatch([]*database.LiveRoute{ + + err = db.UpsertLiveRouteBatch([]*database.LiveRoute{ { ID: uuid.New(), Prefix: "198.51.100.0/24", MaskLength: 24, IPVersion: 4, - OriginASN: 64500, PeerIP: "192.0.2.1", ASPath: []int{64500}, NextHop: "192.0.2.1", - LastUpdated: ts, + OriginASN: 64500, PeerIP: "192.0.2.1", ASPath: []int{64500}, + NextHop: "192.0.2.1", LastUpdated: ts, }, { ID: uuid.New(), Prefix: "2001:db8::/32", MaskLength: 32, IPVersion: 6, - OriginASN: 64501, PeerIP: "2001:db8::1", ASPath: []int{64501}, NextHop: "2001:db8::1", - LastUpdated: ts, + OriginASN: 64501, PeerIP: testPeerV6, ASPath: []int{64501}, + NextHop: testPeerV6, LastUpdated: ts, }, - }); err != nil { + }) + if err != nil { t.Fatalf("UpsertLiveRouteBatch: %v", err) } - if err := db.Close(); err != nil { + + err = db.Close() + if err != nil { t.Fatalf("Close: %v", err) } s := New(db, streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{}) handlers := map[string]http.HandlerFunc{ - "status.json": s.handleStatusJSON(), - "stats": s.handleStats(), + statusJSONHandlerName: s.handleStatusJSON(), + statsHandlerName: s.handleStats(), } for name, handler := range handlers { rec := httptest.NewRecorder() - handler(rec, httptest.NewRequest(http.MethodGet, "/", nil)) + handler(rec, httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil)) + if rec.Code != http.StatusOK { - t.Errorf("%s: status %d, want %d; body %s", name, rec.Code, http.StatusOK, rec.Body) + t.Errorf("%s: status %d, want %d; body %s", + name, rec.Code, http.StatusOK, rec.Body) continue } + //nolint:tagliatelle // field names are the existing JSON API var body struct { Data struct { - IPv4Routes int `json:"ipv4_routes"` - IPv6Routes int `json:"ipv6_routes"` - IPv4PrefixDistribution []database.PrefixDistribution `json:"ipv4_prefix_distribution"` - IPv6PrefixDistribution []database.PrefixDistribution `json:"ipv6_prefix_distribution"` + IPv4Routes int `json:"ipv4_routes"` + IPv6Routes int `json:"ipv6_routes"` + IPv4Distribution []database.PrefixDistribution `json:"ipv4_prefix_distribution"` + IPv6Distribution []database.PrefixDistribution `json:"ipv6_prefix_distribution"` } `json:"data"` } - if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + + err := json.Unmarshal(rec.Body.Bytes(), &body) + if err != nil { t.Fatalf("%s: decoding the answer: %v", name, err) } if body.Data.IPv4Routes != 1 || body.Data.IPv6Routes != 1 { - t.Errorf("%s: routes = (v4 %d, v6 %d), want (1, 1)", name, body.Data.IPv4Routes, body.Data.IPv6Routes) + t.Errorf("%s: routes = (v4 %d, v6 %d), want (1, 1)", + name, body.Data.IPv4Routes, body.Data.IPv6Routes) } + wantV4 := []database.PrefixDistribution{{MaskLength: 24, Count: 1}} - if !slices.Equal(body.Data.IPv4PrefixDistribution, wantV4) { - t.Errorf("%s: IPv4 distribution = %v, want %v", name, body.Data.IPv4PrefixDistribution, wantV4) + if !slices.Equal(body.Data.IPv4Distribution, wantV4) { + t.Errorf("%s: IPv4 distribution = %v, want %v", + name, body.Data.IPv4Distribution, wantV4) } + wantV6 := []database.PrefixDistribution{{MaskLength: 32, Count: 1}} - if !slices.Equal(body.Data.IPv6PrefixDistribution, wantV6) { - t.Errorf("%s: IPv6 distribution = %v, want %v", name, body.Data.IPv6PrefixDistribution, wantV6) + if !slices.Equal(body.Data.IPv6Distribution, wantV6) { + t.Errorf("%s: IPv6 distribution = %v, want %v", + name, body.Data.IPv6Distribution, wantV6) } } } @@ -159,25 +191,30 @@ func TestStatsHandlersAnswerFromMemory(t *testing.T) { // "2001:DB8:0:0::/32" was not found although "2001:db8::/32" was stored, and // "foo/24" answered 500 (https://git.eeqj.de/sneak/routewatch/issues/71). func TestPrefixDetailHandlersParsePrefix(t *testing.T) { + t.Parallel() + db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New()) if err != nil { t.Fatalf("database.New: %v", err) } + t.Cleanup(func() { _ = db.Close() }) ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) - if err := db.UpsertLiveRouteBatch([]*database.LiveRoute{ + + err = db.UpsertLiveRouteBatch([]*database.LiveRoute{ { - ID: uuid.New(), Prefix: "192.0.2.0/24", MaskLength: 24, IPVersion: 4, - OriginASN: 64500, PeerIP: "198.51.100.1", ASPath: []int{64500}, NextHop: "198.51.100.1", - LastUpdated: ts, + ID: uuid.New(), Prefix: testPrefixV4, MaskLength: 24, IPVersion: 4, + OriginASN: 64500, PeerIP: testPeerV4, ASPath: []int{64500}, + NextHop: testPeerV4, LastUpdated: ts, }, { ID: uuid.New(), Prefix: "2001:db8::/32", MaskLength: 32, IPVersion: 6, - OriginASN: 64501, PeerIP: "2001:db8::1", ASPath: []int{64501}, NextHop: "2001:db8::1", - LastUpdated: ts, + OriginASN: 64501, PeerIP: testPeerV6, ASPath: []int{64501}, + NextHop: testPeerV6, LastUpdated: ts, }, - }); err != nil { + }) + if err != nil { t.Fatalf("UpsertLiveRouteBatch: %v", err) } @@ -197,20 +234,29 @@ func TestPrefixDetailHandlersParsePrefix(t *testing.T) { for _, tt := range tests { rec := httptest.NewRecorder() - s.router.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, tt.path, nil)) + s.router.ServeHTTP(rec, httptest.NewRequestWithContext( + t.Context(), http.MethodGet, tt.path, nil)) + if rec.Code != tt.want { - t.Errorf("%s: status %d, want %d; body %s", tt.path, rec.Code, tt.want, rec.Body) + t.Errorf("%s: status %d, want %d; body %s", + tt.path, rec.Code, tt.want, rec.Body) } } } +// errASDetailsLookupFailed is what failingASDetailsDB answers every AS details +// lookup with. +var errASDetailsLookupFailed = errors.New("AS details lookup failed") + // failingASDetailsDB wraps a real store and fails every AS details lookup. type failingASDetailsDB struct { database.Store } -func (failingASDetailsDB) GetASDetailsContext(_ context.Context, _ int) (*database.ASN, []database.LiveRoute, error) { - return nil, nil, errors.New("AS details lookup failed") +func (failingASDetailsDB) GetASDetailsContext( + _ context.Context, _ int, +) (*database.ASN, []database.LiveRoute, error) { + return nil, nil, errASDetailsLookupFailed } // TestPrefixDetailShowsOriginNames checks that the prefix page shows each @@ -220,51 +266,66 @@ func (failingASDetailsDB) GetASDetailsContext(_ context.Context, _ int) (*databa // (https://git.eeqj.de/sneak/routewatch/issues/67). The AS path column shows // handles too, so the test looks for the origin list's "(handle)" form. func TestPrefixDetailShowsOriginNames(t *testing.T) { + t.Parallel() + db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New()) if err != nil { t.Fatalf("database.New: %v", err) } + t.Cleanup(func() { _ = db.Close() }) ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) origins := []int{15169, 13335} + for _, asn := range origins { - if _, err := db.GetOrCreateASN(asn, ts); err != nil { + _, err := db.GetOrCreateASN(asn, ts) + if err != nil { t.Fatalf("GetOrCreateASN(%d): %v", asn, err) } } - if err := db.UpsertLiveRouteBatch([]*database.LiveRoute{ + + err = db.UpsertLiveRouteBatch([]*database.LiveRoute{ { - ID: uuid.New(), Prefix: "192.0.2.0/24", MaskLength: 24, IPVersion: 4, - OriginASN: origins[0], PeerIP: "198.51.100.1", ASPath: []int{origins[0]}, NextHop: "198.51.100.1", - LastUpdated: ts, + ID: uuid.New(), Prefix: testPrefixV4, MaskLength: 24, IPVersion: 4, + OriginASN: origins[0], PeerIP: testPeerV4, + ASPath: []int{origins[0]}, NextHop: testPeerV4, LastUpdated: ts, }, { - ID: uuid.New(), Prefix: "192.0.2.0/24", MaskLength: 24, IPVersion: 4, - OriginASN: origins[1], PeerIP: "198.51.100.2", ASPath: []int{origins[1]}, NextHop: "198.51.100.2", - LastUpdated: ts, + ID: uuid.New(), Prefix: testPrefixV4, MaskLength: 24, IPVersion: 4, + OriginASN: origins[1], PeerIP: "198.51.100.2", + ASPath: []int{origins[1]}, NextHop: "198.51.100.2", LastUpdated: ts, }, - }); err != nil { + }) + if err != nil { t.Fatalf("UpsertLiveRouteBatch: %v", err) } - s := New(failingASDetailsDB{db}, streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{}) + s := New(failingASDetailsDB{db}, streamer.New(logger.New(), metrics.New()), + logger.New(), &config.Config{}) rec := httptest.NewRecorder() - s.router.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/prefix/192.0.2.0/24", nil)) + s.router.ServeHTTP(rec, httptest.NewRequestWithContext( + t.Context(), http.MethodGet, "/prefix/"+testPrefixV4, nil)) + if rec.Code != http.StatusOK { t.Fatalf("status %d, want %d; body %s", rec.Code, http.StatusOK, rec.Body) } page := rec.Body.String() + for _, asn := range origins { info, ok := asinfo.Get(asn) if !ok { t.Fatalf("AS%d is not in the AS info data", asn) } - if want := "(" + html.EscapeString(info.Handle) + ")"; !strings.Contains(page, want) { + + want := "(" + html.EscapeString(info.Handle) + ")" + if !strings.Contains(page, want) { t.Errorf("page does not show AS%d's handle %q", asn, want) } - if want := html.EscapeString(info.Description); !strings.Contains(page, want) { + + want = html.EscapeString(info.Description) + if !strings.Contains(page, want) { t.Errorf("page does not show AS%d's description %q", asn, want) } } @@ -282,14 +343,18 @@ const shortRequestDeadline = 50 * time.Millisecond // written by the time limit middleware from its own goroutine while the // handler could still be writing. func TestRequestPastTimeLimitAnswers504(t *testing.T) { - s := New(nil, streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{}) + t.Parallel() + + s := New(nil, streamer.New(logger.New(), metrics.New()), logger.New(), + &config.Config{}) waitForTimeLimit := func(_ http.ResponseWriter, r *http.Request) { <-r.Context().Done() } // Registered the way setupRoutes registers pages and JSON routes. s.router.Get("/slow-page", waitForTimeLimit) - s.router.Get("/slow.json", JSONValidationMiddleware(http.HandlerFunc(waitForTimeLimit)).ServeHTTP) + s.router.Get("/slow.json", + JSONValidationMiddleware(http.HandlerFunc(waitForTimeLimit)).ServeHTTP) tests := []struct { path string @@ -300,14 +365,18 @@ func TestRequestPastTimeLimitAnswers504(t *testing.T) { } for _, tt := range tests { - ctx, cancel := context.WithTimeout(context.Background(), shortRequestDeadline) + ctx, cancel := context.WithTimeout(context.Background(), + shortRequestDeadline) rec := httptest.NewRecorder() - s.router.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, tt.path, nil).WithContext(ctx)) + s.router.ServeHTTP(rec, + httptest.NewRequestWithContext(ctx, http.MethodGet, tt.path, nil)) cancel() if rec.Code != http.StatusGatewayTimeout { - t.Errorf("%s: status %d, want %d; body %s", tt.path, rec.Code, http.StatusGatewayTimeout, rec.Body) + t.Errorf("%s: status %d, want %d; body %s", + tt.path, rec.Code, http.StatusGatewayTimeout, rec.Body) } + if tt.wantJSONBody { checkJSONErrorCode(t, tt.path, rec, http.StatusGatewayTimeout) } @@ -320,25 +389,32 @@ func TestRequestPastTimeLimitAnswers504(t *testing.T) { // body. Before the fix status.json answered 408, and /api/v1/stats wrote // nothing, which the JSON check on its route turned into 500. func TestStatsHandlersAnswer504WhenDatabaseReadRunsOut(t *testing.T) { + t.Parallel() + release := make(chan struct{}) defer close(release) + db := blockingStatsDB{release: release} - s := New(db, streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{}) + s := New(db, streamer.New(logger.New(), metrics.New()), logger.New(), + &config.Config{}) handlers := map[string]http.HandlerFunc{ - "status.json": s.handleStatusJSON(), - "stats": s.handleStats(), + statusJSONHandlerName: s.handleStatusJSON(), + statsHandlerName: s.handleStats(), } for name, handler := range handlers { - ctx, cancel := context.WithTimeout(context.Background(), shortRequestDeadline) + ctx, cancel := context.WithTimeout(context.Background(), + shortRequestDeadline) rec := httptest.NewRecorder() - handler(rec, httptest.NewRequest(http.MethodGet, "/", nil).WithContext(ctx)) + handler(rec, httptest.NewRequestWithContext(ctx, http.MethodGet, "/", nil)) cancel() if rec.Code != http.StatusGatewayTimeout { - t.Errorf("%s: status %d, want %d; body %s", name, rec.Code, http.StatusGatewayTimeout, rec.Body) + t.Errorf("%s: status %d, want %d; body %s", + name, rec.Code, http.StatusGatewayTimeout, rec.Body) } + checkJSONErrorCode(t, name, rec, http.StatusGatewayTimeout) } } @@ -360,25 +436,33 @@ func (slowDB) GetStatsContext(ctx context.Context) (database.Stats, error) { return database.Stats{}, nil } -func (slowDB) GetIPInfoContext(ctx context.Context, _ string) (*database.IPInfo, error) { +func (slowDB) GetIPInfoContext( + ctx context.Context, _ string, +) (*database.IPInfo, error) { <-ctx.Done() return nil, ctx.Err() } -func (slowDB) GetASDetailsContext(ctx context.Context, asn int) (*database.ASN, []database.LiveRoute, error) { +func (slowDB) GetASDetailsContext( + ctx context.Context, asn int, +) (*database.ASN, []database.LiveRoute, error) { <-ctx.Done() return &database.ASN{ASN: asn}, nil, nil } -func (slowDB) GetPrefixDetailsContext(ctx context.Context, _ string) ([]database.LiveRoute, map[int]database.ASN, error) { +func (slowDB) GetPrefixDetailsContext( + ctx context.Context, _ string, +) ([]database.LiveRoute, map[int]database.ASN, error) { <-ctx.Done() return nil, nil, database.ErrNoRoute } -func (slowDB) GetRandomPrefixesByLengthContext(ctx context.Context, _, _, _ int) ([]database.LiveRoute, error) { +func (slowDB) GetRandomPrefixesByLengthContext( + ctx context.Context, _, _, _ int, +) ([]database.LiveRoute, error) { <-ctx.Done() return nil, nil @@ -393,7 +477,10 @@ func (slowDB) GetRandomPrefixesByLengthContext(ctx context.Context, _, _, _ int) // body. The health check is left out: it answers 503 when its own 2-second // ping runs out. func TestRoutesAnswer504WhenDatabaseReadIsCutOff(t *testing.T) { - s := New(slowDB{}, streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{}) + t.Parallel() + + s := New(slowDB{}, streamer.New(logger.New(), metrics.New()), logger.New(), + &config.Config{}) tests := []struct { path string @@ -425,11 +512,15 @@ type slowSecondReadDB struct { database.Store } -func (slowSecondReadDB) GetASDetailsContext(_ context.Context, asn int) (*database.ASN, []database.LiveRoute, error) { +func (slowSecondReadDB) GetASDetailsContext( + _ context.Context, asn int, +) (*database.ASN, []database.LiveRoute, error) { return &database.ASN{ASN: asn}, nil, nil } -func (slowSecondReadDB) GetASPeersContext(ctx context.Context, _ int) ([]database.ASPeer, error) { +func (slowSecondReadDB) GetASPeersContext( + ctx context.Context, _ int, +) ([]database.ASPeer, error) { <-ctx.Done() return nil, ctx.Err() @@ -439,7 +530,9 @@ func (slowSecondReadDB) GetStatsContext(_ context.Context) (database.Stats, erro return database.Stats{}, nil } -func (slowSecondReadDB) GetWHOISStats(ctx context.Context, _ time.Duration) (*database.WHOISStats, error) { +func (slowSecondReadDB) GetWHOISStats( + ctx context.Context, _ time.Duration, +) (*database.WHOISStats, error) { <-ctx.Done() return nil, ctx.Err() @@ -462,7 +555,10 @@ func (idleASNFetcher) GetStats() ASNFetcherStats { // message. Without the check after the second read, the AS page answers 504 // with an empty body and the stats routes answer 200 without the WHOIS stats. func TestRoutesAnswer504WhenSecondDatabaseReadIsCutOff(t *testing.T) { - s := New(slowSecondReadDB{}, streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{}) + t.Parallel() + + s := New(slowSecondReadDB{}, streamer.New(logger.New(), metrics.New()), + logger.New(), &config.Config{}) s.SetASNFetcher(idleASNFetcher{}) tests := []struct { @@ -482,27 +578,40 @@ func TestRoutesAnswer504WhenSecondDatabaseReadIsCutOff(t *testing.T) { // checkRouteAnswers504 sends a request for path through the server's router // with a short deadline and checks that it answers 504 with the time limit // message: the JSON error body on JSON routes, the plain message on pages. -func checkRouteAnswers504(t *testing.T, s *Server, path string, wantJSONBody bool) { +func checkRouteAnswers504( + t *testing.T, s *Server, path string, wantJSONBody bool, +) { t.Helper() ctx, cancel := context.WithTimeout(context.Background(), shortRequestDeadline) defer cancel() + rec := httptest.NewRecorder() - s.router.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil).WithContext(ctx)) + s.router.ServeHTTP(rec, + httptest.NewRequestWithContext(ctx, http.MethodGet, path, nil)) if rec.Code != http.StatusGatewayTimeout { - t.Errorf("%s: status %d, want %d; body %s", path, rec.Code, http.StatusGatewayTimeout, rec.Body) + t.Errorf("%s: status %d, want %d; body %s", + path, rec.Code, http.StatusGatewayTimeout, rec.Body) } + if wantJSONBody { checkJSONErrorCode(t, path, rec, http.StatusGatewayTimeout) - } else if body := strings.TrimSpace(rec.Body.String()); body != requestTimeoutMessage { + + return + } + + body := strings.TrimSpace(rec.Body.String()) + if body != requestTimeoutMessage { t.Errorf("%s: body %q, want %q", path, body, requestTimeoutMessage) } } // checkJSONErrorCode checks that rec holds the JSON error body with the given // code. -func checkJSONErrorCode(t *testing.T, name string, rec *httptest.ResponseRecorder, want int) { +func checkJSONErrorCode( + t *testing.T, name string, rec *httptest.ResponseRecorder, want int, +) { t.Helper() var body struct { @@ -511,13 +620,17 @@ func checkJSONErrorCode(t *testing.T, name string, rec *httptest.ResponseRecorde Code int `json:"code"` } `json:"error"` } - if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + + err := json.Unmarshal(rec.Body.Bytes(), &body) + if err != nil { t.Errorf("%s: decoding the answer %q: %v", name, rec.Body, err) return } - if body.Status != "error" || body.Error.Code != want { - t.Errorf("%s: answer %s, want status \"error\" and error code %d", name, rec.Body, want) + + if body.Status != jsonStatusError || body.Error.Code != want { + t.Errorf("%s: answer %s, want status \"error\" and error code %d", + name, rec.Body, want) } } @@ -525,12 +638,15 @@ func checkJSONErrorCode(t *testing.T, name string, rec *httptest.ResponseRecorde // current count. func settledGoroutineCount() int { prev := runtime.NumGoroutine() + for range 20 { time.Sleep(10 * time.Millisecond) + cur := runtime.NumGoroutine() if cur == prev { return cur } + prev = cur } @@ -543,6 +659,7 @@ func waitForGoroutines(target int) bool { if runtime.NumGoroutine() <= target { return true } + time.Sleep(10 * time.Millisecond) } diff --git a/internal/server/middleware.go b/internal/server/middleware.go index bd336fb..d2a185d 100644 --- a/internal/server/middleware.go +++ b/internal/server/middleware.go @@ -15,6 +15,7 @@ import ( // responseWriter wraps http.ResponseWriter to capture the response type responseWriter struct { http.ResponseWriter + body *bytes.Buffer statusCode int written bool @@ -87,8 +88,10 @@ func JSONResponseMiddleware(next http.Handler) http.Handler { } // Parse the original response - var originalResponse map[string]interface{} - if err := json.Unmarshal(rw.body.Bytes(), &originalResponse); err != nil { + var originalResponse map[string]any + + err := json.Unmarshal(rw.body.Bytes(), &originalResponse) + if err != nil { // If we can't parse it, just write original w.WriteHeader(rw.statusCode) _, _ = w.Write(rw.body.Bytes()) @@ -97,7 +100,7 @@ func JSONResponseMiddleware(next http.Handler) http.Handler { } // Add @meta field - originalResponse["@meta"] = map[string]interface{}{ + originalResponse["@meta"] = map[string]any{ "time_zone": "UTC", "api_version": 1, "execution_time": fmt.Sprintf("%d ms", execTime.Milliseconds()), @@ -160,8 +163,10 @@ func JSONValidationMiddleware(next http.Handler) http.Handler { // For JSON responses, validate the JSON if rw.body.Len() > 0 { - var testParse interface{} - if err := json.Unmarshal(rw.body.Bytes(), &testParse); err == nil { + var testParse any + + err := json.Unmarshal(rw.body.Bytes(), &testParse) + if err == nil { // Valid JSON, write it out w.Header().Set("Content-Type", "application/json") w.WriteHeader(rw.statusCode) @@ -180,9 +185,11 @@ func JSONValidationMiddleware(next http.Handler) http.Handler { if statusCode == http.StatusOK { statusCode = http.StatusInternalServerError } + if deadlinePassed(r.Context()) { statusCode = http.StatusGatewayTimeout } + w.WriteHeader(statusCode) errorMsg := "Internal server error" @@ -190,11 +197,11 @@ func JSONValidationMiddleware(next http.Handler) http.Handler { errorMsg = requestTimeoutMessage } - response := map[string]interface{}{ - "status": "error", - "error": map[string]interface{}{ - "msg": errorMsg, - "code": statusCode, + response := map[string]any{ + jsonKeyStatus: jsonStatusError, + jsonKeyError: map[string]any{ + jsonKeyMsg: errorMsg, + jsonKeyCode: statusCode, }, } _ = json.NewEncoder(w).Encode(response) @@ -204,6 +211,7 @@ func JSONValidationMiddleware(next http.Handler) http.Handler { // statusWriter wraps http.ResponseWriter to capture the status code type statusWriter struct { http.ResponseWriter + statusCode int written bool } @@ -213,6 +221,7 @@ func (sw *statusWriter) WriteHeader(statusCode int) { sw.statusCode = statusCode sw.written = true } + sw.ResponseWriter.WriteHeader(statusCode) } @@ -252,11 +261,12 @@ func RequestLoggerMiddleware(logger *slog.Logger) func(http.Handler) http.Handle // Slow query threshold (1 second) const slowQueryThreshold = 1 * time.Second - if sw.statusCode >= http.StatusInternalServerError { + switch { + case sw.statusCode >= http.StatusInternalServerError: logLevel = slog.LevelError - } else if sw.statusCode >= http.StatusBadRequest { + case sw.statusCode >= http.StatusBadRequest: logLevel = slog.LevelWarn - } else if duration >= slowQueryThreshold { + case duration >= slowQueryThreshold: logLevel = slog.LevelWarn } diff --git a/internal/server/routes.go b/internal/server/routes.go index 9227772..aca1005 100644 --- a/internal/server/routes.go +++ b/internal/server/routes.go @@ -27,7 +27,10 @@ func (s *Server) setupRoutes() { r.Get("/", s.handleIndex()) r.Get("/status", s.handleStatusHTML()) r.Get("/status.json", JSONValidationMiddleware(s.handleStatusJSON()).ServeHTTP) - r.Get("/.well-known/healthcheck.json", JSONValidationMiddleware(s.handleHealthCheck()).ServeHTTP) + r.Get( + "/.well-known/healthcheck.json", + JSONValidationMiddleware(s.handleHealthCheck()).ServeHTTP, + ) // AS and prefix detail pages r.Get("/as/{asn}", s.handleASDetail()) diff --git a/internal/server/server.go b/internal/server/server.go index 16ba2b3..d709b7a 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -3,6 +3,7 @@ package server import ( "context" + "errors" "fmt" "net" "net/http" @@ -42,7 +43,12 @@ type Server struct { } // New creates a new HTTP server -func New(db database.Store, streamer *streamer.Streamer, logger *logger.Logger, cfg *config.Config) *Server { +func New( + db database.Store, + streamer *streamer.Streamer, + logger *logger.Logger, + cfg *config.Config, +) *Server { s := &Server{ db: db, streamer: streamer, @@ -57,8 +63,8 @@ func New(db database.Store, streamer *streamer.Streamer, logger *logger.Logger, // Start listens on the configured port and serves HTTP in the background. // It returns an error if it cannot listen, for example because the port is -// already in use. -func (s *Server) Start() error { +// already in use. ctx bounds only opening the port, not serving. +func (s *Server) Start(ctx context.Context) error { const ( network = "tcp" readHeaderTimeout = 40 * time.Second @@ -78,7 +84,9 @@ func (s *Server) Start() error { s.logger.Info("Starting HTTP server", "port", s.port, "addr", s.srv.Addr) - listener, err := net.Listen(network, s.srv.Addr) + var listenConfig net.ListenConfig + + listener, err := listenConfig.Listen(ctx, network, s.srv.Addr) if err != nil { return fmt.Errorf("starting HTTP server: %w", err) } @@ -86,7 +94,8 @@ func (s *Server) Start() error { s.logger.Info("HTTP server listening", "addr", listener.Addr().String()) go func() { - if err := s.srv.Serve(listener); err != nil && err != http.ErrServerClosed { + err := s.srv.Serve(listener) + if err != nil && !errors.Is(err, http.ErrServerClosed) { s.logger.Error("HTTP server error", "error", err) } }() diff --git a/internal/server/server_test.go b/internal/server/server_test.go index ff7c7d2..1f6964b 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -1,4 +1,4 @@ -package server +package server_test import ( "context" @@ -8,6 +8,7 @@ import ( "git.eeqj.de/sneak/routewatch/internal/config" "git.eeqj.de/sneak/routewatch/internal/logger" "git.eeqj.de/sneak/routewatch/internal/metrics" + "git.eeqj.de/sneak/routewatch/internal/server" "git.eeqj.de/sneak/routewatch/internal/streamer" ) @@ -16,18 +17,31 @@ import ( // listened in a goroutine, only logged the failure and returned nil, so the // daemon kept running without its web server. func TestStartFailsWhenPortIsHeld(t *testing.T) { - held, err := net.Listen("tcp", ":0") + t.Parallel() + + var listenConfig net.ListenConfig + + held, err := listenConfig.Listen(t.Context(), "tcp", ":0") if err != nil { t.Fatalf("failed to open the test listener: %v", err) } defer func() { _ = held.Close() }() - port := held.Addr().(*net.TCPAddr).Port + addr, ok := held.Addr().(*net.TCPAddr) + if !ok { + t.Fatalf("test listener address %v is not a TCP address", held.Addr()) + } + + port := addr.Port log := logger.New() - s := New(nil, streamer.New(log, metrics.New()), log, &config.Config{Port: port}) + s := server.New(nil, streamer.New(log, metrics.New()), log, + &config.Config{Port: port}) + defer func() { _ = s.Stop(context.Background()) }() - if err := s.Start(); err == nil { - t.Fatalf("Start returned nil while port %d was held by another listener", port) + err = s.Start(t.Context()) + if err == nil { + t.Fatalf("Start returned nil while port %d was held by another listener", + port) } } diff --git a/internal/streamer/streamer.go b/internal/streamer/streamer.go index 7c475c4..4b43ff1 100644 --- a/internal/streamer/streamer.go +++ b/internal/streamer/streamer.go @@ -1,5 +1,6 @@ -// Package streamer implements an HTTP client that connects to the RIPE RIS Live streaming API, -// parses BGP UPDATE messages from the JSON stream, and dispatches them to registered handlers. +// Package streamer implements an HTTP client that connects to the RIPE RIS Live +// streaming API, parses BGP UPDATE messages from the JSON stream, and +// dispatches them to registered handlers. package streamer import ( @@ -25,50 +26,60 @@ import ( // countingReader wraps an io.Reader and counts bytes read type countingReader struct { reader io.Reader - count int64 + count atomic.Int64 } // Read implements io.Reader and counts bytes func (c *countingReader) Read(p []byte) (int, error) { n, err := c.reader.Read(p) - atomic.AddInt64(&c.count, int64(n)) + c.count.Add(int64(n)) return n, err } // Count returns the total bytes read func (c *countingReader) Count() int64 { - return atomic.LoadInt64(&c.count) + return c.count.Load() } // Configuration constants for the RIS Live streamer. const ( risLiveURL = "https://ris-live.ripe.net/v1/stream/?format=json&" + "client=https%3A%2F%2Fgit.eeqj.de%2Fsneak%2Froutewatch" - metricsWindowSize = 60 // seconds for rolling average - metricsUpdateRate = time.Second - minBackoffDelay = 5 * time.Second - maxBackoffDelay = 320 * time.Second - streamIdleTimeout = 60 * time.Second // Reconnect when no line arrives for this long - metricsLogInterval = 10 * time.Second - bytesPerKB = 1024 - bytesPerMB = 1024 * 1024 - maxLineSize = bytesPerMB // Well above an extended BGP message (RFC 8654) in hex plus its prefixes - maxConcurrentHandlers = 800 // Maximum number of concurrent message handlers + metricsWindowSize = 60 // seconds for rolling average + metricsUpdateRate = time.Second + minBackoffDelay = 5 * time.Second + maxBackoffDelay = 320 * time.Second + // Reconnect when no line arrives for this long + streamIdleTimeout = 60 * time.Second + metricsLogInterval = 10 * time.Second + bytesPerKB = 1024 + bytesPerMB = 1024 * 1024 + // Well above an extended BGP message (RFC 8654) in hex plus its prefixes + maxLineSize = bytesPerMB + maxConcurrentHandlers = 800 // Maximum number of concurrent message handlers // Backpressure constants backpressureThreshold = 0.5 // Start dropping at 50% queue utilization backpressureSlope = 2.0 // Slope for linear drop probability increase ) -// errStreamEnded is returned by stream when RIS Live ends the response. -var errStreamEnded = errors.New("RIS Live ended the stream") +var ( + errAlreadyRunning = errors.New("streamer already running") + errUnexpectedStatus = errors.New("unexpected status code") + // errStreamEnded is returned by stream when RIS Live ends the response. + errStreamEnded = errors.New("RIS Live ended the stream") + // errStreamIdle is returned by stream when no line arrived for the idle + // timeout. + errStreamIdle = errors.New("no line received") +) // MessageHandler defines the interface for processing RIS messages. // Implementations must specify which message types they want to receive, // how to process messages, and their desired queue capacity. type MessageHandler interface { - // WantsMessage returns true if this handler wants to process messages of the given type. + // WantsMessage returns true if this handler wants to process messages of + // the given type. WantsMessage(messageType string) bool // HandleMessage processes a RIS message. This method is called from a dedicated @@ -114,18 +125,20 @@ type handlerInfo struct { // backoff, dispatches messages to registered handlers via per-handler queues, and // implements backpressure to prevent queue overflow during high traffic periods. type Streamer struct { - logger *logger.Logger - client *http.Client - url string - idleTimeout time.Duration // Drop the connection when no line arrives for this long - minBackoff time.Duration // First wait before reconnecting; doubles up to maxBackoffDelay + logger *logger.Logger + client *http.Client + url string + // Drop the connection when no line arrives for this long + idleTimeout time.Duration + // First wait before reconnecting; doubles up to maxBackoffDelay + minBackoff time.Duration handlers []*handlerInfo rawHandler RawMessageHandler mu sync.RWMutex cancel context.CancelFunc running bool metrics *metrics.Tracker - totalDropped uint64 // Total dropped messages across all handlers + totalDropped atomic.Uint64 // Total dropped messages across all handlers random *rand.Rand // Random number generator for backpressure drops bgpPeers map[string]bool // Track active BGP peers by peer IP bgpPeersMu sync.RWMutex // Protects bgpPeers map @@ -133,7 +146,8 @@ type Streamer struct { // New creates a new Streamer instance configured to connect to the RIS Live API. // The logger is used for structured logging of connection events and errors. -// The metrics tracker is used to record message counts, bytes received, and connection status. +// The metrics tracker is used to record message counts, bytes received, and +// connection status. func New(logger *logger.Logger, metrics *metrics.Tracker) *Streamer { return &Streamer{ logger: logger, @@ -167,7 +181,8 @@ func (s *Streamer) RegisterHandler(handler MessageHandler) { handler: handler, queue: make(chan *ristypes.RISMessage, handler.QueueCapacity()), metrics: handlerMetrics{ - minTime: time.Duration(math.MaxInt64), // Initialize to max so first value sets the floor + // Initialize to max so first value sets the floor + minTime: time.Duration(math.MaxInt64), }, } @@ -185,22 +200,24 @@ func (s *Streamer) RegisterHandler(handler MessageHandler) { func (s *Streamer) RegisterRawHandler(handler RawMessageHandler) { s.mu.Lock() defer s.mu.Unlock() + s.rawHandler = handler } // Start begins streaming BGP updates from the RIS Live API in a background goroutine. // It starts worker goroutines for each registered handler and manages automatic // reconnection with exponential backoff on connection failures. -// Returns an error if the streamer is already running. -func (s *Streamer) Start() error { +// Returns an error if the streamer is already running. It streams until Stop: +// the end of ctx does not stop it. +func (s *Streamer) Start(ctx context.Context) error { s.mu.Lock() defer s.mu.Unlock() if s.running { - return fmt.Errorf("streamer already running") + return errAlreadyRunning } - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(context.WithoutCancel(ctx)) s.cancel = cancel s.running = true @@ -230,44 +247,21 @@ func (s *Streamer) Stop() { return } + s.cancel() s.cancel = nil // Close all handler queues to signal workers to stop for _, info := range s.handlers { close(info.queue) } + s.running = false s.mu.Unlock() s.metrics.SetConnected(false) } -// runHandlerWorker processes messages for a specific handler -func (s *Streamer) runHandlerWorker(info *handlerInfo) { - for msg := range info.queue { - start := time.Now() - info.handler.HandleMessage(msg) - elapsed := time.Since(start) - - // Update metrics - info.metrics.mu.Lock() - info.metrics.processedCount++ - info.metrics.totalTime += elapsed - - // Update min time - if elapsed < info.metrics.minTime { - info.metrics.minTime = elapsed - } - - // Update max time - if elapsed > info.metrics.maxTime { - info.metrics.maxTime = elapsed - } - info.metrics.mu.Unlock() - } -} - -// IsRunning reports whether the streamer is currently connected and processing messages. -// This is safe to call concurrently from multiple goroutines. +// IsRunning reports whether the streamer is currently connected and processing +// messages. This is safe to call concurrently from multiple goroutines. func (s *Streamer) IsRunning() bool { s.mu.RLock() defer s.mu.RUnlock() @@ -328,12 +322,15 @@ func (s *Streamer) GetHandlerStats() []HandlerStats { // Calculate average time if info.metrics.processedCount > 0 { processedCount := info.metrics.processedCount + const maxInt64 = 1<<63 - 1 + if processedCount > maxInt64 { processedCount = maxInt64 } - //nolint:gosec // processedCount is explicitly bounded above - hs.AvgProcessTime = info.metrics.totalTime / time.Duration(processedCount) + + hs.AvgProcessTime = info.metrics.totalTime / + time.Duration(processedCount) } info.metrics.mu.Unlock() @@ -344,11 +341,38 @@ func (s *Streamer) GetHandlerStats() []HandlerStats { return stats } -// GetDroppedMessages returns the total number of messages dropped across all handlers -// due to queue overflow or backpressure. This counter is monotonically increasing -// and is safe to call concurrently. +// GetDroppedMessages returns the total number of messages dropped across all +// handlers due to queue overflow or backpressure. This counter is monotonically +// increasing and is safe to call concurrently. func (s *Streamer) GetDroppedMessages() uint64 { - return atomic.LoadUint64(&s.totalDropped) + return s.totalDropped.Load() +} + +// runHandlerWorker processes messages for a specific handler +func (s *Streamer) runHandlerWorker(info *handlerInfo) { + for msg := range info.queue { + start := time.Now() + + info.handler.HandleMessage(msg) + + elapsed := time.Since(start) + + // Update metrics + info.metrics.mu.Lock() + info.metrics.processedCount++ + info.metrics.totalTime += elapsed + + // Update min time + if elapsed < info.metrics.minTime { + info.metrics.minTime = elapsed + } + + // Update max time + if elapsed > info.metrics.maxTime { + info.metrics.maxTime = elapsed + } + info.metrics.mu.Unlock() + } } // logMetrics logs the current streaming statistics @@ -357,7 +381,8 @@ func (s *Streamer) logMetrics() { uptime := time.Since(metrics.ConnectedSince) const bitsPerMegabit = 1000000 - totalDropped := atomic.LoadUint64(&s.totalDropped) + + totalDropped := s.totalDropped.Load() s.logger.Info( "Stream statistics", @@ -383,16 +408,19 @@ func (s *Streamer) logMetrics() { // Log per-handler statistics s.mu.RLock() + for i, info := range s.handlers { info.metrics.mu.Lock() if info.metrics.processedCount > 0 { // Safe conversion: processedCount is bounded by maxInt64 processedCount := info.metrics.processedCount + const maxInt64 = 1<<63 - 1 + if processedCount > maxInt64 { processedCount = maxInt64 } - //nolint:gosec // processedCount is explicitly bounded above + avgTime := info.metrics.totalTime / time.Duration(processedCount) s.logger.Info( "Handler statistics", @@ -409,6 +437,7 @@ func (s *Streamer) logMetrics() { } info.metrics.mu.Unlock() } + s.mu.RUnlock() } @@ -417,7 +446,8 @@ func (s *Streamer) updateMetrics(messageBytes int) { s.metrics.RecordMessage(int64(messageBytes)) } -// streamWithReconnect handles streaming with automatic reconnection and exponential backoff +// streamWithReconnect handles streaming with automatic reconnection and +// exponential backoff func (s *Streamer) streamWithReconnect(ctx context.Context) { backoffDelay := s.minBackoff consecutiveFailures := 0 @@ -455,6 +485,7 @@ func (s *Streamer) streamWithReconnect(ctx context.Context) { if streamDuration > 30*time.Second { s.logger.Info("Resetting backoff delay due to successful connection", "stream_duration", streamDuration) + backoffDelay = s.minBackoff consecutiveFailures = 0 } else { @@ -494,7 +525,7 @@ func (s *Streamer) stream(ctx context.Context) error { idleTimer := time.AfterFunc(s.idleTimeout, connCancel) defer idleTimer.Stop() - req, err := http.NewRequestWithContext(connCtx, "GET", s.url, nil) + req, err := http.NewRequestWithContext(connCtx, http.MethodGet, s.url, nil) if err != nil { return fmt.Errorf("failed to create request: %w", err) } @@ -507,13 +538,14 @@ func (s *Streamer) stream(ctx context.Context) error { return fmt.Errorf("failed to connect to RIS Live: %w", err) } defer func() { - if err := resp.Body.Close(); err != nil { + err := resp.Body.Close() + if err != nil { s.logger.Error("Failed to close response body", "error", err) } }() if resp.StatusCode != http.StatusOK { - return fmt.Errorf("unexpected status code: %d", resp.StatusCode) + return fmt.Errorf("%w: %d", errUnexpectedStatus, resp.StatusCode) } // Wrap body with counting reader to track actual wire bytes @@ -527,7 +559,9 @@ func (s *Streamer) stream(ctx context.Context) error { return fmt.Errorf("failed to create gzip reader: %w", err) } defer func() { _ = gzReader.Close() }() + reader = gzReader + s.logger.Info("Connected to RIS Live stream", "compression", "gzip") } else { s.logger.Info("Connected to RIS Live stream", "compression", "none") @@ -535,9 +569,6 @@ func (s *Streamer) stream(ctx context.Context) error { s.metrics.SetConnected(true) - // Track wire bytes for metrics updates - var lastWireBytes int64 - // Start metrics logging goroutine metricsTicker := time.NewTicker(metricsLogInterval) defer metricsTicker.Stop() @@ -554,25 +585,44 @@ func (s *Streamer) stream(ctx context.Context) error { }() // Wire byte update ticker - update metrics with actual wire bytes periodically + go s.recordWireBytes(connCtx, wireCounter) + + return s.readStream(ctx, connCtx, reader, idleTimer) +} + +// recordWireBytes adds the bytes wireCounter has counted to the metrics once a +// second until ctx is cancelled. +func (s *Streamer) recordWireBytes( + ctx context.Context, wireCounter *countingReader, +) { wireUpdateTicker := time.NewTicker(time.Second) defer wireUpdateTicker.Stop() - go func() { - for { - select { - case <-wireUpdateTicker.C: - currentBytes := wireCounter.Count() - delta := currentBytes - lastWireBytes - if delta > 0 { - s.metrics.RecordWireBytes(delta) - lastWireBytes = currentBytes - } - case <-connCtx.Done(): - return - } - } - }() + var lastWireBytes int64 + for { + select { + case <-wireUpdateTicker.C: + currentBytes := wireCounter.Count() + + delta := currentBytes - lastWireBytes + if delta > 0 { + s.metrics.RecordWireBytes(delta) + + lastWireBytes = currentBytes + } + case <-ctx.Done(): + return + } + } +} + +// readStream hands each line of the stream to processLine until the stream +// ends, a line fails, or ctx is cancelled. Each line restarts idleTimer, which +// cancels connCtx when it fires. +func (s *Streamer) readStream( + ctx, connCtx context.Context, reader io.Reader, idleTimer *time.Timer, +) error { scanner := bufio.NewScanner(reader) scanner.Buffer(nil, maxLineSize) @@ -592,129 +642,10 @@ func (s *Streamer) stream(ctx context.Context) error { continue } - // Update metrics with decompressed message size - s.updateMetrics(len(line)) - - // Call raw handler if registered - s.mu.RLock() - rawHandler := s.rawHandler - s.mu.RUnlock() - - if rawHandler != nil { - // Call raw handler synchronously to preserve order - rawHandler(string(line)) + err := s.processLine(ctx, line) + if err != nil { + return err } - - // Parse the message first - var wrapper ristypes.RISLiveMessage - if err := json.Unmarshal(line, &wrapper); err != nil { - // Log the error and return to trigger reconnection - s.logger.Error("Failed to parse JSON", - "error", err, - "line", string(line), - "line_length", len(line)) - - return fmt.Errorf("JSON parse error: %w", err) - } - - // Check if it's a ris_message wrapper - if wrapper.Type != "ris_message" { - s.logger.Error("Unexpected wrapper type", - "type", wrapper.Type, - "line", string(line), - ) - - continue - } - - // Get the actual message - msg := wrapper.Data - - // Parse the timestamp - msg.ParsedTimestamp = time.Unix(int64(msg.Timestamp), 0).UTC() - - // Process based on message type - switch msg.Type { - case ristypes.MessageTypeUpdate: - // Process BGP UPDATE messages - // Will be dispatched to handlers - case "RIS_PEER_STATE": - // RIS peer state messages - silently ignore - continue - case "KEEPALIVE": - // BGP keepalive messages - silently process - continue - case "OPEN": - // BGP open messages - track peer as active - s.bgpPeersMu.Lock() - s.bgpPeers[msg.Peer] = true - peerCount := len(s.bgpPeers) - s.bgpPeersMu.Unlock() - s.metrics.SetBGPPeerCount(peerCount) - - s.logger.Info("BGP session opened", - "peer", msg.Peer, - "peer_asn", msg.PeerASN, - "total_peers", peerCount, - ) - - continue - case "NOTIFICATION": - // BGP notification messages (session closed) - s.bgpPeersMu.Lock() - delete(s.bgpPeers, msg.Peer) - peerCount := len(s.bgpPeers) - s.bgpPeersMu.Unlock() - s.metrics.SetBGPPeerCount(peerCount) - - s.logger.Warn("BGP notification", - "peer", msg.Peer, - "peer_asn", msg.PeerASN, - "total_peers", peerCount, - ) - - continue - case ristypes.MessageTypeState: - // The BGP session with the peer changed state, for example went - // down. Dispatched to the handlers that want it. - default: - s.logger.Warn("Unknown message type, skipping", - "type", msg.Type, - ) - - continue - } - - // Dispatch to interested handlers. Stop cancels ctx and closes the - // queues under the write lock, so if ctx is cancelled here, under the - // read lock, the queues are closed and must not be sent to. - s.mu.RLock() - if ctx.Err() != nil { - s.mu.RUnlock() - - return ctx.Err() - } - for _, info := range s.handlers { - if !info.handler.WantsMessage(msg.Type) { - continue - } - - if !s.queueMessage(info, &msg) { - atomic.AddUint64(&info.metrics.droppedCount, 1) - atomic.AddUint64(&s.totalDropped, 1) - - continue - } - - // Update high water mark if needed - queueLen := len(info.queue) - info.metrics.mu.Lock() - if queueLen > info.metrics.queueHighWaterMark { - info.metrics.queueHighWaterMark = queueLen - } - info.metrics.mu.Unlock() - } - s.mu.RUnlock() } if ctx.Err() != nil { @@ -722,16 +653,161 @@ func (s *Streamer) stream(ctx context.Context) error { } if connCtx.Err() != nil { - return fmt.Errorf("no line received for %s", s.idleTimeout) + return fmt.Errorf("%w for %s", errStreamIdle, s.idleTimeout) } - if err := scanner.Err(); err != nil { + err := scanner.Err() + if err != nil { return fmt.Errorf("scanner error: %w", err) } return errStreamEnded } +// processLine passes one line of the stream to the raw handler, parses it, +// and dispatches an UPDATE or STATE message to the handlers. An error ends the +// connection. +func (s *Streamer) processLine(ctx context.Context, line []byte) error { + // Update metrics with decompressed message size + s.updateMetrics(len(line)) + + // Call raw handler if registered + s.mu.RLock() + rawHandler := s.rawHandler + s.mu.RUnlock() + + if rawHandler != nil { + // Call raw handler synchronously to preserve order + rawHandler(string(line)) + } + + // Parse the message first + var wrapper ristypes.RISLiveMessage + + err := json.Unmarshal(line, &wrapper) + if err != nil { + // Log the error and return to trigger reconnection + s.logger.Error("Failed to parse JSON", + "error", err, + "line", string(line), + "line_length", len(line)) + + return fmt.Errorf("JSON parse error: %w", err) + } + + // Check if it's a ris_message wrapper + if wrapper.Type != "ris_message" { + s.logger.Error("Unexpected wrapper type", + "type", wrapper.Type, + "line", string(line), + ) + + return nil + } + + // Get the actual message + msg := wrapper.Data + + // Parse the timestamp + msg.ParsedTimestamp = time.Unix(int64(msg.Timestamp), 0).UTC() + + // Process based on message type + switch msg.Type { + case ristypes.MessageTypeUpdate: + // Process BGP UPDATE messages + // Will be dispatched to handlers + case "RIS_PEER_STATE": + // RIS peer state messages - silently ignore + return nil + case "KEEPALIVE": + // BGP keepalive messages - silently process + return nil + case "OPEN": + // BGP open messages - track peer as active + s.bgpPeersMu.Lock() + s.bgpPeers[msg.Peer] = true + peerCount := len(s.bgpPeers) + s.bgpPeersMu.Unlock() + s.metrics.SetBGPPeerCount(peerCount) + + s.logger.Info("BGP session opened", + "peer", msg.Peer, + "peer_asn", msg.PeerASN, + "total_peers", peerCount, + ) + + return nil + case "NOTIFICATION": + // BGP notification messages (session closed) + s.bgpPeersMu.Lock() + delete(s.bgpPeers, msg.Peer) + peerCount := len(s.bgpPeers) + s.bgpPeersMu.Unlock() + s.metrics.SetBGPPeerCount(peerCount) + + s.logger.Warn("BGP notification", + "peer", msg.Peer, + "peer_asn", msg.PeerASN, + "total_peers", peerCount, + ) + + return nil + case ristypes.MessageTypeState: + // The BGP session with the peer changed state, for example went + // down. Dispatched to the handlers that want it. + default: + s.logger.Warn("Unknown message type, skipping", + "type", msg.Type, + ) + + return nil + } + + return s.dispatchMessage(ctx, &msg) +} + +// dispatchMessage queues msg for each handler that wants it, unless +// queueMessage drops it for that handler. +func (s *Streamer) dispatchMessage( + ctx context.Context, msg *ristypes.RISMessage, +) error { + // Stop cancels ctx and closes the queues under the write lock, so if ctx + // is cancelled here, under the read lock, the queues are closed and must + // not be sent to. + s.mu.RLock() + + if ctx.Err() != nil { + s.mu.RUnlock() + + return ctx.Err() + } + + for _, info := range s.handlers { + if !info.handler.WantsMessage(msg.Type) { + continue + } + + if !s.queueMessage(info, msg) { + atomic.AddUint64(&info.metrics.droppedCount, 1) + s.totalDropped.Add(1) + + continue + } + + // Update high water mark if needed + queueLen := len(info.queue) + info.metrics.mu.Lock() + if queueLen > info.metrics.queueHighWaterMark { + info.metrics.queueHighWaterMark = queueLen + } + info.metrics.mu.Unlock() + } + + s.mu.RUnlock() + + return nil +} + // queueMessage puts msg on the handler's queue and reports false if it dropped // msg instead. It never waits for room, so a full queue holds up neither the // read loop nor Stop. @@ -746,7 +822,9 @@ func (s *Streamer) stream(ctx context.Context) error { // // The held list stays small: a STATE message is a few hundred bytes and the // feed sends about ten a second, so a queue full for a minute holds about 600. -func (s *Streamer) queueMessage(info *handlerInfo, msg *ristypes.RISMessage) bool { +func (s *Streamer) queueMessage( + info *handlerInfo, msg *ristypes.RISMessage, +) bool { for len(info.heldState) > 0 && trySend(info.queue, info.heldState[0]) { info.heldState = info.heldState[1:] } @@ -776,7 +854,8 @@ func trySend(queue chan *ristypes.RISMessage, msg *ristypes.RISMessage) bool { } } -// shouldDropForBackpressure determines if a message should be dropped based on queue utilization +// shouldDropForBackpressure determines if a message should be dropped based on +// queue utilization func (s *Streamer) shouldDropForBackpressure(info *handlerInfo) bool { // Calculate queue utilization queueLen := len(info.queue) diff --git a/internal/streamer/streamer_test.go b/internal/streamer/streamer_internal_test.go similarity index 78% rename from internal/streamer/streamer_test.go rename to internal/streamer/streamer_internal_test.go index 42e1f18..e3b545e 100644 --- a/internal/streamer/streamer_test.go +++ b/internal/streamer/streamer_internal_test.go @@ -18,6 +18,8 @@ import ( ) func TestNewStreamer(t *testing.T) { + t.Parallel() + logger := logger.New() metricsTracker := metrics.New() s := New(logger, metricsTracker) @@ -56,10 +58,13 @@ func TestNewStreamer(t *testing.T) { // goroutine count returns to its starting value. Each connection starts two // ticker goroutines; before the fix they lived until the streamer's lifetime // context was cancelled, so every reconnect leaked two. +// +//nolint:paralleltest // counts goroutines, so no other test may run alongside func TestStreamDoesNotLeakTickersAcrossReconnects(t *testing.T) { // The handler returns immediately, so the response body is empty and each // stream call ends at once, standing in for a dropped connection. - srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {})) + srv := httptest.NewServer(http.HandlerFunc( + func(_ http.ResponseWriter, _ *http.Request) {})) defer srv.Close() s := New(logger.New(), metrics.New()) @@ -67,19 +72,23 @@ func TestStreamDoesNotLeakTickersAcrossReconnects(t *testing.T) { // One warm-up connection so any persistent HTTP transport goroutine exists // before we take the baseline. - if err := s.stream(context.Background()); !errors.Is(err, errStreamEnded) { + err := s.stream(context.Background()) + if !errors.Is(err, errStreamEnded) { t.Fatalf("warm-up stream returned %v, want %v", err, errStreamEnded) } + s.client.CloseIdleConnections() baseline := settledGoroutineCount() const reconnects = 20 for range reconnects { - if err := s.stream(context.Background()); !errors.Is(err, errStreamEnded) { + err := s.stream(context.Background()) + if !errors.Is(err, errStreamEnded) { t.Fatalf("stream returned %v, want %v", err, errStreamEnded) } } + s.client.CloseIdleConnections() if !waitForGoroutines(baseline) { @@ -89,8 +98,8 @@ func TestStreamDoesNotLeakTickersAcrossReconnects(t *testing.T) { } // updateLine is one UPDATE message as RIS Live sends it. -const updateLine = `{"type":"ris_message","data":{"type":"UPDATE","peer":"192.0.2.1",` + - `"peer_asn":"64496","timestamp":1700000000}}` + "\n" +const updateLine = `{"type":"ris_message","data":{"type":"UPDATE",` + + `"peer":"192.0.2.1","peer_asn":"64496","timestamp":1700000000}}` + "\n" // updateHandler wants UPDATE messages. For each one it handles it sends on // handled if there is room, and otherwise does nothing; a nil handled is @@ -99,8 +108,11 @@ type updateHandler struct { handled chan struct{} } -func (updateHandler) WantsMessage(messageType string) bool { return messageType == "UPDATE" } -func (updateHandler) QueueCapacity() int { return 10 } +func (updateHandler) WantsMessage(messageType string) bool { + return messageType == "UPDATE" +} + +func (updateHandler) QueueCapacity() int { return 10 } func (h updateHandler) HandleMessage(*ristypes.RISMessage) { select { @@ -109,11 +121,21 @@ func (h updateHandler) HandleMessage(*ristypes.RISMessage) { } } +// flush sends what w has buffered to the client. The writers httptest servers +// hand to handlers can always do that. +func flush(w http.ResponseWriter) { + flusher, ok := w.(http.Flusher) + if ok { + flusher.Flush() + } +} + // stateDownLine is the STATE message RIS Live sends when the collector's BGP // session with the peer stateDownPeer goes down. const ( stateDownPeer = "192.0.2.1" - stateDownLine = `{"type":"ris_message","data":{"type":"STATE","peer":"` + stateDownPeer + `",` + + stateDownLine = `{"type":"ris_message","data":{"type":"STATE",` + + `"peer":"` + stateDownPeer + `",` + `"peer_asn":"64496","timestamp":1700000000,"state":"down"}}` + "\n" ) @@ -124,31 +146,39 @@ type stateHandler struct{} func (stateHandler) WantsMessage(messageType string) bool { return messageType == ristypes.MessageTypeState } -func (stateHandler) QueueCapacity() int { return 10 } + +func (stateHandler) QueueCapacity() int { return 10 } + func (stateHandler) HandleMessage(*ristypes.RISMessage) {} // TestStreamHandsStateMessagesToHandlersThatWantThem has the server send one // STATE message whose state is down. The streamer must queue it, with its // peer and state, for a handler that wants STATE messages. func TestStreamHandsStateMessagesToHandlersThatWantThem(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = io.WriteString(w, stateDownLine) - })) + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, stateDownLine) + })) defer srv.Close() s := New(logger.New(), metrics.New()) s.url = srv.URL s.RegisterHandler(stateHandler{}) - if err := s.stream(context.Background()); !errors.Is(err, errStreamEnded) { + err := s.stream(context.Background()) + if !errors.Is(err, errStreamEnded) { t.Fatalf("stream returned %v, want %v", err, errStreamEnded) } select { case msg := <-s.handlers[0].queue: - if msg.Type != ristypes.MessageTypeState || msg.Peer != stateDownPeer || msg.State != ristypes.StateDown { - t.Errorf("queued message has type %q, peer %q, state %q; want %q, %q, %q", - msg.Type, msg.Peer, msg.State, ristypes.MessageTypeState, stateDownPeer, ristypes.StateDown) + if msg.Type != ristypes.MessageTypeState || msg.Peer != stateDownPeer || + msg.State != ristypes.StateDown { + t.Errorf("queued message has type %q, peer %q, state %q; "+ + "want %q, %q, %q", msg.Type, msg.Peer, msg.State, + ristypes.MessageTypeState, stateDownPeer, ristypes.StateDown) } default: t.Fatal("the STATE message was not queued for the handler that wants it") @@ -161,26 +191,37 @@ func TestStreamHandsStateMessagesToHandlersThatWantThem(t *testing.T) { // everything queued before it and before anything handed over after it. The // update handed over while a STATE message is held is dropped. func TestHeldStateMessagesKeepTheirPlaceInTheQueue(t *testing.T) { + t.Parallel() + s := New(logger.New(), metrics.New()) s.RegisterHandler(stateHandler{}) + info := s.handlers[0] before := &ristypes.RISMessage{Type: ristypes.MessageTypeUpdate} for len(info.queue) < cap(info.queue) { info.queue <- before } - firstDown := &ristypes.RISMessage{Type: ristypes.MessageTypeState, State: ristypes.StateDown} - secondDown := &ristypes.RISMessage{Type: ristypes.MessageTypeState, State: ristypes.StateDown} + + firstDown := &ristypes.RISMessage{ + Type: ristypes.MessageTypeState, State: ristypes.StateDown, + } + secondDown := &ristypes.RISMessage{ + Type: ristypes.MessageTypeState, State: ristypes.StateDown, + } updateWhileHeld := &ristypes.RISMessage{Type: ristypes.MessageTypeUpdate} after := &ristypes.RISMessage{Type: ristypes.MessageTypeUpdate} if !s.queueMessage(info, firstDown) { t.Fatal("the first STATE message was dropped on a full queue") } + <-info.queue // the handler makes room for one message + if !s.queueMessage(info, secondDown) { t.Fatal("the second STATE message was dropped") } + if s.queueMessage(info, updateWhileHeld) { t.Fatal("an update was queued while a STATE message was held") } @@ -191,6 +232,7 @@ func TestHeldStateMessagesKeepTheirPlaceInTheQueue(t *testing.T) { t.Fatal("a message overtook those queued before the first STATE message") } } + if !s.queueMessage(info, after) { t.Fatal("an update was dropped on a queue with room") } @@ -198,12 +240,16 @@ func TestHeldStateMessagesKeepTheirPlaceInTheQueue(t *testing.T) { if <-info.queue != firstDown { t.Fatal("the first STATE message did not come next") } + if <-info.queue != secondDown { t.Fatal("the second STATE message did not come after the first") } + if <-info.queue != after { - t.Fatal("the update handed over after the STATE messages did not come after them") + t.Fatal("the update handed over after the STATE messages did not come " + + "after them") } + if len(info.queue) != 0 { t.Errorf("%d messages left on the queue, want none", len(info.queue)) } @@ -215,6 +261,7 @@ func newStreamerWithFullStateQueue(url string) *Streamer { s := New(logger.New(), metrics.New()) s.url = url s.RegisterHandler(stateHandler{}) + queue := s.handlers[0].queue for len(queue) < cap(queue) { queue <- &ristypes.RISMessage{Type: ristypes.MessageTypeState} @@ -229,24 +276,33 @@ func newStreamerWithFullStateQueue(url string) *Streamer { // for that handler, and the read loop must go on to queue the UPDATE for the // handler that wants it. func TestOtherHandlersKeepReceivingWhileAStateMessageIsHeld(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = io.WriteString(w, stateDownLine+updateLine) - })) + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, stateDownLine+updateLine) + })) defer srv.Close() s := newStreamerWithFullStateQueue(srv.URL) s.RegisterHandler(updateHandler{}) - if err := s.stream(context.Background()); !errors.Is(err, errStreamEnded) { + err := s.stream(context.Background()) + if !errors.Is(err, errStreamEnded) { t.Fatalf("stream returned %v, want %v", err, errStreamEnded) } held := s.handlers[0].heldState - if len(held) != 1 || held[0].Peer != stateDownPeer || held[0].State != ristypes.StateDown { - t.Errorf("held %d STATE messages, want the down from %s alone", len(held), stateDownPeer) + if len(held) != 1 || held[0].Peer != stateDownPeer || + held[0].State != ristypes.StateDown { + t.Errorf("held %d STATE messages, want the down from %s alone", + len(held), stateDownPeer) } - if got := len(s.handlers[1].queue); got != 1 { - t.Errorf("%d messages queued for the handler that wants UPDATE messages, want 1", got) + + got := len(s.handlers[1].queue) + if got != 1 { + t.Errorf("%d messages queued for the handler that wants UPDATE "+ + "messages, want 1", got) } } @@ -254,21 +310,27 @@ func TestOtherHandlersKeepReceivingWhileAStateMessageIsHeld(t *testing.T) { // messages and hold the response open while the queue of the handler that // wants them is full. Stop must return at once and end the stream. func TestStopReturnsAtOnceWithStateMessagesHeld(t *testing.T) { + t.Parallel() + const ( lines = 2 maxWait = 5 * time.Second ) - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = io.WriteString(w, stateDownLine+stateDownLine) - w.(http.Flusher).Flush() - <-r.Context().Done() - })) + + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, stateDownLine+stateDownLine) + + flush(w) + <-r.Context().Done() + })) defer srv.Close() s := newStreamerWithFullStateQueue(srv.URL) // The raw handler sees each line before the read loop hands it to the // queues, so once it has seen the second line, the first is held. linesSeen := make(chan struct{}, lines) + s.RegisterRawHandler(func(string) { linesSeen <- struct{}{} }) // As in TestStopBeforeMessageReachesQueues, setting cancel as Start does @@ -276,25 +338,30 @@ func TestStopReturnsAtOnceWithStateMessagesHeld(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) s.cancel = cancel streamErr := make(chan error, 1) + go func() { streamErr <- s.stream(ctx) }() for range lines { select { case <-linesSeen: case <-time.After(maxWait): - t.Fatalf("the read loop did not read %d lines within %s", lines, maxWait) + t.Fatalf("the read loop did not read %d lines within %s", + lines, maxWait) } } stopped := make(chan struct{}) + go func() { s.Stop() close(stopped) }() + select { case <-stopped: case <-time.After(maxWait): - t.Fatalf("Stop did not return within %s while a STATE message was held", maxWait) + t.Fatalf("Stop did not return within %s while a STATE message was held", + maxWait) } select { @@ -313,9 +380,12 @@ func TestStopReturnsAtOnceWithStateMessagesHeld(t *testing.T) { // used to end in "send on closed channel". The raw handler runs in that gap on // the read loop itself, so calling Stop from it hits the gap every time. func TestStopBeforeMessageReachesQueues(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = io.WriteString(w, updateLine) - })) + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, updateLine) + })) defer srv.Close() s := New(logger.New(), metrics.New()) @@ -329,7 +399,8 @@ func TestStopBeforeMessageReachesQueues(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) s.cancel = cancel - if err := s.stream(ctx); !errors.Is(err, context.Canceled) { + err := s.stream(ctx) + if !errors.Is(err, context.Canceled) { t.Fatalf("stream returned %v, want %v", err, context.Canceled) } @@ -341,6 +412,8 @@ func TestStopBeforeMessageReachesQueues(t *testing.T) { // normally, as when RIS Live restarts. The streamer must treat that as a // disconnect and connect again. func TestReconnectWhenStreamEnds(t *testing.T) { + t.Parallel() + endResponse := func(http.ResponseWriter, *http.Request) {} checkReconnect(t, endResponse, streamIdleTimeout) } @@ -349,9 +422,12 @@ func TestReconnectWhenStreamEnds(t *testing.T) { // open without sending anything. The streamer must drop it once the idle // timeout passes and connect again. func TestReconnectWhenStreamGoesIdle(t *testing.T) { + t.Parallel() + const idleTimeout = 50 * time.Millisecond + holdResponse := func(w http.ResponseWriter, r *http.Request) { - w.(http.Flusher).Flush() + flush(w) <-r.Context().Done() } checkReconnect(t, holdResponse, idleTimeout) @@ -361,7 +437,10 @@ func TestReconnectWhenStreamGoesIdle(t *testing.T) { // request and never answer it, not even with headers. The streamer must give // up on it once the idle timeout passes and connect again. func TestReconnectWhenServerNeverAnswers(t *testing.T) { + t.Parallel() + const idleTimeout = 50 * time.Millisecond + neverAnswer := func(_ http.ResponseWriter, r *http.Request) { <-r.Context().Done() } @@ -373,26 +452,32 @@ func TestReconnectWhenServerNeverAnswers(t *testing.T) { // line must start the idle timeout over, so the stream lasts until the server // ends it instead of being dropped as idle. func TestStreamNotDroppedWhileLinesArrive(t *testing.T) { + t.Parallel() + const ( idleTimeout = 100 * time.Millisecond lineInterval = idleTimeout / 10 sendFor = 5 * idleTimeout ) - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - deadline := time.Now().Add(sendFor) - for time.Now().Before(deadline) { - _, _ = io.WriteString(w, updateLine) - w.(http.Flusher).Flush() - time.Sleep(lineInterval) - } - })) + + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + deadline := time.Now().Add(sendFor) + for time.Now().Before(deadline) { + _, _ = io.WriteString(w, updateLine) + + flush(w) + time.Sleep(lineInterval) + } + })) defer srv.Close() s := New(logger.New(), metrics.New()) s.url = srv.URL s.idleTimeout = idleTimeout - if err := s.stream(context.Background()); !errors.Is(err, errStreamEnded) { + err := s.stream(context.Background()) + if !errors.Is(err, errStreamEnded) { t.Fatalf("stream returned %v, want %v", err, errStreamEnded) } } @@ -404,24 +489,31 @@ func TestStreamNotDroppedWhileLinesArrive(t *testing.T) { // handler's queue and read on until the server ends the stream, instead of // dropping the connection at the long line. func TestStreamReadsLongLine(t *testing.T) { + t.Parallel() + const rawHexLength = 200 * bytesPerKB + longLine := `{"type":"ris_message","data":{"type":"UPDATE","peer":"192.0.2.1",` + `"peer_asn":"64496","timestamp":1700000000,"raw":"` + strings.Repeat("0", rawHexLength) + `"}}` + "\n" - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = io.WriteString(w, longLine) - })) + + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, longLine) + })) defer srv.Close() s := New(logger.New(), metrics.New()) s.url = srv.URL s.RegisterHandler(updateHandler{}) - if err := s.stream(context.Background()); !errors.Is(err, errStreamEnded) { + err := s.stream(context.Background()) + if !errors.Is(err, errStreamEnded) { t.Fatalf("stream returned %v, want %v", err, errStreamEnded) } - if queued := len(s.handlers[0].queue); queued != 1 { + queued := len(s.handlers[0].queue) + if queued != 1 { t.Errorf("handler queue holds %d messages, want 1", queued) } } @@ -431,28 +523,36 @@ func TestStreamReadsLongLine(t *testing.T) { // line and then stays open. The streamer must come back for a second // response, must not be marked connected when it does, and must handle the // UPDATE in it. -func checkReconnect(t *testing.T, firstResponse http.HandlerFunc, idleTimeout time.Duration) { +func checkReconnect( + t *testing.T, firstResponse http.HandlerFunc, idleTimeout time.Duration, +) { t.Helper() const maxWait = 5 * time.Second tracker := metrics.New() connectedAtReconnect := make(chan bool, 1) - var requests atomic.Int32 - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if requests.Add(1) == 1 { - firstResponse(w, r) - return - } - select { - case connectedAtReconnect <- tracker.IsConnected(): - default: - } - _, _ = io.WriteString(w, updateLine) - w.(http.Flusher).Flush() - <-r.Context().Done() - })) + var requests atomic.Int32 + + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + if requests.Add(1) == 1 { + firstResponse(w, r) + + return + } + + select { + case connectedAtReconnect <- tracker.IsConnected(): + default: + } + + _, _ = io.WriteString(w, updateLine) + + flush(w) + <-r.Context().Done() + })) defer srv.Close() s := New(logger.New(), tracker) @@ -461,7 +561,9 @@ func checkReconnect(t *testing.T, firstResponse http.HandlerFunc, idleTimeout ti s.minBackoff = time.Millisecond handler := updateHandler{handled: make(chan struct{}, 1)} s.RegisterHandler(handler) - if err := s.Start(); err != nil { + + err := s.Start(t.Context()) + if err != nil { t.Fatalf("Start returned error: %v", err) } defer s.Stop() @@ -473,7 +575,8 @@ func checkReconnect(t *testing.T, firstResponse http.HandlerFunc, idleTimeout ti } if <-connectedAtReconnect { - t.Error("streamer was still marked connected when it came back for a second response") + t.Error("streamer was still marked connected when it came back " + + "for a second response") } } @@ -481,12 +584,15 @@ func checkReconnect(t *testing.T, firstResponse http.HandlerFunc, idleTimeout ti // current count. func settledGoroutineCount() int { prev := runtime.NumGoroutine() + for range 20 { time.Sleep(10 * time.Millisecond) + cur := runtime.NumGoroutine() if cur == prev { return cur } + prev = cur } @@ -499,6 +605,7 @@ func waitForGoroutines(target int) bool { if runtime.NumGoroutine() <= target { return true } + time.Sleep(10 * time.Millisecond) } diff --git a/internal/templates/templates.go b/internal/templates/templates.go index b388343..16d6737 100644 --- a/internal/templates/templates.go +++ b/internal/templates/templates.go @@ -60,6 +60,7 @@ func timeSince(t time.Time) string { if duration < time.Minute { return "just now" } + if duration < time.Hour { minutes := int(duration.Minutes()) if minutes == 1 { @@ -68,6 +69,7 @@ func timeSince(t time.Time) string { return duration.Truncate(time.Minute).String() + " ago" } + if duration < hoursPerDay*time.Hour { hours := int(duration.Hours()) if hours == 1 { @@ -76,10 +78,12 @@ func timeSince(t time.Time) string { return duration.Truncate(time.Hour).String() + " ago" } + days := int(duration.Hours() / hoursPerDay) if days == 1 { return "1 day ago" } + if days < daysPerMonth { return duration.Truncate(hoursPerDay*time.Hour).String() + " ago" } @@ -118,7 +122,7 @@ func initTemplates() { "appLicense": func() string { return version.License }, "appRepoURL": func() string { return version.RepoURL }, "appGitRevision": func() string { return version.GitRevisionShort }, - "appGitCommitURL": func() string { return version.CommitURL() }, + "appGitCommitURL": version.CommitURL, } // Parse index template @@ -134,19 +138,22 @@ func initTemplates() { } // Parse AS detail template - defaultTemplates.ASDetail, err = template.New("asDetail").Funcs(funcs).Parse(asDetailHTML) + defaultTemplates.ASDetail, err = template.New("asDetail"). + Funcs(funcs).Parse(asDetailHTML) if err != nil { panic("failed to parse AS detail template: " + err.Error()) } // Parse prefix detail template - defaultTemplates.PrefixDetail, err = template.New("prefixDetail").Funcs(funcs).Parse(prefixDetailHTML) + defaultTemplates.PrefixDetail, err = template.New("prefixDetail"). + Funcs(funcs).Parse(prefixDetailHTML) if err != nil { panic("failed to parse prefix detail template: " + err.Error()) } // Parse prefix length template - defaultTemplates.PrefixLength, err = template.New("prefixLength").Funcs(funcs).Parse(prefixLengthHTML) + defaultTemplates.PrefixLength, err = template.New("prefixLength"). + Funcs(funcs).Parse(prefixLengthHTML) if err != nil { panic("failed to parse prefix length template: " + err.Error()) } diff --git a/internal/version/version.go b/internal/version/version.go index b32b2d8..20c44d5 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -3,7 +3,7 @@ package version // Build-time variables set via ldflags // -//nolint:gochecknoglobals // These must be variables to allow ldflags injection at build time +//nolint:gochecknoglobals // variables so that ldflags can set them at build time var ( // GitRevision is the git commit hash GitRevision = "unknown" diff --git a/internal/whois/whois.go b/internal/whois/whois.go index 1c4e55b..3713c9f 100644 --- a/internal/whois/whois.go +++ b/internal/whois/whois.go @@ -98,7 +98,8 @@ func (c *Client) LookupASN(ctx context.Context, asn int) (*ASNInfo, error) { if err != nil { // Return partial data from IANA if RIR query fails info := c.parseResponse(asn, rir, ianaResp) - info.RawResponse = ianaResp + "\n--- RIR query failed: " + err.Error() + " ---\n" + info.RawResponse = fmt.Sprintf("%s\n--- RIR query failed: %v ---\n", + ianaResp, err) return info, nil } @@ -120,28 +121,33 @@ func (c *Client) query(ctx context.Context, server, query string) (string, error defer func() { _ = conn.Close() }() // Set deadlines - if err := conn.SetWriteDeadline(time.Now().Add(writeTimeout)); err != nil { + err = conn.SetWriteDeadline(time.Now().Add(writeTimeout)) + if err != nil { return "", fmt.Errorf("set write deadline: %w", err) } // Send query - if _, err := fmt.Fprintf(conn, "%s\r\n", query); err != nil { + _, err = fmt.Fprintf(conn, "%s\r\n", query) + if err != nil { return "", fmt.Errorf("write query: %w", err) } // Read response - if err := conn.SetReadDeadline(time.Now().Add(readTimeout)); err != nil { + err = conn.SetReadDeadline(time.Now().Add(readTimeout)) + if err != nil { return "", fmt.Errorf("set read deadline: %w", err) } var sb strings.Builder + scanner := bufio.NewScanner(conn) for scanner.Scan() { sb.WriteString(scanner.Text()) sb.WriteString("\n") } - if err := scanner.Err(); err != nil { + err = scanner.Err() + if err != nil { return sb.String(), fmt.Errorf("read response: %w", err) } @@ -149,10 +155,10 @@ func (c *Client) query(ctx context.Context, server, query string) (string, error } // parseIANAReferral extracts the RIR and WHOIS server from an IANA response. -func (c *Client) parseIANAReferral(response string) (rir, whoisServer string) { - lines := strings.Split(response, "\n") +func (c *Client) parseIANAReferral(response string) (string, string) { + var rir string - for _, line := range lines { + for line := range strings.SplitSeq(response, "\n") { line = strings.TrimSpace(line) // Look for "refer:" line @@ -183,18 +189,9 @@ func (c *Client) parseIANAReferral(response string) (rir, whoisServer string) { // Also check organisation line for RIR hints if strings.HasPrefix(strings.ToLower(line), "organisation:") { - org := strings.ToLower(line) - switch { - case strings.Contains(org, "arin"): - rir = RIRARIN - case strings.Contains(org, "ripe"): - rir = RIRRIPE - case strings.Contains(org, "apnic"): - rir = RIRAPNIC - case strings.Contains(org, "lacnic"): - rir = RIRLACNIC - case strings.Contains(org, "afrinic"): - rir = RIRAFRNIC + orgRIR := rirInOrganisationLine(strings.ToLower(line)) + if orgRIR != "" { + rir = orgRIR } } } @@ -202,6 +199,25 @@ func (c *Client) parseIANAReferral(response string) (rir, whoisServer string) { return rir, "" } +// rirInOrganisationLine returns the RIR a lowercased IANA "organisation:" +// line names, or "" when it names none. +func rirInOrganisationLine(org string) string { + switch { + case strings.Contains(org, "arin"): + return RIRARIN + case strings.Contains(org, "ripe"): + return RIRRIPE + case strings.Contains(org, "apnic"): + return RIRAPNIC + case strings.Contains(org, "lacnic"): + return RIRLACNIC + case strings.Contains(org, "afrinic"): + return RIRAFRNIC + default: + return "" + } +} + // parseResponse extracts ASN information from a WHOIS response. func (c *Client) parseResponse(asn int, rir, response string) *ASNInfo { info := &ASNInfo{ @@ -211,11 +227,13 @@ func (c *Client) parseResponse(asn int, rir, response string) *ASNInfo { } lines := strings.Split(response, "\n") + var addressLines []string for _, line := range lines { line = strings.TrimSpace(line) - if line == "" || strings.HasPrefix(line, "%") || strings.HasPrefix(line, "#") { + if line == "" || strings.HasPrefix(line, "%") || + strings.HasPrefix(line, "#") { continue } @@ -232,63 +250,14 @@ func (c *Client) parseResponse(asn int, rir, response string) *ASNInfo { continue } - switch key { - // AS Name (varies by RIR) - case "asname", "as-name": - if info.ASName == "" { - info.ASName = value - } - - // Organization - case "orgname", "org-name", "owner": - if info.OrgName == "" { - info.OrgName = value - } - case "orgid", "org-id", "org": - if info.OrgID == "" { - info.OrgID = value - } - // Address (collect multiple lines) - case "address": + if key == "address" { addressLines = append(addressLines, value) - // Country - case "country": - if info.CountryCode == "" && len(value) == 2 { - info.CountryCode = strings.ToUpper(value) - } - - // Abuse contact - case "orgabuseemail", "abuse-mailbox": - if info.AbuseEmail == "" { - info.AbuseEmail = value - } - case "orgabusephone": - if info.AbusePhone == "" { - info.AbusePhone = value - } - - // Tech contact - case "orgtechemail": - if info.TechEmail == "" { - info.TechEmail = value - } - case "orgtechphone": - if info.TechPhone == "" { - info.TechPhone = value - } - - // Registration dates - case "regdate", "created": - if info.RegDate == nil { - info.RegDate = c.parseDate(value) - } - case "updated", "last-modified", "changed": - if info.LastMod == nil { - info.LastMod = c.parseDate(value) - } + continue } + + c.setResponseField(info, key, value) } // Combine address lines @@ -304,6 +273,65 @@ func (c *Client) parseResponse(asn int, rir, response string) *ASNInfo { return info } +// setResponseField stores the value of one "key: value" line of a WHOIS +// response in info, keeping the first value seen for each field. +func (c *Client) setResponseField(info *ASNInfo, key, value string) { + switch key { + // Country + case "country": + if info.CountryCode == "" && len(value) == 2 { + info.CountryCode = strings.ToUpper(value) + } + + // Registration dates + case "regdate", "created": + if info.RegDate == nil { + info.RegDate = c.parseDate(value) + } + case "updated", "last-modified", "changed": + if info.LastMod == nil { + info.LastMod = c.parseDate(value) + } + + default: + field := textFieldForKey(info, key) + if field != nil && *field == "" { + *field = value + } + } +} + +// textFieldForKey returns the text field of info that a WHOIS response key +// fills, or nil when the key fills none. +func textFieldForKey(info *ASNInfo, key string) *string { + switch key { + // AS Name (varies by RIR) + case "asname", "as-name": + return &info.ASName + + // Organization + case "orgname", "org-name", "owner": + return &info.OrgName + case "orgid", "org-id", "org": + return &info.OrgID + + // Abuse contact + case "orgabuseemail", "abuse-mailbox": + return &info.AbuseEmail + case "orgabusephone": + return &info.AbusePhone + + // Tech contact + case "orgtechemail": + return &info.TechEmail + case "orgtechphone": + return &info.TechPhone + + default: + return nil + } +} + // parseDate attempts to parse various date formats used in WHOIS responses. func (c *Client) parseDate(value string) *time.Time { // Common formats @@ -319,15 +347,15 @@ func (c *Client) parseDate(value string) *time.Time { value = strings.TrimSpace(value) // Handle "YYYYMMDD" format from LACNIC if len(value) == lacnicDateFormatLen { - if _, err := time.Parse("20060102", value); err == nil { - t, _ := time.Parse("20060102", value) - + t, err := time.Parse("20060102", value) + if err == nil { return &t } } for _, format := range formats { - if t, err := time.Parse(format, value); err == nil { + t, err := time.Parse(format, value) + if err == nil { return &t } } diff --git a/package.json b/package.json new file mode 100644 index 0000000..fb8b622 --- /dev/null +++ b/package.json @@ -0,0 +1,6 @@ +{ + "private": true, + "devDependencies": { + "prettier": "3.8.1" + } +} diff --git a/pkg/asinfo/asinfo.go b/pkg/asinfo/asinfo.go index 500ec20..372f2e1 100644 --- a/pkg/asinfo/asinfo.go +++ b/pkg/asinfo/asinfo.go @@ -1,4 +1,3 @@ -// Package asinfo provides information about Autonomous Systems (AS) package asinfo import ( @@ -54,7 +53,9 @@ func initRegistry() { } var asInfos []Info - if err := json.Unmarshal(jsonData, &asInfos); err != nil { + + err = json.Unmarshal(jsonData, &asInfos) + if err != nil { panic("failed to unmarshal embedded AS data: " + err.Error()) } @@ -137,6 +138,7 @@ func Search(query string) []Info { defer defaultRegistry.mu.RUnlock() var results []Info + for _, info := range defaultRegistry.byASN { if contains(info.Handle, query) || contains(info.Description, query) { results = append(results, *info) diff --git a/pkg/asinfo/asinfo_test.go b/pkg/asinfo/asinfo_test.go index 5d8c205..3dabfa6 100644 --- a/pkg/asinfo/asinfo_test.go +++ b/pkg/asinfo/asinfo_test.go @@ -1,10 +1,15 @@ -package asinfo +package asinfo_test import ( + "strings" "testing" + + "git.eeqj.de/sneak/routewatch/pkg/asinfo" ) func TestGet(t *testing.T) { + t.Parallel() + tests := []struct { name string asn int @@ -38,20 +43,26 @@ func TestGet(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - info, ok := Get(tt.asn) + t.Parallel() + + info, ok := asinfo.Get(tt.asn) if ok != tt.wantOK { t.Errorf("Get(%d) ok = %v, want %v", tt.asn, ok, tt.wantOK) + return } if ok && info.Description != tt.wantDesc { - t.Errorf("Get(%d) description = %q, want %q", tt.asn, info.Description, tt.wantDesc) + t.Errorf("Get(%d) description = %q, want %q", + tt.asn, info.Description, tt.wantDesc) } }) } } func TestGetDescription(t *testing.T) { + t.Parallel() + tests := []struct { asn int want string @@ -62,7 +73,10 @@ func TestGetDescription(t *testing.T) { for _, tt := range tests { t.Run("", func(t *testing.T) { - if got := GetDescription(tt.asn); got != tt.want { + t.Parallel() + + got := asinfo.GetDescription(tt.asn) + if got != tt.want { t.Errorf("GetDescription(%d) = %q, want %q", tt.asn, got, tt.want) } }) @@ -70,6 +84,8 @@ func TestGetDescription(t *testing.T) { } func TestGetHandle(t *testing.T) { + t.Parallel() + tests := []struct { asn int want string @@ -81,7 +97,10 @@ func TestGetHandle(t *testing.T) { for _, tt := range tests { t.Run("", func(t *testing.T) { - if got := GetHandle(tt.asn); got != tt.want { + t.Parallel() + + got := asinfo.GetHandle(tt.asn) + if got != tt.want { t.Errorf("GetHandle(%d) = %q, want %q", tt.asn, got, tt.want) } }) @@ -89,18 +108,23 @@ func TestGetHandle(t *testing.T) { } func TestTotal(t *testing.T) { - total := Total() + t.Parallel() + + total := asinfo.Total() if total < 100000 { t.Errorf("Total() = %d, expected > 100000", total) } // Verify it's consistent - if total2 := Total(); total2 != total { + total2 := asinfo.Total() + if total2 != total { t.Errorf("Total() returned different values: %d vs %d", total, total2) } } func TestSearch(t *testing.T) { + t.Parallel() + tests := []struct { name string query string @@ -130,16 +154,21 @@ func TestSearch(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - results := Search(tt.query) + t.Parallel() + + results := asinfo.Search(tt.query) if len(results) < tt.wantMin { - t.Errorf("Search(%q) returned %d results, want at least %d", tt.query, len(results), tt.wantMin) + t.Errorf("Search(%q) returned %d results, want at least %d", + tt.query, len(results), tt.wantMin) } // Verify all results contain the query if tt.query != "" { for _, r := range results { - if !contains(r.Handle, tt.query) && !contains(r.Description, tt.query) { - t.Errorf("Search result ASN %d doesn't contain query %q", r.ASN, tt.query) + if !strings.Contains(r.Handle, tt.query) && + !strings.Contains(r.Description, tt.query) { + t.Errorf("Search result ASN %d doesn't contain query %q", + r.ASN, tt.query) } } } @@ -148,13 +177,17 @@ func TestSearch(t *testing.T) { } func TestDataIntegrity(t *testing.T) { + t.Parallel() + // Verify no duplicate ASNs - all := All() + all := asinfo.All() seen := make(map[int]bool) + for _, info := range all { if seen[info.ASN] { t.Errorf("Duplicate ASN found: %d", info.ASN) } + seen[info.ASN] = true } @@ -163,6 +196,7 @@ func TestDataIntegrity(t *testing.T) { if info.Handle == "" && info.ASN != 0 { t.Errorf("ASN %d has empty handle", info.ASN) } + if info.Description == "" { t.Errorf("ASN %d has empty description", info.ASN) } @@ -174,8 +208,9 @@ func BenchmarkGet(b *testing.B) { asns := []int{1, 15169, 13335, 32934, 8075, 16509} b.ResetTimer() - for i := 0; i < b.N; i++ { - Get(asns[i%len(asns)]) + + for i := range b.N { + asinfo.Get(asns[i%len(asns)]) } } @@ -183,7 +218,8 @@ func BenchmarkSearch(b *testing.B) { queries := []string{"Google", "Amazon", "Microsoft", "University"} b.ResetTimer() - for i := 0; i < b.N; i++ { - Search(queries[i%len(queries)]) + + for i := range b.N { + asinfo.Search(queries[i%len(queries)]) } } diff --git a/script/bootstrap b/script/bootstrap index 3a156e5..79ebb69 100755 --- a/script/bootstrap +++ b/script/bootstrap @@ -2,14 +2,27 @@ # script/bootstrap: install all dependencies needed to build and develop # this repo. Idempotent: every install is guarded by a check so already # installed tools are skipped. Base tooling comes from nix, apt, brew, -# or apk (detected in that order); assumes NOTHING is present (not git, -# make, or go). +# or apk (detected in that order); assumes nothing is present. The +# installed node is used only when its major version is the pinned one; +# otherwise (node missing, or another major) the pinned version is +# installed via nvm (installing nvm itself first, from a hash-verified +# release archive, never curl | sh), and yarn and the packages are +# installed under it. Go formats the code and builds it on the host; the +# linter and the tests run only in Docker, as phases of the Dockerfile. set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" +# Pinned versions, 2026-07-06 +NODE_VERSION="22.17.0" +NVM_VERSION="0.40.3" +# sha256 of https://github.com/nvm-sh/nvm/archive/refs/tags/v0.40.3.tar.gz +NVM_SHA256="5f4d6aaa04a177dc93c985e31dbc411ab6b8c6e1e21d8015dbc1372625fcd1d0" +YARN_VERSION="1.22.22" + PKGMGR="" SUDO="" +APT_UPDATED="" detect_pkgmgr() { [ -n "$PKGMGR" ] && return 0 @@ -38,7 +51,14 @@ pkg_install() { detect_pkgmgr case "$PKGMGR" in nix) nix-env -iA "nixpkgs.$1" ;; - apt) $SUDO env DEBIAN_FRONTEND=noninteractive apt-get install -y "$2" ;; + apt) + # Package lists may be empty (fresh images); refresh once per run. + if [ -z "$APT_UPDATED" ]; then + $SUDO env DEBIAN_FRONTEND=noninteractive apt-get update + APT_UPDATED=1 + fi + $SUDO env DEBIAN_FRONTEND=noninteractive apt-get install -y "$2" + ;; brew) brew install "$3" ;; apk) apk add --no-cache "$4" ;; esac @@ -48,22 +68,100 @@ missing() { ! command -v "$1" >/dev/null 2>&1 } +# verify_sha256 +verify_sha256() { + if command -v sha256sum >/dev/null 2>&1; then + actual="$(sha256sum "$1" | cut -d' ' -f1)" + else + actual="$(shasum -a 256 "$1" | cut -d' ' -f1)" + fi + if [ "$actual" != "$2" ]; then + echo "bootstrap: sha256 mismatch for $1" >&2 + echo " expected: $2" >&2 + echo " actual: $actual" >&2 + exit 1 + fi +} + +# nvm is a bash script; run a command in a bash with nvm loaded +nvm_sh() { + bash -c ". \"\$HOME/.nvm/nvm.sh\" && $*" +} + +ensure_nvm() { + [ -s "$HOME/.nvm/nvm.sh" ] && return 0 + # nvm prerequisites; nvm itself requires bash + if missing bash; then pkg_install bash bash bash bash; fi + if missing curl; then pkg_install curl curl curl curl; fi + if missing git; then pkg_install git git git git; fi + tmp="$(mktemp -d)" + curl -fsSL -o "$tmp/nvm.tar.gz" \ + "https://github.com/nvm-sh/nvm/archive/refs/tags/v${NVM_VERSION}.tar.gz" + verify_sha256 "$tmp/nvm.tar.gz" "$NVM_SHA256" + mkdir -p "$HOME/.nvm" + tar -xzf "$tmp/nvm.tar.gz" -C "$HOME/.nvm" --strip-components=1 + rm -rf "$tmp" +} + +# True when the node on PATH has the pinned major version. Only the +# major is compared, as in the shared script/bootstrap of sneak/prompts. +# Another major is not used: the pinned yarn 1 prints a deprecation +# warning under node 24. script/fmt and script/fmt-check carry this +# function unchanged, to run the yarn installed here. +node_is_pinned_major() { + if ! command -v node >/dev/null 2>&1; then return 1; fi + installed="$(node --version)" + installed="${installed#v}" + [ "${installed%%.*}" = "${NODE_VERSION%%.*}" ] +} + +ensure_node() { + if node_is_pinned_major; then + echo "bootstrap: using node $(node --version)" + return 0 + fi + ensure_nvm + nvm_sh "nvm install $NODE_VERSION" + nvm_sh "nvm use $NODE_VERSION >/dev/null && \ + echo \"bootstrap: using node \$(node --version)\"" +} + +ensure_yarn() { + if ! node_is_pinned_major; then + nvm_sh "nvm use $NODE_VERSION >/dev/null && corepack enable && \ + corepack prepare yarn@$YARN_VERSION --activate" + return 0 + fi + if ! missing yarn; then return 0; fi + if ! missing corepack; then + corepack enable + corepack prepare "yarn@$YARN_VERSION" --activate + else + npm install -g "yarn@$YARN_VERSION" + fi +} + +install_js_deps() { + if node_is_pinned_major; then + yarn install --frozen-lockfile + else + nvm_sh "nvm use $NODE_VERSION >/dev/null && cd \"$ROOT\" && \ + yarn install --frozen-lockfile" + fi +} + main() { cd "$ROOT" - # Base tooling (every repo) - if missing git; then pkg_install git git git git; fi if missing make; then pkg_install gnumake make make make; fi + if missing git; then pkg_install git git git git; fi # Go toolchain if missing go; then pkg_install go golang go go; fi - # golangci-lint: packaged in nix, brew, and apk. There is no apt - # package; on apt systems install it manually from a hash-verified - # GitHub release archive (never curl | sh). - if missing golangci-lint; then - pkg_install golangci-lint golangci-lint golangci-lint golangci-lint - fi + ensure_node + ensure_yarn + install_js_deps go mod download diff --git a/script/check b/script/check index cc046f7..92875f7 100755 --- a/script/check +++ b/script/check @@ -1,7 +1,8 @@ #!/bin/sh # script/check: run all checks (test, lint, fmt-check). Our own -# extension to scripts-to-rule-them-all. Must not modify any files. -# Generic: usually needs no adaptation. +# extension to scripts-to-rule-them-all. test and lint are Docker +# phases; fmt-check is native, because a formatter writes the working +# tree. Must not modify any files. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" diff --git a/script/cibuild b/script/cibuild index 3da5857..2656771 100755 --- a/script/cibuild +++ b/script/cibuild @@ -1,14 +1,33 @@ #!/bin/sh -# script/cibuild: run the CI build. The Dockerfile runs script/check -# (via make check), so a successful build implies all checks pass. -# Generic: needs no adaptation. The Gitea workflow runs this on push. +# script/cibuild: run the CI build. It bootstraps first: a CI runner +# checks out and runs this and nothing else, and script/fmt-check runs +# the formatter on the host, which a pristine checkout cannot do. +# A new CACHEBUST value reruns every RUN step after the ARG CACHEBUST +# line, since a cached gate phase is a check that did not run; unlike +# --no-cache, it keeps the layers above that line and the cache mounts. +# This relies on every stage of the Dockerfile declaring ARG CACHEBUST +# before every step that lints, tests or compiles; a stage without that +# line is served from cache on an unchanged tree, with no warning. set -eu -ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" - docker build . + "$SCRIPT_DIR/bootstrap" + "$SCRIPT_DIR/check" + # The version and the tag each get their own line: a failing + # command substitution inside an argument does not trip `set -e`, + # so the inline form degrades silently to an empty constant. The + # VERSION build argument takes precedence over the version a build + # stage derives from the .git in the context. + version="$(git describe --tags --always --dirty 2>/dev/null || true)" + [ -n "$version" ] || version="unknown" + tag="$("$SCRIPT_DIR/projectname")" + docker build --build-arg CACHEBUST="$(date +%s)-$$" \ + --build-arg VERSION="$version" \ + -t "$tag" . } main "$@" diff --git a/script/docker b/script/docker index 07b626c..cc00d9c 100755 --- a/script/docker +++ b/script/docker @@ -1,8 +1,12 @@ #!/bin/sh # script/docker: build the Docker image tagged with the project name. # Identical in all repos; the tag comes from script/projectname. -# --no-cache because the gate phases the final stage depends on are RUN -# steps, and a cached one is a check that did not run. +# A new CACHEBUST value reruns every RUN step after the ARG CACHEBUST +# line, since a cached gate phase is a check that did not run; unlike +# --no-cache, it keeps the layers above that line and the cache mounts. +# This relies on every stage of the Dockerfile declaring ARG CACHEBUST +# before every step that lints, tests or compiles; a stage without that +# line is served from cache on an unchanged tree, with no warning. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" @@ -10,15 +14,17 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" - # Own line: a failing command substitution inside an argument does - # not trip `set -e`, so the inline form degrades silently to an - # empty constant. The VERSION build argument takes precedence over - # the version a build stage derives from the .git in the context. + # The version and the tag each get their own line: a failing + # command substitution inside an argument does not trip `set -e`, + # so the inline form degrades silently to an empty constant. The + # VERSION build argument takes precedence over the version a build + # stage derives from the .git in the context. version="$(git describe --tags --always --dirty 2>/dev/null || true)" [ -n "$version" ] || version="unknown" - docker build --no-cache \ + tag="$("$SCRIPT_DIR/projectname")" + docker build --build-arg CACHEBUST="$(date +%s)-$$" \ --build-arg VERSION="$version" \ - -t "$("$SCRIPT_DIR/projectname")" . + -t "$tag" . } main "$@" diff --git a/script/fmt b/script/fmt index e95d111..9906883 100755 --- a/script/fmt +++ b/script/fmt @@ -1,12 +1,43 @@ #!/bin/sh -# script/fmt: format all files (writes). +# script/fmt: format all files (writes): Go with go fmt, markdown with +# prettier. set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" +# Must match the pin in script/bootstrap. +NODE_VERSION="22.17.0" + +# True when the node on PATH has the pinned major version: the same +# function as in script/bootstrap, which uses it to decide where it +# installs yarn. +node_is_pinned_major() { + if ! command -v node >/dev/null 2>&1; then return 1; fi + installed="$(node --version)" + installed="${installed#v}" + [ "${installed%%.*}" = "${NODE_VERSION%%.*}" ] +} + +# Run the yarn script/bootstrap installed: the yarn on PATH when the +# node on PATH has the pinned major version, and otherwise yarn under +# the pinned node in nvm, which bootstrap leaves off the PATH of the +# shell that called it. nvm is a bash script, hence the subshell. +run_yarn() { + if node_is_pinned_major && command -v yarn >/dev/null 2>&1; then + exec yarn "$@" + fi + if node_is_pinned_major || [ ! -s "$HOME/.nvm/nvm.sh" ]; then + echo "fmt: no yarn; run script/bootstrap first" >&2 + exit 1 + fi + exec bash -c '. "$HOME/.nvm/nvm.sh" && nvm use "$1" >/dev/null && + shift && exec yarn "$@"' bash "$NODE_VERSION" "$@" +} + main() { cd "$ROOT" go fmt ./... + run_yarn run prettier --write '**/*.md' --tab-width 4 --prose-wrap always } main "$@" diff --git a/script/fmt-check b/script/fmt-check index 57e8e5c..924ccad 100755 --- a/script/fmt-check +++ b/script/fmt-check @@ -5,6 +5,35 @@ set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" +# Must match the pin in script/bootstrap. +NODE_VERSION="22.17.0" + +# True when the node on PATH has the pinned major version: the same +# function as in script/bootstrap, which uses it to decide where it +# installs yarn. +node_is_pinned_major() { + if ! command -v node >/dev/null 2>&1; then return 1; fi + installed="$(node --version)" + installed="${installed#v}" + [ "${installed%%.*}" = "${NODE_VERSION%%.*}" ] +} + +# Run the yarn script/bootstrap installed: the yarn on PATH when the +# node on PATH has the pinned major version, and otherwise yarn under +# the pinned node in nvm, which bootstrap leaves off the PATH of the +# shell that called it. nvm is a bash script, hence the subshell. +run_yarn() { + if node_is_pinned_major && command -v yarn >/dev/null 2>&1; then + exec yarn "$@" + fi + if node_is_pinned_major || [ ! -s "$HOME/.nvm/nvm.sh" ]; then + echo "fmt-check: no yarn; run script/bootstrap first" >&2 + exit 1 + fi + exec bash -c '. "$HOME/.nvm/nvm.sh" && nvm use "$1" >/dev/null && + shift && exec yarn "$@"' bash "$NODE_VERSION" "$@" +} + main() { cd "$ROOT" unformatted="$(gofmt -l .)" @@ -13,6 +42,7 @@ main() { echo "$unformatted" >&2 exit 1 fi + run_yarn run prettier --check '**/*.md' --tab-width 4 --prose-wrap always } main "$@" diff --git a/script/lint b/script/lint index a675833..451a067 100755 --- a/script/lint +++ b/script/lint @@ -1,13 +1,28 @@ #!/bin/sh -# script/lint: run the linters. +# script/lint: run the linter. Linting is a phase of the Dockerfile and +# this builds that phase alone; the linter is never installed or run on +# a developer host, where a shared result cache and a host-global lock +# make its answer untrustworthy. +# +# The phase is not the last stage in the file, so it is built only when +# --target names it. A new CACHEBUST value reruns every RUN step after +# the ARG CACHEBUST line, since a cached lint layer is a lint that did +# not run; unlike --no-cache, it keeps the layers above that line and +# the cache mounts. This relies on every stage of the Dockerfile +# declaring ARG CACHEBUST before every step that lints, tests or +# compiles; a stage without that line is served from cache on an +# unchanged tree, with no warning. +# --output type=cacheonly writes no image, since nothing uses one. set -eu -ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" - go vet ./... - golangci-lint run + docker build --build-arg CACHEBUST="$(date +%s)-$$" \ + --target lint \ + --output type=cacheonly . } main "$@" diff --git a/script/test b/script/test index bffb59e..1622a11 100755 --- a/script/test +++ b/script/test @@ -1,17 +1,24 @@ #!/bin/sh -# script/test: run the test suite. On failure, rerun verbosely so the -# failing tests are visible in the output. +# script/test: run the test suite. Testing is a phase of the Dockerfile +# and this builds that phase alone, on the same terms as script/lint: +# --target because a phase that is not the last stage is built only when +# named, and a new CACHEBUST value because a cached test layer is a test +# that did not run; unlike --no-cache, it keeps the layers above the ARG +# CACHEBUST line and the cache mounts. This relies on every stage of the +# Dockerfile declaring ARG CACHEBUST before every step that lints, tests +# or compiles; a stage without that line is served from cache on an +# unchanged tree, with no warning. --output type=cacheonly writes no +# image, since nothing uses one. set -eu -ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" - go test -short -timeout 30s -race -cover ./... || { - echo "--- Rerunning with -v for details ---" - go test -short -timeout 30s -race -v ./... - exit 1 - } + docker build --build-arg CACHEBUST="$(date +%s)-$$" \ + --target test \ + --output type=cacheonly . } main "$@" diff --git a/yarn.lock b/yarn.lock new file mode 100644 index 0000000..d846639 --- /dev/null +++ b/yarn.lock @@ -0,0 +1,8 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +prettier@3.8.1: + version "3.8.1" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.8.1.tgz#edf48977cf991558f4fcbd8a3ba6015ba2a3a173" + integrity sha512-UOnG6LftzbdaHZcKoPFtOcCKztrQ57WkHDeRD9t/PTQtmT0NHSeWWepj6pS0z/N7+08BHFDQVUrfmfMRcZwbMg==