Compare commits
1
Commits
next
..
bc7c5e18a5
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bc7c5e18a5 |
+10
-85
@@ -1,107 +1,32 @@
|
|||||||
# .dockerignore does NOT use .gitignore semantics. Docker matches with
|
# Docker does not read .gitignore, and a pattern here matches from the root of
|
||||||
# moby/patternmatcher: filepath.Match plus `**`, so `*` does not cross
|
# the build context only: a pattern meant for every directory needs `**/`.
|
||||||
# `/` and an unprefixed pattern is anchored at the context root. Every
|
|
||||||
# depth-independent pattern therefore needs `**/`, or `config/.env` and
|
|
||||||
# `certs/server.key` still ship while this file reads as solved. Only
|
|
||||||
# genuinely root-anchored entries go unprefixed. Never transplant these
|
|
||||||
# into .gitignore, where `**/` is wrong.
|
|
||||||
#
|
|
||||||
# Matching is case-sensitive, so secrets use character ranges rather
|
|
||||||
# than an ALL-CAPS twin, which would still miss `Server.Key`.
|
|
||||||
#
|
|
||||||
# Extend with this repo's own host-built artifacts, written anchored:
|
|
||||||
# `/myapp`, never `**/myapp`, which also matches `cmd/myapp/` and
|
|
||||||
# deletes the package directory from the context.
|
|
||||||
|
|
||||||
# .git is sent without its config. Without a VERSION build argument the
|
# .git is sent without its config. Without a VERSION build argument the
|
||||||
# stage that compiles runs `git describe --tags --always` on .git, which
|
# stage that compiles runs `git describe --tags --always` on .git, which
|
||||||
# does not need .git/config; that file can hold a credential, such as a
|
# does not need .git/config; that file can hold a credential, such as a
|
||||||
# password in a remote URL or the token the CI checkout step stores there.
|
# password in a remote URL or the token the CI checkout step stores there.
|
||||||
# Each submodule keeps a config with the same exposure in its git directory
|
.git/config
|
||||||
# under .git/modules/, nested again for a submodule's own submodules, or in
|
|
||||||
# its own .git directory when it keeps one.
|
|
||||||
# KNOWN GAP: a submodule whose name has a `config` segment (`config`,
|
|
||||||
# `deploy/config`, `config/lib`) loses its whole git directory, because
|
|
||||||
# `**/.git/modules/**/config` also matches that segment's directory
|
|
||||||
# under .git/modules/. Go's version stamping then fails the build;
|
|
||||||
# nothing leaks. Name such a submodule without that segment:
|
|
||||||
# `git submodule add --name`.
|
|
||||||
**/.git/config
|
|
||||||
**/.git/modules/**/config
|
|
||||||
|
|
||||||
# Agent scratch: one full checkout of the repo per in-flight agent.
|
# Local build and debug output: `make build`, `make run`, `make asupdate`, test
|
||||||
# Anchored because it occurs once where agents run at the repo root.
|
# binaries, coverage profiles and source archives.
|
||||||
# KNOWN GAP: a repo running agents in subdirectories still ships
|
|
||||||
# `services/api/.claude/` and must add its own anchored entry.
|
|
||||||
.claude
|
|
||||||
|
|
||||||
# Environment files. `*.env` covers bare `.env` and the `prod.env`
|
|
||||||
# convention. Re-include a committed template with a negation if the
|
|
||||||
# build needs one: `!docs/example.env`.
|
|
||||||
**/*.[eE][nN][vV]
|
|
||||||
**/.[eE][nN][vV].*
|
|
||||||
**/.[eE][nN][vV][rR][cC]
|
|
||||||
|
|
||||||
# Private keys and the bundles carrying them. Public certificates
|
|
||||||
# (*.crt, *.cer) are deliberately absent: they are legitimate inputs.
|
|
||||||
**/*.[pP][eE][mM]
|
|
||||||
**/*.[kK][eE][yY]
|
|
||||||
**/*.[pP]12
|
|
||||||
**/*.[pP][fF][xX]
|
|
||||||
**/[iI][dD]_[rR][sS][aA]
|
|
||||||
**/[iI][dD]_[dD][sS][aA]
|
|
||||||
**/[iI][dD]_[eE][cC][dD][sS][aA]
|
|
||||||
**/[iI][dD]_[eE][cC][dD][sS][aA]_[sS][kK]
|
|
||||||
**/[iI][dD]_[eE][dD]25519
|
|
||||||
**/[iI][dD]_[eE][dD]25519_[sS][kK]
|
|
||||||
|
|
||||||
# Dependencies: restored inside the image, never copied in.
|
|
||||||
**/node_modules
|
|
||||||
|
|
||||||
# OS metadata.
|
|
||||||
**/.DS_Store
|
|
||||||
**/Thumbs.db
|
|
||||||
|
|
||||||
# Editor state: never a build input, and it churns COPY.
|
|
||||||
**/*.swp
|
|
||||||
**/*.swo
|
|
||||||
**/*~
|
|
||||||
**/*.bak
|
|
||||||
**/.idea
|
|
||||||
**/.vscode
|
|
||||||
**/*.sublime-*
|
|
||||||
|
|
||||||
# This repository's host-built artifacts: `make build` writes bin/, `go
|
|
||||||
# build` in the root the binaries, `make run` log.txt, `make asupdate` its
|
|
||||||
# .tmp file; logs, test binaries, coverage profiles, source archives, and
|
|
||||||
# the binaries and libraries a build for another platform writes.
|
|
||||||
/bin
|
/bin
|
||||||
/routewatch
|
|
||||||
/asinfo-gen
|
|
||||||
/streamdumper
|
|
||||||
/log.txt
|
/log.txt
|
||||||
/out
|
/out
|
||||||
/pkg/asinfo/asdata.json
|
/pkg/asinfo/asdata.json
|
||||||
**/*.tar.zst
|
**/*.tar.zst
|
||||||
**/*.log
|
|
||||||
**/*.test
|
**/*.test
|
||||||
**/*.out
|
**/*.out
|
||||||
**/*.tmp
|
**/*.tmp
|
||||||
**/*.exe
|
|
||||||
**/*.dll
|
|
||||||
**/*.so
|
|
||||||
**/*.dylib
|
|
||||||
|
|
||||||
# Local databases. The image carries a source archive of the whole build
|
# Local databases and secrets. The image carries a source archive of the whole
|
||||||
# context, so these would otherwise ship inside it. The daemon's database
|
# build context, so these would otherwise ship inside it.
|
||||||
# is db.sqlite, with its -wal and -shm files while it runs.
|
|
||||||
**/*.db
|
**/*.db
|
||||||
**/*.db-journal
|
**/*.db-journal
|
||||||
**/*.db-wal
|
**/*.db-wal
|
||||||
**/*.sqlite
|
**/.env
|
||||||
**/*.sqlite-wal
|
|
||||||
**/*.sqlite-shm
|
|
||||||
|
|
||||||
# A local Go workspace points at directories outside the build context.
|
# A local Go workspace points at directories outside the build context.
|
||||||
/go.work
|
/go.work
|
||||||
/go.work.sum
|
/go.work.sum
|
||||||
|
|
||||||
|
**/.DS_Store
|
||||||
|
|||||||
@@ -10,9 +10,3 @@ insert_final_newline = true
|
|||||||
|
|
||||||
[Makefile]
|
[Makefile]
|
||||||
indent_style = tab
|
indent_style = tab
|
||||||
|
|
||||||
[*.go]
|
|
||||||
indent_style = tab
|
|
||||||
|
|
||||||
# This repository's own sections, such as one for another language it
|
|
||||||
# uses, go below this comment, and a re-vendor keeps them.
|
|
||||||
|
|||||||
@@ -1,20 +1,9 @@
|
|||||||
name: check
|
name: check
|
||||||
on: [push]
|
on: [push]
|
||||||
# Free the shared runner: a new push cancels only the same branch's older run.
|
|
||||||
concurrency:
|
|
||||||
group: ${{ github.workflow }}-${{ github.ref }}
|
|
||||||
cancel-in-progress: true
|
|
||||||
jobs:
|
jobs:
|
||||||
check:
|
check:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
# Free the shared runner from a hung build.
|
|
||||||
timeout-minutes: 20
|
|
||||||
steps:
|
steps:
|
||||||
# actions/checkout v4.2.2, 2026-02-22
|
# actions/checkout v4.2.2, 2026-02-28
|
||||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
|
||||||
# script/cibuild needs no token, so none is left in .git/config.
|
|
||||||
with:
|
|
||||||
persist-credentials: false
|
|
||||||
# All history and tags, so git describe finds the version tag.
|
|
||||||
fetch-depth: 0
|
|
||||||
- run: script/cibuild
|
- run: script/cibuild
|
||||||
|
|||||||
+27
-69
@@ -1,83 +1,41 @@
|
|||||||
# OS
|
# Binaries for programs and plugins
|
||||||
.DS_Store
|
|
||||||
Thumbs.db
|
|
||||||
|
|
||||||
# Editors
|
|
||||||
*.swp
|
|
||||||
*.swo
|
|
||||||
*~
|
|
||||||
*.bak
|
|
||||||
.idea/
|
|
||||||
.vscode/
|
|
||||||
*.sublime-*
|
|
||||||
|
|
||||||
# Agent scratch (worktrees of this repo, created and destroyed by
|
|
||||||
# in-flight tooling). Unanchored: .gitignore patterns already match at
|
|
||||||
# every depth, so no prefix is wanted here. This is not a .dockerignore
|
|
||||||
# entry and must not be given a `**/` prefix on the way into one.
|
|
||||||
.claude/
|
|
||||||
|
|
||||||
# Node
|
|
||||||
node_modules/
|
|
||||||
|
|
||||||
# Secrets. Unanchored like every entry above, so each matches at every
|
|
||||||
# depth. Matching is case-sensitive on Linux, so names use character
|
|
||||||
# ranges rather than a lowercase form that misses `Server.Key`.
|
|
||||||
|
|
||||||
# Environment files. `*.env` covers bare `.env` and the `prod.env`
|
|
||||||
# convention. Only the templates `example.env` and `sample.env` are
|
|
||||||
# re-included below. A repository that commits any other template adds
|
|
||||||
# its own negation at the end of this file, for example `!.env.example`.
|
|
||||||
*.[eE][nN][vV]
|
|
||||||
.[eE][nN][vV].*
|
|
||||||
.[eE][nN][vV][rR][cC]
|
|
||||||
!example.env
|
|
||||||
!sample.env
|
|
||||||
|
|
||||||
# Private keys and the bundles carrying them.
|
|
||||||
*.[pP][eE][mM]
|
|
||||||
*.[kK][eE][yY]
|
|
||||||
*.[pP]12
|
|
||||||
*.[pP][fF][xX]
|
|
||||||
[iI][dD]_[rR][sS][aA]
|
|
||||||
[iI][dD]_[dD][sS][aA]
|
|
||||||
[iI][dD]_[eE][cC][dD][sS][aA]
|
|
||||||
[iI][dD]_[eE][cC][dD][sS][aA]_[sS][kK]
|
|
||||||
[iI][dD]_[eE][dD]25519
|
|
||||||
[iI][dD]_[eE][dD]25519_[sS][kK]
|
|
||||||
|
|
||||||
# This repository's own entries, such as its build outputs, go below
|
|
||||||
# this comment, and a re-vendor keeps them. Anchor a binary built at the
|
|
||||||
# root: `/myapp`, never `myapp`, which also ignores `cmd/myapp/`.
|
|
||||||
|
|
||||||
# Go: logs, test binaries, coverage profiles, and the binaries `make build`
|
|
||||||
# and `go build` in the repository root write.
|
|
||||||
*.log
|
|
||||||
*.out
|
|
||||||
*.test
|
|
||||||
*.exe
|
*.exe
|
||||||
*.exe~
|
*.exe~
|
||||||
*.dll
|
*.dll
|
||||||
*.so
|
*.so
|
||||||
*.dylib
|
*.dylib
|
||||||
/bin/
|
/bin/
|
||||||
/routewatch
|
|
||||||
/asinfo-gen
|
.DS_Store
|
||||||
/streamdumper
|
log.txt
|
||||||
|
|
||||||
|
# Test binary, built with `go test -c`
|
||||||
|
*.test
|
||||||
|
|
||||||
|
# Output of the go coverage tool, specifically when used with LiteIDE
|
||||||
|
*.out
|
||||||
|
|
||||||
|
# Dependency directories (remove the comment below to include it)
|
||||||
|
# vendor/
|
||||||
|
|
||||||
|
# Go workspace file
|
||||||
go.work
|
go.work
|
||||||
go.work.sum
|
go.work.sum
|
||||||
|
|
||||||
# Local databases, temporary files and debug output. The daemon's
|
# env file
|
||||||
# database is db.sqlite, with its -wal and -shm files while it runs.
|
.env
|
||||||
|
|
||||||
|
# Database files
|
||||||
*.db
|
*.db
|
||||||
*.db-journal
|
*.db-journal
|
||||||
*.db-wal
|
*.db-wal
|
||||||
*.sqlite
|
|
||||||
*.sqlite-wal
|
|
||||||
*.sqlite-shm
|
|
||||||
*.tmp
|
|
||||||
log.txt
|
|
||||||
out
|
|
||||||
|
|
||||||
# Raw AS data (only the gzipped version is committed).
|
# Temporary files
|
||||||
|
*.tmp
|
||||||
|
|
||||||
|
# Raw AS data (we only commit the gzipped version)
|
||||||
pkg/asinfo/asdata.json
|
pkg/asinfo/asdata.json
|
||||||
|
|
||||||
|
# Debug output files
|
||||||
|
out
|
||||||
|
log.txt
|
||||||
|
|||||||
+80
-89
@@ -1,101 +1,92 @@
|
|||||||
version: "2"
|
version: "2"
|
||||||
|
|
||||||
# Config schema uses the golangci-lint v2 layout (settings live under
|
|
||||||
# linters.settings, not top-level linters-settings) so that the
|
|
||||||
# thresholds below are actually applied by golangci-lint >= v2.
|
|
||||||
|
|
||||||
run:
|
run:
|
||||||
timeout: 5m
|
go: "1.24"
|
||||||
modules-download-mode: readonly
|
tests: false
|
||||||
|
|
||||||
linters:
|
linters:
|
||||||
default: all
|
|
||||||
enable:
|
enable:
|
||||||
# Successor to the deprecated gomodguard. Named explicitly, rather than
|
# Additional linters requested
|
||||||
# left to `default: all`, because it carries the module policy below.
|
- testifylint # Checks usage of github.com/stretchr/testify
|
||||||
- gomodguard_v2
|
- usetesting # usetesting is an analyzer that detects using os.Setenv instead of t.Setenv since Go 1.17
|
||||||
disable:
|
# - tagliatelle # Disabled: we need snake_case for external API compatibility
|
||||||
# Genuinely incompatible with project patterns
|
- nlreturn # nlreturn checks for a new line before return and branch statements
|
||||||
- exhaustruct # Requires all struct fields
|
- nilnil # Checks that there is no simultaneous return of nil error and an invalid value
|
||||||
- exhaustruct_v5 # Requires all struct fields (successor to exhaustruct)
|
- nestif # Reports deeply nested if statements
|
||||||
- godot # Requires comments to end with periods
|
- mnd # An analyzer to detect magic numbers
|
||||||
- wrapcheck # Too verbose for internal packages
|
- lll # Reports long lines
|
||||||
- varnamelen # Short names like db, id are idiomatic Go
|
- intrange # intrange is a linter to find places where for loops could make use of an integer range
|
||||||
# Deprecated: the warning is attached to the old name, so it is
|
- gochecknoglobals # Check that no global variables exist
|
||||||
# silenced by disabling that name, not by enabling the successor.
|
|
||||||
- wsl # Deprecated, replaced by wsl_v5
|
# Default/existing linters that are commonly useful
|
||||||
- gomodguard # Deprecated, replaced by gomodguard_v2
|
- govet
|
||||||
# Misses findings at random in v2.14.0; back once a pinned release fixes it
|
- errcheck
|
||||||
- canonicalheader
|
- staticcheck
|
||||||
settings:
|
- unused
|
||||||
|
- ineffassign
|
||||||
|
- misspell
|
||||||
|
- revive
|
||||||
|
- gosec
|
||||||
|
- unconvert
|
||||||
|
- unparam
|
||||||
|
|
||||||
|
linters-settings:
|
||||||
lll:
|
lll:
|
||||||
line-length: 88
|
line-length: 120
|
||||||
funlen:
|
|
||||||
lines: 80
|
nestif:
|
||||||
statements: 50
|
min-complexity: 4
|
||||||
cyclop:
|
|
||||||
max-complexity: 15
|
nlreturn:
|
||||||
dupl:
|
block-size: 2
|
||||||
threshold: 100
|
|
||||||
depguard:
|
revive:
|
||||||
# Test-support code must not be compiled into the shipped binary. A
|
|
||||||
# test-support package exists to hand a test privileges the program
|
|
||||||
# itself must never have, so a file that is not a test must not import
|
|
||||||
# one. Test files, and the files inside a package whose directory name
|
|
||||||
# ends in `test`, are where that code belongs, and are exempt.
|
|
||||||
#
|
|
||||||
# The deny list below is the one part of this file a repository is
|
|
||||||
# expected to extend, and the only part it may. depguard matches an
|
|
||||||
# import path against a list of prefixes, so it cannot be told "any path
|
|
||||||
# whose last segment ends in test"; a repository's own test-support
|
|
||||||
# packages have to be named here one at a time, by full import path,
|
|
||||||
# under a module path that differs from repository to repository. Add
|
|
||||||
# them; change nothing else.
|
|
||||||
rules:
|
rules:
|
||||||
test-support:
|
- name: var-naming
|
||||||
list-mode: lax
|
arguments:
|
||||||
files:
|
- []
|
||||||
- "$all"
|
- []
|
||||||
- "!$test"
|
- "upperCaseConst=true"
|
||||||
- "!**/*test/**"
|
|
||||||
deny:
|
tagliatelle:
|
||||||
- pkg: net/http/httptest
|
case:
|
||||||
desc: >-
|
rules:
|
||||||
Test-support code belongs in test files and in packages whose
|
json: snake
|
||||||
directory name ends in test, not in the shipped binary.
|
yaml: snake
|
||||||
# Only decisions already recorded in the Go package defaults are
|
xml: snake
|
||||||
# listed here. Every entry matches the module path exactly.
|
bson: snake
|
||||||
gomodguard_v2:
|
|
||||||
blocked:
|
testifylint:
|
||||||
- module: github.com/rs/zerolog
|
enable-all: true
|
||||||
recommendations:
|
|
||||||
- log/slog
|
usetesting: {}
|
||||||
reason: "Structured logging is stdlib log/slog."
|
|
||||||
# One entry per pre-fork module path, because the later releases
|
|
||||||
# are separate paths. A prefix match would be shorter but would
|
|
||||||
# also reach github.com/go-redis/redismock, the test double for
|
|
||||||
# the successor these entries recommend.
|
|
||||||
- module: github.com/go-redis/redis
|
|
||||||
recommendations:
|
|
||||||
- github.com/redis/go-redis/v9
|
|
||||||
reason: "Pre-fork module; use the maintained go-redis v9."
|
|
||||||
- module: github.com/go-redis/redis/v7
|
|
||||||
recommendations:
|
|
||||||
- github.com/redis/go-redis/v9
|
|
||||||
reason: "Pre-fork module; use the maintained go-redis v9."
|
|
||||||
- module: github.com/go-redis/redis/v8
|
|
||||||
recommendations:
|
|
||||||
- github.com/redis/go-redis/v9
|
|
||||||
reason: "Pre-fork module; use the maintained go-redis v9."
|
|
||||||
- module: github.com/sergi/go-diff
|
|
||||||
recommendations:
|
|
||||||
- github.com/aymanbagabas/go-udiff
|
|
||||||
reason: "No unified diff output; use go-udiff."
|
|
||||||
- module: github.com/hexops/gotextdiff
|
|
||||||
recommendations:
|
|
||||||
- github.com/aymanbagabas/go-udiff
|
|
||||||
reason: "Unmaintained fork; use go-udiff."
|
|
||||||
|
|
||||||
issues:
|
issues:
|
||||||
max-issues-per-linter: 0
|
max-issues-per-linter: 0
|
||||||
max-same-issues: 0
|
max-same-issues: 0
|
||||||
|
exclude-rules:
|
||||||
|
# Exclude unused parameter warnings for cobra command signatures
|
||||||
|
- text: "parameter '(args|cmd)' seems to be unused"
|
||||||
|
linters:
|
||||||
|
- revive
|
||||||
|
|
||||||
|
# Allow ALL_CAPS constant names
|
||||||
|
- text: "don't use ALL_CAPS in Go names"
|
||||||
|
linters:
|
||||||
|
- revive
|
||||||
|
|
||||||
|
# Allow snake_case JSON tags for external API compatibility
|
||||||
|
- path: "internal/types/ris.go"
|
||||||
|
linters:
|
||||||
|
- tagliatelle
|
||||||
|
|
||||||
|
# Allow snake_case JSON tags for database models
|
||||||
|
- path: "internal/database/models.go"
|
||||||
|
linters:
|
||||||
|
- tagliatelle
|
||||||
|
|
||||||
|
# Allow generic package name for types that define data structures
|
||||||
|
- path: "internal/types/"
|
||||||
|
text: "avoid meaningless package names"
|
||||||
|
linters:
|
||||||
|
- revive
|
||||||
|
|||||||
@@ -1,95 +0,0 @@
|
|||||||
# IMPORTANT RULES
|
|
||||||
|
|
||||||
- Claude is an inanimate tool. The spam that Claude attempts to insert into
|
|
||||||
commit messages (which it erroneously refers to as "attribution") is not
|
|
||||||
attribution, as I am the sole author of code created using Claude. It is
|
|
||||||
corporate advertising for Anthropic and is therefore completely unacceptable
|
|
||||||
in commit messages.
|
|
||||||
|
|
||||||
- Tests should always be run before committing code. No commits should be made
|
|
||||||
that do not pass tests.
|
|
||||||
|
|
||||||
- Code should always be formatted before committing. Do not commit unformatted
|
|
||||||
code.
|
|
||||||
|
|
||||||
- Code should always be linted and linter errors fixed before committing. NEVER
|
|
||||||
commit code that does not pass the linter. DO NOT modify the linter config
|
|
||||||
unless specifically instructed.
|
|
||||||
|
|
||||||
- The test suite is fast and local. When running tests, NEVER run individual
|
|
||||||
parts of the test suite, always run the whole thing by running "make test".
|
|
||||||
|
|
||||||
- Do not stop working on a task until you have reached the definition of done
|
|
||||||
provided to you in the initial instruction. Don't do part or most of the work,
|
|
||||||
do all of the work until the criteria for done are met.
|
|
||||||
|
|
||||||
- When you complete each task, if the tests are passing and the code is
|
|
||||||
formatted and there are no linter errors, always commit and push your work.
|
|
||||||
Use a good commit message and don't mention any author or co-author
|
|
||||||
attribution.
|
|
||||||
|
|
||||||
- Do not create additional files in the root directory of the project without
|
|
||||||
asking permission first. Configuration files, documentation, and build files
|
|
||||||
are acceptable in the root, but source code and other files should be
|
|
||||||
organized in appropriate subdirectories.
|
|
||||||
|
|
||||||
- Do not use bare strings or numbers in code, especially if they appear anywhere
|
|
||||||
more than once. Always define a constant (usually at the top of the file) and
|
|
||||||
give it a descriptive name, then use that constant in the code instead of the
|
|
||||||
bare string or number.
|
|
||||||
|
|
||||||
- If you are fixing a bug, write a test first that reproduces the bug and fails,
|
|
||||||
and then fix the bug in the code, using the test to verify that the fix
|
|
||||||
worked.
|
|
||||||
|
|
||||||
- When implementing new features, be aware of potential side-effects (such as
|
|
||||||
state files on disk, data in the database, etc.) and ensure that it is
|
|
||||||
possible to mock or stub these side-effects in tests when designing an API.
|
|
||||||
|
|
||||||
- When dealing with dates and times or timestamps, always use, display, and
|
|
||||||
store UTC. Set the local timezone to UTC on startup. If the user needs to see
|
|
||||||
the time in a different timezone, store the user's timezone in a separate
|
|
||||||
field and convert the UTC time to the user's timezone when displaying it. For
|
|
||||||
internal use and internal applications and administrative purposes, always
|
|
||||||
display UTC.
|
|
||||||
|
|
||||||
- When implementing programs, put the main.go in ./cmd/<program_name>/main.go
|
|
||||||
and put the program's code in ./internal/<program_name>/. This allows for
|
|
||||||
multiple programs to be implemented in the same repository without cluttering
|
|
||||||
the root directory. main.go should simply import and call
|
|
||||||
<program_name>.CLIEntry(). The full implementation should be in
|
|
||||||
./internal/<program_name>/.
|
|
||||||
|
|
||||||
- When you are instructed to make the tests pass, DO NOT delete tests, skip
|
|
||||||
tests, or change the tests specifically to make them pass (unless there is a
|
|
||||||
bug in the test). This is cheating, and it is bad. You should only be
|
|
||||||
modifying the test if it is incorrect or if the test is no longer relevant. In
|
|
||||||
almost all cases, you should be fixing the code that is being tested, or
|
|
||||||
updating the tests to match a refactored implementation.
|
|
||||||
|
|
||||||
- Always write a `Makefile` with the default target being `test`, and with a
|
|
||||||
`fmt` target that formats the code. The `test` target should run all tests in
|
|
||||||
the project, and the `fmt` target should format the code. `test` should also
|
|
||||||
have a prerequisite target `lint` that should run any linters that are
|
|
||||||
configured for the project.
|
|
||||||
|
|
||||||
- After each completed bugfix or feature, the code must be committed. Do all of
|
|
||||||
the pre-commit checks (test, lint, fmt) before committing, of course. After
|
|
||||||
each commit, push to the remote.
|
|
||||||
|
|
||||||
- Always write tests, even if they are extremely simple and just check for
|
|
||||||
correct syntax (ability to compile/import). If you are writing a new feature,
|
|
||||||
write a test for it. You don't need to target complete coverage, but you
|
|
||||||
should at least test any new functionality you add.
|
|
||||||
|
|
||||||
- Always use structured logging. Log any relevant state/context with the
|
|
||||||
messages (but do not log secrets). If stdout is not a terminal, output the
|
|
||||||
structured logs in jsonl format. Use go's log/slog.
|
|
||||||
|
|
||||||
- You do not need to summarize your changes in the chat after making them.
|
|
||||||
Making the changes and committing them is sufficient. If anything out of the
|
|
||||||
ordinary happened, please explain it, but in the normal case where you found
|
|
||||||
and fixed the bug, or implemented the feature, there is no need for the
|
|
||||||
end-of-change summary.
|
|
||||||
|
|
||||||
- When testing daemons, use a 15 second timeout always.
|
|
||||||
@@ -1,20 +0,0 @@
|
|||||||
---
|
|
||||||
title: API Policies
|
|
||||||
last_modified: 2026-10-27
|
|
||||||
---
|
|
||||||
|
|
||||||
A value in an API response is never a string with a label or unit in it. A value
|
|
||||||
of a native type is sent as that type, and its unit goes in the field name, for
|
|
||||||
example `_ms`.
|
|
||||||
|
|
||||||
Wrong:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{ "execution_time": "0 ms" }
|
|
||||||
```
|
|
||||||
|
|
||||||
Right:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{ "execution_time_ms": 0 }
|
|
||||||
```
|
|
||||||
@@ -0,0 +1,97 @@
|
|||||||
|
# IMPORTANT RULES
|
||||||
|
|
||||||
|
* Claude is an inanimate tool. The spam that Claude attempts to insert into
|
||||||
|
commit messages (which it erroneously refers to as "attribution") is not
|
||||||
|
attribution, as I am the sole author of code created using Claude. It is
|
||||||
|
corporate advertising for Anthropic and is therefore completely
|
||||||
|
unacceptable in commit messages.
|
||||||
|
|
||||||
|
* Tests should always be run before committing code. No commits should be
|
||||||
|
made that do not pass tests.
|
||||||
|
|
||||||
|
* Code should always be formatted before committing. Do not commit
|
||||||
|
unformatted code.
|
||||||
|
|
||||||
|
* Code should always be linted and linter errors fixed before committing.
|
||||||
|
NEVER commit code that does not pass the linter. DO NOT modify the linter
|
||||||
|
config unless specifically instructed.
|
||||||
|
|
||||||
|
* The test suite is fast and local. When running tests, NEVER run
|
||||||
|
individual parts of the test suite, always run the whole thing by running
|
||||||
|
"make test".
|
||||||
|
|
||||||
|
* Do not stop working on a task until you have reached the definition of
|
||||||
|
done provided to you in the initial instruction. Don't do part or most of
|
||||||
|
the work, do all of the work until the criteria for done are met.
|
||||||
|
|
||||||
|
* When you complete each task, if the tests are passing and the code is
|
||||||
|
formatted and there are no linter errors, always commit and push your
|
||||||
|
work. Use a good commit message and don't mention any author or co-author
|
||||||
|
attribution.
|
||||||
|
|
||||||
|
* Do not create additional files in the root directory of the project
|
||||||
|
without asking permission first. Configuration files, documentation, and
|
||||||
|
build files are acceptable in the root, but source code and other files
|
||||||
|
should be organized in appropriate subdirectories.
|
||||||
|
|
||||||
|
* Do not use bare strings or numbers in code, especially if they appear
|
||||||
|
anywhere more than once. Always define a constant (usually at the top of
|
||||||
|
the file) and give it a descriptive name, then use that constant in the
|
||||||
|
code instead of the bare string or number.
|
||||||
|
|
||||||
|
* If you are fixing a bug, write a test first that reproduces the bug and
|
||||||
|
fails, and then fix the bug in the code, using the test to verify that the
|
||||||
|
fix worked.
|
||||||
|
|
||||||
|
* When implementing new features, be aware of potential side-effects (such
|
||||||
|
as state files on disk, data in the database, etc.) and ensure that it is
|
||||||
|
possible to mock or stub these side-effects in tests when designing an
|
||||||
|
API.
|
||||||
|
|
||||||
|
* When dealing with dates and times or timestamps, always use, display, and
|
||||||
|
store UTC. Set the local timezone to UTC on startup. If the user needs
|
||||||
|
to see the time in a different timezone, store the user's timezone in a
|
||||||
|
separate field and convert the UTC time to the user's timezone when
|
||||||
|
displaying it. For internal use and internal applications and
|
||||||
|
administrative purposes, always display UTC.
|
||||||
|
|
||||||
|
* When implementing programs, put the main.go in
|
||||||
|
./cmd/<program_name>/main.go and put the program's code in
|
||||||
|
./internal/<program_name>/. This allows for multiple programs to be
|
||||||
|
implemented in the same repository without cluttering the root directory.
|
||||||
|
main.go should simply import and call <program_name>.CLIEntry(). The
|
||||||
|
full implementation should be in ./internal/<program_name>/.
|
||||||
|
|
||||||
|
* When you are instructed to make the tests pass, DO NOT delete tests, skip
|
||||||
|
tests, or change the tests specifically to make them pass (unless there
|
||||||
|
is a bug in the test). This is cheating, and it is bad. You should only
|
||||||
|
be modifying the test if it is incorrect or if the test is no longer
|
||||||
|
relevant. In almost all cases, you should be fixing the code that is
|
||||||
|
being tested, or updating the tests to match a refactored implementation.
|
||||||
|
|
||||||
|
* Always write a `Makefile` with the default target being `test`, and with a
|
||||||
|
`fmt` target that formats the code. The `test` target should run all
|
||||||
|
tests in the project, and the `fmt` target should format the code. `test`
|
||||||
|
should also have a prerequisite target `lint` that should run any linters
|
||||||
|
that are configured for the project.
|
||||||
|
|
||||||
|
* After each completed bugfix or feature, the code must be committed. Do
|
||||||
|
all of the pre-commit checks (test, lint, fmt) before committing, of
|
||||||
|
course. After each commit, push to the remote.
|
||||||
|
|
||||||
|
* Always write tests, even if they are extremely simple and just check for
|
||||||
|
correct syntax (ability to compile/import). If you are writing a new
|
||||||
|
feature, write a test for it. You don't need to target complete coverage,
|
||||||
|
but you should at least test any new functionality you add.
|
||||||
|
|
||||||
|
* Always use structured logging. Log any relevant state/context with the
|
||||||
|
messages (but do not log secrets). If stdout is not a terminal, output
|
||||||
|
the structured logs in jsonl format. Use go's log/slog.
|
||||||
|
|
||||||
|
* You do not need to summarize your changes in the chat after making them.
|
||||||
|
Making the changes and committing them is sufficient. If anything out of
|
||||||
|
the ordinary happened, please explain it, but in the normal case where you
|
||||||
|
found and fixed the bug, or implemented the feature, there is no need for
|
||||||
|
the end-of-change summary.
|
||||||
|
|
||||||
|
* When testing daemons, use a 15 second timeout always.
|
||||||
+41
-75
@@ -1,46 +1,22 @@
|
|||||||
# In every stage ARG CACHEBUST comes right after the steps that install
|
# Lint stage — fast feedback on formatting and lint issues.
|
||||||
# system packages and download Go's modules, or right after FROM in a
|
# The golangci-lint image bundles Go, gcc and make, so it can run go vet on
|
||||||
# stage with neither, and every step that lints, tests or compiles
|
# the CGO sqlite package and golangci-lint without extra installs.
|
||||||
# comes after it. script/ passes a new value on every build, which
|
# golangci/golangci-lint:v2.7.2 (Go 1.25.5), 2026-09-21
|
||||||
# reruns every RUN step after it. Docker keeps the layers above it
|
FROM golangci/golangci-lint@sha256:5d6d5c70a61f1356adfd9dd6316ce286799fefc9d743421356ff1b00842368ba AS lint
|
||||||
# until the base image, their own lines or go.mod and go.sum change.
|
|
||||||
# The steps that compile mount Go's build cache, and golangci-lint run
|
|
||||||
# also mounts golangci-lint's; both survive from build to build.
|
|
||||||
|
|
||||||
# Lint phase. The image is Debian-based and has gcc, so it can check the
|
|
||||||
# cgo SQLite package without extra installs.
|
|
||||||
# golangci/golangci-lint:v2.14.0 (Go 1.27.0), 2026-10-19
|
|
||||||
FROM golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint
|
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
|
|
||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
RUN go mod download
|
RUN go mod download
|
||||||
ARG CACHEBUST
|
|
||||||
COPY . .
|
|
||||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
|
||||||
--mount=type=cache,target=/root/.cache/golangci-lint \
|
|
||||||
golangci-lint run --config .golangci.yml ./...
|
|
||||||
|
|
||||||
# Test phase. -race needs cgo and so a C compiler, which the Debian Go
|
|
||||||
# image ships and the alpine one does not. -count=1 because the mounted
|
|
||||||
# cache also holds test results, which go test would otherwise replay.
|
|
||||||
# -short skips the live RIS Live feed test, so the suite needs no network.
|
|
||||||
# golang:1.27.2-bookworm, 2026-10-19
|
|
||||||
FROM golang@sha256:5cf287a799e6b94384bad13d16b14904c531f51ba65792237e122ce42b392f61 AS test
|
|
||||||
WORKDIR /src
|
|
||||||
COPY go.mod go.sum ./
|
|
||||||
RUN go mod download
|
|
||||||
ARG CACHEBUST
|
|
||||||
COPY . .
|
COPY . .
|
||||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
|
||||||
go test -short -count=1 -timeout 90s -race -cover ./... || \
|
|
||||||
{ echo "--- Rerunning with -v for details ---"; \
|
|
||||||
go test -short -count=1 -timeout 90s -race -v ./...; exit 1; }
|
|
||||||
|
|
||||||
# Build stage. Nothing is wanted from either phase above; the copies
|
RUN make fmt-check
|
||||||
# are what make BuildKit build them first, so nothing after them can
|
RUN make lint
|
||||||
# run unless lint and test passed.
|
|
||||||
# golang:1.27.2-bookworm, 2026-10-19
|
# Build stage
|
||||||
FROM golang@sha256:5cf287a799e6b94384bad13d16b14904c531f51ba65792237e122ce42b392f61 AS builder
|
# golang:1.24-bookworm, 2026-09-21
|
||||||
|
FROM golang@sha256:1a6d4452c65dea36aac2e2d606b01b4a029ec90cc1ae53890540ce6173ea77ac AS builder
|
||||||
|
|
||||||
# Install build dependencies (zstd for archive, gcc for CGO/sqlite3)
|
# Install build dependencies (zstd for archive, gcc for CGO/sqlite3)
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
@@ -50,38 +26,36 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
|||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
COPY go.mod go.sum ./
|
|
||||||
RUN go mod download
|
# Force BuildKit to run the lint stage before compiling or testing.
|
||||||
ARG CACHEBUST
|
|
||||||
COPY --from=lint /src/go.sum /dev/null
|
COPY --from=lint /src/go.sum /dev/null
|
||||||
COPY --from=test /src/go.sum /dev/null
|
|
||||||
# A tar-stream context keeps the sender's file owners, which git refuses.
|
# Copy everything
|
||||||
RUN git config --system --add safe.directory /src
|
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
# Vendor dependencies for the source archive (must be after copying source)
|
# Vendor dependencies (must be after copying source)
|
||||||
RUN go mod vendor
|
RUN go mod download && go mod vendor
|
||||||
|
|
||||||
|
# Run the test suite in the build stage: -race needs cgo and the C compiler
|
||||||
|
# installed above. The suite is offline (the live-feed test is opt-in).
|
||||||
|
RUN make test
|
||||||
|
|
||||||
# Build the binary with CGO enabled (required for sqlite3). The version the
|
# Build the binary with CGO enabled (required for sqlite3). The version the
|
||||||
# page footer shows is the VERSION build arg when one is given, otherwise
|
# page footer shows is the VERSION build argument when one is given, otherwise
|
||||||
# `git describe --tags --always` on the .git in the build context: the tag
|
# `git describe --tags --always` of the .git in the build context (git comes
|
||||||
# on a tagged commit, tag-N-gHASH after one, the short commit when no tag is
|
# with this image): the tag on a tagged commit, tag-N-gHASH after one, the
|
||||||
# reachable. With .git present, a version that is still empty, dev or
|
# short commit when no tag is reachable. A context that carries .git and still
|
||||||
# unknown fails the build: git is missing or could not read the checkout.
|
# yields no version fails the build. The footer links to the full commit.
|
||||||
# Without .git the footer says unknown. The footer links to the full commit.
|
|
||||||
ARG VERSION
|
ARG VERSION
|
||||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
RUN version="${VERSION:-$(git describe --tags --always || echo unknown)}"; \
|
||||||
VERSION="${VERSION:-$(git describe --tags --always)}"; \
|
if [ -e .git ] && { [ -z "$version" ] || [ "$version" = dev ] || \
|
||||||
if [ -e .git ]; then \
|
[ "$version" = unknown ]; }; then \
|
||||||
case "$VERSION" in ""|dev|unknown) \
|
echo "no version could be derived although the build context carries .git" >&2; \
|
||||||
echo "version is '$VERSION' although .git is present" >&2; \
|
exit 1; \
|
||||||
exit 1 ;; \
|
|
||||||
esac; \
|
|
||||||
fi; \
|
fi; \
|
||||||
REVISION="$(git rev-parse --verify HEAD || echo unknown)"; \
|
|
||||||
CGO_ENABLED=1 GOOS=linux go build -o /routewatch -ldflags "\
|
CGO_ENABLED=1 GOOS=linux go build -o /routewatch -ldflags "\
|
||||||
-X git.eeqj.de/sneak/routewatch/internal/version.GitRevision=$REVISION \
|
-X git.eeqj.de/sneak/routewatch/internal/version.GitRevision=$(git rev-parse --verify HEAD || echo unknown) \
|
||||||
-X git.eeqj.de/sneak/routewatch/internal/version.GitRevisionShort=${VERSION:-unknown}" \
|
-X git.eeqj.de/sneak/routewatch/internal/version.GitRevisionShort=$version" \
|
||||||
./cmd/routewatch
|
./cmd/routewatch
|
||||||
|
|
||||||
# Create source archive with vendored dependencies
|
# Create source archive with vendored dependencies
|
||||||
@@ -90,7 +64,7 @@ RUN tar --zstd -cf /routewatch-source.tar.zst \
|
|||||||
--exclude='*.tar.zst' \
|
--exclude='*.tar.zst' \
|
||||||
.
|
.
|
||||||
|
|
||||||
# Runtime stage, and the last one
|
# Runtime stage
|
||||||
# debian:bookworm-slim, 2026-09-21
|
# debian:bookworm-slim, 2026-09-21
|
||||||
FROM debian@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251
|
FROM debian@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251
|
||||||
|
|
||||||
@@ -102,8 +76,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
|||||||
curl \
|
curl \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
ARG CACHEBUST
|
|
||||||
|
|
||||||
# Create non-root user
|
# Create non-root user
|
||||||
RUN useradd -r -u 1000 -m routewatch
|
RUN useradd -r -u 1000 -m routewatch
|
||||||
|
|
||||||
@@ -121,11 +93,10 @@ RUN chown -R routewatch:routewatch /app
|
|||||||
|
|
||||||
ENV XDG_DATA_HOME=/var/lib
|
ENV XDG_DATA_HOME=/var/lib
|
||||||
|
|
||||||
# Cap the Go heap at 2.5 GiB so the runtime collects harder before the
|
# Cap the Go heap at 1.5 GiB so the runtime collects harder before the
|
||||||
# container's memory limit is reached. A full routing table takes about 1 GB
|
# container's memory limit is reached. setpriv in the entrypoint preserves this
|
||||||
# of it (README "Memory"). setpriv in the entrypoint preserves this the way it
|
# the way it does XDG_DATA_HOME above.
|
||||||
# does XDG_DATA_HOME above.
|
ENV GOMEMLIMIT=1536MiB
|
||||||
ENV GOMEMLIMIT=2560MiB
|
|
||||||
|
|
||||||
# Cap glibc's malloc arenas. The SQLite C library allocates and frees millions
|
# Cap glibc's malloc arenas. The SQLite C library allocates and frees millions
|
||||||
# of small page-cache chunks from many threads; glibc otherwise creates up to
|
# of small page-cache chunks from many threads; glibc otherwise creates up to
|
||||||
@@ -140,13 +111,8 @@ EXPOSE 8080
|
|||||||
|
|
||||||
COPY ./entrypoint.sh /entrypoint.sh
|
COPY ./entrypoint.sh /entrypoint.sh
|
||||||
|
|
||||||
# Health check using the health endpoint, on the port PORT names. During the
|
# Health check using the health endpoint, on the port PORT names
|
||||||
# start period, Docker 25 and later checks every 5 seconds, and the first pass
|
HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
|
||||||
# makes the container healthy. After it, Docker checks every 30 seconds. The
|
|
||||||
# start period covers the 60 seconds after which upaas reads the health
|
|
||||||
# (README "Running under upaas"). Startup reads the routing table first,
|
|
||||||
# which takes about 25 seconds when the table is full.
|
|
||||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \
|
|
||||||
CMD curl -sf "http://localhost:${PORT:-8080}/.well-known/healthcheck.json" || exit 1
|
CMD curl -sf "http://localhost:${PORT:-8080}/.well-known/healthcheck.json" || exit 1
|
||||||
|
|
||||||
ENTRYPOINT ["/bin/bash", "/entrypoint.sh" ]
|
ENTRYPOINT ["/bin/bash", "/entrypoint.sh" ]
|
||||||
|
|||||||
@@ -52,6 +52,7 @@ clean:
|
|||||||
run: build
|
run: build
|
||||||
DEBUG=routewatch ./bin/routewatch 2>&1 | tee log.txt
|
DEBUG=routewatch ./bin/routewatch 2>&1 | tee log.txt
|
||||||
|
|
||||||
# Rebuild the AS list built into RouteWatch from as.csv (needs the network).
|
|
||||||
asupdate:
|
asupdate:
|
||||||
@script/asupdate
|
@echo "Updating AS info data..."
|
||||||
|
@go run cmd/asinfo-gen/main.go | gzip > pkg/asinfo/asdata.json.gz.tmp && \
|
||||||
|
mv pkg/asinfo/asdata.json.gz.tmp pkg/asinfo/asdata.json.gz
|
||||||
|
|||||||
@@ -1,46 +1,10 @@
|
|||||||
# RouteWatch
|
# RouteWatch
|
||||||
|
|
||||||
RouteWatch is an MIT-licensed Go daemon by @sneak that monitors the BGP routing
|
RouteWatch is an MIT-licensed Go daemon by @sneak that monitors the BGP routing
|
||||||
table in real time. It works like this:
|
table in real time: it streams BGP UPDATE messages from the RIPE RIS Live
|
||||||
|
service, maintains a live routing table in SQLite, and provides HTTP APIs for
|
||||||
|
querying routing information.
|
||||||
|
|
||||||
- It streams BGP UPDATE messages from the RIPE RIS Live service.
|
|
||||||
- It keeps the whole routing table in memory, with a copy in SQLite.
|
|
||||||
- It serves web pages and a JSON API for looking routes up.
|
|
||||||
|
|
||||||
## Rationale
|
|
||||||
|
|
||||||
RouteWatch exists to serve our other services as an internal API for information
|
|
||||||
about IP addresses.
|
|
||||||
|
|
||||||
RouteWatch follows BGP routing table updates from RIPE RIS Live. RIS Live
|
|
||||||
streams the updates that RIPE's route collectors receive, as they happen, from
|
|
||||||
the moment a client connects. It reports changes, not the routing table they add
|
|
||||||
up to. RouteWatch applies those changes to a routing table that it keeps in
|
|
||||||
memory, with a copy in its own SQLite database. From that table, RouteWatch sees
|
|
||||||
directly:
|
|
||||||
|
|
||||||
- which AS each IP address is routed to
|
|
||||||
- which announced prefix holds each IP address, and how big that prefix is
|
|
||||||
|
|
||||||
RouteWatch adds details of each AS from WHOIS, the public directory of the
|
|
||||||
internet registries. The details include the organisation that holds the AS.
|
|
||||||
RouteWatch keeps them in its database and looks them up again once they are 30
|
|
||||||
days old. An IP lookup answers from those stored details and never waits for
|
|
||||||
WHOIS.
|
|
||||||
|
|
||||||
So our own services can ask RouteWatch about an IP address, instead of each
|
|
||||||
working it out for itself.
|
|
||||||
|
|
||||||
RIS Live sends no routing table when a client connects. So the routing table
|
|
||||||
holds only the routes announced since RouteWatch first started. An IP lookup
|
|
||||||
finds no route for an address until a prefix that holds it is announced. The
|
|
||||||
routing table also misses these updates:
|
|
||||||
|
|
||||||
- those sent while RouteWatch is stopped
|
|
||||||
- those that RouteWatch drops when it falls behind
|
|
||||||
|
|
||||||
A lookup or a prefix page can therefore show a route that has since been
|
|
||||||
withdrawn.
|
|
||||||
|
|
||||||
## Features
|
## Features
|
||||||
|
|
||||||
@@ -48,8 +12,6 @@ withdrawn.
|
|||||||
- Maintains live IPv4 and IPv6 routing tables
|
- Maintains live IPv4 and IPv6 routing tables
|
||||||
- Tracks AS peering relationships
|
- Tracks AS peering relationships
|
||||||
- HTTP API for IP-to-AS lookups, prefix details, and AS information
|
- HTTP API for IP-to-AS lookups, prefix details, and AS information
|
||||||
- IP lookups report the public blocklists an address is on, from copies kept in
|
|
||||||
memory and on disk
|
|
||||||
- Automatic reconnection with exponential backoff
|
- Automatic reconnection with exponential backoff
|
||||||
- Batched database writes for high performance
|
- Batched database writes for high performance
|
||||||
- Backpressure handling to prevent memory exhaustion
|
- Backpressure handling to prevent memory exhaustion
|
||||||
@@ -75,273 +37,21 @@ DEBUG=routewatch ./routewatch
|
|||||||
|
|
||||||
## HTTP Endpoints
|
## HTTP Endpoints
|
||||||
|
|
||||||
Every route answers `HEAD` as it answers `GET`, without the body.
|
|
||||||
|
|
||||||
The pages and the API read routes from the routing table that RouteWatch keeps
|
|
||||||
in memory. For each prefix and origin AS, the table keeps the two newest routes
|
|
||||||
in full, with their AS paths and times. These are the kept routes. The table
|
|
||||||
keeps only the address of every other peer that carries the prefix. When both
|
|
||||||
kept routes of an origin AS are withdrawn, that origin has no kept route until
|
|
||||||
one of its peers sends an update.
|
|
||||||
|
|
||||||
### Web Interface
|
### Web Interface
|
||||||
|
- `GET /` - Redirects to /status
|
||||||
- `GET /` - Home page (HTML): headline figures, lookups by AS number and by IP
|
- `GET /status` - HTML status dashboard
|
||||||
address, and a search of AS names. A card shows the `curl` command that looks
|
- `GET /status.json` - JSON statistics
|
||||||
up the visitor's own address at `/ip/{addr}`, and that command's live answer.
|
- `GET /as/{asn}` - AS detail page (HTML)
|
||||||
Other cards show figures of the routing table, with links to the AS and prefix
|
- `GET /prefix/{prefix}` - Prefix detail page (HTML)
|
||||||
pages:
|
- `GET /prefixlength/{length}` - IPv4 prefixes by mask length
|
||||||
- the newest and the oldest kept route, with its prefix, origin AS and time
|
- `GET /prefixlength6/{length}` - IPv6 prefixes by mask length
|
||||||
- the AS last seen in an AS path most recently, and the one last seen
|
- `GET /ip/{ip}` - Redirects to prefix containing the IP
|
||||||
longest ago
|
|
||||||
- the AS that originates the most prefixes
|
|
||||||
- how many ASes originate a single prefix, and the most recently updated of
|
|
||||||
them
|
|
||||||
- the number of bogon announcements, linked to `/bogons`
|
|
||||||
- the number of prefixes with more than one origin AS, linked to
|
|
||||||
`/multiple-origins`
|
|
||||||
- well-known networks, such as Apple, Google, Cogent and MIT, each with its
|
|
||||||
number of prefixes and the time of its newest kept route, newest first
|
|
||||||
|
|
||||||
RouteWatch works these figures out once a minute, so they can be a minute
|
|
||||||
old.
|
|
||||||
|
|
||||||
- `GET /bogons` - The bogon announcements (HTML): the routes whose prefix or
|
|
||||||
origin AS is reserved. A prefix is reserved when it lies inside one of these:
|
|
||||||
- a block that the IANA special-purpose address registries mark as not
|
|
||||||
globally reachable
|
|
||||||
- the multicast space, `224.0.0.0/4` and `ff00::/8`, which the IANA IPv4 and
|
|
||||||
IPv6 address space registries list as Multicast
|
|
||||||
|
|
||||||
A reserved AS number is one of these:
|
|
||||||
- an AS number for private use or for documentation
|
|
||||||
- `AS23456`
|
|
||||||
- the last AS number of the 16-bit or the 32-bit range
|
|
||||||
|
|
||||||
Address space and AS numbers that the IANA has not handed out yet are not
|
|
||||||
counted. Each row shows the prefix, the origin AS, the number of peers and
|
|
||||||
why it is a bogon. The page shows at most 1,000 rows, in prefix order, and
|
|
||||||
says how many there are in all.
|
|
||||||
|
|
||||||
- `GET /multiple-origins` - The prefixes announced by more than one origin AS
|
|
||||||
(HTML). Each row shows the origin ASes and how many peers carry the prefix
|
|
||||||
with each. Like `/bogons`, it shows at most 1,000 rows, in prefix order, and
|
|
||||||
says how many there are in all.
|
|
||||||
- `GET /status` - HTML status dashboard, which reads `/api/v1/stats`
|
|
||||||
- `GET /status.json` - JSON statistics: feed, memory, database and route
|
|
||||||
figures. The route figures come from the routing table. `oldest_route` and
|
|
||||||
`newest_route` are the times of the oldest and the newest kept routes.
|
|
||||||
`origins_without_kept_route` counts, across all prefixes, the origin ASes that
|
|
||||||
have no kept route. Other figures include these:
|
|
||||||
- `uptime_seconds`: the number of seconds since the RIS Live feed last
|
|
||||||
connected, or 0 when it has never connected
|
|
||||||
- `memory_usage_bytes`: the size of the Go heap in use, in bytes
|
|
||||||
- `whois_stats.current_interval_seconds`: the WHOIS fetcher's wait between
|
|
||||||
two fetches, in seconds, which can include a fraction of a second
|
|
||||||
|
|
||||||
- `GET /.well-known/healthcheck.json` - Health check (JSON): 200 when the
|
|
||||||
database answers and the RIS Live feed is connected, otherwise 503;
|
|
||||||
`data.status` is `ok` or `error` and `data.checks` says which check failed
|
|
||||||
- `GET /as/{asn}` - AS detail page (HTML): the prefixes the AS originates and
|
|
||||||
its peers. When the database has no record of the AS, its handle and
|
|
||||||
description come from the AS list built into RouteWatch. The page then shows
|
|
||||||
no first-seen date.
|
|
||||||
- `GET /prefix/{prefix}/{length}` - Prefix detail page (HTML), such as
|
|
||||||
`/prefix/192.0.2.0/24`: its origin ASes and every peer that carries it. Only
|
|
||||||
the kept routes show an AS path, a next hop and a time.
|
|
||||||
- `GET /prefixlength/{length}` - Up to 500 IPv4 prefixes with that mask length,
|
|
||||||
picked at random (HTML). Each shows the origin AS that most peers carry it
|
|
||||||
with, and the age of its newest kept route.
|
|
||||||
- `GET /prefixlength6/{length}` - The same for IPv6
|
|
||||||
- `GET /ip/{addr}` - The same JSON answer as `/api/v1/ip/{ip}`
|
|
||||||
- `GET /ip/` - The same, for the caller's own address: the address the request
|
|
||||||
came from or, when that is a trusted proxy (`TRUSTED_PROXIES`, see
|
|
||||||
Configuration), the client address in its `X-Forwarded-For`
|
|
||||||
|
|
||||||
### API v1
|
### API v1
|
||||||
|
- `GET /api/v1/stats` - Detailed statistics with handler metrics
|
||||||
- `GET /api/v1/stats` - The `/status.json` figures plus connection, garbage
|
- `GET /api/v1/ip/{ip}` - Look up AS information for an IP address
|
||||||
collector, stream and handler queue statistics. `connection_duration_seconds`
|
- `GET /api/v1/as/{asn}` - Get prefixes announced by an AS
|
||||||
is the number of seconds the RIS Live feed has been connected, or null while
|
- `GET /api/v1/prefix/{prefix}` - Get routes for a specific prefix
|
||||||
it is not connected. `stream.route_churn_per_min` is the number of prefix
|
|
||||||
announcements and withdrawals, from all peers, in the last full clock minute.
|
|
||||||
- `GET /api/v1/ip/{ip}` - Looks up each address. `{ip}` is an IP address, a host
|
|
||||||
name, or a comma-separated list of at most 10 of these. `results` has one
|
|
||||||
entry for each address. Every entry has the same fields:
|
|
||||||
- `ip`: the address
|
|
||||||
- `ptr`: the address's PTR names, or an empty list when it has none
|
|
||||||
- `blocklists`: the public blocklists the address is on (see Blocklists), or
|
|
||||||
an empty list when it is on none
|
|
||||||
|
|
||||||
Every other field is about the address's live route. Each is null when the
|
|
||||||
address has no live route. The fields about the route:
|
|
||||||
- `netblock`: the most specific prefix that holds the address
|
|
||||||
- `mask_length`: the netblock's mask length
|
|
||||||
- `ip_version`: the netblock's IP version, 4 or 6
|
|
||||||
- `num_peers`: the number of peers that carry the netblock
|
|
||||||
- `asn`: the origin AS that the most peers carry the netblock with
|
|
||||||
- `num_origins`: the number of origin ASes that peers carry the netblock
|
|
||||||
with
|
|
||||||
- `origin_asns`: those origin ASes, in AS number order
|
|
||||||
- `first_seen`: when RouteWatch first saw the netblock, or null when the
|
|
||||||
database has not stored the netblock yet
|
|
||||||
- `last_seen`: the time of the netblock's newest kept route, or null when
|
|
||||||
the netblock keeps no route
|
|
||||||
|
|
||||||
The fields about the AS in `asn` come from the database. Each is also null
|
|
||||||
when the database does not have it:
|
|
||||||
- `handle`: the AS's handle
|
|
||||||
- `description`: the AS's description
|
|
||||||
- `as_name`: the AS's name in its WHOIS details
|
|
||||||
- `org_name`: the name of the organisation that holds the AS
|
|
||||||
- `org_id`: the organisation's ID at its registry
|
|
||||||
- `address`: the organisation's postal address
|
|
||||||
- `country_code`: the organisation's country code
|
|
||||||
- `abuse_email`: the address that takes abuse reports for the AS
|
|
||||||
- `rir`: the regional internet registry of the AS
|
|
||||||
|
|
||||||
Of the addresses a host name resolves to, only public ones are looked up. A
|
|
||||||
name with none, such as one that resolves to a private address, is reported
|
|
||||||
as `host name has no public address`, without its addresses. `errors` has
|
|
||||||
one entry for each host name that failed. The answer is 400 when:
|
|
||||||
- `{ip}` lists more than 10
|
|
||||||
- `{ip}` comes to more than 10 addresses once its names are resolved
|
|
||||||
- nothing in `{ip}` is an IP address or a host name with a public address
|
|
||||||
|
|
||||||
A PTR lookup gets 2 seconds. An address whose lookup takes longer is
|
|
||||||
answered without PTR names.
|
|
||||||
|
|
||||||
- `GET /api/v1/as/{asn}` - The AS and the prefixes it originates, IPv4 and IPv6
|
|
||||||
apart, each with the time of its newest kept route. A prefix that keeps no
|
|
||||||
route with the AS has no `last_updated`. When the database has no record of
|
|
||||||
the AS, its handle and description come from the AS list built into
|
|
||||||
RouteWatch. The answer then has no `first_seen` or `last_seen`.
|
|
||||||
- `GET /api/v1/prefix/{prefix}/{length}` - One route for each peer that carries
|
|
||||||
the prefix, and the same routes grouped by origin AS in `origins`. Only the
|
|
||||||
kept routes have `as_path`, `next_hop` and `last_updated`.
|
|
||||||
- `GET /api/v1/search/as?q={text}` - Finds the ASes whose handle or description
|
|
||||||
contains `{text}`, ignoring case. `data` holds:
|
|
||||||
- `query`: `{text}` without its leading and trailing spaces
|
|
||||||
- `total`: the number of ASes that match
|
|
||||||
- `results`: the first 50 of them in AS number order, each with `asn`,
|
|
||||||
`handle` and `description`
|
|
||||||
|
|
||||||
The search reads the AS list built into RouteWatch, not the database. The
|
|
||||||
list comes from
|
|
||||||
[ipverse/as-metadata](https://github.com/ipverse/as-metadata). A match can
|
|
||||||
therefore be an AS that the database has no record of. A `q` that is
|
|
||||||
missing, empty or only spaces answers 400.
|
|
||||||
|
|
||||||
A JSON answer is `{"status": "ok", "data": ...}`, or for an error
|
|
||||||
`{"status": "error", "error": {"msg": ..., "code": ...}}`. The health check is
|
|
||||||
the exception: its 503 answer is also `{"status": "ok", "data": ...}`, with
|
|
||||||
`error` in `data.status`. Every JSON answer also carries an `@meta` object:
|
|
||||||
|
|
||||||
- `time_zone`: always `UTC`
|
|
||||||
- `api_version`: `1`
|
|
||||||
- `execution_time_ms`: the number of milliseconds the request took
|
|
||||||
|
|
||||||
Every value with a unit is a number whose field name ends in the unit, such as
|
|
||||||
`_ms`, `_seconds` or `_bytes`.
|
|
||||||
|
|
||||||
A request that runs past its time limit answers 504: the limit is 30 seconds,
|
|
||||||
and 4 seconds for the database reads behind `/status.json` and `/api/v1/stats`.
|
|
||||||
An AS number, prefix or mask length that does not parse answers 400, as does a
|
|
||||||
mask length out of range for its IP version. An AS answers 404 when it
|
|
||||||
originates no prefix and has no record in the database or in the AS list built
|
|
||||||
into RouteWatch. AS 0 always answers 404, because it means no known origin. The
|
|
||||||
AS name search never lists it. A prefix with no live route answers 404 too. The
|
|
||||||
HTML pages give these errors as plain text. Under `/api/v1` and `/ip/`, a path
|
|
||||||
that matches no route answers 404 with the message `Not found`. A request with
|
|
||||||
`POST`, `PUT`, `PATCH`, `DELETE`, `OPTIONS`, `TRACE` or `CONNECT` answers 405
|
|
||||||
with `Method not allowed` on these routes:
|
|
||||||
|
|
||||||
- the routes under `/api/v1` and `/ip/`
|
|
||||||
- `/status.json`
|
|
||||||
- `/.well-known/healthcheck.json`
|
|
||||||
|
|
||||||
A 500 answer says only `Internal server error`. The error for a host name that
|
|
||||||
does not resolve says only `could not resolve host name`. The detail goes to the
|
|
||||||
log.
|
|
||||||
|
|
||||||
Every answer carries `Strict-Transport-Security`, `Content-Security-Policy`,
|
|
||||||
`X-Frame-Options`, `X-Content-Type-Options`, `Referrer-Policy` and
|
|
||||||
`Permissions-Policy`. The Content-Security-Policy lets a page load content only
|
|
||||||
from the server, and does not allow inline content with `'unsafe-inline'`: the
|
|
||||||
pages keep their CSS and JavaScript in inline `<style>` and `<script>` blocks,
|
|
||||||
so each answer's policy names a random nonce made for that answer, and the
|
|
||||||
browser applies or runs only the blocks that carry it. The pages therefore use
|
|
||||||
no `style` attributes, which a nonce cannot cover.
|
|
||||||
|
|
||||||
An origin AS of 0 means the route has no known origin. Its AS path ends in an AS
|
|
||||||
set of several members, any of which may have originated it. The 0 appears in
|
|
||||||
these fields:
|
|
||||||
|
|
||||||
- `origin_asn` in a route
|
|
||||||
- `asn` and `origin_asns` in an IP lookup
|
|
||||||
- the key `0` in a prefix's `origins`
|
|
||||||
|
|
||||||
The pages show such a route as having no known origin.
|
|
||||||
|
|
||||||
## Blocklists
|
|
||||||
|
|
||||||
The IP lookup reports which public blocklists list each address. Each result has
|
|
||||||
a `blocklists` list with one entry for each list the address is on:
|
|
||||||
|
|
||||||
- `name`: the list's name, from the lists below
|
|
||||||
- `description`: what being on the list means
|
|
||||||
- `updated`: when RouteWatch last downloaded the list, or last heard from its
|
|
||||||
publisher that its copy was current
|
|
||||||
|
|
||||||
An address on no list has an empty `blocklists`. An address with no live route
|
|
||||||
still has a result with its `blocklists`.
|
|
||||||
|
|
||||||
The lists:
|
|
||||||
|
|
||||||
- `dshield`: the 20 networks that DShield saw scan the most targets in the last
|
|
||||||
three days. From [DShield](https://isc.sans.edu/feeds_doc.html), downloaded
|
|
||||||
every hour.
|
|
||||||
- `feodo_tracker`: botnet command-and-control servers. From abuse.ch
|
|
||||||
[Feodo Tracker](https://feodotracker.abuse.ch/blocklist/), downloaded every 15
|
|
||||||
minutes.
|
|
||||||
- `firehol_abusers_30d`: addresses reported for comment, form or forum spam in
|
|
||||||
the last 30 days. From
|
|
||||||
[FireHOL](https://iplists.firehol.org/?ipset=firehol_abusers_30d), downloaded
|
|
||||||
every 5 hours.
|
|
||||||
- `firehol_level3`: addresses reported for attacks, spyware or viruses in the
|
|
||||||
last 30 days. From
|
|
||||||
[FireHOL](https://iplists.firehol.org/?ipset=firehol_level3), downloaded every
|
|
||||||
3 hours.
|
|
||||||
- `spamhaus_drop` and `spamhaus_dropv6`: IPv4 and IPv6 networks that Spamhaus
|
|
||||||
says spammers or cyber criminals have hijacked or lease. From
|
|
||||||
[Spamhaus DROP](https://www.spamhaus.org/blocklists/do-not-route-or-peer/),
|
|
||||||
downloaded once a day. Spamhaus merged its EDROP list into DROP.
|
|
||||||
- `tor_exits`: Tor exit nodes, where traffic from the Tor network leaves. From
|
|
||||||
the [Tor Project](https://check.torproject.org/torbulkexitlist), downloaded
|
|
||||||
every hour.
|
|
||||||
|
|
||||||
No list is downloaded more often than its publisher updates it or allows. Each
|
|
||||||
can be downloaded whole without an account or a key.
|
|
||||||
|
|
||||||
How RouteWatch keeps the lists:
|
|
||||||
|
|
||||||
- It holds each list in memory, so a lookup makes no outside request.
|
|
||||||
- It keeps a copy of each list in the `blocklists` directory of the state
|
|
||||||
directory (see Configuration). Each copy is a file named after its list.
|
|
||||||
- Beside each copy, a file ending in `.validators` keeps the `Last-Modified` and
|
|
||||||
`ETag` the publisher sent with that copy.
|
|
||||||
- At startup it loads each list from its copy. It downloads a list at once when
|
|
||||||
the copy is older than the list's interval, or when there is no copy.
|
|
||||||
- A download sends the copy's `Last-Modified` as `If-Modified-Since` and its
|
|
||||||
`ETag` as `If-None-Match`. The publisher then sends the list only when it has
|
|
||||||
changed. Without either, the download asks for the whole list.
|
|
||||||
- A download fails when it takes over a minute or its file is over 16 MiB.
|
|
||||||
- When a download fails, or its file does not parse, RouteWatch keeps the
|
|
||||||
previous copy and logs the failure. It tries again one interval later.
|
|
||||||
|
|
||||||
A successful IP lookup answer carries `Cache-Control` with a `max-age` of 15
|
|
||||||
minutes, the shortest list interval. The answer for the caller's own address
|
|
||||||
(`/ip/`) is `private`, and the others are `public`.
|
|
||||||
|
|
||||||
## Code Structure
|
## Code Structure
|
||||||
|
|
||||||
@@ -352,20 +62,15 @@ routewatch/
|
|||||||
│ ├── asinfo-gen/ # Utility to generate AS info data
|
│ ├── asinfo-gen/ # Utility to generate AS info data
|
||||||
│ └── streamdumper/ # Debug utility for raw stream output
|
│ └── streamdumper/ # Debug utility for raw stream output
|
||||||
├── internal/
|
├── internal/
|
||||||
│ ├── blocklist/ # Public IP blocklists: downloads and lookups
|
|
||||||
│ ├── bogon/ # Reserved prefixes and AS numbers
|
|
||||||
│ ├── routewatch/ # Core application logic
|
│ ├── routewatch/ # Core application logic
|
||||||
│ ├── server/ # HTTP server and handlers
|
│ ├── server/ # HTTP server and handlers
|
||||||
│ ├── database/ # SQLite storage layer
|
│ ├── database/ # SQLite storage layer
|
||||||
│ ├── routetable/ # The whole routing table in memory
|
|
||||||
│ ├── streamer/ # RIPE RIS Live client
|
│ ├── streamer/ # RIPE RIS Live client
|
||||||
│ ├── ristypes/ # BGP message data structures
|
│ ├── ristypes/ # BGP message data structures
|
||||||
│ ├── logger/ # Structured logging wrapper
|
│ ├── logger/ # Structured logging wrapper
|
||||||
│ ├── metrics/ # Performance metrics tracking
|
│ ├── metrics/ # Performance metrics tracking
|
||||||
│ ├── config/ # Configuration management
|
│ ├── config/ # Configuration management
|
||||||
│ ├── templates/ # HTML templates
|
│ └── templates/ # HTML templates
|
||||||
│ ├── version/ # Build version information
|
|
||||||
│ └── whois/ # WHOIS lookups of AS details
|
|
||||||
└── pkg/
|
└── pkg/
|
||||||
└── asinfo/ # AS information lookup (public API)
|
└── asinfo/ # AS information lookup (public API)
|
||||||
```
|
```
|
||||||
@@ -374,36 +79,31 @@ routewatch/
|
|||||||
|
|
||||||
### Component Relationships
|
### Component Relationships
|
||||||
|
|
||||||
`internal/routewatch/app.go` builds and starts these parts:
|
|
||||||
|
|
||||||
```
|
```
|
||||||
┌──────────────┐ ┌──────────────┐ ┌──────────────┐
|
┌─────────────────────────────────────────────────────────────────┐
|
||||||
│ Streamer │───▶│ Handlers │───▶│ Database │
|
│ RouteWatch │
|
||||||
│ │ │ │ │ │
|
│ (internal/routewatch/app.go - main orchestrator) │
|
||||||
│ RIS Live │ │ - ASHandler │ │ SQLite with │
|
├─────────────────────────────────────────────────────────────────┤
|
||||||
│ HTTP stream │ │ - PeerHandler│ │ WAL mode │
|
│ │
|
||||||
│ client │ │ - PrefixHdlr │ │ │
|
│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │
|
||||||
│ │ │ - PeeringHdlr│ │ Tables: │
|
│ │ Streamer │───▶│ Handlers │───▶│ Database │ │
|
||||||
└──────────────┘ └──────┬───────┘ │ - asns │
|
│ │ │ │ │ │ │ │
|
||||||
│ │ - prefixes │
|
│ │ RIS Live │ │ - ASHandler │ │ SQLite with │ │
|
||||||
▼ │ - routes │
|
│ │ WebSocket │ │ - PeerHandler│ │ WAL mode │ │
|
||||||
┌──────────────┐ │ - route_peers│
|
│ │ client │ │ - PrefixHdlr │ │ │ │
|
||||||
│ Routing table│───▶│ - peerings │
|
│ │ │ │ - PeeringHdlr│ │ Tables: │ │
|
||||||
│ in memory │ │ - bgp_peers │
|
│ └──────────────┘ └──────────────┘ │ - asns │ │
|
||||||
└──────────────┘ └──────────────┘
|
│ │ - prefixes │ │
|
||||||
▲ ▲
|
│ ┌──────────────┐ ┌──────────────┐ │ - live_routes│ │
|
||||||
┌──────────────┐ ┌──────┴───────┐ │
|
│ │ Server │───▶│ Handlers │───▶│ - peerings │ │
|
||||||
│ Server │───▶│ Handlers │───────────┘
|
│ │ │ │ │ │ - bgp_peers │ │
|
||||||
│ │ │ │
|
│ │ Chi router │ │ Status, API │ └──────────────┘ │
|
||||||
│ Chi router │ │ Status, API │
|
│ │ port 8080 │ │ AS, Prefix │ │
|
||||||
│ port 8080 │ │ AS, Prefix │
|
│ └──────────────┘ └──────────────┘ │
|
||||||
└──────────────┘ └──────────────┘
|
│ │
|
||||||
|
└─────────────────────────────────────────────────────────────────┘
|
||||||
```
|
```
|
||||||
|
|
||||||
The prefix handler keeps the routing table in memory current. It writes the
|
|
||||||
table's changes to the database every second. The web handlers read routes from
|
|
||||||
the routing table, and everything else from the database.
|
|
||||||
|
|
||||||
### Execution Flow
|
### Execution Flow
|
||||||
|
|
||||||
1. **Startup** (`cmd/routewatch/main.go` → `internal/routewatch/cli.go`)
|
1. **Startup** (`cmd/routewatch/main.go` → `internal/routewatch/cli.go`)
|
||||||
@@ -411,198 +111,102 @@ the routing table, and everything else from the database.
|
|||||||
- Signal handlers registered for graceful shutdown
|
- Signal handlers registered for graceful shutdown
|
||||||
|
|
||||||
2. **Initialization** (`internal/routewatch/app.go`)
|
2. **Initialization** (`internal/routewatch/app.go`)
|
||||||
- Database created with SQLite schema (WAL mode, page caches as in "Memory")
|
- Database created with SQLite schema (WAL mode, 3GB cache)
|
||||||
- The routing table read from the `routes` table into memory, before the
|
|
||||||
streamer connects. A full table takes about 25 seconds to read.
|
|
||||||
- Message handlers registered with the streamer
|
- Message handlers registered with the streamer
|
||||||
- HTTP server started on configured port
|
- HTTP server started on configured port
|
||||||
|
|
||||||
3. **Message Processing Pipeline**
|
3. **Message Processing Pipeline**
|
||||||
|
|
||||||
```
|
```
|
||||||
RIS Live Stream → JSON Parser → Message Dispatcher → Handler Queues
|
RIS Live Stream → JSON Parser → Message Dispatcher → Handler Queues → Batch Writers → SQLite
|
||||||
├→ Batch Writers → SQLite
|
|
||||||
└→ PrefixHandler → Routing Table in Memory → SQLite, every second
|
|
||||||
```
|
```
|
||||||
|
|
||||||
- Streamer connects to `ris-live.ripe.net` via HTTP
|
- Streamer connects to `ris-live.ripe.net` via HTTP
|
||||||
- Parses BGP UPDATE messages from JSON stream
|
- Parses BGP UPDATE messages from JSON stream
|
||||||
- Dispatches to registered handlers based on message type
|
- Dispatches to registered handlers based on message type
|
||||||
- Each handler has its own queue with backpressure handling
|
- Each handler has its own queue with backpressure handling
|
||||||
- Handlers batch writes for efficiency: the AS, peer and prefix handlers
|
- Handlers batch writes for efficiency (25K-30K ops, 1-2s timeout)
|
||||||
write 30,000, 10,000 and 25,000 updates at a time, or after 2, 2 and 1
|
|
||||||
seconds; the peering handler writes the peerings it has collected every 30
|
|
||||||
seconds
|
|
||||||
|
|
||||||
4. **Handler Details**
|
4. **Handler Details**
|
||||||
- **ASHandler**: Tracks all ASNs seen in AS paths
|
- **ASHandler**: Tracks all ASNs seen in AS paths
|
||||||
- **PeerHandler**: Records BGP peer information
|
- **PeerHandler**: Records BGP peer information
|
||||||
- **PrefixHandler**: Keeps the routing table in memory current. It applies
|
- **PrefixHandler**: Maintains live routing table (upserts on announcement, deletes on withdrawal)
|
||||||
each update to the table as it handles it:
|
|
||||||
- An announcement gives the peer a route to the prefix.
|
|
||||||
- A withdrawal removes the peer's route. So does an announcement whose
|
|
||||||
AS path is empty or holds AS 0 anywhere.
|
|
||||||
- A peer whose BGP session goes down leaves every prefix.
|
|
||||||
|
|
||||||
Every second it writes the changed prefix and origin AS pairs to the
|
|
||||||
`routes` table. A write takes at most 20,000 pairs, and the rest follow
|
|
||||||
in the next writes. It also stores each new prefix with the time it was
|
|
||||||
first seen. Peers are known by their IP address. A peer IP can have
|
|
||||||
sessions on two collectors. A session going down on one then also
|
|
||||||
removes the routes the other still carries, until they are announced
|
|
||||||
again.
|
|
||||||
|
|
||||||
- **PeeringHandler**: Extracts AS peering relationships from AS paths
|
- **PeeringHandler**: Extracts AS peering relationships from AS paths
|
||||||
|
|
||||||
5. **HTTP Request Flow**
|
5. **HTTP Request Flow**
|
||||||
```
|
```
|
||||||
Request → Chi Router → Middleware (timeout, logging) → Handler → Routing Table or Database → Response
|
Request → Chi Router → Middleware (timeout, logging) → Handler → Database Query → Response
|
||||||
```
|
```
|
||||||
|
|
||||||
### Key Design Patterns
|
### Key Design Patterns
|
||||||
|
|
||||||
- **Routing Table in Memory**: `internal/routetable` holds the whole routing
|
- **Batched Writes**: All database operations are batched for performance
|
||||||
table. For each prefix and origin AS it keeps:
|
- **Backpressure**: Probabilistic message dropping when queues exceed 50% capacity
|
||||||
- every peer whose current route to the prefix has that origin AS, one bit
|
- **Graceful Shutdown**: 60-second timeout, flushes all pending batches
|
||||||
per peer;
|
- **Reconnection**: Exponential backoff (5s-320s) with reset after 30s of stable connection
|
||||||
- the newest two of those routes in full, with AS path and time.
|
- **IP Lookup**: the most specific live route is found by looking up the
|
||||||
|
address's prefix at each mask length, longest first, in the prefix index
|
||||||
Of the other routes, only the peer is kept. The prefix handler writes the
|
(at most 25 lookups for IPv4, 113 for IPv6); prefixes are stored in the
|
||||||
changed pairs to the `routes` table every second, one row per prefix and
|
text form Go's `net/netip` prints. Routes shorter than /8 for IPv4 or /16
|
||||||
origin AS. Startup reads that table back into memory before the streamer
|
for IPv6, such as a default route, are not used.
|
||||||
connects. A crash loses only the changes not yet written.
|
|
||||||
|
|
||||||
- **Batched Writes**: The handlers write the feed's updates to the database in
|
|
||||||
batches for performance
|
|
||||||
- **Backpressure**: Probabilistic message dropping when queues exceed 50%
|
|
||||||
capacity
|
|
||||||
- **Graceful Shutdown**: on SIGTERM or SIGINT the streamer stops, the handlers
|
|
||||||
write the batches they hold, the HTTP server stops and the database is
|
|
||||||
checkpointed and closed. The daemon gives this 8 seconds and exits with status
|
|
||||||
0; "Running under upaas" says what a stop loses, logs and leaves undone when
|
|
||||||
the 8 seconds run out.
|
|
||||||
- **Reconnection**: Any end of the stream other than shutdown, including RIS
|
|
||||||
Live ending the response normally, marks the feed disconnected and reconnects
|
|
||||||
with exponential backoff (5s-320s). The backoff resets after a stream that
|
|
||||||
lasted more than 30s. A stream with no line for 60s is dropped and
|
|
||||||
reconnected.
|
|
||||||
- **IP Lookup**: the routing table in memory finds the most specific prefix that
|
|
||||||
holds the address. It looks up the address's prefix at each mask length,
|
|
||||||
longest first: at most 25 lookups for IPv4 and 113 for IPv6. Prefixes shorter
|
|
||||||
than /8 for IPv4 or /16 for IPv6, such as a default route, are not used.
|
|
||||||
|
|
||||||
### Database Schema
|
### Database Schema
|
||||||
|
|
||||||
```sql
|
```sql
|
||||||
-- Core tables
|
-- Core tables
|
||||||
asns(asn PRIMARY KEY, handle, description,
|
asns(id, number, handle, description, first_seen, last_seen)
|
||||||
as_name, org_name, org_id, address, country_code, abuse_email,
|
prefixes_v4(id, prefix, mask_length, first_seen, last_seen)
|
||||||
abuse_phone, tech_email, tech_phone, rir, rir_registration_date,
|
prefixes_v6(id, prefix, mask_length, first_seen, last_seen)
|
||||||
rir_last_modified, whois_raw,
|
|
||||||
first_seen, last_seen, whois_updated_at)
|
|
||||||
prefixes_v4(prefix PRIMARY KEY, first_seen) WITHOUT ROWID
|
|
||||||
prefixes_v6(prefix PRIMARY KEY, first_seen) WITHOUT ROWID
|
|
||||||
|
|
||||||
-- The routing table, one row per prefix and origin AS that some peer
|
-- Live routing tables (one per IP version)
|
||||||
-- carries: the peer set, and the newest two routes in full
|
live_routes_v4(id, prefix, mask_length, origin_asn, peer_ip, as_path,
|
||||||
routes(prefix, origin_asn, peers,
|
next_hop, last_updated)
|
||||||
newest_peer, newest_as_path, newest_time,
|
live_routes_v6(id, prefix, mask_length, origin_asn, peer_ip, as_path,
|
||||||
older_peer, older_as_path, older_time,
|
next_hop, last_updated)
|
||||||
PRIMARY KEY (prefix, origin_asn)) WITHOUT ROWID
|
|
||||||
-- The peers of the routes table, each with its number in the peer sets
|
|
||||||
route_peers(number PRIMARY KEY, address UNIQUE)
|
|
||||||
|
|
||||||
-- Relationship tracking
|
-- Relationship tracking
|
||||||
peerings(id PRIMARY KEY, as_a, as_b, first_seen, last_seen,
|
peerings(id, as_a, as_b, first_seen, last_seen)
|
||||||
UNIQUE(as_a, as_b))
|
bgp_peers(id, peer_ip, peer_asn, last_message_type, last_seen)
|
||||||
bgp_peers(id PRIMARY KEY, peer_ip UNIQUE, peer_asn, first_seen, last_seen,
|
|
||||||
last_message_type)
|
|
||||||
|
|
||||||
-- Nothing in the daemon writes this table
|
|
||||||
announcements(id PRIMARY KEY, prefix_id, peer_asn, origin_asn, path,
|
|
||||||
next_hop, timestamp, is_withdrawal)
|
|
||||||
```
|
```
|
||||||
|
|
||||||
The indexes are in `internal/database/schema.sql`.
|
|
||||||
|
|
||||||
## Configuration
|
## Configuration
|
||||||
|
|
||||||
Configuration is handled via environment variables and OS-specific paths:
|
Configuration is handled via environment variables and OS-specific paths:
|
||||||
|
|
||||||
| Variable | Default | Description |
|
| Variable | Default | Description |
|
||||||
| ------------------ | -------------------------------- | ----------------------------------------------------------- |
|
|----------|----------|-------------|
|
||||||
| `PORT` | `8080` | HTTP server port, a whole number from 1 to 65535 |
|
| `PORT` | `8080` | HTTP server port, a whole number from 1 to 65535 |
|
||||||
| `DEBUG` | (empty) | Set to `routewatch` for debug logging |
|
| `DEBUG` | (empty) | Set to `routewatch` for debug logging |
|
||||||
| `XDG_DATA_HOME` | `/var/lib` (in the Docker image) | Base of the state directory; must be an absolute path |
|
| `XDG_DATA_HOME` | `/var/lib` (in the Docker image) | Base of the state directory; must be an absolute path |
|
||||||
| `GOMEMLIMIT` | `2560MiB` (in the Docker image) | Go soft memory limit; see Memory |
|
| `GOMEMLIMIT` | `1536MiB` (in the Docker image) | Go soft memory limit; see Memory |
|
||||||
| `MALLOC_ARENA_MAX` | `2` (in the Docker image) | glibc malloc arena cap, a positive whole number; see Memory |
|
| `MALLOC_ARENA_MAX` | `2` (in the Docker image) | glibc malloc arena cap, a positive whole number; see Memory |
|
||||||
| `TRUSTED_PROXIES` | the RFC 1918 ranges | Addresses and CIDR ranges of trusted proxies; see below |
|
|
||||||
|
|
||||||
A variable that is set to an invalid value stops the start with an error and a
|
A variable that is set to an invalid value stops the start with an error and a
|
||||||
non-zero exit. An empty variable counts as unset. `TRUSTED_PROXIES` is the one
|
non-zero exit. An empty variable counts as unset.
|
||||||
exception: set but empty, it makes the server trust no proxy.
|
|
||||||
|
|
||||||
`TRUSTED_PROXIES` names the reverse proxies whose `X-Forwarded-For` the server
|
State directory (database location):
|
||||||
believes:
|
- macOS: `~/Library/Application Support/routewatch/`
|
||||||
|
- Linux: `/var/lib/berlin.sneak.app.routewatch/` when running as root,
|
||||||
- When it is not set, the server trusts the private ranges of RFC 1918:
|
otherwise `$XDG_DATA_HOME/berlin.sneak.app.routewatch/` (with `XDG_DATA_HOME`
|
||||||
- `10.0.0.0/8`
|
unset, `~/.local/share/berlin.sneak.app.routewatch/`). In the Docker image
|
||||||
- `172.16.0.0/12`
|
this is `/var/lib/berlin.sneak.app.routewatch/`.
|
||||||
- `192.168.0.0/16`
|
|
||||||
- When it lists addresses and CIDR ranges, comma-separated, such as
|
|
||||||
`10.0.0.0/8,192.0.2.1`, the server trusts those instead.
|
|
||||||
- When it is set but empty (`TRUSTED_PROXIES=`), the server trusts no proxy.
|
|
||||||
|
|
||||||
For a request from a trusted proxy, the request log and `/ip/` use the last
|
|
||||||
address in `X-Forwarded-For` that is not a trusted proxy. Each proxy adds the
|
|
||||||
address it received the request from to the end, so the entries before it are
|
|
||||||
whatever the client sent. For a request from any other address they use the
|
|
||||||
address the request came from, whatever forwarding headers it carries.
|
|
||||||
`X-Real-IP` and `True-Client-IP` are not read.
|
|
||||||
|
|
||||||
State directory (the database and the blocklist copies):
|
|
||||||
|
|
||||||
- macOS: `~/Library/Application Support/berlin.sneak.app.routewatch/`
|
|
||||||
- Linux: `/var/lib/berlin.sneak.app.routewatch/` when running as root, otherwise
|
|
||||||
`$XDG_DATA_HOME/berlin.sneak.app.routewatch/` (with `XDG_DATA_HOME` unset,
|
|
||||||
`~/.local/share/berlin.sneak.app.routewatch/`). In the Docker image this is
|
|
||||||
`/var/lib/berlin.sneak.app.routewatch/`.
|
|
||||||
|
|
||||||
## Memory
|
## Memory
|
||||||
|
|
||||||
The daemon keeps the whole routing table in memory. The image sets ceilings that
|
The daemon holds a live routing table, so its memory grows with the size of the
|
||||||
keep it inside a 5 GiB container.
|
data it tracks. The image sets ceilings that keep it inside a 5 GiB container.
|
||||||
|
|
||||||
Budget:
|
Budget:
|
||||||
|
- Go heap: a 1.5 GiB soft limit (`GOMEMLIMIT=1536MiB`, set in the image).
|
||||||
- Go heap: a 2.5 GiB soft limit (`GOMEMLIMIT=2560MiB`, set in the image).
|
- SQLite: at most 640 MiB of page cache across the connection pool (64 MiB per
|
||||||
- The routing table takes about 700 bytes per prefix and origin AS. A full
|
connection, 10 connections) and a 1.5 GiB hard heap limit for the C library.
|
||||||
table has about 1.4 million of these pairs, so it takes about 1 GB.
|
- glibc allocator: the SQLite C library runs on glibc `malloc`, which frees
|
||||||
- The rest of the heap takes up to about 0.5 GiB. Most of it is the
|
page-cache chunks back to per-arena free lists rather than to the kernel, so
|
||||||
handlers' batches and queues, described below.
|
process RSS tracks the high-water mark of those arenas, not SQLite's live
|
||||||
- The remaining 1 GiB is room for the garbage collector. With much less, it
|
heap. glibc creates up to eight arenas per core, so on a many-core host the
|
||||||
would run almost all the time.
|
retained memory — and thus RSS — grows with the core count. The image sets
|
||||||
- SQLite: 656 MiB of page cache.
|
`MALLOC_ARENA_MAX=2` to bound it; the two-arena cap costs nothing here because
|
||||||
- The connection that does every write has 512 MiB. It keeps the pages the
|
database writes are already serialized.
|
||||||
writes touch from one write to the next. With a full routing table, the
|
|
||||||
`routes` table takes about 450 MB, so its pages fit.
|
|
||||||
- Each of the 9 reading connections has 16 MiB. SQLite empties a reading
|
|
||||||
connection's cache whenever another connection has written.
|
|
||||||
- A 768 MiB soft heap limit sits above the caches, so it does not shrink
|
|
||||||
them. A 1 GiB hard heap limit caps the C library.
|
|
||||||
- glibc allocator: the SQLite C library allocates memory with glibc `malloc`.
|
|
||||||
- glibc keeps freed page-cache chunks in per-arena free lists instead of
|
|
||||||
returning them to the kernel. So the process's RSS follows the high-water
|
|
||||||
mark of those arenas, not SQLite's live heap.
|
|
||||||
- glibc creates up to eight arenas per core, so on a many-core host RSS
|
|
||||||
grows with the core count.
|
|
||||||
- The image sets `MALLOC_ARENA_MAX=2` to bound it. Two arenas cost nothing
|
|
||||||
here, because database writes already run one at a time.
|
|
||||||
- About 0.2 GiB for everything else in the runtime.
|
- About 0.2 GiB for everything else in the runtime.
|
||||||
|
|
||||||
The ceilings add up to 3.7 GiB: the Go heap's 2.5 GiB, SQLite's 1 GiB and the
|
|
||||||
0.2 GiB for the rest.
|
|
||||||
|
|
||||||
Run the container with a memory limit of 5 GiB and swap disabled:
|
Run the container with a memory limit of 5 GiB and swap disabled:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@@ -612,36 +216,20 @@ docker run --memory=5g --memory-swap=5g ...
|
|||||||
or the equivalent in your deployment tool. This leaves headroom above the
|
or the equivalent in your deployment tool. This leaves headroom above the
|
||||||
ceilings for spikes and the kernel page cache.
|
ceilings for spikes and the kernel page cache.
|
||||||
|
|
||||||
To replace the image's Go soft limit, set `GOMEMLIMIT` in the environment, for
|
Override the Go soft limit by setting `GOMEMLIMIT` in the environment (for
|
||||||
example `-e GOMEMLIMIT=3GiB`. A limit close to the routing table's size makes
|
example `-e GOMEMLIMIT=1GiB`); this replaces the image default. `MALLOC_ARENA_MAX`
|
||||||
the garbage collector run almost all the time. `MALLOC_ARENA_MAX` can be
|
can be overridden the same way, but raising it lets RSS climb again on a
|
||||||
overridden the same way, but raising it lets RSS climb again on a many-core
|
many-core host.
|
||||||
host.
|
|
||||||
|
|
||||||
What happens at each limit:
|
What happens at each limit:
|
||||||
|
|
||||||
- Go soft limit: as the heap approaches `GOMEMLIMIT`, the runtime runs garbage
|
- Go soft limit: as the heap approaches `GOMEMLIMIT`, the runtime runs garbage
|
||||||
collection more aggressively rather than growing further.
|
collection more aggressively rather than growing further.
|
||||||
- SQLite: at the 768 MiB soft heap limit it recycles its page cache instead of
|
- SQLite: at a 1 GiB soft heap limit it recycles its page cache instead of
|
||||||
allocating more. At the 1 GiB hard heap limit a statement fails with an
|
allocating more; at the 1.5 GiB hard heap limit a statement fails with an
|
||||||
out-of-memory error, and the handler logs it. The prefix handler writes the
|
out-of-memory error, and the handler logs it and drops that batch. The process
|
||||||
routing table's changes again on a later write. The other writes drop their
|
keeps running.
|
||||||
batch. The process keeps running.
|
|
||||||
- Handler queues: each of the four handler queues holds at most 20,000 messages.
|
- Handler queues: each of the four handler queues holds at most 20,000 messages.
|
||||||
When a queue fills, the streamer drops messages instead of blocking, except a
|
When a queue fills, the streamer drops messages instead of blocking.
|
||||||
`STATE` message, which it holds, in feed order, until there is room: a few
|
|
||||||
hundred bytes each, about ten a second.
|
|
||||||
- Handler batches: while the AS, peer or prefix handler writes a batch to the
|
|
||||||
database, the updates that arrive collect in its next batch. That batch stops
|
|
||||||
at ten times the batch size:
|
|
||||||
- 300,000 updates for the AS handler (`asnBatchMaxSize`, about 10 MiB);
|
|
||||||
- 100,000 for the peer handler (`peerBatchMaxSize`, about 10 MiB);
|
|
||||||
- 250,000 for the prefix handler (`prefixBatchMaxSize`, about 15 MiB).
|
|
||||||
|
|
||||||
At that size the handler takes nothing more off its queue until the write
|
|
||||||
ends. The queue then fills, and the streamer drops messages as above. With
|
|
||||||
the batches being written, the three handlers' batches take at most about
|
|
||||||
0.1 GiB of the Go heap.
|
|
||||||
|
|
||||||
With `DEBUG=routewatch` the daemon logs a `System stats` line every 60 seconds
|
With `DEBUG=routewatch` the daemon logs a `System stats` line every 60 seconds
|
||||||
with the goroutine count and Go memory figures.
|
with the goroutine count and Go memory figures.
|
||||||
@@ -652,35 +240,15 @@ What the [upaas](https://git.eeqj.de/sneak/upaas) app needs:
|
|||||||
|
|
||||||
- Container port: `8080`.
|
- Container port: `8080`.
|
||||||
- Volume: one, at container path `/var/lib/berlin.sneak.app.routewatch`.
|
- Volume: one, at container path `/var/lib/berlin.sneak.app.routewatch`.
|
||||||
- Environment: nothing. The server trusts a reverse proxy at an RFC 1918 address
|
- Environment: nothing is required. Leave `XDG_DATA_HOME`, `GOMEMLIMIT` and
|
||||||
without `TRUSTED_PROXIES` (see Configuration). Set it only when the proxy
|
|
||||||
connects from any other address. Leave `XDG_DATA_HOME`, `GOMEMLIMIT` and
|
|
||||||
`MALLOC_ARENA_MAX` at the image's values. `DEBUG=routewatch` is optional and
|
`MALLOC_ARENA_MAX` at the image's values. `DEBUG=routewatch` is optional and
|
||||||
adds the `System stats` memory line to the log.
|
adds the `System stats` memory line to the log.
|
||||||
- Memory Limit: `5g`, the 5 GiB limit from Memory above. upaas sets no swap
|
- Memory Limit: `5g`, the 5 GiB limit from Memory above. upaas sets no swap
|
||||||
limit, so on a host with swap Docker allows the same amount of swap again.
|
limit, so on a host with swap Docker allows the same amount of swap again.
|
||||||
- Health check: the image's `HEALTHCHECK` requests
|
- Health check: the image's `HEALTHCHECK` requests
|
||||||
`/.well-known/healthcheck.json` on the container port. upaas reads the
|
`/.well-known/healthcheck.json` on the container port. upaas reads the
|
||||||
container's health 60 seconds after a deploy and fails the deploy unless it is
|
container's health 60 seconds after a deploy and fails the deploy unless it
|
||||||
`healthy`. For the first 60 seconds, Docker 25 and later runs the check every
|
is `healthy`.
|
||||||
5 seconds, and the first pass makes the container healthy. Startup first reads
|
|
||||||
the routing table, which takes about 25 seconds when the table is full.
|
|
||||||
- Stop grace: the container needs 9 seconds between SIGTERM and SIGKILL. upaas
|
|
||||||
gives 10 (`stopTimeoutSeconds` in its `internal/docker/client.go`), as
|
|
||||||
`docker stop` does by default. The daemon allows 8 seconds for the handlers to
|
|
||||||
write what they hold and for the requests in progress to finish. Then it stops
|
|
||||||
the HTTP server, closes the database and exits with status 0. A longer grace
|
|
||||||
changes nothing. What is not written after 8 seconds is lost, and the streamer
|
|
||||||
and each handler log how much.
|
|
||||||
- The prefix handler's last write takes at most 20,000 of the routing
|
|
||||||
table's changed pairs. A full-table peer's session going down changes
|
|
||||||
about a million pairs, which take about a minute to write. A stop in that
|
|
||||||
minute leaves the rest unwritten.
|
|
||||||
- A lost withdrawal or peer down leaves its routes in the `routes` table,
|
|
||||||
and the next start reads them back. They stay until the peer sends another
|
|
||||||
update for the prefix, or its session goes down.
|
|
||||||
- When the 8 seconds run out the database is not closed. When the database
|
|
||||||
next opens, SQLite rolls back any write the exit cut off.
|
|
||||||
|
|
||||||
## Development
|
## Development
|
||||||
|
|
||||||
@@ -695,7 +263,7 @@ make fmt
|
|||||||
make lint
|
make lint
|
||||||
|
|
||||||
# Build
|
# Build
|
||||||
make build
|
make
|
||||||
```
|
```
|
||||||
|
|
||||||
## Entrypoints
|
## Entrypoints
|
||||||
@@ -703,45 +271,35 @@ make build
|
|||||||
This repository adheres to the
|
This repository adheres to the
|
||||||
[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
|
[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
|
||||||
standard: normalized scripts in `script/` are the entrypoints for the
|
standard: normalized scripts in `script/` are the entrypoints for the
|
||||||
development workflow, and the Makefile targets are thin shims that call them. We
|
development workflow, and the Makefile targets are thin shims that call
|
||||||
provide:
|
them. We provide:
|
||||||
|
|
||||||
- `script/bootstrap` — install all development dependencies (git, make, go, the
|
- `script/bootstrap` — install all development dependencies (go,
|
||||||
pinned node and yarn, prettier from `yarn.lock`, Go module download)
|
golangci-lint, Go module download)
|
||||||
- `script/setup` — make a fresh clone ready for development: runs
|
- `script/setup` — make a fresh clone ready for development: runs
|
||||||
`script/bootstrap`, then `script/install-precommit`
|
`script/bootstrap`, then `script/install-precommit`
|
||||||
- `script/projectname` — print the project name (used for the Docker image tag)
|
- `script/projectname` — print the project name (used for the Docker
|
||||||
- `script/test` — build the `test` phase of the `Dockerfile` alone, which runs
|
image tag)
|
||||||
`go test -short -count=1 -timeout 90s -race -cover ./...` (verbose rerun on
|
- `script/test` — run the test suite
|
||||||
failure). The `-short` flag skips the live-network integration test so the run
|
(`go test -short -timeout 30s -race -cover ./...`, verbose rerun on
|
||||||
is deterministic and offline.
|
failure). The `-short` flag skips the live-network integration test so the
|
||||||
- `script/lint` — build the `lint` phase of the `Dockerfile` alone, which runs
|
default run is deterministic and offline.
|
||||||
golangci-lint v2.14.0 with `.golangci.yml`
|
- `script/lint` — run `go vet ./...` and `golangci-lint run`
|
||||||
- `script/fmt` — format the Go code with `go fmt` and the markdown with prettier
|
- `script/fmt` — format all code (writes)
|
||||||
(writes)
|
|
||||||
- `script/fmt-check` — check formatting (read-only)
|
- `script/fmt-check` — check formatting (read-only)
|
||||||
- `script/check` — run `script/test`, `script/lint`, and `script/fmt-check`
|
- `script/check` — run `script/test`, `script/lint`, and
|
||||||
- `script/docker` — build the Docker image tagged via `script/projectname`
|
`script/fmt-check`
|
||||||
- `script/cibuild` — CI entrypoint: runs `script/bootstrap` and `script/check`,
|
- `script/docker` — build the Docker image tagged via
|
||||||
then builds the image
|
`script/projectname`
|
||||||
- `script/precommit` — pre-commit gate: `go mod tidy` (must not change `go.mod`
|
- `script/cibuild` — CI entrypoint: `docker build .`
|
||||||
or `go.sum`) and `go fmt` (writes), then `script/check`
|
- `script/precommit` — pre-commit gate: `go mod tidy` + `go fmt` (must
|
||||||
- `script/install-precommit` — install the git pre-commit hook that runs
|
not change files), then `script/check`
|
||||||
`script/precommit`
|
- `script/install-precommit` — install the git pre-commit hook that
|
||||||
- `script/asupdate` — rebuild the AS list built into RouteWatch,
|
runs `script/precommit`
|
||||||
`pkg/asinfo/asdata.json.gz`, from the `as.csv` file of
|
|
||||||
[ipverse/as-metadata](https://github.com/ipverse/as-metadata). It needs the
|
|
||||||
network. It runs `cmd/asinfo-gen`, which fails when it finds no AS. When
|
|
||||||
`cmd/asinfo-gen` fails, the old list stays and the script exits non-zero.
|
|
||||||
|
|
||||||
Tests and the linter run only in Docker, as the `test` and `lint` phases of the
|
The live-network integration test `TestRouteWatchLiveFeed` streams the RIPE
|
||||||
`Dockerfile`; the image cannot be built unless both pass. Every `docker build`
|
RIS feed for a few seconds and is skipped in short mode. To run it on demand,
|
||||||
in `script/` passes a new `CACHEBUST` value, so each run executes every check,
|
invoke `go test` directly without `-short`:
|
||||||
while Go's build cache and golangci-lint's cache are kept between builds.
|
|
||||||
|
|
||||||
The live-network integration test `TestRouteWatchLiveFeed` streams the RIPE RIS
|
|
||||||
feed for a few seconds and is skipped in short mode. To run it on demand, invoke
|
|
||||||
`go test` directly without `-short`:
|
|
||||||
`go test -run TestRouteWatchLiveFeed ./internal/routewatch/`.
|
`go test -run TestRouteWatchLiveFeed ./internal/routewatch/`.
|
||||||
|
|
||||||
## License
|
## License
|
||||||
|
|||||||
+86
-537
@@ -1,6 +1,6 @@
|
|||||||
---
|
---
|
||||||
title: Repository Policies
|
title: Repository Policies
|
||||||
last_modified: 2026-10-27
|
last_modified: 2026-07-06
|
||||||
---
|
---
|
||||||
|
|
||||||
This document covers repository structure, tooling, and workflow standards. Code
|
This document covers repository structure, tooling, and workflow standards. Code
|
||||||
@@ -12,7 +12,6 @@ style conventions are in separate documents:
|
|||||||
- [JavaScript](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE_JS.md)
|
- [JavaScript](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE_JS.md)
|
||||||
- [Python](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE_PYTHON.md)
|
- [Python](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE_PYTHON.md)
|
||||||
- [Go HTTP Server Conventions](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/GO_HTTP_SERVER_CONVENTIONS.md)
|
- [Go HTTP Server Conventions](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/GO_HTTP_SERVER_CONVENTIONS.md)
|
||||||
- [API Policies](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/API_POLICIES.md)
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -55,37 +54,21 @@ style conventions are in separate documents:
|
|||||||
`cibuild`. `script/bootstrap` installs all dependencies idempotently and
|
`cibuild`. `script/bootstrap` installs all dependencies idempotently and
|
||||||
assumes nothing is present: base tools come from nix, apt, brew, or apk
|
assumes nothing is present: base tools come from nix, apt, brew, or apk
|
||||||
(detected in that order; apt runs noninteractive). For node it uses the
|
(detected in that order; apt runs noninteractive). For node it uses the
|
||||||
installed node only when its major version is the pinned one; otherwise (node
|
installed node if present; otherwise it installs a PINNED node version via
|
||||||
missing, or another major, such as the node 24 the CI runner image ships) it
|
nvm, first installing nvm itself if missing — from a hash-verified GitHub
|
||||||
installs the PINNED node version via nvm and installs yarn and the packages
|
release archive (never `curl | sh`), with bash installed as an explicit
|
||||||
under it. nvm itself is installed first if missing, from a hash-verified
|
prerequisite since nvm requires bash. yarn is then pinned via
|
||||||
GitHub release archive (never `curl | sh`), with bash installed as an explicit
|
`corepack prepare yarn@<version> --activate`. Never install "latest" or "lts";
|
||||||
prerequisite since nvm requires bash. Only the major version is compared
|
always exact versions. `script/cibuild` runs the CI build: it changes to the
|
||||||
because the `Dockerfile` stages start from a node image whose exact version is
|
repo root and runs `docker build .`; the Gitea workflow calls it. Four further
|
||||||
not the pin, and nvm cannot install a prebuilt node on alpine. yarn is pinned
|
scripts are our own extensions to the standard: `script/check` runs
|
||||||
via `corepack prepare yarn@<version> --activate`. Never install "latest" or
|
`script/test`, `script/lint`, and `script/fmt-check`; `script/precommit` is
|
||||||
"lts"; always exact versions. `script/cibuild` runs the CI build: it changes
|
what the git pre-commit hook runs, and it calls `script/check`;
|
||||||
to the repo root, runs `script/bootstrap`, runs `script/check`, and builds the
|
`script/install-precommit` installs the git pre-commit hook (the `make hooks`
|
||||||
image with the version; the Gitea workflow calls it. **`script/cibuild` runs
|
target shims to it); and `script/projectname` (literally that filename) simply
|
||||||
`script/bootstrap` first**, because the workflow checks out the repo and runs
|
outputs the project's name. Scripts that need the name call
|
||||||
nothing else, while `script/fmt-check` runs the formatter on the host: on a
|
`script/projectname` — e.g. `script/docker` assembles its image tag from it —
|
||||||
pristine checkout with nothing installed the run dies there, after the
|
so those scripts stay byte-identical across all repos. Repo-type-specific
|
||||||
containerised gates have passed. **The bootstrap alone is not enough**: when
|
|
||||||
`script/bootstrap` installs node and yarn under nvm it leaves neither on the
|
|
||||||
`PATH` of the shell that called it, so a bare `yarn` either exits 127 or runs
|
|
||||||
under another node. The host entrypoints that need yarn — `script/fmt` and
|
|
||||||
`script/fmt-check` — therefore use the same test as `script/bootstrap`: when
|
|
||||||
the node on `PATH` has the pinned major version they run the `yarn` on `PATH`,
|
|
||||||
and otherwise they load nvm through `$HOME/.nvm/nvm.sh` and run yarn under the
|
|
||||||
pinned node. A runner carrying nothing but docker and git then gets through
|
|
||||||
`script/check`. Four further scripts are our own extensions to the standard:
|
|
||||||
`script/check` runs `script/test`, `script/lint` and `script/fmt-check`;
|
|
||||||
`script/precommit` is what the git pre-commit hook runs, and it calls
|
|
||||||
`script/check`; `script/install-precommit` installs the git pre-commit hook
|
|
||||||
(the `make hooks` target shims to it); and `script/projectname` (literally
|
|
||||||
that filename) simply outputs the project's name. Scripts that need the name
|
|
||||||
call `script/projectname` — e.g. `script/docker` assembles its image tag from
|
|
||||||
it — so those scripts stay byte-identical across all repos. Repo-type-specific
|
|
||||||
pre-commit extras (e.g. `go mod tidy` verification in Go repos) belong in
|
pre-commit extras (e.g. `go mod tidy` verification in Go repos) belong in
|
||||||
`script/precommit`, not in the hook itself. Model scripts are at
|
`script/precommit`, not in the hook itself. Model scripts are at
|
||||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/script/<name>`. The README
|
`https://git.eeqj.de/sneak/prompts/raw/branch/main/script/<name>`. The README
|
||||||
@@ -106,359 +89,87 @@ style conventions are in separate documents:
|
|||||||
contributor should be able to understand the entire development workflow by
|
contributor should be able to understand the entire development workflow by
|
||||||
reading the Makefile.
|
reading the Makefile.
|
||||||
|
|
||||||
- Every repo should have a `Dockerfile`, and it carries the repo's gates: a
|
- Every repo should have a `Dockerfile`. All Dockerfiles must run `make check`
|
||||||
`lint` phase and a `test` phase, with the final stage depending on both so the
|
as a build step so the build fails if the branch is not green. For non-server
|
||||||
image cannot be built unless they pass. For non-server repos the final stage
|
repos, the Dockerfile should bring up a development environment and run
|
||||||
brings up a development environment; for server repos it is the runtime image.
|
`make check`. For server repos, `make check` should run as an early build
|
||||||
The gate phases and the build stage start from their pinned base images and
|
stage before the final image is assembled. Dockerfiles install development
|
||||||
install what those images lack either inline, as the canonical Go `Dockerfile`
|
prerequisites by running `script/bootstrap` rather than duplicating installs
|
||||||
below does for `git`, or by running `script/bootstrap`, as the `prompts`
|
inline; COPY `script/` and the dependency manifests (`package.json` +
|
||||||
repo's own `Dockerfile` does for its yarn packages. The development
|
`yarn.lock`, `go.mod` + `go.sum`, etc.) before running it so the bootstrap
|
||||||
environment stage installs development prerequisites by running
|
layer stays cached until dependencies change.
|
||||||
`script/bootstrap` rather than duplicating its installs inline. A stage that
|
|
||||||
runs `script/bootstrap` COPYs `script/` and the dependency manifests
|
|
||||||
(`package.json` + `yarn.lock`, `go.mod` + `go.sum`, etc.) before running it.
|
|
||||||
|
|
||||||
- **Linting and testing run in Docker, as phases of the `Dockerfile`.** There is
|
- **Dockerfiles must use a separate lint stage for fail-fast feedback.** Go
|
||||||
no separate lint file. `script/lint` and `script/test` each build one phase
|
repos use a multistage build where linting runs in an independent stage based
|
||||||
and nothing else:
|
on the `golangci/golangci-lint` image (pinned by hash). This stage runs
|
||||||
|
`make fmt-check` and `make lint` before the full build begins. The build stage
|
||||||
|
then declares an explicit dependency on the lint stage via
|
||||||
|
`COPY --from=lint /src/go.sum /dev/null`, which forces BuildKit to complete
|
||||||
|
linting before proceeding to compilation and tests. This ensures lint failures
|
||||||
|
surface in seconds rather than minutes, without blocking on dependency
|
||||||
|
download or compilation in the build stage.
|
||||||
|
|
||||||
```sh
|
The standard pattern for a Go repo Dockerfile is:
|
||||||
docker build --build-arg CACHEBUST="$(date +%s)-$$" \
|
|
||||||
--target lint --output type=cacheonly .
|
|
||||||
docker build --build-arg CACHEBUST="$(date +%s)-$$" \
|
|
||||||
--target test --output type=cacheonly .
|
|
||||||
```
|
|
||||||
|
|
||||||
**A stage that is not the last one in the file is built only when the final
|
|
||||||
stage's chain depends on it, or when `--target` names it.** That is why the
|
|
||||||
two gates are always invoked by name here, and why the final stage carries a
|
|
||||||
`COPY --from=` of a harmless file from each of them: without that edge a
|
|
||||||
plain `docker build .` builds the last stage alone and exits 0 having linted
|
|
||||||
and tested nothing.
|
|
||||||
|
|
||||||
**The gate builds write no image.** With `--output type=cacheonly` the phase
|
|
||||||
runs and a failing step fails the build, but the result is not exported.
|
|
||||||
Nothing uses those images, and writing one out is slow: a Go test phase's
|
|
||||||
image holds the toolchain. A build given neither `--output` nor `-t` writes
|
|
||||||
an untagged image and leaves it dangling, on every developer host and every
|
|
||||||
CI runner. `script/cibuild` and `script/docker` build the image that ships
|
|
||||||
and tag it, so each build replaces the previous image; each assigns the tag
|
|
||||||
on its own line before the build, so `set -e` stops it where
|
|
||||||
`script/projectname` fails.
|
|
||||||
|
|
||||||
Inside a phase the tool is invoked directly — `golangci-lint`, `go test`,
|
|
||||||
`eslint`, `prettier` — never through `make lint` or `script/test`, which are
|
|
||||||
themselves a `docker build` and would recurse into a daemon that does not
|
|
||||||
exist in a build step. Formatting is the exception and stays on the host:
|
|
||||||
`script/fmt` writes the working tree, and `script/fmt-check` is its
|
|
||||||
read-only twin.
|
|
||||||
|
|
||||||
**No lint verdict may come from a host invocation of the linter.** On a
|
|
||||||
shared host golangci-lint reads a result cache keyed on file content rather
|
|
||||||
than location, so a second checkout of the same content is served the first
|
|
||||||
one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs
|
|
||||||
exit non-zero with `parallel golangci-lint is running` — a status a caller
|
|
||||||
cannot tell from real findings. Both have produced wrong verdicts in this
|
|
||||||
org, in both directions. A container has its own `TMPDIR`, so the lock is
|
|
||||||
out of reach. The lint phase does keep golangci-lint's cache between check
|
|
||||||
builds (below), but it lints every checkout at `/src`, and a finding is
|
|
||||||
taken from that cache only for the same files, settings and golangci-lint
|
|
||||||
version.
|
|
||||||
|
|
||||||
- **Any build that runs checks is given a new `CACHEBUST` value.** Docker
|
|
||||||
invalidates a `COPY` layer only when the copied content changes, so on an
|
|
||||||
unchanged tree the check `RUN` is served from cache, nothing executes, and the
|
|
||||||
build still exits 0. Every `docker build` in `script/` therefore passes
|
|
||||||
`--build-arg CACHEBUST="$(date +%s)-$$"`, the time and the script's process
|
|
||||||
id, so no two builds share a value, not even the three in one
|
|
||||||
`script/cibuild`. `script/lint`, `script/test`, `script/cibuild` and
|
|
||||||
`script/docker` are the four, and there is no fifth — `script/check` runs the
|
|
||||||
two gate phases and `script/fmt-check`, and builds no image of its own. In
|
|
||||||
every stage of the `Dockerfile`, every step that lints, tests or compiles
|
|
||||||
comes after `ARG CACHEBUST`. `ARG CACHEBUST` comes right after these steps:
|
|
||||||
- the steps that install system packages
|
|
||||||
- the steps that download Go's modules
|
|
||||||
- the step that runs `script/bootstrap` to install a Python `test` phase's
|
|
||||||
packages from `Pipfile.lock`
|
|
||||||
|
|
||||||
A stage with none of these steps declares `ARG CACHEBUST` right after
|
|
||||||
`FROM`. A step that installs anything else as well, such as a
|
|
||||||
`script/bootstrap` that also installs node packages, comes after it too. A
|
|
||||||
new value reruns every `RUN` step after that line; `COPY` steps are compared
|
|
||||||
by content and need nothing. Docker keeps the layers above the line until
|
|
||||||
the base image, their own lines or the files they copy change. A stage
|
|
||||||
without the `ARG` line is served from cache on an unchanged tree, so a
|
|
||||||
repository re-vendors the scripts and adds the `ARG` lines in one commit.
|
|
||||||
The scripts do not pass `--no-cache`: it reruns every step too, but it also
|
|
||||||
builds the layers above the line again and empties every cache mount, so
|
|
||||||
Go's build cache and golangci-lint's, which the canonical Go `Dockerfile`
|
|
||||||
below mounts so that they survive between check builds, would start empty
|
|
||||||
every time. A bare `docker build .` is not evidence that anything ran: a
|
|
||||||
sub-second build reporting success is a cache hit, not a result. Never
|
|
||||||
invalidate by pruning — `docker builder prune` and friends destroy a build
|
|
||||||
cache shared with every other build on the host. When a check is added or
|
|
||||||
changed, prove it works by planting a defect it must catch and watching the
|
|
||||||
run fail on it, then revert the defect. A green run alone shows neither that
|
|
||||||
the check ran nor that it covers what it should.
|
|
||||||
|
|
||||||
- **The gate phases are separate stages, and the build stage depends on both.**
|
|
||||||
The lint phase is based on the `golangci/golangci-lint` image (pinned by
|
|
||||||
hash), so lint failures surface in seconds rather than after a full compile,
|
|
||||||
and the test phase is based on the Debian Go image. The canonical Go repo
|
|
||||||
`Dockerfile`:
|
|
||||||
|
|
||||||
```dockerfile
|
```dockerfile
|
||||||
# In every stage ARG CACHEBUST comes right after the steps that install
|
# Lint stage — fast feedback on formatting and lint issues
|
||||||
# system packages and download Go's modules, or right after FROM in a
|
|
||||||
# stage with neither, and every step that lints, tests or compiles
|
|
||||||
# comes after it. script/ passes a new value on every build, which
|
|
||||||
# reruns every RUN step after it. Docker keeps the layers above it
|
|
||||||
# until the base image, their own lines or go.mod and go.sum change.
|
|
||||||
# The steps that compile mount Go's build cache, and golangci-lint run
|
|
||||||
# also mounts golangci-lint's; both survive from build to build.
|
|
||||||
|
|
||||||
# Lint phase
|
|
||||||
# golangci/golangci-lint:v2.x.x, YYYY-MM-DD
|
# golangci/golangci-lint:v2.x.x, YYYY-MM-DD
|
||||||
FROM golangci/golangci-lint@sha256:... AS lint
|
FROM golangci/golangci-lint@sha256:... AS lint
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
RUN go mod download
|
RUN go mod download
|
||||||
ARG CACHEBUST
|
|
||||||
COPY . .
|
COPY . .
|
||||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
RUN make fmt-check
|
||||||
--mount=type=cache,target=/root/.cache/golangci-lint \
|
RUN make lint
|
||||||
golangci-lint run --config .golangci.yml ./...
|
|
||||||
|
|
||||||
# Test phase. -race needs cgo and so a C compiler, which the Debian Go
|
# Build stage
|
||||||
# image ships and the alpine one does not. -count=1 because the mounted
|
|
||||||
# cache also holds test results, which go test would otherwise replay.
|
|
||||||
# golang:1.x, YYYY-MM-DD
|
|
||||||
FROM golang@sha256:... AS test
|
|
||||||
WORKDIR /src
|
|
||||||
COPY go.mod go.sum ./
|
|
||||||
RUN go mod download
|
|
||||||
ARG CACHEBUST
|
|
||||||
COPY . .
|
|
||||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
|
||||||
go test -count=1 -timeout 90s -race -cover ./... || \
|
|
||||||
{ echo "--- Rerunning with -v for details ---"; \
|
|
||||||
go test -count=1 -timeout 90s -race -v ./...; exit 1; }
|
|
||||||
|
|
||||||
# Build stage. Nothing is wanted from either phase above; the copies
|
|
||||||
# are what make BuildKit build them first, so nothing after them can
|
|
||||||
# run unless lint and test passed.
|
|
||||||
# golang:1.x-alpine, YYYY-MM-DD
|
# golang:1.x-alpine, YYYY-MM-DD
|
||||||
FROM golang@sha256:... AS builder
|
FROM golang@sha256:... AS builder
|
||||||
RUN apk add --no-cache git
|
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
|
|
||||||
|
# Force BuildKit to run the lint stage before proceeding
|
||||||
|
COPY --from=lint /src/go.sum /dev/null
|
||||||
|
|
||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
RUN go mod download
|
RUN go mod download
|
||||||
ARG CACHEBUST
|
|
||||||
COPY --from=lint /src/go.sum /dev/null
|
|
||||||
COPY --from=test /src/go.sum /dev/null
|
|
||||||
# A tar-stream context keeps the sender's file owners, which git refuses.
|
|
||||||
RUN git config --system --add safe.directory /src
|
|
||||||
COPY . .
|
COPY . .
|
||||||
|
RUN make test
|
||||||
|
|
||||||
# The VERSION build arg when one is given, otherwise
|
ARG VERSION=dev
|
||||||
# `git describe --tags --always` on the .git in the build context. With
|
RUN CGO_ENABLED=0 go build -trimpath \
|
||||||
# .git present, a version that is still empty, dev or unknown fails the
|
|
||||||
# build: git is missing or could not read the checkout.
|
|
||||||
ARG VERSION
|
|
||||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
|
||||||
VERSION="${VERSION:-$(git describe --tags --always)}"; \
|
|
||||||
if [ -e .git ]; then \
|
|
||||||
case "$VERSION" in ""|dev|unknown) \
|
|
||||||
echo "version is '$VERSION' although .git is present" >&2; \
|
|
||||||
exit 1 ;; \
|
|
||||||
esac; \
|
|
||||||
fi; \
|
|
||||||
CGO_ENABLED=0 go build -trimpath \
|
|
||||||
-ldflags="-s -w -X main.Version=${VERSION}" \
|
-ldflags="-s -w -X main.Version=${VERSION}" \
|
||||||
-o /app ./cmd/app/
|
-o /app ./cmd/app/
|
||||||
|
|
||||||
# Runtime stage, and the last one
|
# Runtime stage
|
||||||
FROM alpine@sha256:...
|
FROM alpine@sha256:...
|
||||||
ARG CACHEBUST
|
|
||||||
COPY --from=builder /app /usr/local/bin/app
|
COPY --from=builder /app /usr/local/bin/app
|
||||||
ENTRYPOINT ["app"]
|
ENTRYPOINT ["app"]
|
||||||
```
|
```
|
||||||
|
|
||||||
Key points:
|
Key points:
|
||||||
- The lint phase uses the `golangci/golangci-lint` image directly (it has
|
- The lint stage uses the `golangci/golangci-lint` image directly (it
|
||||||
both Go and the linter), so nothing needs installing.
|
includes both Go and the linter), so there is no need to install the
|
||||||
- `COPY --from=<phase> /src/go.sum /dev/null` is a no-op copy whose only
|
linter separately.
|
||||||
purpose is the ordering edge. BuildKit runs stages in parallel by default,
|
- `COPY --from=lint /src/go.sum /dev/null` is a no-op file copy that creates
|
||||||
and a stage nothing depends on is not built at all, so without these two
|
a stage dependency. BuildKit runs stages in parallel by default; without
|
||||||
lines a red gate would not fail the build.
|
this line, the build stage would not wait for lint to finish and a lint
|
||||||
- Keep the runtime stage last, and if you add a stage after it, give it the
|
failure might not fail the overall build.
|
||||||
same two copies. A plain `docker build .` builds the last stage's chain
|
|
||||||
and nothing else.
|
|
||||||
- Check builds keep these, and none of them can hand back a stale result:
|
|
||||||
- The layers above `ARG CACHEBUST`: the system packages and Go's
|
|
||||||
modules. Docker installs the packages again when the base image or the
|
|
||||||
install line changes, and downloads the modules again when `go.mod` or
|
|
||||||
`go.sum` changes. Go checks every module against `go.sum` as it
|
|
||||||
downloads it, and no later build can change a kept layer. The modules
|
|
||||||
are kept in a layer rather than a cache mount because a cache mount is
|
|
||||||
shared by every build on the host, and Go builds from whatever files
|
|
||||||
it holds without checking them again (only `go mod verify` does), so a
|
|
||||||
file changed there would go unnoticed.
|
|
||||||
- Go's build cache (`/root/.cache/go-build`, where Go keeps it for
|
|
||||||
root), mounted on the steps that compile: `golangci-lint run`, both
|
|
||||||
`go test` runs and `go build`. Go reuses a compiled package only when
|
|
||||||
all its inputs are unchanged. The cache also holds test results, which
|
|
||||||
is why both `go test` runs carry `-count=1`.
|
|
||||||
- golangci-lint's cache (`/root/.cache/golangci-lint`, where the pinned
|
|
||||||
image keeps it), mounted on `golangci-lint run`. golangci-lint reuses
|
|
||||||
a package's findings only when the package's files and those of every
|
|
||||||
package it imports, `go.mod`, the linters and their settings in
|
|
||||||
`.golangci.yml`, and the golangci-lint version are all unchanged.
|
|
||||||
- In a Python `test` phase, the layer of the step that runs
|
|
||||||
`script/bootstrap` to install the packages from `Pipfile.lock`, as
|
|
||||||
described for that phase below.
|
|
||||||
- If the project uses `//go:embed` directives that reference build artifacts
|
- If the project uses `//go:embed` directives that reference build artifacts
|
||||||
(e.g. a web frontend compiled in a separate stage), the lint phase must
|
(e.g. a web frontend compiled in a separate stage), the lint stage must
|
||||||
create placeholder files so the embed directives resolve. Example:
|
create placeholder files so the embed directives resolve. Example:
|
||||||
`RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`.
|
`RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`.
|
||||||
- If the project requires CGO or system libraries for linting, install them
|
The lint stage should not depend on the actual build output — it exists to
|
||||||
in the lint phase. The `golangci/golangci-lint` image is Debian-based and
|
fail fast.
|
||||||
has no `apk`, so install with `apt-get` under the Debian package name
|
- If the project requires CGO or system libraries for linting (e.g.
|
||||||
(`libvips-dev`, where alpine says `vips-dev`), and delete the package
|
`vips-dev`), install them in the lint stage with `apk add`.
|
||||||
lists in the same `RUN`, so the layer does not keep them. The test phase,
|
- The build stage runs `make test` after compilation setup. Tests run in the
|
||||||
on the Debian Go image, installs the libraries its tests need the same
|
build stage, not the lint stage, because they may require compiled
|
||||||
way. Either install goes right after `FROM`, above `ARG CACHEBUST`, so
|
artifacts or heavier dependencies.
|
||||||
Docker keeps it until the base image or the install line changes:
|
|
||||||
|
|
||||||
```dockerfile
|
|
||||||
FROM golangci/golangci-lint@sha256:... AS lint
|
|
||||||
RUN apt-get update \
|
|
||||||
&& apt-get install -y --no-install-recommends libvips-dev \
|
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
|
||||||
WORKDIR /src
|
|
||||||
COPY go.mod go.sum ./
|
|
||||||
RUN go mod download
|
|
||||||
ARG CACHEBUST
|
|
||||||
```
|
|
||||||
|
|
||||||
- `.dockerignore` lets `.git` into the build context. It keeps out every git
|
|
||||||
`config` at any depth (`**/.git/config`, `**/.git/modules/**/config`): the
|
|
||||||
repository's own, each submodule's under `.git/modules/`, and that of a
|
|
||||||
submodule keeping its own `.git` directory. `git describe` does not need
|
|
||||||
them, and each can hold a credential: a password in a remote URL, or the
|
|
||||||
token the CI checkout step stores there. A submodule whose name has a
|
|
||||||
`config` segment (`config`, `deploy/config`, `config/lib`) loses its whole
|
|
||||||
git directory to `**/.git/modules/**/config`, and Go's version stamping
|
|
||||||
then fails the build: give it a name without that segment
|
|
||||||
(`git submodule add --name`). The stage that compiles has `git` (the
|
|
||||||
Debian Go image has it; an alpine one needs `apk add --no-cache git`) and
|
|
||||||
takes the version from the `VERSION` build argument when one is given,
|
|
||||||
otherwise from `git describe --tags --always`. That gives the tag on a
|
|
||||||
tagged commit; on a later commit, the tag, the number of commits since it
|
|
||||||
and the short commit (`v1.2.3-4-gabc1234`); and the short commit when no
|
|
||||||
tag is reachable. The stage that compiles also marks its working directory
|
|
||||||
safe for git (`git config --system --add safe.directory /src`): a context
|
|
||||||
sent as a tar stream keeps the sender's file owners, and git refuses a
|
|
||||||
checkout owned by another user, so the version would come out empty.
|
|
||||||
`ARG VERSION` has no default, and the build fails if the context carries
|
|
||||||
`.git` and the version still comes out empty, `dev` or `unknown`. A plain
|
|
||||||
`docker build .` with no build arguments must succeed; a Dockerfile that
|
|
||||||
refuses an empty build argument drops that refusal and keeps the argument.
|
|
||||||
A checkout whose `.git` is a file (a linked worktree, or a repository
|
|
||||||
checked out as a submodule) is the exception: that file points to a git
|
|
||||||
directory outside the build context, so the build cannot read the version
|
|
||||||
and a plain `docker build .` fails; pass the version with
|
|
||||||
`--build-arg VERSION=...`, as `script/docker` and `script/cibuild` already
|
|
||||||
do.
|
|
||||||
|
|
||||||
- **A Python `test` phase that installs its packages from `Pipfile.lock` does it
|
|
||||||
above `ARG CACHEBUST`, so check builds keep them.** The steps are
|
|
||||||
`COPY script/ script/`, `COPY Pipfile Pipfile.lock ./` and
|
|
||||||
`RUN script/bootstrap`, with `ARG CACHEBUST` right after them. Docker keeps
|
|
||||||
what that `RUN` installed until the base image, `script/`, `Pipfile`,
|
|
||||||
`Pipfile.lock` or a line up to and including the `RUN` changes. The tests run
|
|
||||||
after `ARG CACHEBUST`, on every check build. In every other stage, a step that
|
|
||||||
installs Python packages comes after `ARG CACHEBUST`.
|
|
||||||
- When `script/bootstrap` runs in that step, it installs nothing beyond
|
|
||||||
system packages, pipenv and its dependencies, and the packages from
|
|
||||||
`Pipfile.lock`. It installs no Python there: the tests use the base
|
|
||||||
image's Python, which the image's hash pins. If it installs anything else
|
|
||||||
there, the step goes after `ARG CACHEBUST`.
|
|
||||||
- Every external reference must be pinned by hash, so `script/bootstrap`
|
|
||||||
installs pipenv and its dependencies by hash. It does so with
|
|
||||||
`pip install --require-hashes -r` and a requirements file in `script/`
|
|
||||||
that lists each of them with `==` and its `--hash`. pip refuses to install
|
|
||||||
a dependency the file does not list.
|
|
||||||
- It then runs `pipenv install --deploy`, with `--dev` when the tests need
|
|
||||||
the development packages. `--deploy` installs only what `Pipfile.lock`
|
|
||||||
names, checks every package against the lockfile's hashes, and fails when
|
|
||||||
the lockfile is out of date with `Pipfile`.
|
|
||||||
- A kept layer holds nothing a new install would refuse: pip checked pipenv
|
|
||||||
and its dependencies, and pipenv checked the lockfile's packages, against
|
|
||||||
their hashes. pipenv creates its virtual environment with the `virtualenv`
|
|
||||||
package. The packages that `virtualenv` adds come from wheels inside that
|
|
||||||
package, so the hash pip checked for it covers them. Two things in the
|
|
||||||
layer are not fixed by a hash. A package with no ready-built wheel for the
|
|
||||||
image is built from source. pip downloads the build tools without a hash
|
|
||||||
check, so a kept layer's build of such a package may differ from a new
|
|
||||||
one. And system packages keep the versions the package manager gave when
|
|
||||||
the layer was built; a new install may get newer ones.
|
|
||||||
- As with Go's modules, the packages are kept in a layer, not a cache mount.
|
|
||||||
A cache mount is shared by every build on the host, and nothing checks its
|
|
||||||
files again.
|
|
||||||
|
|
||||||
In this example, `script/bootstrap` finds `make` and `git` already
|
|
||||||
installed, installs pipenv and its dependencies by hash, and runs
|
|
||||||
`pipenv install --dev --deploy`:
|
|
||||||
|
|
||||||
```dockerfile
|
|
||||||
# Test phase. The 90-second timeout covers both pytest runs.
|
|
||||||
# python:3.x-slim, YYYY-MM-DD
|
|
||||||
FROM python@sha256:... AS test
|
|
||||||
# make and git are base tools that script/bootstrap installs when they
|
|
||||||
# are missing. Installed here, they stay in their own layer when
|
|
||||||
# script/ changes.
|
|
||||||
RUN apt-get update \
|
|
||||||
&& apt-get install -y --no-install-recommends make git \
|
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
|
||||||
WORKDIR /app
|
|
||||||
COPY script/ script/
|
|
||||||
COPY Pipfile Pipfile.lock ./
|
|
||||||
RUN script/bootstrap
|
|
||||||
ARG CACHEBUST
|
|
||||||
COPY . .
|
|
||||||
RUN timeout 90 sh -c 'pipenv run pytest || { \
|
|
||||||
echo "--- Rerunning with -v for details ---"; \
|
|
||||||
pipenv run pytest -v; exit 1; }'
|
|
||||||
```
|
|
||||||
|
|
||||||
- Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that
|
- Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that
|
||||||
runs `script/cibuild` on push, and checks out the repo as its only other step,
|
runs `script/cibuild` (which runs `docker build .`) on push. Since the
|
||||||
with `persist-credentials: false`: `script/cibuild` needs no token, and
|
Dockerfile already runs `make check`, a successful build implies all checks
|
||||||
without it the checkout leaves the job's token in `.git/config` for every
|
pass.
|
||||||
later step. The checkout step also sets `fetch-depth: 0`, which fetches the
|
|
||||||
tags `git describe` needs: by default it clones shallow with no tags, and a
|
|
||||||
tagged repository's CI build would stamp a bare short commit id. The
|
|
||||||
workflow's `concurrency` block groups runs by workflow and branch
|
|
||||||
(`${{ github.workflow }}-${{ github.ref }}`) with `cancel-in-progress: true`,
|
|
||||||
so a new push cancels the older run on the same branch, queued or running, and
|
|
||||||
no other: runs for replaced commits do not hold up the shared runner.
|
|
||||||
`script/cibuild` bootstraps, runs the gate phases, and then builds the image,
|
|
||||||
so a successful run means every check passed; a bare `docker build .` does not
|
|
||||||
carry the same guarantee, because its gate phases may come from the cache. The
|
|
||||||
image build gets a `CACHEBUST` value of its own and so runs the gate phases a
|
|
||||||
second time. That is the price of the rule above, and it is worth paying: the
|
|
||||||
image that ships is built from a run of its own gates rather than from a cache
|
|
||||||
entry. The `check` job sets `timeout-minutes: 20`, so a hung build frees the
|
|
||||||
shared runner after 20 minutes. That allows for the three Docker builds
|
|
||||||
described above (the test phase, the lint phase, then the image), each held to
|
|
||||||
the 5-minute Docker build limit below, plus the bootstrap. A separate workflow
|
|
||||||
limited to `main` by a `branches` list under `on: push` cannot be checked by
|
|
||||||
review: to try a change to it, add the feature branch to that list and push,
|
|
||||||
then remove the branch from the list again before merging. Keep any job in it
|
|
||||||
that publishes behind `if: github.ref_name == 'main'`, so the run from the
|
|
||||||
feature branch publishes nothing.
|
|
||||||
|
|
||||||
- Use platform-standard formatters: `black` for Python, `prettier` for
|
- Use platform-standard formatters: `black` for Python, `prettier` for
|
||||||
JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with
|
JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with
|
||||||
@@ -478,21 +189,14 @@ style conventions are in separate documents:
|
|||||||
module under test to verify it compiles/parses. There is no excuse for
|
module under test to verify it compiles/parses. There is no excuse for
|
||||||
`make test` to be a no-op.
|
`make test` to be a no-op.
|
||||||
|
|
||||||
- `make test` must complete in under 60 seconds. That is the hard cap, and a
|
- `make test` must complete in under 20 seconds. Add a 30-second timeout in the
|
||||||
suite that exceeds it fails. Under 20 seconds is the target. A suite between
|
Makefile.
|
||||||
20 and 60 seconds is still green, but the overage must be filed as an
|
|
||||||
improvement bug against that repo. Add a 90-second timeout to the test
|
|
||||||
invocation (`go test -timeout 90s`). The backstop deliberately sits above the
|
|
||||||
hard cap so that it catches a genuinely hung test rather than a merely slow
|
|
||||||
one.
|
|
||||||
|
|
||||||
- **The test command should use the conditional verbose rerun pattern.** Run
|
- **`make test` should use the conditional verbose rerun pattern.** Run tests
|
||||||
tests without `-v` (verbose) first. If tests fail, automatically rerun with
|
without `-v` (verbose) first. If tests fail, automatically rerun with `-v` to
|
||||||
`-v` to show full output. This keeps CI logs and `docker build` output clean
|
show full output. This keeps CI logs and `docker build` output clean on
|
||||||
on success (just package/suite summaries) while providing full diagnostic
|
success (just package/suite summaries) while providing full diagnostic detail
|
||||||
detail on failure (every test case, every assertion). The command lives in the
|
on failure (every test case, every assertion). The general shell pattern:
|
||||||
`test` phase of the `Dockerfile`, since `script/test` builds that phase; the
|
|
||||||
Makefile form below is the same pattern for any repo-local invocation:
|
|
||||||
|
|
||||||
```makefile
|
```makefile
|
||||||
test:
|
test:
|
||||||
@@ -505,25 +209,11 @@ style conventions are in separate documents:
|
|||||||
|
|
||||||
```makefile
|
```makefile
|
||||||
test:
|
test:
|
||||||
@go test -count=1 -timeout 90s -race -cover ./... || \
|
@go test -timeout 30s -race -cover ./... || \
|
||||||
{ echo "--- Rerunning with -v for details ---"; \
|
{ echo "--- Rerunning with -v for details ---"; \
|
||||||
go test -count=1 -timeout 90s -race -v ./...; exit 1; }
|
go test -timeout 30s -race -v ./...; exit 1; }
|
||||||
```
|
```
|
||||||
|
|
||||||
`-count=1` is required on both invocations: it defeats Go's test _result_
|
|
||||||
cache, so neither run can report a stored pass in place of running the
|
|
||||||
tests. It leaves the build cache alone, so it costs the runtime of the suite
|
|
||||||
and no recompilation.
|
|
||||||
|
|
||||||
That cache is Go's own, separate from Docker's layer cache. Go stores a
|
|
||||||
passing result in its cache directory (`GOCACHE`), and when the same tests
|
|
||||||
run again on unchanged code it prints that result, marked `(cached)`,
|
|
||||||
without running them. The directory lasts from one run to the next on a
|
|
||||||
developer's machine, where this target runs, and in the `test` phase of the
|
|
||||||
`Dockerfile`, which mounts it to keep compiled packages between check
|
|
||||||
builds, so the phase's `go test` runs carry `-count=1` too. A new
|
|
||||||
`CACHEBUST` value (above) is what makes that step run on an unchanged tree.
|
|
||||||
|
|
||||||
Python example:
|
Python example:
|
||||||
|
|
||||||
```makefile
|
```makefile
|
||||||
@@ -549,89 +239,10 @@ style conventions are in separate documents:
|
|||||||
must be in `.gitignore`. No exceptions.
|
must be in `.gitignore`. No exceptions.
|
||||||
|
|
||||||
- `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`),
|
- `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`),
|
||||||
editor files (`.swp`, `*~`), in-repo agent scratch directories (`.claude/`),
|
editor files (`.swp`, `*~`), language build artifacts, and `node_modules/`.
|
||||||
`node_modules/`, and the repo's own build outputs. Fetch the standard
|
Fetch the standard `.gitignore` from
|
||||||
`.gitignore` from
|
|
||||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when setting up
|
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when setting up
|
||||||
a new repo. A repo's `.gitignore` is the standard file followed by the repo's
|
a new repo.
|
||||||
own entries, such as its binaries; a re-vendor replaces the standard part and
|
|
||||||
keeps those entries. These patterns are written to `.gitignore`'s own
|
|
||||||
semantics, in which an unanchored pattern already matches at every depth; they
|
|
||||||
are not a `.dockerignore` and must not be transplanted into one unmodified.
|
|
||||||
|
|
||||||
- **`.dockerignore` does not use `.gitignore` semantics, and copying patterns
|
|
||||||
across unmodified leaves secrets in the build context.** Docker matches with
|
|
||||||
`moby/patternmatcher`: `filepath.Match` semantics plus a `**` extension, so
|
|
||||||
`*` does not cross `/` and a pattern without a leading `**/` is anchored at
|
|
||||||
the build-context root. A `.dockerignore` listing `.env`, `*.pem` and `*.key`
|
|
||||||
therefore excludes only the copies at the repository root, while `config/.env`
|
|
||||||
and `certs/server.key` still reach the context and can land in an image layer
|
|
||||||
— which is more dangerous than a short file with no secret patterns at all,
|
|
||||||
because it reads as solved and stops anyone looking. Give every
|
|
||||||
depth-independent pattern the `**/` prefix and leave only genuinely
|
|
||||||
root-anchored entries unprefixed: `.claude`, and the repo's own host-built
|
|
||||||
binary, written `/myapp` and never `**/myapp`, which would also match
|
|
||||||
`cmd/myapp/` and delete the package directory from the context. Matching is
|
|
||||||
case-sensitive, and an ALL-CAPS twin per pattern still misses `Server.Key`, so
|
|
||||||
secret names use character ranges — `**/*.[kK][eE][yY]`, `**/*.[pP][eE][mM]`,
|
|
||||||
and likewise for `.envrc` and the extensionless SSH keys. Where such a pattern
|
|
||||||
also catches something the build needs, re-include it with a negation
|
|
||||||
(`!docs/example.env`); deleting the pattern reopens the exposure for every
|
|
||||||
other file it covers. Fetch the standard `.dockerignore` from
|
|
||||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.dockerignore` and extend
|
|
||||||
it with the repo's own artifacts.
|
|
||||||
|
|
||||||
- **In-repo agent scratch belongs in both files, written to each file's own
|
|
||||||
semantics.** `.claude/` holds one worktree per in-flight agent — an entire
|
|
||||||
additional checkout of the repo — so under `COPY . .` the build context
|
|
||||||
inflates by a multiple of the repo and another session's unreviewed work can
|
|
||||||
be copied into an image layer. In `.gitignore` the entry is `.claude/`,
|
|
||||||
unanchored. In `.dockerignore` it is `.claude`, anchored and with **no** `**/`
|
|
||||||
prefix, because the prefixed form would also delete any nested directory of
|
|
||||||
that name from the build. Anchoring carries a known gap that the canonical
|
|
||||||
`.dockerignore` states in its own comment, since consuming repos receive the
|
|
||||||
file and not the tracker: the directory is created in the agent's working
|
|
||||||
directory, so a repo running agents in subdirectories still ships
|
|
||||||
`services/api/.claude/` and must add its own anchored entry there.
|
|
||||||
|
|
||||||
- **A plain `docker build .` of a clone stamps the version that
|
|
||||||
`git describe --tags --always` gives**, derived from the `.git` in the build
|
|
||||||
context as the canonical `Dockerfile` above shows. Without its failure check,
|
|
||||||
a missing `git` or an unreadable checkout would leave `-X main.Version=` empty
|
|
||||||
and the build would still exit 0. `script/docker` and `script/cibuild` pass
|
|
||||||
the version they compute on the host; it takes precedence. They do this
|
|
||||||
byte-identically across repos:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
# The version and the tag each get their own line: a failing command
|
|
||||||
# substitution inside an argument does not trip `set -e`, so the inline
|
|
||||||
# form degrades to an empty constant.
|
|
||||||
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
|
|
||||||
[ -n "$version" ] || version="unknown"
|
|
||||||
tag="$(script/projectname)"
|
|
||||||
docker build --build-arg CACHEBUST="$(date +%s)-$$" \
|
|
||||||
--build-arg VERSION="$version" \
|
|
||||||
-t "$tag" .
|
|
||||||
```
|
|
||||||
|
|
||||||
`--always` makes an untagged repo yield an abbreviated commit hash rather
|
|
||||||
than failing, and the `[ -n "$version" ]` line is the single place the
|
|
||||||
fallback is applied — a live check that fires on a build from an export with
|
|
||||||
no `.git` and on a repository with no commits yet. Do not fold it into the
|
|
||||||
substitution as `|| echo unknown`, which makes the guard unreachable. The
|
|
||||||
Dockerfile's side is `ARG VERSION` in the stage that compiles, declared
|
|
||||||
there because `ARG` is stage-scoped; passing `VERSION` to a repo whose
|
|
||||||
Dockerfile declares no such `ARG` is ignored and costs nothing, which is why
|
|
||||||
the scripts stay byte-identical. One consequence for CI: the standard
|
|
||||||
checkout action clones shallow and fetches no tags, so the canonical
|
|
||||||
`.gitea/workflows/check.yml` sets `fetch-depth: 0` on its checkout step.
|
|
||||||
|
|
||||||
- **Verify `.dockerignore` by enumerating the image, not by reading the
|
|
||||||
patterns.** Plant files at the root _and_ at least two directories deep, build
|
|
||||||
a probe image that does `COPY . .`, and list what actually landed
|
|
||||||
(`docker run --rm --entrypoint find IMAGE /app`). The `transferring context`
|
|
||||||
size is not a substitute: a nested secret is a few bytes, and BuildKit
|
|
||||||
transfers only the delta from the previous build.
|
|
||||||
|
|
||||||
- **No build artifacts in version control.** Code-derived data (compiled
|
- **No build artifacts in version control.** Code-derived data (compiled
|
||||||
bundles, minified output, generated assets) must never be committed to the
|
bundles, minified output, generated assets) must never be committed to the
|
||||||
@@ -647,61 +258,9 @@ style conventions are in separate documents:
|
|||||||
- Make all changes on a feature branch. You can do whatever you want on a
|
- Make all changes on a feature branch. You can do whatever you want on a
|
||||||
feature branch.
|
feature branch.
|
||||||
|
|
||||||
- `.golangci.yml` is standardized. The vendored copy in a consuming repo must
|
- `.golangci.yml` is standardized and must _NEVER_ be modified by an agent, only
|
||||||
_NEVER_ be modified by an agent: fetch it from
|
manually by the user. Fetch from
|
||||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml` and keep it
|
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml`.
|
||||||
byte-identical, so that no repo can quietly loosen its own linting. Linter
|
|
||||||
configuration changes are made to the canonical copy in the `prompts` repo and
|
|
||||||
reach consuming repos by re-vendoring; an agent may open a PR against
|
|
||||||
canonical, which only the user merges. One list is exempt from byte-identity,
|
|
||||||
because it cannot be written once for every repo: the `deny` list of the
|
|
||||||
`test-support` depguard rule, where a repo names its own test-support packages
|
|
||||||
by full import path. A repo adds entries there and changes nothing else, and a
|
|
||||||
re-vendor carries its entries forward. The canonical golangci-lint version is
|
|
||||||
v2.14.0 (released 2026-09-24), pinned as the digest of the lint phase's base
|
|
||||||
image
|
|
||||||
(`golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f`,
|
|
||||||
which reports `2.14.0 built with go1.27.0 from 114493f9`). A module's `go`
|
|
||||||
directive must not name a newer Go minor version than the one golangci-lint
|
|
||||||
was built with, or golangci-lint refuses to lint it: this release lints
|
|
||||||
`go 1.27.1` but not `go 1.28`. That digest is the only pin, since no repo
|
|
||||||
installs golangci-lint on the host. A repo sets the lint phase digest to the
|
|
||||||
one named here and re-vendors `.golangci.yml` in the same commit, whichever of
|
|
||||||
the two prompted the change: the canonical copy can name linters that an older
|
|
||||||
golangci-lint rejects, and a newer golangci-lint can add linters that
|
|
||||||
`default: all` switches on until the canonical copy disables them.
|
|
||||||
|
|
||||||
- **`script/bootstrap` installs a pinned tool by comparing versions, never by
|
|
||||||
testing presence.** An `if ! command -v <tool>; then install; fi` guard tests
|
|
||||||
`PATH` only, so on an already-provisioned machine the pin is inert and a
|
|
||||||
version bump is a silent no-op — while the Dockerfile, installing into a clean
|
|
||||||
image, gets the pinned version, so a local `make check` and `make docker` can
|
|
||||||
disagree about what the tool even is. The canonical form:
|
|
||||||
- compares the installed version against the pin over the **whole** version
|
|
||||||
token; a parser that stops at the first `-` reports `2.12.2` for a host
|
|
||||||
running `2.12.2-rc1` and skips the install;
|
|
||||||
- treats absent, non-zero, empty or unrecognised `--version` output as a
|
|
||||||
mismatch, so the failure direction is a redundant install and never a
|
|
||||||
skipped one;
|
|
||||||
- after installing, re-resolves the binary the way callers do — `hash -r`,
|
|
||||||
then through `PATH`, not through the directory the installer wrote to —
|
|
||||||
and fails naming the resolved path, since an install that a shadowing
|
|
||||||
binary hides succeeds while changing nothing any caller sees;
|
|
||||||
- is actually called, and prints the version on both success paths: a
|
|
||||||
function defined and never invoked has the same exit status and the same
|
|
||||||
empty output as one that worked.
|
|
||||||
|
|
||||||
Node is the exception to the whole-token comparison: the canonical
|
|
||||||
`script/bootstrap` compares only its major version, because the node 22
|
|
||||||
alpine image the `Dockerfile` stages start from is not the exact pin, and
|
|
||||||
nvm cannot install a prebuilt node on alpine.
|
|
||||||
|
|
||||||
Keep it POSIX sh: no arrays, no `[[`, no `grep -P`.
|
|
||||||
|
|
||||||
A Go tool a repo needs on the host is installed with `go install` pinned to
|
|
||||||
a commit hash (`go install <package>@<commit hash>`). It is never tracked as
|
|
||||||
a `go.mod` tool dependency or through a `tools.go` file, either of which
|
|
||||||
pulls the tool's own dependencies into the repo's `go.mod` and `go.sum`.
|
|
||||||
|
|
||||||
- When pinning images or packages by hash, add a comment above the reference
|
- When pinning images or packages by hash, add a comment above the reference
|
||||||
with the version and date (YYYY-MM-DD).
|
with the version and date (YYYY-MM-DD).
|
||||||
@@ -812,21 +371,15 @@ style conventions are in separate documents:
|
|||||||
Never edit existing migrations after release.
|
Never edit existing migrations after release.
|
||||||
|
|
||||||
- All repos should have an `.editorconfig` enforcing the project's indentation
|
- All repos should have an `.editorconfig` enforcing the project's indentation
|
||||||
settings: the standard file from
|
settings.
|
||||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.editorconfig`, which sets
|
|
||||||
tabs for `Makefile` and Go files, followed by the repo's own sections, such as
|
|
||||||
one for another language it uses. A re-vendor replaces the standard part and
|
|
||||||
keeps those sections.
|
|
||||||
|
|
||||||
- Avoid putting files in the repo root unless necessary. Root should contain
|
- Avoid putting files in the repo root unless necessary. Root should contain
|
||||||
only project-level config files (`README.md`, `AGENTS.md`, `Makefile`,
|
only project-level config files (`README.md`, `Makefile`, `Dockerfile`,
|
||||||
`Dockerfile`, `LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`,
|
`LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, and
|
||||||
and language-specific config). Everything else goes in a subdirectory.
|
language-specific config). Everything else goes in a subdirectory. Canonical
|
||||||
Canonical subdirectory names:
|
subdirectory names:
|
||||||
- `bin/` — executable scripts and tools
|
- `bin/` — executable scripts and tools
|
||||||
- `cmd/` — Go command entrypoints; thin only: one `main.go` per binary whose
|
- `cmd/` — Go command entrypoints
|
||||||
body is a single call into `internal/` or `pkg/`, no project logic in
|
|
||||||
`cmd/`
|
|
||||||
- `configs/` — configuration templates and examples
|
- `configs/` — configuration templates and examples
|
||||||
- `deploy/` — deployment manifests (k8s, compose, terraform)
|
- `deploy/` — deployment manifests (k8s, compose, terraform)
|
||||||
- `docs/` — documentation and markdown (README.md stays in root)
|
- `docs/` — documentation and markdown (README.md stays in root)
|
||||||
@@ -853,7 +406,3 @@ style conventions are in separate documents:
|
|||||||
- Go: `go.mod`, `go.sum`, `.golangci.yml`
|
- Go: `go.mod`, `go.sum`, `.golangci.yml`
|
||||||
- JS: `package.json`, `yarn.lock`, `.prettierrc`, `.prettierignore`
|
- JS: `package.json`, `yarn.lock`, `.prettierrc`, `.prettierignore`
|
||||||
- Python: `pyproject.toml`
|
- Python: `pyproject.toml`
|
||||||
|
|
||||||
- Guidance for coding agents lives in one `AGENTS.md` at the repository root. It
|
|
||||||
is never committed under a file or directory named after one agent tool, such
|
|
||||||
as `CLAUDE.md` or `.claude/`, and never split into separate memory files.
|
|
||||||
|
|||||||
@@ -1,336 +1,122 @@
|
|||||||
# Workflow
|
# Workflow
|
||||||
|
|
||||||
- branch (from `main`)
|
* branch (from `main`)
|
||||||
- do the work in Next Step
|
* do the work in Next Step
|
||||||
- move Next Step to the top of Completed Steps
|
* move Next Step to the top of Completed Steps
|
||||||
- move the top item of Future Steps into Next Step
|
* move the top item of Future Steps into Next Step
|
||||||
- commit (`TODO.md` changes in the same commit as the work)
|
* commit (`TODO.md` changes in the same commit as the work)
|
||||||
- merge to `main` if the branch is not protected, otherwise open a PR
|
* merge to `main` if the branch is not protected, otherwise open a PR
|
||||||
- push
|
* push
|
||||||
|
|
||||||
# Status
|
# Status
|
||||||
|
|
||||||
pre-1.0. No git tags. `main` last took `next` on 2026-10-06, through
|
pre-1.0. No git tags. The Docker build runs the format check, the linter
|
||||||
https://git.eeqj.de/sneak/routewatch/pulls/6. The linter and the tests are
|
and the tests, and the Gitea workflow runs that build on every push. The
|
||||||
phases of the Docker build, and the Gitea workflow runs `script/cibuild` (the
|
image sets memory ceilings for a 5 GiB container (README "Memory") and the
|
||||||
format check of the Go code and the markdown, both phases, then the image) on
|
README says how to run it under upaas (README "Running under upaas"). A
|
||||||
every push. The image sets memory ceilings for a 5 GiB container (README
|
35-hour run of `3898daa` on the live feed peaked at about 1 GiB, without a
|
||||||
"Memory") and the README says how to run it under upaas (README "Running under
|
container memory limit.
|
||||||
upaas"). A 25-hour run of `a9b6860` on the live feed peaked at about 0.6 GiB
|
|
||||||
(https://git.eeqj.de/sneak/routewatch/issues/3). A run of `825afa2` on the live
|
|
||||||
feed found the prefix handler dropping about a quarter of its updates because
|
|
||||||
its route writes were slower than the feed
|
|
||||||
(https://git.eeqj.de/sneak/routewatch/issues/96); the cheaper writes since then
|
|
||||||
are measured by a benchmark, not yet on the live feed.
|
|
||||||
|
|
||||||
# Next Step
|
# Next Step
|
||||||
|
|
||||||
Run `next` on the live feed, as README "Running under upaas" describes, until
|
`next` waits for sneak to merge it to `main` through
|
||||||
the routing table stops growing. Post its size and memory figures on
|
https://git.eeqj.de/sneak/routewatch/pulls/6. After that, setting
|
||||||
https://git.eeqj.de/sneak/routewatch/issues/122. The same run makes the final
|
routewatch up under upaas on fsn1app1 and deploying it are his
|
||||||
check https://git.eeqj.de/sneak/routewatch/issues/96 names: whether the prefix
|
(https://git.eeqj.de/sneak/routewatch/issues/31), and so is the run under a
|
||||||
handler still drops updates after the first hour. If it does, a new issue
|
real 5 GiB limit (https://git.eeqj.de/sneak/routewatch/issues/3).
|
||||||
follows with the numbers. Then an audit of the code at the commit that run
|
|
||||||
tested.
|
|
||||||
|
|
||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
- 2026-10-27: the tests of `internal/routewatch` no longer fail on a loaded
|
- 2026-10-03: `/api/v1/stats` serves the prefix distribution from memory,
|
||||||
host. Five of them gave `HandleMessage` two seconds to apply a full prefix
|
seeded at startup and adjusted on every live-route write, so a request no
|
||||||
batch, which under the race detector can take longer. A test now waits up to
|
longer reads every live route (closes #30)
|
||||||
15 seconds for each step it expects. The tests of the handlers' limits give
|
- 2026-10-03: a new database is created with `auto_vacuum` set to
|
||||||
the handlers small ones, so the package no longer runs past its 90-second
|
incremental, through the connection string so it is set before the file
|
||||||
limit (closes #213, #215)
|
is first written, and each periodic incremental vacuum now returns up to
|
||||||
- 2026-10-27: the server works out the home page's cards every minute until it
|
1000 free pages; the late `PRAGMA auto_vacuum` in `Initialize`, which
|
||||||
stops. It used to work them out only once, at startup: the end of the context
|
SQLite ignored, is gone (closes #43)
|
||||||
that `Start` is given stopped the refreshes (closes #211)
|
- 2026-10-03: looking up an IP address no longer reads every IPv6 route:
|
||||||
- 2026-10-27: README "Rationale" says why RouteWatch exists: to answer our other
|
both families find the most specific live route with at most 33 or 129
|
||||||
services' questions about IP addresses, from its own routing table and the
|
lookups on the prefix index, and the IPv4 range columns are gone. Prefixes
|
||||||
WHOIS details it keeps. It still says what RIS Live does not give (closes
|
from the feed are stored in one text form, so an IPv6 withdrawal, which the
|
||||||
#204)
|
feed sends uncompressed, now removes its route (closes #48)
|
||||||
- 2026-10-27: every JSON answer carries `@meta`. A panic on a JSON route answers
|
- 2026-10-02: a plain `docker build .` stamps the commit's tag or short
|
||||||
500 with the JSON error body. `/status.json` and the health check answer 405
|
commit (`git describe --tags --always`) into the page footer instead of
|
||||||
with the JSON error body to any method they do not take. The 500 and the 405
|
`unknown`: `.dockerignore` sends `.git` without `.git/config`, a `VERSION`
|
||||||
used to have an empty body. The home page's IP lookup shows the whole JSON
|
build argument takes precedence, and `make build` stamps the same value
|
||||||
answer, `@meta` included. It used to show only the answer's `data`, or only
|
(closes #46)
|
||||||
the message of an error answer. A test checks `@meta` on every JSON route, in
|
|
||||||
its successes and its errors (closes #201)
|
|
||||||
- 2026-10-27: every value in a JSON answer that has a unit is a number, with the
|
|
||||||
unit at the end of its field name. `@meta.execution_time` is now
|
|
||||||
`execution_time_ms`. The stats' `uptime`, `connection_duration` and
|
|
||||||
`memory_usage` are now `uptime_seconds`, `connection_duration_seconds` and
|
|
||||||
`memory_usage_bytes`. `whois_stats.current_interval` is now
|
|
||||||
`current_interval_seconds`. They used to be strings such as "12 ms",
|
|
||||||
"1h18m34s" or "1.1 GB". `connection_duration_seconds` is null while the feed
|
|
||||||
is not connected. `API_POLICIES.md` is vendored from `sneak/prompts`, and
|
|
||||||
`REPO_POLICIES.md` re-vendored (closes #199)
|
|
||||||
- 2026-10-27: every result of the IP lookup has the same fields. A field without
|
|
||||||
a value is null. It used to be left out. An address with no live route has
|
|
||||||
every route and AS field set to null, and `ptr` is an empty list when there
|
|
||||||
are no PTR names. New fields give the number of origin ASes of the netblock,
|
|
||||||
in `num_origins`, and which they are, in `origin_asns` (closes #200)
|
|
||||||
- 2026-10-27: with `TRUSTED_PROXIES` unset, the server trusts reverse proxies at
|
|
||||||
the private addresses of RFC 1918. `/ip/`, the home page's `curl` card and the
|
|
||||||
request log then give the client's address from `X-Forwarded-For`. They used
|
|
||||||
to give the proxy's address. A list in `TRUSTED_PROXIES` replaces the default,
|
|
||||||
and `TRUSTED_PROXIES` set but empty trusts no proxy (closes #198)
|
|
||||||
- 2026-10-27: the home page has cards with figures of the routing table: the
|
|
||||||
newest and oldest kept route, the newest and oldest updated AS, the AS with
|
|
||||||
the most prefixes, the ASes with a single prefix, and fifteen well-known
|
|
||||||
networks. Two cards count the bogon announcements and the prefixes with more
|
|
||||||
than one origin AS, and link to the new pages `/bogons` and
|
|
||||||
`/multiple-origins` that list them. A new package, `internal/bogon`, tells
|
|
||||||
which prefixes and AS numbers are reserved. The server works the figures out
|
|
||||||
once a minute (closes #154)
|
|
||||||
- 2026-10-27: the per-peer route tables are gone, with the code that wrote,
|
|
||||||
counted and read them. The prefix handler applies each update to the routing
|
|
||||||
table in memory and writes the table's changes every second. It also stores
|
|
||||||
each new prefix with its first-seen time. The image's memory limits are set
|
|
||||||
for a full routing table of about 1 GB: `GOMEMLIMIT` is 2.5 GiB, the writing
|
|
||||||
connection's page cache 512 MiB, and SQLite's heap limits 768 MiB and 1 GiB
|
|
||||||
(closes #122)
|
|
||||||
- 2026-10-27: the pages, the API and the stats read every route from the routing
|
|
||||||
table in memory. For a prefix with several origin ASes, the IP lookup gives
|
|
||||||
the one most peers carry. Its `last_seen` is the newest kept route's time. The
|
|
||||||
prefix page and API list every peer that carries the prefix, with an AS path,
|
|
||||||
next hop and time only for the kept routes. The stats count the origins whose
|
|
||||||
kept routes were all withdrawn, in `origins_without_kept_route` (part of #122)
|
|
||||||
- 2026-10-27: the prefix, AS and peer handlers make a timed write on every tick
|
|
||||||
of their timers, every 1, 2 and 2 seconds. They used to skip every other tick.
|
|
||||||
A full batch's write restarts the timer, so the next timed write comes one
|
|
||||||
interval after that write began (closes #192)
|
|
||||||
- 2026-10-27: once the stop deadline has passed, the AS, peer and peering
|
|
||||||
handlers start no write either. A full batch or a tick of the timer that
|
|
||||||
waited while a write ran past the deadline could still start one. The flush
|
|
||||||
goroutines of the prefix, AS and peer handlers now run the same code (closes
|
|
||||||
#189)
|
|
||||||
- 2026-10-27: once the stop deadline has passed, the prefix handler starts no
|
|
||||||
flush. A batch that filled while a write ran past the deadline could still be
|
|
||||||
written after it (closes #188)
|
|
||||||
- 2026-10-27: the prefix handler applies each update to the routing table in
|
|
||||||
memory, and writes the table's changes to the database every second. A write
|
|
||||||
takes at most 20,000 changed pairs, and the rest follow in the next writes. A
|
|
||||||
failed write is written again later. Startup reads the table before the
|
|
||||||
streamer connects. The pages still read the per-peer route tables (part of
|
|
||||||
#122)
|
|
||||||
- 2026-10-27: the database keeps a copy of the whole routing table in a new
|
|
||||||
`routes` table, one row per prefix and origin AS. The peers' numbers are in a
|
|
||||||
new `route_peers` table. One function writes the pairs that changed in memory,
|
|
||||||
and another reads the whole table back into memory at startup. Nothing uses
|
|
||||||
them yet (part of #122)
|
|
||||||
- 2026-10-27: a new package, `internal/routetable`, holds the whole routing
|
|
||||||
table in memory. For each prefix and origin AS it keeps the peers that carry
|
|
||||||
it, one bit per peer, and the two newest of their routes. Nothing uses it yet
|
|
||||||
(part of #122)
|
|
||||||
- 2026-10-27: each result of the IP lookup lists the public blocklists the
|
|
||||||
address is on, in `blocklists`. The lists are DShield, Feodo Tracker, two
|
|
||||||
FireHOL lists, Spamhaus DROP and the Tor exit list. RouteWatch downloads each
|
|
||||||
list on its own schedule and answers from copies in memory. The copies are
|
|
||||||
also kept in the state directory's `blocklists` directory. IP lookup answers
|
|
||||||
carry `Cache-Control`. An address with no live route now has a result too,
|
|
||||||
without route fields, so the IP lookup no longer answers 404 (closes #158)
|
|
||||||
- 2026-10-27: the home page has an IP information API card. It shows the `curl`
|
|
||||||
command that looks up the visitor's own address at `/ip/{addr}`, and that
|
|
||||||
command's live answer (closes #156)
|
|
||||||
- 2026-10-27: on the AS page, a click on the heading of the IPv4 or the IPv6
|
|
||||||
prefix list folds or unfolds the list. Both lists start open (closes #155)
|
|
||||||
- 2026-10-27: the AS page lists its IPv4 and its IPv6 prefixes by mask length,
|
|
||||||
shortest first, and prefixes of one mask length by address. They used to be
|
|
||||||
listed by address (closes #153)
|
|
||||||
- 2026-10-27: on `/` and `/status`, a spinner beside the status turns one frame
|
|
||||||
each time the stats show more messages than the time before. A page that has
|
|
||||||
not seen more messages for three refreshes shows "stale" and the time since
|
|
||||||
then instead, in red (closes #152)
|
|
||||||
- 2026-10-27: the WHOIS details of a Brazilian AS keep their accented letters.
|
|
||||||
LACNIC's answer for such an AS comes from NIC.br, Brazil's registry, with each
|
|
||||||
accented letter replaced. The details now come from NIC.br's own server,
|
|
||||||
`whois.registro.br`. When that query fails, the details from LACNIC's answer
|
|
||||||
are kept and a warning is logged. After NIC.br answers that RouteWatch is over
|
|
||||||
its query limit, RouteWatch stops asking NIC.br for an hour (closes #174)
|
|
||||||
- 2026-10-27: `make asupdate` stops with an error when the download of `as.csv`
|
|
||||||
breaks off. The generator used to retry the failed read forever. A record that
|
|
||||||
does not parse is still logged and skipped (closes #177)
|
|
||||||
- 2026-10-27: `make asupdate` runs the new `script/asupdate`. It replaces the AS
|
|
||||||
list built into RouteWatch only when the generator succeeds and finds at least
|
|
||||||
one AS. A failed download used to replace the list with an empty file. The
|
|
||||||
generator reads `as.csv` from `ipverse/as-metadata`, the source's new name
|
|
||||||
(closes #173)
|
|
||||||
- 2026-10-27: the WHOIS details of an AS at RIPE, APNIC or AFRINIC come only
|
|
||||||
from its `aut-num` object and the organisation object it names. RIPE and APNIC
|
|
||||||
start their answer with the `as-block` object of the AS's block. Its dates
|
|
||||||
were stored as the AS's (closes #168)
|
|
||||||
- 2026-10-27: AS names keep their accented letters. The AS list built into
|
|
||||||
RouteWatch is rebuilt from today's `as.csv`, which added a country code
|
|
||||||
column. That file no longer lists AS 0 or the documentation AS numbers. A
|
|
||||||
WHOIS answer that is not valid UTF-8 is decoded as Latin-1, which RIPE and
|
|
||||||
APNIC send, before it is parsed and stored (closes #165)
|
|
||||||
- 2026-10-27: the Age column on the AS and prefix pages reads in words, such as
|
|
||||||
"5 minutes ago" or "2 weeks ago", instead of Go durations such as "360h0m0s
|
|
||||||
ago". An age under a minute reads in seconds, and one over a month in months
|
|
||||||
or years instead of a date (closes #163)
|
|
||||||
- 2026-10-26: a write to a stored AS, peering or peer keeps the earlier of the
|
|
||||||
two first-seen times and the later of the two last-seen times. An old update
|
|
||||||
from a collector catching up on a backlog no longer moves last seen back
|
|
||||||
(closes #162)
|
|
||||||
- 2026-10-26: the status page shows "Route Churn/min", the number of prefix
|
|
||||||
announcements and withdrawals, from all peers, in the last full clock minute.
|
|
||||||
`/api/v1/stats` reports it as `route_churn_per_min` in `stream`, which
|
|
||||||
replaces the moving average `route_churn_per_sec`. The messages and updates
|
|
||||||
per second on `/` and `/status` are whole numbers (closes #160)
|
|
||||||
- 2026-10-26: the WHOIS fetcher backs off more slowly. A failure makes its wait
|
|
||||||
1.5 times as long, up to one minute instead of five. Its queries to RIPE,
|
|
||||||
APNIC and AFRINIC leave out person and role objects, which RIPE counts against
|
|
||||||
a daily limit per address. A refused query is logged with the RIR's error line
|
|
||||||
(closes #159)
|
|
||||||
- 2026-10-26: the home page's AS name search lists the ASes whose handle or
|
|
||||||
description holds the text, as links to their pages, from the new
|
|
||||||
`/api/v1/search/as`; the search ignores case. When the database has no record
|
|
||||||
of an AS, its page takes the handle and description from the AS list built
|
|
||||||
into RouteWatch and still lists the AS's prefixes (closes #151)
|
|
||||||
- 2026-10-19: README "HTTP Endpoints" lists every route with what it answers,
|
|
||||||
"Database Schema" matches `schema.sql`, and the rest of the README matches the
|
|
||||||
code; this file brought up to date (closes #72)
|
|
||||||
- 2026-10-19: route writes are cheaper: the live route and prefix tables lose
|
|
||||||
their random UUID keys and the prefix tables their duplicate index, a route
|
|
||||||
update sets only the columns that change, and every write goes through one
|
|
||||||
connection with a page cache of 1 GiB (README "Memory"). A benchmark writes
|
|
||||||
the prefix handler's largest batch faster than the feed sends it (closes #96)
|
|
||||||
- 2026-10-19: the WAL checkpoint, the incremental vacuum and ANALYZE take the
|
|
||||||
database lock every batch write holds, so a vacuum no longer fails with
|
|
||||||
"database is locked" and an ANALYZE no longer makes a batch write lose its
|
|
||||||
updates; ANALYZE runs as `PRAGMA optimize` with an analysis limit (closes #94)
|
|
||||||
- 2026-10-19: shutdown stops the streamer first, and each handler writes
|
|
||||||
everything left on its queue before it stops, so the updates waiting there are
|
|
||||||
no longer lost (closes #69)
|
|
||||||
- 2026-10-19: a path that ends in an AS set of several members has origin AS 0,
|
|
||||||
no known origin, instead of the set's last member, and the set adds no
|
|
||||||
peerings; the pages show such a route as having no known origin (closes #75)
|
|
||||||
- 2026-10-19: `bgp_peer_count` drops to 0 when the last peer goes down (closes
|
|
||||||
#92)
|
|
||||||
- 2026-10-19: `bgp_peer_count` counts every peer the feed has heard from, not
|
|
||||||
only those whose session opened since startup, and drops a peer whose session
|
|
||||||
goes down (closes #74)
|
|
||||||
- 2026-10-19: the shared files are re-vendored from `sneak/prompts` at
|
|
||||||
`0653579`: golangci-lint v2.14.0 and its configuration, lint and test as
|
|
||||||
`Dockerfile` phases that keep Go's and golangci-lint's caches, markdown
|
|
||||||
formatted with prettier, agent guidance in `AGENTS.md`; the code is fixed for
|
|
||||||
what the new linter configuration finds (closes #52)
|
|
||||||
- 2026-10-19: the streamer reads feed lines up to 1 MiB, so a line over 64 KiB
|
|
||||||
no longer ends the connection (closes #76)
|
|
||||||
- 2026-10-19: a peer's live routes are removed when its BGP session goes down;
|
|
||||||
`STATE` messages are never dropped for backpressure, and one that finds its
|
|
||||||
queue full waits for room ahead of the handler's other messages (closes #56)
|
|
||||||
- 2026-10-19: the prefix-length pages pick their sample with a bounded number of
|
|
||||||
index lookups instead of reading every route of that length, and list each
|
|
||||||
prefix once (closes #66)
|
|
||||||
- 2026-10-19: a request past its time limit answers 504 instead of 408, and
|
|
||||||
`/api/v1/stats` answers 504 instead of 500 when its read runs out of time
|
|
||||||
(closes #70)
|
|
||||||
- 2026-10-19: the AS, peer and prefix handlers write each batch outside the lock
|
|
||||||
their message handling needs, so messages keep coming off their queues during
|
|
||||||
a write; meanwhile the next batch grows to at most ten batch sizes (closes
|
|
||||||
#65)
|
|
||||||
- 2026-10-19: a live route is identified by prefix and peer, so an announcement
|
|
||||||
with a new origin AS replaces the peer's route and a withdrawal removes it
|
|
||||||
whatever its origin AS (closes #55)
|
|
||||||
- 2026-10-14: the streamer reconnects when RIS Live ends the stream or sends
|
|
||||||
nothing for 60 seconds, and the health check reports the feed disconnected
|
|
||||||
meanwhile (closes #53)
|
|
||||||
- 2026-10-14: the prefix page takes the names of the origin ASes from the prefix
|
|
||||||
query instead of reading every route of each origin AS (closes #67)
|
|
||||||
- 2026-10-14: the daemon exits with status 1 when its web server cannot listen
|
|
||||||
on its port, instead of running on without it (closes #73)
|
|
||||||
- 2026-10-14: the prefix pages parse the prefix before looking it up: one that
|
|
||||||
does not parse answers 400, and any valid way of writing a prefix finds its
|
|
||||||
routes (closes #71)
|
|
||||||
- 2026-10-14: the IP lookup ignores routes shorter than /8 for IPv4 and /16 for
|
|
||||||
IPv6, so a default route is no longer named as an address's owner (closes #57)
|
|
||||||
- 2026-10-14: a prefix batch keeps each peer's last update for a prefix, instead
|
|
||||||
of one update per prefix across all peers (closes #54)
|
|
||||||
- 2026-10-14: `reconnect_count` in `/api/v1/stats` counts every reconnect
|
|
||||||
(closes #68)
|
|
||||||
- 2026-10-03: `/api/v1/stats` serves the prefix distribution from memory, seeded
|
|
||||||
at startup and adjusted on every live-route write, so a request no longer
|
|
||||||
reads every live route (closes #30)
|
|
||||||
- 2026-10-03: a new database is created with `auto_vacuum` set to incremental,
|
|
||||||
through the connection string so it is set before the file is first written,
|
|
||||||
and each periodic incremental vacuum now returns up to 1000 free pages; the
|
|
||||||
late `PRAGMA auto_vacuum` in `Initialize`, which SQLite ignored, is gone
|
|
||||||
(closes #43)
|
|
||||||
- 2026-10-03: looking up an IP address no longer reads every IPv6 route: both
|
|
||||||
families find the most specific live route with at most 33 or 129 lookups on
|
|
||||||
the prefix index, and the IPv4 range columns are gone. Prefixes from the feed
|
|
||||||
are stored in one text form, so an IPv6 withdrawal, which the feed sends
|
|
||||||
uncompressed, now removes its route (closes #48)
|
|
||||||
- 2026-10-02: a plain `docker build .` stamps the commit's tag or short commit
|
|
||||||
(`git describe --tags --always`) into the page footer instead of `unknown`:
|
|
||||||
`.dockerignore` sends `.git` without `.git/config`, a `VERSION` build argument
|
|
||||||
takes precedence, and `make build` stamps the same value (closes #46)
|
|
||||||
- 2026-09-29: the entrypoint creates the data directory if it is missing and
|
- 2026-09-29: the entrypoint creates the data directory if it is missing and
|
||||||
stops the start if a step fails; README "Running under upaas" no longer asks
|
stops the start if a step fails; README "Running under upaas" no longer
|
||||||
for the host directory to be created first (closes #42)
|
asks for the host directory to be created first (closes #42)
|
||||||
- 2026-09-29: `.dockerignore` keeps `.git`, local build output, local databases
|
- 2026-09-29: `.dockerignore` keeps `.git`, local build output, local
|
||||||
and `.env` out of the Docker build context, and so out of the source archive
|
databases and `.env` out of the Docker build context, and so out of the
|
||||||
in the image (closes #39)
|
source archive in the image (closes #39)
|
||||||
- 2026-09-29: README first line names the MIT license and the author; the
|
- 2026-09-29: README first line names the MIT license and the author;
|
||||||
License section now says MIT and links `LICENSE`, and an Author section was
|
the License section now says MIT and links `LICENSE`, and an Author
|
||||||
added; this file brought up to date (closes #38)
|
section was added; this file brought up to date (closes #38)
|
||||||
- 2026-09-29: MIT `LICENSE` (closes #1)
|
- 2026-09-29: MIT `LICENSE` (closes #1)
|
||||||
- 2026-09-28: stopping the daemon while the feed is flowing no longer panics
|
- 2026-09-28: stopping the daemon while the feed is flowing no longer
|
||||||
with "send on closed channel": the read loop checks for a stop just before
|
panics with "send on closed channel": the read loop checks for a stop
|
||||||
handing a message to the handler queues, and a second `Stop` no longer closes
|
just before handing a message to the handler queues, and a second
|
||||||
the queues again (closes #34)
|
`Stop` no longer closes the queues again (closes #34)
|
||||||
- 2026-09-28: `docker stop` no longer kills the daemon 2 seconds after the stop
|
- 2026-09-28: `docker stop` no longer kills the daemon 2 seconds after the
|
||||||
signal: the entrypoint switches to the `routewatch` user with `setpriv`
|
stop signal: the entrypoint switches to the `routewatch` user with
|
||||||
instead of `runuser`, so the daemon receives the signal itself and gets the
|
`setpriv` instead of `runuser`, so the daemon receives the signal itself
|
||||||
whole wait `docker stop` allows, up to its own 60-second limit (closes #33)
|
and gets the whole wait `docker stop` allows, up to its own 60-second
|
||||||
|
limit (closes #33)
|
||||||
- 2026-09-28: ready to run under upaas: a set but invalid `PORT`,
|
- 2026-09-28: ready to run under upaas: a set but invalid `PORT`,
|
||||||
`XDG_DATA_HOME` or `MALLOC_ARENA_MAX` stops the start, the health check
|
`XDG_DATA_HOME` or `MALLOC_ARENA_MAX` stops the start, the health
|
||||||
follows `PORT`, README "Running under upaas" section (closes #31)
|
check follows `PORT`, README "Running under upaas" section (closes
|
||||||
- 2026-09-22: realtime in-memory database statistics: counts seeded at startup
|
#31)
|
||||||
and adjusted on every write, oldest/newest route timestamps via index-end
|
- 2026-09-22: realtime in-memory database statistics: counts seeded at
|
||||||
lookups; `/api/v1/stats` no longer scans the tables (closes #27)
|
startup and adjusted on every write, oldest/newest route timestamps via
|
||||||
- 2026-09-21: batch writes take the write lock when their transaction begins
|
index-end lookups; `/api/v1/stats` no longer scans the tables (closes
|
||||||
(`_txlock=immediate`), so they wait out a WAL checkpoint instead of failing
|
#27)
|
||||||
with "database is locked" (closes #25)
|
- 2026-09-21: batch writes take the write lock when their transaction
|
||||||
- 2026-09-21: `MALLOC_ARENA_MAX=2` in the image caps glibc malloc arenas, so
|
begins (`_txlock=immediate`), so they wait out a WAL checkpoint instead
|
||||||
memory outside the Go runtime no longer grows with the core count (closes #23)
|
of failing with "database is locked" (closes #25)
|
||||||
- 2026-09-21: `GOMEMLIMIT=1536MiB` in the image; README Memory section with the
|
- 2026-09-21: `MALLOC_ARENA_MAX=2` in the image caps glibc malloc arenas,
|
||||||
memory budget and the 5 GiB container limit (closes #13)
|
so memory outside the Go runtime no longer grows with the core count
|
||||||
- 2026-09-21: the four handler queues hold at most 20,000 messages each, down
|
(closes #23)
|
||||||
from 100,000 (closes #11)
|
- 2026-09-21: `GOMEMLIMIT=1536MiB` in the image; README Memory section
|
||||||
- 2026-09-21: two goroutine leaks fixed: the stats handlers after a timeout and
|
with the memory budget and the 5 GiB container limit (closes #13)
|
||||||
the streamer's tickers on every reconnect (closes #12)
|
- 2026-09-21: the four handler queues hold at most 20,000 messages each,
|
||||||
- 2026-09-21: parsed RIS messages no longer keep the unused `Community` and
|
down from 100,000 (closes #11)
|
||||||
`Raw` fields (closes #9)
|
- 2026-09-21: two goroutine leaks fixed: the stats handlers after a
|
||||||
- 2026-09-21: `.editorconfig`, and a Gitea workflow that runs `script/cibuild`
|
timeout and the streamer's tickers on every reconnect (closes #12)
|
||||||
on every push (closes #14)
|
- 2026-09-21: parsed RIS messages no longer keep the unused `Community`
|
||||||
- 2026-09-21: the peering handler's AS-path map holds at most 500,000 paths and
|
and `Raw` fields (closes #9)
|
||||||
is swapped for an empty one every 30 seconds instead of copied (closes #10)
|
- 2026-09-21: `.editorconfig`, and a Gitea workflow that runs
|
||||||
- 2026-09-21: SQLite memory bounded across the whole connection pool: a 64 MiB
|
`script/cibuild` on every push (closes #14)
|
||||||
page cache on each connection, 1 GiB soft and 1.5 GiB hard heap limits (closes
|
- 2026-09-21: the peering handler's AS-path map holds at most 500,000
|
||||||
#8)
|
paths and is swapped for an empty one every 30 seconds instead of copied
|
||||||
- 2026-09-21: the Docker build runs the format check, the linter and the tests,
|
(closes #10)
|
||||||
linting in a separate stage on a golangci-lint image pinned by digest (closes
|
- 2026-09-21: SQLite memory bounded across the whole connection pool: a
|
||||||
#5)
|
64 MiB page cache on each connection, 1 GiB soft and 1.5 GiB hard heap
|
||||||
|
limits (closes #8)
|
||||||
|
- 2026-09-21: the Docker build runs the format check, the linter and the
|
||||||
|
tests, linting in a separate stage on a golangci-lint image pinned by
|
||||||
|
digest (closes #5)
|
||||||
- 2026-09-21: `make test` skips the live-network feed test (`-short`), so
|
- 2026-09-21: `make test` skips the live-network feed test (`-short`), so
|
||||||
`make check` no longer depends on the network (closes #2)
|
`make check` no longer depends on the network (closes #2)
|
||||||
- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints, Makefile
|
- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints,
|
||||||
shims, README Entrypoints section
|
Makefile shims, README Entrypoints section
|
||||||
- 2026-02-22: repo policy compliance: required policy files, .gitignore update,
|
- 2026-02-22: repo policy compliance: required policy files, .gitignore
|
||||||
Makefile fmt-check/check/docker/hooks targets, gofmt pass
|
update, Makefile fmt-check/check/docker/hooks targets, gofmt pass
|
||||||
(repo-policies-compliance, unmerged)
|
(repo-policies-compliance, unmerged)
|
||||||
- 2026-01-01: WAL checkpointing: periodic (5s), on startup with logging,
|
- 2026-01-01: WAL checkpointing: periodic (5s), on startup with logging,
|
||||||
TRUNCATE mode; fixed slow queries
|
TRUNCATE mode; fixed slow queries
|
||||||
- 2025-12-31: status page: navbar, home page with search, oldest/newest route
|
- 2025-12-31: status page: navbar, home page with search, oldest/newest
|
||||||
timestamps, NULL handling in WHOIS stats; container runs as routewatch user
|
route timestamps, NULL handling in WHOIS stats; container runs as
|
||||||
with proper state dir
|
routewatch user with proper state dir
|
||||||
- 2025-12-30: structured HTTP request logging, increased timeouts, CIDR prefix
|
- 2025-12-30: structured HTTP request logging, increased timeouts,
|
||||||
URL routing fixes, status page metrics and footer
|
CIDR prefix URL routing fixes, status page metrics and footer
|
||||||
- 2025-12-29: Dockerfile multi-stage build with source archive; SQLite
|
- 2025-12-29: Dockerfile multi-stage build with source archive; SQLite
|
||||||
incremental vacuum for non-blocking space reclamation
|
incremental vacuum for non-blocking space reclamation
|
||||||
- 2025-07-27: initial RouteWatch BGP stream monitor
|
- 2025-07-27: initial RouteWatch BGP stream monitor
|
||||||
|
|
||||||
# Future Steps
|
# Future Steps
|
||||||
|
|
||||||
None.
|
- Production memory under 5 GiB: whether to test under a real 5 GiB
|
||||||
|
container limit on fsn1app1 is open for sneak
|
||||||
|
(https://git.eeqj.de/sneak/routewatch/issues/3)
|
||||||
|
|||||||
+26
-73
@@ -2,11 +2,8 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"encoding/csv"
|
"encoding/csv"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
"io"
|
||||||
"log"
|
"log"
|
||||||
"net/http"
|
"net/http"
|
||||||
@@ -15,21 +12,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
const asCsvURL = "https://github.com/ipverse/as-metadata/raw/refs/heads/master/as.csv"
|
const asCsvURL = "https://github.com/ipverse/asn-info/raw/refs/heads/master/as.csv"
|
||||||
|
|
||||||
// The first columns of as.csv, the ones the generator reads. It reads past
|
|
||||||
// the columns after them, such as the country code.
|
|
||||||
const (
|
|
||||||
asnColumn = "asn"
|
|
||||||
handleColumn = "handle"
|
|
||||||
descriptionColumn = "description"
|
|
||||||
columnsRead = 3
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
errUnexpectedHeader = errors.New("unexpected CSV header")
|
|
||||||
errNoASes = errors.New("no AS in the CSV")
|
|
||||||
)
|
|
||||||
|
|
||||||
// ASInfo represents information about an Autonomous System
|
// ASInfo represents information about an Autonomous System
|
||||||
type ASInfo struct {
|
type ASInfo struct {
|
||||||
@@ -44,82 +27,49 @@ func main() {
|
|||||||
|
|
||||||
// Fetch the CSV data
|
// Fetch the CSV data
|
||||||
log.Println("Fetching AS CSV data...")
|
log.Println("Fetching AS CSV data...")
|
||||||
|
resp, err := http.Get(asCsvURL)
|
||||||
req, err := http.NewRequestWithContext(
|
|
||||||
context.Background(), http.MethodGet, asCsvURL, nil)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("Failed to fetch CSV: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, err := http.DefaultClient.Do(req)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("Failed to fetch CSV: %v", err)
|
log.Fatalf("Failed to fetch CSV: %v", err)
|
||||||
}
|
}
|
||||||
|
defer func() {
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
}()
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
if resp.StatusCode != http.StatusOK {
|
||||||
log.Fatalf("Failed to fetch CSV: HTTP %d", resp.StatusCode)
|
log.Fatalf("Failed to fetch CSV: HTTP %d", resp.StatusCode)
|
||||||
}
|
}
|
||||||
|
|
||||||
asInfos, err := parseCSV(resp.Body)
|
// Parse CSV
|
||||||
|
reader := csv.NewReader(resp.Body)
|
||||||
// Closed here rather than deferred: log.Fatalf exits without running
|
|
||||||
// deferred calls.
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("Failed to parse CSV: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Printf("Parsed %d AS records", len(asInfos))
|
|
||||||
|
|
||||||
// Convert to JSON and output to stdout
|
|
||||||
encoder := json.NewEncoder(os.Stdout)
|
|
||||||
encoder.SetIndent("", "\t")
|
|
||||||
|
|
||||||
err = encoder.Encode(asInfos)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("Failed to encode JSON: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseCSV reads as.csv from r and returns its ASes, with the handle and the
|
|
||||||
// description as the file spells them. A header whose first columns are not
|
|
||||||
// the AS number, the handle and the description is an error. A record that
|
|
||||||
// does not parse is logged and skipped. A read that fails, such as on a broken
|
|
||||||
// download, is an error. A file left with no AS is an error.
|
|
||||||
func parseCSV(r io.Reader) ([]ASInfo, error) {
|
|
||||||
reader := csv.NewReader(r)
|
|
||||||
|
|
||||||
|
// Read header
|
||||||
header, err := reader.Read()
|
header, err := reader.Read()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("read CSV header: %w", err)
|
log.Fatalf("Failed to read CSV header: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(header) < columnsRead || header[0] != asnColumn ||
|
if len(header) != 3 || header[0] != "asn" || header[1] != "handle" || header[2] != "description" {
|
||||||
header[1] != handleColumn || header[2] != descriptionColumn {
|
log.Fatalf("Unexpected CSV header: %v", header)
|
||||||
return nil, fmt.Errorf("%w: %v", errUnexpectedHeader, header)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Read all records
|
||||||
var asInfos []ASInfo
|
var asInfos []ASInfo
|
||||||
|
|
||||||
for {
|
for {
|
||||||
// The reader returns an error for a record with a different number
|
|
||||||
// of fields than the header.
|
|
||||||
record, err := reader.Read()
|
record, err := reader.Read()
|
||||||
if errors.Is(err, io.EOF) {
|
if err == io.EOF {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
if err != nil {
|
||||||
if _, ok := errors.AsType[*csv.ParseError](err); ok {
|
|
||||||
log.Printf("Error reading CSV record: %v", err)
|
log.Printf("Error reading CSV record: %v", err)
|
||||||
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
// Any other error comes from the download, such as a broken
|
const expectedFields = 3
|
||||||
// connection. Every later read returns it again.
|
if len(record) != expectedFields {
|
||||||
if err != nil {
|
log.Printf("Skipping invalid record: %v", record)
|
||||||
return nil, fmt.Errorf("read CSV record: %w", err)
|
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
asn, err := strconv.Atoi(record[0])
|
asn, err := strconv.Atoi(record[0])
|
||||||
@@ -136,9 +86,12 @@ func parseCSV(r io.Reader) ([]ASInfo, error) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(asInfos) == 0 {
|
log.Printf("Parsed %d AS records", len(asInfos))
|
||||||
return nil, errNoASes
|
|
||||||
}
|
|
||||||
|
|
||||||
return asInfos, nil
|
// Convert to JSON and output to stdout
|
||||||
|
encoder := json.NewEncoder(os.Stdout)
|
||||||
|
encoder.SetIndent("", "\t")
|
||||||
|
if err := encoder.Encode(asInfos); err != nil {
|
||||||
|
log.Fatalf("Failed to encode JSON: %v", err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,142 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"io"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"testing/iotest"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// parseTimeLimit is how long a test waits for parseCSV to return before it
|
|
||||||
// fails.
|
|
||||||
const parseTimeLimit = 2 * time.Second
|
|
||||||
|
|
||||||
// TestParseCSVReadsTodaysLayout parses an as.csv with the country code column
|
|
||||||
// that the source added after the description. It expects each AS with its
|
|
||||||
// handle and description spelled exactly as in the file, accents included.
|
|
||||||
func TestParseCSVReadsTodaysLayout(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const asCSV = `asn,handle,description,country-code
|
|
||||||
1,LVLT-1,Level 3 Parent LLC,US
|
|
||||||
1251,FUNDACAO-AMPARO-PESQUISA,FUNDAÇÃO DE AMPARO À PESQUISA DO ESTADO SÃO PAULO,BR
|
|
||||||
`
|
|
||||||
|
|
||||||
got, err := parseCSV(strings.NewReader(asCSV))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("parseCSV: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
want := []ASInfo{
|
|
||||||
{ASN: 1, Handle: "LVLT-1", Description: "Level 3 Parent LLC"},
|
|
||||||
{
|
|
||||||
ASN: 1251,
|
|
||||||
Handle: "FUNDACAO-AMPARO-PESQUISA",
|
|
||||||
Description: "FUNDAÇÃO DE AMPARO À PESQUISA DO ESTADO " +
|
|
||||||
"SÃO PAULO",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
if !slices.Equal(got, want) {
|
|
||||||
t.Errorf("parseCSV = %+v, want %+v", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestParseCSVRejectsAnotherHeader parses files whose first columns are not
|
|
||||||
// the AS number, the handle and the description. It expects an error rather
|
|
||||||
// than a list read from the wrong columns.
|
|
||||||
func TestParseCSVRejectsAnotherHeader(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, asCSV := range []string{
|
|
||||||
"asn,handle\n1,LVLT-1\n",
|
|
||||||
"asn,description,handle\n1,Level 3 Parent LLC,LVLT-1\n",
|
|
||||||
} {
|
|
||||||
_, err := parseCSV(strings.NewReader(asCSV))
|
|
||||||
if err == nil {
|
|
||||||
t.Errorf("parseCSV(%q): no error", asCSV)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestParseCSVRejectsAFileWithNoASes parses files that hold no AS, one with
|
|
||||||
// only the header and one whose only record has no valid AS number. It
|
|
||||||
// expects errNoASes, so that make asupdate keeps the old list.
|
|
||||||
func TestParseCSVRejectsAFileWithNoASes(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, asCSV := range []string{
|
|
||||||
"asn,handle,description,country-code\n",
|
|
||||||
"asn,handle,description,country-code\nAS1,LVLT-1,Level 3,US\n",
|
|
||||||
} {
|
|
||||||
_, err := parseCSV(strings.NewReader(asCSV))
|
|
||||||
if !errors.Is(err, errNoASes) {
|
|
||||||
t.Errorf("parseCSV(%q) = %v, want %v", asCSV, err, errNoASes)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestParseCSVSkipsARecordThatDoesNotParse parses an as.csv with a record
|
|
||||||
// that has too few fields between two valid records. It expects both valid
|
|
||||||
// ASes and no error, so that one bad record does not stop make asupdate.
|
|
||||||
func TestParseCSVSkipsARecordThatDoesNotParse(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const asCSV = "asn,handle,description,country-code\n" +
|
|
||||||
"1,LVLT-1,Level 3 Parent LLC,US\n" +
|
|
||||||
"2,UDEL-DCN\n" +
|
|
||||||
"3,MIT-GATEWAYS,Massachusetts Institute of Technology,US\n"
|
|
||||||
|
|
||||||
got, err := parseCSV(strings.NewReader(asCSV))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("parseCSV: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
want := []ASInfo{
|
|
||||||
{ASN: 1, Handle: "LVLT-1", Description: "Level 3 Parent LLC"},
|
|
||||||
{
|
|
||||||
ASN: 3,
|
|
||||||
Handle: "MIT-GATEWAYS",
|
|
||||||
Description: "Massachusetts Institute of Technology",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
if !slices.Equal(got, want) {
|
|
||||||
t.Errorf("parseCSV = %+v, want %+v", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestParseCSVStopsWhenTheDownloadBreaksOff parses an as.csv whose download
|
|
||||||
// breaks off after the first record. From then on every read fails, as on a
|
|
||||||
// broken connection. It expects parseCSV to return that error, so that make
|
|
||||||
// asupdate keeps the old list.
|
|
||||||
func TestParseCSVStopsWhenTheDownloadBreaksOff(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const asCSV = "asn,handle,description,country-code\n" +
|
|
||||||
"1,LVLT-1,Level 3 Parent LLC,US\n"
|
|
||||||
|
|
||||||
download := io.MultiReader(
|
|
||||||
strings.NewReader(asCSV), iotest.ErrReader(io.ErrUnexpectedEOF))
|
|
||||||
|
|
||||||
// parseCSV runs on its own goroutine, so that a parseCSV that never
|
|
||||||
// returns fails the test instead of hanging it.
|
|
||||||
parsed := make(chan error, 1)
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
_, err := parseCSV(download)
|
|
||||||
parsed <- err
|
|
||||||
}()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case err := <-parsed:
|
|
||||||
if !errors.Is(err, io.ErrUnexpectedEOF) {
|
|
||||||
t.Errorf("parseCSV = %v, want %v", err, io.ErrUnexpectedEOF)
|
|
||||||
}
|
|
||||||
case <-time.After(parseTimeLimit):
|
|
||||||
t.Fatalf("parseCSV did not return within %v", parseTimeLimit)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -26,17 +26,9 @@ func main() {
|
|||||||
|
|
||||||
// Register raw message handler that prints to stdout
|
// Register raw message handler that prints to stdout
|
||||||
s.RegisterRawHandler(func(line string) {
|
s.RegisterRawHandler(func(line string) {
|
||||||
_, _ = fmt.Fprintln(os.Stdout, line)
|
fmt.Println(line)
|
||||||
})
|
})
|
||||||
|
|
||||||
// Start streaming before any defer: log.Fatal exits without running
|
|
||||||
// deferred calls.
|
|
||||||
err := s.Start(context.Background())
|
|
||||||
if err != nil {
|
|
||||||
log.Fatal("Failed to start streamer:", err)
|
|
||||||
}
|
|
||||||
defer s.Stop(context.Background())
|
|
||||||
|
|
||||||
// Set up context with cancellation
|
// Set up context with cancellation
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
defer cancel()
|
defer cancel()
|
||||||
@@ -44,13 +36,18 @@ func main() {
|
|||||||
// Handle signals
|
// Handle signals
|
||||||
sigChan := make(chan os.Signal, 1)
|
sigChan := make(chan os.Signal, 1)
|
||||||
signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
|
signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
<-sigChan
|
<-sigChan
|
||||||
log.Println("Received shutdown signal")
|
log.Println("Received shutdown signal")
|
||||||
cancel()
|
cancel()
|
||||||
}()
|
}()
|
||||||
|
|
||||||
|
// Start streaming
|
||||||
|
if err := s.Start(); err != nil {
|
||||||
|
log.Fatal("Failed to start streamer:", err)
|
||||||
|
}
|
||||||
|
defer s.Stop()
|
||||||
|
|
||||||
// Wait for context cancellation
|
// Wait for context cancellation
|
||||||
<-ctx.Done()
|
<-ctx.Done()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,19 +1,15 @@
|
|||||||
module git.eeqj.de/sneak/routewatch
|
module git.eeqj.de/sneak/routewatch
|
||||||
|
|
||||||
go 1.27.1
|
go 1.24.4
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/bits-and-blooms/bitset v1.26.0
|
|
||||||
github.com/dustin/go-humanize v1.0.1
|
github.com/dustin/go-humanize v1.0.1
|
||||||
github.com/go-chi/chi/v5 v5.2.2
|
github.com/go-chi/chi/v5 v5.2.2
|
||||||
github.com/google/btree v1.1.3
|
|
||||||
github.com/google/uuid v1.6.0
|
github.com/google/uuid v1.6.0
|
||||||
github.com/mattn/go-sqlite3 v1.14.29
|
github.com/mattn/go-sqlite3 v1.14.29
|
||||||
github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9
|
github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9
|
||||||
go.uber.org/fx v1.24.0
|
go.uber.org/fx v1.24.0
|
||||||
go4.org/netipx v0.0.0-20260823151212-3075585bcbeb
|
|
||||||
golang.org/x/term v0.33.0
|
golang.org/x/term v0.33.0
|
||||||
golang.org/x/text v0.43.0
|
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
|||||||
@@ -1,13 +1,9 @@
|
|||||||
github.com/bits-and-blooms/bitset v1.26.0 h1:WPvcPZbP1yuKofmjs2SYIFO1EjzIGhCP9ducElLsh5Q=
|
|
||||||
github.com/bits-and-blooms/bitset v1.26.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
|
|
||||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||||
github.com/go-chi/chi/v5 v5.2.2 h1:CMwsvRVTbXVytCk1Wd72Zy1LAsAh9GxMmSNWLHCG618=
|
github.com/go-chi/chi/v5 v5.2.2 h1:CMwsvRVTbXVytCk1Wd72Zy1LAsAh9GxMmSNWLHCG618=
|
||||||
github.com/go-chi/chi/v5 v5.2.2/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops=
|
github.com/go-chi/chi/v5 v5.2.2/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops=
|
||||||
github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg=
|
|
||||||
github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
|
|
||||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||||
github.com/mattn/go-sqlite3 v1.14.29 h1:1O6nRLJKvsi1H2Sj0Hzdfojwt8GiGKm+LOfLaBFaouQ=
|
github.com/mattn/go-sqlite3 v1.14.29 h1:1O6nRLJKvsi1H2Sj0Hzdfojwt8GiGKm+LOfLaBFaouQ=
|
||||||
@@ -28,13 +24,9 @@ go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
|
|||||||
go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||||
go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
|
go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
|
||||||
go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
|
go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
|
||||||
go4.org/netipx v0.0.0-20260823151212-3075585bcbeb h1:XBM4hvfwGAttkkiTIFfeigdfcL1xIfdKXqFdgiHGtDs=
|
|
||||||
go4.org/netipx v0.0.0-20260823151212-3075585bcbeb/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y=
|
|
||||||
golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
|
golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
|
||||||
golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
||||||
golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg=
|
golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg=
|
||||||
golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0=
|
golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0=
|
||||||
golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc=
|
|
||||||
golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s=
|
|
||||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
|
|||||||
@@ -1,184 +0,0 @@
|
|||||||
// Package blocklist downloads public IP blocklists on a schedule, keeps the
|
|
||||||
// last good copy of each on disk and in memory, and answers which lists
|
|
||||||
// contain an address.
|
|
||||||
package blocklist
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"log/slog"
|
|
||||||
"net/netip"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"go4.org/netipx"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Format is how a list's file is written.
|
|
||||||
type Format int
|
|
||||||
|
|
||||||
const (
|
|
||||||
// FormatLines has one address or network per line, such as 192.0.2.1 or
|
|
||||||
// 192.0.2.0/24. Lines that start with # are comments.
|
|
||||||
FormatLines Format = iota
|
|
||||||
|
|
||||||
// FormatSpamhausJSON has one JSON object per line, with a network in its
|
|
||||||
// cidr field. The last line holds the file's metadata and no network.
|
|
||||||
FormatSpamhausJSON
|
|
||||||
|
|
||||||
// FormatDShield has one network per line, as tab-separated fields that
|
|
||||||
// start with the network's first and last address. Lines that start
|
|
||||||
// with # are comments.
|
|
||||||
FormatDShield
|
|
||||||
)
|
|
||||||
|
|
||||||
// List is one public blocklist.
|
|
||||||
type List struct {
|
|
||||||
// Name names the list in lookup answers, and names its copy on disk.
|
|
||||||
Name string
|
|
||||||
|
|
||||||
// Description says in one line what being on the list means.
|
|
||||||
Description string
|
|
||||||
|
|
||||||
// URL is where the publisher serves the whole list.
|
|
||||||
URL string
|
|
||||||
|
|
||||||
// Interval is how long a copy is used before the list is downloaded
|
|
||||||
// again.
|
|
||||||
Interval time.Duration
|
|
||||||
|
|
||||||
// Format is how the list's file is written.
|
|
||||||
Format Format
|
|
||||||
}
|
|
||||||
|
|
||||||
// Listing says that a list contains an address.
|
|
||||||
type Listing struct {
|
|
||||||
Name string `json:"name"`
|
|
||||||
Description string `json:"description"`
|
|
||||||
|
|
||||||
// Updated is when routewatch last downloaded the list, or last heard from
|
|
||||||
// its publisher that its copy was still current.
|
|
||||||
Updated time.Time `json:"updated"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// listCopy is a list's last good copy in memory.
|
|
||||||
type listCopy struct {
|
|
||||||
addresses *netipx.IPSet
|
|
||||||
updated time.Time
|
|
||||||
|
|
||||||
// validators identify the version of the list this copy holds.
|
|
||||||
validators validators
|
|
||||||
}
|
|
||||||
|
|
||||||
// Params is what New needs.
|
|
||||||
type Params struct {
|
|
||||||
// Dir is the directory that holds the copies on disk.
|
|
||||||
Dir string
|
|
||||||
|
|
||||||
// Lists are the lists to download and look addresses up in.
|
|
||||||
Lists []List
|
|
||||||
|
|
||||||
Logger *slog.Logger
|
|
||||||
}
|
|
||||||
|
|
||||||
// Blocklists keeps a copy of each list and answers which lists contain an
|
|
||||||
// address. Start loads the copies from disk and starts downloading the lists.
|
|
||||||
type Blocklists struct {
|
|
||||||
dir string
|
|
||||||
lists []List
|
|
||||||
logger *slog.Logger
|
|
||||||
|
|
||||||
// copies holds each list's last good copy, by list name. A list with no
|
|
||||||
// good copy yet has none.
|
|
||||||
mu sync.RWMutex
|
|
||||||
copies map[string]listCopy
|
|
||||||
|
|
||||||
// stop ends the goroutine Start runs, and done is closed when it returns.
|
|
||||||
stop context.CancelFunc
|
|
||||||
done chan struct{}
|
|
||||||
}
|
|
||||||
|
|
||||||
// New returns a Blocklists with no copies. Start loads them.
|
|
||||||
func New(params Params) *Blocklists {
|
|
||||||
return &Blocklists{
|
|
||||||
dir: params.Dir,
|
|
||||||
lists: params.Lists,
|
|
||||||
logger: params.Logger.With("component", "blocklists"),
|
|
||||||
copies: make(map[string]listCopy),
|
|
||||||
done: make(chan struct{}),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Lookup returns a Listing for each list whose copy contains addr, in the
|
|
||||||
// order of the lists given to New. It makes no outside request. When no list
|
|
||||||
// contains addr, it returns an empty list, not nil.
|
|
||||||
func (b *Blocklists) Lookup(addr netip.Addr) []Listing {
|
|
||||||
b.mu.RLock()
|
|
||||||
defer b.mu.RUnlock()
|
|
||||||
|
|
||||||
listings := []Listing{}
|
|
||||||
|
|
||||||
for _, list := range b.lists {
|
|
||||||
c, ok := b.copies[list.Name]
|
|
||||||
if ok && c.addresses.Contains(addr.Unmap()) {
|
|
||||||
listings = append(listings, Listing{
|
|
||||||
Name: list.Name,
|
|
||||||
Description: list.Description,
|
|
||||||
Updated: c.updated,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return listings
|
|
||||||
}
|
|
||||||
|
|
||||||
// ShortestInterval returns the shortest Interval of the lists. That is the
|
|
||||||
// soonest a copy can change, and so how long a lookup answer stays current.
|
|
||||||
func (b *Blocklists) ShortestInterval() time.Duration {
|
|
||||||
var shortest time.Duration
|
|
||||||
|
|
||||||
for i, list := range b.lists {
|
|
||||||
if i == 0 || list.Interval < shortest {
|
|
||||||
shortest = list.Interval
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return shortest
|
|
||||||
}
|
|
||||||
|
|
||||||
// updated returns when the named list's copy was last downloaded or found
|
|
||||||
// current, or the zero time when the list has no copy.
|
|
||||||
func (b *Blocklists) updated(name string) time.Time {
|
|
||||||
b.mu.RLock()
|
|
||||||
defer b.mu.RUnlock()
|
|
||||||
|
|
||||||
return b.copies[name].updated
|
|
||||||
}
|
|
||||||
|
|
||||||
// copyValidators returns the validators of the named list's copy. They are
|
|
||||||
// empty when the list has no copy.
|
|
||||||
func (b *Blocklists) copyValidators(name string) validators {
|
|
||||||
b.mu.RLock()
|
|
||||||
defer b.mu.RUnlock()
|
|
||||||
|
|
||||||
return b.copies[name].validators
|
|
||||||
}
|
|
||||||
|
|
||||||
// setCopy makes c the named list's copy.
|
|
||||||
func (b *Blocklists) setCopy(name string, c listCopy) {
|
|
||||||
b.mu.Lock()
|
|
||||||
defer b.mu.Unlock()
|
|
||||||
|
|
||||||
b.copies[name] = c
|
|
||||||
}
|
|
||||||
|
|
||||||
// setUpdated records that the named list's copy was found current at t.
|
|
||||||
func (b *Blocklists) setUpdated(name string, t time.Time) {
|
|
||||||
b.mu.Lock()
|
|
||||||
defer b.mu.Unlock()
|
|
||||||
|
|
||||||
c, ok := b.copies[name]
|
|
||||||
if ok {
|
|
||||||
c.updated = t
|
|
||||||
b.copies[name] = c
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,494 +0,0 @@
|
|||||||
package blocklist
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"io"
|
|
||||||
"io/fs"
|
|
||||||
"log/slog"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"net/netip"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The test list's name and description, the address its good version lists,
|
|
||||||
// and the address a newer version lists instead.
|
|
||||||
const (
|
|
||||||
testListName = "test_list"
|
|
||||||
testListDescription = "On the test list"
|
|
||||||
listedAddr = "192.0.2.1"
|
|
||||||
goodListBody = "# A test list\n" + listedAddr + "\n"
|
|
||||||
newListedAddr = "192.0.2.2"
|
|
||||||
newListBody = "# A test list\n" + newListedAddr + "\n"
|
|
||||||
)
|
|
||||||
|
|
||||||
// testETag is the ETag a publisher sends with a version of the test list.
|
|
||||||
const testETag = `"v1"`
|
|
||||||
|
|
||||||
// maxWait is how long a test waits for the download goroutine.
|
|
||||||
const maxWait = 10 * time.Second
|
|
||||||
|
|
||||||
// publisher stands in for a list's publisher. It answers every request with
|
|
||||||
// its status and body, and keeps the headers of each request. A 200 answer
|
|
||||||
// carries lastModified and etag, when set, as Last-Modified and ETag. As
|
|
||||||
// from a real publisher, a request whose If-None-Match or If-Modified-Since
|
|
||||||
// matches them is answered "not modified".
|
|
||||||
type publisher struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
status int
|
|
||||||
body string
|
|
||||||
lastModified time.Time
|
|
||||||
etag string
|
|
||||||
headers []http.Header
|
|
||||||
}
|
|
||||||
|
|
||||||
// newPublisher starts a publisher that answers 200 with body, and returns it
|
|
||||||
// and its URL.
|
|
||||||
func newPublisher(t *testing.T, body string) (*publisher, string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
p := &publisher{status: http.StatusOK, body: body}
|
|
||||||
srv := httptest.NewServer(p)
|
|
||||||
t.Cleanup(srv.Close)
|
|
||||||
|
|
||||||
return p, srv.URL
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *publisher) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|
||||||
p.mu.Lock()
|
|
||||||
defer p.mu.Unlock()
|
|
||||||
|
|
||||||
p.headers = append(p.headers, r.Header.Clone())
|
|
||||||
|
|
||||||
if p.status != http.StatusOK {
|
|
||||||
w.WriteHeader(p.status)
|
|
||||||
_, _ = io.WriteString(w, p.body)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if p.etag != "" {
|
|
||||||
w.Header().Set(etagHeader, p.etag)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ServeContent sends Last-Modified unless lastModified is zero, and
|
|
||||||
// answers "not modified" when the request's conditions match.
|
|
||||||
http.ServeContent(w, r, testListName, p.lastModified,
|
|
||||||
strings.NewReader(p.body))
|
|
||||||
}
|
|
||||||
|
|
||||||
// answer makes the publisher answer later requests with status and body.
|
|
||||||
func (p *publisher) answer(status int, body string) {
|
|
||||||
p.mu.Lock()
|
|
||||||
defer p.mu.Unlock()
|
|
||||||
|
|
||||||
p.status, p.body = status, body
|
|
||||||
}
|
|
||||||
|
|
||||||
// publish makes the publisher answer later requests with a version of the
|
|
||||||
// list: body, with lastModified and etag. A zero lastModified or an empty
|
|
||||||
// etag is not sent.
|
|
||||||
func (p *publisher) publish(body string, lastModified time.Time, etag string) {
|
|
||||||
p.mu.Lock()
|
|
||||||
defer p.mu.Unlock()
|
|
||||||
|
|
||||||
p.status, p.body = http.StatusOK, body
|
|
||||||
p.lastModified, p.etag = lastModified, etag
|
|
||||||
}
|
|
||||||
|
|
||||||
// requests returns the headers of the requests the publisher has had.
|
|
||||||
func (p *publisher) requests() []http.Header {
|
|
||||||
p.mu.Lock()
|
|
||||||
defer p.mu.Unlock()
|
|
||||||
|
|
||||||
return append([]http.Header(nil), p.headers...)
|
|
||||||
}
|
|
||||||
|
|
||||||
// testList returns a list served at url, written in format.
|
|
||||||
func testList(url string, format Format) List {
|
|
||||||
return List{
|
|
||||||
Name: testListName,
|
|
||||||
Description: testListDescription,
|
|
||||||
URL: url,
|
|
||||||
Interval: time.Hour,
|
|
||||||
Format: format,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func newTestBlocklists(dir string, list List) *Blocklists {
|
|
||||||
return New(Params{
|
|
||||||
Dir: dir,
|
|
||||||
Lists: []List{list},
|
|
||||||
Logger: slog.New(slog.DiscardHandler),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// lookup looks addr up in b.
|
|
||||||
func lookup(t *testing.T, b *Blocklists, addr string) []Listing {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
return b.Lookup(netip.MustParseAddr(addr))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLookupFindsListedAddresses downloads a list in each format and looks
|
|
||||||
// up addresses on it, inside its networks, and off it, in IPv4 and IPv6.
|
|
||||||
// A listed address has one listing that names the list; any other has an
|
|
||||||
// empty list of listings.
|
|
||||||
func TestLookupFindsListedAddresses(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
format Format
|
|
||||||
body string
|
|
||||||
listed []string
|
|
||||||
unlisted []string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "single addresses",
|
|
||||||
format: FormatLines,
|
|
||||||
body: "# comment\n192.0.2.1\n\n2001:db8::1\n",
|
|
||||||
listed: []string{"192.0.2.1", "::ffff:192.0.2.1", "2001:db8::1"},
|
|
||||||
unlisted: []string{"192.0.2.2", "2001:db8::2"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "networks",
|
|
||||||
format: FormatLines,
|
|
||||||
body: "# comment\n198.51.100.0/24\n2001:db8:1::/48\n",
|
|
||||||
listed: []string{"198.51.100.0", "198.51.100.255", "2001:db8:1::5"},
|
|
||||||
unlisted: []string{"198.51.101.0", "2001:db8:2::1"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "Spamhaus networks",
|
|
||||||
format: FormatSpamhausJSON,
|
|
||||||
body: `{"cidr":"203.0.113.0/25","sblid":"SBL1","rir":"arin"}` + "\n" +
|
|
||||||
`{"cidr":"2001:db8:3::/48","sblid":"SBL2","rir":"ripencc"}` + "\n" +
|
|
||||||
`{"type":"metadata","timestamp":1793074442,"records":2}` + "\n",
|
|
||||||
listed: []string{"203.0.113.1", "2001:db8:3::1"},
|
|
||||||
unlisted: []string{"203.0.113.200", "2001:db8:4::1"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "DShield networks",
|
|
||||||
format: FormatDShield,
|
|
||||||
body: "# DShield.org Recommended Block List\n" +
|
|
||||||
"192.0.2.0\t192.0.2.255\t24\t327\tEXAMPLE\tUS\tNone\n",
|
|
||||||
listed: []string{"192.0.2.0", "192.0.2.77"},
|
|
||||||
unlisted: []string{"192.0.3.1", "2001:db8::1"},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
_, url := newPublisher(t, tt.body)
|
|
||||||
b := newTestBlocklists(t.TempDir(), testList(url, tt.format))
|
|
||||||
|
|
||||||
err := b.download(t.Context(), b.lists[0])
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("download: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, addr := range tt.listed {
|
|
||||||
listings := lookup(t, b, addr)
|
|
||||||
if len(listings) != 1 || listings[0].Name != testListName ||
|
|
||||||
listings[0].Description != testListDescription ||
|
|
||||||
listings[0].Updated.IsZero() {
|
|
||||||
t.Errorf("%s: listings %+v, want one for %s", addr,
|
|
||||||
listings, testListName)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, addr := range tt.unlisted {
|
|
||||||
listings := lookup(t, b, addr)
|
|
||||||
if listings == nil || len(listings) != 0 {
|
|
||||||
t.Errorf("%s: listings %#v, want an empty list", addr, listings)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestFailedDownloadKeepsThePreviousCopy downloads a good version of a list,
|
|
||||||
// then one that fails. The good copy stays in memory and on disk, and no
|
|
||||||
// part of the failed download is left beside it.
|
|
||||||
func TestFailedDownloadKeepsThePreviousCopy(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
status int
|
|
||||||
body string
|
|
||||||
}{
|
|
||||||
{"server error", http.StatusInternalServerError, "unavailable"},
|
|
||||||
{"not a list", http.StatusOK, "<!DOCTYPE html>\n<html></html>\n"},
|
|
||||||
{
|
|
||||||
"over the size limit", http.StatusOK,
|
|
||||||
strings.Repeat(listedAddr+"\n", maxDownloadSize/len(listedAddr)),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
p, url := newPublisher(t, goodListBody)
|
|
||||||
b := newTestBlocklists(dir, testList(url, FormatLines))
|
|
||||||
|
|
||||||
err := b.download(t.Context(), b.lists[0])
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("first download: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
before := lookup(t, b, listedAddr)
|
|
||||||
|
|
||||||
p.answer(tt.status, tt.body)
|
|
||||||
|
|
||||||
err = b.download(t.Context(), b.lists[0])
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("second download returned no error")
|
|
||||||
}
|
|
||||||
|
|
||||||
after := lookup(t, b, listedAddr)
|
|
||||||
if len(after) != 1 || len(before) != 1 ||
|
|
||||||
!after[0].Updated.Equal(before[0].Updated) {
|
|
||||||
t.Errorf("listings %+v after the failure, want %+v", after, before)
|
|
||||||
}
|
|
||||||
|
|
||||||
copies := os.DirFS(dir)
|
|
||||||
|
|
||||||
onDisk, err := fs.ReadFile(copies, testListName)
|
|
||||||
if err != nil || string(onDisk) != goodListBody {
|
|
||||||
t.Errorf("copy on disk %q (%v), want %q", onDisk, err, goodListBody)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = fs.Stat(copies, testListName+downloadSuffix)
|
|
||||||
if !errors.Is(err, fs.ErrNotExist) {
|
|
||||||
t.Errorf("the failed download's file is still there (%v)", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRestartLoadsCopiesFromDisk downloads a list, then starts a new
|
|
||||||
// Blocklists on the same directory, as a restart does. The new one answers
|
|
||||||
// from the copy on disk at once, and does not download the list again while
|
|
||||||
// the copy is younger than the list's interval.
|
|
||||||
func TestRestartLoadsCopiesFromDisk(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
p, url := newPublisher(t, goodListBody)
|
|
||||||
list := testList(url, FormatLines)
|
|
||||||
|
|
||||||
first := newTestBlocklists(dir, list)
|
|
||||||
|
|
||||||
err := first.download(t.Context(), list)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("download: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
downloaded := lookup(t, first, listedAddr)[0].Updated
|
|
||||||
|
|
||||||
second := newTestBlocklists(dir, list)
|
|
||||||
second.Start(t.Context())
|
|
||||||
second.Stop(t.Context())
|
|
||||||
|
|
||||||
listings := lookup(t, second, listedAddr)
|
|
||||||
if len(listings) != 1 ||
|
|
||||||
listings[0].Updated.Sub(downloaded).Abs() > time.Second {
|
|
||||||
t.Errorf("after the restart, listings %+v, want one updated at %s",
|
|
||||||
listings, downloaded)
|
|
||||||
}
|
|
||||||
|
|
||||||
if got := len(p.requests()); got != 1 {
|
|
||||||
t.Errorf("the publisher had %d requests, want 1", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestStartRefreshesAnOldCopy starts on a copy older than its list's
|
|
||||||
// interval. The list is downloaded at once, with routewatch's User-Agent and
|
|
||||||
// the Last-Modified and ETag the publisher sent with the copy, kept on disk.
|
|
||||||
// The publisher answers that the copy is still current, so the copy is kept
|
|
||||||
// and marked as updated now.
|
|
||||||
func TestStartRefreshesAnOldCopy(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
p, url := newPublisher(t, goodListBody)
|
|
||||||
list := testList(url, FormatLines)
|
|
||||||
|
|
||||||
lastModified := time.Now().Add(-3 * list.Interval).UTC().Truncate(time.Second)
|
|
||||||
p.publish(goodListBody, lastModified, testETag)
|
|
||||||
|
|
||||||
err := newTestBlocklists(dir, list).download(t.Context(), list)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("download: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
old := time.Now().Add(-2 * list.Interval).UTC().Truncate(time.Second)
|
|
||||||
|
|
||||||
err = os.Chtimes(filepath.Join(dir, testListName), old, old)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Chtimes: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
b := newTestBlocklists(dir, list)
|
|
||||||
b.Start(t.Context())
|
|
||||||
|
|
||||||
deadline := time.Now().Add(maxWait)
|
|
||||||
for !lookup(t, b, listedAddr)[0].Updated.After(old) {
|
|
||||||
if time.Now().After(deadline) {
|
|
||||||
t.Fatalf("the copy was not marked updated within %s", maxWait)
|
|
||||||
}
|
|
||||||
|
|
||||||
time.Sleep(time.Millisecond)
|
|
||||||
}
|
|
||||||
|
|
||||||
b.Stop(t.Context())
|
|
||||||
|
|
||||||
requests := p.requests()
|
|
||||||
last := requests[len(requests)-1]
|
|
||||||
|
|
||||||
want := lastModified.Format(http.TimeFormat)
|
|
||||||
if got := last.Get(ifModifiedSinceHeader); got != want {
|
|
||||||
t.Errorf("If-Modified-Since %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
if got := last.Get(ifNoneMatchHeader); got != testETag {
|
|
||||||
t.Errorf("If-None-Match %q, want %q", got, testETag)
|
|
||||||
}
|
|
||||||
|
|
||||||
if got := last.Get(userAgentHeader); got != userAgent {
|
|
||||||
t.Errorf("User-Agent %q, want %q", got, userAgent)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestNewVersionOlderThanTheLastDownloadReplacesTheCopy downloads a list.
|
|
||||||
// The publisher then puts out a new version whose Last-Modified is earlier
|
|
||||||
// than that download, as when a cache in front of the publisher passes the
|
|
||||||
// version on late. The next download sends the Last-Modified of the copy,
|
|
||||||
// not the time of the download, so it gets the new version.
|
|
||||||
func TestNewVersionOlderThanTheLastDownloadReplacesTheCopy(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
p, url := newPublisher(t, goodListBody)
|
|
||||||
list := testList(url, FormatLines)
|
|
||||||
b := newTestBlocklists(t.TempDir(), list)
|
|
||||||
|
|
||||||
firstVersion := time.Now().Add(-2 * list.Interval).UTC().Truncate(time.Second)
|
|
||||||
p.publish(goodListBody, firstVersion, "")
|
|
||||||
|
|
||||||
err := b.download(t.Context(), list)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("first download: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
p.publish(newListBody, firstVersion.Add(list.Interval), "")
|
|
||||||
|
|
||||||
err = b.download(t.Context(), list)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("second download: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if listings := lookup(t, b, newListedAddr); len(listings) != 1 {
|
|
||||||
t.Errorf("%s: listings %+v, want one from the new version",
|
|
||||||
newListedAddr, listings)
|
|
||||||
}
|
|
||||||
|
|
||||||
want := firstVersion.Format(http.TimeFormat)
|
|
||||||
if got := p.requests()[1].Get(ifModifiedSinceHeader); got != want {
|
|
||||||
t.Errorf("If-Modified-Since %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestDownloadWithoutLastModifiedOrETagSendsNoCondition downloads a list
|
|
||||||
// twice from a publisher that sends neither Last-Modified nor ETag. Neither
|
|
||||||
// request carries If-Modified-Since or If-None-Match, so the second download
|
|
||||||
// gets the whole list again.
|
|
||||||
func TestDownloadWithoutLastModifiedOrETagSendsNoCondition(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
p, url := newPublisher(t, goodListBody)
|
|
||||||
list := testList(url, FormatLines)
|
|
||||||
b := newTestBlocklists(t.TempDir(), list)
|
|
||||||
|
|
||||||
err := b.download(t.Context(), list)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("first download: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = b.download(t.Context(), list)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("second download: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for i, request := range p.requests() {
|
|
||||||
if request.Get(ifModifiedSinceHeader) != "" ||
|
|
||||||
request.Get(ifNoneMatchHeader) != "" {
|
|
||||||
t.Errorf("request %d has a condition: %v", i, request)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestStopCancelsADownloadInProgress stops while the publisher has not
|
|
||||||
// answered. Stop cancels the download and returns once the download
|
|
||||||
// goroutine has stopped.
|
|
||||||
func TestStopCancelsADownloadInProgress(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
arrived := make(chan struct{})
|
|
||||||
|
|
||||||
var once sync.Once
|
|
||||||
|
|
||||||
srv := httptest.NewServer(http.HandlerFunc(
|
|
||||||
func(_ http.ResponseWriter, r *http.Request) {
|
|
||||||
once.Do(func() { close(arrived) })
|
|
||||||
<-r.Context().Done()
|
|
||||||
}))
|
|
||||||
t.Cleanup(srv.Close)
|
|
||||||
|
|
||||||
b := newTestBlocklists(t.TempDir(), testList(srv.URL, FormatLines))
|
|
||||||
b.Start(t.Context())
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-arrived:
|
|
||||||
case <-time.After(maxWait):
|
|
||||||
t.Fatalf("no download began within %s", maxWait)
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(t.Context(), maxWait)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
b.Stop(ctx)
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-b.done:
|
|
||||||
default:
|
|
||||||
t.Error("Stop returned before the download goroutine stopped")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPublicListsAreComplete checks that every public list has what a
|
|
||||||
// download and a lookup answer need, and a unique name.
|
|
||||||
func TestPublicListsAreComplete(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
names := make(map[string]bool)
|
|
||||||
|
|
||||||
for _, list := range Lists() {
|
|
||||||
if list.Name == "" || list.Description == "" || list.URL == "" ||
|
|
||||||
list.Interval <= 0 || names[list.Name] {
|
|
||||||
t.Errorf("list %+v is incomplete or has a name used before", list)
|
|
||||||
}
|
|
||||||
|
|
||||||
names[list.Name] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,379 +0,0 @@
|
|||||||
package blocklist
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"io/fs"
|
|
||||||
"net/http"
|
|
||||||
"os"
|
|
||||||
"slices"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"go4.org/netipx"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Limits of one download.
|
|
||||||
const (
|
|
||||||
// downloadTimeout is how long one download may take.
|
|
||||||
downloadTimeout = time.Minute
|
|
||||||
|
|
||||||
// maxDownloadSize is the size of the largest list file accepted, in
|
|
||||||
// bytes. The largest list, FireHOL abusers, is about 2.5 MiB.
|
|
||||||
maxDownloadSize = 16 << 20
|
|
||||||
)
|
|
||||||
|
|
||||||
// The request headers a download sets.
|
|
||||||
const (
|
|
||||||
userAgentHeader = "User-Agent"
|
|
||||||
ifModifiedSinceHeader = "If-Modified-Since"
|
|
||||||
ifNoneMatchHeader = "If-None-Match"
|
|
||||||
|
|
||||||
// userAgent names routewatch to the publishers.
|
|
||||||
userAgent = "routewatch (+https://git.eeqj.de/sneak/routewatch)"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The response headers that identify the version of a list a publisher sent.
|
|
||||||
const (
|
|
||||||
lastModifiedHeader = "Last-Modified"
|
|
||||||
etagHeader = "ETag"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
// downloadSuffix ends the name of the file a download is written to,
|
|
||||||
// beside the list's copy, until it has parsed.
|
|
||||||
downloadSuffix = ".download"
|
|
||||||
|
|
||||||
// validatorsSuffix ends the name of the file beside a list's copy that
|
|
||||||
// keeps the copy's validators.
|
|
||||||
validatorsSuffix = ".validators"
|
|
||||||
|
|
||||||
// dirPermissions are those of the directory that holds the copies.
|
|
||||||
dirPermissions = 0o750
|
|
||||||
|
|
||||||
// validatorsPermissions are those of the file that keeps a copy's
|
|
||||||
// validators.
|
|
||||||
validatorsPermissions = 0o600
|
|
||||||
)
|
|
||||||
|
|
||||||
// validators are the Last-Modified and ETag headers a publisher sent with a
|
|
||||||
// version of a list. A download sends them back, as If-Modified-Since and
|
|
||||||
// If-None-Match, so that the publisher sends the list only when it has
|
|
||||||
// changed. They are kept as the publisher wrote them.
|
|
||||||
//
|
|
||||||
//nolint:tagliatelle // snake_case keys, as the Go style guide asks
|
|
||||||
type validators struct {
|
|
||||||
LastModified string `json:"last_modified,omitempty"`
|
|
||||||
ETag string `json:"etag,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
var (
|
|
||||||
errUnexpectedStatus = errors.New("unexpected HTTP status")
|
|
||||||
errTooLarge = errors.New("list is over the size limit")
|
|
||||||
)
|
|
||||||
|
|
||||||
// Start loads each list's copy from disk. It then starts the goroutine that
|
|
||||||
// downloads each list once its copy is older than its Interval, and a list
|
|
||||||
// with no copy at once. The goroutine runs until Stop: the end of ctx does
|
|
||||||
// not stop it.
|
|
||||||
func (b *Blocklists) Start(ctx context.Context) {
|
|
||||||
err := os.MkdirAll(b.dir, dirPermissions)
|
|
||||||
if err != nil {
|
|
||||||
b.logger.Error("Failed to create the blocklists directory",
|
|
||||||
"dir", b.dir, "error", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
nextDownload := make([]time.Time, len(b.lists))
|
|
||||||
|
|
||||||
for i, list := range b.lists {
|
|
||||||
loadErr := b.loadCopy(list)
|
|
||||||
if loadErr != nil && !errors.Is(loadErr, fs.ErrNotExist) {
|
|
||||||
b.logger.Error("Failed to load a blocklist copy",
|
|
||||||
"list", list.Name, "error", loadErr)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A list with no copy has the zero time, so it is due at once.
|
|
||||||
nextDownload[i] = b.updated(list.Name).Add(list.Interval)
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx, b.stop = context.WithCancel(context.WithoutCancel(ctx))
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
b.run(ctx, nextDownload)
|
|
||||||
}()
|
|
||||||
|
|
||||||
b.logger.Info("Blocklist downloads started", "dir", b.dir)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stop stops the downloads, cancelling one in progress, and waits until the
|
|
||||||
// goroutine has returned or ctx ends. The copies in memory stay, so Lookup
|
|
||||||
// still answers.
|
|
||||||
func (b *Blocklists) Stop(ctx context.Context) {
|
|
||||||
b.stop()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-b.done:
|
|
||||||
b.logger.Info("Blocklist downloads stopped")
|
|
||||||
case <-ctx.Done():
|
|
||||||
b.logger.Warn("Stop deadline passed before the blocklist downloads " +
|
|
||||||
"stopped")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// run downloads each list when nextDownload says, until ctx ends. After each
|
|
||||||
// try, whether it worked or not, the list's next download is an Interval
|
|
||||||
// later.
|
|
||||||
func (b *Blocklists) run(ctx context.Context, nextDownload []time.Time) {
|
|
||||||
defer close(b.done)
|
|
||||||
|
|
||||||
if len(nextDownload) == 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
for {
|
|
||||||
soonest := slices.MinFunc(nextDownload, time.Time.Compare)
|
|
||||||
timer := time.NewTimer(time.Until(soonest))
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
timer.Stop()
|
|
||||||
|
|
||||||
return
|
|
||||||
case <-timer.C:
|
|
||||||
}
|
|
||||||
|
|
||||||
for i, list := range b.lists {
|
|
||||||
if time.Now().Before(nextDownload[i]) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
err := b.download(ctx, list)
|
|
||||||
if ctx.Err() != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
b.logger.Error("Blocklist download failed; keeping the "+
|
|
||||||
"previous copy", "list", list.Name, "error", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
nextDownload[i] = time.Now().Add(list.Interval)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// download asks the publisher for list, sending the validators of the list's
|
|
||||||
// copy when it has them. A new version that parses becomes the list's copy,
|
|
||||||
// on disk and in memory. An answer that the copy is still current marks the
|
|
||||||
// copy as updated now. Anything else leaves the copy as it is and returns an
|
|
||||||
// error.
|
|
||||||
func (b *Blocklists) download(ctx context.Context, list List) error {
|
|
||||||
ctx, cancel := context.WithTimeout(ctx, downloadTimeout)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, list.URL, nil)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
req.Header.Set(userAgentHeader, userAgent)
|
|
||||||
|
|
||||||
held := b.copyValidators(list.Name)
|
|
||||||
if held.LastModified != "" {
|
|
||||||
req.Header.Set(ifModifiedSinceHeader, held.LastModified)
|
|
||||||
}
|
|
||||||
|
|
||||||
if held.ETag != "" {
|
|
||||||
req.Header.Set(ifNoneMatchHeader, held.ETag)
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, err := http.DefaultClient.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() { _ = resp.Body.Close() }()
|
|
||||||
|
|
||||||
switch resp.StatusCode {
|
|
||||||
case http.StatusOK:
|
|
||||||
return b.replaceCopy(list, resp.Body, validators{
|
|
||||||
LastModified: resp.Header.Get(lastModifiedHeader),
|
|
||||||
ETag: resp.Header.Get(etagHeader),
|
|
||||||
})
|
|
||||||
case http.StatusNotModified:
|
|
||||||
return b.markCurrent(list)
|
|
||||||
default:
|
|
||||||
return fmt.Errorf("%w: %s", errUnexpectedStatus, resp.Status)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// replaceCopy writes a new version of list, read from body, beside the list's
|
|
||||||
// copy. Only when it parses does it replace the copy, on disk and in memory.
|
|
||||||
// sent are the validators the publisher sent with the new version.
|
|
||||||
func (b *Blocklists) replaceCopy(list List, body io.Reader, sent validators) error {
|
|
||||||
root, err := os.OpenRoot(b.dir)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() { _ = root.Close() }()
|
|
||||||
|
|
||||||
downloadName := list.Name + downloadSuffix
|
|
||||||
|
|
||||||
addresses, err := writeAndParse(root, downloadName, body, list.Format)
|
|
||||||
if err != nil {
|
|
||||||
_ = root.Remove(downloadName)
|
|
||||||
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// The old copy's validators are removed before the copy is replaced, so
|
|
||||||
// that they are never kept with the new copy.
|
|
||||||
err = root.Remove(list.Name + validatorsSuffix)
|
|
||||||
if err != nil && !errors.Is(err, fs.ErrNotExist) {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = root.Rename(downloadName, list.Name)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
b.setCopy(list.Name, listCopy{
|
|
||||||
addresses: addresses, updated: time.Now().UTC(), validators: sent,
|
|
||||||
})
|
|
||||||
b.logger.Info("Downloaded a new version of a blocklist", "list", list.Name)
|
|
||||||
|
|
||||||
// Without this file, the first download after a restart asks for the
|
|
||||||
// whole list.
|
|
||||||
err = writeValidators(root, list.Name, sent)
|
|
||||||
if err != nil {
|
|
||||||
b.logger.Error("Failed to keep the validators of a blocklist copy",
|
|
||||||
"list", list.Name, "error", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeAndParse writes body to the file name in root, up to maxDownloadSize
|
|
||||||
// bytes, and parses the file as format.
|
|
||||||
func writeAndParse(
|
|
||||||
root *os.Root, name string, body io.Reader, format Format,
|
|
||||||
) (*netipx.IPSet, error) {
|
|
||||||
f, err := root.Create(name)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() { _ = f.Close() }()
|
|
||||||
|
|
||||||
written, err := io.Copy(f, io.LimitReader(body, maxDownloadSize+1))
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if written > maxDownloadSize {
|
|
||||||
return nil, errTooLarge
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = f.Seek(0, io.SeekStart)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return parse(f, format)
|
|
||||||
}
|
|
||||||
|
|
||||||
// markCurrent records that the publisher says the list's copy is still
|
|
||||||
// current: the copy, on disk and in memory, is marked as updated now.
|
|
||||||
func (b *Blocklists) markCurrent(list List) error {
|
|
||||||
root, err := os.OpenRoot(b.dir)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() { _ = root.Close() }()
|
|
||||||
|
|
||||||
now := time.Now().UTC()
|
|
||||||
|
|
||||||
err = root.Chtimes(list.Name, now, now)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
b.setUpdated(list.Name, now)
|
|
||||||
b.logger.Debug("Blocklist copy is still current", "list", list.Name)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// loadCopy makes the list's copy on disk, with the validators kept beside it,
|
|
||||||
// its copy in memory. The file's modification time is when the copy was
|
|
||||||
// updated.
|
|
||||||
func (b *Blocklists) loadCopy(list List) error {
|
|
||||||
root, err := os.OpenRoot(b.dir)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() { _ = root.Close() }()
|
|
||||||
|
|
||||||
f, err := root.Open(list.Name)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() { _ = f.Close() }()
|
|
||||||
|
|
||||||
info, err := f.Stat()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
addresses, err := parse(f, list.Format)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
held, err := readValidators(root, list.Name)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
b.setCopy(list.Name, listCopy{
|
|
||||||
addresses: addresses, updated: info.ModTime().UTC(), validators: held,
|
|
||||||
})
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeValidators keeps held in the file beside the named list's copy.
|
|
||||||
func writeValidators(root *os.Root, name string, held validators) error {
|
|
||||||
data, err := json.Marshal(held)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return root.WriteFile(name+validatorsSuffix, data, validatorsPermissions)
|
|
||||||
}
|
|
||||||
|
|
||||||
// readValidators returns the validators kept beside the named list's copy.
|
|
||||||
// They are empty when there is no such file.
|
|
||||||
func readValidators(root *os.Root, name string) (validators, error) {
|
|
||||||
var held validators
|
|
||||||
|
|
||||||
data, err := root.ReadFile(name + validatorsSuffix)
|
|
||||||
if errors.Is(err, fs.ErrNotExist) {
|
|
||||||
return held, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return held, err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = json.Unmarshal(data, &held)
|
|
||||||
|
|
||||||
return held, err
|
|
||||||
}
|
|
||||||
@@ -1,93 +0,0 @@
|
|||||||
package blocklist
|
|
||||||
|
|
||||||
import "time"
|
|
||||||
|
|
||||||
// How often each list is downloaded. No list is downloaded more often than
|
|
||||||
// its publisher updates it or allows.
|
|
||||||
const (
|
|
||||||
// DShield asks that its feeds be downloaded at most once an hour:
|
|
||||||
// https://isc.sans.edu/feeds_doc.html
|
|
||||||
dshieldInterval = time.Hour
|
|
||||||
|
|
||||||
// abuse.ch makes the list every 5 minutes and recommends a download at
|
|
||||||
// least every 15: https://feodotracker.abuse.ch/blocklist/
|
|
||||||
feodoTrackerInterval = 15 * time.Minute
|
|
||||||
|
|
||||||
// FireHOL's page for each list gives how often the list changes on
|
|
||||||
// average: about every 5 hours for abusers and every 3 hours for level
|
|
||||||
// 3. FireHOL sets no download limit.
|
|
||||||
// https://iplists.firehol.org/?ipset=firehol_abusers_30d
|
|
||||||
// https://iplists.firehol.org/?ipset=firehol_level3
|
|
||||||
fireHOLAbusersInterval = 5 * time.Hour
|
|
||||||
fireHOLLevel3Interval = 3 * time.Hour
|
|
||||||
|
|
||||||
// Spamhaus asks that DROP be downloaded at most once a day:
|
|
||||||
// https://www.spamhaus.org/blocklists/do-not-route-or-peer/
|
|
||||||
spamhausDROPInterval = 24 * time.Hour
|
|
||||||
|
|
||||||
// Tor publishes a new exit list every hour and sets no download limit:
|
|
||||||
// https://collector.torproject.org/recent/exit-lists/
|
|
||||||
torExitsInterval = time.Hour
|
|
||||||
)
|
|
||||||
|
|
||||||
// Lists returns the public blocklists routewatch downloads. Each can be
|
|
||||||
// downloaded whole without an account or a key. Spamhaus merged its EDROP
|
|
||||||
// list into DROP, so DROP covers both.
|
|
||||||
func Lists() []List {
|
|
||||||
return []List{
|
|
||||||
{
|
|
||||||
Name: "dshield",
|
|
||||||
Description: "In one of the 20 networks that DShield saw scan " +
|
|
||||||
"the most targets in the last three days",
|
|
||||||
URL: "https://feeds.dshield.org/block.txt",
|
|
||||||
Interval: dshieldInterval,
|
|
||||||
Format: FormatDShield,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "feodo_tracker",
|
|
||||||
Description: "A botnet command-and-control server",
|
|
||||||
URL: "https://feodotracker.abuse.ch/downloads/ipblocklist.txt",
|
|
||||||
Interval: feodoTrackerInterval,
|
|
||||||
Format: FormatLines,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "firehol_abusers_30d",
|
|
||||||
Description: "Reported for comment, form or forum spam in the " +
|
|
||||||
"last 30 days",
|
|
||||||
URL: "https://iplists.firehol.org/files/firehol_abusers_30d.netset",
|
|
||||||
Interval: fireHOLAbusersInterval,
|
|
||||||
Format: FormatLines,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "firehol_level3",
|
|
||||||
Description: "Reported for attacks, spyware or viruses in the " +
|
|
||||||
"last 30 days",
|
|
||||||
URL: "https://iplists.firehol.org/files/firehol_level3.netset",
|
|
||||||
Interval: fireHOLLevel3Interval,
|
|
||||||
Format: FormatLines,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "spamhaus_drop",
|
|
||||||
Description: "In an IPv4 network that Spamhaus says spammers " +
|
|
||||||
"or cyber criminals have hijacked or lease",
|
|
||||||
URL: "https://www.spamhaus.org/drop/drop_v4.json",
|
|
||||||
Interval: spamhausDROPInterval,
|
|
||||||
Format: FormatSpamhausJSON,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "spamhaus_dropv6",
|
|
||||||
Description: "In an IPv6 network that Spamhaus says spammers " +
|
|
||||||
"or cyber criminals have hijacked or lease",
|
|
||||||
URL: "https://www.spamhaus.org/drop/drop_v6.json",
|
|
||||||
Interval: spamhausDROPInterval,
|
|
||||||
Format: FormatSpamhausJSON,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "tor_exits",
|
|
||||||
Description: "A Tor exit node, where traffic from the Tor network leaves",
|
|
||||||
URL: "https://check.torproject.org/torbulkexitlist",
|
|
||||||
Interval: torExitsInterval,
|
|
||||||
Format: FormatLines,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,158 +0,0 @@
|
|||||||
package blocklist
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bufio"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net/netip"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"go4.org/netipx"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
// commentPrefix starts a comment line in a list file.
|
|
||||||
commentPrefix = "#"
|
|
||||||
|
|
||||||
// networkSeparator separates the address and the mask length of a
|
|
||||||
// network, as in 192.0.2.0/24.
|
|
||||||
networkSeparator = "/"
|
|
||||||
|
|
||||||
// dshieldFieldSeparator separates the fields of a DShield line.
|
|
||||||
dshieldFieldSeparator = "\t"
|
|
||||||
|
|
||||||
// spamhausMetadataType is the type of the line that ends a Spamhaus
|
|
||||||
// file. That line holds the file's metadata and no network.
|
|
||||||
spamhausMetadataType = "metadata"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
errUnknownFormat = errors.New("unknown list format")
|
|
||||||
errNoNetwork = errors.New("no network on the line")
|
|
||||||
errBadRange = errors.New("first and last address make no range")
|
|
||||||
)
|
|
||||||
|
|
||||||
// parse reads a list file written in format and returns the addresses it
|
|
||||||
// holds. A line it cannot read fails the whole file, so a file that is not
|
|
||||||
// the list, such as an error page, never replaces a good copy.
|
|
||||||
func parse(r io.Reader, format Format) (*netipx.IPSet, error) {
|
|
||||||
var builder netipx.IPSetBuilder
|
|
||||||
|
|
||||||
scanner := bufio.NewScanner(r)
|
|
||||||
|
|
||||||
for lineNumber := 1; scanner.Scan(); lineNumber++ {
|
|
||||||
line := strings.TrimSpace(scanner.Text())
|
|
||||||
if line == "" || strings.HasPrefix(line, commentPrefix) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
err := addLine(&builder, line, format)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("line %d: %w", lineNumber, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
err := scanner.Err()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return builder.IPSet()
|
|
||||||
}
|
|
||||||
|
|
||||||
// addLine adds the address or network on one line of a list file to builder.
|
|
||||||
func addLine(builder *netipx.IPSetBuilder, line string, format Format) error {
|
|
||||||
switch format {
|
|
||||||
case FormatLines:
|
|
||||||
return addAddressOrNetwork(builder, line)
|
|
||||||
case FormatSpamhausJSON:
|
|
||||||
return addSpamhausNetwork(builder, line)
|
|
||||||
case FormatDShield:
|
|
||||||
return addDShieldNetwork(builder, line)
|
|
||||||
default:
|
|
||||||
return errUnknownFormat
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// addAddressOrNetwork adds an address, such as 192.0.2.1, or a network, such
|
|
||||||
// as 192.0.2.0/24, to builder.
|
|
||||||
func addAddressOrNetwork(builder *netipx.IPSetBuilder, text string) error {
|
|
||||||
if strings.Contains(text, networkSeparator) {
|
|
||||||
network, err := netip.ParsePrefix(text)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
builder.AddPrefix(network)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
addr, err := netip.ParseAddr(text)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
builder.Add(addr)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// addSpamhausNetwork adds the network on a line of a Spamhaus file to
|
|
||||||
// builder. The metadata line adds nothing.
|
|
||||||
func addSpamhausNetwork(builder *netipx.IPSetBuilder, line string) error {
|
|
||||||
var entry struct {
|
|
||||||
CIDR string `json:"cidr"`
|
|
||||||
Type string `json:"type"`
|
|
||||||
}
|
|
||||||
|
|
||||||
err := json.Unmarshal([]byte(line), &entry)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if entry.Type == spamhausMetadataType {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
network, err := netip.ParsePrefix(entry.CIDR)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
builder.AddPrefix(network)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// addDShieldNetwork adds the network on a line of a DShield file to builder.
|
|
||||||
// The line's first two fields are the network's first and last address.
|
|
||||||
func addDShieldNetwork(builder *netipx.IPSetBuilder, line string) error {
|
|
||||||
firstField, rest, found := strings.Cut(line, dshieldFieldSeparator)
|
|
||||||
if !found {
|
|
||||||
return errNoNetwork
|
|
||||||
}
|
|
||||||
|
|
||||||
lastField, _, _ := strings.Cut(rest, dshieldFieldSeparator)
|
|
||||||
|
|
||||||
first, err := netip.ParseAddr(firstField)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
last, err := netip.ParseAddr(lastField)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
network := netipx.IPRangeFrom(first, last)
|
|
||||||
if !network.IsValid() {
|
|
||||||
return errBadRange
|
|
||||||
}
|
|
||||||
|
|
||||||
builder.AddRange(network)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,202 +0,0 @@
|
|||||||
// Package bogon tells whether a prefix or an AS number is reserved: set aside
|
|
||||||
// for a special purpose, so that no network should announce it on the
|
|
||||||
// internet. A route is a bogon announcement when its prefix is reserved or its
|
|
||||||
// origin AS is.
|
|
||||||
//
|
|
||||||
// Most reserved prefixes come from the IANA special-purpose address registries
|
|
||||||
// for IPv4 and IPv6. The two multicast blocks, 224.0.0.0/4 and ff00::/8, come
|
|
||||||
// from the IANA IPv4 and IPv6 address space registries. The reserved AS
|
|
||||||
// numbers come from the IANA special-purpose AS numbers registry:
|
|
||||||
// - the AS numbers for private use
|
|
||||||
// - the AS numbers for documentation
|
|
||||||
// - AS 23456, which stands in for a 32-bit AS number
|
|
||||||
// - the last AS number of the 16-bit and of the 32-bit range
|
|
||||||
//
|
|
||||||
// Address space and AS numbers that the IANA has not handed out yet are not
|
|
||||||
// counted.
|
|
||||||
package bogon
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/netip"
|
|
||||||
"strconv"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Prefix returns why prefix is reserved, and true, when it lies inside a
|
|
||||||
// reserved block. The most specific registry block that holds prefix decides,
|
|
||||||
// because some blocks that may be announced lie inside reserved ones. It
|
|
||||||
// returns "" and false when prefix is not reserved.
|
|
||||||
func Prefix(prefix netip.Prefix) (string, bool) {
|
|
||||||
var holder *block
|
|
||||||
|
|
||||||
for i, b := range registryBlocks {
|
|
||||||
inside := b.prefix.Bits() <= prefix.Bits() && b.prefix.Contains(prefix.Addr())
|
|
||||||
if inside && (holder == nil || b.prefix.Bits() > holder.prefix.Bits()) {
|
|
||||||
holder = ®istryBlocks[i]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if holder == nil || !holder.reserved {
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
|
|
||||||
return holder.name + " (" + holder.prefix.String() + ")", true
|
|
||||||
}
|
|
||||||
|
|
||||||
// ASN returns why asn is reserved, and true, when it is. It returns "" and
|
|
||||||
// false when asn is not reserved.
|
|
||||||
func ASN(asn int) (string, bool) {
|
|
||||||
for _, r := range reservedASRanges {
|
|
||||||
if asn < r.first || asn > r.last {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if r.first == r.last {
|
|
||||||
return r.name + " (AS" + strconv.Itoa(asn) + ")", true
|
|
||||||
}
|
|
||||||
|
|
||||||
return r.name + " (AS" + strconv.Itoa(r.first) + "-AS" +
|
|
||||||
strconv.Itoa(r.last) + ")", true
|
|
||||||
}
|
|
||||||
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
|
|
||||||
// The names that the registries give to more than one row.
|
|
||||||
const (
|
|
||||||
privateUse = "Private-Use"
|
|
||||||
documentation = "Documentation"
|
|
||||||
benchmarking = "Benchmarking"
|
|
||||||
ietfProtocolAssignments = "IETF Protocol Assignments"
|
|
||||||
portControlAnycast = "Port Control Protocol Anycast"
|
|
||||||
natTraversalAnycast = "Traversal Using Relays around NAT Anycast"
|
|
||||||
nat64Discovery = "NAT64/DNS64 Discovery"
|
|
||||||
amt = "AMT"
|
|
||||||
as112DirectDelegation = "Direct Delegation AS112 Service"
|
|
||||||
ipv4IPv6Translation = "IPv4-IPv6 Translat."
|
|
||||||
multicast = "Multicast"
|
|
||||||
)
|
|
||||||
|
|
||||||
// block is one row of an IANA address registry.
|
|
||||||
type block struct {
|
|
||||||
prefix netip.Prefix
|
|
||||||
name string
|
|
||||||
// reserved says that no network should announce the row's addresses on
|
|
||||||
// the internet.
|
|
||||||
reserved bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// reservedBlock returns a reserved row: a multicast block, or a special-purpose
|
|
||||||
// row whose Globally Reachable column is False.
|
|
||||||
func reservedBlock(prefix, name string) block {
|
|
||||||
return block{prefix: netip.MustParsePrefix(prefix), name: name, reserved: true}
|
|
||||||
}
|
|
||||||
|
|
||||||
// reachableBlock returns a special-purpose row whose Globally Reachable column
|
|
||||||
// is True or N/A. Such a row matters only inside a reserved block, whose
|
|
||||||
// addresses it lets be announced.
|
|
||||||
func reachableBlock(prefix, name string) block {
|
|
||||||
return block{prefix: netip.MustParsePrefix(prefix), name: name}
|
|
||||||
}
|
|
||||||
|
|
||||||
// registryBlocks are the rows of the IANA special-purpose address registries,
|
|
||||||
// without the deprecated ones:
|
|
||||||
// - https://www.iana.org/assignments/iana-ipv4-special-registry/
|
|
||||||
// - https://www.iana.org/assignments/iana-ipv6-special-registry/
|
|
||||||
//
|
|
||||||
// They also hold the multicast blocks, which the IANA address space registries
|
|
||||||
// list as Multicast:
|
|
||||||
// - https://www.iana.org/assignments/ipv4-address-space/
|
|
||||||
// - https://www.iana.org/assignments/ipv6-address-space/
|
|
||||||
//
|
|
||||||
//nolint:gochecknoglobals // a fixed list, parsed once
|
|
||||||
var registryBlocks = []block{
|
|
||||||
reservedBlock("0.0.0.0/8", "This network"),
|
|
||||||
reservedBlock("0.0.0.0/32", "This host on this network"),
|
|
||||||
reservedBlock("10.0.0.0/8", privateUse),
|
|
||||||
reservedBlock("100.64.0.0/10", "Shared Address Space"),
|
|
||||||
reservedBlock("127.0.0.0/8", "Loopback"),
|
|
||||||
reservedBlock("169.254.0.0/16", "Link Local"),
|
|
||||||
reservedBlock("172.16.0.0/12", privateUse),
|
|
||||||
reservedBlock("192.0.0.0/24", ietfProtocolAssignments),
|
|
||||||
reservedBlock("192.0.0.0/29", "IPv4 Service Continuity Prefix"),
|
|
||||||
reservedBlock("192.0.0.8/32", "IPv4 dummy address"),
|
|
||||||
reachableBlock("192.0.0.9/32", portControlAnycast),
|
|
||||||
reachableBlock("192.0.0.10/32", natTraversalAnycast),
|
|
||||||
reservedBlock("192.0.0.170/32", nat64Discovery),
|
|
||||||
reservedBlock("192.0.0.171/32", nat64Discovery),
|
|
||||||
reservedBlock("192.0.2.0/24", "Documentation (TEST-NET-1)"),
|
|
||||||
reachableBlock("192.31.196.0/24", "AS112-v4"),
|
|
||||||
reachableBlock("192.52.193.0/24", amt),
|
|
||||||
reservedBlock("192.88.99.2/32", "6a44-relay anycast address"),
|
|
||||||
reservedBlock("192.168.0.0/16", privateUse),
|
|
||||||
reachableBlock("192.175.48.0/24", as112DirectDelegation),
|
|
||||||
reservedBlock("198.18.0.0/15", benchmarking),
|
|
||||||
reservedBlock("198.51.100.0/24", "Documentation (TEST-NET-2)"),
|
|
||||||
reservedBlock("203.0.113.0/24", "Documentation (TEST-NET-3)"),
|
|
||||||
reservedBlock("224.0.0.0/4", multicast),
|
|
||||||
reservedBlock("240.0.0.0/4", "Reserved"),
|
|
||||||
reservedBlock("255.255.255.255/32", "Limited Broadcast"),
|
|
||||||
|
|
||||||
reservedBlock("::1/128", "Loopback Address"),
|
|
||||||
reservedBlock("::/128", "Unspecified Address"),
|
|
||||||
reservedBlock("::ffff:0:0/96", "IPv4-mapped Address"),
|
|
||||||
reachableBlock("64:ff9b::/96", ipv4IPv6Translation),
|
|
||||||
reservedBlock("64:ff9b:1::/48", ipv4IPv6Translation),
|
|
||||||
reservedBlock("100::/64", "Discard-Only Address Block"),
|
|
||||||
reservedBlock("100:0:0:1::/64", "Dummy IPv6 Prefix"),
|
|
||||||
reservedBlock("2001::/23", ietfProtocolAssignments),
|
|
||||||
reachableBlock("2001::/32", "TEREDO"),
|
|
||||||
reachableBlock("2001:1::1/128", portControlAnycast),
|
|
||||||
reachableBlock("2001:1::2/128", natTraversalAnycast),
|
|
||||||
reachableBlock("2001:1::3/128", "DNS-SD Service Registration Protocol Anycast"),
|
|
||||||
reservedBlock("2001:2::/48", benchmarking),
|
|
||||||
reachableBlock("2001:3::/32", amt),
|
|
||||||
reachableBlock("2001:4:112::/48", "AS112-v6"),
|
|
||||||
reachableBlock("2001:20::/28", "ORCHIDv2"),
|
|
||||||
reachableBlock("2001:30::/28", "Drone Remote ID Protocol Entity Tags (DETs) Prefix"),
|
|
||||||
reservedBlock("2001:db8::/32", documentation),
|
|
||||||
reachableBlock("2002::/16", "6to4"),
|
|
||||||
reachableBlock("2620:4f:8000::/48", as112DirectDelegation),
|
|
||||||
reservedBlock("3fff::/20", documentation),
|
|
||||||
reservedBlock("5f00::/16", "Segment Routing (SRv6) SIDs"),
|
|
||||||
reservedBlock("fc00::/7", "Unique-Local"),
|
|
||||||
reservedBlock("fe80::/10", "Link-Local Unicast"),
|
|
||||||
reservedBlock("ff00::/8", multicast),
|
|
||||||
}
|
|
||||||
|
|
||||||
// asRange is a range of reserved AS numbers, first to last.
|
|
||||||
type asRange struct {
|
|
||||||
first, last int
|
|
||||||
name string
|
|
||||||
}
|
|
||||||
|
|
||||||
// The reserved AS numbers of the IANA special-purpose AS numbers registry.
|
|
||||||
const (
|
|
||||||
asTrans = 23456
|
|
||||||
firstDocumentationAS16 = 64496
|
|
||||||
lastDocumentationAS16 = 64511
|
|
||||||
firstPrivateUseAS16 = 64512
|
|
||||||
lastPrivateUseAS16 = 65534
|
|
||||||
lastAS16 = 65535
|
|
||||||
firstDocumentationAS32 = 65536
|
|
||||||
lastDocumentationAS32 = 65551
|
|
||||||
firstPrivateUseAS32 = 4200000000
|
|
||||||
lastPrivateUseAS32 = 4294967294
|
|
||||||
lastAS32 = 4294967295
|
|
||||||
)
|
|
||||||
|
|
||||||
// reservedASRanges are the reserved rows of the IANA special-purpose AS numbers
|
|
||||||
// registry, https://www.iana.org/assignments/iana-as-numbers-special-registry/.
|
|
||||||
// AS 0 is left out: RouteWatch uses it for a route with no known origin. AS 112
|
|
||||||
// is left out because the AS112 project announces routes with it.
|
|
||||||
//
|
|
||||||
//nolint:gochecknoglobals // a fixed list
|
|
||||||
var reservedASRanges = []asRange{
|
|
||||||
{first: asTrans, last: asTrans, name: "AS_TRANS"},
|
|
||||||
{first: firstDocumentationAS16, last: lastDocumentationAS16, name: documentation},
|
|
||||||
{first: firstPrivateUseAS16, last: lastPrivateUseAS16, name: privateUse},
|
|
||||||
{first: lastAS16, last: lastAS16, name: "Last 16-bit AS number"},
|
|
||||||
{first: firstDocumentationAS32, last: lastDocumentationAS32, name: documentation},
|
|
||||||
{first: firstPrivateUseAS32, last: lastPrivateUseAS32, name: privateUse},
|
|
||||||
{first: lastAS32, last: lastAS32, name: "Last 32-bit AS number"},
|
|
||||||
}
|
|
||||||
@@ -1,95 +0,0 @@
|
|||||||
package bogon_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/netip"
|
|
||||||
"strconv"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/bogon"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestPrefix(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
prefix string
|
|
||||||
// want is why the prefix is reserved, or "" when it is not.
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{"10.1.2.0/24", "Private-Use (10.0.0.0/8)"},
|
|
||||||
{"10.0.0.0/8", "Private-Use (10.0.0.0/8)"},
|
|
||||||
{"192.0.2.0/24", "Documentation (TEST-NET-1) (192.0.2.0/24)"},
|
|
||||||
{"2001:db8:1::/48", "Documentation (2001:db8::/32)"},
|
|
||||||
{"fd00::/8", "Unique-Local (fc00::/7)"},
|
|
||||||
// Multicast space, from the IANA address space registries.
|
|
||||||
{"224.0.0.0/4", "Multicast (224.0.0.0/4)"},
|
|
||||||
{"239.255.0.0/16", "Multicast (224.0.0.0/4)"},
|
|
||||||
{"ff00::/8", "Multicast (ff00::/8)"},
|
|
||||||
{"ff02::/16", "Multicast (ff00::/8)"},
|
|
||||||
// The most specific block names the reason.
|
|
||||||
{"0.0.0.0/32", "This host on this network (0.0.0.0/32)"},
|
|
||||||
{"2001:2::/48", "Benchmarking (2001:2::/48)"},
|
|
||||||
// Blocks that may be announced, inside reserved blocks.
|
|
||||||
{"192.0.0.9/32", ""},
|
|
||||||
{"2001:4:112::/48", ""},
|
|
||||||
// Blocks that may be announced, inside no reserved block.
|
|
||||||
{"192.175.48.0/24", ""},
|
|
||||||
{"2002::/16", ""},
|
|
||||||
// A prefix that holds a reserved block is not inside it.
|
|
||||||
{"192.0.0.0/16", ""},
|
|
||||||
{"0.0.0.0/0", ""},
|
|
||||||
// Prefixes anyone may announce.
|
|
||||||
{"8.8.8.0/24", ""},
|
|
||||||
{"2001:4860::/32", ""},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range tests {
|
|
||||||
t.Run(tc.prefix, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
reason, reserved := bogon.Prefix(netip.MustParsePrefix(tc.prefix))
|
|
||||||
if reason != tc.want || reserved != (tc.want != "") {
|
|
||||||
t.Errorf("Prefix(%s) = %q, %v; want %q", tc.prefix, reason,
|
|
||||||
reserved, tc.want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestASN(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
asn int
|
|
||||||
// want is why the AS number is reserved, or "" when it is not.
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{23456, "AS_TRANS (AS23456)"},
|
|
||||||
{64496, "Documentation (AS64496-AS64511)"},
|
|
||||||
{64512, "Private-Use (AS64512-AS65534)"},
|
|
||||||
{65534, "Private-Use (AS64512-AS65534)"},
|
|
||||||
{65535, "Last 16-bit AS number (AS65535)"},
|
|
||||||
{65551, "Documentation (AS65536-AS65551)"},
|
|
||||||
{4200000000, "Private-Use (AS4200000000-AS4294967294)"},
|
|
||||||
{4294967295, "Last 32-bit AS number (AS4294967295)"},
|
|
||||||
// AS 0 means no known origin, and the AS112 project uses AS 112.
|
|
||||||
{0, ""},
|
|
||||||
{112, ""},
|
|
||||||
// Not handed out yet, which is not counted.
|
|
||||||
{65552, ""},
|
|
||||||
{15169, ""},
|
|
||||||
{64495, ""},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range tests {
|
|
||||||
t.Run(strconv.Itoa(tc.asn), func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
reason, reserved := bogon.ASN(tc.asn)
|
|
||||||
if reason != tc.want || reserved != (tc.want != "") {
|
|
||||||
t.Errorf("ASN(%d) = %q, %v; want %q", tc.asn, reason, reserved,
|
|
||||||
tc.want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+10
-89
@@ -2,14 +2,11 @@
|
|||||||
package config
|
package config
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/netip"
|
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"runtime"
|
"runtime"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -30,22 +27,6 @@ const (
|
|||||||
maxPort = 65535
|
maxPort = 65535
|
||||||
)
|
)
|
||||||
|
|
||||||
// The private address ranges of RFC 1918. The HTTP server trusts a reverse
|
|
||||||
// proxy in any of them when TRUSTED_PROXIES is not set.
|
|
||||||
const (
|
|
||||||
privateRange10 = "10.0.0.0/8"
|
|
||||||
privateRange172 = "172.16.0.0/12"
|
|
||||||
privateRange192 = "192.168.0.0/16"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
errRelativeXDGDataHome = errors.New("XDG_DATA_HOME must be an absolute path")
|
|
||||||
errUnsupportedOS = errors.New("unsupported operating system")
|
|
||||||
errInvalidPort = errors.New("PORT must be a whole number")
|
|
||||||
errInvalidProxy = errors.New(
|
|
||||||
"TRUSTED_PROXIES must list IP addresses and CIDR ranges, comma-separated")
|
|
||||||
)
|
|
||||||
|
|
||||||
// Config holds configuration for the entire application
|
// Config holds configuration for the entire application
|
||||||
type Config struct {
|
type Config struct {
|
||||||
// StateDir is the directory for all application state (database, snapshots)
|
// StateDir is the directory for all application state (database, snapshots)
|
||||||
@@ -54,21 +35,14 @@ type Config struct {
|
|||||||
// Port is the TCP port the HTTP server listens on
|
// Port is the TCP port the HTTP server listens on
|
||||||
Port int
|
Port int
|
||||||
|
|
||||||
// TrustedProxies are the reverse proxies whose X-Forwarded-For header
|
|
||||||
// the HTTP server believes
|
|
||||||
TrustedProxies []netip.Prefix
|
|
||||||
|
|
||||||
// MaxRuntime is the maximum runtime (0 = run forever)
|
// MaxRuntime is the maximum runtime (0 = run forever)
|
||||||
MaxRuntime time.Duration
|
MaxRuntime time.Duration
|
||||||
|
|
||||||
// EnableBatchedDatabaseWrites enables batched database operations for
|
// EnableBatchedDatabaseWrites enables batched database operations for better performance
|
||||||
// better performance
|
|
||||||
EnableBatchedDatabaseWrites bool
|
EnableBatchedDatabaseWrites bool
|
||||||
|
|
||||||
// RouteExpirationTimeout is how long a route can go without being
|
// RouteExpirationTimeout is how long a route can go without being refreshed before expiring
|
||||||
// refreshed before expiring
|
// Default is 2 hours which is conservative for BGP (typical BGP hold time is 90-180 seconds)
|
||||||
// Default is 2 hours which is conservative for BGP (typical BGP hold
|
|
||||||
// time is 90-180 seconds)
|
|
||||||
RouteExpirationTimeout time.Duration
|
RouteExpirationTimeout time.Duration
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -84,21 +58,12 @@ func New() (*Config, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
trustedProxies, err := getTrustedProxies()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return &Config{
|
return &Config{
|
||||||
StateDir: stateDir,
|
StateDir: stateDir,
|
||||||
Port: port,
|
Port: port,
|
||||||
TrustedProxies: trustedProxies,
|
MaxRuntime: 0, // Run forever by default
|
||||||
// Run forever by default
|
EnableBatchedDatabaseWrites: true, // Enable batching by default
|
||||||
MaxRuntime: 0,
|
RouteExpirationTimeout: defaultRouteExpirationMinutes * time.Minute, // For active route monitoring
|
||||||
// Enable batching by default
|
|
||||||
EnableBatchedDatabaseWrites: true,
|
|
||||||
// For active route monitoring
|
|
||||||
RouteExpirationTimeout: defaultRouteExpirationMinutes * time.Minute,
|
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -124,7 +89,7 @@ func getStateDirectory() (string, error) {
|
|||||||
// the database somewhere unexpected.
|
// the database somewhere unexpected.
|
||||||
xdgData := os.Getenv("XDG_DATA_HOME")
|
xdgData := os.Getenv("XDG_DATA_HOME")
|
||||||
if xdgData != "" && !filepath.IsAbs(xdgData) {
|
if xdgData != "" && !filepath.IsAbs(xdgData) {
|
||||||
return "", fmt.Errorf("%w, got %q", errRelativeXDGDataHome, xdgData)
|
return "", fmt.Errorf("XDG_DATA_HOME must be an absolute path, got %q", xdgData)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Unix-like: /var/lib/berlin.sneak.app.routewatch if root, else XDG_DATA_HOME
|
// Unix-like: /var/lib/berlin.sneak.app.routewatch if root, else XDG_DATA_HOME
|
||||||
@@ -146,7 +111,7 @@ func getStateDirectory() (string, error) {
|
|||||||
return filepath.Join(home, ".local", "share", AppIdentifier), nil
|
return filepath.Join(home, ".local", "share", AppIdentifier), nil
|
||||||
|
|
||||||
default:
|
default:
|
||||||
return "", fmt.Errorf("%w: %s", errUnsupportedOS, runtime.GOOS)
|
return "", fmt.Errorf("unsupported operating system: %s", runtime.GOOS)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -160,60 +125,16 @@ func getPort() (int, error) {
|
|||||||
// ParseUint, unlike Atoi, refuses a sign: the health check URL cannot use "+9090"
|
// ParseUint, unlike Atoi, refuses a sign: the health check URL cannot use "+9090"
|
||||||
port, err := strconv.ParseUint(value, 10, 0)
|
port, err := strconv.ParseUint(value, 10, 0)
|
||||||
if err != nil || port < 1 || port > maxPort {
|
if err != nil || port < 1 || port > maxPort {
|
||||||
return 0, fmt.Errorf("%w from 1 to %d, got %q",
|
return 0, fmt.Errorf("PORT must be a whole number from 1 to %d, got %q", maxPort, value)
|
||||||
errInvalidPort, maxPort, value)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return int(port), nil
|
return int(port), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// getTrustedProxies returns the reverse proxies the HTTP server trusts. When
|
|
||||||
// TRUSTED_PROXIES is not set, they are the private ranges of RFC 1918. When it
|
|
||||||
// is set, they are the addresses and CIDR ranges it lists, such as
|
|
||||||
// "10.0.0.0/8,192.0.2.1", and none when it is empty. An address is returned as
|
|
||||||
// a range holding only that address.
|
|
||||||
func getTrustedProxies() ([]netip.Prefix, error) {
|
|
||||||
value, set := os.LookupEnv("TRUSTED_PROXIES")
|
|
||||||
if !set {
|
|
||||||
return []netip.Prefix{
|
|
||||||
netip.MustParsePrefix(privateRange10),
|
|
||||||
netip.MustParsePrefix(privateRange172),
|
|
||||||
netip.MustParsePrefix(privateRange192),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if value == "" {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var proxies []netip.Prefix
|
|
||||||
|
|
||||||
for entry := range strings.SplitSeq(value, ",") {
|
|
||||||
entry = strings.TrimSpace(entry)
|
|
||||||
|
|
||||||
addr, err := netip.ParseAddr(entry)
|
|
||||||
if err == nil {
|
|
||||||
proxies = append(proxies, netip.PrefixFrom(addr, addr.BitLen()))
|
|
||||||
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
prefix, err := netip.ParsePrefix(entry)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w, got %q", errInvalidProxy, entry)
|
|
||||||
}
|
|
||||||
|
|
||||||
proxies = append(proxies, prefix)
|
|
||||||
}
|
|
||||||
|
|
||||||
return proxies, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// EnsureDirectories creates all necessary directories if they don't exist
|
// EnsureDirectories creates all necessary directories if they don't exist
|
||||||
func (c *Config) EnsureDirectories() error {
|
func (c *Config) EnsureDirectories() error {
|
||||||
// Ensure state directory exists
|
// Ensure state directory exists
|
||||||
err := os.MkdirAll(c.StateDir, dirPermissions)
|
if err := os.MkdirAll(c.StateDir, dirPermissions); err != nil {
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to create state directory: %w", err)
|
return fmt.Errorf("failed to create state directory: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,153 +0,0 @@
|
|||||||
package config
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/netip"
|
|
||||||
"os"
|
|
||||||
"runtime"
|
|
||||||
"slices"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestNewReadsPort(t *testing.T) {
|
|
||||||
tests := map[string]int{
|
|
||||||
"": defaultPort,
|
|
||||||
"1": 1,
|
|
||||||
"9090": 9090,
|
|
||||||
"65535": 65535,
|
|
||||||
}
|
|
||||||
|
|
||||||
for value, want := range tests {
|
|
||||||
t.Run(value, func(t *testing.T) {
|
|
||||||
t.Setenv("PORT", value)
|
|
||||||
t.Setenv("XDG_DATA_HOME", "")
|
|
||||||
|
|
||||||
cfg, err := New()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("New() with PORT=%q: %v", value, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if cfg.Port != want {
|
|
||||||
t.Errorf("New() with PORT=%q: Port = %d, want %d", value, cfg.Port, want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewRefusesInvalidPort(t *testing.T) {
|
|
||||||
for _, value := range []string{"0", "65536", "-1", "+9090", "http", "80.5"} {
|
|
||||||
t.Run(value, func(t *testing.T) {
|
|
||||||
t.Setenv("PORT", value)
|
|
||||||
t.Setenv("XDG_DATA_HOME", "")
|
|
||||||
|
|
||||||
_, err := New()
|
|
||||||
if err == nil {
|
|
||||||
t.Errorf("New() with PORT=%q returned no error", value)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestNewTrustsPrivateRangesByDefault checks that with TRUSTED_PROXIES unset
|
|
||||||
// the trusted proxies are the private ranges of RFC 1918. Before the fix no
|
|
||||||
// proxy was trusted, so a reverse proxy at a private address was taken for the
|
|
||||||
// client (https://git.eeqj.de/sneak/routewatch/issues/198).
|
|
||||||
func TestNewTrustsPrivateRangesByDefault(t *testing.T) {
|
|
||||||
t.Setenv("PORT", "")
|
|
||||||
t.Setenv("XDG_DATA_HOME", "")
|
|
||||||
|
|
||||||
// t.Setenv puts the variable back when the test ends.
|
|
||||||
t.Setenv("TRUSTED_PROXIES", "")
|
|
||||||
|
|
||||||
err := os.Unsetenv("TRUSTED_PROXIES")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg, err := New()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("New() without TRUSTED_PROXIES: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
want := []netip.Prefix{
|
|
||||||
netip.MustParsePrefix("10.0.0.0/8"),
|
|
||||||
netip.MustParsePrefix("172.16.0.0/12"),
|
|
||||||
netip.MustParsePrefix("192.168.0.0/16"),
|
|
||||||
}
|
|
||||||
if !slices.Equal(cfg.TrustedProxies, want) {
|
|
||||||
t.Errorf("New() without TRUSTED_PROXIES: TrustedProxies = %v, want %v",
|
|
||||||
cfg.TrustedProxies, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestNewReadsTrustedProxies checks that a list in TRUSTED_PROXIES replaces
|
|
||||||
// the default, and that TRUSTED_PROXIES set but empty trusts no proxy.
|
|
||||||
func TestNewReadsTrustedProxies(t *testing.T) {
|
|
||||||
t.Setenv("PORT", "")
|
|
||||||
t.Setenv("XDG_DATA_HOME", "")
|
|
||||||
|
|
||||||
t.Setenv("TRUSTED_PROXIES", "10.0.0.0/8, 192.0.2.1,2001:db8::/32")
|
|
||||||
|
|
||||||
cfg, err := New()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("New() with TRUSTED_PROXIES: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
want := []netip.Prefix{
|
|
||||||
netip.MustParsePrefix("10.0.0.0/8"),
|
|
||||||
netip.MustParsePrefix("192.0.2.1/32"),
|
|
||||||
netip.MustParsePrefix("2001:db8::/32"),
|
|
||||||
}
|
|
||||||
if !slices.Equal(cfg.TrustedProxies, want) {
|
|
||||||
t.Errorf("TrustedProxies = %v, want %v", cfg.TrustedProxies, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Setenv("TRUSTED_PROXIES", "")
|
|
||||||
|
|
||||||
cfg, err = New()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("New() with TRUSTED_PROXIES set but empty: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(cfg.TrustedProxies) != 0 {
|
|
||||||
t.Errorf("New() with TRUSTED_PROXIES set but empty: TrustedProxies = %v, "+
|
|
||||||
"want none", cfg.TrustedProxies)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewRefusesInvalidTrustedProxies(t *testing.T) {
|
|
||||||
invalid := []string{"proxy.example", "10.0.0.0/33", "192.0.2.1:80", "192.0.2.1,"}
|
|
||||||
for _, value := range invalid {
|
|
||||||
t.Run(value, func(t *testing.T) {
|
|
||||||
t.Setenv("PORT", "")
|
|
||||||
t.Setenv("XDG_DATA_HOME", "")
|
|
||||||
t.Setenv("TRUSTED_PROXIES", value)
|
|
||||||
|
|
||||||
_, err := New()
|
|
||||||
if err == nil {
|
|
||||||
t.Errorf("New() with TRUSTED_PROXIES=%q returned no error", value)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewRefusesRelativeXDGDataHome(t *testing.T) {
|
|
||||||
if runtime.GOOS == "darwin" {
|
|
||||||
t.Skip("macOS does not read XDG_DATA_HOME")
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Setenv("PORT", "")
|
|
||||||
|
|
||||||
t.Setenv("XDG_DATA_HOME", "relative/path")
|
|
||||||
|
|
||||||
_, err := New()
|
|
||||||
if err == nil {
|
|
||||||
t.Error("New() with a relative XDG_DATA_HOME returned no error")
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Setenv("XDG_DATA_HOME", "/var/lib")
|
|
||||||
|
|
||||||
_, err = New()
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("New() with XDG_DATA_HOME=/var/lib: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import (
|
||||||
|
"runtime"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNewReadsPort(t *testing.T) {
|
||||||
|
tests := map[string]int{
|
||||||
|
"": defaultPort,
|
||||||
|
"1": 1,
|
||||||
|
"9090": 9090,
|
||||||
|
"65535": 65535,
|
||||||
|
}
|
||||||
|
|
||||||
|
for value, want := range tests {
|
||||||
|
t.Run(value, func(t *testing.T) {
|
||||||
|
t.Setenv("PORT", value)
|
||||||
|
t.Setenv("XDG_DATA_HOME", "")
|
||||||
|
|
||||||
|
cfg, err := New()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("New() with PORT=%q: %v", value, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if cfg.Port != want {
|
||||||
|
t.Errorf("New() with PORT=%q: Port = %d, want %d", value, cfg.Port, want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewRefusesInvalidPort(t *testing.T) {
|
||||||
|
for _, value := range []string{"0", "65536", "-1", "+9090", "http", "80.5"} {
|
||||||
|
t.Run(value, func(t *testing.T) {
|
||||||
|
t.Setenv("PORT", value)
|
||||||
|
t.Setenv("XDG_DATA_HOME", "")
|
||||||
|
|
||||||
|
if _, err := New(); err == nil {
|
||||||
|
t.Errorf("New() with PORT=%q returned no error", value)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewRefusesRelativeXDGDataHome(t *testing.T) {
|
||||||
|
if runtime.GOOS == "darwin" {
|
||||||
|
t.Skip("macOS does not read XDG_DATA_HOME")
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Setenv("PORT", "")
|
||||||
|
|
||||||
|
t.Setenv("XDG_DATA_HOME", "relative/path")
|
||||||
|
if _, err := New(); err == nil {
|
||||||
|
t.Error("New() with a relative XDG_DATA_HOME returned no error")
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Setenv("XDG_DATA_HOME", "/var/lib")
|
||||||
|
if _, err := New(); err != nil {
|
||||||
|
t.Errorf("New() with XDG_DATA_HOME=/var/lib: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
+91
-15
@@ -6,10 +6,11 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
)
|
)
|
||||||
|
|
||||||
// liveCounts holds the running row counts that the stats endpoints report.
|
// liveCounts holds the running row counts and the prefix distribution that the
|
||||||
// They are seeded once at startup from the tables and then adjusted on every
|
// stats endpoints report. They are seeded once at startup from the tables and
|
||||||
// write, so a stats read serves them from memory instead of running a query
|
// then adjusted on every write, so a stats read serves them from memory instead
|
||||||
// over the tables. The COUNT(*) scans (issue 27) grew with the database until
|
// of running a query over the tables. The COUNT(*) scans (issue 27) and then
|
||||||
|
// the prefix distribution query (issue 30) each grew with the database until
|
||||||
// they took the whole request timeout and made /api/v1/stats return 500.
|
// they took the whole request timeout and made /api/v1/stats return 500.
|
||||||
//
|
//
|
||||||
// A single mutex guards all fields so the stats reader takes a consistent
|
// A single mutex guards all fields so the stats reader takes a consistent
|
||||||
@@ -22,11 +23,18 @@ type liveCounts struct {
|
|||||||
prefixesV6 int
|
prefixesV6 int
|
||||||
peerings int
|
peerings int
|
||||||
peers int
|
peers int
|
||||||
|
routesV4 int
|
||||||
|
routesV6 int
|
||||||
|
// The prefix distribution: for each mask length, the number of distinct
|
||||||
|
// prefixes that have at least one live route.
|
||||||
|
distributionV4 [ipv4Bits + 1]int
|
||||||
|
distributionV6 [ipv6Bits + 1]int
|
||||||
}
|
}
|
||||||
|
|
||||||
// seed sets every count to the value read from the tables at startup. It runs
|
// seed sets every count to the value read from the tables at startup. It runs
|
||||||
// before any writer, so it needs no coordination with the adjust methods.
|
// before any writer, so it needs no coordination with the adjust methods.
|
||||||
func (c *liveCounts) seed(asns, prefixesV4, prefixesV6, peerings, peers int) {
|
func (c *liveCounts) seed(asns, prefixesV4, prefixesV6, peerings, peers, routesV4, routesV6 int,
|
||||||
|
distributionV4, distributionV6 []PrefixDistribution) {
|
||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
defer c.mu.Unlock()
|
defer c.mu.Unlock()
|
||||||
|
|
||||||
@@ -35,6 +43,14 @@ func (c *liveCounts) seed(asns, prefixesV4, prefixesV6, peerings, peers int) {
|
|||||||
c.prefixesV6 = prefixesV6
|
c.prefixesV6 = prefixesV6
|
||||||
c.peerings = peerings
|
c.peerings = peerings
|
||||||
c.peers = peers
|
c.peers = peers
|
||||||
|
c.routesV4 = routesV4
|
||||||
|
c.routesV6 = routesV6
|
||||||
|
for _, entry := range distributionV4 {
|
||||||
|
addAtMaskLength(c.distributionV4[:], entry.MaskLength, entry.Count)
|
||||||
|
}
|
||||||
|
for _, entry := range distributionV6 {
|
||||||
|
addAtMaskLength(c.distributionV6[:], entry.MaskLength, entry.Count)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// addASNs adds n to the ASN count.
|
// addASNs adds n to the ASN count.
|
||||||
@@ -66,7 +82,40 @@ func (c *liveCounts) addPeers(n int) {
|
|||||||
c.mu.Unlock()
|
c.mu.Unlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
// fill copies the counts into a Stats, including the derived total, under a
|
// addRoutes adds to the IPv4 and IPv6 live-route counts. Deletions pass
|
||||||
|
// negative values.
|
||||||
|
func (c *liveCounts) addRoutes(v4, v6 int) {
|
||||||
|
c.mu.Lock()
|
||||||
|
c.routesV4 += v4
|
||||||
|
c.routesV6 += v6
|
||||||
|
c.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// addToDistribution adds n to the IPv4 and IPv6 prefix distributions once for
|
||||||
|
// each listed mask length. A write lists the mask lengths of the prefixes it
|
||||||
|
// gave their first live route with n = 1, and of the prefixes it left with no
|
||||||
|
// live route with n = -1.
|
||||||
|
func (c *liveCounts) addToDistribution(maskLengthsV4, maskLengthsV6 []int, n int) {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
|
||||||
|
for _, maskLength := range maskLengthsV4 {
|
||||||
|
addAtMaskLength(c.distributionV4[:], maskLength, n)
|
||||||
|
}
|
||||||
|
for _, maskLength := range maskLengthsV6 {
|
||||||
|
addAtMaskLength(c.distributionV6[:], maskLength, n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// addAtMaskLength adds n to counts[maskLength]. A mask length the array has no
|
||||||
|
// entry for is ignored, so a malformed route cannot crash the daemon.
|
||||||
|
func addAtMaskLength(counts []int, maskLength, n int) {
|
||||||
|
if maskLength >= 0 && maskLength < len(counts) {
|
||||||
|
counts[maskLength] += n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// fill copies the counts into a Stats, including the derived totals, under a
|
||||||
// single read lock so the reader sees one consistent snapshot.
|
// single read lock so the reader sees one consistent snapshot.
|
||||||
func (c *liveCounts) fill(s *Stats) {
|
func (c *liveCounts) fill(s *Stats) {
|
||||||
c.mu.RLock()
|
c.mu.RLock()
|
||||||
@@ -78,6 +127,24 @@ func (c *liveCounts) fill(s *Stats) {
|
|||||||
s.Prefixes = c.prefixesV4 + c.prefixesV6
|
s.Prefixes = c.prefixesV4 + c.prefixesV6
|
||||||
s.Peerings = c.peerings
|
s.Peerings = c.peerings
|
||||||
s.Peers = c.peers
|
s.Peers = c.peers
|
||||||
|
s.IPv4Routes = c.routesV4
|
||||||
|
s.IPv6Routes = c.routesV6
|
||||||
|
s.LiveRoutes = c.routesV4 + c.routesV6
|
||||||
|
s.IPv4PrefixDistribution = distributionList(c.distributionV4[:])
|
||||||
|
s.IPv6PrefixDistribution = distributionList(c.distributionV6[:])
|
||||||
|
}
|
||||||
|
|
||||||
|
// distributionList lists the mask lengths that have at least one prefix, in
|
||||||
|
// ascending order, the way the distribution query returns them.
|
||||||
|
func distributionList(counts []int) []PrefixDistribution {
|
||||||
|
var list []PrefixDistribution
|
||||||
|
for maskLength, count := range counts {
|
||||||
|
if count > 0 {
|
||||||
|
list = append(list, PrefixDistribution{MaskLength: maskLength, Count: count})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return list
|
||||||
}
|
}
|
||||||
|
|
||||||
// countRows returns the number of rows in the named table. It is used only at
|
// countRows returns the number of rows in the named table. It is used only at
|
||||||
@@ -85,43 +152,52 @@ func (c *liveCounts) fill(s *Stats) {
|
|||||||
func (d *Database) countRows(ctx context.Context, table string) (int, error) {
|
func (d *Database) countRows(ctx context.Context, table string) (int, error) {
|
||||||
var n int
|
var n int
|
||||||
// table is one of a fixed set of literals below, never external input.
|
// table is one of a fixed set of literals below, never external input.
|
||||||
err := d.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM "+table).Scan(&n)
|
if err := d.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM "+table).Scan(&n); err != nil {
|
||||||
if err != nil {
|
|
||||||
return 0, fmt.Errorf("failed to count %s: %w", table, err)
|
return 0, fmt.Errorf("failed to count %s: %w", table, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return n, nil
|
return n, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// seedCounts reads the current row counts from the tables into the in-memory
|
// seedCounts reads the current row counts and prefix distribution from the
|
||||||
// counters. It runs once at startup, before the streamer begins writing.
|
// tables into the in-memory counters. It runs once at startup, before the
|
||||||
|
// streamer begins writing.
|
||||||
func (d *Database) seedCounts(ctx context.Context) error {
|
func (d *Database) seedCounts(ctx context.Context) error {
|
||||||
asns, err := d.countRows(ctx, "asns")
|
asns, err := d.countRows(ctx, "asns")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
prefixesV4, err := d.countRows(ctx, "prefixes_v4")
|
prefixesV4, err := d.countRows(ctx, "prefixes_v4")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
prefixesV6, err := d.countRows(ctx, "prefixes_v6")
|
prefixesV6, err := d.countRows(ctx, "prefixes_v6")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
peerings, err := d.countRows(ctx, "peerings")
|
peerings, err := d.countRows(ctx, "peerings")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
peers, err := d.countRows(ctx, "bgp_peers")
|
peers, err := d.countRows(ctx, "bgp_peers")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
routesV4, err := d.countRows(ctx, "live_routes_v4")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
routesV6, err := d.countRows(ctx, "live_routes_v6")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
distributionV4, distributionV6, err := d.GetPrefixDistributionContext(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
d.counts.seed(asns, prefixesV4, prefixesV6, peerings, peers)
|
d.counts.seed(asns, prefixesV4, prefixesV6, peerings, peers, routesV4, routesV6,
|
||||||
|
distributionV4, distributionV6)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,235 +0,0 @@
|
|||||||
package database
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/config"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
|
||||||
)
|
|
||||||
|
|
||||||
// testV4PeerIP is the peer the IPv4 rows in these tests are learned from.
|
|
||||||
const testV4PeerIP = "192.0.2.1"
|
|
||||||
|
|
||||||
// testV6Prefix is the IPv6 prefix of these tests.
|
|
||||||
const testV6Prefix = "2001:db8::/32"
|
|
||||||
|
|
||||||
// TestLiveCountsTrackWritesInRealtime checks that the stats counts start at
|
|
||||||
// zero and reflect each write the moment it commits (no recompute, no timer).
|
|
||||||
// These counts are what /api/v1/stats reports; before this change the
|
|
||||||
// endpoint recomputed them with a COUNT(*) over each table on every request.
|
|
||||||
func TestLiveCountsTrackWritesInRealtime(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
cfg := &config.Config{StateDir: t.TempDir()}
|
|
||||||
|
|
||||||
db, err := New(cfg, logger.New())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to create database: %v", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = db.Close() }()
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
empty, err := db.GetStatsContext(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GetStatsContext on empty database: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if empty.ASNs != 0 || empty.Prefixes != 0 || empty.Peerings != 0 ||
|
|
||||||
empty.Peers != 0 {
|
|
||||||
t.Fatalf("empty database counts nonzero: %+v", empty)
|
|
||||||
}
|
|
||||||
|
|
||||||
ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
|
||||||
|
|
||||||
writeASNsPrefixesAndPeers(t, db, ts)
|
|
||||||
|
|
||||||
stats, err := db.GetStatsContext(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GetStatsContext: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
assertCounts(t, "after inserts", stats, wantCounts{
|
|
||||||
asns: 2, prefixes: 2, peerings: 1, peers: 1,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeASNsPrefixesAndPeers writes two ASNs, two prefixes, one peer and one
|
|
||||||
// peering: the rows the stats count.
|
|
||||||
func writeASNsPrefixesAndPeers(t *testing.T, db *Database, ts time.Time) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
seen := SeenTimes{First: ts, Last: ts}
|
|
||||||
|
|
||||||
err := db.GetOrCreateASNBatch(map[int]SeenTimes{64500: seen, 64501: seen})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GetOrCreateASNBatch: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = db.UpdatePrefixesBatch(map[string]time.Time{
|
|
||||||
"198.51.100.0/24": ts,
|
|
||||||
testV6Prefix: ts,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("UpdatePrefixesBatch: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = db.UpdatePeerBatch(map[string]PeerUpdate{
|
|
||||||
testV4PeerIP: {
|
|
||||||
PeerIP: testV4PeerIP, PeerASN: 64500,
|
|
||||||
MessageType: "UPDATE", Seen: seen,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("UpdatePeerBatch: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = db.RecordPeering(64500, 64501, seen)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("RecordPeering: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLiveCountsSeededFromDatabaseAtStartup writes rows, reopens the same
|
|
||||||
// database file, and checks the counts come back from the seed scan rather
|
|
||||||
// than starting at zero.
|
|
||||||
func TestLiveCountsSeededFromDatabaseAtStartup(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
cfg := &config.Config{StateDir: t.TempDir()}
|
|
||||||
|
|
||||||
db, err := New(cfg, logger.New())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to create database: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
writeASNsPrefixesAndPeers(t, db, time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC))
|
|
||||||
|
|
||||||
err = db.Close()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Close: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
reopened, err := New(cfg, logger.New())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to reopen database: %v", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = reopened.Close() }()
|
|
||||||
|
|
||||||
stats, err := reopened.GetStatsContext(context.Background())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GetStatsContext after reopen: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
assertCounts(t, "after reopen", stats, wantCounts{
|
|
||||||
asns: 2, prefixes: 2, peerings: 1, peers: 1,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLiveCountsConcurrentReadWrite runs writers and stats readers at once so
|
|
||||||
// the race detector proves the counters are safe under concurrent use.
|
|
||||||
func TestLiveCountsConcurrentReadWrite(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
cfg := &config.Config{StateDir: t.TempDir()}
|
|
||||||
|
|
||||||
db, err := New(cfg, logger.New())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to create database: %v", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = db.Close() }()
|
|
||||||
|
|
||||||
ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
|
||||||
|
|
||||||
const (
|
|
||||||
writers = 4
|
|
||||||
prefixesPerWrite = 25
|
|
||||||
)
|
|
||||||
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
wg.Add(writers)
|
|
||||||
|
|
||||||
for w := range writers {
|
|
||||||
go func(base int) {
|
|
||||||
defer wg.Done()
|
|
||||||
|
|
||||||
for i := range prefixesPerWrite {
|
|
||||||
prefix := fmt.Sprintf("10.%d.%d.0/24", base, i)
|
|
||||||
|
|
||||||
err := db.UpdatePrefixesBatch(map[string]time.Time{prefix: ts})
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("UpdatePrefixesBatch: %v", err)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}(w)
|
|
||||||
}
|
|
||||||
|
|
||||||
var readerWG sync.WaitGroup
|
|
||||||
readerWG.Add(1)
|
|
||||||
|
|
||||||
stop := make(chan struct{})
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
defer readerWG.Done()
|
|
||||||
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-stop:
|
|
||||||
return
|
|
||||||
default:
|
|
||||||
_, err := db.GetStatsContext(context.Background())
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("GetStatsContext: %v", err)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
wg.Wait()
|
|
||||||
close(stop)
|
|
||||||
readerWG.Wait()
|
|
||||||
|
|
||||||
stats, err := db.GetStatsContext(context.Background())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GetStatsContext: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if want := writers * prefixesPerWrite; stats.IPv4Prefixes != want {
|
|
||||||
t.Errorf("IPv4Prefixes = %d, want %d", stats.IPv4Prefixes, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type wantCounts struct {
|
|
||||||
asns int
|
|
||||||
prefixes int
|
|
||||||
peerings int
|
|
||||||
peers int
|
|
||||||
}
|
|
||||||
|
|
||||||
func assertCounts(t *testing.T, when string, got Stats, want wantCounts) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
if got.ASNs != want.asns {
|
|
||||||
t.Errorf("%s: ASNs = %d, want %d", when, got.ASNs, want.asns)
|
|
||||||
}
|
|
||||||
|
|
||||||
if got.Prefixes != want.prefixes {
|
|
||||||
t.Errorf("%s: Prefixes = %d, want %d", when, got.Prefixes, want.prefixes)
|
|
||||||
}
|
|
||||||
|
|
||||||
if got.Peerings != want.peerings {
|
|
||||||
t.Errorf("%s: Peerings = %d, want %d", when, got.Peerings, want.peerings)
|
|
||||||
}
|
|
||||||
|
|
||||||
if got.Peers != want.peers {
|
|
||||||
t.Errorf("%s: Peers = %d, want %d", when, got.Peers, want.peers)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,511 @@
|
|||||||
|
package database
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"slices"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.eeqj.de/sneak/routewatch/internal/config"
|
||||||
|
"git.eeqj.de/sneak/routewatch/internal/logger"
|
||||||
|
"github.com/google/uuid"
|
||||||
|
)
|
||||||
|
|
||||||
|
// mkV4Route builds an IPv4 live route with its mask length taken from the
|
||||||
|
// prefix.
|
||||||
|
func mkV4Route(t *testing.T, prefix string, asn int, ts time.Time) *LiveRoute {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
maskLength, err := prefixMaskLength(prefix)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prefixMaskLength(%s): %v", prefix, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &LiveRoute{
|
||||||
|
ID: uuid.New(),
|
||||||
|
Prefix: prefix,
|
||||||
|
MaskLength: maskLength,
|
||||||
|
IPVersion: ipVersionV4,
|
||||||
|
OriginASN: asn,
|
||||||
|
PeerIP: "192.0.2.1",
|
||||||
|
ASPath: []int{asn},
|
||||||
|
NextHop: "192.0.2.254",
|
||||||
|
LastUpdated: ts,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// mkV6Route builds an IPv6 live route.
|
||||||
|
func mkV6Route(prefix string, asn int, ts time.Time) *LiveRoute {
|
||||||
|
return &LiveRoute{
|
||||||
|
ID: uuid.New(),
|
||||||
|
Prefix: prefix,
|
||||||
|
MaskLength: 32,
|
||||||
|
IPVersion: ipVersionV6,
|
||||||
|
OriginASN: asn,
|
||||||
|
PeerIP: "2001:db8::1",
|
||||||
|
ASPath: []int{asn},
|
||||||
|
NextHop: "2001:db8::ffff",
|
||||||
|
LastUpdated: ts,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestLiveCountsTrackWritesInRealtime checks that the stats counts start at
|
||||||
|
// zero, reflect each write the moment it commits (no recompute, no timer), do
|
||||||
|
// not move when a route is merely re-announced, and drop when a route is
|
||||||
|
// deleted. These counts are what /api/v1/stats reports; before this change the
|
||||||
|
// endpoint recomputed them with a COUNT(*) over each table on every request.
|
||||||
|
func TestLiveCountsTrackWritesInRealtime(t *testing.T) {
|
||||||
|
cfg := &config.Config{StateDir: t.TempDir()}
|
||||||
|
|
||||||
|
db, err := New(cfg, logger.New())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to create database: %v", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = db.Close() }()
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
empty, err := db.GetStatsContext(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetStatsContext on empty database: %v", err)
|
||||||
|
}
|
||||||
|
if empty.ASNs != 0 || empty.Prefixes != 0 || empty.Peerings != 0 ||
|
||||||
|
empty.Peers != 0 || empty.LiveRoutes != 0 {
|
||||||
|
t.Fatalf("empty database counts nonzero: %+v", empty)
|
||||||
|
}
|
||||||
|
|
||||||
|
ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
||||||
|
|
||||||
|
if err := db.GetOrCreateASNBatch(map[int]time.Time{64500: ts, 64501: ts}); err != nil {
|
||||||
|
t.Fatalf("GetOrCreateASNBatch: %v", err)
|
||||||
|
}
|
||||||
|
if err := db.UpdatePrefixesBatch(map[string]time.Time{
|
||||||
|
"198.51.100.0/24": ts,
|
||||||
|
"2001:db8::/32": ts,
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("UpdatePrefixesBatch: %v", err)
|
||||||
|
}
|
||||||
|
if err := db.UpdatePeerBatch(map[string]PeerUpdate{
|
||||||
|
"192.0.2.1": {PeerIP: "192.0.2.1", PeerASN: 64500, MessageType: "UPDATE", Timestamp: ts},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("UpdatePeerBatch: %v", err)
|
||||||
|
}
|
||||||
|
if err := db.RecordPeering(64500, 64501, ts); err != nil {
|
||||||
|
t.Fatalf("RecordPeering: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
routes := []*LiveRoute{
|
||||||
|
mkV4Route(t, "198.51.100.0/24", 64500, ts),
|
||||||
|
mkV4Route(t, "203.0.113.0/24", 64501, ts.Add(time.Minute)),
|
||||||
|
mkV6Route("2001:db8::/32", 64502, ts.Add(2*time.Minute)),
|
||||||
|
}
|
||||||
|
if err := db.UpsertLiveRouteBatch(routes); err != nil {
|
||||||
|
t.Fatalf("UpsertLiveRouteBatch: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
stats, err := db.GetStatsContext(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetStatsContext: %v", err)
|
||||||
|
}
|
||||||
|
assertCounts(t, "after inserts", stats, wantCounts{
|
||||||
|
asns: 2, prefixes: 2, peerings: 1, peers: 1,
|
||||||
|
ipv4Routes: 2, ipv6Routes: 1, liveRoutes: 3,
|
||||||
|
})
|
||||||
|
|
||||||
|
// Re-announcing the same routes is an update, not an insert: counts hold.
|
||||||
|
if err := db.UpsertLiveRouteBatch(routes); err != nil {
|
||||||
|
t.Fatalf("UpsertLiveRouteBatch (re-announce): %v", err)
|
||||||
|
}
|
||||||
|
stats, err = db.GetStatsContext(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetStatsContext: %v", err)
|
||||||
|
}
|
||||||
|
assertCounts(t, "after re-announce", stats, wantCounts{
|
||||||
|
asns: 2, prefixes: 2, peerings: 1, peers: 1,
|
||||||
|
ipv4Routes: 2, ipv6Routes: 1, liveRoutes: 3,
|
||||||
|
})
|
||||||
|
|
||||||
|
// A withdrawal removes one route.
|
||||||
|
if err := db.DeleteLiveRouteBatch([]LiveRouteDeletion{
|
||||||
|
{Prefix: "203.0.113.0/24", OriginASN: 64501, PeerIP: "192.0.2.1", IPVersion: ipVersionV4},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("DeleteLiveRouteBatch: %v", err)
|
||||||
|
}
|
||||||
|
stats, err = db.GetStatsContext(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetStatsContext: %v", err)
|
||||||
|
}
|
||||||
|
assertCounts(t, "after delete", stats, wantCounts{
|
||||||
|
asns: 2, prefixes: 2, peerings: 1, peers: 1,
|
||||||
|
ipv4Routes: 1, ipv6Routes: 1, liveRoutes: 2,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestLiveCountsSeededFromDatabaseAtStartup writes rows, reopens the same
|
||||||
|
// database file, and checks the counts and the prefix distribution come back
|
||||||
|
// from the seed scan rather than starting at zero.
|
||||||
|
func TestLiveCountsSeededFromDatabaseAtStartup(t *testing.T) {
|
||||||
|
cfg := &config.Config{StateDir: t.TempDir()}
|
||||||
|
|
||||||
|
db, err := New(cfg, logger.New())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to create database: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
||||||
|
if err := db.GetOrCreateASNBatch(map[int]time.Time{64500: ts, 64501: ts, 64502: ts}); err != nil {
|
||||||
|
t.Fatalf("GetOrCreateASNBatch: %v", err)
|
||||||
|
}
|
||||||
|
if err := db.UpsertLiveRouteBatch([]*LiveRoute{
|
||||||
|
mkV4Route(t, "198.51.100.0/24", 64500, ts),
|
||||||
|
mkV6Route("2001:db8::/32", 64502, ts),
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("UpsertLiveRouteBatch: %v", err)
|
||||||
|
}
|
||||||
|
if err := db.Close(); err != nil {
|
||||||
|
t.Fatalf("Close: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
reopened, err := New(cfg, logger.New())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to reopen database: %v", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = reopened.Close() }()
|
||||||
|
|
||||||
|
stats, err := reopened.GetStatsContext(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetStatsContext after reopen: %v", err)
|
||||||
|
}
|
||||||
|
if stats.ASNs != 3 {
|
||||||
|
t.Errorf("seeded ASNs = %d, want 3", stats.ASNs)
|
||||||
|
}
|
||||||
|
if stats.IPv4Routes != 1 || stats.IPv6Routes != 1 || stats.LiveRoutes != 2 {
|
||||||
|
t.Errorf("seeded routes = (v4 %d, v6 %d, total %d), want (1, 1, 2)",
|
||||||
|
stats.IPv4Routes, stats.IPv6Routes, stats.LiveRoutes)
|
||||||
|
}
|
||||||
|
assertDistribution(t, "seeded IPv4 distribution", stats.IPv4PrefixDistribution,
|
||||||
|
[]PrefixDistribution{{MaskLength: 24, Count: 1}})
|
||||||
|
assertDistribution(t, "seeded IPv6 distribution", stats.IPv6PrefixDistribution,
|
||||||
|
[]PrefixDistribution{{MaskLength: 32, Count: 1}})
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestPrefixDistributionTracksWrites checks that the prefix distribution the
|
||||||
|
// stats read reports stays exact across each kind of live-route write, and that
|
||||||
|
// after every step it equals what the distribution query reads from the route
|
||||||
|
// tables. The steps run once through the batch methods the prefix handler uses
|
||||||
|
// and once through the single-route methods.
|
||||||
|
func TestPrefixDistributionTracksWrites(t *testing.T) {
|
||||||
|
ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
||||||
|
shared := mkV4Route(t, "198.51.100.0/24", 64500, ts)
|
||||||
|
sharedSecondPeer := mkV4Route(t, "198.51.100.0/24", 64500, ts)
|
||||||
|
sharedSecondPeer.PeerIP = "192.0.2.2"
|
||||||
|
other := mkV4Route(t, "203.0.113.0/24", 64501, ts)
|
||||||
|
wide := mkV4Route(t, "172.16.0.0/16", 64502, ts)
|
||||||
|
v6 := mkV6Route("2001:db8::/32", 64503, ts)
|
||||||
|
v6SecondPeer := mkV6Route("2001:db8::/32", 64503, ts)
|
||||||
|
v6SecondPeer.PeerIP = "2001:db8::2"
|
||||||
|
|
||||||
|
all := []PrefixDistribution{{MaskLength: 16, Count: 1}, {MaskLength: 24, Count: 2}}
|
||||||
|
v6Only := []PrefixDistribution{{MaskLength: 32, Count: 1}}
|
||||||
|
|
||||||
|
steps := []struct {
|
||||||
|
name string
|
||||||
|
announce []*LiveRoute
|
||||||
|
withdraw []*LiveRoute
|
||||||
|
wantV4 []PrefixDistribution
|
||||||
|
wantV6 []PrefixDistribution
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "new routes",
|
||||||
|
announce: []*LiveRoute{shared, other, wide, v6},
|
||||||
|
wantV4: all, wantV6: v6Only,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "re-announcement",
|
||||||
|
announce: []*LiveRoute{shared, other, wide, v6},
|
||||||
|
wantV4: all, wantV6: v6Only,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "second peer announces a prefix that has a live route",
|
||||||
|
announce: []*LiveRoute{sharedSecondPeer, v6SecondPeer},
|
||||||
|
wantV4: all, wantV6: v6Only,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "withdrawal of a route that is not the last for its prefix",
|
||||||
|
withdraw: []*LiveRoute{shared, v6},
|
||||||
|
wantV4: all, wantV6: v6Only,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "withdrawal of the last route for a prefix",
|
||||||
|
withdraw: []*LiveRoute{sharedSecondPeer, v6SecondPeer},
|
||||||
|
wantV4: []PrefixDistribution{{MaskLength: 16, Count: 1}, {MaskLength: 24, Count: 1}},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "withdrawal of every remaining route",
|
||||||
|
withdraw: []*LiveRoute{other, wide},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "two peers announce a new prefix together",
|
||||||
|
announce: []*LiveRoute{shared, sharedSecondPeer, v6, v6SecondPeer},
|
||||||
|
wantV4: []PrefixDistribution{{MaskLength: 24, Count: 1}},
|
||||||
|
wantV6: v6Only,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "both routes for a prefix withdrawn together",
|
||||||
|
withdraw: []*LiveRoute{shared, sharedSecondPeer, v6, v6SecondPeer},
|
||||||
|
},
|
||||||
|
// The feed often withdraws a route that is not live. That must not take
|
||||||
|
// the prefix out of the distribution. A count wrongly taken below zero is
|
||||||
|
// left out of the answer, so the next step shows it: its announcement
|
||||||
|
// at the same mask length would then not be counted.
|
||||||
|
{
|
||||||
|
name: "withdrawal of routes that are not live",
|
||||||
|
withdraw: []*LiveRoute{other, v6},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "announcement after a withdrawal of routes that are not live",
|
||||||
|
announce: []*LiveRoute{other, v6},
|
||||||
|
wantV4: []PrefixDistribution{{MaskLength: 24, Count: 1}},
|
||||||
|
wantV6: v6Only,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, batch := range []bool{true, false} {
|
||||||
|
name := "single-route writes"
|
||||||
|
if batch {
|
||||||
|
name = "batch writes"
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to create database: %v", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = db.Close() }()
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
for _, step := range steps {
|
||||||
|
if err := announceRoutes(db, batch, step.announce); err != nil {
|
||||||
|
t.Fatalf("%s: announce: %v", step.name, err)
|
||||||
|
}
|
||||||
|
if err := withdrawRoutes(db, batch, step.withdraw); err != nil {
|
||||||
|
t.Fatalf("%s: withdraw: %v", step.name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
stats, err := db.GetStatsContext(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: GetStatsContext: %v", step.name, err)
|
||||||
|
}
|
||||||
|
queryV4, queryV6, err := db.GetPrefixDistributionContext(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: GetPrefixDistributionContext: %v", step.name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertDistribution(t, step.name+": IPv4 distribution", stats.IPv4PrefixDistribution, step.wantV4)
|
||||||
|
assertDistribution(t, step.name+": IPv6 distribution", stats.IPv6PrefixDistribution, step.wantV6)
|
||||||
|
assertDistribution(t, step.name+": IPv4 distribution query", queryV4, step.wantV4)
|
||||||
|
assertDistribution(t, step.name+": IPv6 distribution query", queryV6, step.wantV6)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// announceRoutes writes routes in one UpsertLiveRouteBatch, or with one
|
||||||
|
// UpsertLiveRoute each.
|
||||||
|
func announceRoutes(db *Database, batch bool, routes []*LiveRoute) error {
|
||||||
|
if batch {
|
||||||
|
return db.UpsertLiveRouteBatch(routes)
|
||||||
|
}
|
||||||
|
for _, route := range routes {
|
||||||
|
if err := db.UpsertLiveRoute(route); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// withdrawRoutes removes routes in one DeleteLiveRouteBatch, or with one
|
||||||
|
// DeleteLiveRoute each. It names each route by prefix and peer only, with no
|
||||||
|
// origin ASN, as a withdrawal from the feed does when its message carries no AS
|
||||||
|
// path.
|
||||||
|
func withdrawRoutes(db *Database, batch bool, routes []*LiveRoute) error {
|
||||||
|
if !batch {
|
||||||
|
for _, route := range routes {
|
||||||
|
if err := db.DeleteLiveRoute(route.Prefix, 0, route.PeerIP); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
deletions := make([]LiveRouteDeletion, 0, len(routes))
|
||||||
|
for _, route := range routes {
|
||||||
|
deletions = append(deletions, LiveRouteDeletion{
|
||||||
|
Prefix: route.Prefix, PeerIP: route.PeerIP, IPVersion: route.IPVersion,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return db.DeleteLiveRouteBatch(deletions)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestStatsRouteTimestamps checks the oldest/newest route timestamps are read
|
||||||
|
// from the right rows across both tables and parse into time.Time. The old
|
||||||
|
// MIN/MAX union query read its result into *time.Time, which the driver could
|
||||||
|
// not parse, so it logged a warning every call and left both timestamps nil.
|
||||||
|
func TestStatsRouteTimestamps(t *testing.T) {
|
||||||
|
cfg := &config.Config{StateDir: t.TempDir()}
|
||||||
|
|
||||||
|
db, err := New(cfg, logger.New())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to create database: %v", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = db.Close() }()
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
empty, err := db.GetStatsContext(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetStatsContext on empty database: %v", err)
|
||||||
|
}
|
||||||
|
if empty.OldestRoute != nil || empty.NewestRoute != nil {
|
||||||
|
t.Fatalf("empty database timestamps = (%v, %v), want (nil, nil)",
|
||||||
|
empty.OldestRoute, empty.NewestRoute)
|
||||||
|
}
|
||||||
|
|
||||||
|
base := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
||||||
|
oldest := base
|
||||||
|
newest := base.Add(2 * time.Minute)
|
||||||
|
if err := db.UpsertLiveRouteBatch([]*LiveRoute{
|
||||||
|
mkV4Route(t, "198.51.100.0/24", 64500, base.Add(time.Minute)),
|
||||||
|
mkV4Route(t, "203.0.113.0/24", 64501, oldest),
|
||||||
|
mkV6Route("2001:db8::/32", 64502, newest),
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("UpsertLiveRouteBatch: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
stats, err := db.GetStatsContext(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetStatsContext: %v", err)
|
||||||
|
}
|
||||||
|
if stats.OldestRoute == nil || !stats.OldestRoute.Equal(oldest) {
|
||||||
|
t.Errorf("OldestRoute = %v, want %v", stats.OldestRoute, oldest)
|
||||||
|
}
|
||||||
|
if stats.NewestRoute == nil || !stats.NewestRoute.Equal(newest) {
|
||||||
|
t.Errorf("NewestRoute = %v, want %v", stats.NewestRoute, newest)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestLiveCountsConcurrentReadWrite runs writers and stats readers at once so
|
||||||
|
// the race detector proves the counters are safe under concurrent use.
|
||||||
|
func TestLiveCountsConcurrentReadWrite(t *testing.T) {
|
||||||
|
cfg := &config.Config{StateDir: t.TempDir()}
|
||||||
|
|
||||||
|
db, err := New(cfg, logger.New())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to create database: %v", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = db.Close() }()
|
||||||
|
|
||||||
|
ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
||||||
|
|
||||||
|
const writers = 4
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
|
||||||
|
wg.Add(writers)
|
||||||
|
for w := range writers {
|
||||||
|
go func(base int) {
|
||||||
|
defer wg.Done()
|
||||||
|
for i := range 25 {
|
||||||
|
asn := 65000 + base*100 + i
|
||||||
|
route := mkV6Route("2001:db8::/32", asn, ts)
|
||||||
|
route.PeerIP = "2001:db8::" + uuid.NewString()
|
||||||
|
if err := db.UpsertLiveRoute(route); err != nil {
|
||||||
|
t.Errorf("UpsertLiveRoute: %v", err)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}(w)
|
||||||
|
}
|
||||||
|
|
||||||
|
var readerWG sync.WaitGroup
|
||||||
|
readerWG.Add(1)
|
||||||
|
stop := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
defer readerWG.Done()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-stop:
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
if _, err := db.GetStatsContext(context.Background()); err != nil {
|
||||||
|
t.Errorf("GetStatsContext: %v", err)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
wg.Wait()
|
||||||
|
close(stop)
|
||||||
|
readerWG.Wait()
|
||||||
|
|
||||||
|
stats, err := db.GetStatsContext(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetStatsContext: %v", err)
|
||||||
|
}
|
||||||
|
if want := writers * 25; stats.IPv6Routes != want {
|
||||||
|
t.Errorf("IPv6Routes = %d, want %d", stats.IPv6Routes, want)
|
||||||
|
}
|
||||||
|
// Every writer announced the same prefix, so it counts once.
|
||||||
|
assertDistribution(t, "IPv6 distribution", stats.IPv6PrefixDistribution,
|
||||||
|
[]PrefixDistribution{{MaskLength: 32, Count: 1}})
|
||||||
|
}
|
||||||
|
|
||||||
|
type wantCounts struct {
|
||||||
|
asns int
|
||||||
|
prefixes int
|
||||||
|
peerings int
|
||||||
|
peers int
|
||||||
|
ipv4Routes int
|
||||||
|
ipv6Routes int
|
||||||
|
liveRoutes int
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertCounts(t *testing.T, when string, got Stats, want wantCounts) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
if got.ASNs != want.asns {
|
||||||
|
t.Errorf("%s: ASNs = %d, want %d", when, got.ASNs, want.asns)
|
||||||
|
}
|
||||||
|
if got.Prefixes != want.prefixes {
|
||||||
|
t.Errorf("%s: Prefixes = %d, want %d", when, got.Prefixes, want.prefixes)
|
||||||
|
}
|
||||||
|
if got.Peerings != want.peerings {
|
||||||
|
t.Errorf("%s: Peerings = %d, want %d", when, got.Peerings, want.peerings)
|
||||||
|
}
|
||||||
|
if got.Peers != want.peers {
|
||||||
|
t.Errorf("%s: Peers = %d, want %d", when, got.Peers, want.peers)
|
||||||
|
}
|
||||||
|
if got.IPv4Routes != want.ipv4Routes {
|
||||||
|
t.Errorf("%s: IPv4Routes = %d, want %d", when, got.IPv4Routes, want.ipv4Routes)
|
||||||
|
}
|
||||||
|
if got.IPv6Routes != want.ipv6Routes {
|
||||||
|
t.Errorf("%s: IPv6Routes = %d, want %d", when, got.IPv6Routes, want.ipv6Routes)
|
||||||
|
}
|
||||||
|
if got.LiveRoutes != want.liveRoutes {
|
||||||
|
t.Errorf("%s: LiveRoutes = %d, want %d", when, got.LiveRoutes, want.liveRoutes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertDistribution(t *testing.T, what string, got, want []PrefixDistribution) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
if !slices.Equal(got, want) {
|
||||||
|
t.Errorf("%s = %v, want %v", what, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
+1368
-840
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,391 @@
|
|||||||
|
package database
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"database/sql"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net/netip"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.eeqj.de/sneak/routewatch/internal/config"
|
||||||
|
"git.eeqj.de/sneak/routewatch/internal/logger"
|
||||||
|
"github.com/google/uuid"
|
||||||
|
)
|
||||||
|
|
||||||
|
// tempStoreMemory is the PRAGMA temp_store value meaning "hold temp B-trees in
|
||||||
|
// memory"; the DSN change must leave temp_store below this so they spill to disk.
|
||||||
|
const tempStoreMemory = 2
|
||||||
|
|
||||||
|
// autoVacuumIncremental is the PRAGMA auto_vacuum value meaning "incremental".
|
||||||
|
const autoVacuumIncremental = 2
|
||||||
|
|
||||||
|
// vacuumTestRoutes is how many routes the vacuum test writes and then deletes,
|
||||||
|
// enough to leave many free pages in the file.
|
||||||
|
const vacuumTestRoutes = 2000
|
||||||
|
|
||||||
|
// heldConnections is how many pooled connections the pragma test holds open at
|
||||||
|
// once so each is a distinct SQLite connection that parsed the DSN.
|
||||||
|
const heldConnections = 5
|
||||||
|
|
||||||
|
// testPeerIP is the peer every route in the IP lookup test is learned from.
|
||||||
|
const testPeerIP = "192.0.2.254"
|
||||||
|
|
||||||
|
// Parameters for the checkpoint-contention regression test.
|
||||||
|
const (
|
||||||
|
// contentionIterations is how many batch writes race the checkpoint loop.
|
||||||
|
contentionIterations = 400
|
||||||
|
// contendedASNCount is the small set of ASNs the batches reuse, so most
|
||||||
|
// batches update existing rows and exercise the read-before-write path.
|
||||||
|
contendedASNCount = 16
|
||||||
|
// asnSecondBand offsets a second ASN per batch so each batch writes more
|
||||||
|
// than one row.
|
||||||
|
asnSecondBand = 100
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestGetIPInfoFindsMostSpecificLiveRoute stores nested live prefixes for both
|
||||||
|
// families and checks that a lookup returns the most specific one covering the
|
||||||
|
// address, ErrNoRoute when none covers it, and the next less specific prefix
|
||||||
|
// once the only route of the most specific one is withdrawn.
|
||||||
|
func TestGetIPInfoFindsMostSpecificLiveRoute(t *testing.T) {
|
||||||
|
cfg := &config.Config{StateDir: t.TempDir()}
|
||||||
|
|
||||||
|
db, err := New(cfg, logger.New())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to create database: %v", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = db.Close() }()
|
||||||
|
|
||||||
|
// Nested live prefixes, each originated by its own AS.
|
||||||
|
origins := map[string]int{
|
||||||
|
"10.0.0.0/8": 64500,
|
||||||
|
"10.1.0.0/16": 64501,
|
||||||
|
"10.1.2.0/24": 64502,
|
||||||
|
"2001:db8::/32": 64500,
|
||||||
|
"2001:db8:1::/48": 64501,
|
||||||
|
"2001:db8:1:2::/64": 64502,
|
||||||
|
}
|
||||||
|
ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
||||||
|
routes := make([]*LiveRoute, 0, len(origins))
|
||||||
|
for prefix, asn := range origins {
|
||||||
|
routes = append(routes, &LiveRoute{
|
||||||
|
ID: uuid.New(),
|
||||||
|
Prefix: prefix,
|
||||||
|
MaskLength: netip.MustParsePrefix(prefix).Bits(),
|
||||||
|
IPVersion: detectIPVersion(prefix),
|
||||||
|
OriginASN: asn,
|
||||||
|
PeerIP: testPeerIP,
|
||||||
|
ASPath: []int{asn},
|
||||||
|
NextHop: testPeerIP,
|
||||||
|
LastUpdated: ts,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if err := db.UpsertLiveRouteBatch(routes); err != nil {
|
||||||
|
t.Fatalf("UpsertLiveRouteBatch: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// lookup checks that ip resolves to the live prefix want, or to ErrNoRoute
|
||||||
|
// when want is empty.
|
||||||
|
lookup := func(ip, want string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
info, err := db.GetIPInfo(ip)
|
||||||
|
if want == "" {
|
||||||
|
if !errors.Is(err, ErrNoRoute) {
|
||||||
|
t.Errorf("GetIPInfo(%s) = %+v, %v; want ErrNoRoute", ip, info, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("GetIPInfo(%s): %v", ip, err)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if info.Netblock != want || info.MaskLength != netip.MustParsePrefix(want).Bits() ||
|
||||||
|
info.ASN != origins[want] {
|
||||||
|
t.Errorf("GetIPInfo(%s) = %s (mask %d) AS%d, want %s AS%d",
|
||||||
|
ip, info.Netblock, info.MaskLength, info.ASN, want, origins[want])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
lookup("10.1.2.3", "10.1.2.0/24")
|
||||||
|
lookup("::ffff:10.1.2.3", "10.1.2.0/24")
|
||||||
|
lookup("10.1.3.4", "10.1.0.0/16")
|
||||||
|
lookup("10.2.0.1", "10.0.0.0/8")
|
||||||
|
lookup("192.0.2.1", "")
|
||||||
|
lookup("2001:db8:1:2::3", "2001:db8:1:2::/64")
|
||||||
|
lookup("2001:db8:1:3::4", "2001:db8:1::/48")
|
||||||
|
lookup("2001:db8:2::1", "2001:db8::/32")
|
||||||
|
lookup("2001:db9::1", "")
|
||||||
|
|
||||||
|
err = db.DeleteLiveRouteBatch([]LiveRouteDeletion{
|
||||||
|
{Prefix: "10.1.2.0/24", OriginASN: 64502, PeerIP: testPeerIP, IPVersion: ipVersionV4},
|
||||||
|
{Prefix: "2001:db8:1:2::/64", OriginASN: 64502, PeerIP: testPeerIP, IPVersion: ipVersionV6},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("DeleteLiveRouteBatch: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
lookup("10.1.2.3", "10.1.0.0/16")
|
||||||
|
lookup("2001:db8:1:2::3", "2001:db8:1::/48")
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestGetIPInfoIgnoresShortRoutes checks that the IP lookup does not report a
|
||||||
|
// default route, or a route shorter than /8 for IPv4 or /16 for IPv6, as the
|
||||||
|
// route for an address, and that a route of exactly /8 or /16, or longer, still
|
||||||
|
// is. The prefixes are written out so that a wrong limit fails the test.
|
||||||
|
func TestGetIPInfoIgnoresShortRoutes(t *testing.T) {
|
||||||
|
cfg := &config.Config{StateDir: t.TempDir()}
|
||||||
|
|
||||||
|
db, err := New(cfg, logger.New())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to create database: %v", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = db.Close() }()
|
||||||
|
|
||||||
|
ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
||||||
|
announce := func(prefix string, asn int) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
err := db.UpsertLiveRouteBatch([]*LiveRoute{{
|
||||||
|
ID: uuid.New(),
|
||||||
|
Prefix: prefix,
|
||||||
|
MaskLength: netip.MustParsePrefix(prefix).Bits(),
|
||||||
|
IPVersion: detectIPVersion(prefix),
|
||||||
|
OriginASN: asn,
|
||||||
|
PeerIP: testPeerIP,
|
||||||
|
ASPath: []int{asn},
|
||||||
|
NextHop: testPeerIP,
|
||||||
|
LastUpdated: ts,
|
||||||
|
}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("UpsertLiveRouteBatch(%s): %v", prefix, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// lookup checks that ip resolves to the live prefix netblock from asn.
|
||||||
|
lookup := func(ip, netblock string, asn int) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
info, err := db.GetIPInfo(ip)
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("GetIPInfo(%s): %v", ip, err)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if info.Netblock != netblock || info.ASN != asn {
|
||||||
|
t.Errorf("GetIPInfo(%s) = %s AS%d, want %s AS%d", ip, info.Netblock, info.ASN, netblock, asn)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Default and very short routes, and routes one bit shorter than each limit.
|
||||||
|
announce("2000::/3", 64500)
|
||||||
|
announce("0.0.0.0/0", 64500)
|
||||||
|
announce("2000::/15", 64500)
|
||||||
|
announce("192.0.0.0/7", 64500)
|
||||||
|
|
||||||
|
for _, ip := range []string{"2001:db8::1", "192.0.2.1"} {
|
||||||
|
if info, err := db.GetIPInfo(ip); !errors.Is(err, ErrNoRoute) {
|
||||||
|
t.Errorf("GetIPInfo(%s) with only short routes = %+v, %v; want ErrNoRoute", ip, info, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Routes of exactly the minimum length are used.
|
||||||
|
announce("2001::/16", 64501)
|
||||||
|
announce("192.0.0.0/8", 64501)
|
||||||
|
|
||||||
|
lookup("2001:db8::1", "2001::/16", 64501)
|
||||||
|
lookup("192.0.2.1", "192.0.0.0/8", 64501)
|
||||||
|
|
||||||
|
announce("2001:db8::/32", 64502)
|
||||||
|
|
||||||
|
lookup("2001:db8::1", "2001:db8::/32", 64502)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestConnectionPoolPragmas holds several pooled connections open at once and
|
||||||
|
// checks each one carries the per-connection settings from the DSN, plus the
|
||||||
|
// process-wide hard heap limit, and sees the new file with auto_vacuum
|
||||||
|
// incremental.
|
||||||
|
func TestConnectionPoolPragmas(t *testing.T) {
|
||||||
|
cfg := &config.Config{StateDir: t.TempDir()}
|
||||||
|
|
||||||
|
db, err := New(cfg, logger.New())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to create database: %v", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = db.Close() }()
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
// Hold distinct connections open simultaneously so the pool must open a new
|
||||||
|
// one (each parsing the DSN) rather than hand back the same connection.
|
||||||
|
conns := make([]*sql.Conn, 0, heldConnections)
|
||||||
|
defer func() {
|
||||||
|
for _, c := range conns {
|
||||||
|
_ = c.Close()
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
for i := 0; i < heldConnections; i++ {
|
||||||
|
c, err := db.db.Conn(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to open connection %d: %v", i, err)
|
||||||
|
}
|
||||||
|
conns = append(conns, c)
|
||||||
|
}
|
||||||
|
|
||||||
|
for i, c := range conns {
|
||||||
|
var cacheSize int
|
||||||
|
if err := c.QueryRowContext(ctx, "PRAGMA cache_size").Scan(&cacheSize); err != nil {
|
||||||
|
t.Fatalf("conn %d: failed to read cache_size: %v", i, err)
|
||||||
|
}
|
||||||
|
if cacheSize != sqliteCacheSizeKiB {
|
||||||
|
t.Errorf("conn %d: cache_size = %d, want %d", i, cacheSize, sqliteCacheSizeKiB)
|
||||||
|
}
|
||||||
|
|
||||||
|
var busyTimeout int
|
||||||
|
if err := c.QueryRowContext(ctx, "PRAGMA busy_timeout").Scan(&busyTimeout); err != nil {
|
||||||
|
t.Fatalf("conn %d: failed to read busy_timeout: %v", i, err)
|
||||||
|
}
|
||||||
|
if busyTimeout != sqliteBusyTimeoutMs {
|
||||||
|
t.Errorf("conn %d: busy_timeout = %d, want %d", i, busyTimeout, sqliteBusyTimeoutMs)
|
||||||
|
}
|
||||||
|
|
||||||
|
var tempStore int
|
||||||
|
if err := c.QueryRowContext(ctx, "PRAGMA temp_store").Scan(&tempStore); err != nil {
|
||||||
|
t.Fatalf("conn %d: failed to read temp_store: %v", i, err)
|
||||||
|
}
|
||||||
|
if tempStore == tempStoreMemory {
|
||||||
|
t.Errorf("conn %d: temp_store = %d, want anything but %d (MEMORY)", i, tempStore, tempStoreMemory)
|
||||||
|
}
|
||||||
|
|
||||||
|
var hardHeapLimit int64
|
||||||
|
if err := c.QueryRowContext(ctx, "PRAGMA hard_heap_limit").Scan(&hardHeapLimit); err != nil {
|
||||||
|
t.Fatalf("conn %d: failed to read hard_heap_limit: %v", i, err)
|
||||||
|
}
|
||||||
|
if hardHeapLimit != sqliteHardHeapLimitBytes {
|
||||||
|
t.Errorf("conn %d: hard_heap_limit = %d, want %d", i, hardHeapLimit, sqliteHardHeapLimitBytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
var autoVacuum int
|
||||||
|
if err := c.QueryRowContext(ctx, "PRAGMA auto_vacuum").Scan(&autoVacuum); err != nil {
|
||||||
|
t.Fatalf("conn %d: failed to read auto_vacuum: %v", i, err)
|
||||||
|
}
|
||||||
|
if autoVacuum != autoVacuumIncremental {
|
||||||
|
t.Errorf("conn %d: auto_vacuum = %d, want %d (incremental)", i, autoVacuum, autoVacuumIncremental)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestVacuumReturnsFreePages checks that after routes are deleted from a new
|
||||||
|
// database, one Vacuum call returns every page they used. The deletes leave
|
||||||
|
// fewer free pages than the 1000 Vacuum frees per call, so none may remain.
|
||||||
|
// With auto_vacuum off (issue https://git.eeqj.de/sneak/routewatch/issues/43)
|
||||||
|
// the free pages stayed in the file, and with the PRAGMA run by ExecContext
|
||||||
|
// Vacuum freed only one page per call.
|
||||||
|
func TestVacuumReturnsFreePages(t *testing.T) {
|
||||||
|
cfg := &config.Config{StateDir: t.TempDir()}
|
||||||
|
|
||||||
|
db, err := New(cfg, logger.New())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to create database: %v", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = db.Close() }()
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
ts := time.Now().UTC()
|
||||||
|
const asn = 64500
|
||||||
|
|
||||||
|
routes := make([]*LiveRoute, 0, vacuumTestRoutes)
|
||||||
|
deletions := make([]LiveRouteDeletion, 0, vacuumTestRoutes)
|
||||||
|
for i := range vacuumTestRoutes {
|
||||||
|
route := mkV4Route(t, fmt.Sprintf("10.%d.%d.0/24", i/256, i%256), asn, ts)
|
||||||
|
routes = append(routes, route)
|
||||||
|
deletions = append(deletions, LiveRouteDeletion{
|
||||||
|
Prefix: route.Prefix,
|
||||||
|
OriginASN: asn,
|
||||||
|
PeerIP: route.PeerIP,
|
||||||
|
IPVersion: ipVersionV4,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := db.UpsertLiveRouteBatch(routes); err != nil {
|
||||||
|
t.Fatalf("failed to write routes: %v", err)
|
||||||
|
}
|
||||||
|
if err := db.DeleteLiveRouteBatch(deletions); err != nil {
|
||||||
|
t.Fatalf("failed to delete routes: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var before int
|
||||||
|
if err := db.db.QueryRowContext(ctx, "PRAGMA freelist_count").Scan(&before); err != nil {
|
||||||
|
t.Fatalf("failed to read freelist_count: %v", err)
|
||||||
|
}
|
||||||
|
if before == 0 {
|
||||||
|
t.Fatalf("no free pages after deleting %d routes", vacuumTestRoutes)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := db.Vacuum(ctx); err != nil {
|
||||||
|
t.Fatalf("Vacuum failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var after int
|
||||||
|
if err := db.db.QueryRowContext(ctx, "PRAGMA freelist_count").Scan(&after); err != nil {
|
||||||
|
t.Fatalf("failed to read freelist_count: %v", err)
|
||||||
|
}
|
||||||
|
if after != 0 {
|
||||||
|
t.Errorf("free pages after Vacuum = %d of %d, want 0", after, before)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestBatchWriteDuringCheckpoint reproduces issue #25. A batch write reads
|
||||||
|
// (SELECT) before it writes (INSERT/UPDATE). Under the default deferred locking
|
||||||
|
// the transaction begins as a reader and, when the maintainer's WAL checkpoint
|
||||||
|
// holds the write lock, its upgrade to writer fails immediately with "database
|
||||||
|
// is locked" without honouring busy_timeout, dropping the batch. With
|
||||||
|
// _txlock=immediate the transaction takes the write lock at BEGIN and waits, so
|
||||||
|
// no batch is dropped. The checkpoint runs without the Database mutex, exactly
|
||||||
|
// as the background maintainer does in production.
|
||||||
|
func TestBatchWriteDuringCheckpoint(t *testing.T) {
|
||||||
|
cfg := &config.Config{StateDir: t.TempDir()}
|
||||||
|
|
||||||
|
db, err := New(cfg, logger.New())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to create database: %v", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = db.Close() }()
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
_ = db.Checkpoint(ctx) // errors are the checkpoint's own to absorb
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
ts := time.Now().UTC()
|
||||||
|
for i := 0; i < contentionIterations; i++ {
|
||||||
|
asns := map[int]time.Time{
|
||||||
|
i % contendedASNCount: ts,
|
||||||
|
(i % contendedASNCount) + asnSecondBand: ts,
|
||||||
|
}
|
||||||
|
if err := db.GetOrCreateASNBatch(asns); err != nil {
|
||||||
|
cancel()
|
||||||
|
wg.Wait()
|
||||||
|
t.Fatalf("batch write failed under checkpoint contention: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
cancel()
|
||||||
|
wg.Wait()
|
||||||
|
}
|
||||||
@@ -1,61 +0,0 @@
|
|||||||
package database
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/config"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestDockerignoreExcludesDatabaseFiles checks that .dockerignore keeps the
|
|
||||||
// database file, and the WAL and shared-memory files SQLite writes next to it,
|
|
||||||
// out of the build context wherever they sit in the clone. The image ships the
|
|
||||||
// whole build context as a source archive.
|
|
||||||
func TestDockerignoreExcludesDatabaseFiles(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to create database: %v", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = db.Close() }()
|
|
||||||
|
|
||||||
data, err := os.ReadFile("../../.dockerignore")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to read .dockerignore: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Docker reads `**/` as any number of directories and the rest of the
|
|
||||||
// pattern as filepath.Match does, so `**/` followed by a pattern that
|
|
||||||
// matches a file's name excludes that file at any depth.
|
|
||||||
var anyDepth []string
|
|
||||||
|
|
||||||
for line := range strings.Lines(string(data)) {
|
|
||||||
pattern, ok := strings.CutPrefix(strings.TrimSpace(line), "**/")
|
|
||||||
if ok {
|
|
||||||
anyDepth = append(anyDepth, pattern)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
name := filepath.Base(db.path)
|
|
||||||
|
|
||||||
for _, file := range []string{name, name + "-wal", name + "-shm"} {
|
|
||||||
excluded := false
|
|
||||||
|
|
||||||
for _, pattern := range anyDepth {
|
|
||||||
matched, err := filepath.Match(pattern, file)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("bad .dockerignore pattern **/%s: %v", pattern, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
excluded = excluded || matched
|
|
||||||
}
|
|
||||||
|
|
||||||
if !excluded {
|
|
||||||
t.Errorf(".dockerignore has no **/ pattern that excludes %s", file)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,18 +1,14 @@
|
|||||||
// Package database provides SQLite storage for BGP routing data: ASNs,
|
// Package database provides SQLite storage for BGP routing data including ASNs,
|
||||||
// prefixes, announcements, peerings and BGP peers. Its routes table holds a
|
// prefixes, announcements, peerings, and live route tables.
|
||||||
// copy of the whole routing table that internal/routetable keeps in memory.
|
|
||||||
package database
|
package database
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"net/netip"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/routetable"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// Stats contains database statistics: counts of ASNs, prefixes, peerings and
|
// Stats contains database statistics including counts for ASNs, prefixes,
|
||||||
// peers, and the file size.
|
// peerings, peers, and live routes, as well as file size and prefix distribution data.
|
||||||
type Stats struct {
|
type Stats struct {
|
||||||
ASNs int
|
ASNs int
|
||||||
Prefixes int
|
Prefixes int
|
||||||
@@ -21,17 +17,22 @@ type Stats struct {
|
|||||||
Peerings int
|
Peerings int
|
||||||
Peers int
|
Peers int
|
||||||
FileSizeBytes int64
|
FileSizeBytes int64
|
||||||
|
LiveRoutes int
|
||||||
|
IPv4Routes int
|
||||||
|
IPv6Routes int
|
||||||
|
OldestRoute *time.Time
|
||||||
|
NewestRoute *time.Time
|
||||||
|
IPv4PrefixDistribution []PrefixDistribution
|
||||||
|
IPv6PrefixDistribution []PrefixDistribution
|
||||||
}
|
}
|
||||||
|
|
||||||
// Store defines the interface for database operations. It provides methods for
|
// Store defines the interface for database operations. It provides methods for
|
||||||
// managing ASNs, prefixes, announcements, peerings, BGP peers, and the copy of
|
// managing ASNs, prefixes, announcements, peerings, BGP peers, and live routes.
|
||||||
// the routing table. Implementations must be safe for concurrent use.
|
// Implementations must be safe for concurrent use.
|
||||||
//
|
|
||||||
//nolint:interfacebloat // the one seam handlers and tests depend on
|
|
||||||
type Store interface {
|
type Store interface {
|
||||||
// ASN operations
|
// ASN operations
|
||||||
GetOrCreateASN(number int, timestamp time.Time) (*ASN, error)
|
GetOrCreateASN(number int, timestamp time.Time) (*ASN, error)
|
||||||
GetOrCreateASNBatch(asns map[int]SeenTimes) error
|
GetOrCreateASNBatch(asns map[int]time.Time) error
|
||||||
|
|
||||||
// Prefix operations
|
// Prefix operations
|
||||||
GetOrCreatePrefix(prefix string, timestamp time.Time) (*Prefix, error)
|
GetOrCreatePrefix(prefix string, timestamp time.Time) (*Prefix, error)
|
||||||
@@ -41,7 +42,7 @@ type Store interface {
|
|||||||
RecordAnnouncement(announcement *Announcement) error
|
RecordAnnouncement(announcement *Announcement) error
|
||||||
|
|
||||||
// Peering operations
|
// Peering operations
|
||||||
RecordPeering(asA, asB int, seen SeenTimes) error
|
RecordPeering(asA, asB int, timestamp time.Time) error
|
||||||
|
|
||||||
// Statistics
|
// Statistics
|
||||||
GetStats() (Stats, error)
|
GetStats() (Stats, error)
|
||||||
@@ -51,9 +52,19 @@ type Store interface {
|
|||||||
UpdatePeer(peerIP string, peerASN int, messageType string, timestamp time.Time) error
|
UpdatePeer(peerIP string, peerASN int, messageType string, timestamp time.Time) error
|
||||||
UpdatePeerBatch(peers map[string]PeerUpdate) error
|
UpdatePeerBatch(peers map[string]PeerUpdate) error
|
||||||
|
|
||||||
// Routing table operations: the database's copy of the table in memory
|
// Live route operations
|
||||||
WriteRouteChanges(changes []routetable.Change, newPeers map[string]uint) error
|
UpsertLiveRoute(route *LiveRoute) error
|
||||||
ReadRouteTable(ctx context.Context) (*routetable.Table, error)
|
UpsertLiveRouteBatch(routes []*LiveRoute) error
|
||||||
|
DeleteLiveRoute(prefix string, originASN int, peerIP string) error
|
||||||
|
DeleteLiveRouteBatch(deletions []LiveRouteDeletion) error
|
||||||
|
GetPrefixDistribution() (ipv4 []PrefixDistribution, ipv6 []PrefixDistribution, err error)
|
||||||
|
GetPrefixDistributionContext(ctx context.Context) (ipv4 []PrefixDistribution, ipv6 []PrefixDistribution, err error)
|
||||||
|
GetLiveRouteCounts() (ipv4Count, ipv6Count int, err error)
|
||||||
|
GetLiveRouteCountsContext(ctx context.Context) (ipv4Count, ipv6Count int, err error)
|
||||||
|
|
||||||
|
// IP lookup operations
|
||||||
|
GetIPInfo(ip string) (*IPInfo, error)
|
||||||
|
GetIPInfoContext(ctx context.Context, ip string) (*IPInfo, error)
|
||||||
|
|
||||||
// ASN WHOIS operations
|
// ASN WHOIS operations
|
||||||
GetNextStaleASN(ctx context.Context, staleThreshold time.Duration) (int, error)
|
GetNextStaleASN(ctx context.Context, staleThreshold time.Duration) (int, error)
|
||||||
@@ -61,12 +72,14 @@ type Store interface {
|
|||||||
GetWHOISStats(ctx context.Context, staleThreshold time.Duration) (*WHOISStats, error)
|
GetWHOISStats(ctx context.Context, staleThreshold time.Duration) (*WHOISStats, error)
|
||||||
|
|
||||||
// AS and prefix detail operations
|
// AS and prefix detail operations
|
||||||
GetAS(ctx context.Context, asn int) (*ASN, bool, error)
|
GetASDetails(asn int) (*ASN, []LiveRoute, error)
|
||||||
GetLastSeenASes(ctx context.Context) (*ASN, *ASN, error)
|
GetASDetailsContext(ctx context.Context, asn int) (*ASN, []LiveRoute, error)
|
||||||
GetASWHOIS(ctx context.Context, asn int) (*ASN, error)
|
|
||||||
GetPrefixFirstSeen(ctx context.Context, prefix netip.Prefix) (time.Time, error)
|
|
||||||
GetASPeers(asn int) ([]ASPeer, error)
|
GetASPeers(asn int) ([]ASPeer, error)
|
||||||
GetASPeersContext(ctx context.Context, asn int) ([]ASPeer, error)
|
GetASPeersContext(ctx context.Context, asn int) ([]ASPeer, error)
|
||||||
|
GetPrefixDetails(prefix string) ([]LiveRoute, error)
|
||||||
|
GetPrefixDetailsContext(ctx context.Context, prefix string) ([]LiveRoute, error)
|
||||||
|
GetRandomPrefixesByLength(maskLength, ipVersion, limit int) ([]LiveRoute, error)
|
||||||
|
GetRandomPrefixesByLengthContext(ctx context.Context, maskLength, ipVersion, limit int) ([]LiveRoute, error)
|
||||||
|
|
||||||
// Lifecycle
|
// Lifecycle
|
||||||
Close() error
|
Close() error
|
||||||
|
|||||||
+74
-44
@@ -1,3 +1,4 @@
|
|||||||
|
// Package database provides SQLite storage for BGP routing data.
|
||||||
package database
|
package database
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -8,8 +9,6 @@ import (
|
|||||||
|
|
||||||
// ASN represents an Autonomous System Number with its metadata including
|
// ASN represents an Autonomous System Number with its metadata including
|
||||||
// handle, description, WHOIS data, and first/last seen timestamps.
|
// handle, description, WHOIS data, and first/last seen timestamps.
|
||||||
//
|
|
||||||
//nolint:tagliatelle // field names are the existing JSON API
|
|
||||||
type ASN struct {
|
type ASN struct {
|
||||||
ASN int `json:"asn"`
|
ASN int `json:"asn"`
|
||||||
Handle string `json:"handle"`
|
Handle string `json:"handle"`
|
||||||
@@ -28,73 +27,104 @@ type ASN struct {
|
|||||||
RIRRegDate *time.Time `json:"rir_registration_date,omitempty"`
|
RIRRegDate *time.Time `json:"rir_registration_date,omitempty"`
|
||||||
RIRLastMod *time.Time `json:"rir_last_modified,omitempty"`
|
RIRLastMod *time.Time `json:"rir_last_modified,omitempty"`
|
||||||
WHOISRaw string `json:"whois_raw,omitempty"`
|
WHOISRaw string `json:"whois_raw,omitempty"`
|
||||||
// Timestamps. They are zero, and left out of JSON, for an AS that has no
|
// Timestamps
|
||||||
// record in the asns table.
|
FirstSeen time.Time `json:"first_seen"`
|
||||||
FirstSeen time.Time `json:"first_seen,omitzero"`
|
LastSeen time.Time `json:"last_seen"`
|
||||||
LastSeen time.Time `json:"last_seen,omitzero"`
|
|
||||||
WHOISUpdatedAt *time.Time `json:"whois_updated_at,omitempty"`
|
WHOISUpdatedAt *time.Time `json:"whois_updated_at,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Prefix represents an IP prefix (CIDR block) with its IP version (4 or 6)
|
// Prefix represents an IP prefix (CIDR block) with its IP version (4 or 6)
|
||||||
// and the time it was first seen.
|
// and first/last seen timestamps.
|
||||||
type Prefix struct {
|
type Prefix struct {
|
||||||
Prefix string
|
ID uuid.UUID `json:"id"`
|
||||||
IPVersion int // 4 or 6
|
Prefix string `json:"prefix"`
|
||||||
FirstSeen time.Time
|
IPVersion int `json:"ip_version"` // 4 or 6
|
||||||
|
FirstSeen time.Time `json:"first_seen"`
|
||||||
|
LastSeen time.Time `json:"last_seen"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Announcement represents a BGP announcement or withdrawal event,
|
// Announcement represents a BGP announcement or withdrawal event,
|
||||||
// containing the prefix, AS path, origin ASN, peer ASN, next hop, and timestamp.
|
// containing the prefix, AS path, origin ASN, peer ASN, next hop, and timestamp.
|
||||||
type Announcement struct {
|
type Announcement struct {
|
||||||
ID uuid.UUID
|
ID uuid.UUID `json:"id"`
|
||||||
PrefixID uuid.UUID
|
PrefixID uuid.UUID `json:"prefix_id"`
|
||||||
PeerASN int
|
PeerASN int `json:"peer_asn"`
|
||||||
OriginASN int
|
OriginASN int `json:"origin_asn"`
|
||||||
Path string // JSON-encoded AS path
|
Path string `json:"path"` // JSON-encoded AS path
|
||||||
NextHop string
|
NextHop string `json:"next_hop"`
|
||||||
Timestamp time.Time
|
Timestamp time.Time `json:"timestamp"`
|
||||||
IsWithdrawal bool
|
IsWithdrawal bool `json:"is_withdrawal"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ASNPeering represents a peering relationship between two ASNs,
|
// ASNPeering represents a peering relationship between two ASNs,
|
||||||
// stored with the lower ASN as ASA and the higher as ASB.
|
// stored with the lower ASN as ASA and the higher as ASB.
|
||||||
type ASNPeering struct {
|
type ASNPeering struct {
|
||||||
ID uuid.UUID
|
ID uuid.UUID `json:"id"`
|
||||||
ASA int
|
ASA int `json:"as_a"`
|
||||||
ASB int
|
ASB int `json:"as_b"`
|
||||||
FirstSeen time.Time
|
FirstSeen time.Time `json:"first_seen"`
|
||||||
LastSeen time.Time
|
LastSeen time.Time `json:"last_seen"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// SeenTimes is the earliest and the latest time one batch of updates saw an AS,
|
// LiveRoute represents a route in the live routing table
|
||||||
// a peer or a peering. A row the batch adds is stored first seen at First and
|
type LiveRoute struct {
|
||||||
// last seen at Last. A row already stored takes First as its first_seen only
|
ID uuid.UUID `json:"id"`
|
||||||
// when First is earlier, and Last as its last_seen only when Last is later.
|
Prefix string `json:"prefix"`
|
||||||
type SeenTimes struct {
|
MaskLength int `json:"mask_length"`
|
||||||
First time.Time
|
IPVersion int `json:"ip_version"`
|
||||||
Last time.Time
|
OriginASN int `json:"origin_asn"`
|
||||||
|
PeerIP string `json:"peer_ip"`
|
||||||
|
ASPath []int `json:"as_path"`
|
||||||
|
NextHop string `json:"next_hop"`
|
||||||
|
LastUpdated time.Time `json:"last_updated"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add makes t the earliest time of s when it is earlier than First, and the
|
// PrefixDistribution represents the distribution of prefixes by mask length
|
||||||
// latest when it is later than Last. The zero SeenTimes holds no time yet, so
|
type PrefixDistribution struct {
|
||||||
// the first Add sets both.
|
MaskLength int `json:"mask_length"`
|
||||||
func (s *SeenTimes) Add(t time.Time) {
|
Count int `json:"count"`
|
||||||
if s.First.IsZero() || t.Before(s.First) {
|
|
||||||
s.First = t
|
|
||||||
}
|
|
||||||
|
|
||||||
if s.Last.IsZero() || t.After(s.Last) {
|
|
||||||
s.Last = t
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// PeerUpdate represents parameters for updating a peer. PeerASN and
|
// IPInfo represents comprehensive IP information for the /ip endpoint
|
||||||
// MessageType are those of the peer's latest update.
|
type IPInfo struct {
|
||||||
|
IP string `json:"ip"`
|
||||||
|
PTR []string `json:"ptr,omitempty"`
|
||||||
|
Netblock string `json:"netblock"`
|
||||||
|
MaskLength int `json:"mask_length"`
|
||||||
|
IPVersion int `json:"ip_version"`
|
||||||
|
NumPeers int `json:"num_peers"`
|
||||||
|
// AS information
|
||||||
|
ASN int `json:"asn"`
|
||||||
|
ASName string `json:"as_name,omitempty"`
|
||||||
|
Handle string `json:"handle,omitempty"`
|
||||||
|
Description string `json:"description,omitempty"`
|
||||||
|
OrgName string `json:"org_name,omitempty"`
|
||||||
|
OrgID string `json:"org_id,omitempty"`
|
||||||
|
Address string `json:"address,omitempty"`
|
||||||
|
CountryCode string `json:"country_code,omitempty"`
|
||||||
|
AbuseEmail string `json:"abuse_email,omitempty"`
|
||||||
|
RIR string `json:"rir,omitempty"`
|
||||||
|
// Timestamps
|
||||||
|
FirstSeen time.Time `json:"first_seen"`
|
||||||
|
LastSeen time.Time `json:"last_seen"`
|
||||||
|
// Indicates if WHOIS data needs refresh (not serialized)
|
||||||
|
NeedsWHOISRefresh bool `json:"-"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// LiveRouteDeletion represents parameters for deleting a live route
|
||||||
|
type LiveRouteDeletion struct {
|
||||||
|
Prefix string
|
||||||
|
OriginASN int
|
||||||
|
PeerIP string
|
||||||
|
IPVersion int
|
||||||
|
}
|
||||||
|
|
||||||
|
// PeerUpdate represents parameters for updating a peer
|
||||||
type PeerUpdate struct {
|
type PeerUpdate struct {
|
||||||
PeerIP string
|
PeerIP string
|
||||||
PeerASN int
|
PeerASN int
|
||||||
MessageType string
|
MessageType string
|
||||||
Seen SeenTimes
|
Timestamp time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
// ASNWHOISUpdate contains WHOIS data for updating an ASN record.
|
// ASNWHOISUpdate contains WHOIS data for updating an ASN record.
|
||||||
|
|||||||
@@ -1,339 +0,0 @@
|
|||||||
package database
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"database/sql"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"net/netip"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/bits-and-blooms/bitset"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/routetable"
|
|
||||||
)
|
|
||||||
|
|
||||||
// keptRoutesPerRow is how many kept routes a row of the routes table has
|
|
||||||
// columns for: the newest and the older one.
|
|
||||||
const keptRoutesPerRow = 2
|
|
||||||
|
|
||||||
// errUnknownRoutePeer is returned for a peer that route_peers lacks: on a
|
|
||||||
// write, the peer of a kept route; on a read, a peer number in a row.
|
|
||||||
var errUnknownRoutePeer = errors.New("peer not in route_peers")
|
|
||||||
|
|
||||||
// The statements of the routes and route_peers tables (schema.sql).
|
|
||||||
const (
|
|
||||||
insertRoutePeerSQL = `INSERT INTO route_peers (number, address)
|
|
||||||
VALUES (?, ?)`
|
|
||||||
routePeerNumberSQL = `SELECT number FROM route_peers WHERE address = ?`
|
|
||||||
routePeersSQL = `SELECT number, address FROM route_peers`
|
|
||||||
|
|
||||||
upsertRouteSQL = `INSERT INTO routes (prefix, origin_asn, peers,
|
|
||||||
newest_peer, newest_as_path, newest_time,
|
|
||||||
older_peer, older_as_path, older_time)
|
|
||||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
||||||
ON CONFLICT (prefix, origin_asn) DO UPDATE SET peers = excluded.peers,
|
|
||||||
newest_peer = excluded.newest_peer,
|
|
||||||
newest_as_path = excluded.newest_as_path,
|
|
||||||
newest_time = excluded.newest_time,
|
|
||||||
older_peer = excluded.older_peer,
|
|
||||||
older_as_path = excluded.older_as_path,
|
|
||||||
older_time = excluded.older_time`
|
|
||||||
deleteRouteSQL = `DELETE FROM routes WHERE prefix = ? AND origin_asn = ?`
|
|
||||||
routesSQL = `SELECT prefix, origin_asn, peers,
|
|
||||||
newest_peer, newest_as_path, newest_time,
|
|
||||||
older_peer, older_as_path, older_time FROM routes`
|
|
||||||
)
|
|
||||||
|
|
||||||
// WriteRouteChanges writes, in one transaction, what one call of the routing
|
|
||||||
// table's TakeChanges returned. It stores each new peer with its number, writes
|
|
||||||
// the row of each changed prefix and origin AS pair, and deletes the row of
|
|
||||||
// each removed one. Route times are stored to the second, as the feed gives
|
|
||||||
// them.
|
|
||||||
//
|
|
||||||
// When it fails, it writes nothing. The caller then writes these changes and
|
|
||||||
// peers again, because later changes can name the peers it did not store.
|
|
||||||
func (d *Database) WriteRouteChanges(
|
|
||||||
changes []routetable.Change, newPeers map[string]uint,
|
|
||||||
) error {
|
|
||||||
if len(changes) == 0 && len(newPeers) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
d.lock("WriteRouteChanges")
|
|
||||||
defer d.unlock()
|
|
||||||
|
|
||||||
tx, err := d.beginTx()
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to begin transaction: %w", err)
|
|
||||||
}
|
|
||||||
defer d.rollback(tx)
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
// The new peers first, so that the rows can look up their numbers.
|
|
||||||
for address, number := range newPeers {
|
|
||||||
_, err := tx.ExecContext(ctx, insertRoutePeerSQL, number, address)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to store peer %s: %w", address, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
err = writeRouteRows(ctx, tx, changes)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = tx.Commit()
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to commit transaction: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeRouteRows upserts the row of each change in tx, or deletes it for a
|
|
||||||
// removed pair.
|
|
||||||
func writeRouteRows(
|
|
||||||
ctx context.Context, tx *loggingTx, changes []routetable.Change,
|
|
||||||
) error {
|
|
||||||
upsert, err := tx.PrepareContext(ctx, upsertRouteSQL)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to prepare route upsert: %w", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = upsert.Close() }()
|
|
||||||
|
|
||||||
del, err := tx.PrepareContext(ctx, deleteRouteSQL)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to prepare route delete: %w", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = del.Close() }()
|
|
||||||
|
|
||||||
peerNumber, err := tx.PrepareContext(ctx, routePeerNumberSQL)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to prepare peer number lookup: %w", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = peerNumber.Close() }()
|
|
||||||
|
|
||||||
for _, change := range changes {
|
|
||||||
prefix := change.Prefix.String()
|
|
||||||
|
|
||||||
if change.Removed {
|
|
||||||
_, err := del.ExecContext(ctx, prefix, change.OriginASN)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to delete route %s: %w", prefix, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
row, err := routeRow(ctx, peerNumber, change)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to write route %s: %w", prefix, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = upsert.ExecContext(ctx, row...)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to write route %s: %w", prefix, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// routeRow returns the values of the routes row of change, in the order of
|
|
||||||
// upsertRouteSQL. peerNumber looks up a peer's number by its address. The
|
|
||||||
// table keeps at most keptRoutesPerRow routes of a pair.
|
|
||||||
func routeRow(
|
|
||||||
ctx context.Context, peerNumber *sql.Stmt, change routetable.Change,
|
|
||||||
) ([]any, error) {
|
|
||||||
peers, err := change.Peers.MarshalBinary()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
row := []any{change.Prefix.String(), change.OriginASN, peers}
|
|
||||||
|
|
||||||
for i := range keptRoutesPerRow {
|
|
||||||
if i >= len(change.Routes) {
|
|
||||||
row = append(row, nil, nil, nil)
|
|
||||||
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
r := change.Routes[i]
|
|
||||||
|
|
||||||
var number uint
|
|
||||||
|
|
||||||
err := peerNumber.QueryRowContext(ctx, r.Peer).Scan(&number)
|
|
||||||
if errors.Is(err, sql.ErrNoRows) {
|
|
||||||
return nil, fmt.Errorf("%w: %s", errUnknownRoutePeer, r.Peer)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
path, err := json.Marshal(r.ASPath)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
row = append(row, number, string(path), r.Time.Unix())
|
|
||||||
}
|
|
||||||
|
|
||||||
return row, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ReadRouteTable returns the routing table that the routes and route_peers
|
|
||||||
// tables hold, as startup reads it. It reads the peers first, so that the
|
|
||||||
// table keeps their numbers, then loads each row into the table as it reads
|
|
||||||
// it. The two reads are separate, so nothing may write the tables meanwhile.
|
|
||||||
// It fails on a row that names a peer number route_peers lacks.
|
|
||||||
func (d *Database) ReadRouteTable(ctx context.Context) (*routetable.Table, error) {
|
|
||||||
peerNumbers, err := d.readRoutePeers(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
addresses := make(map[uint]string, len(peerNumbers))
|
|
||||||
numbers := new(bitset.BitSet)
|
|
||||||
|
|
||||||
for address, number := range peerNumbers {
|
|
||||||
addresses[number] = address
|
|
||||||
numbers.Set(number)
|
|
||||||
}
|
|
||||||
|
|
||||||
table := routetable.New(routetable.Params{PeerNumbers: peerNumbers})
|
|
||||||
|
|
||||||
rows, err := d.db.QueryContext(ctx, routesSQL)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to read routes: %w", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = rows.Close() }()
|
|
||||||
|
|
||||||
for rows.Next() {
|
|
||||||
change, err := scanRouteRow(rows, addresses, numbers)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
table.Load(change)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = rows.Err()
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to read routes: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return table, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// readRoutePeers returns the number of each peer in route_peers, by address.
|
|
||||||
func (d *Database) readRoutePeers(ctx context.Context) (map[string]uint, error) {
|
|
||||||
rows, err := d.db.QueryContext(ctx, routePeersSQL)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to read route peers: %w", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = rows.Close() }()
|
|
||||||
|
|
||||||
peerNumbers := make(map[string]uint)
|
|
||||||
|
|
||||||
for rows.Next() {
|
|
||||||
var (
|
|
||||||
number uint
|
|
||||||
address string
|
|
||||||
)
|
|
||||||
|
|
||||||
err := rows.Scan(&number, &address)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to read a route peer: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
peerNumbers[address] = number
|
|
||||||
}
|
|
||||||
|
|
||||||
err = rows.Err()
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to read route peers: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return peerNumbers, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// keptRouteColumns are the three columns of one kept route in a routes row.
|
|
||||||
type keptRouteColumns struct {
|
|
||||||
peer sql.Null[uint]
|
|
||||||
asPath sql.NullString
|
|
||||||
time sql.NullInt64
|
|
||||||
}
|
|
||||||
|
|
||||||
// scanRouteRow returns the content of the routes row rows is at, with each kept
|
|
||||||
// route's peer named by its address in addresses. It fails when the row names a
|
|
||||||
// peer number that addresses lacks. numbers holds the numbers of addresses as a
|
|
||||||
// set, against which a row's whole peer set is checked at once.
|
|
||||||
func scanRouteRow(
|
|
||||||
rows *sql.Rows, addresses map[uint]string, numbers *bitset.BitSet,
|
|
||||||
) (routetable.Change, error) {
|
|
||||||
var (
|
|
||||||
change routetable.Change
|
|
||||||
prefix string
|
|
||||||
peers []byte
|
|
||||||
kept [keptRoutesPerRow]keptRouteColumns
|
|
||||||
)
|
|
||||||
|
|
||||||
err := rows.Scan(&prefix, &change.OriginASN, &peers,
|
|
||||||
&kept[0].peer, &kept[0].asPath, &kept[0].time,
|
|
||||||
&kept[1].peer, &kept[1].asPath, &kept[1].time)
|
|
||||||
if err != nil {
|
|
||||||
return change, fmt.Errorf("failed to read a route: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
change.Prefix, err = netip.ParsePrefix(prefix)
|
|
||||||
if err != nil {
|
|
||||||
return change, fmt.Errorf("failed to read route %s: %w", prefix, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
change.Peers = new(bitset.BitSet)
|
|
||||||
|
|
||||||
err = change.Peers.UnmarshalBinary(peers)
|
|
||||||
if err != nil {
|
|
||||||
return change, fmt.Errorf("failed to read the peers of route %s: %w",
|
|
||||||
prefix, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !numbers.IsSuperSet(change.Peers) {
|
|
||||||
unknown, _ := change.Peers.Difference(numbers).NextSet(0)
|
|
||||||
|
|
||||||
return change, fmt.Errorf("%w: number %d in the peers of route %s",
|
|
||||||
errUnknownRoutePeer, unknown, prefix)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, columns := range kept {
|
|
||||||
if !columns.peer.Valid {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
address, known := addresses[columns.peer.V]
|
|
||||||
if !known {
|
|
||||||
return change, fmt.Errorf("%w: number %d in route %s",
|
|
||||||
errUnknownRoutePeer, columns.peer.V, prefix)
|
|
||||||
}
|
|
||||||
|
|
||||||
r := routetable.Route{
|
|
||||||
Peer: address,
|
|
||||||
Time: time.Unix(columns.time.Int64, 0).UTC(),
|
|
||||||
}
|
|
||||||
|
|
||||||
err := json.Unmarshal([]byte(columns.asPath.String), &r.ASPath)
|
|
||||||
if err != nil {
|
|
||||||
return change, fmt.Errorf("failed to read an AS path of route %s: %w",
|
|
||||||
prefix, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
change.Routes = append(change.Routes, r)
|
|
||||||
}
|
|
||||||
|
|
||||||
return change, nil
|
|
||||||
}
|
|
||||||
@@ -1,527 +0,0 @@
|
|||||||
package database
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"math"
|
|
||||||
"math/rand/v2"
|
|
||||||
"net/netip"
|
|
||||||
"os"
|
|
||||||
"reflect"
|
|
||||||
"slices"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/bits-and-blooms/bitset"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/config"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/routetable"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Peers, origin ASes and prefixes of the routes table tests.
|
|
||||||
const (
|
|
||||||
peerA = "192.0.2.1"
|
|
||||||
peerB = "192.0.2.2"
|
|
||||||
peerC = "2001:db8::3"
|
|
||||||
peerD = "2001:db8::4"
|
|
||||||
peerE = "2001:db8::5"
|
|
||||||
|
|
||||||
asA = 64500
|
|
||||||
asB = 64501
|
|
||||||
// transitAS is the first AS of every test path, the peer's own.
|
|
||||||
transitAS = 64496
|
|
||||||
|
|
||||||
tablePrefixV4 = "10.1.2.0/24"
|
|
||||||
tablePrefixShortV4 = "10.9.0.0/16"
|
|
||||||
tablePrefixV6 = "2001:db8:1::/48"
|
|
||||||
)
|
|
||||||
|
|
||||||
// tableTime returns the time second seconds after the start of every routes
|
|
||||||
// table test.
|
|
||||||
func tableTime(second int) time.Time {
|
|
||||||
return time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC).
|
|
||||||
Add(time.Duration(second) * time.Second)
|
|
||||||
}
|
|
||||||
|
|
||||||
// announceTo has peer announce prefix with origin AS asn second seconds after
|
|
||||||
// the start.
|
|
||||||
func announceTo(
|
|
||||||
table *routetable.Table, prefix string, asn int, peer string, second int,
|
|
||||||
) {
|
|
||||||
table.Announce(netip.MustParsePrefix(prefix), asn, routetable.Route{
|
|
||||||
Peer: peer, ASPath: []int{transitAS, asn}, Time: tableTime(second),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// newRoutesTestDatabase returns a new database in a temporary directory.
|
|
||||||
func newRoutesTestDatabase(t *testing.T) *Database {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to create database: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Cleanup(func() { _ = db.Close() })
|
|
||||||
|
|
||||||
return db
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeTableChanges writes what table changed since the last call.
|
|
||||||
func writeTableChanges(t *testing.T, db *Database, table *routetable.Table) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
changes, newPeers := table.TakeChanges(math.MaxInt)
|
|
||||||
|
|
||||||
err := db.WriteRouteChanges(changes, newPeers)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("WriteRouteChanges: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReadRouteTableGivesBackWhatWasWritten(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := newRoutesTestDatabase(t)
|
|
||||||
source := routetable.New(routetable.Params{})
|
|
||||||
|
|
||||||
// Two origins of one prefix: the routes of peerC and peerB are kept for
|
|
||||||
// asA, peerA keeps only its bit.
|
|
||||||
announceTo(source, tablePrefixV4, asA, peerA, 10)
|
|
||||||
announceTo(source, tablePrefixV4, asA, peerB, 20)
|
|
||||||
announceTo(source, tablePrefixV4, asA, peerC, 30)
|
|
||||||
announceTo(source, tablePrefixV4, asB, peerD, 40)
|
|
||||||
// Two routes with the same time.
|
|
||||||
announceTo(source, tablePrefixV6, asA, peerA, 50)
|
|
||||||
announceTo(source, tablePrefixV6, asA, peerC, 50)
|
|
||||||
// An origin whose kept routes were withdrawn while peerA carries it.
|
|
||||||
announceTo(source, tablePrefixShortV4, asB, peerA, 10)
|
|
||||||
announceTo(source, tablePrefixShortV4, asB, peerB, 20)
|
|
||||||
announceTo(source, tablePrefixShortV4, asB, peerC, 30)
|
|
||||||
source.Withdraw(netip.MustParsePrefix(tablePrefixShortV4), peerB)
|
|
||||||
source.Withdraw(netip.MustParsePrefix(tablePrefixShortV4), peerC)
|
|
||||||
|
|
||||||
writeTableChanges(t, db, source)
|
|
||||||
checkReadTable(t, db, source)
|
|
||||||
|
|
||||||
// A peer moves to another origin, one goes down and a new one comes.
|
|
||||||
announceTo(source, tablePrefixV4, asB, peerA, 60)
|
|
||||||
source.PeerDown(peerC)
|
|
||||||
announceTo(source, tablePrefixV6, asB, peerE, 70)
|
|
||||||
|
|
||||||
writeTableChanges(t, db, source)
|
|
||||||
checkReadTable(t, db, source)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReadRouteTableGivesATableThatTakesLaterUpdates(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := newRoutesTestDatabase(t)
|
|
||||||
source := routetable.New(routetable.Params{})
|
|
||||||
|
|
||||||
// peerA and peerB carry tablePrefixV4, peerB and peerC tablePrefixV6.
|
|
||||||
announceTo(source, tablePrefixV4, asA, peerA, 10)
|
|
||||||
announceTo(source, tablePrefixV4, asA, peerB, 20)
|
|
||||||
announceTo(source, tablePrefixV6, asA, peerB, 30)
|
|
||||||
announceTo(source, tablePrefixV6, asA, peerC, 40)
|
|
||||||
|
|
||||||
writeTableChanges(t, db, source)
|
|
||||||
read := checkReadTable(t, db, source)
|
|
||||||
|
|
||||||
// The same updates to both tables: a loaded peer withdraws a prefix,
|
|
||||||
// another one goes down, and a new peer comes.
|
|
||||||
for _, table := range []*routetable.Table{source, read} {
|
|
||||||
table.Withdraw(netip.MustParsePrefix(tablePrefixV4), peerA)
|
|
||||||
table.PeerDown(peerB)
|
|
||||||
announceTo(table, tablePrefixShortV4, asB, peerD, 50)
|
|
||||||
}
|
|
||||||
|
|
||||||
checkSameReads(t, read, source)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReadRouteTableFailsOnAPeerWithoutANumber(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := newRoutesTestDatabase(t)
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
|
|
||||||
// Only the routes of peerB and peerC are kept, so only the peer set names
|
|
||||||
// peerA.
|
|
||||||
announceTo(table, tablePrefixV4, asA, peerA, 10)
|
|
||||||
announceTo(table, tablePrefixV4, asA, peerB, 20)
|
|
||||||
announceTo(table, tablePrefixV4, asA, peerC, 30)
|
|
||||||
writeTableChanges(t, db, table)
|
|
||||||
|
|
||||||
_, err := db.db.ExecContext(t.Context(),
|
|
||||||
"DELETE FROM route_peers WHERE address = ?", peerA)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("deleting a route peer: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = db.ReadRouteTable(t.Context())
|
|
||||||
if !errors.Is(err, errUnknownRoutePeer) {
|
|
||||||
t.Fatalf("ReadRouteTable = %v, want %v", err, errUnknownRoutePeer)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkReadTable checks that the table ReadRouteTable reads answers every read
|
|
||||||
// as want does, and that reading it recorded no change. It returns that table.
|
|
||||||
func checkReadTable(
|
|
||||||
t *testing.T, db *Database, want *routetable.Table,
|
|
||||||
) *routetable.Table {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
got, err := db.ReadRouteTable(t.Context())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ReadRouteTable: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
checkSameReads(t, got, want)
|
|
||||||
|
|
||||||
changes, newPeers := got.TakeChanges(math.MaxInt)
|
|
||||||
if len(changes) != 0 || len(newPeers) != 0 {
|
|
||||||
t.Errorf("the read recorded changes %+v and new peers %v, want none",
|
|
||||||
changes, newPeers)
|
|
||||||
}
|
|
||||||
|
|
||||||
return got
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkSameReads checks that got answers every read as want does, for the
|
|
||||||
// prefixes and the origin ASes the tests use.
|
|
||||||
func checkSameReads(t *testing.T, got, want *routetable.Table) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
if !reflect.DeepEqual(got.Counts(), want.Counts()) {
|
|
||||||
t.Errorf("Counts() = %+v, want %+v", got.Counts(), want.Counts())
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, text := range []string{
|
|
||||||
tablePrefixV4, tablePrefixShortV4, tablePrefixV6,
|
|
||||||
} {
|
|
||||||
prefix := netip.MustParsePrefix(text)
|
|
||||||
|
|
||||||
if !reflect.DeepEqual(got.Origins(prefix), want.Origins(prefix)) {
|
|
||||||
t.Errorf("Origins(%s) = %+v, want %+v",
|
|
||||||
prefix, got.Origins(prefix), want.Origins(prefix))
|
|
||||||
}
|
|
||||||
|
|
||||||
gotSummary, gotFound := got.Lookup(prefix.Addr())
|
|
||||||
wantSummary, wantFound := want.Lookup(prefix.Addr())
|
|
||||||
|
|
||||||
if gotFound != wantFound || !reflect.DeepEqual(gotSummary, wantSummary) {
|
|
||||||
t.Errorf("Lookup(%s) = %+v, %v; want %+v, %v", prefix.Addr(),
|
|
||||||
gotSummary, gotFound, wantSummary, wantFound)
|
|
||||||
}
|
|
||||||
|
|
||||||
ipVersion := ipVersionV6
|
|
||||||
if prefix.Addr().Is4() {
|
|
||||||
ipVersion = ipVersionV4
|
|
||||||
}
|
|
||||||
|
|
||||||
gotSample := sortedSample(got, ipVersion, prefix.Bits())
|
|
||||||
wantSample := sortedSample(want, ipVersion, prefix.Bits())
|
|
||||||
|
|
||||||
if !reflect.DeepEqual(gotSample, wantSample) {
|
|
||||||
t.Errorf("SamplePrefixes(%d, %d) = %+v, want %+v",
|
|
||||||
ipVersion, prefix.Bits(), gotSample, wantSample)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, asn := range []int{asA, asB} {
|
|
||||||
if !reflect.DeepEqual(got.OriginatedPrefixes(asn),
|
|
||||||
want.OriginatedPrefixes(asn)) {
|
|
||||||
t.Errorf("OriginatedPrefixes(%d) = %+v, want %+v", asn,
|
|
||||||
got.OriginatedPrefixes(asn), want.OriginatedPrefixes(asn))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// sortedSample returns every prefix of the IP version and mask length in
|
|
||||||
// table, in prefix order.
|
|
||||||
func sortedSample(
|
|
||||||
table *routetable.Table, ipVersion, maskLength int,
|
|
||||||
) []routetable.PrefixSummary {
|
|
||||||
// More than the tests have of any mask length.
|
|
||||||
const all = 100
|
|
||||||
|
|
||||||
sample := table.SamplePrefixes(ipVersion, maskLength, all)
|
|
||||||
slices.SortFunc(sample, func(a, b routetable.PrefixSummary) int {
|
|
||||||
return a.Prefix.Compare(b.Prefix)
|
|
||||||
})
|
|
||||||
|
|
||||||
return sample
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestWriteRouteChangesDeletesTheRowOfARemovedPair(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := newRoutesTestDatabase(t)
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
|
|
||||||
announceTo(table, tablePrefixV4, asA, peerA, 10)
|
|
||||||
announceTo(table, tablePrefixV4, asB, peerB, 20)
|
|
||||||
writeTableChanges(t, db, table)
|
|
||||||
|
|
||||||
// The prefix's only peer with asA withdraws it.
|
|
||||||
table.Withdraw(netip.MustParsePrefix(tablePrefixV4), peerA)
|
|
||||||
writeTableChanges(t, db, table)
|
|
||||||
|
|
||||||
var origins []int
|
|
||||||
|
|
||||||
rows, err := db.db.QueryContext(t.Context(),
|
|
||||||
"SELECT origin_asn FROM routes WHERE prefix = ?", tablePrefixV4)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("reading the routes: %v", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = rows.Close() }()
|
|
||||||
|
|
||||||
for rows.Next() {
|
|
||||||
var origin int
|
|
||||||
|
|
||||||
err := rows.Scan(&origin)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("reading a route: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
origins = append(origins, origin)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = rows.Err()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("reading the routes: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !slices.Equal(origins, []int{asB}) {
|
|
||||||
t.Errorf("rows of %s have origins %v, want only %d",
|
|
||||||
tablePrefixV4, origins, asB)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// routePeer is one row of route_peers.
|
|
||||||
type routePeer struct {
|
|
||||||
number uint
|
|
||||||
address string
|
|
||||||
}
|
|
||||||
|
|
||||||
// readRoutePeerRows returns the rows of route_peers in number order.
|
|
||||||
func readRoutePeerRows(t *testing.T, db *Database) []routePeer {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
rows, err := db.db.QueryContext(t.Context(),
|
|
||||||
"SELECT number, address FROM route_peers ORDER BY number")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("reading the route peers: %v", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = rows.Close() }()
|
|
||||||
|
|
||||||
var peers []routePeer
|
|
||||||
|
|
||||||
for rows.Next() {
|
|
||||||
var peer routePeer
|
|
||||||
|
|
||||||
err := rows.Scan(&peer.number, &peer.address)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("reading a route peer: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
peers = append(peers, peer)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = rows.Err()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("reading the route peers: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return peers
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestWriteRouteChangesStoresNewPeers(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := newRoutesTestDatabase(t)
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
|
|
||||||
announceTo(table, tablePrefixV4, asA, peerA, 10)
|
|
||||||
announceTo(table, tablePrefixV6, asA, peerC, 20)
|
|
||||||
writeTableChanges(t, db, table)
|
|
||||||
|
|
||||||
// peerA is stored already; peerB is new.
|
|
||||||
announceTo(table, tablePrefixV4, asA, peerA, 30)
|
|
||||||
announceTo(table, tablePrefixV4, asA, peerB, 40)
|
|
||||||
writeTableChanges(t, db, table)
|
|
||||||
|
|
||||||
want := []routePeer{{0, peerA}, {1, peerC}, {2, peerB}}
|
|
||||||
|
|
||||||
got := readRoutePeerRows(t, db)
|
|
||||||
if !slices.Equal(got, want) {
|
|
||||||
t.Errorf("route_peers = %+v, want %+v", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestWriteRouteChangesWritesNothingWhenItFails(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := newRoutesTestDatabase(t)
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
|
|
||||||
announceTo(table, tablePrefixV4, asA, peerA, 10)
|
|
||||||
announceTo(table, tablePrefixV6, asA, peerB, 20)
|
|
||||||
|
|
||||||
// peerB's number is missing, so its kept route cannot be written.
|
|
||||||
changes, newPeers := table.TakeChanges(math.MaxInt)
|
|
||||||
delete(newPeers, peerB)
|
|
||||||
|
|
||||||
err := db.WriteRouteChanges(changes, newPeers)
|
|
||||||
if !errors.Is(err, errUnknownRoutePeer) {
|
|
||||||
t.Fatalf("WriteRouteChanges = %v, want %v", err, errUnknownRoutePeer)
|
|
||||||
}
|
|
||||||
|
|
||||||
peers := readRoutePeerRows(t, db)
|
|
||||||
if len(peers) != 0 {
|
|
||||||
t.Errorf("route_peers = %+v after the failed write, want none", peers)
|
|
||||||
}
|
|
||||||
|
|
||||||
var routes int
|
|
||||||
|
|
||||||
err = db.db.QueryRowContext(t.Context(),
|
|
||||||
"SELECT COUNT(*) FROM routes").Scan(&routes)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("counting the routes: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if routes != 0 {
|
|
||||||
t.Errorf("routes holds %d rows after the failed write, want none", routes)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Parameters for BenchmarkReadRouteTable.
|
|
||||||
const (
|
|
||||||
// benchTablePairs is how many prefix and origin AS pairs the table holds:
|
|
||||||
// one origin AS for each prefix. Every benchTableV6Every-th prefix is an
|
|
||||||
// IPv6 /48, the others IPv4 /24s.
|
|
||||||
benchTablePairs = 200000
|
|
||||||
benchTableV6Every = 5
|
|
||||||
// benchTablePeers is how many peers there are, about as many as RIS has.
|
|
||||||
// benchTablePeersPerPair is how many of them, picked at random, carry each
|
|
||||||
// pair: RIS's 460 million routes over 1.4 million pairs.
|
|
||||||
benchTablePeers = 1400
|
|
||||||
benchTablePeersPerPair = 330
|
|
||||||
// benchTableBatch is how many changes each write holds.
|
|
||||||
benchTableBatch = 10000
|
|
||||||
)
|
|
||||||
|
|
||||||
// BenchmarkReadRouteTable times the read of a routing table of 200,000 prefix
|
|
||||||
// and origin AS pairs, each carried by 330 of 1,400 peers and keeping two
|
|
||||||
// routes. It writes that table first. It reports the bytes of the database
|
|
||||||
// file per pair. It is run by hand, not by make test, for example in the test
|
|
||||||
// image:
|
|
||||||
//
|
|
||||||
// go test -run '^$' -bench ReadRouteTable -benchtime 5x ./internal/database/
|
|
||||||
func BenchmarkReadRouteTable(b *testing.B) {
|
|
||||||
db, err := New(&config.Config{StateDir: b.TempDir()}, logger.New())
|
|
||||||
if err != nil {
|
|
||||||
b.Fatalf("failed to create database: %v", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = db.Close() }()
|
|
||||||
|
|
||||||
//nolint:gosec // The benchmark needs the same table every run.
|
|
||||||
rng := rand.New(rand.NewPCG(1, 2))
|
|
||||||
|
|
||||||
newPeers := make(map[string]uint, benchTablePeers)
|
|
||||||
for number := range uint(benchTablePeers) {
|
|
||||||
newPeers[benchPeerAddress(number)] = number
|
|
||||||
}
|
|
||||||
|
|
||||||
for first := 0; first < benchTablePairs; first += benchTableBatch {
|
|
||||||
changes := make([]routetable.Change, 0, benchTableBatch)
|
|
||||||
for pair := first; pair < first+benchTableBatch; pair++ {
|
|
||||||
changes = append(changes, benchTableChange(rng, pair))
|
|
||||||
}
|
|
||||||
|
|
||||||
err := db.WriteRouteChanges(changes, newPeers)
|
|
||||||
if err != nil {
|
|
||||||
b.Fatalf("WriteRouteChanges: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
newPeers = nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// The WAL is copied into the database file, whose size is reported.
|
|
||||||
err = db.Checkpoint(b.Context())
|
|
||||||
if err != nil {
|
|
||||||
b.Fatalf("Checkpoint: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
file, err := os.Stat(db.path)
|
|
||||||
if err != nil {
|
|
||||||
b.Fatalf("reading the database file's size: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for b.Loop() {
|
|
||||||
table, err := db.ReadRouteTable(b.Context())
|
|
||||||
if err != nil {
|
|
||||||
b.Fatalf("ReadRouteTable: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
counts := table.Counts()
|
|
||||||
if counts.IPv4Prefixes+counts.IPv6Prefixes != benchTablePairs {
|
|
||||||
b.Fatalf("read %d prefixes, want %d",
|
|
||||||
counts.IPv4Prefixes+counts.IPv6Prefixes, benchTablePairs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// After the loop: b.Loop drops the metrics reported before it starts.
|
|
||||||
b.ReportMetric(float64(file.Size())/benchTablePairs, "file-bytes/pair")
|
|
||||||
}
|
|
||||||
|
|
||||||
// benchPeerAddress returns the address of the peer numbered number.
|
|
||||||
func benchPeerAddress(number uint) string {
|
|
||||||
const addressesPerBlock = 256
|
|
||||||
|
|
||||||
return fmt.Sprintf("198.18.%d.%d",
|
|
||||||
number/addressesPerBlock, number%addressesPerBlock)
|
|
||||||
}
|
|
||||||
|
|
||||||
// benchTableChange returns the content of the pair numbered pair: its prefix,
|
|
||||||
// its origin AS, benchTablePeersPerPair peers picked at random, and the routes
|
|
||||||
// of two of them, each with an AS path of four ASes.
|
|
||||||
func benchTableChange(rng *rand.Rand, pair int) routetable.Change {
|
|
||||||
const (
|
|
||||||
firstOrigin = 64500
|
|
||||||
transitASes = 50000
|
|
||||||
olderSeconds = 10
|
|
||||||
)
|
|
||||||
|
|
||||||
text := fmt.Sprintf("%d.%d.%d.0/24", 1+pair>>16, (pair>>8)%256, pair%256)
|
|
||||||
if pair%benchTableV6Every == 0 {
|
|
||||||
text = fmt.Sprintf("2001:%x:%x::/48", pair>>16, pair%65536)
|
|
||||||
}
|
|
||||||
|
|
||||||
prefix := netip.MustParsePrefix(text)
|
|
||||||
origin := firstOrigin + pair%transitASes
|
|
||||||
peers := bitset.New(benchTablePeers)
|
|
||||||
|
|
||||||
for peers.Count() < benchTablePeersPerPair {
|
|
||||||
peers.Set(rng.UintN(benchTablePeers))
|
|
||||||
}
|
|
||||||
|
|
||||||
change := routetable.Change{Prefix: prefix, OriginASN: origin, Peers: peers}
|
|
||||||
|
|
||||||
// The routes of the first two peers, newest first.
|
|
||||||
for i, number := range slices.Collect(peers.EachSet())[:keptRoutesPerRow] {
|
|
||||||
change.Routes = append(change.Routes, routetable.Route{
|
|
||||||
Peer: benchPeerAddress(number),
|
|
||||||
ASPath: []int{
|
|
||||||
firstOrigin + rng.IntN(transitASes),
|
|
||||||
firstOrigin + rng.IntN(transitASes),
|
|
||||||
firstOrigin + rng.IntN(transitASes),
|
|
||||||
origin,
|
|
||||||
},
|
|
||||||
Time: tableTime(pair - i*olderSeconds),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return change
|
|
||||||
}
|
|
||||||
@@ -27,18 +27,21 @@ CREATE TABLE IF NOT EXISTS asns (
|
|||||||
whois_updated_at DATETIME -- when we last fetched WHOIS data
|
whois_updated_at DATETIME -- when we last fetched WHOIS data
|
||||||
);
|
);
|
||||||
|
|
||||||
-- IPv4 prefixes table. Every read and write finds a row by its prefix, so the
|
-- IPv4 prefixes table
|
||||||
-- rows are stored by prefix (WITHOUT ROWID) and need no index of their own.
|
|
||||||
CREATE TABLE IF NOT EXISTS prefixes_v4 (
|
CREATE TABLE IF NOT EXISTS prefixes_v4 (
|
||||||
prefix TEXT PRIMARY KEY,
|
id TEXT PRIMARY KEY,
|
||||||
first_seen DATETIME NOT NULL
|
prefix TEXT UNIQUE NOT NULL,
|
||||||
) WITHOUT ROWID;
|
first_seen DATETIME NOT NULL,
|
||||||
|
last_seen DATETIME NOT NULL
|
||||||
|
);
|
||||||
|
|
||||||
-- IPv6 prefixes table, stored as prefixes_v4 is.
|
-- IPv6 prefixes table
|
||||||
CREATE TABLE IF NOT EXISTS prefixes_v6 (
|
CREATE TABLE IF NOT EXISTS prefixes_v6 (
|
||||||
prefix TEXT PRIMARY KEY,
|
id TEXT PRIMARY KEY,
|
||||||
first_seen DATETIME NOT NULL
|
prefix TEXT UNIQUE NOT NULL,
|
||||||
) WITHOUT ROWID;
|
first_seen DATETIME NOT NULL,
|
||||||
|
last_seen DATETIME NOT NULL
|
||||||
|
);
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS announcements (
|
CREATE TABLE IF NOT EXISTS announcements (
|
||||||
id TEXT PRIMARY KEY,
|
id TEXT PRIMARY KEY,
|
||||||
@@ -72,6 +75,11 @@ CREATE TABLE IF NOT EXISTS bgp_peers (
|
|||||||
last_message_type TEXT
|
last_message_type TEXT
|
||||||
);
|
);
|
||||||
|
|
||||||
|
-- Indexes for prefixes_v4 table
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_prefixes_v4_prefix ON prefixes_v4(prefix);
|
||||||
|
|
||||||
|
-- Indexes for prefixes_v6 table
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_prefixes_v6_prefix ON prefixes_v6(prefix);
|
||||||
CREATE INDEX IF NOT EXISTS idx_announcements_timestamp ON announcements(timestamp);
|
CREATE INDEX IF NOT EXISTS idx_announcements_timestamp ON announcements(timestamp);
|
||||||
CREATE INDEX IF NOT EXISTS idx_announcements_prefix_id ON announcements(prefix_id);
|
CREATE INDEX IF NOT EXISTS idx_announcements_prefix_id ON announcements(prefix_id);
|
||||||
CREATE INDEX IF NOT EXISTS idx_announcements_peer_asn ON announcements(peer_asn);
|
CREATE INDEX IF NOT EXISTS idx_announcements_peer_asn ON announcements(peer_asn);
|
||||||
@@ -83,40 +91,50 @@ CREATE INDEX IF NOT EXISTS idx_peerings_lookup ON peerings(as_a, as_b);
|
|||||||
-- Indexes for asns table
|
-- Indexes for asns table
|
||||||
CREATE INDEX IF NOT EXISTS idx_asns_asn ON asns(asn);
|
CREATE INDEX IF NOT EXISTS idx_asns_asn ON asns(asn);
|
||||||
CREATE INDEX IF NOT EXISTS idx_asns_whois_updated_at ON asns(whois_updated_at);
|
CREATE INDEX IF NOT EXISTS idx_asns_whois_updated_at ON asns(whois_updated_at);
|
||||||
-- Finds the AS last seen most recently and the one last seen longest ago, for
|
|
||||||
-- the home page.
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_asns_last_seen ON asns(last_seen);
|
|
||||||
|
|
||||||
-- Indexes for bgp_peers table
|
-- Indexes for bgp_peers table
|
||||||
CREATE INDEX IF NOT EXISTS idx_bgp_peers_asn ON bgp_peers(peer_asn);
|
CREATE INDEX IF NOT EXISTS idx_bgp_peers_asn ON bgp_peers(peer_asn);
|
||||||
CREATE INDEX IF NOT EXISTS idx_bgp_peers_last_seen ON bgp_peers(last_seen);
|
CREATE INDEX IF NOT EXISTS idx_bgp_peers_last_seen ON bgp_peers(last_seen);
|
||||||
CREATE INDEX IF NOT EXISTS idx_bgp_peers_ip ON bgp_peers(peer_ip);
|
CREATE INDEX IF NOT EXISTS idx_bgp_peers_ip ON bgp_peers(peer_ip);
|
||||||
|
|
||||||
-- The whole routing table, as internal/routetable keeps it in memory: one row
|
-- IPv4 routing table maintained by PrefixHandler
|
||||||
-- for each prefix and origin AS that some peer carries. Startup reads every
|
CREATE TABLE IF NOT EXISTS live_routes_v4 (
|
||||||
-- row, and the rows of the pairs that change are written back.
|
id TEXT PRIMARY KEY,
|
||||||
CREATE TABLE IF NOT EXISTS routes (
|
prefix TEXT NOT NULL,
|
||||||
prefix TEXT NOT NULL, -- in the text form Go's net/netip prints
|
mask_length INTEGER NOT NULL, -- CIDR mask length (0-32)
|
||||||
origin_asn INTEGER NOT NULL,
|
origin_asn INTEGER NOT NULL,
|
||||||
-- The peers whose current route to the prefix has this origin AS: a set
|
peer_ip TEXT NOT NULL,
|
||||||
-- of their numbers in route_peers, as the bitset package's MarshalBinary
|
as_path TEXT NOT NULL, -- JSON array
|
||||||
-- writes it.
|
next_hop TEXT NOT NULL,
|
||||||
peers BLOB NOT NULL,
|
last_updated DATETIME NOT NULL,
|
||||||
-- The newest of those routes and the one before it, the two kept in full.
|
UNIQUE(prefix, origin_asn, peer_ip)
|
||||||
-- Each has its peer's number in route_peers, its AS path as a JSON array,
|
|
||||||
-- and its time in seconds since 1970, UTC. A route that is not kept has
|
|
||||||
-- NULL in its three columns.
|
|
||||||
newest_peer INTEGER,
|
|
||||||
newest_as_path TEXT,
|
|
||||||
newest_time INTEGER,
|
|
||||||
older_peer INTEGER,
|
|
||||||
older_as_path TEXT,
|
|
||||||
older_time INTEGER,
|
|
||||||
PRIMARY KEY (prefix, origin_asn)
|
|
||||||
) WITHOUT ROWID;
|
|
||||||
|
|
||||||
-- The peers of the routes table, each with the number it has there.
|
|
||||||
CREATE TABLE IF NOT EXISTS route_peers (
|
|
||||||
number INTEGER PRIMARY KEY,
|
|
||||||
address TEXT NOT NULL UNIQUE
|
|
||||||
);
|
);
|
||||||
|
|
||||||
|
-- IPv6 routing table maintained by PrefixHandler
|
||||||
|
CREATE TABLE IF NOT EXISTS live_routes_v6 (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
prefix TEXT NOT NULL,
|
||||||
|
mask_length INTEGER NOT NULL, -- CIDR mask length (0-128)
|
||||||
|
origin_asn INTEGER NOT NULL,
|
||||||
|
peer_ip TEXT NOT NULL,
|
||||||
|
as_path TEXT NOT NULL, -- JSON array
|
||||||
|
next_hop TEXT NOT NULL,
|
||||||
|
last_updated DATETIME NOT NULL,
|
||||||
|
UNIQUE(prefix, origin_asn, peer_ip)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Indexes for live_routes_v4 table
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v4_prefix ON live_routes_v4(prefix);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v4_mask_length ON live_routes_v4(mask_length);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v4_origin_asn ON live_routes_v4(origin_asn);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v4_last_updated ON live_routes_v4(last_updated);
|
||||||
|
-- Index to optimize prefix distribution queries
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v4_mask_prefix ON live_routes_v4(mask_length, prefix);
|
||||||
|
|
||||||
|
-- Indexes for live_routes_v6 table
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v6_prefix ON live_routes_v6(prefix);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v6_mask_length ON live_routes_v6(mask_length);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v6_origin_asn ON live_routes_v6(origin_asn);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v6_last_updated ON live_routes_v6(last_updated);
|
||||||
|
-- Index to optimize prefix distribution queries
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v6_mask_prefix ON live_routes_v6(mask_length, prefix);
|
||||||
@@ -18,12 +18,30 @@ func logSlowQuery(logger *logger.Logger, query string, start time.Time) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// exec runs a statement on the writing connection, with slow query logging
|
// queryRow wraps QueryRow with slow query logging
|
||||||
func (d *Database) exec(query string, args ...any) error {
|
// nolint:unused // kept for consistency with other query wrappers
|
||||||
|
func (d *Database) queryRow(query string, args ...interface{}) *sql.Row {
|
||||||
start := time.Now()
|
start := time.Now()
|
||||||
defer logSlowQuery(d.logger, query, start)
|
defer logSlowQuery(d.logger, query, start)
|
||||||
|
|
||||||
_, err := d.writer.ExecContext(context.Background(), query, args...)
|
return d.db.QueryRow(query, args...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// query wraps Query with slow query logging
|
||||||
|
// nolint:unused // kept for future use to ensure all queries go through slow query logging
|
||||||
|
func (d *Database) query(query string, args ...interface{}) (*sql.Rows, error) {
|
||||||
|
start := time.Now()
|
||||||
|
defer logSlowQuery(d.logger, query, start)
|
||||||
|
|
||||||
|
return d.db.Query(query, args...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// exec wraps Exec with slow query logging
|
||||||
|
func (d *Database) exec(query string, args ...interface{}) error {
|
||||||
|
start := time.Now()
|
||||||
|
defer logSlowQuery(d.logger, query, start)
|
||||||
|
|
||||||
|
_, err := d.db.Exec(query, args...)
|
||||||
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -31,51 +49,51 @@ func (d *Database) exec(query string, args ...any) error {
|
|||||||
// loggingTx wraps sql.Tx to log slow queries
|
// loggingTx wraps sql.Tx to log slow queries
|
||||||
type loggingTx struct {
|
type loggingTx struct {
|
||||||
*sql.Tx
|
*sql.Tx
|
||||||
|
|
||||||
logger *logger.Logger
|
logger *logger.Logger
|
||||||
}
|
}
|
||||||
|
|
||||||
// QueryRow wraps sql.Tx.QueryRow to log slow queries
|
// QueryRow wraps sql.Tx.QueryRow to log slow queries
|
||||||
func (tx *loggingTx) QueryRow(query string, args ...any) *sql.Row {
|
func (tx *loggingTx) QueryRow(query string, args ...interface{}) *sql.Row {
|
||||||
start := time.Now()
|
start := time.Now()
|
||||||
defer logSlowQuery(tx.logger, query, start)
|
defer logSlowQuery(tx.logger, query, start)
|
||||||
|
|
||||||
return tx.Tx.QueryRowContext(context.Background(), query, args...)
|
return tx.Tx.QueryRow(query, args...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Query wraps sql.Tx.Query to log slow queries
|
||||||
|
func (tx *loggingTx) Query(query string, args ...interface{}) (*sql.Rows, error) {
|
||||||
|
start := time.Now()
|
||||||
|
defer logSlowQuery(tx.logger, query, start)
|
||||||
|
|
||||||
|
return tx.Tx.Query(query, args...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// QueryContext wraps sql.Tx.QueryContext to log slow queries
|
||||||
|
func (tx *loggingTx) QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error) {
|
||||||
|
start := time.Now()
|
||||||
|
defer logSlowQuery(tx.logger, query, start)
|
||||||
|
|
||||||
|
return tx.Tx.QueryContext(ctx, query, args...)
|
||||||
}
|
}
|
||||||
|
|
||||||
// QueryRowContext wraps sql.Tx.QueryRowContext to log slow queries
|
// QueryRowContext wraps sql.Tx.QueryRowContext to log slow queries
|
||||||
func (tx *loggingTx) QueryRowContext(
|
func (tx *loggingTx) QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row {
|
||||||
ctx context.Context, query string, args ...any,
|
|
||||||
) *sql.Row {
|
|
||||||
start := time.Now()
|
start := time.Now()
|
||||||
defer logSlowQuery(tx.logger, query, start)
|
defer logSlowQuery(tx.logger, query, start)
|
||||||
|
|
||||||
return tx.Tx.QueryRowContext(ctx, query, args...)
|
return tx.Tx.QueryRowContext(ctx, query, args...)
|
||||||
}
|
}
|
||||||
|
|
||||||
// QueryContext wraps sql.Tx.QueryContext to log slow queries
|
|
||||||
func (tx *loggingTx) QueryContext(
|
|
||||||
ctx context.Context, query string, args ...any,
|
|
||||||
) (*sql.Rows, error) {
|
|
||||||
start := time.Now()
|
|
||||||
defer logSlowQuery(tx.logger, query, start)
|
|
||||||
|
|
||||||
//nolint:sqlclosecheck // returns the rows, which the caller closes
|
|
||||||
return tx.Tx.QueryContext(ctx, query, args...)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Exec wraps sql.Tx.Exec to log slow queries
|
// Exec wraps sql.Tx.Exec to log slow queries
|
||||||
func (tx *loggingTx) Exec(query string, args ...any) (sql.Result, error) {
|
func (tx *loggingTx) Exec(query string, args ...interface{}) (sql.Result, error) {
|
||||||
start := time.Now()
|
start := time.Now()
|
||||||
defer logSlowQuery(tx.logger, query, start)
|
defer logSlowQuery(tx.logger, query, start)
|
||||||
|
|
||||||
return tx.Tx.ExecContext(context.Background(), query, args...)
|
return tx.Tx.Exec(query, args...)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExecContext wraps sql.Tx.ExecContext to log slow queries
|
// ExecContext wraps sql.Tx.ExecContext to log slow queries
|
||||||
func (tx *loggingTx) ExecContext(
|
func (tx *loggingTx) ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error) {
|
||||||
ctx context.Context, query string, args ...any,
|
|
||||||
) (sql.Result, error) {
|
|
||||||
start := time.Now()
|
start := time.Now()
|
||||||
defer logSlowQuery(tx.logger, query, start)
|
defer logSlowQuery(tx.logger, query, start)
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
package database
|
package database
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
@@ -18,3 +20,14 @@ func detectIPVersion(prefix string) int {
|
|||||||
|
|
||||||
return ipVersionV4
|
return ipVersionV4
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// prefixMaskLength returns the mask length of a prefix such as 192.0.2.0/24.
|
||||||
|
func prefixMaskLength(prefix string) (int, error) {
|
||||||
|
_, network, err := net.ParseCIDR(prefix)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("invalid prefix %s: %w", prefix, err)
|
||||||
|
}
|
||||||
|
maskLength, _ := network.Mask.Size()
|
||||||
|
|
||||||
|
return maskLength, nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -74,7 +74,6 @@ func getSourceAttrs() []slog.Attr {
|
|||||||
|
|
||||||
// Get the function name
|
// Get the function name
|
||||||
fn := runtime.FuncForPC(pc)
|
fn := runtime.FuncForPC(pc)
|
||||||
|
|
||||||
var funcName string
|
var funcName string
|
||||||
if fn != nil {
|
if fn != nil {
|
||||||
funcName = filepath.Base(fn.Name())
|
funcName = filepath.Base(fn.Name())
|
||||||
|
|||||||
+12
-19
@@ -34,7 +34,7 @@ type Tracker struct {
|
|||||||
// Announcement/withdrawal metrics
|
// Announcement/withdrawal metrics
|
||||||
announcementCounter metrics.Counter
|
announcementCounter metrics.Counter
|
||||||
withdrawalCounter metrics.Counter
|
withdrawalCounter metrics.Counter
|
||||||
churnPerMinute minuteCounter // prefix announcements and withdrawals
|
churnRate metrics.Meter // combined announcements + withdrawals per second
|
||||||
|
|
||||||
// BGP peer tracking
|
// BGP peer tracking
|
||||||
bgpPeerCount atomic.Int32
|
bgpPeerCount atomic.Int32
|
||||||
@@ -56,7 +56,7 @@ func New() *Tracker {
|
|||||||
ipv6UpdateRate: metrics.NewMeter(),
|
ipv6UpdateRate: metrics.NewMeter(),
|
||||||
announcementCounter: metrics.NewCounter(),
|
announcementCounter: metrics.NewCounter(),
|
||||||
withdrawalCounter: metrics.NewCounter(),
|
withdrawalCounter: metrics.NewCounter(),
|
||||||
churnPerMinute: minuteCounter{now: time.Now},
|
churnRate: metrics.NewMeter(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -64,7 +64,6 @@ func New() *Tracker {
|
|||||||
// first counts one reconnect.
|
// first counts one reconnect.
|
||||||
func (t *Tracker) SetConnected(connected bool) {
|
func (t *Tracker) SetConnected(connected bool) {
|
||||||
t.isConnected.Store(connected)
|
t.isConnected.Store(connected)
|
||||||
|
|
||||||
if !connected {
|
if !connected {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -121,11 +120,9 @@ func (t *Tracker) GetStreamMetrics() StreamMetrics {
|
|||||||
if msgCount >= 0 {
|
if msgCount >= 0 {
|
||||||
totalMessages = uint64(msgCount)
|
totalMessages = uint64(msgCount)
|
||||||
}
|
}
|
||||||
|
|
||||||
if byteCount >= 0 {
|
if byteCount >= 0 {
|
||||||
totalBytes = uint64(byteCount)
|
totalBytes = uint64(byteCount)
|
||||||
}
|
}
|
||||||
|
|
||||||
if wireByteCount >= 0 {
|
if wireByteCount >= 0 {
|
||||||
totalWireBytes = uint64(wireByteCount)
|
totalWireBytes = uint64(wireByteCount)
|
||||||
}
|
}
|
||||||
@@ -156,20 +153,20 @@ func (t *Tracker) RecordIPv6Update() {
|
|||||||
// RecordAnnouncement records a route announcement
|
// RecordAnnouncement records a route announcement
|
||||||
func (t *Tracker) RecordAnnouncement() {
|
func (t *Tracker) RecordAnnouncement() {
|
||||||
t.announcementCounter.Inc(1)
|
t.announcementCounter.Inc(1)
|
||||||
t.churnPerMinute.add()
|
t.churnRate.Mark(1)
|
||||||
}
|
}
|
||||||
|
|
||||||
// RecordWithdrawal records a route withdrawal
|
// RecordWithdrawal records a route withdrawal
|
||||||
func (t *Tracker) RecordWithdrawal() {
|
func (t *Tracker) RecordWithdrawal() {
|
||||||
t.withdrawalCounter.Inc(1)
|
t.withdrawalCounter.Inc(1)
|
||||||
t.churnPerMinute.add()
|
t.churnRate.Mark(1)
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetBGPPeerCount updates the current BGP peer count
|
// SetBGPPeerCount updates the current BGP peer count
|
||||||
func (t *Tracker) SetBGPPeerCount(count int) {
|
func (t *Tracker) SetBGPPeerCount(count int) {
|
||||||
// BGP peer count is always small (< 1000), so int32 is safe
|
// BGP peer count is always small (< 1000), so int32 is safe
|
||||||
if count >= 0 && count < 1<<31 {
|
if count > 0 && count < 1<<31 {
|
||||||
t.bgpPeerCount.Store(int32(count))
|
t.bgpPeerCount.Store(int32(count)) //nolint:gosec // count is validated
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -198,10 +195,9 @@ func (t *Tracker) GetWithdrawalCount() uint64 {
|
|||||||
return uint64(count)
|
return uint64(count)
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetChurnPerMinute returns the number of prefix announcements and
|
// GetChurnRate returns the route churn rate per second
|
||||||
// withdrawals, from all peers, in the last full clock minute.
|
func (t *Tracker) GetChurnRate() float64 {
|
||||||
func (t *Tracker) GetChurnPerMinute() int {
|
return t.churnRate.Rate1()
|
||||||
return t.churnPerMinute.lastFullMinute()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetRouteMetrics returns current route update metrics
|
// GetRouteMetrics returns current route update metrics
|
||||||
@@ -218,8 +214,7 @@ type StreamMetrics struct {
|
|||||||
TotalMessages uint64
|
TotalMessages uint64
|
||||||
// TotalBytes is the total number of decompressed bytes received since startup
|
// TotalBytes is the total number of decompressed bytes received since startup
|
||||||
TotalBytes uint64
|
TotalBytes uint64
|
||||||
// TotalWireBytes is the total number of bytes received on the wire (before
|
// TotalWireBytes is the total number of bytes received on the wire (before decompression)
|
||||||
// decompression)
|
|
||||||
TotalWireBytes uint64
|
TotalWireBytes uint64
|
||||||
// ConnectedSince is the time when the current connection was established
|
// ConnectedSince is the time when the current connection was established
|
||||||
ConnectedSince time.Time
|
ConnectedSince time.Time
|
||||||
@@ -227,11 +222,9 @@ type StreamMetrics struct {
|
|||||||
Connected bool
|
Connected bool
|
||||||
// MessagesPerSec is the rate of messages received per second (1-minute average)
|
// MessagesPerSec is the rate of messages received per second (1-minute average)
|
||||||
MessagesPerSec float64
|
MessagesPerSec float64
|
||||||
// BitsPerSec is the rate of decompressed bits received per second (1-minute
|
// BitsPerSec is the rate of decompressed bits received per second (1-minute average)
|
||||||
// average)
|
|
||||||
BitsPerSec float64
|
BitsPerSec float64
|
||||||
// WireBitsPerSec is the rate of bits received on the wire per second
|
// WireBitsPerSec is the rate of bits received on the wire per second (1-minute average)
|
||||||
// (1-minute average)
|
|
||||||
WireBitsPerSec float64
|
WireBitsPerSec float64
|
||||||
// ReconnectCount is the number of reconnections since startup
|
// ReconnectCount is the number of reconnections since startup
|
||||||
ReconnectCount uint64
|
ReconnectCount uint64
|
||||||
|
|||||||
@@ -1,18 +1,12 @@
|
|||||||
package metrics_test
|
package metrics
|
||||||
|
|
||||||
import (
|
import "testing"
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/metrics"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestReconnectCount connects three times with a disconnect between each, the
|
// TestReconnectCount connects three times with a disconnect between each, the
|
||||||
// way the streamer does, and expects every connection after the first to count
|
// way the streamer does, and expects every connection after the first to count
|
||||||
// one reconnect.
|
// one reconnect.
|
||||||
func TestReconnectCount(t *testing.T) {
|
func TestReconnectCount(t *testing.T) {
|
||||||
t.Parallel()
|
tracker := New()
|
||||||
|
|
||||||
tracker := metrics.New()
|
|
||||||
|
|
||||||
tracker.SetConnected(true)
|
tracker.SetConnected(true)
|
||||||
tracker.SetConnected(false)
|
tracker.SetConnected(false)
|
||||||
@@ -25,18 +19,3 @@ func TestReconnectCount(t *testing.T) {
|
|||||||
t.Fatalf("reconnect count = %d, want %d", got, wantReconnects)
|
t.Fatalf("reconnect count = %d, want %d", got, wantReconnects)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestBGPPeerCountDropsToZero sets a peer count of 1 and then 0, the way the
|
|
||||||
// streamer does when the last peer's session ends, and expects 0 back.
|
|
||||||
func TestBGPPeerCountDropsToZero(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
tracker := metrics.New()
|
|
||||||
|
|
||||||
tracker.SetBGPPeerCount(1)
|
|
||||||
tracker.SetBGPPeerCount(0)
|
|
||||||
|
|
||||||
if got := tracker.GetBGPPeerCount(); got != 0 {
|
|
||||||
t.Fatalf("peer count = %d after setting 0, want 0", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,59 +0,0 @@
|
|||||||
package metrics
|
|
||||||
|
|
||||||
import (
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// minuteCounter counts events in each clock minute and reports the count of
|
|
||||||
// the last full minute. It is safe for concurrent use.
|
|
||||||
type minuteCounter struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
now func() time.Time
|
|
||||||
|
|
||||||
// minute is the start of the minute that count belongs to.
|
|
||||||
minute time.Time
|
|
||||||
count int
|
|
||||||
// lastMinuteCount is the count of the minute just before minute.
|
|
||||||
lastMinuteCount int
|
|
||||||
}
|
|
||||||
|
|
||||||
// add counts one event in the current minute.
|
|
||||||
func (c *minuteCounter) add() {
|
|
||||||
c.mu.Lock()
|
|
||||||
defer c.mu.Unlock()
|
|
||||||
|
|
||||||
c.moveToCurrentMinute()
|
|
||||||
c.count++
|
|
||||||
}
|
|
||||||
|
|
||||||
// lastFullMinute returns the number of events in the minute that ended most
|
|
||||||
// recently.
|
|
||||||
func (c *minuteCounter) lastFullMinute() int {
|
|
||||||
c.mu.Lock()
|
|
||||||
defer c.mu.Unlock()
|
|
||||||
|
|
||||||
c.moveToCurrentMinute()
|
|
||||||
|
|
||||||
return c.lastMinuteCount
|
|
||||||
}
|
|
||||||
|
|
||||||
// moveToCurrentMinute starts a new count when the clock has moved into
|
|
||||||
// another minute. The count it ends becomes the last full minute's count when
|
|
||||||
// its minute is the one just before the current minute. Otherwise the last
|
|
||||||
// full minute had no events. The caller holds mu.
|
|
||||||
func (c *minuteCounter) moveToCurrentMinute() {
|
|
||||||
minute := c.now().Truncate(time.Minute)
|
|
||||||
if minute.Equal(c.minute) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if minute.Sub(c.minute) == time.Minute {
|
|
||||||
c.lastMinuteCount = c.count
|
|
||||||
} else {
|
|
||||||
c.lastMinuteCount = 0
|
|
||||||
}
|
|
||||||
|
|
||||||
c.minute = minute
|
|
||||||
c.count = 0
|
|
||||||
}
|
|
||||||
@@ -1,86 +0,0 @@
|
|||||||
package metrics
|
|
||||||
|
|
||||||
import (
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestChurnPerMinuteCountsLastFullMinute announces and withdraws prefixes on
|
|
||||||
// an injected clock and expects each minute's whole count once that minute
|
|
||||||
// has ended, and 0 for a minute with none.
|
|
||||||
func TestChurnPerMinuteCountsLastFullMinute(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
now := time.Date(2026, 10, 26, 12, 0, 30, 0, time.UTC)
|
|
||||||
tracker := New()
|
|
||||||
tracker.churnPerMinute.now = func() time.Time { return now }
|
|
||||||
|
|
||||||
expectChurn := func(want int, why string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
got := tracker.GetChurnPerMinute()
|
|
||||||
if got != want {
|
|
||||||
t.Fatalf("at %s churn = %d, want %d: %s",
|
|
||||||
now.Format(time.TimeOnly), got, want, why)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const firstMinuteChurn = 2
|
|
||||||
|
|
||||||
tracker.RecordAnnouncement()
|
|
||||||
tracker.RecordWithdrawal()
|
|
||||||
expectChurn(0, "12:00 has not ended")
|
|
||||||
|
|
||||||
now = time.Date(2026, 10, 26, 12, 1, 59, 0, time.UTC)
|
|
||||||
|
|
||||||
tracker.RecordAnnouncement()
|
|
||||||
expectChurn(firstMinuteChurn, "12:00 had one announcement and one withdrawal")
|
|
||||||
|
|
||||||
now = time.Date(2026, 10, 26, 12, 2, 0, 0, time.UTC)
|
|
||||||
|
|
||||||
expectChurn(1, "12:01 had one announcement")
|
|
||||||
|
|
||||||
now = time.Date(2026, 10, 26, 12, 4, 0, 0, time.UTC)
|
|
||||||
|
|
||||||
expectChurn(0, "12:03 had none")
|
|
||||||
|
|
||||||
tracker.RecordWithdrawal()
|
|
||||||
|
|
||||||
now = time.Date(2026, 10, 26, 12, 6, 0, 0, time.UTC)
|
|
||||||
|
|
||||||
expectChurn(0, "12:05 had none, though 12:04 had one withdrawal")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestChurnPerMinuteCountsConcurrentRecords announces prefixes from several
|
|
||||||
// goroutines at once and expects every one counted.
|
|
||||||
func TestChurnPerMinuteCountsConcurrentRecords(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const (
|
|
||||||
goroutines = 8
|
|
||||||
recordsPerGoroutine = 1000
|
|
||||||
)
|
|
||||||
|
|
||||||
now := time.Date(2026, 10, 26, 12, 0, 0, 0, time.UTC)
|
|
||||||
tracker := New()
|
|
||||||
tracker.churnPerMinute.now = func() time.Time { return now }
|
|
||||||
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
for range goroutines {
|
|
||||||
wg.Go(func() {
|
|
||||||
for range recordsPerGoroutine {
|
|
||||||
tracker.RecordAnnouncement()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
wg.Wait()
|
|
||||||
|
|
||||||
now = now.Add(time.Minute)
|
|
||||||
|
|
||||||
want := goroutines * recordsPerGoroutine
|
|
||||||
if got := tracker.GetChurnPerMinute(); got != want {
|
|
||||||
t.Fatalf("churn = %d, want %d", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+30
-103
@@ -1,120 +1,53 @@
|
|||||||
// Package ristypes defines the data structures for RIS Live BGP messages and
|
// Package ristypes defines the data structures for RIS Live BGP messages and announcements.
|
||||||
// announcements.
|
|
||||||
package ristypes
|
package ristypes
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"slices"
|
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Message types and the session state the handlers act on, as the feed writes
|
// ASPath represents a BGP AS path as a slice of AS numbers.
|
||||||
// them in a message's type and state fields.
|
// It handles JSON unmarshaling of both simple arrays and nested AS sets,
|
||||||
const (
|
// flattening any nested structures into a single sequence of AS numbers.
|
||||||
// MessageTypeUpdate is a BGP UPDATE: announcements and withdrawals.
|
type ASPath []int
|
||||||
MessageTypeUpdate = "UPDATE"
|
|
||||||
|
|
||||||
// MessageTypeState is a change in the state of the BGP session between
|
// UnmarshalJSON implements the json.Unmarshaler interface for ASPath.
|
||||||
// the collector and the peer.
|
// It handles both simple integer arrays [1, 2, 3] and nested AS sets
|
||||||
MessageTypeState = "STATE"
|
// like [1, [2, 3], 4], flattening them into a single slice of integers.
|
||||||
|
func (p *ASPath) UnmarshalJSON(data []byte) error {
|
||||||
|
// First try to unmarshal as a simple array of integers
|
||||||
|
var simple []int
|
||||||
|
if err := json.Unmarshal(data, &simple); err == nil {
|
||||||
|
*p = ASPath(simple)
|
||||||
|
|
||||||
// StateDown is the state of a STATE message whose session has ended: the
|
return nil
|
||||||
// collector no longer has any of the peer's routes, and sends no
|
|
||||||
// withdrawals for them.
|
|
||||||
StateDown = "down"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ASPath is a BGP AS path as the feed writes it: an array of AS numbers in
|
|
||||||
// which an AS set is an array of its members, such as [1, 2, [3, 4]]. A path
|
|
||||||
// that holds AS 0 decodes as an empty one (UnmarshalJSON).
|
|
||||||
type ASPath struct {
|
|
||||||
// Sequence is the path's AS numbers in order. A set of one member that
|
|
||||||
// ends the path counts as that AS and is its last element. The members of
|
|
||||||
// a set anywhere else are flattened into it.
|
|
||||||
Sequence []int
|
|
||||||
|
|
||||||
// Set holds the members of a set of several members that ends the path.
|
|
||||||
// Any of them may be the origin, and they are not neighbours of each
|
|
||||||
// other or of the AS before them.
|
|
||||||
Set []int
|
|
||||||
}
|
|
||||||
|
|
||||||
// NoOrigin is the origin AS of a route whose origin is not known. AS 0 is
|
|
||||||
// reserved, so no AS has this number.
|
|
||||||
const NoOrigin = 0
|
|
||||||
|
|
||||||
// Origin returns the AS that originated the route: the last AS of the
|
|
||||||
// sequence, or NoOrigin when the path ends in a set of several members or is
|
|
||||||
// empty.
|
|
||||||
func (p *ASPath) Origin() int {
|
|
||||||
if len(p.Set) > 0 || len(p.Sequence) == 0 {
|
|
||||||
return NoOrigin
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return p.Sequence[len(p.Sequence)-1]
|
// If that fails, unmarshal as array of interfaces and flatten
|
||||||
}
|
var raw []interface{}
|
||||||
|
if err := json.Unmarshal(data, &raw); err != nil {
|
||||||
// UnmarshalJSON implements the json.Unmarshaler interface for ASPath. A path
|
|
||||||
// that holds AS 0 anywhere, a member of an AS set included, decodes as an empty
|
|
||||||
// path: RFC 7607 makes such a path malformed and its announcements
|
|
||||||
// withdrawals. On an announcement with an empty path the prefix handler
|
|
||||||
// deletes the peer's route, and the AS and peering handlers record nothing.
|
|
||||||
func (p *ASPath) UnmarshalJSON(data []byte) error {
|
|
||||||
path, err := decodeASPath(data)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if slices.Contains(path.Sequence, 0) || slices.Contains(path.Set, 0) {
|
// Flatten the array
|
||||||
path = ASPath{}
|
result := make([]int, 0)
|
||||||
}
|
for _, item := range raw {
|
||||||
|
|
||||||
*p = path
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// decodeASPath decodes an AS path as the feed writes it.
|
|
||||||
func decodeASPath(data []byte) (ASPath, error) {
|
|
||||||
// Most paths have no AS set and decode as a plain array of integers.
|
|
||||||
var sequence []int
|
|
||||||
|
|
||||||
err := json.Unmarshal(data, &sequence)
|
|
||||||
if err == nil {
|
|
||||||
return ASPath{Sequence: sequence}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var raw []any
|
|
||||||
|
|
||||||
err = json.Unmarshal(data, &raw)
|
|
||||||
if err != nil {
|
|
||||||
return ASPath{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
path := ASPath{Sequence: make([]int, 0, len(raw))}
|
|
||||||
|
|
||||||
for i, item := range raw {
|
|
||||||
switch v := item.(type) {
|
switch v := item.(type) {
|
||||||
case float64:
|
case float64:
|
||||||
path.Sequence = append(path.Sequence, int(v))
|
result = append(result, int(v))
|
||||||
case []any:
|
case []interface{}:
|
||||||
set := make([]int, 0, len(v))
|
// Nested array - flatten it
|
||||||
|
for _, nested := range v {
|
||||||
for _, member := range v {
|
if num, ok := nested.(float64); ok {
|
||||||
if num, ok := member.(float64); ok {
|
result = append(result, int(num))
|
||||||
set = append(set, int(num))
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if i == len(raw)-1 && len(set) > 1 {
|
|
||||||
path.Set = set
|
|
||||||
} else {
|
|
||||||
path.Sequence = append(path.Sequence, set...)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return path, nil
|
*p = ASPath(result)
|
||||||
|
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// RISLiveMessage represents the outer wrapper message from the RIPE RIS Live stream.
|
// RISLiveMessage represents the outer wrapper message from the RIPE RIS Live stream.
|
||||||
@@ -129,8 +62,6 @@ type RISLiveMessage struct {
|
|||||||
// It contains metadata about the BGP session (peer, ASN, host) along with
|
// It contains metadata about the BGP session (peer, ASN, host) along with
|
||||||
// the actual BGP update data including AS path, communities, announcements,
|
// the actual BGP update data including AS path, communities, announcements,
|
||||||
// and withdrawals.
|
// and withdrawals.
|
||||||
//
|
|
||||||
//nolint:tagliatelle // field names are the RIS Live feed's
|
|
||||||
type RISMessage struct {
|
type RISMessage struct {
|
||||||
Type string `json:"type"`
|
Type string `json:"type"`
|
||||||
Timestamp float64 `json:"timestamp"`
|
Timestamp float64 `json:"timestamp"`
|
||||||
@@ -142,7 +73,7 @@ type RISMessage struct {
|
|||||||
RRC string `json:"rrc,omitempty"`
|
RRC string `json:"rrc,omitempty"`
|
||||||
MrtTime float64 `json:"mrt_time,omitempty"`
|
MrtTime float64 `json:"mrt_time,omitempty"`
|
||||||
SocketTime float64 `json:"socket_time,omitempty"`
|
SocketTime float64 `json:"socket_time,omitempty"`
|
||||||
Path ASPath `json:"path,omitzero"`
|
Path ASPath `json:"path,omitempty"`
|
||||||
// Community and Raw are present in the feed but read by no handler.
|
// Community and Raw are present in the feed but read by no handler.
|
||||||
// They are the largest fields on a message that lives in up to four
|
// They are the largest fields on a message that lives in up to four
|
||||||
// handler queues, so json:"-" keeps them out of the decoded message
|
// handler queues, so json:"-" keeps them out of the decoded message
|
||||||
@@ -154,15 +85,11 @@ type RISMessage struct {
|
|||||||
Announcements []RISAnnouncement `json:"announcements,omitempty"`
|
Announcements []RISAnnouncement `json:"announcements,omitempty"`
|
||||||
Withdrawals []string `json:"withdrawals,omitempty"`
|
Withdrawals []string `json:"withdrawals,omitempty"`
|
||||||
Raw string `json:"-"`
|
Raw string `json:"-"`
|
||||||
// State is set on STATE messages: the session's new state, such as StateDown.
|
|
||||||
State string `json:"state,omitempty"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// RISAnnouncement represents a BGP route announcement within a RIS message.
|
// RISAnnouncement represents a BGP route announcement within a RIS message.
|
||||||
// It contains the next hop IP address and the list of prefixes being announced
|
// It contains the next hop IP address and the list of prefixes being announced
|
||||||
// via that next hop.
|
// via that next hop.
|
||||||
//
|
|
||||||
//nolint:tagliatelle // field names are the RIS Live feed's
|
|
||||||
type RISAnnouncement struct {
|
type RISAnnouncement struct {
|
||||||
NextHop string `json:"next_hop"`
|
NextHop string `json:"next_hop"`
|
||||||
Prefixes []string `json:"prefixes"`
|
Prefixes []string `json:"prefixes"`
|
||||||
|
|||||||
@@ -1,13 +1,10 @@
|
|||||||
package ristypes_test
|
package ristypes
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bufio"
|
"bufio"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"os"
|
"os"
|
||||||
"slices"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/ristypes"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// messageExamplesPath is the captured RIS Live feed used as decode fixtures,
|
// messageExamplesPath is the captured RIS Live feed used as decode fixtures,
|
||||||
@@ -18,58 +15,44 @@ const messageExamplesPath = "../../docs/message-examples.json"
|
|||||||
// streamer does and checks that the fields no handler reads (Community, Raw)
|
// streamer does and checks that the fields no handler reads (Community, Raw)
|
||||||
// stay empty while the fields handlers use (Path, Announcements) still decode.
|
// stay empty while the fields handlers use (Path, Announcements) still decode.
|
||||||
func TestDecodeDropsCommunityAndRaw(t *testing.T) {
|
func TestDecodeDropsCommunityAndRaw(t *testing.T) {
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
f, err := os.Open(messageExamplesPath)
|
f, err := os.Open(messageExamplesPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("open fixtures: %v", err)
|
t.Fatalf("open fixtures: %v", err)
|
||||||
}
|
}
|
||||||
|
defer f.Close()
|
||||||
defer func() {
|
|
||||||
_ = f.Close()
|
|
||||||
}()
|
|
||||||
|
|
||||||
var messages, withPath, withAnnouncements int
|
var messages, withPath, withAnnouncements int
|
||||||
|
|
||||||
scanner := bufio.NewScanner(f)
|
scanner := bufio.NewScanner(f)
|
||||||
scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
||||||
|
|
||||||
for scanner.Scan() {
|
for scanner.Scan() {
|
||||||
line := scanner.Bytes()
|
line := scanner.Bytes()
|
||||||
if len(line) == 0 {
|
if len(line) == 0 {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
var wrapper ristypes.RISLiveMessage
|
var wrapper RISLiveMessage
|
||||||
|
if err := json.Unmarshal(line, &wrapper); err != nil {
|
||||||
err := json.Unmarshal(line, &wrapper)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unmarshal message %d: %v", messages+1, err)
|
t.Fatalf("unmarshal message %d: %v", messages+1, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
messages++
|
messages++
|
||||||
|
|
||||||
msg := wrapper.Data
|
msg := wrapper.Data
|
||||||
if msg.Community != nil {
|
if msg.Community != nil {
|
||||||
t.Errorf("message %d: Community decoded, want empty: %v",
|
t.Errorf("message %d: Community decoded, want empty: %v", messages, msg.Community)
|
||||||
messages, msg.Community)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if msg.Raw != "" {
|
if msg.Raw != "" {
|
||||||
t.Errorf("message %d: Raw decoded, want empty", messages)
|
t.Errorf("message %d: Raw decoded, want empty", messages)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(msg.Path.Sequence) > 0 {
|
if len(msg.Path) > 0 {
|
||||||
withPath++
|
withPath++
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(msg.Announcements) > 0 {
|
if len(msg.Announcements) > 0 {
|
||||||
withAnnouncements++
|
withAnnouncements++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if err := scanner.Err(); err != nil {
|
||||||
err = scanner.Err()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("scan fixtures: %v", err)
|
t.Fatalf("scan fixtures: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -79,99 +62,6 @@ func TestDecodeDropsCommunityAndRaw(t *testing.T) {
|
|||||||
// The fixtures include announcement messages; a used field must still decode,
|
// The fixtures include announcement messages; a used field must still decode,
|
||||||
// otherwise an empty Community/Raw would prove nothing.
|
// otherwise an empty Community/Raw would prove nothing.
|
||||||
if withPath == 0 || withAnnouncements == 0 {
|
if withPath == 0 || withAnnouncements == 0 {
|
||||||
t.Fatalf("used fields did not decode: withPath=%d withAnnouncements=%d",
|
t.Fatalf("used fields did not decode: withPath=%d withAnnouncements=%d", withPath, withAnnouncements)
|
||||||
withPath, withAnnouncements)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestASPathKeepsATrailingSetApart decodes AS paths as the feed writes them, an
|
|
||||||
// AS set as an array inside the path. A set of several members that ends the
|
|
||||||
// path is kept apart from the sequence, and the path has no origin, since any
|
|
||||||
// member may be it. A set of one member counts as that AS, and a set anywhere
|
|
||||||
// else is flattened into the sequence.
|
|
||||||
func TestASPathKeepsATrailingSetApart(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
path string
|
|
||||||
wantSequence []int
|
|
||||||
wantSet []int
|
|
||||||
wantOrigin int
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
path: "[64496, 64497]",
|
|
||||||
wantSequence: []int{64496, 64497},
|
|
||||||
wantOrigin: 64497,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
path: "[64496, 64497, [64498, 64499]]",
|
|
||||||
wantSequence: []int{64496, 64497},
|
|
||||||
wantSet: []int{64498, 64499},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
path: "[64496, 64497, [64498]]",
|
|
||||||
wantSequence: []int{64496, 64497, 64498},
|
|
||||||
wantOrigin: 64498,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
path: "[64496, [64497, 64498], 64499]",
|
|
||||||
wantSequence: []int{64496, 64497, 64498, 64499},
|
|
||||||
wantOrigin: 64499,
|
|
||||||
},
|
|
||||||
{path: "[]"},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.path, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var got ristypes.ASPath
|
|
||||||
|
|
||||||
err := json.Unmarshal([]byte(tt.path), &got)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unmarshal %s: %v", tt.path, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !slices.Equal(got.Sequence, tt.wantSequence) ||
|
|
||||||
!slices.Equal(got.Set, tt.wantSet) {
|
|
||||||
t.Errorf("%s decoded to sequence %v and set %v, want %v and %v",
|
|
||||||
tt.path, got.Sequence, got.Set, tt.wantSequence, tt.wantSet)
|
|
||||||
}
|
|
||||||
|
|
||||||
origin := got.Origin()
|
|
||||||
if origin != tt.wantOrigin {
|
|
||||||
t.Errorf("origin of %s = %d, want %d", tt.path, origin, tt.wantOrigin)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestASPathWithASZeroDecodesAsAnEmptyPath decodes AS paths that hold AS 0
|
|
||||||
// before their end, at their end, or in an AS set. RFC 7607 makes such a path
|
|
||||||
// malformed, so each decodes as an empty path.
|
|
||||||
func TestASPathWithASZeroDecodesAsAnEmptyPath(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, path := range []string{
|
|
||||||
"[64496, 0, 64497]",
|
|
||||||
"[64496, 0]",
|
|
||||||
"[64496, [0, 64497], 64498]",
|
|
||||||
"[64496, [64497, 0]]",
|
|
||||||
} {
|
|
||||||
t.Run(path, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var got ristypes.ASPath
|
|
||||||
|
|
||||||
err := json.Unmarshal([]byte(path), &got)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unmarshal %s: %v", path, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(got.Sequence) > 0 || len(got.Set) > 0 {
|
|
||||||
t.Errorf("%s decoded to sequence %v and set %v, want an empty path",
|
|
||||||
path, got.Sequence, got.Set)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,155 +0,0 @@
|
|||||||
package routetable_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"math"
|
|
||||||
"math/rand/v2"
|
|
||||||
"net/netip"
|
|
||||||
"runtime"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/routetable"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Parameters for BenchmarkApplyMixedBatch.
|
|
||||||
const (
|
|
||||||
// benchPeers is how many peers there are, about as many as RIS has.
|
|
||||||
benchPeers = 1400
|
|
||||||
// benchPrefixes is how many IPv4 /24 prefixes the table holds, and
|
|
||||||
// benchPeersPerPrefix how many peers, picked at random, carry each.
|
|
||||||
benchPrefixes = 100000
|
|
||||||
benchPeersPerPrefix = 20
|
|
||||||
// benchBatchSize is how many updates the timed batch holds, as many as
|
|
||||||
// the prefix handler's largest batch (prefixBatchMaxSize in
|
|
||||||
// internal/routewatch).
|
|
||||||
benchBatchSize = 250000
|
|
||||||
// In the timed batch every benchWithdrawalEvery-th update is a
|
|
||||||
// withdrawal, and every benchNewOriginEvery-th announcement has an
|
|
||||||
// origin AS other than the prefix's.
|
|
||||||
benchWithdrawalEvery = 5
|
|
||||||
benchNewOriginEvery = 7
|
|
||||||
// benchOrigins is how many origin ASes the prefixes are spread over,
|
|
||||||
// from benchFirstASN on: 20 prefixes each, about as many as an origin AS
|
|
||||||
// has on average in the full table.
|
|
||||||
benchOrigins = 5000
|
|
||||||
benchFirstASN = 64500
|
|
||||||
// benchUpdatesPerSecond is how many updates share one time.
|
|
||||||
benchUpdatesPerSecond = 1000
|
|
||||||
benchMaskLength = 24
|
|
||||||
)
|
|
||||||
|
|
||||||
// benchUpdate is one update of the benchmark.
|
|
||||||
type benchUpdate struct {
|
|
||||||
prefix netip.Prefix
|
|
||||||
origin int
|
|
||||||
route routetable.Route
|
|
||||||
withdraw bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// BenchmarkApplyMixedBatch times one batch of 250,000 updates of routes picked
|
|
||||||
// at random: announcements, withdrawals and announcements with a new origin AS.
|
|
||||||
// The table it applies them to holds 100,000 IPv4 prefixes from 5,000 origin
|
|
||||||
// ASes, each carried by 20 of 1,400 peers. It reports the heap bytes that table
|
|
||||||
// takes per prefix and origin AS pair. It is run by hand, not by make test, for
|
|
||||||
// example in the test image:
|
|
||||||
//
|
|
||||||
// go test -run '^$' -bench MixedBatch -benchtime 10x ./internal/routetable/
|
|
||||||
func BenchmarkApplyMixedBatch(b *testing.B) {
|
|
||||||
//nolint:gosec // The benchmark needs the same updates every run.
|
|
||||||
rng := rand.New(rand.NewPCG(1, 2))
|
|
||||||
peers := make([]string, benchPeers)
|
|
||||||
|
|
||||||
for i := range peers {
|
|
||||||
peers[i] = fmt.Sprintf("198.18.%d.%d", i/256, i%256)
|
|
||||||
}
|
|
||||||
|
|
||||||
before := heapBytes()
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
updates := 0
|
|
||||||
|
|
||||||
for prefix := range benchPrefixes {
|
|
||||||
for range benchPeersPerPrefix {
|
|
||||||
peer := peers[rng.IntN(benchPeers)]
|
|
||||||
apply(table, benchAnnouncement(rng, prefix, peer, updates, false))
|
|
||||||
updates++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The record of changes is handed over every second while routewatch
|
|
||||||
// runs, so it is not part of the table's size.
|
|
||||||
table.TakeChanges(math.MaxInt)
|
|
||||||
|
|
||||||
bytesPerPair := float64(heapBytes()-before) / benchPrefixes
|
|
||||||
|
|
||||||
batch := make([]benchUpdate, 0, benchBatchSize)
|
|
||||||
for i := range benchBatchSize {
|
|
||||||
update := benchAnnouncement(rng, rng.IntN(benchPrefixes),
|
|
||||||
peers[rng.IntN(benchPeers)], updates+i, i%benchNewOriginEvery == 0)
|
|
||||||
update.withdraw = i%benchWithdrawalEvery == 0
|
|
||||||
batch = append(batch, update)
|
|
||||||
}
|
|
||||||
|
|
||||||
for b.Loop() {
|
|
||||||
for _, update := range batch {
|
|
||||||
apply(table, update)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// After the loop: b.Loop drops the metrics reported before it starts.
|
|
||||||
b.ReportMetric(bytesPerPair, "bytes/pair")
|
|
||||||
}
|
|
||||||
|
|
||||||
// benchAnnouncement returns the announcement of the prefix numbered prefix, a
|
|
||||||
// /24 from 1.0.0.0/24 on, by peer. It is the updates-th update of the
|
|
||||||
// benchmark, and its AS path holds four ASes from the peer's AS to the origin
|
|
||||||
// AS. newOrigin gives it an origin AS other than the prefix's.
|
|
||||||
func benchAnnouncement(
|
|
||||||
rng *rand.Rand, prefix int, peer string, updates int, newOrigin bool,
|
|
||||||
) benchUpdate {
|
|
||||||
origin := benchFirstASN + prefix%benchOrigins
|
|
||||||
if newOrigin {
|
|
||||||
origin += benchOrigins
|
|
||||||
}
|
|
||||||
|
|
||||||
start := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
|
||||||
|
|
||||||
return benchUpdate{
|
|
||||||
prefix: netip.MustParsePrefix(fmt.Sprintf("%d.%d.%d.0/%d",
|
|
||||||
1+prefix>>16, (prefix>>8)%256, prefix%256, benchMaskLength)),
|
|
||||||
origin: origin,
|
|
||||||
route: routetable.Route{
|
|
||||||
Peer: peer,
|
|
||||||
ASPath: []int{
|
|
||||||
benchFirstASN + rng.IntN(benchOrigins),
|
|
||||||
benchFirstASN + rng.IntN(benchOrigins),
|
|
||||||
benchFirstASN + rng.IntN(benchOrigins),
|
|
||||||
origin,
|
|
||||||
},
|
|
||||||
Time: start.Add(time.Duration(updates/benchUpdatesPerSecond) *
|
|
||||||
time.Second),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// apply applies update to table.
|
|
||||||
func apply(table *routetable.Table, update benchUpdate) {
|
|
||||||
if update.withdraw {
|
|
||||||
table.Withdraw(update.prefix, update.route.Peer)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
table.Announce(update.prefix, update.origin, update.route)
|
|
||||||
}
|
|
||||||
|
|
||||||
// heapBytes returns the bytes of the live heap objects, after a collection.
|
|
||||||
func heapBytes() uint64 {
|
|
||||||
runtime.GC()
|
|
||||||
|
|
||||||
var stats runtime.MemStats
|
|
||||||
|
|
||||||
runtime.ReadMemStats(&stats)
|
|
||||||
|
|
||||||
return stats.HeapAlloc
|
|
||||||
}
|
|
||||||
@@ -1,259 +0,0 @@
|
|||||||
package routetable
|
|
||||||
|
|
||||||
import (
|
|
||||||
"cmp"
|
|
||||||
"net/netip"
|
|
||||||
"slices"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/bogon"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/ristypes"
|
|
||||||
)
|
|
||||||
|
|
||||||
// walkBatch is how many prefixes the walk over the whole table reads in one
|
|
||||||
// hold of the read lock. The updates waiting for the lock get it in between.
|
|
||||||
const walkBatch = 10000
|
|
||||||
|
|
||||||
// Overview holds figures about the whole table. Working them out takes a walk
|
|
||||||
// over every prefix, so a caller asks for them now and then and keeps them.
|
|
||||||
type Overview struct {
|
|
||||||
// NewestRoute and OldestRoute are the newest and the oldest kept routes,
|
|
||||||
// or nil when the table keeps none. Of several with the same time, it is
|
|
||||||
// any one of them.
|
|
||||||
NewestRoute *PrefixRoute
|
|
||||||
OldestRoute *PrefixRoute
|
|
||||||
|
|
||||||
// MostPrefixesASN is the AS that originates the most prefixes, and
|
|
||||||
// MostPrefixes is how many it originates. Of several with as many, it is
|
|
||||||
// the lowest AS number. AS 0, no known origin, counts neither here nor
|
|
||||||
// in the single-prefix figures.
|
|
||||||
MostPrefixesASN int
|
|
||||||
MostPrefixes int
|
|
||||||
|
|
||||||
// SinglePrefixASes counts the ASes that originate one prefix.
|
|
||||||
SinglePrefixASes int
|
|
||||||
// NewestSinglePrefix is the newest kept route of those ASes, or nil when
|
|
||||||
// they keep none.
|
|
||||||
NewestSinglePrefix *PrefixRoute
|
|
||||||
|
|
||||||
// Bogons are the bogon announcements, in prefix order: at most the
|
|
||||||
// number Overview is asked for. BogonCount counts them all.
|
|
||||||
Bogons []Bogon
|
|
||||||
BogonCount int
|
|
||||||
|
|
||||||
// MultipleOrigins are the prefixes that peers carry with more than one
|
|
||||||
// origin AS, in prefix order: at most the number Overview is asked for.
|
|
||||||
// MultipleOriginCount counts them all.
|
|
||||||
MultipleOrigins []MultipleOriginPrefix
|
|
||||||
MultipleOriginCount int
|
|
||||||
}
|
|
||||||
|
|
||||||
// PrefixRoute is the time of a kept route, with its prefix and origin AS.
|
|
||||||
type PrefixRoute struct {
|
|
||||||
Prefix netip.Prefix
|
|
||||||
OriginASN int
|
|
||||||
Time time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// Bogon is a bogon announcement: a prefix and origin AS pair whose prefix or
|
|
||||||
// origin AS is reserved, as package bogon tells.
|
|
||||||
type Bogon struct {
|
|
||||||
Prefix netip.Prefix
|
|
||||||
OriginASN int
|
|
||||||
// Peers is how many peers carry the prefix with this origin AS.
|
|
||||||
Peers int
|
|
||||||
// Reasons say why the prefix, the origin AS or both are reserved.
|
|
||||||
Reasons []string
|
|
||||||
}
|
|
||||||
|
|
||||||
// MultipleOriginPrefix is a prefix that peers carry with more than one origin
|
|
||||||
// AS.
|
|
||||||
type MultipleOriginPrefix struct {
|
|
||||||
Prefix netip.Prefix
|
|
||||||
// Origins are its origin ASes, in AS number order.
|
|
||||||
Origins []OriginPeers
|
|
||||||
}
|
|
||||||
|
|
||||||
// OriginPeers is one origin AS of a prefix, and how many peers carry the prefix
|
|
||||||
// with it.
|
|
||||||
type OriginPeers struct {
|
|
||||||
ASN int
|
|
||||||
Peers int
|
|
||||||
}
|
|
||||||
|
|
||||||
// Overview works out the table's overview. It lists at most maxListed bogon
|
|
||||||
// announcements and at most maxListed prefixes with more than one origin AS:
|
|
||||||
// the first in prefix order. It walks over every prefix, but holds the read
|
|
||||||
// lock for walkBatch prefixes at a time.
|
|
||||||
func (t *Table) Overview(maxListed int) Overview {
|
|
||||||
var overview Overview
|
|
||||||
|
|
||||||
t.walkPrefixes(func(prefix netip.Prefix, origins []*origin) {
|
|
||||||
overview.noteKeptRoutes(prefix, origins)
|
|
||||||
overview.noteBogons(prefix, origins)
|
|
||||||
overview.noteMultipleOrigins(prefix, origins)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.countPrefixesPerOrigin(&overview)
|
|
||||||
|
|
||||||
slices.SortFunc(overview.Bogons, func(a, b Bogon) int {
|
|
||||||
return cmp.Or(a.Prefix.Compare(b.Prefix),
|
|
||||||
cmp.Compare(a.OriginASN, b.OriginASN))
|
|
||||||
})
|
|
||||||
overview.BogonCount = len(overview.Bogons)
|
|
||||||
overview.Bogons = overview.Bogons[:min(maxListed, overview.BogonCount)]
|
|
||||||
|
|
||||||
slices.SortFunc(overview.MultipleOrigins, func(a, b MultipleOriginPrefix) int {
|
|
||||||
return a.Prefix.Compare(b.Prefix)
|
|
||||||
})
|
|
||||||
overview.MultipleOriginCount = len(overview.MultipleOrigins)
|
|
||||||
overview.MultipleOrigins =
|
|
||||||
overview.MultipleOrigins[:min(maxListed, overview.MultipleOriginCount)]
|
|
||||||
|
|
||||||
return overview
|
|
||||||
}
|
|
||||||
|
|
||||||
// walkPrefixes calls visit with every prefix of the table and its origins. It
|
|
||||||
// holds the read lock, but lets it go after every walkBatch prefixes, so the
|
|
||||||
// updates waiting for it get in. A prefix that is added or removed in such a
|
|
||||||
// gap may or may not be visited, as Go allows for a map that changes during a
|
|
||||||
// range over it. visit must not keep origins once it returns.
|
|
||||||
func (t *Table) walkPrefixes(visit func(prefix netip.Prefix, origins []*origin)) {
|
|
||||||
t.mu.RLock()
|
|
||||||
defer t.mu.RUnlock()
|
|
||||||
|
|
||||||
visited := 0
|
|
||||||
|
|
||||||
for _, f := range []*family{&t.ipv4, &t.ipv6} {
|
|
||||||
for _, prefixes := range f.byLength {
|
|
||||||
for prefix, origins := range prefixes {
|
|
||||||
visit(prefix, origins)
|
|
||||||
|
|
||||||
visited++
|
|
||||||
if visited%walkBatch == 0 {
|
|
||||||
t.mu.RUnlock()
|
|
||||||
t.mu.RLock()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// countPrefixesPerOrigin works out the overview's figures of how many prefixes
|
|
||||||
// each AS originates. It holds the read lock once, for one look at each origin
|
|
||||||
// AS, of which there are far fewer than prefixes.
|
|
||||||
func (t *Table) countPrefixesPerOrigin(overview *Overview) {
|
|
||||||
t.mu.RLock()
|
|
||||||
defer t.mu.RUnlock()
|
|
||||||
|
|
||||||
for asn, prefixes := range t.byOrigin {
|
|
||||||
if asn == ristypes.NoOrigin {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
count := len(prefixes)
|
|
||||||
if count > overview.MostPrefixes ||
|
|
||||||
(count == overview.MostPrefixes && asn < overview.MostPrefixesASN) {
|
|
||||||
overview.MostPrefixesASN = asn
|
|
||||||
overview.MostPrefixes = count
|
|
||||||
}
|
|
||||||
|
|
||||||
if count == 1 {
|
|
||||||
overview.SinglePrefixASes++
|
|
||||||
overview.noteSinglePrefix(asn, prefixes)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// noteSinglePrefix makes the newest kept route of asn, whose one prefix is in
|
|
||||||
// prefixes, the overview's newest single-prefix route when it is newer.
|
|
||||||
func (overview *Overview) noteSinglePrefix(
|
|
||||||
asn int, prefixes map[netip.Prefix]*origin,
|
|
||||||
) {
|
|
||||||
for prefix, o := range prefixes {
|
|
||||||
newest := o.newestTime()
|
|
||||||
if newest == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if overview.NewestSinglePrefix == nil ||
|
|
||||||
newest.After(overview.NewestSinglePrefix.Time) {
|
|
||||||
overview.NewestSinglePrefix = &PrefixRoute{
|
|
||||||
Prefix: prefix, OriginASN: asn, Time: *newest,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// noteKeptRoutes makes the newest and the oldest kept routes of prefix's
|
|
||||||
// origins the overview's newest and oldest routes when they are newer or older.
|
|
||||||
func (overview *Overview) noteKeptRoutes(prefix netip.Prefix, origins []*origin) {
|
|
||||||
for _, o := range origins {
|
|
||||||
if len(o.routes) == 0 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
newest := o.routes[0].time
|
|
||||||
if overview.NewestRoute == nil || newest.After(overview.NewestRoute.Time) {
|
|
||||||
overview.NewestRoute = &PrefixRoute{
|
|
||||||
Prefix: prefix, OriginASN: o.asn, Time: newest,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
oldest := o.routes[len(o.routes)-1].time
|
|
||||||
if overview.OldestRoute == nil || oldest.Before(overview.OldestRoute.Time) {
|
|
||||||
overview.OldestRoute = &PrefixRoute{
|
|
||||||
Prefix: prefix, OriginASN: o.asn, Time: oldest,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// noteBogons adds each origin of prefix to the overview's bogon announcements
|
|
||||||
// when the prefix or the origin AS is reserved.
|
|
||||||
func (overview *Overview) noteBogons(prefix netip.Prefix, origins []*origin) {
|
|
||||||
prefixReason, prefixReserved := bogon.Prefix(prefix)
|
|
||||||
|
|
||||||
for _, o := range origins {
|
|
||||||
asnReason, asnReserved := bogon.ASN(o.asn)
|
|
||||||
if !prefixReserved && !asnReserved {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
announcement := Bogon{
|
|
||||||
Prefix: prefix, OriginASN: o.asn, Peers: int(o.peers.Count()),
|
|
||||||
}
|
|
||||||
|
|
||||||
if prefixReserved {
|
|
||||||
announcement.Reasons = append(announcement.Reasons, prefixReason)
|
|
||||||
}
|
|
||||||
|
|
||||||
if asnReserved {
|
|
||||||
announcement.Reasons = append(announcement.Reasons, asnReason)
|
|
||||||
}
|
|
||||||
|
|
||||||
overview.Bogons = append(overview.Bogons, announcement)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// noteMultipleOrigins adds prefix to the overview's prefixes with more than one
|
|
||||||
// origin AS when it has more than one.
|
|
||||||
func (overview *Overview) noteMultipleOrigins(
|
|
||||||
prefix netip.Prefix, origins []*origin,
|
|
||||||
) {
|
|
||||||
if len(origins) <= 1 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
entry := MultipleOriginPrefix{
|
|
||||||
Prefix: prefix,
|
|
||||||
Origins: make([]OriginPeers, 0, len(origins)),
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, o := range origins {
|
|
||||||
entry.Origins = append(entry.Origins,
|
|
||||||
OriginPeers{ASN: o.asn, Peers: int(o.peers.Count())})
|
|
||||||
}
|
|
||||||
|
|
||||||
overview.MultipleOrigins = append(overview.MultipleOrigins, entry)
|
|
||||||
}
|
|
||||||
@@ -1,133 +0,0 @@
|
|||||||
package routetable
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/netip"
|
|
||||||
"runtime"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestWalkPrefixesVisitsEveryPrefix walks a table with more prefixes than two
|
|
||||||
// of the walk's batches, while another goroutine keeps adding and removing
|
|
||||||
// other prefixes of the same mask length. The walk lets the lock go between
|
|
||||||
// batches, so those updates get in while it runs. Every prefix that was in
|
|
||||||
// the table all along must be visited once.
|
|
||||||
func TestWalkPrefixesVisitsEveryPrefix(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const (
|
|
||||||
asn = 1000
|
|
||||||
peer = "192.0.2.1"
|
|
||||||
// prefixCount is more prefixes than two of the walk's batches.
|
|
||||||
prefixCount = 2*walkBatch + 1
|
|
||||||
)
|
|
||||||
|
|
||||||
routeTime := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
|
||||||
table := New(Params{})
|
|
||||||
prefixes := make([]netip.Prefix, 0, prefixCount)
|
|
||||||
addr := netip.MustParseAddr("1.0.0.0")
|
|
||||||
|
|
||||||
for range prefixCount {
|
|
||||||
prefix := netip.PrefixFrom(addr, addr.BitLen())
|
|
||||||
table.Announce(prefix, asn, Route{Peer: peer, Time: routeTime})
|
|
||||||
prefixes = append(prefixes, prefix)
|
|
||||||
addr = addr.Next()
|
|
||||||
}
|
|
||||||
|
|
||||||
stop := make(chan struct{})
|
|
||||||
updated := make(chan struct{})
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
defer close(updated)
|
|
||||||
|
|
||||||
other := netip.MustParseAddr("2.0.0.0")
|
|
||||||
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-stop:
|
|
||||||
return
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
prefix := netip.PrefixFrom(other, other.BitLen())
|
|
||||||
table.Announce(prefix, asn, Route{Peer: peer, Time: routeTime})
|
|
||||||
table.Withdraw(prefix, peer)
|
|
||||||
|
|
||||||
other = other.Next()
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
visits := make(map[netip.Prefix]int)
|
|
||||||
|
|
||||||
table.walkPrefixes(func(prefix netip.Prefix, _ []*origin) {
|
|
||||||
visits[prefix]++
|
|
||||||
})
|
|
||||||
|
|
||||||
close(stop)
|
|
||||||
<-updated
|
|
||||||
|
|
||||||
for _, prefix := range prefixes {
|
|
||||||
if visits[prefix] != 1 {
|
|
||||||
t.Fatalf("the walk visited %s %d times, want once", prefix,
|
|
||||||
visits[prefix])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestWalkPrefixesLetsUpdatesIn starts an update during the walk's first
|
|
||||||
// batch. The update must have been made by the time the walk visits the first
|
|
||||||
// prefix of its second batch, because the walk lets the lock go in between.
|
|
||||||
func TestWalkPrefixesLetsUpdatesIn(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const (
|
|
||||||
asn = 1000
|
|
||||||
peer = "192.0.2.1"
|
|
||||||
)
|
|
||||||
|
|
||||||
routeTime := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
|
||||||
table := New(Params{})
|
|
||||||
addr := netip.MustParseAddr("1.0.0.0")
|
|
||||||
|
|
||||||
for range walkBatch + 1 {
|
|
||||||
prefix := netip.PrefixFrom(addr, addr.BitLen())
|
|
||||||
table.Announce(prefix, asn, Route{Peer: peer, Time: routeTime})
|
|
||||||
|
|
||||||
addr = addr.Next()
|
|
||||||
}
|
|
||||||
|
|
||||||
updatePrefix := netip.MustParsePrefix("2.0.0.0/8")
|
|
||||||
updateDone := make(chan struct{})
|
|
||||||
visited := 0
|
|
||||||
madeBeforeSecondBatch := false
|
|
||||||
|
|
||||||
table.walkPrefixes(func(netip.Prefix, []*origin) {
|
|
||||||
visited++
|
|
||||||
|
|
||||||
switch visited {
|
|
||||||
case 1:
|
|
||||||
go func() {
|
|
||||||
defer close(updateDone)
|
|
||||||
|
|
||||||
table.Announce(updatePrefix, asn, Route{Peer: peer, Time: routeTime})
|
|
||||||
}()
|
|
||||||
|
|
||||||
// Wait until the update waits for the lock. TryRLock fails
|
|
||||||
// once a writer waits.
|
|
||||||
for table.mu.TryRLock() {
|
|
||||||
table.mu.RUnlock()
|
|
||||||
runtime.Gosched()
|
|
||||||
}
|
|
||||||
case walkBatch + 1:
|
|
||||||
// The walk holds the read lock, so the table can be read.
|
|
||||||
madeBeforeSecondBatch = table.origins(updatePrefix) != nil
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
<-updateDone
|
|
||||||
|
|
||||||
if !madeBeforeSecondBatch {
|
|
||||||
t.Fatal("an update started during the walk's first batch was not made " +
|
|
||||||
"before its second batch")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,231 +0,0 @@
|
|||||||
package routetable_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/netip"
|
|
||||||
"reflect"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/routetable"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Origin ASes that are not reserved, unlike asA, asB and asC, which are
|
|
||||||
// documentation AS numbers and so make bogon announcements.
|
|
||||||
const (
|
|
||||||
asOne = 1000
|
|
||||||
asTwo = 2000
|
|
||||||
asThree = 3000
|
|
||||||
asFour = 4000
|
|
||||||
asFive = 5000
|
|
||||||
|
|
||||||
// noKnownOrigin is the origin AS of a route whose path ends in an AS
|
|
||||||
// set.
|
|
||||||
noKnownOrigin = 0
|
|
||||||
// privateUseAS and asTrans are reserved AS numbers.
|
|
||||||
privateUseAS = 64512
|
|
||||||
asTrans = 23456
|
|
||||||
|
|
||||||
// maxListed is how many bogon announcements and prefixes with more than
|
|
||||||
// one origin AS the tests ask Overview for.
|
|
||||||
maxListed = 10
|
|
||||||
)
|
|
||||||
|
|
||||||
// prefixRoute returns the PrefixRoute of prefix with origin AS asn, second
|
|
||||||
// seconds after the start.
|
|
||||||
func prefixRoute(prefix string, asn, second int) *routetable.PrefixRoute {
|
|
||||||
return &routetable.PrefixRoute{
|
|
||||||
Prefix: netip.MustParsePrefix(prefix), OriginASN: asn, Time: at(second),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOverviewOfAnEmptyTable(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
got := routetable.New(routetable.Params{}).Overview(maxListed)
|
|
||||||
if !reflect.DeepEqual(got, routetable.Overview{}) {
|
|
||||||
t.Errorf("Overview() = %+v, want no figures", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestOverviewNewestAndOldestRoutes checks that the newest and the oldest kept
|
|
||||||
// routes are found among every origin's kept routes, and that withdrawn routes
|
|
||||||
// no longer count.
|
|
||||||
func TestOverviewNewestAndOldestRoutes(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
announceAll(table, "1.0.0.0/24", asOne, 20, peerA, peerB)
|
|
||||||
// The oldest is the older of this origin's two kept routes.
|
|
||||||
announceAll(table, "2.0.0.0/24", asTwo, 10, peerA, peerB)
|
|
||||||
announceAll(table, "2001:4860::/32", asThree, 40, peerC)
|
|
||||||
// Withdrawn routes, newer and older than every other.
|
|
||||||
announceAll(table, "3.0.0.0/24", asOne, 50, peerA)
|
|
||||||
announceAll(table, "4.0.0.0/24", asOne, 5, peerB)
|
|
||||||
table.Withdraw(netip.MustParsePrefix("3.0.0.0/24"), peerA)
|
|
||||||
table.Withdraw(netip.MustParsePrefix("4.0.0.0/24"), peerB)
|
|
||||||
|
|
||||||
got := table.Overview(maxListed)
|
|
||||||
|
|
||||||
want := prefixRoute("2001:4860::/32", asThree, 40)
|
|
||||||
if !reflect.DeepEqual(got.NewestRoute, want) {
|
|
||||||
t.Errorf("NewestRoute = %+v, want %+v", got.NewestRoute, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
want = prefixRoute("2.0.0.0/24", asTwo, 10)
|
|
||||||
if !reflect.DeepEqual(got.OldestRoute, want) {
|
|
||||||
t.Errorf("OldestRoute = %+v, want %+v", got.OldestRoute, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestOverviewPrefixesPerOrigin checks the AS that originates the most
|
|
||||||
// prefixes and the ASes that originate one.
|
|
||||||
func TestOverviewPrefixesPerOrigin(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
// asOne and asTwo originate two prefixes each; the lower AS number counts.
|
|
||||||
announceAll(table, "1.0.0.0/24", asTwo, 10, peerA)
|
|
||||||
announceAll(table, "1.0.1.0/24", asTwo, 10, peerA)
|
|
||||||
announceAll(table, "2.0.0.0/24", asOne, 10, peerA)
|
|
||||||
announceAll(table, "2.0.1.0/24", asOne, 10, peerB)
|
|
||||||
// Three ASes originate one prefix. asThree's route is the newest, and
|
|
||||||
// asFive's origin keeps no route: peers B and C withdrew theirs.
|
|
||||||
announceAll(table, "3.0.0.0/24", asThree, 30, peerA)
|
|
||||||
announceAll(table, "4.0.0.0/24", asFour, 20, peerA)
|
|
||||||
announceAll(table, "5.0.0.0/24", asFive, 40, peerA, peerB, peerC)
|
|
||||||
table.Withdraw(netip.MustParsePrefix("5.0.0.0/24"), peerB)
|
|
||||||
table.Withdraw(netip.MustParsePrefix("5.0.0.0/24"), peerC)
|
|
||||||
|
|
||||||
got := table.Overview(maxListed)
|
|
||||||
|
|
||||||
if got.MostPrefixesASN != asOne || got.MostPrefixes != 2 {
|
|
||||||
t.Errorf("most prefixes: AS%d with %d, want AS%d with 2",
|
|
||||||
got.MostPrefixesASN, got.MostPrefixes, asOne)
|
|
||||||
}
|
|
||||||
|
|
||||||
if got.SinglePrefixASes != 3 {
|
|
||||||
t.Errorf("SinglePrefixASes = %d, want 3", got.SinglePrefixASes)
|
|
||||||
}
|
|
||||||
|
|
||||||
want := prefixRoute("3.0.0.0/24", asThree, 30)
|
|
||||||
if !reflect.DeepEqual(got.NewestSinglePrefix, want) {
|
|
||||||
t.Errorf("NewestSinglePrefix = %+v, want %+v", got.NewestSinglePrefix, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestOverviewLeavesOutNoKnownOrigin checks that AS 0, which stands for no
|
|
||||||
// known origin, counts in neither the most nor the single-prefix figures. It
|
|
||||||
// originates one prefix, with a route newer than asOne's, and asOne would lose
|
|
||||||
// the tie for the most prefixes to it.
|
|
||||||
func TestOverviewLeavesOutNoKnownOrigin(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
announceAll(table, "1.0.0.0/24", asOne, 10, peerA)
|
|
||||||
announceAll(table, "2.0.0.0/24", noKnownOrigin, 20, peerA)
|
|
||||||
|
|
||||||
got := table.Overview(maxListed)
|
|
||||||
|
|
||||||
if got.MostPrefixesASN != asOne || got.SinglePrefixASes != 1 {
|
|
||||||
t.Errorf("most prefixes AS%d, %d single-prefix ASes; want AS%d, 1",
|
|
||||||
got.MostPrefixesASN, got.SinglePrefixASes, asOne)
|
|
||||||
}
|
|
||||||
|
|
||||||
want := prefixRoute("1.0.0.0/24", asOne, 10)
|
|
||||||
if !reflect.DeepEqual(got.NewestSinglePrefix, want) {
|
|
||||||
t.Errorf("NewestSinglePrefix = %+v, want %+v", got.NewestSinglePrefix, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestOverviewBogons checks which routes are bogon announcements and why, in
|
|
||||||
// prefix order, and that a smaller maxListed lists fewer but counts them all.
|
|
||||||
func TestOverviewBogons(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
// A reserved prefix, carried by two peers.
|
|
||||||
announceAll(table, "192.168.0.0/16", asOne, 10, peerA, peerB)
|
|
||||||
// A reserved origin AS.
|
|
||||||
announceAll(table, "1.0.0.0/24", privateUseAS, 10, peerA)
|
|
||||||
// Both.
|
|
||||||
announceAll(table, "10.0.0.0/8", asTrans, 10, peerA)
|
|
||||||
// Neither: a normal route, and a block that may be announced although it
|
|
||||||
// lies inside a reserved one.
|
|
||||||
announceAll(table, "1.0.1.0/24", asOne, 10, peerA)
|
|
||||||
announceAll(table, "2001:4:112::/48", asTwo, 10, peerA)
|
|
||||||
|
|
||||||
want := []routetable.Bogon{
|
|
||||||
{
|
|
||||||
Prefix: netip.MustParsePrefix("1.0.0.0/24"), OriginASN: privateUseAS,
|
|
||||||
Peers: 1, Reasons: []string{"Private-Use (AS64512-AS65534)"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Prefix: netip.MustParsePrefix("10.0.0.0/8"), OriginASN: asTrans,
|
|
||||||
Peers: 1,
|
|
||||||
Reasons: []string{
|
|
||||||
"Private-Use (10.0.0.0/8)", "AS_TRANS (AS23456)",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Prefix: netip.MustParsePrefix("192.168.0.0/16"), OriginASN: asOne,
|
|
||||||
Peers: 2, Reasons: []string{"Private-Use (192.168.0.0/16)"},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
got := table.Overview(maxListed)
|
|
||||||
if !reflect.DeepEqual(got.Bogons, want) || got.BogonCount != len(want) {
|
|
||||||
t.Errorf("Overview(%d): %d bogons %+v, want %d %+v", maxListed,
|
|
||||||
got.BogonCount, got.Bogons, len(want), want)
|
|
||||||
}
|
|
||||||
|
|
||||||
const fewer = 2
|
|
||||||
|
|
||||||
got = table.Overview(fewer)
|
|
||||||
if !reflect.DeepEqual(got.Bogons, want[:fewer]) || got.BogonCount != len(want) {
|
|
||||||
t.Errorf("Overview(%d): %d bogons %+v, want %d %+v", fewer,
|
|
||||||
got.BogonCount, got.Bogons, len(want), want[:fewer])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestOverviewMultipleOrigins checks the prefixes with more than one origin
|
|
||||||
// AS, in prefix order, and that a smaller maxListed lists fewer but counts them
|
|
||||||
// all.
|
|
||||||
func TestOverviewMultipleOrigins(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
announceAll(table, "2001:4860::/32", asThree, 10, peerA)
|
|
||||||
announceAll(table, "2001:4860::/32", asTwo, 10, peerB)
|
|
||||||
announceAll(table, "1.0.0.0/24", asOne, 10, peerA, peerB)
|
|
||||||
announceAll(table, "1.0.0.0/24", asTwo, 30, peerC)
|
|
||||||
// One origin only.
|
|
||||||
announceAll(table, "2.0.0.0/24", asOne, 10, peerA, peerB)
|
|
||||||
|
|
||||||
want := []routetable.MultipleOriginPrefix{
|
|
||||||
{
|
|
||||||
Prefix: netip.MustParsePrefix("1.0.0.0/24"),
|
|
||||||
Origins: []routetable.OriginPeers{
|
|
||||||
{ASN: asOne, Peers: 2}, {ASN: asTwo, Peers: 1},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Prefix: netip.MustParsePrefix("2001:4860::/32"),
|
|
||||||
Origins: []routetable.OriginPeers{
|
|
||||||
{ASN: asTwo, Peers: 1}, {ASN: asThree, Peers: 1},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
got := table.Overview(maxListed)
|
|
||||||
if !reflect.DeepEqual(got.MultipleOrigins, want) ||
|
|
||||||
got.MultipleOriginCount != len(want) {
|
|
||||||
t.Errorf("Overview(%d): %d prefixes %+v, want %d %+v", maxListed,
|
|
||||||
got.MultipleOriginCount, got.MultipleOrigins, len(want), want)
|
|
||||||
}
|
|
||||||
|
|
||||||
got = table.Overview(1)
|
|
||||||
if !reflect.DeepEqual(got.MultipleOrigins, want[:1]) ||
|
|
||||||
got.MultipleOriginCount != len(want) {
|
|
||||||
t.Errorf("Overview(1): %d prefixes %+v, want %d %+v",
|
|
||||||
got.MultipleOriginCount, got.MultipleOrigins, len(want), want[:1])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,238 +0,0 @@
|
|||||||
package routetable
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/netip"
|
|
||||||
"slices"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/bits-and-blooms/bitset"
|
|
||||||
)
|
|
||||||
|
|
||||||
// PrefixSummary is what the table answers about one prefix.
|
|
||||||
type PrefixSummary struct {
|
|
||||||
Prefix netip.Prefix
|
|
||||||
// OriginASN is the origin AS that the most peers carry the prefix with.
|
|
||||||
// When several have as many peers, it is the lowest AS number.
|
|
||||||
OriginASN int
|
|
||||||
// OriginASNs are all the origin ASes that peers carry the prefix with, in
|
|
||||||
// AS number order.
|
|
||||||
OriginASNs []int
|
|
||||||
// Peers is how many peers carry the prefix.
|
|
||||||
Peers int
|
|
||||||
// Newest is the time of the newest kept route of the prefix, or nil when
|
|
||||||
// it keeps none.
|
|
||||||
Newest *time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// OriginatedPrefix is one prefix an AS originates.
|
|
||||||
type OriginatedPrefix struct {
|
|
||||||
Prefix netip.Prefix
|
|
||||||
// Newest is the time of the newest kept route of the prefix with that
|
|
||||||
// origin AS, or nil when it keeps none.
|
|
||||||
Newest *time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// Counts are the table's figures for the stats.
|
|
||||||
type Counts struct {
|
|
||||||
// Pairs counts the prefix and origin AS pairs that some peer carries.
|
|
||||||
Pairs int
|
|
||||||
// Peers counts the peers that have a number: those given in
|
|
||||||
// Params.PeerNumbers and those numbered since.
|
|
||||||
Peers int
|
|
||||||
IPv4Prefixes int
|
|
||||||
IPv6Prefixes int
|
|
||||||
// IPv4PrefixesByLength[n] and IPv6PrefixesByLength[n] are how many
|
|
||||||
// prefixes of mask length n some peer carries.
|
|
||||||
IPv4PrefixesByLength []int
|
|
||||||
IPv6PrefixesByLength []int
|
|
||||||
// IPv4Routes and IPv6Routes count peer and prefix pairs: each peer that
|
|
||||||
// carries a prefix counts once for it.
|
|
||||||
IPv4Routes int
|
|
||||||
IPv6Routes int
|
|
||||||
// OriginsWithoutKeptRoute counts the prefix and origin AS pairs that peers
|
|
||||||
// carry but whose kept routes were all withdrawn.
|
|
||||||
OriginsWithoutKeptRoute int
|
|
||||||
// OldestKeptRoute and NewestKeptRoute are the times of the oldest and the
|
|
||||||
// newest kept routes, or nil when the table keeps none.
|
|
||||||
OldestKeptRoute *time.Time
|
|
||||||
NewestKeptRoute *time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// Lookup returns the most specific prefix that holds addr and that some peer
|
|
||||||
// carries. It ignores prefixes shorter than /8 for IPv4 and /16 for IPv6. It
|
|
||||||
// makes one map lookup per mask length, and reports false when no prefix
|
|
||||||
// holds addr.
|
|
||||||
func (t *Table) Lookup(addr netip.Addr) (PrefixSummary, bool) {
|
|
||||||
t.mu.RLock()
|
|
||||||
defer t.mu.RUnlock()
|
|
||||||
|
|
||||||
// An IPv4 address written in IPv6 form (::ffff:192.0.2.1) is looked up as
|
|
||||||
// IPv4.
|
|
||||||
addr = addr.Unmap()
|
|
||||||
f := t.familyOf(addr)
|
|
||||||
|
|
||||||
for bits := addr.BitLen(); bits >= f.minLookupLength; bits-- {
|
|
||||||
prefix := netip.PrefixFrom(addr, bits).Masked()
|
|
||||||
|
|
||||||
origins, found := f.byLength[bits][prefix]
|
|
||||||
if found {
|
|
||||||
return summarize(prefix, origins), true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return PrefixSummary{}, false
|
|
||||||
}
|
|
||||||
|
|
||||||
// OriginatedPrefixes returns the prefixes AS asn originates, in prefix order.
|
|
||||||
func (t *Table) OriginatedPrefixes(asn int) []OriginatedPrefix {
|
|
||||||
t.mu.RLock()
|
|
||||||
defer t.mu.RUnlock()
|
|
||||||
|
|
||||||
prefixes := make([]OriginatedPrefix, 0, len(t.byOrigin[asn]))
|
|
||||||
for prefix, o := range t.byOrigin[asn] {
|
|
||||||
prefixes = append(prefixes,
|
|
||||||
OriginatedPrefix{Prefix: prefix, Newest: o.newestTime()})
|
|
||||||
}
|
|
||||||
|
|
||||||
slices.SortFunc(prefixes, func(a, b OriginatedPrefix) int {
|
|
||||||
return a.Prefix.Compare(b.Prefix)
|
|
||||||
})
|
|
||||||
|
|
||||||
return prefixes
|
|
||||||
}
|
|
||||||
|
|
||||||
// Origins returns each origin AS of prefix, in AS number order, with its peers
|
|
||||||
// and its kept routes. It returns nil when no peer carries the prefix.
|
|
||||||
func (t *Table) Origins(prefix netip.Prefix) []Origin {
|
|
||||||
t.mu.RLock()
|
|
||||||
defer t.mu.RUnlock()
|
|
||||||
|
|
||||||
if !prefix.IsValid() {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var origins []Origin
|
|
||||||
|
|
||||||
for _, o := range t.origins(prefix.Masked()) {
|
|
||||||
origins = append(origins, Origin{
|
|
||||||
ASN: o.asn,
|
|
||||||
Peers: t.addresses(&o.peers),
|
|
||||||
Routes: t.exportRoutes(o.routes),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return origins
|
|
||||||
}
|
|
||||||
|
|
||||||
// SamplePrefixes returns up to limit of the prefixes of IP version ipVersion
|
|
||||||
// (4 or 6) and mask length maskLength. It reads only the prefixes it returns.
|
|
||||||
// Go starts each walk over a map at a random place, so each call returns
|
|
||||||
// another sample.
|
|
||||||
func (t *Table) SamplePrefixes(ipVersion, maskLength, limit int) []PrefixSummary {
|
|
||||||
t.mu.RLock()
|
|
||||||
defer t.mu.RUnlock()
|
|
||||||
|
|
||||||
var f *family
|
|
||||||
|
|
||||||
switch ipVersion {
|
|
||||||
case ipVersion4:
|
|
||||||
f = &t.ipv4
|
|
||||||
case ipVersion6:
|
|
||||||
f = &t.ipv6
|
|
||||||
default:
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if maskLength < 0 || maskLength >= len(f.byLength) || limit <= 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
prefixes := f.byLength[maskLength]
|
|
||||||
sample := make([]PrefixSummary, 0, min(limit, len(prefixes)))
|
|
||||||
|
|
||||||
for prefix, origins := range prefixes {
|
|
||||||
if len(sample) == limit {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
|
|
||||||
sample = append(sample, summarize(prefix, origins))
|
|
||||||
}
|
|
||||||
|
|
||||||
return sample
|
|
||||||
}
|
|
||||||
|
|
||||||
// Counts returns the table's figures for the stats. The writes keep them up to
|
|
||||||
// date, so this reads no prefix.
|
|
||||||
func (t *Table) Counts() Counts {
|
|
||||||
t.mu.RLock()
|
|
||||||
defer t.mu.RUnlock()
|
|
||||||
|
|
||||||
counts := Counts{
|
|
||||||
Peers: len(t.peerNumbers),
|
|
||||||
IPv4PrefixesByLength: t.ipv4.prefixesByLength(),
|
|
||||||
IPv6PrefixesByLength: t.ipv6.prefixesByLength(),
|
|
||||||
IPv4Routes: t.ipv4.routes,
|
|
||||||
IPv6Routes: t.ipv6.routes,
|
|
||||||
OriginsWithoutKeptRoute: t.originsWithoutKeptRoute,
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, n := range counts.IPv4PrefixesByLength {
|
|
||||||
counts.IPv4Prefixes += n
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, n := range counts.IPv6PrefixesByLength {
|
|
||||||
counts.IPv6Prefixes += n
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, prefixes := range t.byOrigin {
|
|
||||||
counts.Pairs += len(prefixes)
|
|
||||||
}
|
|
||||||
|
|
||||||
oldest, found := t.keptTimes.Min()
|
|
||||||
if found {
|
|
||||||
counts.OldestKeptRoute = &oldest.time
|
|
||||||
}
|
|
||||||
|
|
||||||
newest, found := t.keptTimes.Max()
|
|
||||||
if found {
|
|
||||||
counts.NewestKeptRoute = &newest.time
|
|
||||||
}
|
|
||||||
|
|
||||||
return counts
|
|
||||||
}
|
|
||||||
|
|
||||||
// addresses returns the addresses of the peers in peers, in number order.
|
|
||||||
func (t *Table) addresses(peers *bitset.BitSet) []string {
|
|
||||||
addresses := make([]string, 0, peers.Count())
|
|
||||||
for number := range peers.EachSet() {
|
|
||||||
addresses = append(addresses, t.peerAddresses[number])
|
|
||||||
}
|
|
||||||
|
|
||||||
return addresses
|
|
||||||
}
|
|
||||||
|
|
||||||
// summarize returns the summary of prefix, whose origins are in AS number
|
|
||||||
// order.
|
|
||||||
func summarize(prefix netip.Prefix, origins []*origin) PrefixSummary {
|
|
||||||
summary := PrefixSummary{Prefix: prefix, OriginASNs: make([]int, 0, len(origins))}
|
|
||||||
mostPeers := 0
|
|
||||||
|
|
||||||
for _, o := range origins {
|
|
||||||
summary.OriginASNs = append(summary.OriginASNs, o.asn)
|
|
||||||
|
|
||||||
peers := int(o.peers.Count())
|
|
||||||
summary.Peers += peers
|
|
||||||
|
|
||||||
if peers > mostPeers {
|
|
||||||
mostPeers = peers
|
|
||||||
summary.OriginASN = o.asn
|
|
||||||
}
|
|
||||||
|
|
||||||
newest := o.newestTime()
|
|
||||||
if newest != nil && (summary.Newest == nil || newest.After(*summary.Newest)) {
|
|
||||||
summary.Newest = newest
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return summary
|
|
||||||
}
|
|
||||||
@@ -1,268 +0,0 @@
|
|||||||
package routetable_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/netip"
|
|
||||||
"reflect"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/routetable"
|
|
||||||
)
|
|
||||||
|
|
||||||
// announceAll has each listed peer announce prefix with origin AS asn, the
|
|
||||||
// peer at index i i*10 seconds after start.
|
|
||||||
func announceAll(
|
|
||||||
table *routetable.Table, prefix string, asn, start int, peers ...string,
|
|
||||||
) {
|
|
||||||
const secondsBetweenPeers = 10
|
|
||||||
|
|
||||||
for i, peer := range peers {
|
|
||||||
table.Announce(netip.MustParsePrefix(prefix), asn,
|
|
||||||
routeOf(peer, asn, start+i*secondsBetweenPeers))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLookup(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
announceAll(table, "10.0.0.0/8", asA, 10, peerA)
|
|
||||||
// Three peers with two origins: asB has more of them.
|
|
||||||
announceAll(table, "10.1.0.0/16", asA, 10, peerA)
|
|
||||||
announceAll(table, "10.1.0.0/16", asB, 20, peerB, peerC)
|
|
||||||
// Two origins with one peer each: the lower AS number wins.
|
|
||||||
announceAll(table, prefixV4, asB, 20, peerB)
|
|
||||||
announceAll(table, prefixV4, asA, 10, peerA)
|
|
||||||
// An origin whose kept routes were withdrawn: no newest time.
|
|
||||||
announceAll(table, "10.9.0.0/16", asA, 10, peerA, peerB, peerC)
|
|
||||||
table.Withdraw(netip.MustParsePrefix("10.9.0.0/16"), peerB)
|
|
||||||
table.Withdraw(netip.MustParsePrefix("10.9.0.0/16"), peerC)
|
|
||||||
announceAll(table, "2001:db8::/32", asA, 10, peerA)
|
|
||||||
announceAll(table, prefixV6, asB, 20, peerB)
|
|
||||||
announceAll(table, "3000::/16", asA, 10, peerA)
|
|
||||||
// Shorter than the lookup uses.
|
|
||||||
announceAll(table, "192.0.0.0/7", asA, 10, peerA)
|
|
||||||
announceAll(table, "2000::/15", asA, 10, peerA)
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
addr string
|
|
||||||
// want is the summary's prefix, or empty when none is found.
|
|
||||||
want string
|
|
||||||
originASN int
|
|
||||||
originASNs []int
|
|
||||||
peers int
|
|
||||||
newest int // in seconds after the start; 0 for none
|
|
||||||
}{
|
|
||||||
{"10.1.2.3", prefixV4, asA, []int{asA, asB}, 2, 20},
|
|
||||||
{"::ffff:10.1.2.3", prefixV4, asA, []int{asA, asB}, 2, 20},
|
|
||||||
{"10.1.3.4", "10.1.0.0/16", asB, []int{asA, asB}, 3, 30},
|
|
||||||
{"10.2.0.1", "10.0.0.0/8", asA, []int{asA}, 1, 10},
|
|
||||||
{"10.9.0.1", "10.9.0.0/16", asA, []int{asA}, 1, 0},
|
|
||||||
{"192.0.2.1", "", 0, nil, 0, 0},
|
|
||||||
{"2001:db8:1::5", prefixV6, asB, []int{asB}, 1, 20},
|
|
||||||
{"2001:db8:2::1", "2001:db8::/32", asA, []int{asA}, 1, 10},
|
|
||||||
{"3000::1", "3000::/16", asA, []int{asA}, 1, 10},
|
|
||||||
{"2001:db9::1", "", 0, nil, 0, 0},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range tests {
|
|
||||||
t.Run(tc.addr, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
got, found := table.Lookup(netip.MustParseAddr(tc.addr))
|
|
||||||
if tc.want == "" {
|
|
||||||
if found {
|
|
||||||
t.Errorf("Lookup(%s) = %+v, want none", tc.addr, got)
|
|
||||||
}
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
want := routetable.PrefixSummary{
|
|
||||||
Prefix: netip.MustParsePrefix(tc.want),
|
|
||||||
OriginASN: tc.originASN,
|
|
||||||
OriginASNs: tc.originASNs,
|
|
||||||
Peers: tc.peers,
|
|
||||||
}
|
|
||||||
if tc.newest != 0 {
|
|
||||||
want.Newest = timeAt(tc.newest)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !found || !reflect.DeepEqual(got, want) {
|
|
||||||
t.Errorf("Lookup(%s) = %+v, %v; want %+v", tc.addr, got, found, want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOriginatedPrefixes(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
announceAll(table, "2001:db8::/32", asA, 30, peerA)
|
|
||||||
announceAll(table, prefixV4, asA, 10, peerA)
|
|
||||||
// asB's newer route for the same prefix does not count for asA.
|
|
||||||
announceAll(table, prefixV4, asB, 40, peerB)
|
|
||||||
announceAll(table, "10.1.0.0/16", asA, 10, peerA, peerB)
|
|
||||||
announceAll(table, "10.9.0.0/16", asA, 10, peerA, peerB, peerC)
|
|
||||||
table.Withdraw(netip.MustParsePrefix("10.9.0.0/16"), peerB)
|
|
||||||
table.Withdraw(netip.MustParsePrefix("10.9.0.0/16"), peerC)
|
|
||||||
|
|
||||||
want := []routetable.OriginatedPrefix{
|
|
||||||
{Prefix: netip.MustParsePrefix("10.1.0.0/16"), Newest: timeAt(20)},
|
|
||||||
{Prefix: netip.MustParsePrefix(prefixV4), Newest: timeAt(10)},
|
|
||||||
{Prefix: netip.MustParsePrefix("10.9.0.0/16")},
|
|
||||||
{Prefix: netip.MustParsePrefix("2001:db8::/32"), Newest: timeAt(30)},
|
|
||||||
}
|
|
||||||
|
|
||||||
got := table.OriginatedPrefixes(asA)
|
|
||||||
if !reflect.DeepEqual(got, want) {
|
|
||||||
t.Errorf("OriginatedPrefixes(%d) = %+v, want %+v", asA, got, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
got = table.OriginatedPrefixes(asC)
|
|
||||||
if len(got) != 0 {
|
|
||||||
t.Errorf("OriginatedPrefixes(%d) = %+v, want none", asC, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSamplePrefixes(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
|
|
||||||
announced := []string{
|
|
||||||
"10.0.0.0/24", "10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24",
|
|
||||||
"10.0.4.0/24", "10.0.0.0/16",
|
|
||||||
"2001:db8::/48", "2001:db8:1::/48", "2001:db8:2::/48",
|
|
||||||
}
|
|
||||||
for _, prefix := range announced {
|
|
||||||
announceAll(table, prefix, asA, 10, peerA)
|
|
||||||
}
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
ipVersion, maskLength, limit int
|
|
||||||
want int
|
|
||||||
}{
|
|
||||||
{"fewer than there are", 4, 24, 3, 3},
|
|
||||||
{"more than there are", 4, 24, 10, 5},
|
|
||||||
{"IPv6", 6, 48, 10, 3},
|
|
||||||
{"a mask length with none", 4, 20, 10, 0},
|
|
||||||
{"no such IP version", 5, 24, 10, 0},
|
|
||||||
{"a mask length too long", 4, 33, 10, 0},
|
|
||||||
{"a negative mask length", 4, -1, 10, 0},
|
|
||||||
{"a limit of zero", 4, 24, 0, 0},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range tests {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
sample := table.SamplePrefixes(tc.ipVersion, tc.maskLength, tc.limit)
|
|
||||||
if len(sample) != tc.want {
|
|
||||||
t.Fatalf("SamplePrefixes(%d, %d, %d) returned %d prefixes, want %d",
|
|
||||||
tc.ipVersion, tc.maskLength, tc.limit, len(sample), tc.want)
|
|
||||||
}
|
|
||||||
|
|
||||||
seen := make(map[netip.Prefix]bool)
|
|
||||||
|
|
||||||
for _, got := range sample {
|
|
||||||
want := routetable.PrefixSummary{
|
|
||||||
Prefix: got.Prefix, OriginASN: asA, OriginASNs: []int{asA},
|
|
||||||
Peers: 1, Newest: timeAt(10),
|
|
||||||
}
|
|
||||||
if seen[got.Prefix] || got.Prefix.Bits() != tc.maskLength ||
|
|
||||||
!reflect.DeepEqual(got, want) {
|
|
||||||
t.Errorf("SamplePrefixes(%d, %d, %d) returned %+v",
|
|
||||||
tc.ipVersion, tc.maskLength, tc.limit, got)
|
|
||||||
}
|
|
||||||
|
|
||||||
seen[got.Prefix] = true
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCounts(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
checkCounts(t, table, routetable.Counts{
|
|
||||||
IPv4PrefixesByLength: byLength(ipv4Bits, nil),
|
|
||||||
IPv6PrefixesByLength: byLength(ipv6Bits, nil),
|
|
||||||
})
|
|
||||||
|
|
||||||
announceAll(table, prefixV4, asA, 10, peerA, peerB)
|
|
||||||
announceAll(table, "10.2.0.0/16", asA, 15, peerA)
|
|
||||||
announceAll(table, "2001:db8::/32", asA, 5, peerA)
|
|
||||||
announceAll(table, "2001:db8::/32", asA, 25, peerC)
|
|
||||||
checkCounts(t, table, routetable.Counts{
|
|
||||||
Pairs: 3,
|
|
||||||
Peers: 3,
|
|
||||||
IPv4Prefixes: 2,
|
|
||||||
IPv6Prefixes: 1,
|
|
||||||
IPv4PrefixesByLength: byLength(ipv4Bits, map[int]int{16: 1, 24: 1}),
|
|
||||||
IPv6PrefixesByLength: byLength(ipv6Bits, map[int]int{32: 1}),
|
|
||||||
IPv4Routes: 3,
|
|
||||||
IPv6Routes: 2,
|
|
||||||
OldestKeptRoute: timeAt(5),
|
|
||||||
NewestKeptRoute: timeAt(25),
|
|
||||||
})
|
|
||||||
|
|
||||||
// peerC's route drops peerA's from the kept ones, and peerA leaves the
|
|
||||||
// IPv6 prefix.
|
|
||||||
announceAll(table, prefixV4, asA, 30, peerC)
|
|
||||||
table.Withdraw(netip.MustParsePrefix("2001:db8::/32"), peerA)
|
|
||||||
checkCounts(t, table, routetable.Counts{
|
|
||||||
Pairs: 3,
|
|
||||||
Peers: 3,
|
|
||||||
IPv4Prefixes: 2,
|
|
||||||
IPv6Prefixes: 1,
|
|
||||||
IPv4PrefixesByLength: byLength(ipv4Bits, map[int]int{16: 1, 24: 1}),
|
|
||||||
IPv6PrefixesByLength: byLength(ipv6Bits, map[int]int{32: 1}),
|
|
||||||
IPv4Routes: 4,
|
|
||||||
IPv6Routes: 1,
|
|
||||||
OldestKeptRoute: timeAt(15),
|
|
||||||
NewestKeptRoute: timeAt(30),
|
|
||||||
})
|
|
||||||
|
|
||||||
// peerC was the last peer of the IPv6 prefix. It keeps its number.
|
|
||||||
table.PeerDown(peerC)
|
|
||||||
checkCounts(t, table, routetable.Counts{
|
|
||||||
Pairs: 2,
|
|
||||||
Peers: 3,
|
|
||||||
IPv4Prefixes: 2,
|
|
||||||
IPv4PrefixesByLength: byLength(ipv4Bits, map[int]int{16: 1, 24: 1}),
|
|
||||||
IPv6PrefixesByLength: byLength(ipv6Bits, nil),
|
|
||||||
IPv4Routes: 3,
|
|
||||||
OldestKeptRoute: timeAt(15),
|
|
||||||
NewestKeptRoute: timeAt(20),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// The address lengths of the two IP versions.
|
|
||||||
const (
|
|
||||||
ipv4Bits = 32
|
|
||||||
ipv6Bits = 128
|
|
||||||
)
|
|
||||||
|
|
||||||
// byLength returns the prefix counts by mask length, for addresses of bits
|
|
||||||
// bits, with counts[n] prefixes of mask length n and none of any other.
|
|
||||||
func byLength(bits int, counts map[int]int) []int {
|
|
||||||
all := make([]int, bits+1)
|
|
||||||
for n, count := range counts {
|
|
||||||
all[n] = count
|
|
||||||
}
|
|
||||||
|
|
||||||
return all
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkCounts checks that the table's counts are want.
|
|
||||||
func checkCounts(t *testing.T, table *routetable.Table, want routetable.Counts) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
got := table.Counts()
|
|
||||||
if !reflect.DeepEqual(got, want) {
|
|
||||||
t.Errorf("Counts() = %+v, want %+v", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,677 +0,0 @@
|
|||||||
// Package routetable holds the whole routing table in memory. For each prefix
|
|
||||||
// and origin AS, it keeps the peers whose current route to the prefix has that
|
|
||||||
// origin AS. It also keeps the two newest of those routes in full. It applies
|
|
||||||
// announcements, withdrawals and peers' sessions going down. It answers the
|
|
||||||
// route reads that the pages and the API need.
|
|
||||||
package routetable
|
|
||||||
|
|
||||||
import (
|
|
||||||
"cmp"
|
|
||||||
"maps"
|
|
||||||
"net/netip"
|
|
||||||
"slices"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/bits-and-blooms/bitset"
|
|
||||||
"github.com/google/btree"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
// keptRoutesPerOrigin is how many routes of each prefix and origin AS are
|
|
||||||
// kept in full: the newest ones. Every other peer that carries the prefix
|
|
||||||
// with that origin AS keeps only its bit in the peer set.
|
|
||||||
keptRoutesPerOrigin = 2
|
|
||||||
|
|
||||||
// keptTimesDegree is the degree of the B-tree that holds the times of the
|
|
||||||
// kept routes in order: each of its nodes holds up to 63 times.
|
|
||||||
keptTimesDegree = 32
|
|
||||||
|
|
||||||
ipv4Bits = 32
|
|
||||||
ipv6Bits = 128
|
|
||||||
|
|
||||||
// The address lookup ignores prefixes shorter than these mask lengths. A
|
|
||||||
// default route or a route this short only says where a peer sends traffic
|
|
||||||
// it has no better route for; it does not say who holds the address.
|
|
||||||
minLookupMaskLengthV4 = 8
|
|
||||||
minLookupMaskLengthV6 = 16
|
|
||||||
|
|
||||||
// The IP versions SamplePrefixes takes.
|
|
||||||
ipVersion4 = 4
|
|
||||||
ipVersion6 = 6
|
|
||||||
)
|
|
||||||
|
|
||||||
// Route is one peer's route to one prefix. The next hop is the peer itself, so
|
|
||||||
// it is not stored.
|
|
||||||
type Route struct {
|
|
||||||
// Peer is the peer's address.
|
|
||||||
Peer string
|
|
||||||
ASPath []int
|
|
||||||
// Time is when the peer announced the route.
|
|
||||||
Time time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// Origin is one origin AS of a prefix: the peers whose current route to the
|
|
||||||
// prefix has that origin AS, and the newest of those routes in full.
|
|
||||||
type Origin struct {
|
|
||||||
// ASN is the origin AS. 0 means no known origin: the path ends in an AS
|
|
||||||
// set of several members.
|
|
||||||
ASN int
|
|
||||||
// Peers are the peers' addresses, in the order of their numbers: the
|
|
||||||
// order they were first seen in.
|
|
||||||
Peers []string
|
|
||||||
// Routes are the kept routes, newest first: at most two, and none when
|
|
||||||
// both were withdrawn while other peers still carry the prefix.
|
|
||||||
Routes []Route
|
|
||||||
}
|
|
||||||
|
|
||||||
// Change is the current content of one prefix and origin AS pair that changed:
|
|
||||||
// what a database row for the pair holds.
|
|
||||||
type Change struct {
|
|
||||||
Prefix netip.Prefix
|
|
||||||
OriginASN int
|
|
||||||
// Removed says that no peer carries the prefix with this origin AS any
|
|
||||||
// more. Peers and Routes are then nil.
|
|
||||||
Removed bool
|
|
||||||
// Peers holds the number of each peer that carries the prefix with this
|
|
||||||
// origin AS.
|
|
||||||
Peers *bitset.BitSet
|
|
||||||
// Routes are the kept routes, newest first.
|
|
||||||
Routes []Route
|
|
||||||
}
|
|
||||||
|
|
||||||
// Params configures New.
|
|
||||||
type Params struct {
|
|
||||||
// PeerNumbers holds the number each peer was given before, by address. A
|
|
||||||
// peer not in it gets the next number above all of them the first time it
|
|
||||||
// announces a route.
|
|
||||||
PeerNumbers map[string]uint
|
|
||||||
}
|
|
||||||
|
|
||||||
// Table is the routing table. Its methods are safe for concurrent use.
|
|
||||||
type Table struct {
|
|
||||||
mu sync.RWMutex
|
|
||||||
|
|
||||||
ipv4 family
|
|
||||||
ipv6 family
|
|
||||||
// byOrigin holds, for each origin AS, the prefixes it originates.
|
|
||||||
byOrigin map[int]map[netip.Prefix]*origin
|
|
||||||
|
|
||||||
// peerNumbers and peerAddresses give each peer's number by its address
|
|
||||||
// and its address by its number. nextPeerNumber is the number the next
|
|
||||||
// new peer gets.
|
|
||||||
peerNumbers map[string]uint
|
|
||||||
peerAddresses map[uint]string
|
|
||||||
nextPeerNumber uint
|
|
||||||
// peerPrefixes[n] counts the prefixes the peer numbered n carries, for
|
|
||||||
// each n below nextPeerNumber. It is a slice, not a map, because Load
|
|
||||||
// counts once for every peer of every pair it loads: with a map, reading
|
|
||||||
// a full table at startup took several times as long.
|
|
||||||
peerPrefixes []int
|
|
||||||
|
|
||||||
// newPeers and changed are what TakeChanges hands over: the peers
|
|
||||||
// numbered since it last ran, and the prefix and origin AS pairs changed
|
|
||||||
// since it last took them.
|
|
||||||
newPeers map[string]uint
|
|
||||||
changed map[pair]struct{}
|
|
||||||
|
|
||||||
// keptTimes counts the kept routes at each time, in time order, so its
|
|
||||||
// first and last items are the oldest and newest kept routes.
|
|
||||||
keptTimes *btree.BTreeG[keptTime]
|
|
||||||
// originsWithoutKeptRoute counts the origins that peers carry but that
|
|
||||||
// keep no route.
|
|
||||||
originsWithoutKeptRoute int
|
|
||||||
}
|
|
||||||
|
|
||||||
// New returns an empty table that numbers new peers on from
|
|
||||||
// params.PeerNumbers.
|
|
||||||
func New(params Params) *Table {
|
|
||||||
t := &Table{
|
|
||||||
ipv4: newFamily(ipv4Bits, minLookupMaskLengthV4),
|
|
||||||
ipv6: newFamily(ipv6Bits, minLookupMaskLengthV6),
|
|
||||||
byOrigin: make(map[int]map[netip.Prefix]*origin),
|
|
||||||
peerNumbers: make(map[string]uint, len(params.PeerNumbers)),
|
|
||||||
peerAddresses: make(map[uint]string, len(params.PeerNumbers)),
|
|
||||||
newPeers: make(map[string]uint),
|
|
||||||
changed: make(map[pair]struct{}),
|
|
||||||
keptTimes: btree.NewG(keptTimesDegree, func(a, b keptTime) bool {
|
|
||||||
return a.time.Before(b.time)
|
|
||||||
}),
|
|
||||||
}
|
|
||||||
|
|
||||||
for address, number := range params.PeerNumbers {
|
|
||||||
t.peerNumbers[address] = number
|
|
||||||
t.peerAddresses[number] = address
|
|
||||||
t.nextPeerNumber = max(t.nextPeerNumber, number+1)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.peerPrefixes = make([]int, t.nextPeerNumber)
|
|
||||||
|
|
||||||
return t
|
|
||||||
}
|
|
||||||
|
|
||||||
// Announce applies an announcement: the peer's current route to prefix is now
|
|
||||||
// r, with origin AS originASN. A peer that carried the prefix with another
|
|
||||||
// origin AS leaves that origin's peer set, and its route there is dropped. r
|
|
||||||
// is kept in full when it is one of the two newest routes of its prefix and
|
|
||||||
// origin AS; otherwise only the peer's bit is.
|
|
||||||
//
|
|
||||||
// An announcement whose path is empty or holds AS 0 is a withdrawal, and the
|
|
||||||
// caller passes it to Withdraw instead.
|
|
||||||
func (t *Table) Announce(prefix netip.Prefix, originASN int, r Route) {
|
|
||||||
t.mu.Lock()
|
|
||||||
defer t.mu.Unlock()
|
|
||||||
|
|
||||||
if !prefix.IsValid() {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
prefix = prefix.Masked()
|
|
||||||
peer := t.numberPeer(r.Peer)
|
|
||||||
|
|
||||||
current := originWithPeer(t.origins(prefix), peer)
|
|
||||||
if current != nil && current.asn != originASN {
|
|
||||||
t.leave(prefix, current, peer)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.join(prefix, t.findOrAddOrigin(prefix, originASN), route{
|
|
||||||
peer: peer,
|
|
||||||
path: slices.Clone(r.ASPath),
|
|
||||||
time: r.Time,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Withdraw applies a withdrawal: the peer no longer carries prefix. It leaves
|
|
||||||
// the prefix's peer set, and its kept route, if any, is dropped.
|
|
||||||
func (t *Table) Withdraw(prefix netip.Prefix, peer string) {
|
|
||||||
t.mu.Lock()
|
|
||||||
defer t.mu.Unlock()
|
|
||||||
|
|
||||||
number, known := t.peerNumbers[peer]
|
|
||||||
if !known || !prefix.IsValid() {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
prefix = prefix.Masked()
|
|
||||||
|
|
||||||
o := originWithPeer(t.origins(prefix), number)
|
|
||||||
if o != nil {
|
|
||||||
t.leave(prefix, o, number)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// PeerDown applies a peer's session going down: the peer leaves every peer
|
|
||||||
// set, and every route of it is dropped. When the peer carries any prefix, it
|
|
||||||
// visits every prefix in the table.
|
|
||||||
func (t *Table) PeerDown(peer string) {
|
|
||||||
t.mu.Lock()
|
|
||||||
defer t.mu.Unlock()
|
|
||||||
|
|
||||||
number, known := t.peerNumbers[peer]
|
|
||||||
if !known || t.peerPrefixes[number] == 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, f := range []*family{&t.ipv4, &t.ipv6} {
|
|
||||||
for _, prefixes := range f.byLength {
|
|
||||||
for prefix, origins := range prefixes {
|
|
||||||
o := originWithPeer(origins, number)
|
|
||||||
if o != nil {
|
|
||||||
t.leave(prefix, o, number)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Load puts one prefix and origin AS pair that the database holds into the
|
|
||||||
// table, as startup does with each row it reads. The change's peers join the
|
|
||||||
// pair's peer set, and its routes are kept as an announcement's would be.
|
|
||||||
// Unlike an update, Load records no change, so TakeChanges does not hand the
|
|
||||||
// pair back to be written again. Loading a pair twice changes nothing.
|
|
||||||
//
|
|
||||||
// The caller makes sure that every peer change names, in its peer set or in
|
|
||||||
// its routes, has a number in Params.PeerNumbers.
|
|
||||||
func (t *Table) Load(change Change) {
|
|
||||||
t.mu.Lock()
|
|
||||||
defer t.mu.Unlock()
|
|
||||||
|
|
||||||
// A removed pair has no peers, and a pair without peers is not in the
|
|
||||||
// table.
|
|
||||||
if change.Peers == nil || change.Peers.None() || !change.Prefix.IsValid() {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
prefix := change.Prefix.Masked()
|
|
||||||
o := t.findOrAddOrigin(prefix, change.OriginASN)
|
|
||||||
wasWithoutKeptRoute := o.withoutKeptRoute()
|
|
||||||
|
|
||||||
// The peers not yet in the peer set join it in one union, which takes
|
|
||||||
// less time than setting their bits one at a time.
|
|
||||||
joining := change.Peers.Difference(&o.peers)
|
|
||||||
o.peers.InPlaceUnion(joining)
|
|
||||||
t.familyOf(prefix.Addr()).routes += int(joining.Count())
|
|
||||||
|
|
||||||
for peer := range joining.EachSet() {
|
|
||||||
t.peerPrefixes[peer]++
|
|
||||||
}
|
|
||||||
|
|
||||||
// Oldest first, as they were announced, so that two routes with the same
|
|
||||||
// time are kept in the order they had.
|
|
||||||
for _, r := range slices.Backward(change.Routes) {
|
|
||||||
peer := t.peerNumbers[r.Peer]
|
|
||||||
|
|
||||||
t.dropKeptRoute(o, peer)
|
|
||||||
t.keepIfNewest(o, route{
|
|
||||||
peer: peer,
|
|
||||||
path: slices.Clone(r.ASPath),
|
|
||||||
time: r.Time,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
t.countOriginWithoutKeptRoute(o, wasWithoutKeptRoute)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TakeChanges returns at most maxChanges of the prefix and origin AS pairs that
|
|
||||||
// changed since they were last taken, in prefix order, each with its current
|
|
||||||
// content. The other changed pairs wait for a later call. It also returns every
|
|
||||||
// peer numbered since it last ran, by address, so a pair taken later names only
|
|
||||||
// peers already returned. It then forgets what it returned.
|
|
||||||
func (t *Table) TakeChanges(maxChanges int) ([]Change, map[string]uint) {
|
|
||||||
t.mu.Lock()
|
|
||||||
defer t.mu.Unlock()
|
|
||||||
|
|
||||||
changes := make([]Change, 0, min(maxChanges, len(t.changed)))
|
|
||||||
for key := range t.changed {
|
|
||||||
if len(changes) == maxChanges {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
|
|
||||||
changes = append(changes, t.currentContent(key))
|
|
||||||
delete(t.changed, key)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A map keeps the memory of its largest size, so an emptied one is
|
|
||||||
// replaced.
|
|
||||||
if len(t.changed) == 0 {
|
|
||||||
t.changed = make(map[pair]struct{})
|
|
||||||
}
|
|
||||||
|
|
||||||
slices.SortFunc(changes, func(a, b Change) int {
|
|
||||||
return cmp.Or(a.Prefix.Compare(b.Prefix),
|
|
||||||
cmp.Compare(a.OriginASN, b.OriginASN))
|
|
||||||
})
|
|
||||||
|
|
||||||
newPeers := t.newPeers
|
|
||||||
t.newPeers = make(map[string]uint)
|
|
||||||
|
|
||||||
return changes, newPeers
|
|
||||||
}
|
|
||||||
|
|
||||||
// PutBackChanges records again what one TakeChanges call returned, for a
|
|
||||||
// caller that could not write it. The next TakeChanges returns those peers
|
|
||||||
// again, and those pairs wait to be taken again like any changed pair. Each is
|
|
||||||
// then taken with its content at that time, so a later change of a pair
|
|
||||||
// replaces the earlier.
|
|
||||||
func (t *Table) PutBackChanges(changes []Change, newPeers map[string]uint) {
|
|
||||||
t.mu.Lock()
|
|
||||||
defer t.mu.Unlock()
|
|
||||||
|
|
||||||
for _, change := range changes {
|
|
||||||
t.changed[pair{prefix: change.Prefix, asn: change.OriginASN}] = struct{}{}
|
|
||||||
}
|
|
||||||
|
|
||||||
maps.Copy(t.newPeers, newPeers)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ChangeCount returns how many changed prefix and origin AS pairs wait for
|
|
||||||
// TakeChanges.
|
|
||||||
func (t *Table) ChangeCount() int {
|
|
||||||
t.mu.RLock()
|
|
||||||
defer t.mu.RUnlock()
|
|
||||||
|
|
||||||
return len(t.changed)
|
|
||||||
}
|
|
||||||
|
|
||||||
// numberPeer returns the peer's number, and gives it the next one the first
|
|
||||||
// time the table sees it.
|
|
||||||
func (t *Table) numberPeer(address string) uint {
|
|
||||||
number, known := t.peerNumbers[address]
|
|
||||||
if known {
|
|
||||||
return number
|
|
||||||
}
|
|
||||||
|
|
||||||
number = t.nextPeerNumber
|
|
||||||
t.nextPeerNumber++
|
|
||||||
t.peerNumbers[address] = number
|
|
||||||
t.peerAddresses[number] = address
|
|
||||||
t.peerPrefixes = append(t.peerPrefixes, 0)
|
|
||||||
t.newPeers[address] = number
|
|
||||||
|
|
||||||
return number
|
|
||||||
}
|
|
||||||
|
|
||||||
// familyOf returns the part of the table for addr's IP version.
|
|
||||||
func (t *Table) familyOf(addr netip.Addr) *family {
|
|
||||||
if addr.Is4() {
|
|
||||||
return &t.ipv4
|
|
||||||
}
|
|
||||||
|
|
||||||
return &t.ipv6
|
|
||||||
}
|
|
||||||
|
|
||||||
// prefixesOfLength returns the prefixes of the IP version and the mask length
|
|
||||||
// of prefix, a valid masked prefix.
|
|
||||||
func (t *Table) prefixesOfLength(prefix netip.Prefix) map[netip.Prefix][]*origin {
|
|
||||||
return t.familyOf(prefix.Addr()).byLength[prefix.Bits()]
|
|
||||||
}
|
|
||||||
|
|
||||||
// origins returns the origins of prefix, a valid masked prefix.
|
|
||||||
func (t *Table) origins(prefix netip.Prefix) []*origin {
|
|
||||||
return t.prefixesOfLength(prefix)[prefix]
|
|
||||||
}
|
|
||||||
|
|
||||||
// findOrAddOrigin returns the origin of prefix with AS asn, added with no
|
|
||||||
// peers when the prefix has none with that AS.
|
|
||||||
func (t *Table) findOrAddOrigin(prefix netip.Prefix, asn int) *origin {
|
|
||||||
prefixes := t.prefixesOfLength(prefix)
|
|
||||||
origins := prefixes[prefix]
|
|
||||||
|
|
||||||
i, found := slices.BinarySearchFunc(origins, asn,
|
|
||||||
func(o *origin, asn int) int { return cmp.Compare(o.asn, asn) })
|
|
||||||
if found {
|
|
||||||
return origins[i]
|
|
||||||
}
|
|
||||||
|
|
||||||
o := &origin{asn: asn}
|
|
||||||
prefixes[prefix] = slices.Insert(origins, i, o)
|
|
||||||
|
|
||||||
if t.byOrigin[asn] == nil {
|
|
||||||
t.byOrigin[asn] = make(map[netip.Prefix]*origin)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.byOrigin[asn][prefix] = o
|
|
||||||
|
|
||||||
return o
|
|
||||||
}
|
|
||||||
|
|
||||||
// removeOrigin removes o, which no peer carries any more, from prefix, and the
|
|
||||||
// prefix from the table when it has no origin left.
|
|
||||||
func (t *Table) removeOrigin(prefix netip.Prefix, o *origin) {
|
|
||||||
prefixes := t.prefixesOfLength(prefix)
|
|
||||||
|
|
||||||
origins := slices.DeleteFunc(prefixes[prefix],
|
|
||||||
func(other *origin) bool { return other == o })
|
|
||||||
if len(origins) == 0 {
|
|
||||||
delete(prefixes, prefix)
|
|
||||||
} else {
|
|
||||||
prefixes[prefix] = origins
|
|
||||||
}
|
|
||||||
|
|
||||||
delete(t.byOrigin[o.asn], prefix)
|
|
||||||
|
|
||||||
if len(t.byOrigin[o.asn]) == 0 {
|
|
||||||
delete(t.byOrigin, o.asn)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// join puts the peer of r in the peer set of o, an origin of prefix, and keeps
|
|
||||||
// r in full when it is one of the two newest routes of o. The peer's earlier
|
|
||||||
// route, if o keeps one, is replaced: it is no longer the peer's current route.
|
|
||||||
// The pair is recorded as changed only when its peer set or kept routes change.
|
|
||||||
func (t *Table) join(prefix netip.Prefix, o *origin, r route) {
|
|
||||||
if slices.ContainsFunc(o.routes, r.equal) {
|
|
||||||
// The peer announced its kept route again.
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
wasWithoutKeptRoute := o.withoutKeptRoute()
|
|
||||||
|
|
||||||
joined := !o.peers.Test(r.peer)
|
|
||||||
if joined {
|
|
||||||
o.peers.Set(r.peer)
|
|
||||||
|
|
||||||
t.familyOf(prefix.Addr()).routes++
|
|
||||||
t.peerPrefixes[r.peer]++
|
|
||||||
}
|
|
||||||
|
|
||||||
dropped := t.dropKeptRoute(o, r.peer)
|
|
||||||
kept := t.keepIfNewest(o, r)
|
|
||||||
|
|
||||||
if joined || dropped || kept {
|
|
||||||
t.recordChange(prefix, o, wasWithoutKeptRoute)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// leave takes peer out of the peer set of o, an origin of prefix, and drops
|
|
||||||
// its kept route, if any.
|
|
||||||
func (t *Table) leave(prefix netip.Prefix, o *origin, peer uint) {
|
|
||||||
wasWithoutKeptRoute := o.withoutKeptRoute()
|
|
||||||
|
|
||||||
o.peers.Clear(peer)
|
|
||||||
t.dropKeptRoute(o, peer)
|
|
||||||
|
|
||||||
t.familyOf(prefix.Addr()).routes--
|
|
||||||
t.peerPrefixes[peer]--
|
|
||||||
|
|
||||||
t.recordChange(prefix, o, wasWithoutKeptRoute)
|
|
||||||
}
|
|
||||||
|
|
||||||
// keepIfNewest keeps r in o unless both kept routes of o are newer, and says
|
|
||||||
// whether it kept r. When o already keeps two routes, the older one is
|
|
||||||
// dropped, and its peer keeps only its bit.
|
|
||||||
func (t *Table) keepIfNewest(o *origin, r route) bool {
|
|
||||||
// The kept routes stay newest first: r goes before the first one that is
|
|
||||||
// not newer than it.
|
|
||||||
i := 0
|
|
||||||
for i < len(o.routes) && o.routes[i].time.After(r.time) {
|
|
||||||
i++
|
|
||||||
}
|
|
||||||
|
|
||||||
if i == keptRoutesPerOrigin {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(o.routes) == keptRoutesPerOrigin {
|
|
||||||
t.dropKeptRoute(o, o.routes[keptRoutesPerOrigin-1].peer)
|
|
||||||
}
|
|
||||||
|
|
||||||
o.routes = slices.Insert(o.routes, i, r)
|
|
||||||
t.countKeptTime(r.time, 1)
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
// dropKeptRoute drops the peer's route from the kept routes of o, if o keeps
|
|
||||||
// one, and says whether it did.
|
|
||||||
func (t *Table) dropKeptRoute(o *origin, peer uint) bool {
|
|
||||||
i := slices.IndexFunc(o.routes, func(r route) bool { return r.peer == peer })
|
|
||||||
if i < 0 {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
t.countKeptTime(o.routes[i].time, -1)
|
|
||||||
o.routes = slices.Delete(o.routes, i, i+1)
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
// recordChange records that the pair of prefix and the AS of o changed, and
|
|
||||||
// brings the count of origins without a kept route up to date.
|
|
||||||
// wasWithoutKeptRoute says whether o was such an origin before the change. An
|
|
||||||
// origin that no peer carries any more is removed.
|
|
||||||
func (t *Table) recordChange(
|
|
||||||
prefix netip.Prefix, o *origin, wasWithoutKeptRoute bool,
|
|
||||||
) {
|
|
||||||
t.changed[pair{prefix: prefix, asn: o.asn}] = struct{}{}
|
|
||||||
|
|
||||||
t.countOriginWithoutKeptRoute(o, wasWithoutKeptRoute)
|
|
||||||
|
|
||||||
if o.peers.None() {
|
|
||||||
t.removeOrigin(prefix, o)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// countOriginWithoutKeptRoute brings the count of origins without a kept route
|
|
||||||
// up to date after a change to o. wasWithoutKeptRoute says whether o was such
|
|
||||||
// an origin before the change.
|
|
||||||
func (t *Table) countOriginWithoutKeptRoute(o *origin, wasWithoutKeptRoute bool) {
|
|
||||||
if wasWithoutKeptRoute {
|
|
||||||
t.originsWithoutKeptRoute--
|
|
||||||
}
|
|
||||||
|
|
||||||
if o.withoutKeptRoute() {
|
|
||||||
t.originsWithoutKeptRoute++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// countKeptTime adds delta to the number of kept routes with time at.
|
|
||||||
func (t *Table) countKeptTime(at time.Time, delta int) {
|
|
||||||
count, _ := t.keptTimes.Get(keptTime{time: at})
|
|
||||||
count.time = at
|
|
||||||
count.routes += delta
|
|
||||||
|
|
||||||
if count.routes == 0 {
|
|
||||||
t.keptTimes.Delete(count)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
t.keptTimes.ReplaceOrInsert(count)
|
|
||||||
}
|
|
||||||
|
|
||||||
// currentContent returns what the table holds for the pair now.
|
|
||||||
func (t *Table) currentContent(key pair) Change {
|
|
||||||
o := t.byOrigin[key.asn][key.prefix]
|
|
||||||
if o == nil {
|
|
||||||
return Change{Prefix: key.prefix, OriginASN: key.asn, Removed: true}
|
|
||||||
}
|
|
||||||
|
|
||||||
return Change{
|
|
||||||
Prefix: key.prefix,
|
|
||||||
OriginASN: key.asn,
|
|
||||||
Peers: o.peers.Clone(),
|
|
||||||
Routes: t.exportRoutes(o.routes),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// exportRoutes returns copies of routes that name each peer by its address, or
|
|
||||||
// nil when there are none.
|
|
||||||
func (t *Table) exportRoutes(routes []route) []Route {
|
|
||||||
if len(routes) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
exported := make([]Route, 0, len(routes))
|
|
||||||
for _, r := range routes {
|
|
||||||
exported = append(exported, Route{
|
|
||||||
Peer: t.peerAddresses[r.peer],
|
|
||||||
ASPath: slices.Clone(r.path),
|
|
||||||
Time: r.time,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return exported
|
|
||||||
}
|
|
||||||
|
|
||||||
// family is the part of the table for one IP version.
|
|
||||||
type family struct {
|
|
||||||
// byLength[n] holds the prefixes of mask length n that some peer carries,
|
|
||||||
// each with its origins in AS number order.
|
|
||||||
byLength []map[netip.Prefix][]*origin
|
|
||||||
// minLookupLength is the shortest mask length the address lookup uses.
|
|
||||||
minLookupLength int
|
|
||||||
// routes counts the peer and prefix pairs: each peer that carries a
|
|
||||||
// prefix counts once for it.
|
|
||||||
routes int
|
|
||||||
}
|
|
||||||
|
|
||||||
// newFamily returns an empty family for addresses of bits bits.
|
|
||||||
func newFamily(bits, minLookupLength int) family {
|
|
||||||
f := family{
|
|
||||||
byLength: make([]map[netip.Prefix][]*origin, bits+1),
|
|
||||||
minLookupLength: minLookupLength,
|
|
||||||
}
|
|
||||||
|
|
||||||
for n := range f.byLength {
|
|
||||||
f.byLength[n] = make(map[netip.Prefix][]*origin)
|
|
||||||
}
|
|
||||||
|
|
||||||
return f
|
|
||||||
}
|
|
||||||
|
|
||||||
// prefixesByLength returns how many prefixes of each mask length f holds.
|
|
||||||
func (f *family) prefixesByLength() []int {
|
|
||||||
counts := make([]int, len(f.byLength))
|
|
||||||
for n, prefixes := range f.byLength {
|
|
||||||
counts[n] = len(prefixes)
|
|
||||||
}
|
|
||||||
|
|
||||||
return counts
|
|
||||||
}
|
|
||||||
|
|
||||||
// origin is what the table holds for one prefix and origin AS.
|
|
||||||
type origin struct {
|
|
||||||
asn int
|
|
||||||
// peers holds the number of each peer whose current route to the prefix
|
|
||||||
// has this origin AS.
|
|
||||||
peers bitset.BitSet
|
|
||||||
// routes are the newest of those routes, newest first, at most
|
|
||||||
// keptRoutesPerOrigin.
|
|
||||||
routes []route
|
|
||||||
}
|
|
||||||
|
|
||||||
// withoutKeptRoute says whether peers carry o but o keeps no route.
|
|
||||||
func (o *origin) withoutKeptRoute() bool {
|
|
||||||
return o.peers.Any() && len(o.routes) == 0
|
|
||||||
}
|
|
||||||
|
|
||||||
// newestTime returns the time of the newest kept route of o, or nil when o
|
|
||||||
// keeps none.
|
|
||||||
func (o *origin) newestTime() *time.Time {
|
|
||||||
if len(o.routes) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
newest := o.routes[0].time
|
|
||||||
|
|
||||||
return &newest
|
|
||||||
}
|
|
||||||
|
|
||||||
// originWithPeer returns the origin whose peer set holds peer, or nil when
|
|
||||||
// none does. A peer carries a prefix with one origin AS at most.
|
|
||||||
func originWithPeer(origins []*origin, peer uint) *origin {
|
|
||||||
for _, o := range origins {
|
|
||||||
if o.peers.Test(peer) {
|
|
||||||
return o
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// route is a kept route. peer is the peer's number.
|
|
||||||
type route struct {
|
|
||||||
peer uint
|
|
||||||
path []int
|
|
||||||
time time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// equal says whether r and other are the same peer's route with the same path
|
|
||||||
// and time.
|
|
||||||
func (r route) equal(other route) bool {
|
|
||||||
return r.peer == other.peer && r.time.Equal(other.time) &&
|
|
||||||
slices.Equal(r.path, other.path)
|
|
||||||
}
|
|
||||||
|
|
||||||
// pair names one prefix and origin AS.
|
|
||||||
type pair struct {
|
|
||||||
prefix netip.Prefix
|
|
||||||
asn int
|
|
||||||
}
|
|
||||||
|
|
||||||
// keptTime counts the kept routes with one time.
|
|
||||||
type keptTime struct {
|
|
||||||
time time.Time
|
|
||||||
routes int
|
|
||||||
}
|
|
||||||
@@ -1,668 +0,0 @@
|
|||||||
package routetable_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"math"
|
|
||||||
"net/netip"
|
|
||||||
"reflect"
|
|
||||||
"slices"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/bits-and-blooms/bitset"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/routetable"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
peerA = "192.0.2.1"
|
|
||||||
peerB = "192.0.2.2"
|
|
||||||
peerC = "2001:db8::3"
|
|
||||||
peerD = "2001:db8::4"
|
|
||||||
|
|
||||||
asA = 64500
|
|
||||||
asB = 64501
|
|
||||||
asC = 64502
|
|
||||||
// transitAS is the first AS of every test path, the peer's own.
|
|
||||||
transitAS = 64496
|
|
||||||
|
|
||||||
prefixV4 = "10.1.2.0/24"
|
|
||||||
prefixV6 = "2001:db8:1::/48"
|
|
||||||
)
|
|
||||||
|
|
||||||
// at returns the time second seconds after the start of every test.
|
|
||||||
func at(second int) time.Time {
|
|
||||||
return time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC).
|
|
||||||
Add(time.Duration(second) * time.Second)
|
|
||||||
}
|
|
||||||
|
|
||||||
// timeAt returns at(second) for a field that may be nil.
|
|
||||||
func timeAt(second int) *time.Time {
|
|
||||||
t := at(second)
|
|
||||||
|
|
||||||
return &t
|
|
||||||
}
|
|
||||||
|
|
||||||
// routeOf returns the route peer announces with origin AS asn second seconds
|
|
||||||
// after the start.
|
|
||||||
func routeOf(peer string, asn, second int) routetable.Route {
|
|
||||||
return routetable.Route{
|
|
||||||
Peer: peer, ASPath: []int{transitAS, asn}, Time: at(second),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// step is one update a test applies to a table for one prefix.
|
|
||||||
type step func(table *routetable.Table, prefix netip.Prefix)
|
|
||||||
|
|
||||||
// announce is the step of peer announcing prefix with origin AS asn second
|
|
||||||
// seconds after the start.
|
|
||||||
func announce(peer string, asn, second int) step {
|
|
||||||
return func(table *routetable.Table, prefix netip.Prefix) {
|
|
||||||
table.Announce(prefix, asn, routeOf(peer, asn, second))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// withdraw is the step of peer withdrawing prefix.
|
|
||||||
func withdraw(peer string) step {
|
|
||||||
return func(table *routetable.Table, prefix netip.Prefix) {
|
|
||||||
table.Withdraw(prefix, peer)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// peerDown is the step of peer's session going down.
|
|
||||||
func peerDown(peer string) step {
|
|
||||||
return func(table *routetable.Table, _ netip.Prefix) {
|
|
||||||
table.PeerDown(peer)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// rule is one case of the rule that decides what the table holds for a prefix:
|
|
||||||
// the steps applied to an empty table, and the prefix's origins after them.
|
|
||||||
// The peers are numbered in the order they first announce, which is the order
|
|
||||||
// Origin.Peers lists them in.
|
|
||||||
type rule struct {
|
|
||||||
name string
|
|
||||||
steps []step
|
|
||||||
want []routetable.Origin
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkRules runs each case for an IPv4 and an IPv6 prefix.
|
|
||||||
func checkRules(t *testing.T, rules []rule) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
for _, tc := range rules {
|
|
||||||
for _, text := range []string{prefixV4, prefixV6} {
|
|
||||||
t.Run(tc.name+" "+text, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
prefix := netip.MustParsePrefix(text)
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
|
|
||||||
for _, apply := range tc.steps {
|
|
||||||
apply(table, prefix)
|
|
||||||
}
|
|
||||||
|
|
||||||
got := table.Origins(prefix)
|
|
||||||
if !reflect.DeepEqual(got, tc.want) {
|
|
||||||
t.Errorf("Origins(%s) = %+v, want %+v", prefix, got, tc.want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAnnouncementsKeepTheTwoNewestRoutes(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
checkRules(t, []rule{
|
|
||||||
{
|
|
||||||
name: "an announcement is kept",
|
|
||||||
steps: []step{announce(peerA, asA, 10)},
|
|
||||||
want: []routetable.Origin{{ASN: asA, Peers: []string{peerA},
|
|
||||||
Routes: []routetable.Route{routeOf(peerA, asA, 10)}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a lagging peer's older route is not kept",
|
|
||||||
steps: []step{announce(peerA, asA, 20), announce(peerB, asA, 30),
|
|
||||||
announce(peerC, asA, 10)},
|
|
||||||
want: []routetable.Origin{{ASN: asA,
|
|
||||||
Peers: []string{peerA, peerB, peerC}, Routes: []routetable.Route{
|
|
||||||
routeOf(peerB, asA, 30), routeOf(peerA, asA, 20),
|
|
||||||
}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a third newer route drops the older kept one, whose peer " +
|
|
||||||
"keeps its bit",
|
|
||||||
steps: []step{announce(peerA, asA, 10), announce(peerB, asA, 20),
|
|
||||||
announce(peerC, asA, 30)},
|
|
||||||
want: []routetable.Origin{{ASN: asA,
|
|
||||||
Peers: []string{peerA, peerB, peerC}, Routes: []routetable.Route{
|
|
||||||
routeOf(peerC, asA, 30), routeOf(peerB, asA, 20),
|
|
||||||
}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a route as old as the older kept one replaces it",
|
|
||||||
steps: []step{announce(peerA, asA, 10), announce(peerB, asA, 20),
|
|
||||||
announce(peerC, asA, 10)},
|
|
||||||
want: []routetable.Origin{{ASN: asA,
|
|
||||||
Peers: []string{peerA, peerB, peerC}, Routes: []routetable.Route{
|
|
||||||
routeOf(peerB, asA, 20), routeOf(peerC, asA, 10),
|
|
||||||
}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a peer's new route replaces its kept one",
|
|
||||||
steps: []step{announce(peerA, asA, 10), announce(peerB, asA, 20),
|
|
||||||
announce(peerA, asA, 30)},
|
|
||||||
want: []routetable.Origin{{ASN: asA, Peers: []string{peerA, peerB},
|
|
||||||
Routes: []routetable.Route{
|
|
||||||
routeOf(peerA, asA, 30), routeOf(peerB, asA, 20),
|
|
||||||
}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a peer's new route replaces its kept one even when older",
|
|
||||||
steps: []step{announce(peerA, asA, 10), announce(peerB, asA, 20),
|
|
||||||
announce(peerB, asA, 5)},
|
|
||||||
want: []routetable.Origin{{ASN: asA, Peers: []string{peerA, peerB},
|
|
||||||
Routes: []routetable.Route{
|
|
||||||
routeOf(peerA, asA, 10), routeOf(peerB, asA, 5),
|
|
||||||
}}},
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAPeerCarriesAPrefixWithOneOrigin(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
checkRules(t, []rule{
|
|
||||||
{
|
|
||||||
name: "a peer moving to a new origin leaves the old one",
|
|
||||||
steps: []step{announce(peerA, asA, 10), announce(peerB, asA, 20),
|
|
||||||
announce(peerA, asB, 30)},
|
|
||||||
want: []routetable.Origin{
|
|
||||||
{ASN: asA, Peers: []string{peerB},
|
|
||||||
Routes: []routetable.Route{routeOf(peerB, asA, 20)}},
|
|
||||||
{ASN: asB, Peers: []string{peerA},
|
|
||||||
Routes: []routetable.Route{routeOf(peerA, asB, 30)}},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a peer whose route is not kept moves to a new origin",
|
|
||||||
steps: []step{announce(peerA, asA, 10), announce(peerB, asA, 20),
|
|
||||||
announce(peerC, asA, 30), announce(peerA, asB, 40)},
|
|
||||||
want: []routetable.Origin{
|
|
||||||
{ASN: asA, Peers: []string{peerB, peerC},
|
|
||||||
Routes: []routetable.Route{
|
|
||||||
routeOf(peerC, asA, 30), routeOf(peerB, asA, 20),
|
|
||||||
}},
|
|
||||||
{ASN: asB, Peers: []string{peerA},
|
|
||||||
Routes: []routetable.Route{routeOf(peerA, asB, 40)}},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "origins are listed in AS number order",
|
|
||||||
steps: []step{announce(peerA, asC, 10), announce(peerB, asA, 20),
|
|
||||||
announce(peerC, asB, 30)},
|
|
||||||
want: []routetable.Origin{
|
|
||||||
{ASN: asA, Peers: []string{peerB},
|
|
||||||
Routes: []routetable.Route{routeOf(peerB, asA, 20)}},
|
|
||||||
{ASN: asB, Peers: []string{peerC},
|
|
||||||
Routes: []routetable.Route{routeOf(peerC, asB, 30)}},
|
|
||||||
{ASN: asC, Peers: []string{peerA},
|
|
||||||
Routes: []routetable.Route{routeOf(peerA, asC, 10)}},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestWithdrawalsAndPeerDowns(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// threePeers is peerA, peerB and peerC announcing in turn; the routes of
|
|
||||||
// peerC and peerB are kept.
|
|
||||||
threePeers := []step{announce(peerA, asA, 10), announce(peerB, asA, 20),
|
|
||||||
announce(peerC, asA, 30)}
|
|
||||||
|
|
||||||
checkRules(t, []rule{
|
|
||||||
{
|
|
||||||
name: "a withdrawal drops the peer's kept route",
|
|
||||||
steps: append(slices.Clone(threePeers), withdraw(peerC)),
|
|
||||||
want: []routetable.Origin{{ASN: asA, Peers: []string{peerA, peerB},
|
|
||||||
Routes: []routetable.Route{routeOf(peerB, asA, 20)}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a withdrawal of a route that is not kept leaves the kept ones",
|
|
||||||
steps: append(slices.Clone(threePeers), withdraw(peerA)),
|
|
||||||
want: []routetable.Origin{{ASN: asA, Peers: []string{peerB, peerC},
|
|
||||||
Routes: []routetable.Route{
|
|
||||||
routeOf(peerC, asA, 30), routeOf(peerB, asA, 20),
|
|
||||||
}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "an origin whose kept routes are withdrawn keeps its other " +
|
|
||||||
"peers",
|
|
||||||
steps: append(slices.Clone(threePeers), withdraw(peerB),
|
|
||||||
withdraw(peerC)),
|
|
||||||
want: []routetable.Origin{{ASN: asA, Peers: []string{peerA}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "an origin without kept routes keeps the next update",
|
|
||||||
steps: append(slices.Clone(threePeers), withdraw(peerB),
|
|
||||||
withdraw(peerC), announce(peerA, asA, 5)),
|
|
||||||
want: []routetable.Origin{{ASN: asA, Peers: []string{peerA},
|
|
||||||
Routes: []routetable.Route{routeOf(peerA, asA, 5)}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "an origin no peer carries is removed",
|
|
||||||
steps: []step{announce(peerA, asA, 10), announce(peerB, asB, 20),
|
|
||||||
withdraw(peerA)},
|
|
||||||
want: []routetable.Origin{{ASN: asB, Peers: []string{peerB},
|
|
||||||
Routes: []routetable.Route{routeOf(peerB, asB, 20)}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a prefix no peer carries is removed",
|
|
||||||
steps: []step{announce(peerA, asA, 10), withdraw(peerA)},
|
|
||||||
want: nil,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a withdrawal from a peer that does not carry it changes nothing",
|
|
||||||
steps: []step{announce(peerA, asA, 10), withdraw(peerB)},
|
|
||||||
want: []routetable.Origin{{ASN: asA, Peers: []string{peerA},
|
|
||||||
Routes: []routetable.Route{routeOf(peerA, asA, 10)}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a peer going down leaves the prefix",
|
|
||||||
steps: []step{announce(peerA, asA, 10), announce(peerB, asA, 20),
|
|
||||||
peerDown(peerB)},
|
|
||||||
want: []routetable.Origin{{ASN: asA, Peers: []string{peerA},
|
|
||||||
Routes: []routetable.Route{routeOf(peerA, asA, 10)}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a peer going down after it withdrew changes nothing",
|
|
||||||
steps: []step{announce(peerA, asA, 10), announce(peerB, asA, 20),
|
|
||||||
withdraw(peerB), peerDown(peerB)},
|
|
||||||
want: []routetable.Origin{{ASN: asA, Peers: []string{peerA},
|
|
||||||
Routes: []routetable.Route{routeOf(peerA, asA, 10)}}},
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPeerDownLeavesEveryPrefix(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
prefixes := []netip.Prefix{
|
|
||||||
netip.MustParsePrefix(prefixV4),
|
|
||||||
netip.MustParsePrefix("10.2.0.0/16"),
|
|
||||||
netip.MustParsePrefix(prefixV6),
|
|
||||||
}
|
|
||||||
|
|
||||||
// peerA carries every prefix with origin asA, peerB the first one with
|
|
||||||
// origin asB.
|
|
||||||
for _, prefix := range prefixes {
|
|
||||||
table.Announce(prefix, asA, routeOf(peerA, asA, 10))
|
|
||||||
}
|
|
||||||
|
|
||||||
table.Announce(prefixes[0], asB, routeOf(peerB, asB, 20))
|
|
||||||
table.PeerDown(peerA)
|
|
||||||
|
|
||||||
want := []routetable.Origin{{ASN: asB, Peers: []string{peerB},
|
|
||||||
Routes: []routetable.Route{routeOf(peerB, asB, 20)}}}
|
|
||||||
|
|
||||||
got := table.Origins(prefixes[0])
|
|
||||||
if !reflect.DeepEqual(got, want) {
|
|
||||||
t.Errorf("Origins(%s) = %+v, want %+v", prefixes[0], got, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, prefix := range prefixes[1:] {
|
|
||||||
got := table.Origins(prefix)
|
|
||||||
if got != nil {
|
|
||||||
t.Errorf("Origins(%s) = %+v after the only peer went down, want nil",
|
|
||||||
prefix, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
counts := table.Counts()
|
|
||||||
if counts.IPv4Prefixes != 1 || counts.IPv6Prefixes != 0 ||
|
|
||||||
counts.IPv4Routes != 1 || counts.IPv6Routes != 0 {
|
|
||||||
t.Errorf("Counts() = %+v, want one IPv4 prefix and route", counts)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(table.OriginatedPrefixes(asA)) != 0 {
|
|
||||||
t.Errorf("OriginatedPrefixes(%d) = %+v, want none",
|
|
||||||
asA, table.OriginatedPrefixes(asA))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCountsOriginsWithoutKeptRoute(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
prefix := netip.MustParsePrefix(prefixV4)
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
|
|
||||||
steps := []struct {
|
|
||||||
apply step
|
|
||||||
want int
|
|
||||||
}{
|
|
||||||
{announce(peerA, asA, 10), 0},
|
|
||||||
{announce(peerB, asA, 20), 0},
|
|
||||||
{announce(peerC, asA, 30), 0},
|
|
||||||
{withdraw(peerC), 0},
|
|
||||||
{withdraw(peerB), 1},
|
|
||||||
// An update from the last peer is kept.
|
|
||||||
{announce(peerA, asA, 40), 0},
|
|
||||||
{withdraw(peerA), 0},
|
|
||||||
// Once more without kept routes, then without peers: the origin
|
|
||||||
// is removed.
|
|
||||||
{announce(peerA, asA, 10), 0},
|
|
||||||
{announce(peerB, asA, 20), 0},
|
|
||||||
{announce(peerC, asA, 30), 0},
|
|
||||||
{withdraw(peerB), 0},
|
|
||||||
{withdraw(peerC), 1},
|
|
||||||
{peerDown(peerA), 0},
|
|
||||||
}
|
|
||||||
|
|
||||||
for i, s := range steps {
|
|
||||||
s.apply(table, prefix)
|
|
||||||
|
|
||||||
got := table.Counts().OriginsWithoutKeptRoute
|
|
||||||
if got != s.want {
|
|
||||||
t.Errorf("after step %d, OriginsWithoutKeptRoute = %d, want %d",
|
|
||||||
i, got, s.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// change is a routetable.Change with its peer set as a list of peer numbers.
|
|
||||||
type change struct {
|
|
||||||
Prefix string
|
|
||||||
OriginASN int
|
|
||||||
Removed bool
|
|
||||||
Peers []uint
|
|
||||||
Routes []routetable.Route
|
|
||||||
}
|
|
||||||
|
|
||||||
// peerNumbers returns the numbers in set, or nil for a nil set.
|
|
||||||
func peerNumbers(set *bitset.BitSet) []uint {
|
|
||||||
if set == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return slices.Collect(set.EachSet())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTakeChanges(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// peerA was numbered 5 before; the next new peer is 6.
|
|
||||||
table := routetable.New(routetable.Params{
|
|
||||||
PeerNumbers: map[string]uint{peerA: 5},
|
|
||||||
})
|
|
||||||
v4 := netip.MustParsePrefix(prefixV4)
|
|
||||||
v6 := netip.MustParsePrefix(prefixV6)
|
|
||||||
|
|
||||||
table.Announce(v6, asA, routeOf(peerA, asA, 10))
|
|
||||||
table.Announce(v4, asA, routeOf(peerA, asA, 10))
|
|
||||||
table.Announce(v4, asA, routeOf(peerB, asA, 20))
|
|
||||||
|
|
||||||
checkChanges(t, table, []change{
|
|
||||||
{Prefix: prefixV4, OriginASN: asA, Peers: []uint{5, 6},
|
|
||||||
Routes: []routetable.Route{
|
|
||||||
routeOf(peerB, asA, 20), routeOf(peerA, asA, 10),
|
|
||||||
}},
|
|
||||||
{Prefix: prefixV6, OriginASN: asA, Peers: []uint{5},
|
|
||||||
Routes: []routetable.Route{routeOf(peerA, asA, 10)}},
|
|
||||||
}, map[string]uint{peerB: 6})
|
|
||||||
|
|
||||||
// TakeChanges cleared the record.
|
|
||||||
checkChanges(t, table, []change{}, map[string]uint{})
|
|
||||||
|
|
||||||
// peerB moves to asB; peerA withdraws, which removes the pair of v4 and
|
|
||||||
// asA.
|
|
||||||
table.Announce(v4, asB, routeOf(peerB, asB, 30))
|
|
||||||
table.Withdraw(v4, peerA)
|
|
||||||
|
|
||||||
checkChanges(t, table, []change{
|
|
||||||
{Prefix: prefixV4, OriginASN: asA, Removed: true},
|
|
||||||
{Prefix: prefixV4, OriginASN: asB, Peers: []uint{6},
|
|
||||||
Routes: []routetable.Route{routeOf(peerB, asB, 30)}},
|
|
||||||
}, map[string]uint{})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTakeChangesHoldsOnlyPairsThatChanged(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
v4 := netip.MustParsePrefix(prefixV4)
|
|
||||||
v6 := netip.MustParsePrefix(prefixV6)
|
|
||||||
|
|
||||||
// peerA, peerB and peerC are numbered 0, 1 and 2. The routes of peerC and
|
|
||||||
// peerB to v4 are kept; peerA keeps only its bit.
|
|
||||||
announceAll(table, prefixV4, asA, 10, peerA, peerB, peerC)
|
|
||||||
table.Announce(v6, asA, routeOf(peerB, asA, 10))
|
|
||||||
table.TakeChanges(math.MaxInt)
|
|
||||||
|
|
||||||
// peerA announces again, with a time older than both kept routes.
|
|
||||||
table.Announce(v4, asA, routeOf(peerA, asA, 15))
|
|
||||||
checkChanges(t, table, []change{}, map[string]uint{})
|
|
||||||
|
|
||||||
// peerC announces its kept route again.
|
|
||||||
table.Announce(v4, asA, routeOf(peerC, asA, 30))
|
|
||||||
checkChanges(t, table, []change{}, map[string]uint{})
|
|
||||||
|
|
||||||
// peerC announces the same path with a newer time, so its kept route
|
|
||||||
// changes.
|
|
||||||
table.Announce(v4, asA, routeOf(peerC, asA, 40))
|
|
||||||
checkChanges(t, table, []change{
|
|
||||||
{Prefix: prefixV4, OriginASN: asA, Peers: []uint{0, 1, 2},
|
|
||||||
Routes: []routetable.Route{
|
|
||||||
routeOf(peerC, asA, 40), routeOf(peerB, asA, 20),
|
|
||||||
}},
|
|
||||||
}, map[string]uint{})
|
|
||||||
|
|
||||||
// peerA withdraws v6, which it does not carry.
|
|
||||||
table.Withdraw(v6, peerA)
|
|
||||||
checkChanges(t, table, []change{}, map[string]uint{})
|
|
||||||
|
|
||||||
// peerA going down changes v4, the one prefix it carries.
|
|
||||||
table.PeerDown(peerA)
|
|
||||||
checkChanges(t, table, []change{
|
|
||||||
{Prefix: prefixV4, OriginASN: asA, Peers: []uint{1, 2},
|
|
||||||
Routes: []routetable.Route{
|
|
||||||
routeOf(peerC, asA, 40), routeOf(peerB, asA, 20),
|
|
||||||
}},
|
|
||||||
}, map[string]uint{})
|
|
||||||
|
|
||||||
// peerA, which carries nothing now, goes down again.
|
|
||||||
table.PeerDown(peerA)
|
|
||||||
checkChanges(t, table, []change{}, map[string]uint{})
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestTakeChangesTakesAtMostMaxChanges changes three prefix and origin AS
|
|
||||||
// pairs, then takes the changes two at a time. The first call must return two
|
|
||||||
// pairs and both new peers, and the second the third pair and no peer.
|
|
||||||
func TestTakeChangesTakesAtMostMaxChanges(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const maxChanges = 2
|
|
||||||
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
v4 := netip.MustParsePrefix(prefixV4)
|
|
||||||
v6 := netip.MustParsePrefix(prefixV6)
|
|
||||||
|
|
||||||
table.Announce(v4, asA, routeOf(peerA, asA, 10))
|
|
||||||
table.Announce(v4, asB, routeOf(peerB, asB, 10))
|
|
||||||
table.Announce(v6, asA, routeOf(peerA, asA, 10))
|
|
||||||
|
|
||||||
first, firstPeers := table.TakeChanges(maxChanges)
|
|
||||||
second, secondPeers := table.TakeChanges(maxChanges)
|
|
||||||
|
|
||||||
if len(first) != maxChanges || len(second) != 1 {
|
|
||||||
t.Fatalf("TakeChanges(%d) returned %d and then %d pairs, want %d and 1",
|
|
||||||
maxChanges, len(first), len(second), maxChanges)
|
|
||||||
}
|
|
||||||
|
|
||||||
wantPeers := map[string]uint{peerA: 0, peerB: 1}
|
|
||||||
if !reflect.DeepEqual(firstPeers, wantPeers) || len(secondPeers) != 0 {
|
|
||||||
t.Errorf("TakeChanges(%d) returned the new peers %v and then %v, "+
|
|
||||||
"want %v and none", maxChanges, firstPeers, secondPeers, wantPeers)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, c := range first {
|
|
||||||
if c.Prefix == second[0].Prefix && c.OriginASN == second[0].OriginASN {
|
|
||||||
t.Errorf("both calls returned %s with AS%d", c.Prefix, c.OriginASN)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPutBackChanges takes a table's changes and puts them back, as a caller
|
|
||||||
// does whose write failed. The next TakeChanges must return them again, with
|
|
||||||
// each pair's content then, and with the changes since.
|
|
||||||
func TestPutBackChanges(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
v4 := netip.MustParsePrefix(prefixV4)
|
|
||||||
v6 := netip.MustParsePrefix(prefixV6)
|
|
||||||
|
|
||||||
table.Announce(v4, asA, routeOf(peerA, asA, 10))
|
|
||||||
table.Announce(v6, asA, routeOf(peerA, asA, 10))
|
|
||||||
table.PutBackChanges(table.TakeChanges(math.MaxInt))
|
|
||||||
|
|
||||||
if got := table.ChangeCount(); got != 2 {
|
|
||||||
t.Errorf("ChangeCount() = %d after the changes were put back, want 2",
|
|
||||||
got)
|
|
||||||
}
|
|
||||||
|
|
||||||
// peerA withdraws v4, so v4's earlier content must not come back; peerB
|
|
||||||
// announces v6, and is a new peer as well.
|
|
||||||
table.Withdraw(v4, peerA)
|
|
||||||
table.Announce(v6, asA, routeOf(peerB, asA, 20))
|
|
||||||
|
|
||||||
checkChanges(t, table, []change{
|
|
||||||
{Prefix: prefixV4, OriginASN: asA, Removed: true},
|
|
||||||
{Prefix: prefixV6, OriginASN: asA, Peers: []uint{0, 1},
|
|
||||||
Routes: []routetable.Route{
|
|
||||||
routeOf(peerB, asA, 20), routeOf(peerA, asA, 10),
|
|
||||||
}},
|
|
||||||
}, map[string]uint{peerA: 0, peerB: 1})
|
|
||||||
|
|
||||||
if got := table.ChangeCount(); got != 0 {
|
|
||||||
t.Errorf("ChangeCount() = %d after TakeChanges, want 0", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLoad(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const otherV4 = "10.9.0.0/16"
|
|
||||||
|
|
||||||
v4 := netip.MustParsePrefix(prefixV4)
|
|
||||||
v6 := netip.MustParsePrefix(prefixV6)
|
|
||||||
other := netip.MustParsePrefix(otherV4)
|
|
||||||
|
|
||||||
// The routes of peerC and peerB to v4 are kept; peerA keeps only its bit.
|
|
||||||
// The two routes to v6 have the same time. The routes of other were
|
|
||||||
// withdrawn while peerA still carries it.
|
|
||||||
source := routetable.New(routetable.Params{})
|
|
||||||
announceAll(source, prefixV4, asA, 10, peerA, peerB, peerC)
|
|
||||||
announceAll(source, prefixV4, asB, 40, peerD)
|
|
||||||
source.Announce(v6, asA, routeOf(peerA, asA, 50))
|
|
||||||
source.Announce(v6, asA, routeOf(peerC, asA, 50))
|
|
||||||
announceAll(source, otherV4, asB, 10, peerA, peerB, peerC)
|
|
||||||
source.Withdraw(other, peerB)
|
|
||||||
source.Withdraw(other, peerC)
|
|
||||||
|
|
||||||
changes, peers := source.TakeChanges(math.MaxInt)
|
|
||||||
loaded := routetable.New(routetable.Params{PeerNumbers: peers})
|
|
||||||
|
|
||||||
// Loading a pair twice changes nothing.
|
|
||||||
for range 2 {
|
|
||||||
for _, change := range changes {
|
|
||||||
loaded.Load(change)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, prefix := range []netip.Prefix{v4, v6, other} {
|
|
||||||
got, want := loaded.Origins(prefix), source.Origins(prefix)
|
|
||||||
if !reflect.DeepEqual(got, want) {
|
|
||||||
t.Errorf("Origins(%s) = %+v after the load, want %+v", prefix, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if !reflect.DeepEqual(loaded.Counts(), source.Counts()) {
|
|
||||||
t.Errorf("Counts() = %+v after the load, want %+v",
|
|
||||||
loaded.Counts(), source.Counts())
|
|
||||||
}
|
|
||||||
|
|
||||||
checkChanges(t, loaded, []change{}, map[string]uint{})
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestConcurrentUse has goroutines update one table while others read it. The
|
|
||||||
// race detector, which make test runs, fails it when a method uses the table
|
|
||||||
// without its lock.
|
|
||||||
func TestConcurrentUse(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const (
|
|
||||||
// rounds is how many times each goroutine calls its method.
|
|
||||||
rounds = 1000
|
|
||||||
ipVersion = 4
|
|
||||||
sampleSize = 10
|
|
||||||
)
|
|
||||||
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
prefix := netip.MustParsePrefix(prefixV4)
|
|
||||||
|
|
||||||
uses := []func(round int){
|
|
||||||
func(round int) { table.Announce(prefix, asA, routeOf(peerA, asA, round)) },
|
|
||||||
func(round int) { table.Announce(prefix, asB, routeOf(peerB, asB, round)) },
|
|
||||||
func(int) { table.Withdraw(prefix, peerA) },
|
|
||||||
func(int) { table.PeerDown(peerB) },
|
|
||||||
func(int) { table.PutBackChanges(table.TakeChanges(math.MaxInt)) },
|
|
||||||
func(int) { table.ChangeCount() },
|
|
||||||
func(int) { table.Lookup(prefix.Addr()) },
|
|
||||||
func(int) { table.Origins(prefix) },
|
|
||||||
func(int) { table.OriginatedPrefixes(asA) },
|
|
||||||
func(int) { table.SamplePrefixes(ipVersion, prefix.Bits(), sampleSize) },
|
|
||||||
func(int) { table.Counts() },
|
|
||||||
func(int) { table.Overview(sampleSize) },
|
|
||||||
}
|
|
||||||
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
for _, use := range uses {
|
|
||||||
wg.Go(func() {
|
|
||||||
for round := range rounds {
|
|
||||||
use(round)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
wg.Wait()
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkChanges checks that TakeChanges, taking every change, returns want and
|
|
||||||
// wantPeers.
|
|
||||||
func checkChanges(
|
|
||||||
t *testing.T, table *routetable.Table, want []change,
|
|
||||||
wantPeers map[string]uint,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
changes, newPeers := table.TakeChanges(math.MaxInt)
|
|
||||||
|
|
||||||
got := make([]change, 0, len(changes))
|
|
||||||
for _, c := range changes {
|
|
||||||
got = append(got, change{
|
|
||||||
Prefix: c.Prefix.String(), OriginASN: c.OriginASN, Removed: c.Removed,
|
|
||||||
Peers: peerNumbers(c.Peers), Routes: c.Routes,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
if !reflect.DeepEqual(got, want) {
|
|
||||||
t.Errorf("TakeChanges() changes = %+v, want %+v", got, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !reflect.DeepEqual(newPeers, wantPeers) {
|
|
||||||
t.Errorf("TakeChanges() new peers = %v, want %v", newPeers, wantPeers)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+36
-134
@@ -1,43 +1,28 @@
|
|||||||
// Package routewatch contains the primary RouteWatch type that represents a
|
// Package routewatch contains the primary RouteWatch type that represents a running instance
|
||||||
// running instance of the application and contains pointers to its core
|
// of the application and contains pointers to its core dependencies, and is responsible for initialization.
|
||||||
// dependencies, and is responsible for initialization. It also contains the
|
|
||||||
// ASN WHOIS fetcher for background updates and the database maintainer for
|
|
||||||
// background maintenance tasks. The package also holds the handlers that
|
|
||||||
// write the feed's updates to the database in batches, and the flush loop
|
|
||||||
// that the AS, peer and prefix handlers share.
|
|
||||||
package routewatch
|
package routewatch
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"path/filepath"
|
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/blocklist"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/config"
|
"git.eeqj.de/sneak/routewatch/internal/config"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/database"
|
"git.eeqj.de/sneak/routewatch/internal/database"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
"git.eeqj.de/sneak/routewatch/internal/logger"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/metrics"
|
"git.eeqj.de/sneak/routewatch/internal/metrics"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/routetable"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/server"
|
"git.eeqj.de/sneak/routewatch/internal/server"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/streamer"
|
"git.eeqj.de/sneak/routewatch/internal/streamer"
|
||||||
|
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
)
|
)
|
||||||
|
|
||||||
// errNonBatchedNotImplemented is returned by Run when batched database writes
|
// Dependencies contains all dependencies for RouteWatch
|
||||||
// are disabled, since only the batched handlers exist.
|
|
||||||
var errNonBatchedNotImplemented = errors.New("non-batched handlers not implemented")
|
|
||||||
|
|
||||||
// Dependencies contains all dependencies for RouteWatch. Table is the routing
|
|
||||||
// table read from the database at startup, which the server reads too.
|
|
||||||
type Dependencies struct {
|
type Dependencies struct {
|
||||||
fx.In
|
fx.In
|
||||||
|
|
||||||
DB database.Store
|
DB database.Store
|
||||||
Table *routetable.Table
|
|
||||||
Streamer *streamer.Streamer
|
Streamer *streamer.Streamer
|
||||||
Server *server.Server
|
Server *server.Server
|
||||||
Logger *logger.Logger
|
Logger *logger.Logger
|
||||||
@@ -47,7 +32,6 @@ type Dependencies struct {
|
|||||||
// RouteWatch represents the main application instance
|
// RouteWatch represents the main application instance
|
||||||
type RouteWatch struct {
|
type RouteWatch struct {
|
||||||
db database.Store
|
db database.Store
|
||||||
table *routetable.Table
|
|
||||||
streamer *streamer.Streamer
|
streamer *streamer.Streamer
|
||||||
server *server.Server
|
server *server.Server
|
||||||
logger *logger.Logger
|
logger *logger.Logger
|
||||||
@@ -61,18 +45,12 @@ type RouteWatch struct {
|
|||||||
peeringHandler *PeeringHandler
|
peeringHandler *PeeringHandler
|
||||||
asnFetcher *ASNFetcher
|
asnFetcher *ASNFetcher
|
||||||
dbMaintainer *DBMaintainer
|
dbMaintainer *DBMaintainer
|
||||||
blocklists *blocklist.Blocklists
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// blocklistsDirName is the directory in the state directory that holds the
|
|
||||||
// blocklist copies.
|
|
||||||
const blocklistsDirName = "blocklists"
|
|
||||||
|
|
||||||
// New creates a new RouteWatch instance
|
// New creates a new RouteWatch instance
|
||||||
func New(deps Dependencies) *RouteWatch {
|
func New(deps Dependencies) *RouteWatch {
|
||||||
rw := &RouteWatch{
|
rw := &RouteWatch{
|
||||||
db: deps.DB,
|
db: deps.DB,
|
||||||
table: deps.Table,
|
|
||||||
streamer: deps.Streamer,
|
streamer: deps.Streamer,
|
||||||
server: deps.Server,
|
server: deps.Server,
|
||||||
logger: deps.Logger,
|
logger: deps.Logger,
|
||||||
@@ -90,17 +68,14 @@ func (rw *RouteWatch) Run(ctx context.Context) error {
|
|||||||
// Apply runtime limit if specified
|
// Apply runtime limit if specified
|
||||||
if rw.maxRuntime > 0 {
|
if rw.maxRuntime > 0 {
|
||||||
var cancel context.CancelFunc
|
var cancel context.CancelFunc
|
||||||
|
|
||||||
ctx, cancel = context.WithTimeout(ctx, rw.maxRuntime)
|
ctx, cancel = context.WithTimeout(ctx, rw.maxRuntime)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
rw.logger.Info("Running with time limit", "max_runtime", rw.maxRuntime)
|
rw.logger.Info("Running with time limit", "max_runtime", rw.maxRuntime)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Register database handler to process BGP UPDATE messages
|
// Register database handler to process BGP UPDATE messages
|
||||||
if rw.config.EnableBatchedDatabaseWrites {
|
if rw.config.EnableBatchedDatabaseWrites {
|
||||||
rw.logger.Info("Using batched database handlers for improved performance")
|
rw.logger.Info("Using batched database handlers for improved performance")
|
||||||
|
|
||||||
// ASHandler maintains the asns table
|
// ASHandler maintains the asns table
|
||||||
rw.dbHandler = NewASHandler(rw.db, rw.logger)
|
rw.dbHandler = NewASHandler(rw.db, rw.logger)
|
||||||
rw.streamer.RegisterHandler(rw.dbHandler)
|
rw.streamer.RegisterHandler(rw.dbHandler)
|
||||||
@@ -109,10 +84,8 @@ func (rw *RouteWatch) Run(ctx context.Context) error {
|
|||||||
rw.peerHandler = NewPeerHandler(rw.db, rw.logger)
|
rw.peerHandler = NewPeerHandler(rw.db, rw.logger)
|
||||||
rw.streamer.RegisterHandler(rw.peerHandler)
|
rw.streamer.RegisterHandler(rw.peerHandler)
|
||||||
|
|
||||||
// PrefixHandler keeps the routing table in memory current, and writes
|
// PrefixHandler maintains the prefixes and live_routes tables
|
||||||
// its changes to the routes table and new prefixes to the prefix
|
rw.prefixHandler = NewPrefixHandler(rw.db, rw.logger)
|
||||||
// tables
|
|
||||||
rw.prefixHandler = NewPrefixHandler(rw.db, rw.logger, rw.table)
|
|
||||||
rw.prefixHandler.SetMetricsTracker(rw.streamer.GetMetricsTracker())
|
rw.prefixHandler.SetMetricsTracker(rw.streamer.GetMetricsTracker())
|
||||||
rw.streamer.RegisterHandler(rw.prefixHandler)
|
rw.streamer.RegisterHandler(rw.prefixHandler)
|
||||||
|
|
||||||
@@ -123,42 +96,29 @@ func (rw *RouteWatch) Run(ctx context.Context) error {
|
|||||||
// Non-batched handlers not implemented yet
|
// Non-batched handlers not implemented yet
|
||||||
rw.logger.Error("Non-batched handlers not implemented")
|
rw.logger.Error("Non-batched handlers not implemented")
|
||||||
|
|
||||||
return errNonBatchedNotImplemented
|
return fmt.Errorf("non-batched handlers not implemented")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// No longer need routing table handler - PrefixHandler maintains live_routes table
|
||||||
|
|
||||||
// Start streaming
|
// Start streaming
|
||||||
err := rw.streamer.Start(ctx)
|
if err := rw.streamer.Start(); err != nil {
|
||||||
if err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Start ASN WHOIS fetcher for background updates. The HTTP handlers read
|
|
||||||
// the server's fetcher without a lock, so it is set before they run.
|
|
||||||
rw.asnFetcher = NewASNFetcher(rw.db, rw.logger.Logger)
|
|
||||||
rw.asnFetcher.Start(ctx)
|
|
||||||
rw.server.SetASNFetcher(rw.asnFetcher)
|
|
||||||
|
|
||||||
// Load the blocklist copies from disk and start downloading the lists.
|
|
||||||
// The HTTP handlers read the server's blocklists without a lock, so they
|
|
||||||
// are set before the handlers run.
|
|
||||||
blocklists := blocklist.New(blocklist.Params{
|
|
||||||
Dir: filepath.Join(rw.config.StateDir, blocklistsDirName),
|
|
||||||
Lists: blocklist.Lists(),
|
|
||||||
Logger: rw.logger.Logger,
|
|
||||||
})
|
|
||||||
blocklists.Start(ctx)
|
|
||||||
rw.blocklists = blocklists
|
|
||||||
rw.server.SetBlocklists(blocklists)
|
|
||||||
|
|
||||||
// Start HTTP server
|
// Start HTTP server
|
||||||
err = rw.server.Start(ctx)
|
if err := rw.server.Start(); err != nil {
|
||||||
if err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Start ASN WHOIS fetcher for background updates
|
||||||
|
rw.asnFetcher = NewASNFetcher(rw.db, rw.logger.Logger)
|
||||||
|
rw.asnFetcher.Start()
|
||||||
|
rw.server.SetASNFetcher(rw.asnFetcher)
|
||||||
|
|
||||||
// Start database maintenance goroutine
|
// Start database maintenance goroutine
|
||||||
rw.dbMaintainer = NewDBMaintainer(rw.db, rw.logger.Logger)
|
rw.dbMaintainer = NewDBMaintainer(rw.db, rw.logger.Logger)
|
||||||
rw.dbMaintainer.Start(ctx)
|
rw.dbMaintainer.Start()
|
||||||
|
|
||||||
// Wait for context cancellation
|
// Wait for context cancellation
|
||||||
<-ctx.Done()
|
<-ctx.Done()
|
||||||
@@ -166,71 +126,56 @@ func (rw *RouteWatch) Run(ctx context.Context) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Shutdown stops all services in order: the streamer, the handlers, which
|
// Shutdown performs graceful shutdown of all services
|
||||||
// write what they hold, the WHOIS fetcher, the database maintainer, the
|
func (rw *RouteWatch) Shutdown() {
|
||||||
// blocklist downloads and the HTTP server, and last the database, which it
|
|
||||||
// checkpoints and closes. Each stage waits at most until ctx ends, so
|
|
||||||
// Shutdown returns soon after: a handler logs what it could not write by
|
|
||||||
// then, and the HTTP server closes the connections of requests still
|
|
||||||
// running.
|
|
||||||
func (rw *RouteWatch) Shutdown(ctx context.Context) {
|
|
||||||
rw.mu.Lock()
|
rw.mu.Lock()
|
||||||
if rw.shutdown {
|
if rw.shutdown {
|
||||||
rw.mu.Unlock()
|
rw.mu.Unlock()
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
rw.shutdown = true
|
rw.shutdown = true
|
||||||
rw.mu.Unlock()
|
rw.mu.Unlock()
|
||||||
|
|
||||||
// Stop the streamer first: it returns once the handlers have been handed
|
// Stop batched handlers first to flush remaining batches
|
||||||
// every message it queued, or when ctx ends. Then each handler writes
|
|
||||||
// what it holds.
|
|
||||||
rw.streamer.Stop(ctx)
|
|
||||||
|
|
||||||
if rw.dbHandler != nil {
|
if rw.dbHandler != nil {
|
||||||
rw.logger.Info("Flushing database handler")
|
rw.logger.Info("Flushing database handler")
|
||||||
rw.dbHandler.Stop(ctx)
|
rw.dbHandler.Stop()
|
||||||
}
|
}
|
||||||
|
|
||||||
if rw.peerHandler != nil {
|
if rw.peerHandler != nil {
|
||||||
rw.logger.Info("Flushing peer handler")
|
rw.logger.Info("Flushing peer handler")
|
||||||
rw.peerHandler.Stop(ctx)
|
rw.peerHandler.Stop()
|
||||||
}
|
}
|
||||||
|
|
||||||
if rw.prefixHandler != nil {
|
if rw.prefixHandler != nil {
|
||||||
rw.logger.Info("Flushing prefix handler")
|
rw.logger.Info("Flushing prefix handler")
|
||||||
rw.prefixHandler.Stop(ctx)
|
rw.prefixHandler.Stop()
|
||||||
}
|
}
|
||||||
|
|
||||||
if rw.peeringHandler != nil {
|
if rw.peeringHandler != nil {
|
||||||
rw.logger.Info("Flushing peering handler")
|
rw.logger.Info("Flushing peering handler")
|
||||||
rw.peeringHandler.Stop(ctx)
|
rw.peeringHandler.Stop()
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stop ASN WHOIS fetcher
|
// Stop ASN WHOIS fetcher
|
||||||
if rw.asnFetcher != nil {
|
if rw.asnFetcher != nil {
|
||||||
rw.asnFetcher.Stop(ctx)
|
rw.asnFetcher.Stop()
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stop database maintainer
|
// Stop database maintainer
|
||||||
if rw.dbMaintainer != nil {
|
if rw.dbMaintainer != nil {
|
||||||
rw.dbMaintainer.Stop(ctx)
|
rw.dbMaintainer.Stop()
|
||||||
}
|
}
|
||||||
|
|
||||||
if rw.blocklists != nil {
|
// Stop services
|
||||||
rw.blocklists.Stop(ctx)
|
rw.streamer.Stop()
|
||||||
}
|
|
||||||
|
|
||||||
err := rw.server.Stop(ctx)
|
// Stop HTTP server with a timeout
|
||||||
if err != nil {
|
const serverStopTimeout = 5 * time.Second
|
||||||
rw.logger.Error("Stopped the HTTP server before every request finished",
|
stopCtx, cancel := context.WithTimeout(context.Background(), serverStopTimeout)
|
||||||
"error", err)
|
defer cancel()
|
||||||
|
if err := rw.server.Stop(stopCtx); err != nil {
|
||||||
|
rw.logger.Error("Failed to stop HTTP server gracefully", "error", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
rw.closeDatabase(ctx)
|
|
||||||
|
|
||||||
// Log final metrics
|
// Log final metrics
|
||||||
metrics := rw.streamer.GetMetrics()
|
metrics := rw.streamer.GetMetrics()
|
||||||
rw.logger.Info("Final metrics",
|
rw.logger.Info("Final metrics",
|
||||||
@@ -240,54 +185,12 @@ func (rw *RouteWatch) Shutdown(ctx context.Context) {
|
|||||||
"bits_per_sec", metrics.BitsPerSec,
|
"bits_per_sec", metrics.BitsPerSec,
|
||||||
"duration", time.Since(metrics.ConnectedSince),
|
"duration", time.Since(metrics.ConnectedSince),
|
||||||
)
|
)
|
||||||
}
|
|
||||||
|
|
||||||
// closeDatabase checkpoints and closes the database, unless ctx has ended: a
|
|
||||||
// write may then still be running, and both would wait for it. SQLite rolls
|
|
||||||
// back a write cut off by the exit when the database next opens.
|
|
||||||
func (rw *RouteWatch) closeDatabase(ctx context.Context) {
|
|
||||||
if ctx.Err() != nil {
|
|
||||||
rw.logger.Error("Stop deadline passed; the database is not closed")
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
err := rw.db.Checkpoint(ctx)
|
|
||||||
if err != nil {
|
|
||||||
rw.logger.Error("Failed to checkpoint the database", "error", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = rw.db.Close()
|
|
||||||
if err != nil {
|
|
||||||
rw.logger.Error("Failed to close the database", "error", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// readRouteTable reads the routing table from the database and logs how many
|
|
||||||
// prefix and origin AS pairs and peers it holds and how long the read took.
|
|
||||||
// fx calls it once, to build the server and RouteWatch, before the streamer
|
|
||||||
// connects, so the prefix handler applies the feed's updates to all of it. No
|
|
||||||
// start timeout limits the read: fx builds them before it runs a start hook.
|
|
||||||
func readRouteTable(
|
|
||||||
db database.Store, logger *logger.Logger,
|
|
||||||
) (*routetable.Table, error) {
|
|
||||||
start := time.Now()
|
|
||||||
|
|
||||||
table, err := db.ReadRouteTable(context.Background())
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to read the routing table: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
counts := table.Counts()
|
|
||||||
logger.Info("Read the routing table", "pairs", counts.Pairs,
|
|
||||||
"peers", counts.Peers, "duration_ms", time.Since(start).Milliseconds())
|
|
||||||
|
|
||||||
return table, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// getModule provides all fx dependencies
|
// getModule provides all fx dependencies
|
||||||
func getModule() []fx.Option {
|
func getModule() fx.Option {
|
||||||
return []fx.Option{
|
return fx.Options(
|
||||||
fx.Provide(
|
fx.Provide(
|
||||||
logger.New,
|
logger.New,
|
||||||
config.New,
|
config.New,
|
||||||
@@ -296,10 +199,9 @@ func getModule() []fx.Option {
|
|||||||
database.New,
|
database.New,
|
||||||
fx.As(new(database.Store)),
|
fx.As(new(database.Store)),
|
||||||
),
|
),
|
||||||
readRouteTable,
|
|
||||||
streamer.New,
|
streamer.New,
|
||||||
server.New,
|
server.New,
|
||||||
New,
|
New,
|
||||||
),
|
),
|
||||||
}
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,769 +0,0 @@
|
|||||||
package routewatch
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"net/netip"
|
|
||||||
"slices"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/config"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/database"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/metrics"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/ristypes"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/routetable"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/server"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/streamer"
|
|
||||||
)
|
|
||||||
|
|
||||||
// mockStore is a mock implementation of database.Store for testing
|
|
||||||
type mockStore struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
|
|
||||||
// Counters for tracking calls
|
|
||||||
ASNCount int
|
|
||||||
PrefixCount int
|
|
||||||
PeeringCount int
|
|
||||||
WithdrawalCount int
|
|
||||||
|
|
||||||
// Track unique items
|
|
||||||
ASNs map[int]*database.ASN
|
|
||||||
Prefixes map[string]*database.Prefix
|
|
||||||
Peerings map[string]bool // key is "from_to"
|
|
||||||
// TablePrefixes holds the prefixes that the written changes of the
|
|
||||||
// routing table leave some peer carrying.
|
|
||||||
TablePrefixes map[string]bool
|
|
||||||
|
|
||||||
// Track IP versions
|
|
||||||
IPv4Prefixes int
|
|
||||||
IPv6Prefixes int
|
|
||||||
}
|
|
||||||
|
|
||||||
// newMockStore creates a new mock store
|
|
||||||
func newMockStore() *mockStore {
|
|
||||||
return &mockStore{
|
|
||||||
ASNs: make(map[int]*database.ASN),
|
|
||||||
Prefixes: make(map[string]*database.Prefix),
|
|
||||||
Peerings: make(map[string]bool),
|
|
||||||
TablePrefixes: make(map[string]bool),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetOrCreateASN mock implementation
|
|
||||||
func (m *mockStore) GetOrCreateASN(
|
|
||||||
number int, timestamp time.Time,
|
|
||||||
) (*database.ASN, error) {
|
|
||||||
m.mu.Lock()
|
|
||||||
defer m.mu.Unlock()
|
|
||||||
|
|
||||||
if asn, exists := m.ASNs[number]; exists {
|
|
||||||
asn.LastSeen = timestamp
|
|
||||||
|
|
||||||
return asn, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
asn := &database.ASN{
|
|
||||||
ASN: number,
|
|
||||||
FirstSeen: timestamp,
|
|
||||||
LastSeen: timestamp,
|
|
||||||
}
|
|
||||||
m.ASNs[number] = asn
|
|
||||||
m.ASNCount++
|
|
||||||
|
|
||||||
return asn, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// UpdatePrefixesBatch mock implementation
|
|
||||||
func (m *mockStore) UpdatePrefixesBatch(prefixes map[string]time.Time) error {
|
|
||||||
m.mu.Lock()
|
|
||||||
defer m.mu.Unlock()
|
|
||||||
|
|
||||||
for prefix, timestamp := range prefixes {
|
|
||||||
if _, exists := m.Prefixes[prefix]; !exists {
|
|
||||||
const (
|
|
||||||
ipVersionV4 = 4
|
|
||||||
ipVersionV6 = 6
|
|
||||||
)
|
|
||||||
|
|
||||||
ipVersion := ipVersionV4
|
|
||||||
if strings.Contains(prefix, ":") {
|
|
||||||
ipVersion = ipVersionV6
|
|
||||||
}
|
|
||||||
|
|
||||||
m.Prefixes[prefix] = &database.Prefix{
|
|
||||||
Prefix: prefix,
|
|
||||||
IPVersion: ipVersion,
|
|
||||||
FirstSeen: timestamp,
|
|
||||||
}
|
|
||||||
m.PrefixCount++
|
|
||||||
|
|
||||||
if ipVersion == ipVersionV4 {
|
|
||||||
m.IPv4Prefixes++
|
|
||||||
} else {
|
|
||||||
m.IPv6Prefixes++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetOrCreatePrefix mock implementation
|
|
||||||
func (m *mockStore) GetOrCreatePrefix(
|
|
||||||
prefix string, timestamp time.Time,
|
|
||||||
) (*database.Prefix, error) {
|
|
||||||
m.mu.Lock()
|
|
||||||
defer m.mu.Unlock()
|
|
||||||
|
|
||||||
if p, exists := m.Prefixes[prefix]; exists {
|
|
||||||
return p, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
const (
|
|
||||||
ipVersionV4 = 4
|
|
||||||
ipVersionV6 = 6
|
|
||||||
)
|
|
||||||
|
|
||||||
ipVersion := ipVersionV4
|
|
||||||
if strings.Contains(prefix, ":") {
|
|
||||||
ipVersion = ipVersionV6
|
|
||||||
}
|
|
||||||
|
|
||||||
p := &database.Prefix{
|
|
||||||
Prefix: prefix,
|
|
||||||
IPVersion: ipVersion,
|
|
||||||
FirstSeen: timestamp,
|
|
||||||
}
|
|
||||||
m.Prefixes[prefix] = p
|
|
||||||
m.PrefixCount++
|
|
||||||
|
|
||||||
if ipVersion == ipVersionV4 {
|
|
||||||
m.IPv4Prefixes++
|
|
||||||
} else {
|
|
||||||
m.IPv6Prefixes++
|
|
||||||
}
|
|
||||||
|
|
||||||
return p, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// RecordAnnouncement mock implementation
|
|
||||||
func (m *mockStore) RecordAnnouncement(_ *database.Announcement) error {
|
|
||||||
// Not tracking announcements in detail for now
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// RecordPeering mock implementation
|
|
||||||
func (m *mockStore) RecordPeering(asA, asB int, _ database.SeenTimes) error {
|
|
||||||
m.mu.Lock()
|
|
||||||
defer m.mu.Unlock()
|
|
||||||
|
|
||||||
// Normalize
|
|
||||||
if asA > asB {
|
|
||||||
asA, asB = asB, asA
|
|
||||||
}
|
|
||||||
|
|
||||||
key := fmt.Sprintf("%d_%d", asA, asB)
|
|
||||||
if !m.Peerings[key] {
|
|
||||||
m.Peerings[key] = true
|
|
||||||
m.PeeringCount++
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// UpdatePeer mock implementation
|
|
||||||
func (m *mockStore) UpdatePeer(_ string, _ int, _ string, _ time.Time) error {
|
|
||||||
// Simple mock - just return nil
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Close mock implementation
|
|
||||||
func (m *mockStore) Close() error {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetStats returns statistics about the mock store
|
|
||||||
func (m *mockStore) GetStats() (database.Stats, error) {
|
|
||||||
m.mu.Lock()
|
|
||||||
defer m.mu.Unlock()
|
|
||||||
|
|
||||||
return database.Stats{
|
|
||||||
ASNs: len(m.ASNs),
|
|
||||||
Prefixes: len(m.Prefixes),
|
|
||||||
IPv4Prefixes: m.IPv4Prefixes,
|
|
||||||
IPv6Prefixes: m.IPv6Prefixes,
|
|
||||||
Peerings: m.PeeringCount,
|
|
||||||
Peers: 10, // Mock peer count
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetStatsContext returns statistics about the mock store with context support
|
|
||||||
func (m *mockStore) GetStatsContext(_ context.Context) (database.Stats, error) {
|
|
||||||
return m.GetStats()
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetAS returns the stored AS, or only its number when none is stored.
|
|
||||||
func (m *mockStore) GetAS(_ context.Context, asn int) (*database.ASN, bool, error) {
|
|
||||||
m.mu.Lock()
|
|
||||||
defer m.mu.Unlock()
|
|
||||||
|
|
||||||
if asnInfo, exists := m.ASNs[asn]; exists {
|
|
||||||
return asnInfo, true, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return &database.ASN{ASN: asn}, false, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetLastSeenASes returns copies of the stored ASes last seen most recently
|
|
||||||
// and longest ago, or neither when none is stored.
|
|
||||||
func (m *mockStore) GetLastSeenASes(
|
|
||||||
_ context.Context,
|
|
||||||
) (*database.ASN, *database.ASN, error) {
|
|
||||||
m.mu.Lock()
|
|
||||||
defer m.mu.Unlock()
|
|
||||||
|
|
||||||
var newest, oldest *database.ASN
|
|
||||||
|
|
||||||
for _, asn := range m.ASNs {
|
|
||||||
if newest == nil || asn.LastSeen.After(newest.LastSeen) {
|
|
||||||
newest = asn
|
|
||||||
}
|
|
||||||
|
|
||||||
if oldest == nil || asn.LastSeen.Before(oldest.LastSeen) {
|
|
||||||
oldest = asn
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if newest == nil {
|
|
||||||
return nil, nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
newestCopy, oldestCopy := *newest, *oldest
|
|
||||||
|
|
||||||
return &newestCopy, &oldestCopy, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetASWHOIS returns the stored AS, or only its number when none is stored.
|
|
||||||
func (m *mockStore) GetASWHOIS(ctx context.Context, asn int) (*database.ASN, error) {
|
|
||||||
asnInfo, _, err := m.GetAS(ctx, asn)
|
|
||||||
|
|
||||||
return asnInfo, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetPrefixFirstSeen returns the zero time: the mock keeps no prefix times.
|
|
||||||
func (m *mockStore) GetPrefixFirstSeen(
|
|
||||||
_ context.Context, _ netip.Prefix,
|
|
||||||
) (time.Time, error) {
|
|
||||||
return time.Time{}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetASPeers mock implementation
|
|
||||||
func (m *mockStore) GetASPeers(_ int) ([]database.ASPeer, error) {
|
|
||||||
// Return empty peers for now
|
|
||||||
return []database.ASPeer{}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetASPeersContext mock implementation with context support
|
|
||||||
func (m *mockStore) GetASPeersContext(
|
|
||||||
_ context.Context, asn int,
|
|
||||||
) ([]database.ASPeer, error) {
|
|
||||||
return m.GetASPeers(asn)
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetNextStaleASN mock implementation
|
|
||||||
func (m *mockStore) GetNextStaleASN(_ context.Context, _ time.Duration) (int, error) {
|
|
||||||
return 0, database.ErrNoStaleASN
|
|
||||||
}
|
|
||||||
|
|
||||||
// UpdateASNWHOIS mock implementation
|
|
||||||
func (m *mockStore) UpdateASNWHOIS(
|
|
||||||
_ context.Context, _ *database.ASNWHOISUpdate,
|
|
||||||
) error {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetWHOISStats mock implementation
|
|
||||||
func (m *mockStore) GetWHOISStats(
|
|
||||||
_ context.Context, _ time.Duration,
|
|
||||||
) (*database.WHOISStats, error) {
|
|
||||||
m.mu.Lock()
|
|
||||||
defer m.mu.Unlock()
|
|
||||||
|
|
||||||
return &database.WHOISStats{
|
|
||||||
TotalASNs: len(m.ASNs),
|
|
||||||
FreshASNs: 0,
|
|
||||||
StaleASNs: 0,
|
|
||||||
NeverFetched: len(m.ASNs),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// WriteRouteChanges records the prefixes the changes leave some peer carrying,
|
|
||||||
// whatever their origin AS.
|
|
||||||
func (m *mockStore) WriteRouteChanges(
|
|
||||||
changes []routetable.Change, _ map[string]uint,
|
|
||||||
) error {
|
|
||||||
m.mu.Lock()
|
|
||||||
defer m.mu.Unlock()
|
|
||||||
|
|
||||||
for _, change := range changes {
|
|
||||||
if change.Removed {
|
|
||||||
delete(m.TablePrefixes, change.Prefix.String())
|
|
||||||
} else {
|
|
||||||
m.TablePrefixes[change.Prefix.String()] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ReadRouteTable returns an empty routing table.
|
|
||||||
func (m *mockStore) ReadRouteTable(_ context.Context) (*routetable.Table, error) {
|
|
||||||
return routetable.New(routetable.Params{}), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetOrCreateASNBatch mock implementation
|
|
||||||
func (m *mockStore) GetOrCreateASNBatch(asns map[int]database.SeenTimes) error {
|
|
||||||
m.mu.Lock()
|
|
||||||
defer m.mu.Unlock()
|
|
||||||
|
|
||||||
for number, seen := range asns {
|
|
||||||
if _, exists := m.ASNs[number]; !exists {
|
|
||||||
m.ASNs[number] = &database.ASN{
|
|
||||||
ASN: number,
|
|
||||||
FirstSeen: seen.First,
|
|
||||||
LastSeen: seen.Last,
|
|
||||||
}
|
|
||||||
m.ASNCount++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// UpdatePeerBatch mock implementation
|
|
||||||
func (m *mockStore) UpdatePeerBatch(_ map[string]database.PeerUpdate) error {
|
|
||||||
// Simple mock - just return nil
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Vacuum mock implementation
|
|
||||||
func (m *mockStore) Vacuum(_ context.Context) error {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Analyze mock implementation
|
|
||||||
func (m *mockStore) Analyze(_ context.Context) error {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Checkpoint mock implementation
|
|
||||||
func (m *mockStore) Checkpoint(_ context.Context) error {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Ping mock implementation
|
|
||||||
func (m *mockStore) Ping(_ context.Context) error {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRouteWatchLiveFeed(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
if testing.Short() {
|
|
||||||
t.Skip("skipping live RIPE RIS network feed test in short mode; " +
|
|
||||||
"run without -short to include it")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create mock database
|
|
||||||
mockDB := newMockStore()
|
|
||||||
|
|
||||||
defer func() {
|
|
||||||
err := mockDB.Close()
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("failed to close mock database: %v", err)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
logger := logger.New()
|
|
||||||
|
|
||||||
// Create metrics tracker
|
|
||||||
metricsTracker := metrics.New()
|
|
||||||
|
|
||||||
// Create streamer
|
|
||||||
s := streamer.New(logger, metricsTracker)
|
|
||||||
|
|
||||||
// The state directory holds the blocklist copies Run downloads.
|
|
||||||
cfg := &config.Config{
|
|
||||||
StateDir: t.TempDir(),
|
|
||||||
MaxRuntime: 5 * time.Second,
|
|
||||||
EnableBatchedDatabaseWrites: true,
|
|
||||||
}
|
|
||||||
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
|
|
||||||
// Create server
|
|
||||||
srv := server.New(mockDB, table, s, logger, cfg)
|
|
||||||
|
|
||||||
// Create RouteWatch with 5 second limit
|
|
||||||
deps := Dependencies{
|
|
||||||
DB: mockDB,
|
|
||||||
Table: table,
|
|
||||||
Streamer: s,
|
|
||||||
Server: srv,
|
|
||||||
Logger: logger,
|
|
||||||
Config: cfg,
|
|
||||||
}
|
|
||||||
rw := New(deps)
|
|
||||||
|
|
||||||
// Run with context
|
|
||||||
ctx := context.Background()
|
|
||||||
go func() {
|
|
||||||
_ = rw.Run(ctx)
|
|
||||||
}()
|
|
||||||
|
|
||||||
// Wait for the configured duration
|
|
||||||
time.Sleep(5 * time.Second)
|
|
||||||
|
|
||||||
// Force peering processing for test
|
|
||||||
if rw.peeringHandler != nil {
|
|
||||||
rw.peeringHandler.ProcessPeeringsNow()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get statistics
|
|
||||||
stats, err := mockDB.GetStats()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Failed to get stats: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if stats.ASNs == 0 {
|
|
||||||
t.Error("Expected to receive some ASNs from live feed")
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Logf("Received %d unique ASNs in 5 seconds", stats.ASNs)
|
|
||||||
|
|
||||||
if stats.Prefixes == 0 {
|
|
||||||
t.Error("Expected to receive some prefixes from live feed")
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Logf("Received %d unique prefixes (%d IPv4, %d IPv6) in 5 seconds",
|
|
||||||
stats.Prefixes, stats.IPv4Prefixes, stats.IPv6Prefixes)
|
|
||||||
|
|
||||||
if stats.Peerings == 0 {
|
|
||||||
t.Error("Expected to receive some peerings from live feed")
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Logf("Recorded %d AS peering relationships in 5 seconds", stats.Peerings)
|
|
||||||
}
|
|
||||||
|
|
||||||
// gatedPrefixHandler is a PrefixHandler whose worker in the streamer waits on
|
|
||||||
// the first message until release is closed, so the messages after it stay on
|
|
||||||
// the streamer's queue.
|
|
||||||
type gatedPrefixHandler struct {
|
|
||||||
*PrefixHandler
|
|
||||||
|
|
||||||
release chan struct{}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h gatedPrefixHandler) HandleMessage(msg *ristypes.RISMessage) {
|
|
||||||
<-h.release
|
|
||||||
h.PrefixHandler.HandleMessage(msg)
|
|
||||||
}
|
|
||||||
|
|
||||||
// announcementFeed returns the lines RIS Live would send for one UPDATE per
|
|
||||||
// prefix, each announcing that prefix from testPeerIP with origin testASNA,
|
|
||||||
// followed by a KEEPALIVE. The streamer hands a KEEPALIVE to no handler, so
|
|
||||||
// once a raw handler has seen it, every update before it is queued.
|
|
||||||
func announcementFeed(prefixes []string) string {
|
|
||||||
var feed strings.Builder
|
|
||||||
|
|
||||||
for _, prefix := range prefixes {
|
|
||||||
fmt.Fprintf(&feed, `{"type":"ris_message","data":{"type":"UPDATE",`+
|
|
||||||
`"peer":"%s","peer_asn":"%d","timestamp":1700000000,"path":[%d],`+
|
|
||||||
`"announcements":[{"next_hop":"%s","prefixes":["%s"]}]}}`+"\n",
|
|
||||||
testPeerIP, testASNA, testASNA, testPeerIP, prefix)
|
|
||||||
}
|
|
||||||
|
|
||||||
fmt.Fprintf(&feed, `{"type":"ris_message","data":{"type":"KEEPALIVE",`+
|
|
||||||
`"peer":"%s","peer_asn":"%d","timestamp":1700000000}}`+"\n",
|
|
||||||
testPeerIP, testASNA)
|
|
||||||
|
|
||||||
return feed.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestShutdownWritesTheUpdatesQueuedInTheStreamer has a local server send
|
|
||||||
// announcements of three prefixes and keep the response open. The prefix
|
|
||||||
// handler's worker waits on the first one until the server sees the streamer
|
|
||||||
// close the connection, which only Shutdown does, so all three are still in
|
|
||||||
// the streamer when Shutdown starts. Each prefix must be in the database when
|
|
||||||
// Shutdown returns, both in the prefix tables and in the routing table's
|
|
||||||
// writes.
|
|
||||||
func TestShutdownWritesTheUpdatesQueuedInTheStreamer(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
prefixes := []string{"192.0.2.0/24", "198.51.100.0/24", "203.0.113.0/24"}
|
|
||||||
feed := announcementFeed(prefixes)
|
|
||||||
|
|
||||||
release := make(chan struct{})
|
|
||||||
|
|
||||||
srv := httptest.NewServer(http.HandlerFunc(
|
|
||||||
func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
_, _ = io.WriteString(w, feed)
|
|
||||||
_ = http.NewResponseController(w).Flush()
|
|
||||||
|
|
||||||
<-r.Context().Done()
|
|
||||||
close(release)
|
|
||||||
}))
|
|
||||||
defer srv.Close()
|
|
||||||
|
|
||||||
store := newMockStore()
|
|
||||||
log := logger.New()
|
|
||||||
cfg := &config.Config{}
|
|
||||||
s := streamer.New(log, metrics.New())
|
|
||||||
s.SetURL(srv.URL)
|
|
||||||
|
|
||||||
rw := New(Dependencies{
|
|
||||||
DB: store,
|
|
||||||
Streamer: s,
|
|
||||||
Server: server.New(store, nil, s, log, cfg),
|
|
||||||
Logger: log,
|
|
||||||
Config: cfg,
|
|
||||||
})
|
|
||||||
rw.prefixHandler = NewPrefixHandler(store, log, routetable.New(routetable.Params{}))
|
|
||||||
s.RegisterHandler(gatedPrefixHandler{
|
|
||||||
PrefixHandler: rw.prefixHandler, release: release,
|
|
||||||
})
|
|
||||||
|
|
||||||
linesSeen := make(chan struct{}, len(prefixes)+1)
|
|
||||||
|
|
||||||
s.RegisterRawHandler(func(string) { linesSeen <- struct{}{} })
|
|
||||||
|
|
||||||
err := s.Start(t.Context())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Start returned error: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for range cap(linesSeen) {
|
|
||||||
select {
|
|
||||||
case <-linesSeen:
|
|
||||||
case <-time.After(stepTimeout):
|
|
||||||
t.Fatalf("the streamer did not read %d lines within %s",
|
|
||||||
cap(linesSeen), stepTimeout)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
shutDown := make(chan struct{})
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
rw.Shutdown(t.Context())
|
|
||||||
close(shutDown)
|
|
||||||
}()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-shutDown:
|
|
||||||
case <-time.After(stepTimeout):
|
|
||||||
t.Fatalf("Shutdown did not return within %s", stepTimeout)
|
|
||||||
}
|
|
||||||
|
|
||||||
store.mu.Lock()
|
|
||||||
defer store.mu.Unlock()
|
|
||||||
|
|
||||||
for _, prefix := range prefixes {
|
|
||||||
_, inPrefixTables := store.Prefixes[prefix]
|
|
||||||
if !inPrefixTables || !store.TablePrefixes[prefix] {
|
|
||||||
t.Errorf("%s was queued in the streamer when Shutdown started; "+
|
|
||||||
"written to the prefix tables %t, in the routing table's "+
|
|
||||||
"writes %t", prefix, inPrefixTables, store.TablePrefixes[prefix])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// freePort returns a TCP port that was free a moment ago, for a server the
|
|
||||||
// test starts.
|
|
||||||
func freePort(t *testing.T) int {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var listenConfig net.ListenConfig
|
|
||||||
|
|
||||||
listener, err := listenConfig.Listen(t.Context(), "tcp", ":0")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to find a free port: %v", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = listener.Close() }()
|
|
||||||
|
|
||||||
addr, ok := listener.Addr().(*net.TCPAddr)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("listener address %v is not a TCP address", listener.Addr())
|
|
||||||
}
|
|
||||||
|
|
||||||
return addr.Port
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestShutdownDoesNotWaitForABlockedWrite starts the HTTP server and has the
|
|
||||||
// AS handler start a write that the store holds until the test ends, past
|
|
||||||
// Shutdown's deadline. Shutdown must return without waiting for the write, with
|
|
||||||
// the HTTP server stopped, and must neither checkpoint nor close the database:
|
|
||||||
// the real one would wait for the write.
|
|
||||||
// Before the fix Shutdown waited for the write.
|
|
||||||
func TestShutdownDoesNotWaitForABlockedWrite(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const deadline = 100 * time.Millisecond
|
|
||||||
|
|
||||||
blocking := newBlockingStore()
|
|
||||||
defer blocking.releaseWrites()
|
|
||||||
|
|
||||||
store := &closeOrderStore{Store: blocking}
|
|
||||||
log := logger.New()
|
|
||||||
port := freePort(t)
|
|
||||||
cfg := &config.Config{Port: port}
|
|
||||||
s := streamer.New(log, metrics.New())
|
|
||||||
|
|
||||||
// A started server reads its routing table in the background, so it
|
|
||||||
// needs one.
|
|
||||||
rw := New(Dependencies{
|
|
||||||
DB: store,
|
|
||||||
Streamer: s,
|
|
||||||
Server: server.New(store, routetable.New(routetable.Params{}), s, log,
|
|
||||||
cfg),
|
|
||||||
Logger: log,
|
|
||||||
Config: cfg,
|
|
||||||
})
|
|
||||||
|
|
||||||
err := rw.server.Start(t.Context())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to start the HTTP server: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
rw.dbHandler = NewASHandler(store, log)
|
|
||||||
fill := ristypes.ASPath{Sequence: slices.Repeat([]int{testASNA}, asnBatchSize)}
|
|
||||||
startBlockedWrite(t, blocking, rw.dbHandler,
|
|
||||||
[]*ristypes.RISMessage{{Path: fill}})
|
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(t.Context(), deadline)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
shutDown := make(chan struct{})
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
rw.Shutdown(ctx)
|
|
||||||
close(shutDown)
|
|
||||||
}()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-shutDown:
|
|
||||||
case <-time.After(deadline + stepTimeout):
|
|
||||||
t.Fatalf("Shutdown did not return within %s of its deadline while "+
|
|
||||||
"a write was blocked", stepTimeout)
|
|
||||||
}
|
|
||||||
|
|
||||||
var dialer net.Dialer
|
|
||||||
|
|
||||||
addr := net.JoinHostPort("127.0.0.1", strconv.Itoa(port))
|
|
||||||
|
|
||||||
conn, err := dialer.DialContext(t.Context(), "tcp", addr)
|
|
||||||
if err == nil {
|
|
||||||
_ = conn.Close()
|
|
||||||
|
|
||||||
t.Errorf("the HTTP server still accepts connections on %s after "+
|
|
||||||
"Shutdown returned", addr)
|
|
||||||
}
|
|
||||||
|
|
||||||
store.callsMu.Lock()
|
|
||||||
defer store.callsMu.Unlock()
|
|
||||||
|
|
||||||
if slices.Contains(store.calls, callCheckpoint) ||
|
|
||||||
slices.Contains(store.calls, callClose) {
|
|
||||||
t.Errorf("Shutdown made the calls %q past its deadline, want no %s "+
|
|
||||||
"and no %s", store.calls, callCheckpoint, callClose)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// closeOrderStore is a store that records, in order, the AS writes,
|
|
||||||
// checkpoints and closes it is given. It passes the AS writes on to Store.
|
|
||||||
type closeOrderStore struct {
|
|
||||||
database.Store
|
|
||||||
|
|
||||||
callsMu sync.Mutex
|
|
||||||
calls []string
|
|
||||||
}
|
|
||||||
|
|
||||||
// The calls closeOrderStore records.
|
|
||||||
const (
|
|
||||||
callWrite = "AS write"
|
|
||||||
callCheckpoint = "checkpoint"
|
|
||||||
callClose = "close"
|
|
||||||
)
|
|
||||||
|
|
||||||
func (s *closeOrderStore) GetOrCreateASNBatch(
|
|
||||||
asns map[int]database.SeenTimes,
|
|
||||||
) error {
|
|
||||||
s.record(callWrite)
|
|
||||||
|
|
||||||
return s.Store.GetOrCreateASNBatch(asns)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *closeOrderStore) Checkpoint(_ context.Context) error {
|
|
||||||
s.record(callCheckpoint)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *closeOrderStore) Close() error {
|
|
||||||
s.record(callClose)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *closeOrderStore) record(call string) {
|
|
||||||
s.callsMu.Lock()
|
|
||||||
defer s.callsMu.Unlock()
|
|
||||||
|
|
||||||
s.calls = append(s.calls, call)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestShutdownCheckpointsAndClosesTheDatabaseLast hands the AS handler one
|
|
||||||
// message and shuts down with time to spare. The handler's last write must
|
|
||||||
// come first, then a checkpoint, then the close. Before the fix Shutdown
|
|
||||||
// neither checkpointed nor closed the database.
|
|
||||||
func TestShutdownCheckpointsAndClosesTheDatabaseLast(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
store := &closeOrderStore{Store: newMockStore()}
|
|
||||||
log := logger.New()
|
|
||||||
cfg := &config.Config{}
|
|
||||||
s := streamer.New(log, metrics.New())
|
|
||||||
|
|
||||||
rw := New(Dependencies{
|
|
||||||
DB: store,
|
|
||||||
Streamer: s,
|
|
||||||
Server: server.New(store, nil, s, log, cfg),
|
|
||||||
Logger: log,
|
|
||||||
Config: cfg,
|
|
||||||
})
|
|
||||||
rw.dbHandler = NewASHandler(store, log)
|
|
||||||
rw.dbHandler.HandleMessage(&ristypes.RISMessage{
|
|
||||||
Path: ristypes.ASPath{Sequence: []int{testASNA}},
|
|
||||||
})
|
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(t.Context(), stepTimeout)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
rw.Shutdown(ctx)
|
|
||||||
|
|
||||||
store.callsMu.Lock()
|
|
||||||
defer store.callsMu.Unlock()
|
|
||||||
|
|
||||||
want := []string{callWrite, callCheckpoint, callClose}
|
|
||||||
if !slices.Equal(store.calls, want) {
|
|
||||||
t.Errorf("Shutdown made the calls %q, want %q", store.calls, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,481 @@
|
|||||||
|
package routewatch
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.eeqj.de/sneak/routewatch/internal/config"
|
||||||
|
"git.eeqj.de/sneak/routewatch/internal/database"
|
||||||
|
"git.eeqj.de/sneak/routewatch/internal/logger"
|
||||||
|
"git.eeqj.de/sneak/routewatch/internal/metrics"
|
||||||
|
"git.eeqj.de/sneak/routewatch/internal/server"
|
||||||
|
"git.eeqj.de/sneak/routewatch/internal/streamer"
|
||||||
|
"github.com/google/uuid"
|
||||||
|
)
|
||||||
|
|
||||||
|
// mockStore is a mock implementation of database.Store for testing
|
||||||
|
type mockStore struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
|
||||||
|
// Counters for tracking calls
|
||||||
|
ASNCount int
|
||||||
|
PrefixCount int
|
||||||
|
PeeringCount int
|
||||||
|
RouteCount int
|
||||||
|
WithdrawalCount int
|
||||||
|
|
||||||
|
// Track unique items
|
||||||
|
ASNs map[int]*database.ASN
|
||||||
|
Prefixes map[string]*database.Prefix
|
||||||
|
Peerings map[string]bool // key is "from_to"
|
||||||
|
Routes map[string]bool // key is "prefix_origin_peer"
|
||||||
|
|
||||||
|
// Track IP versions
|
||||||
|
IPv4Prefixes int
|
||||||
|
IPv6Prefixes int
|
||||||
|
}
|
||||||
|
|
||||||
|
// newMockStore creates a new mock store
|
||||||
|
func newMockStore() *mockStore {
|
||||||
|
return &mockStore{
|
||||||
|
ASNs: make(map[int]*database.ASN),
|
||||||
|
Prefixes: make(map[string]*database.Prefix),
|
||||||
|
Peerings: make(map[string]bool),
|
||||||
|
Routes: make(map[string]bool),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetOrCreateASN mock implementation
|
||||||
|
func (m *mockStore) GetOrCreateASN(number int, timestamp time.Time) (*database.ASN, error) {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
|
||||||
|
if asn, exists := m.ASNs[number]; exists {
|
||||||
|
asn.LastSeen = timestamp
|
||||||
|
|
||||||
|
return asn, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
asn := &database.ASN{
|
||||||
|
ASN: number,
|
||||||
|
FirstSeen: timestamp,
|
||||||
|
LastSeen: timestamp,
|
||||||
|
}
|
||||||
|
m.ASNs[number] = asn
|
||||||
|
m.ASNCount++
|
||||||
|
|
||||||
|
return asn, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpdatePrefixesBatch mock implementation
|
||||||
|
func (m *mockStore) UpdatePrefixesBatch(prefixes map[string]time.Time) error {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
|
||||||
|
for prefix, timestamp := range prefixes {
|
||||||
|
if p, exists := m.Prefixes[prefix]; exists {
|
||||||
|
p.LastSeen = timestamp
|
||||||
|
} else {
|
||||||
|
const (
|
||||||
|
ipVersionV4 = 4
|
||||||
|
ipVersionV6 = 6
|
||||||
|
)
|
||||||
|
|
||||||
|
ipVersion := ipVersionV4
|
||||||
|
if strings.Contains(prefix, ":") {
|
||||||
|
ipVersion = ipVersionV6
|
||||||
|
}
|
||||||
|
|
||||||
|
m.Prefixes[prefix] = &database.Prefix{
|
||||||
|
ID: uuid.New(),
|
||||||
|
Prefix: prefix,
|
||||||
|
IPVersion: ipVersion,
|
||||||
|
FirstSeen: timestamp,
|
||||||
|
LastSeen: timestamp,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetOrCreatePrefix mock implementation
|
||||||
|
func (m *mockStore) GetOrCreatePrefix(prefix string, timestamp time.Time) (*database.Prefix, error) {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
|
||||||
|
if p, exists := m.Prefixes[prefix]; exists {
|
||||||
|
p.LastSeen = timestamp
|
||||||
|
|
||||||
|
return p, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
ipVersionV4 = 4
|
||||||
|
ipVersionV6 = 6
|
||||||
|
)
|
||||||
|
|
||||||
|
ipVersion := ipVersionV4
|
||||||
|
if strings.Contains(prefix, ":") {
|
||||||
|
ipVersion = ipVersionV6
|
||||||
|
}
|
||||||
|
|
||||||
|
p := &database.Prefix{
|
||||||
|
ID: uuid.New(),
|
||||||
|
Prefix: prefix,
|
||||||
|
IPVersion: ipVersion,
|
||||||
|
FirstSeen: timestamp,
|
||||||
|
LastSeen: timestamp,
|
||||||
|
}
|
||||||
|
m.Prefixes[prefix] = p
|
||||||
|
m.PrefixCount++
|
||||||
|
|
||||||
|
if ipVersion == ipVersionV4 {
|
||||||
|
m.IPv4Prefixes++
|
||||||
|
} else {
|
||||||
|
m.IPv6Prefixes++
|
||||||
|
}
|
||||||
|
|
||||||
|
return p, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordAnnouncement mock implementation
|
||||||
|
func (m *mockStore) RecordAnnouncement(_ *database.Announcement) error {
|
||||||
|
// Not tracking announcements in detail for now
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordPeering mock implementation
|
||||||
|
func (m *mockStore) RecordPeering(asA, asB int, _ time.Time) error {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
|
||||||
|
// Normalize
|
||||||
|
if asA > asB {
|
||||||
|
asA, asB = asB, asA
|
||||||
|
}
|
||||||
|
|
||||||
|
key := fmt.Sprintf("%d_%d", asA, asB)
|
||||||
|
if !m.Peerings[key] {
|
||||||
|
m.Peerings[key] = true
|
||||||
|
m.PeeringCount++
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpdatePeer mock implementation
|
||||||
|
func (m *mockStore) UpdatePeer(peerIP string, peerASN int, messageType string, timestamp time.Time) error {
|
||||||
|
// Simple mock - just return nil
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close mock implementation
|
||||||
|
func (m *mockStore) Close() error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetStats returns statistics about the mock store
|
||||||
|
func (m *mockStore) GetStats() (database.Stats, error) {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
|
||||||
|
return database.Stats{
|
||||||
|
ASNs: len(m.ASNs),
|
||||||
|
Prefixes: len(m.Prefixes),
|
||||||
|
IPv4Prefixes: m.IPv4Prefixes,
|
||||||
|
IPv6Prefixes: m.IPv6Prefixes,
|
||||||
|
Peerings: m.PeeringCount,
|
||||||
|
Peers: 10, // Mock peer count
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetStatsContext returns statistics about the mock store with context support
|
||||||
|
func (m *mockStore) GetStatsContext(ctx context.Context) (database.Stats, error) {
|
||||||
|
return m.GetStats()
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpsertLiveRoute mock implementation
|
||||||
|
func (m *mockStore) UpsertLiveRoute(route *database.LiveRoute) error {
|
||||||
|
// Simple mock - just return nil
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeleteLiveRoute mock implementation
|
||||||
|
func (m *mockStore) DeleteLiveRoute(prefix string, originASN int, peerIP string) error {
|
||||||
|
// Simple mock - just return nil
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetPrefixDistribution mock implementation
|
||||||
|
func (m *mockStore) GetPrefixDistribution() (ipv4 []database.PrefixDistribution, ipv6 []database.PrefixDistribution, err error) {
|
||||||
|
// Return empty distributions for now
|
||||||
|
return nil, nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetPrefixDistributionContext mock implementation with context support
|
||||||
|
func (m *mockStore) GetPrefixDistributionContext(ctx context.Context) (ipv4 []database.PrefixDistribution, ipv6 []database.PrefixDistribution, err error) {
|
||||||
|
return m.GetPrefixDistribution()
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetLiveRouteCounts mock implementation
|
||||||
|
func (m *mockStore) GetLiveRouteCounts() (ipv4Count, ipv6Count int, err error) {
|
||||||
|
// Return mock counts
|
||||||
|
return m.RouteCount / 2, m.RouteCount / 2, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetLiveRouteCountsContext mock implementation with context support
|
||||||
|
func (m *mockStore) GetLiveRouteCountsContext(ctx context.Context) (ipv4Count, ipv6Count int, err error) {
|
||||||
|
return m.GetLiveRouteCounts()
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetASDetails mock implementation
|
||||||
|
func (m *mockStore) GetASDetails(asn int) (*database.ASN, []database.LiveRoute, error) {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
|
||||||
|
// Check if ASN exists
|
||||||
|
if asnInfo, exists := m.ASNs[asn]; exists {
|
||||||
|
// Return empty prefixes for now
|
||||||
|
return asnInfo, []database.LiveRoute{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil, nil, database.ErrNoRoute
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetASDetailsContext mock implementation with context support
|
||||||
|
func (m *mockStore) GetASDetailsContext(ctx context.Context, asn int) (*database.ASN, []database.LiveRoute, error) {
|
||||||
|
return m.GetASDetails(asn)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetPrefixDetails mock implementation
|
||||||
|
func (m *mockStore) GetPrefixDetails(prefix string) ([]database.LiveRoute, error) {
|
||||||
|
// Return empty routes for now
|
||||||
|
return []database.LiveRoute{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetPrefixDetailsContext mock implementation with context support
|
||||||
|
func (m *mockStore) GetPrefixDetailsContext(ctx context.Context, prefix string) ([]database.LiveRoute, error) {
|
||||||
|
return m.GetPrefixDetails(prefix)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *mockStore) GetRandomPrefixesByLength(maskLength, ipVersion, limit int) ([]database.LiveRoute, error) {
|
||||||
|
// Return empty routes for now
|
||||||
|
return []database.LiveRoute{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetRandomPrefixesByLengthContext mock implementation with context support
|
||||||
|
func (m *mockStore) GetRandomPrefixesByLengthContext(ctx context.Context, maskLength, ipVersion, limit int) ([]database.LiveRoute, error) {
|
||||||
|
return m.GetRandomPrefixesByLength(maskLength, ipVersion, limit)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetASPeers mock implementation
|
||||||
|
func (m *mockStore) GetASPeers(asn int) ([]database.ASPeer, error) {
|
||||||
|
// Return empty peers for now
|
||||||
|
return []database.ASPeer{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetASPeersContext mock implementation with context support
|
||||||
|
func (m *mockStore) GetASPeersContext(ctx context.Context, asn int) ([]database.ASPeer, error) {
|
||||||
|
return m.GetASPeers(asn)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetIPInfo mock implementation
|
||||||
|
func (m *mockStore) GetIPInfo(ip string) (*database.IPInfo, error) {
|
||||||
|
return m.GetIPInfoContext(context.Background(), ip)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetIPInfoContext mock implementation with context support
|
||||||
|
func (m *mockStore) GetIPInfoContext(ctx context.Context, ip string) (*database.IPInfo, error) {
|
||||||
|
now := time.Now()
|
||||||
|
return &database.IPInfo{
|
||||||
|
IP: ip,
|
||||||
|
Netblock: "8.8.8.0/24",
|
||||||
|
MaskLength: 24,
|
||||||
|
IPVersion: 4,
|
||||||
|
NumPeers: 3,
|
||||||
|
ASN: 15169,
|
||||||
|
Handle: "GOOGLE",
|
||||||
|
Description: "Google LLC",
|
||||||
|
CountryCode: "US",
|
||||||
|
FirstSeen: now.Add(-24 * time.Hour),
|
||||||
|
LastSeen: now,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetNextStaleASN mock implementation
|
||||||
|
func (m *mockStore) GetNextStaleASN(ctx context.Context, staleThreshold time.Duration) (int, error) {
|
||||||
|
return 0, database.ErrNoStaleASN
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpdateASNWHOIS mock implementation
|
||||||
|
func (m *mockStore) UpdateASNWHOIS(ctx context.Context, update *database.ASNWHOISUpdate) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetWHOISStats mock implementation
|
||||||
|
func (m *mockStore) GetWHOISStats(ctx context.Context, staleThreshold time.Duration) (*database.WHOISStats, error) {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
|
||||||
|
return &database.WHOISStats{
|
||||||
|
TotalASNs: len(m.ASNs),
|
||||||
|
FreshASNs: 0,
|
||||||
|
StaleASNs: 0,
|
||||||
|
NeverFetched: len(m.ASNs),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpsertLiveRouteBatch mock implementation
|
||||||
|
func (m *mockStore) UpsertLiveRouteBatch(routes []*database.LiveRoute) error {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
|
||||||
|
for _, route := range routes {
|
||||||
|
// Track prefix
|
||||||
|
if _, exists := m.Prefixes[route.Prefix]; !exists {
|
||||||
|
m.Prefixes[route.Prefix] = &database.Prefix{
|
||||||
|
ID: uuid.New(),
|
||||||
|
Prefix: route.Prefix,
|
||||||
|
IPVersion: route.IPVersion,
|
||||||
|
FirstSeen: route.LastUpdated,
|
||||||
|
LastSeen: route.LastUpdated,
|
||||||
|
}
|
||||||
|
m.PrefixCount++
|
||||||
|
if route.IPVersion == 4 {
|
||||||
|
m.IPv4Prefixes++
|
||||||
|
} else {
|
||||||
|
m.IPv6Prefixes++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
m.RouteCount++
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeleteLiveRouteBatch mock implementation
|
||||||
|
func (m *mockStore) DeleteLiveRouteBatch(deletions []database.LiveRouteDeletion) error {
|
||||||
|
// Simple mock - just return nil
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetOrCreateASNBatch mock implementation
|
||||||
|
func (m *mockStore) GetOrCreateASNBatch(asns map[int]time.Time) error {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
|
||||||
|
for number, timestamp := range asns {
|
||||||
|
if _, exists := m.ASNs[number]; !exists {
|
||||||
|
m.ASNs[number] = &database.ASN{
|
||||||
|
ASN: number,
|
||||||
|
FirstSeen: timestamp,
|
||||||
|
LastSeen: timestamp,
|
||||||
|
}
|
||||||
|
m.ASNCount++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpdatePeerBatch mock implementation
|
||||||
|
func (m *mockStore) UpdatePeerBatch(peers map[string]database.PeerUpdate) error {
|
||||||
|
// Simple mock - just return nil
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Vacuum mock implementation
|
||||||
|
func (m *mockStore) Vacuum(ctx context.Context) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Analyze mock implementation
|
||||||
|
func (m *mockStore) Analyze(ctx context.Context) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Checkpoint mock implementation
|
||||||
|
func (m *mockStore) Checkpoint(ctx context.Context) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ping mock implementation
|
||||||
|
func (m *mockStore) Ping(ctx context.Context) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRouteWatchLiveFeed(t *testing.T) {
|
||||||
|
if testing.Short() {
|
||||||
|
t.Skip("skipping live RIPE RIS network feed test in short mode; run without -short to include it")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create mock database
|
||||||
|
mockDB := newMockStore()
|
||||||
|
defer mockDB.Close()
|
||||||
|
|
||||||
|
logger := logger.New()
|
||||||
|
|
||||||
|
// Create metrics tracker
|
||||||
|
metricsTracker := metrics.New()
|
||||||
|
|
||||||
|
// Create streamer
|
||||||
|
s := streamer.New(logger, metricsTracker)
|
||||||
|
|
||||||
|
// Create test config with empty state dir (no snapshot loading)
|
||||||
|
cfg := &config.Config{
|
||||||
|
StateDir: "",
|
||||||
|
MaxRuntime: 5 * time.Second,
|
||||||
|
EnableBatchedDatabaseWrites: true,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create server
|
||||||
|
srv := server.New(mockDB, s, logger, cfg)
|
||||||
|
|
||||||
|
// Create RouteWatch with 5 second limit
|
||||||
|
deps := Dependencies{
|
||||||
|
DB: mockDB,
|
||||||
|
Streamer: s,
|
||||||
|
Server: srv,
|
||||||
|
Logger: logger,
|
||||||
|
Config: cfg,
|
||||||
|
}
|
||||||
|
rw := New(deps)
|
||||||
|
|
||||||
|
// Run with context
|
||||||
|
ctx := context.Background()
|
||||||
|
go func() {
|
||||||
|
_ = rw.Run(ctx)
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Wait for the configured duration
|
||||||
|
time.Sleep(5 * time.Second)
|
||||||
|
|
||||||
|
// Force peering processing for test
|
||||||
|
if rw.peeringHandler != nil {
|
||||||
|
rw.peeringHandler.ProcessPeeringsNow()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get statistics
|
||||||
|
stats, err := mockDB.GetStats()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to get stats: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if stats.ASNs == 0 {
|
||||||
|
t.Error("Expected to receive some ASNs from live feed")
|
||||||
|
}
|
||||||
|
t.Logf("Received %d unique ASNs in 5 seconds", stats.ASNs)
|
||||||
|
|
||||||
|
if stats.Prefixes == 0 {
|
||||||
|
t.Error("Expected to receive some prefixes from live feed")
|
||||||
|
}
|
||||||
|
t.Logf("Received %d unique prefixes (%d IPv4, %d IPv6) in 5 seconds", stats.Prefixes, stats.IPv4Prefixes, stats.IPv6Prefixes)
|
||||||
|
|
||||||
|
if stats.Peerings == 0 {
|
||||||
|
t.Error("Expected to receive some peerings from live feed")
|
||||||
|
}
|
||||||
|
t.Logf("Recorded %d AS peering relationships in 5 seconds", stats.Peerings)
|
||||||
|
|
||||||
|
}
|
||||||
@@ -1,3 +1,3 @@
|
|||||||
package routewatch_test
|
package routewatch
|
||||||
|
|
||||||
// Tests for routewatch package are in app_integration_internal_test.go
|
// Tests for routewatch package are in app_integration_test.go
|
||||||
|
|||||||
@@ -1,8 +1,6 @@
|
|||||||
package routewatch
|
package routewatch
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"slices"
|
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -21,12 +19,6 @@ const (
|
|||||||
// asnBatchSize is the number of ASN operations to batch together
|
// asnBatchSize is the number of ASN operations to batch together
|
||||||
asnBatchSize = 30000
|
asnBatchSize = 30000
|
||||||
|
|
||||||
// asnBatchMaxSize is the most ASN operations the batch holds while the
|
|
||||||
// flush goroutine writes the previous one: ten batches, about 10 MiB
|
|
||||||
// (README "Memory"). At this size HandleMessage waits until the flush
|
|
||||||
// goroutine takes the batch.
|
|
||||||
asnBatchMaxSize = 300000
|
|
||||||
|
|
||||||
// asnBatchTimeout is the maximum time to wait before flushing a batch
|
// asnBatchTimeout is the maximum time to wait before flushing a batch
|
||||||
// DO NOT reduce this timeout - larger batches are more efficient
|
// DO NOT reduce this timeout - larger batches are more efficient
|
||||||
asnBatchTimeout = 2 * time.Second
|
asnBatchTimeout = 2 * time.Second
|
||||||
@@ -40,26 +32,12 @@ type ASHandler struct {
|
|||||||
db database.Store
|
db database.Store
|
||||||
logger *logger.Logger
|
logger *logger.Logger
|
||||||
|
|
||||||
// batchMaxSize is asnBatchMaxSize. Only HandleMessage reads it, so a test
|
// Batching
|
||||||
// may lower it before it hands the handler a message.
|
|
||||||
batchMaxSize int
|
|
||||||
|
|
||||||
// Batching. mu guards batch and writing only: the flush goroutine takes
|
|
||||||
// the batch under mu and writes it after releasing mu, so HandleMessage
|
|
||||||
// waits for a database write only while batch is at batchMaxSize.
|
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
batch []asnOp
|
batch []asnOp
|
||||||
// writing is how many operations the flush goroutine took from batch
|
lastFlush time.Time
|
||||||
// and has not finished writing.
|
stopCh chan struct{}
|
||||||
writing int
|
wg sync.WaitGroup
|
||||||
// batchTaken is signalled when the flush goroutine takes batch; its L is
|
|
||||||
// &mu.
|
|
||||||
batchTaken sync.Cond
|
|
||||||
batchFullCh chan struct{} // wakes the flush goroutine when batch is full
|
|
||||||
// stopCh hands Stop's context to the flush goroutine. It holds one, so
|
|
||||||
// Stop does not wait for a write in progress to hand it over.
|
|
||||||
stopCh chan context.Context
|
|
||||||
done chan struct{} // closed when the flush goroutine returns
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type asnOp struct {
|
type asnOp struct {
|
||||||
@@ -73,28 +51,17 @@ type asnOp struct {
|
|||||||
// Stop when finished to ensure all pending operations are flushed and the
|
// Stop when finished to ensure all pending operations are flushed and the
|
||||||
// background goroutine is terminated.
|
// background goroutine is terminated.
|
||||||
func NewASHandler(db database.Store, logger *logger.Logger) *ASHandler {
|
func NewASHandler(db database.Store, logger *logger.Logger) *ASHandler {
|
||||||
return newASHandler(db, logger, asnBatchTimeout)
|
|
||||||
}
|
|
||||||
|
|
||||||
// newASHandler is NewASHandler with the flush timer's interval, batchTimeout,
|
|
||||||
// in place of asnBatchTimeout. A test that the timer must not disturb gives an
|
|
||||||
// interval longer than the test runs.
|
|
||||||
func newASHandler(
|
|
||||||
db database.Store, logger *logger.Logger, batchTimeout time.Duration,
|
|
||||||
) *ASHandler {
|
|
||||||
h := &ASHandler{
|
h := &ASHandler{
|
||||||
db: db,
|
db: db,
|
||||||
logger: logger,
|
logger: logger,
|
||||||
batchMaxSize: asnBatchMaxSize,
|
|
||||||
batch: make([]asnOp, 0, asnBatchSize),
|
batch: make([]asnOp, 0, asnBatchSize),
|
||||||
batchFullCh: make(chan struct{}, 1),
|
lastFlush: time.Now(),
|
||||||
stopCh: make(chan context.Context, 1),
|
stopCh: make(chan struct{}),
|
||||||
done: make(chan struct{}),
|
|
||||||
}
|
}
|
||||||
h.batchTaken.L = &h.mu
|
|
||||||
|
|
||||||
// Start the flush timer goroutine
|
// Start the flush timer goroutine
|
||||||
go flushLoop(batchTimeout, h.batchFullCh, h.stopCh, h.done, h.flushBatch)
|
h.wg.Add(1)
|
||||||
|
go h.flushLoop()
|
||||||
|
|
||||||
return h
|
return h
|
||||||
}
|
}
|
||||||
@@ -104,7 +71,7 @@ func newASHandler(
|
|||||||
// the AS path information needed to track autonomous systems.
|
// the AS path information needed to track autonomous systems.
|
||||||
func (h *ASHandler) WantsMessage(messageType string) bool {
|
func (h *ASHandler) WantsMessage(messageType string) bool {
|
||||||
// We only care about UPDATE messages for the database
|
// We only care about UPDATE messages for the database
|
||||||
return messageType == ristypes.MessageTypeUpdate
|
return messageType == "UPDATE"
|
||||||
}
|
}
|
||||||
|
|
||||||
// QueueCapacity returns the recommended message queue size for this handler.
|
// QueueCapacity returns the recommended message queue size for this handler.
|
||||||
@@ -117,25 +84,22 @@ func (h *ASHandler) QueueCapacity() int {
|
|||||||
|
|
||||||
// HandleMessage processes a RIS Live BGP message by extracting all ASNs from
|
// HandleMessage processes a RIS Live BGP message by extracting all ASNs from
|
||||||
// the AS path and queuing them for batch insertion into the database. The
|
// the AS path and queuing them for batch insertion into the database. The
|
||||||
// origin ASN and all transit ASNs are recorded with their associated
|
// origin ASN (last element in the path) and all transit ASNs are recorded
|
||||||
// timestamps. When the batch reaches the configured size threshold, the flush
|
// with their associated timestamps. The batch is automatically flushed when
|
||||||
// goroutine is woken to write it.
|
// it reaches the configured size threshold.
|
||||||
func (h *ASHandler) HandleMessage(msg *ristypes.RISMessage) {
|
func (h *ASHandler) HandleMessage(msg *ristypes.RISMessage) {
|
||||||
// Use the pre-parsed timestamp
|
// Use the pre-parsed timestamp
|
||||||
timestamp := msg.ParsedTimestamp
|
timestamp := msg.ParsedTimestamp
|
||||||
|
|
||||||
originASN := msg.Path.Origin()
|
// Get origin ASN from path (last element)
|
||||||
|
var originASN int
|
||||||
|
if len(msg.Path) > 0 {
|
||||||
|
originASN = msg.Path[len(msg.Path)-1]
|
||||||
|
}
|
||||||
|
|
||||||
h.mu.Lock()
|
h.mu.Lock()
|
||||||
defer h.mu.Unlock()
|
defer h.mu.Unlock()
|
||||||
|
|
||||||
// While the batch is at its maximum size, wait until the flush goroutine
|
|
||||||
// takes it. Meanwhile the handler takes nothing off its queue, which
|
|
||||||
// fills, and the streamer drops messages for it.
|
|
||||||
for len(h.batch) >= h.batchMaxSize {
|
|
||||||
h.batchTaken.Wait()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Queue origin ASN operation
|
// Queue origin ASN operation
|
||||||
if originASN > 0 {
|
if originASN > 0 {
|
||||||
h.batch = append(h.batch, asnOp{
|
h.batch = append(h.batch, asnOp{
|
||||||
@@ -144,8 +108,8 @@ func (h *ASHandler) HandleMessage(msg *ristypes.RISMessage) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// Also track all ASNs in the path, the members of an AS set included
|
// Also track all ASNs in the path
|
||||||
for _, asn := range slices.Concat(msg.Path.Sequence, msg.Path.Set) {
|
for _, asn := range msg.Path {
|
||||||
if asn > 0 {
|
if asn > 0 {
|
||||||
h.batch = append(h.batch, asnOp{
|
h.batch = append(h.batch, asnOp{
|
||||||
number: asn,
|
number: asn,
|
||||||
@@ -154,84 +118,67 @@ func (h *ASHandler) HandleMessage(msg *ristypes.RISMessage) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// A full batch wakes the flush goroutine. If it is writing, the wake-up
|
// Check if we need to flush
|
||||||
// waits in batchFullCh and the batch keeps growing until the write ends
|
|
||||||
// or the batch reaches batchMaxSize.
|
|
||||||
if len(h.batch) >= asnBatchSize {
|
if len(h.batch) >= asnBatchSize {
|
||||||
select {
|
h.flushBatchLocked()
|
||||||
case h.batchFullCh <- struct{}{}:
|
|
||||||
default: // already woken
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stop stops the flush goroutine, which writes the operations the handler holds
|
// flushLoop runs in a goroutine and periodically flushes batches
|
||||||
// before it returns, and waits for it until ctx ends. Once ctx has ended the
|
func (h *ASHandler) flushLoop() {
|
||||||
// flush goroutine skips that last write, and a write still running is lost
|
defer h.wg.Done()
|
||||||
// when the process exits, so Stop logs how many operations are not written.
|
ticker := time.NewTicker(asnBatchTimeout)
|
||||||
func (h *ASHandler) Stop(ctx context.Context) {
|
defer ticker.Stop()
|
||||||
h.stopCh <- ctx
|
|
||||||
|
|
||||||
|
for {
|
||||||
select {
|
select {
|
||||||
case <-h.done:
|
case <-ticker.C:
|
||||||
case <-ctx.Done():
|
|
||||||
}
|
|
||||||
|
|
||||||
h.mu.Lock()
|
h.mu.Lock()
|
||||||
unwritten := h.writing + len(h.batch)
|
if time.Since(h.lastFlush) >= asnBatchTimeout {
|
||||||
|
h.flushBatchLocked()
|
||||||
|
}
|
||||||
|
h.mu.Unlock()
|
||||||
|
case <-h.stopCh:
|
||||||
|
// Final flush
|
||||||
|
h.mu.Lock()
|
||||||
|
h.flushBatchLocked()
|
||||||
h.mu.Unlock()
|
h.mu.Unlock()
|
||||||
|
|
||||||
if unwritten > 0 {
|
return
|
||||||
h.logger.Error("Stop deadline passed before the AS handler wrote "+
|
}
|
||||||
"its updates", "updates", unwritten)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// takeBatch returns the pending operations and leaves an empty batch in their
|
// flushBatchLocked flushes the ASN batch to the database (must be called with mutex held)
|
||||||
// place for HandleMessage to fill. They count in writing until flushBatch has
|
func (h *ASHandler) flushBatchLocked() {
|
||||||
// written them.
|
|
||||||
func (h *ASHandler) takeBatch() []asnOp {
|
|
||||||
h.mu.Lock()
|
|
||||||
defer h.mu.Unlock()
|
|
||||||
|
|
||||||
if len(h.batch) == 0 {
|
if len(h.batch) == 0 {
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
batch := h.batch
|
|
||||||
h.writing = len(batch)
|
|
||||||
h.batch = make([]asnOp, 0, asnBatchSize)
|
|
||||||
h.batchTaken.Broadcast()
|
|
||||||
|
|
||||||
return batch
|
|
||||||
}
|
|
||||||
|
|
||||||
// flushBatch takes the pending batch and writes it to the database, without
|
|
||||||
// holding the mutex during the write.
|
|
||||||
func (h *ASHandler) flushBatch() {
|
|
||||||
batch := h.takeBatch()
|
|
||||||
if len(batch) == 0 {
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// The earliest and the latest time the batch saw each AS. The batch holds
|
// Process ASNs first (deduped)
|
||||||
// several updates for an AS, from many peers, so a new AS is stored first
|
asnMap := make(map[int]time.Time)
|
||||||
// seen at the earliest of them and last seen at the latest.
|
for _, op := range h.batch {
|
||||||
asns := make(map[int]database.SeenTimes)
|
if existing, ok := asnMap[op.number]; !ok || op.timestamp.After(existing) {
|
||||||
for _, op := range batch {
|
asnMap[op.number] = op.timestamp
|
||||||
seen := asns[op.number]
|
}
|
||||||
seen.Add(op.timestamp)
|
|
||||||
asns[op.number] = seen
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Process all ASNs in a single batch transaction
|
// Process all ASNs in a single batch transaction
|
||||||
err := h.db.GetOrCreateASNBatch(asns)
|
if err := h.db.GetOrCreateASNBatch(asnMap); err != nil {
|
||||||
if err != nil {
|
h.logger.Error("Failed to process ASN batch", "error", err, "count", len(asnMap))
|
||||||
h.logger.Error("Failed to process ASN batch",
|
|
||||||
"error", err, "count", len(asns))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
h.mu.Lock()
|
// Clear batch
|
||||||
h.writing = 0
|
h.batch = h.batch[:0]
|
||||||
h.mu.Unlock()
|
h.lastFlush = time.Now()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop gracefully shuts down the ASHandler by signaling the background flush
|
||||||
|
// goroutine to terminate and waiting for it to complete. Any pending ASN
|
||||||
|
// operations in the current batch are flushed to the database before Stop
|
||||||
|
// returns. This method should be called during application shutdown to ensure
|
||||||
|
// no data is lost.
|
||||||
|
func (h *ASHandler) Stop() {
|
||||||
|
close(h.stopCh)
|
||||||
|
h.wg.Wait()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,68 +0,0 @@
|
|||||||
package routewatch
|
|
||||||
|
|
||||||
import (
|
|
||||||
"maps"
|
|
||||||
"slices"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestASHandlerStoresTheEarliestAndLatestTimesOfANewAS handles two updates
|
|
||||||
// whose path carries an AS the database does not hold yet, the second 5
|
|
||||||
// seconds after the first, in one batch into a real database. The AS is stored
|
|
||||||
// first seen at the time of the first update and last seen at the time of the
|
|
||||||
// second.
|
|
||||||
func TestASHandlerStoresTheEarliestAndLatestTimesOfANewAS(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := newTestDatabase(t)
|
|
||||||
|
|
||||||
// Built without NewASHandler's flush goroutine, so the test flushes itself.
|
|
||||||
h := &ASHandler{db: db, logger: logger.New(), batchMaxSize: asnBatchMaxSize}
|
|
||||||
|
|
||||||
first := peerAnnouncement(testPeerIP, testV4Prefix)
|
|
||||||
second := peerAnnouncement(testPeerIP, testV4Prefix)
|
|
||||||
second.ParsedTimestamp = first.ParsedTimestamp.Add(secondUpdateAfter)
|
|
||||||
|
|
||||||
h.HandleMessage(first)
|
|
||||||
h.HandleMessage(second)
|
|
||||||
h.flushBatch()
|
|
||||||
|
|
||||||
asn, _, err := db.GetAS(t.Context(), testASNA)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GetAS(%d): %v", testASNA, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !asn.FirstSeen.Equal(first.ParsedTimestamp) {
|
|
||||||
t.Errorf("first_seen of AS%d = %s, want %s", testASNA,
|
|
||||||
asn.FirstSeen, first.ParsedTimestamp)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !asn.LastSeen.Equal(second.ParsedTimestamp) {
|
|
||||||
t.Errorf("last_seen of AS%d = %s, want %s", testASNA,
|
|
||||||
asn.LastSeen, second.ParsedTimestamp)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestASHandlerRecordsNothingFromAPathWithASZero handles an update whose AS
|
|
||||||
// path holds AS 0 between two other ASes. RFC 7607 makes such a path
|
|
||||||
// malformed, so neither AS is recorded.
|
|
||||||
func TestASHandlerRecordsNothingFromAPathWithASZero(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
store := newMockStore()
|
|
||||||
|
|
||||||
// Built without NewASHandler's flush goroutine, so the test flushes itself.
|
|
||||||
h := &ASHandler{db: store, logger: logger.New(), batchMaxSize: asnBatchMaxSize}
|
|
||||||
|
|
||||||
msg := peerAnnouncement(testPeerIP, testV4Prefix)
|
|
||||||
msg.Path = decodePath(t, testASNA, 0, testASNB)
|
|
||||||
h.HandleMessage(msg)
|
|
||||||
h.flushBatch()
|
|
||||||
|
|
||||||
if len(store.ASNs) > 0 {
|
|
||||||
t.Errorf("ASes recorded = %v, want none",
|
|
||||||
slices.Sorted(maps.Keys(store.ASNs)))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,8 +1,8 @@
|
|||||||
|
// Package routewatch contains the ASN WHOIS fetcher for background updates.
|
||||||
package routewatch
|
package routewatch
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
@@ -21,16 +21,10 @@ const (
|
|||||||
minInterval = 1 * time.Second
|
minInterval = 1 * time.Second
|
||||||
|
|
||||||
// maxInterval is the maximum interval after failures (backoff cap).
|
// maxInterval is the maximum interval after failures (backoff cap).
|
||||||
maxInterval = time.Minute
|
maxInterval = 5 * time.Minute
|
||||||
|
|
||||||
// A failure multiplies the interval by backoffNumerator divided by
|
// backoffMultiplier is how much to multiply interval on failure.
|
||||||
// backoffDenominator, 1.5, up to maxInterval.
|
backoffMultiplier = 2
|
||||||
backoffNumerator = 3
|
|
||||||
backoffDenominator = 2
|
|
||||||
|
|
||||||
// A success divides the interval by successDivisor, down to
|
|
||||||
// minInterval.
|
|
||||||
successDivisor = 2
|
|
||||||
|
|
||||||
// whoisStaleThreshold is how old WHOIS data can be before refresh.
|
// whoisStaleThreshold is how old WHOIS data can be before refresh.
|
||||||
whoisStaleThreshold = 30 * 24 * time.Hour // 30 days
|
whoisStaleThreshold = 30 * 24 * time.Hour // 30 days
|
||||||
@@ -49,7 +43,7 @@ type ASNFetcher struct {
|
|||||||
logger *slog.Logger
|
logger *slog.Logger
|
||||||
immediateQueue chan int
|
immediateQueue chan int
|
||||||
stopCh chan struct{}
|
stopCh chan struct{}
|
||||||
done chan struct{} // closed when the fetcher goroutine returns
|
wg sync.WaitGroup
|
||||||
|
|
||||||
// fetchMu ensures only one fetch runs at a time
|
// fetchMu ensures only one fetch runs at a time
|
||||||
fetchMu sync.Mutex
|
fetchMu sync.Mutex
|
||||||
@@ -59,8 +53,7 @@ type ASNFetcher struct {
|
|||||||
currentInterval time.Duration
|
currentInterval time.Duration
|
||||||
consecutiveFails int
|
consecutiveFails int
|
||||||
|
|
||||||
// hourly stats tracking; now is the clock, time.Now except in tests
|
// hourly stats tracking
|
||||||
now func() time.Time
|
|
||||||
statsMu sync.Mutex
|
statsMu sync.Mutex
|
||||||
successTimes []time.Time
|
successTimes []time.Time
|
||||||
errorTimes []time.Time
|
errorTimes []time.Time
|
||||||
@@ -68,27 +61,22 @@ type ASNFetcher struct {
|
|||||||
|
|
||||||
// NewASNFetcher creates a new ASN fetcher.
|
// NewASNFetcher creates a new ASN fetcher.
|
||||||
func NewASNFetcher(db database.Store, logger *slog.Logger) *ASNFetcher {
|
func NewASNFetcher(db database.Store, logger *slog.Logger) *ASNFetcher {
|
||||||
logger = logger.With("component", "asn_fetcher")
|
|
||||||
|
|
||||||
return &ASNFetcher{
|
return &ASNFetcher{
|
||||||
db: db,
|
db: db,
|
||||||
whoisClient: whois.NewClient(logger),
|
whoisClient: whois.NewClient(),
|
||||||
logger: logger,
|
logger: logger.With("component", "asn_fetcher"),
|
||||||
immediateQueue: make(chan int, immediateQueueSize),
|
immediateQueue: make(chan int, immediateQueueSize),
|
||||||
stopCh: make(chan struct{}),
|
stopCh: make(chan struct{}),
|
||||||
done: make(chan struct{}),
|
|
||||||
currentInterval: baseInterval,
|
currentInterval: baseInterval,
|
||||||
now: time.Now,
|
|
||||||
successTimes: make([]time.Time, 0),
|
successTimes: make([]time.Time, 0),
|
||||||
errorTimes: make([]time.Time, 0),
|
errorTimes: make([]time.Time, 0),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Start begins the background ASN fetcher goroutine. It runs until Stop:
|
// Start begins the background ASN fetcher goroutine.
|
||||||
// the end of ctx does not stop it.
|
func (f *ASNFetcher) Start() {
|
||||||
func (f *ASNFetcher) Start(ctx context.Context) {
|
f.wg.Add(1)
|
||||||
go f.run(context.WithoutCancel(ctx))
|
go f.run()
|
||||||
|
|
||||||
f.logger.Info("ASN fetcher started",
|
f.logger.Info("ASN fetcher started",
|
||||||
"base_interval", baseInterval,
|
"base_interval", baseInterval,
|
||||||
"min_interval", minInterval,
|
"min_interval", minInterval,
|
||||||
@@ -96,17 +84,11 @@ func (f *ASNFetcher) Start(ctx context.Context) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stop stops the fetcher and waits, until ctx ends, for a lookup in progress to
|
// Stop gracefully shuts down the fetcher.
|
||||||
// finish.
|
func (f *ASNFetcher) Stop() {
|
||||||
func (f *ASNFetcher) Stop(ctx context.Context) {
|
|
||||||
close(f.stopCh)
|
close(f.stopCh)
|
||||||
|
f.wg.Wait()
|
||||||
select {
|
|
||||||
case <-f.done:
|
|
||||||
f.logger.Info("ASN fetcher stopped")
|
f.logger.Info("ASN fetcher stopped")
|
||||||
case <-ctx.Done():
|
|
||||||
f.logger.Warn("Stop deadline passed before the ASN fetcher stopped")
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// QueueImmediate queues an ASN for immediate WHOIS lookup.
|
// QueueImmediate queues an ASN for immediate WHOIS lookup.
|
||||||
@@ -131,7 +113,7 @@ func (f *ASNFetcher) GetStats() server.ASNFetcherStats {
|
|||||||
f.intervalMu.Unlock()
|
f.intervalMu.Unlock()
|
||||||
|
|
||||||
// Prune old entries and count
|
// Prune old entries and count
|
||||||
cutoff := f.now().Add(-statsWindow)
|
cutoff := time.Now().Add(-statsWindow)
|
||||||
f.successTimes = pruneOldTimes(f.successTimes, cutoff)
|
f.successTimes = pruneOldTimes(f.successTimes, cutoff)
|
||||||
f.errorTimes = pruneOldTimes(f.errorTimes, cutoff)
|
f.errorTimes = pruneOldTimes(f.errorTimes, cutoff)
|
||||||
|
|
||||||
@@ -143,15 +125,16 @@ func (f *ASNFetcher) GetStats() server.ASNFetcherStats {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// pruneOldTimes drops the times at or before cutoff from the front of times,
|
// pruneOldTimes removes times older than cutoff and returns the pruned slice.
|
||||||
// which holds times in the order they were recorded, and returns the rest. It
|
|
||||||
// copies nothing, so it is cheap enough to run on every append.
|
|
||||||
func pruneOldTimes(times []time.Time, cutoff time.Time) []time.Time {
|
func pruneOldTimes(times []time.Time, cutoff time.Time) []time.Time {
|
||||||
for len(times) > 0 && !times[0].After(cutoff) {
|
result := make([]time.Time, 0, len(times))
|
||||||
times = times[1:]
|
for _, t := range times {
|
||||||
|
if t.After(cutoff) {
|
||||||
|
result = append(result, t)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return times
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
// getInterval returns the current fetch interval.
|
// getInterval returns the current fetch interval.
|
||||||
@@ -168,7 +151,10 @@ func (f *ASNFetcher) recordSuccess() {
|
|||||||
f.consecutiveFails = 0
|
f.consecutiveFails = 0
|
||||||
|
|
||||||
// Decrease interval by half, but not below minimum
|
// Decrease interval by half, but not below minimum
|
||||||
newInterval := max(f.currentInterval/successDivisor, minInterval)
|
newInterval := f.currentInterval / backoffMultiplier
|
||||||
|
if newInterval < minInterval {
|
||||||
|
newInterval = minInterval
|
||||||
|
}
|
||||||
|
|
||||||
if newInterval != f.currentInterval {
|
if newInterval != f.currentInterval {
|
||||||
f.logger.Debug("Decreased fetch interval",
|
f.logger.Debug("Decreased fetch interval",
|
||||||
@@ -179,11 +165,9 @@ func (f *ASNFetcher) recordSuccess() {
|
|||||||
}
|
}
|
||||||
f.intervalMu.Unlock()
|
f.intervalMu.Unlock()
|
||||||
|
|
||||||
// Record success time for stats, dropping those older than statsWindow
|
// Record success time for stats
|
||||||
f.statsMu.Lock()
|
f.statsMu.Lock()
|
||||||
now := f.now()
|
f.successTimes = append(f.successTimes, time.Now())
|
||||||
f.successTimes = pruneOldTimes(f.successTimes, now.Add(-statsWindow))
|
|
||||||
f.successTimes = append(f.successTimes, now)
|
|
||||||
f.statsMu.Unlock()
|
f.statsMu.Unlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -192,9 +176,11 @@ func (f *ASNFetcher) recordFailure() {
|
|||||||
f.intervalMu.Lock()
|
f.intervalMu.Lock()
|
||||||
f.consecutiveFails++
|
f.consecutiveFails++
|
||||||
|
|
||||||
// Exponential backoff: multiply by 1.5, capped at max
|
// Exponential backoff: multiply by 2, capped at max
|
||||||
newInterval := min(
|
newInterval := f.currentInterval * backoffMultiplier
|
||||||
f.currentInterval*backoffNumerator/backoffDenominator, maxInterval)
|
if newInterval > maxInterval {
|
||||||
|
newInterval = maxInterval
|
||||||
|
}
|
||||||
|
|
||||||
if newInterval != f.currentInterval {
|
if newInterval != f.currentInterval {
|
||||||
f.logger.Debug("Increased fetch interval due to failure",
|
f.logger.Debug("Increased fetch interval due to failure",
|
||||||
@@ -206,17 +192,15 @@ func (f *ASNFetcher) recordFailure() {
|
|||||||
}
|
}
|
||||||
f.intervalMu.Unlock()
|
f.intervalMu.Unlock()
|
||||||
|
|
||||||
// Record error time for stats, dropping those older than statsWindow
|
// Record error time for stats
|
||||||
f.statsMu.Lock()
|
f.statsMu.Lock()
|
||||||
now := f.now()
|
f.errorTimes = append(f.errorTimes, time.Now())
|
||||||
f.errorTimes = pruneOldTimes(f.errorTimes, now.Add(-statsWindow))
|
|
||||||
f.errorTimes = append(f.errorTimes, now)
|
|
||||||
f.statsMu.Unlock()
|
f.statsMu.Unlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
// run is the main background loop.
|
// run is the main background loop.
|
||||||
func (f *ASNFetcher) run(ctx context.Context) {
|
func (f *ASNFetcher) run() {
|
||||||
defer close(f.done)
|
defer f.wg.Done()
|
||||||
|
|
||||||
timer := time.NewTimer(f.getInterval())
|
timer := time.NewTimer(f.getInterval())
|
||||||
defer timer.Stop()
|
defer timer.Stop()
|
||||||
@@ -228,13 +212,13 @@ func (f *ASNFetcher) run(ctx context.Context) {
|
|||||||
|
|
||||||
case asn := <-f.immediateQueue:
|
case asn := <-f.immediateQueue:
|
||||||
// Process immediate request (respects lock)
|
// Process immediate request (respects lock)
|
||||||
f.tryFetch(ctx, asn)
|
f.tryFetch(asn)
|
||||||
// Reset timer after immediate fetch
|
// Reset timer after immediate fetch
|
||||||
timer.Reset(f.getInterval())
|
timer.Reset(f.getInterval())
|
||||||
|
|
||||||
case <-timer.C:
|
case <-timer.C:
|
||||||
// Background fetch of stale/missing ASN
|
// Background fetch of stale/missing ASN
|
||||||
f.fetchNextStale(ctx)
|
f.fetchNextStale()
|
||||||
// Reset timer with potentially updated interval
|
// Reset timer with potentially updated interval
|
||||||
timer.Reset(f.getInterval())
|
timer.Reset(f.getInterval())
|
||||||
}
|
}
|
||||||
@@ -243,7 +227,7 @@ func (f *ASNFetcher) run(ctx context.Context) {
|
|||||||
|
|
||||||
// tryFetch attempts to fetch and update an ASN, respecting the fetch lock.
|
// tryFetch attempts to fetch and update an ASN, respecting the fetch lock.
|
||||||
// Returns true if fetch was successful.
|
// Returns true if fetch was successful.
|
||||||
func (f *ASNFetcher) tryFetch(ctx context.Context, asn int) bool {
|
func (f *ASNFetcher) tryFetch(asn int) bool {
|
||||||
// Try to acquire lock, skip if another fetch is running
|
// Try to acquire lock, skip if another fetch is running
|
||||||
if !f.fetchMu.TryLock() {
|
if !f.fetchMu.TryLock() {
|
||||||
f.logger.Debug("Skipping fetch, another fetch in progress", "asn", asn)
|
f.logger.Debug("Skipping fetch, another fetch in progress", "asn", asn)
|
||||||
@@ -252,11 +236,11 @@ func (f *ASNFetcher) tryFetch(ctx context.Context, asn int) bool {
|
|||||||
}
|
}
|
||||||
defer f.fetchMu.Unlock()
|
defer f.fetchMu.Unlock()
|
||||||
|
|
||||||
return f.fetchAndUpdate(ctx, asn)
|
return f.fetchAndUpdate(asn)
|
||||||
}
|
}
|
||||||
|
|
||||||
// fetchNextStale finds and fetches the next ASN needing WHOIS data.
|
// fetchNextStale finds and fetches the next ASN needing WHOIS data.
|
||||||
func (f *ASNFetcher) fetchNextStale(ctx context.Context) {
|
func (f *ASNFetcher) fetchNextStale() {
|
||||||
// Try to acquire lock, skip if another fetch is running
|
// Try to acquire lock, skip if another fetch is running
|
||||||
if !f.fetchMu.TryLock() {
|
if !f.fetchMu.TryLock() {
|
||||||
f.logger.Debug("Skipping stale fetch, another fetch in progress")
|
f.logger.Debug("Skipping stale fetch, another fetch in progress")
|
||||||
@@ -265,12 +249,12 @@ func (f *ASNFetcher) fetchNextStale(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
defer f.fetchMu.Unlock()
|
defer f.fetchMu.Unlock()
|
||||||
|
|
||||||
lookupCtx, cancel := context.WithTimeout(ctx, time.Minute)
|
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
asn, err := f.db.GetNextStaleASN(lookupCtx, whoisStaleThreshold)
|
asn, err := f.db.GetNextStaleASN(ctx, whoisStaleThreshold)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if !errors.Is(err, database.ErrNoStaleASN) {
|
if err != database.ErrNoStaleASN {
|
||||||
f.logger.Error("Failed to get stale ASN", "error", err)
|
f.logger.Error("Failed to get stale ASN", "error", err)
|
||||||
f.recordFailure()
|
f.recordFailure()
|
||||||
}
|
}
|
||||||
@@ -279,34 +263,18 @@ func (f *ASNFetcher) fetchNextStale(ctx context.Context) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
f.fetchAndUpdate(ctx, asn)
|
f.fetchAndUpdate(asn)
|
||||||
}
|
}
|
||||||
|
|
||||||
// fetchAndUpdate performs a WHOIS lookup and updates the database.
|
// fetchAndUpdate performs a WHOIS lookup and updates the database.
|
||||||
// Returns true if successful. A failed lookup leaves the AS's row as it was,
|
// Returns true if successful.
|
||||||
// so the AS stays stale. An AS IANA refers to no RIR is stored with no
|
func (f *ASNFetcher) fetchAndUpdate(asn int) bool {
|
||||||
// WHOIS data, so it is not looked up again until it is stale; that counts
|
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
||||||
// as neither a success nor a failure.
|
|
||||||
func (f *ASNFetcher) fetchAndUpdate(ctx context.Context, asn int) bool {
|
|
||||||
ctx, cancel := context.WithTimeout(ctx, time.Minute)
|
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
f.logger.Info("Fetching WHOIS data", "asn", asn)
|
f.logger.Info("Fetching WHOIS data", "asn", asn)
|
||||||
|
|
||||||
info, err := f.whoisClient.LookupASN(ctx, asn)
|
info, err := f.whoisClient.LookupASN(ctx, asn)
|
||||||
if errors.Is(err, whois.ErrNoReferral) {
|
|
||||||
f.logger.Info("IANA refers the AS to no RIR", "asn", asn)
|
|
||||||
|
|
||||||
err = f.db.UpdateASNWHOIS(ctx, &database.ASNWHOISUpdate{ASN: asn})
|
|
||||||
if err != nil {
|
|
||||||
f.logger.Error("Failed to update ASN WHOIS data",
|
|
||||||
"asn", asn, "error", err)
|
|
||||||
f.recordFailure()
|
|
||||||
}
|
|
||||||
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
f.logger.Error("WHOIS lookup failed", "asn", asn, "error", err)
|
f.logger.Error("WHOIS lookup failed", "asn", asn, "error", err)
|
||||||
f.recordFailure()
|
f.recordFailure()
|
||||||
|
|||||||
@@ -1,278 +0,0 @@
|
|||||||
package routewatch
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bufio"
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"io"
|
|
||||||
"net"
|
|
||||||
"sync"
|
|
||||||
"syscall"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/database"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The WHOIS servers a lookup asks: IANA first, then the RIR IANA refers it
|
|
||||||
// to.
|
|
||||||
const (
|
|
||||||
ianaWHOISServer = "whois.iana.org:43"
|
|
||||||
ripeWHOISServer = "whois.ripe.net:43"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ianaRefersToRIPE is IANA's answer for an AS in a block it gave to RIPE.
|
|
||||||
const ianaRefersToRIPE = `refer: whois.ripe.net
|
|
||||||
|
|
||||||
as-block: 64496-64511
|
|
||||||
organisation: Assigned by RIPE NCC
|
|
||||||
changed: 2002-09-01
|
|
||||||
source: IANA
|
|
||||||
`
|
|
||||||
|
|
||||||
// ripeAccessDenied is RIPE's answer to a host it no longer serves.
|
|
||||||
const ripeAccessDenied = `% This is the RIPE Database query service.
|
|
||||||
%ERROR:201: access denied for 192.0.2.1
|
|
||||||
`
|
|
||||||
|
|
||||||
// ianaPrivateUseBlock is IANA's answer for a private-use AS: its block and
|
|
||||||
// no referral.
|
|
||||||
const ianaPrivateUseBlock = `as-block: 64512-65534
|
|
||||||
organisation: Reserved for Private Use
|
|
||||||
source: IANA
|
|
||||||
`
|
|
||||||
|
|
||||||
// newFetcherWithWHOISAnswers returns an ASN fetcher on a new database that
|
|
||||||
// holds testASNA, never looked up. The fetcher's WHOIS client connects to a
|
|
||||||
// fake server for each address in answers, which reads the query and sends
|
|
||||||
// answers[address]; connecting to any other address is refused.
|
|
||||||
func newFetcherWithWHOISAnswers(
|
|
||||||
t *testing.T, answers map[string]string,
|
|
||||||
) (*ASNFetcher, *database.Database) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
db := newTestDatabase(t)
|
|
||||||
|
|
||||||
_, err := db.GetOrCreateASN(testASNA, time.Now())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GetOrCreateASN: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var servers sync.WaitGroup
|
|
||||||
|
|
||||||
t.Cleanup(servers.Wait)
|
|
||||||
|
|
||||||
f := NewASNFetcher(db, logger.New().Logger)
|
|
||||||
f.whoisClient.DialContext = func(
|
|
||||||
_ context.Context, _, address string,
|
|
||||||
) (net.Conn, error) {
|
|
||||||
answer, ok := answers[address]
|
|
||||||
if !ok {
|
|
||||||
return nil, syscall.ECONNREFUSED
|
|
||||||
}
|
|
||||||
|
|
||||||
client, server := net.Pipe()
|
|
||||||
|
|
||||||
servers.Go(func() {
|
|
||||||
defer func() { _ = server.Close() }()
|
|
||||||
|
|
||||||
_, _ = bufio.NewReader(server).ReadString('\n')
|
|
||||||
_, _ = io.WriteString(server, answer)
|
|
||||||
})
|
|
||||||
|
|
||||||
return client, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return f, db
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestFetchAndUpdateKeepsTheRowWhenTheLookupFails looks up an AS IANA refers
|
|
||||||
// to RIPE when RIPE's server refuses the connection, and when it answers only
|
|
||||||
// with an error. The AS must stay never looked up, and the fetcher must back
|
|
||||||
// off.
|
|
||||||
func TestFetchAndUpdateKeepsTheRowWhenTheLookupFails(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for name, answers := range map[string]map[string]string{
|
|
||||||
"connection refused": {ianaWHOISServer: ianaRefersToRIPE},
|
|
||||||
"only an error": {
|
|
||||||
ianaWHOISServer: ianaRefersToRIPE,
|
|
||||||
ripeWHOISServer: ripeAccessDenied,
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
f, db := newFetcherWithWHOISAnswers(t, answers)
|
|
||||||
|
|
||||||
if f.fetchAndUpdate(t.Context(), testASNA) {
|
|
||||||
t.Error("fetchAndUpdate reported success")
|
|
||||||
}
|
|
||||||
|
|
||||||
stats, err := db.GetWHOISStats(t.Context(), whoisStaleThreshold)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GetWHOISStats: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if stats.NeverFetched != 1 {
|
|
||||||
t.Errorf("never looked up = %d, want 1: the failed lookup "+
|
|
||||||
"was stored", stats.NeverFetched)
|
|
||||||
}
|
|
||||||
|
|
||||||
want := baseInterval * backoffNumerator / backoffDenominator
|
|
||||||
|
|
||||||
got := f.GetStats()
|
|
||||||
if got.CurrentInterval != want || got.ConsecutiveFails != 1 {
|
|
||||||
t.Errorf("interval %v after %d failures, want %v after 1",
|
|
||||||
got.CurrentInterval, got.ConsecutiveFails, want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRecordStepsTheInterval has the fetcher fail from its base interval of
|
|
||||||
// 15 seconds, then succeed. Each failure makes the interval 1.5 times as
|
|
||||||
// long, up to one minute, and each success halves it, down to one second.
|
|
||||||
func TestRecordStepsTheInterval(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
f := NewASNFetcher(newMockStore(), logger.New().Logger)
|
|
||||||
|
|
||||||
for i, want := range []time.Duration{
|
|
||||||
22500 * time.Millisecond,
|
|
||||||
33750 * time.Millisecond,
|
|
||||||
50625 * time.Millisecond,
|
|
||||||
time.Minute,
|
|
||||||
time.Minute,
|
|
||||||
} {
|
|
||||||
f.recordFailure()
|
|
||||||
|
|
||||||
if got := f.getInterval(); got != want {
|
|
||||||
t.Errorf("interval after %d failures = %v, want %v",
|
|
||||||
i+1, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for i, want := range []time.Duration{
|
|
||||||
30 * time.Second,
|
|
||||||
15 * time.Second,
|
|
||||||
7500 * time.Millisecond,
|
|
||||||
3750 * time.Millisecond,
|
|
||||||
1875 * time.Millisecond,
|
|
||||||
time.Second,
|
|
||||||
time.Second,
|
|
||||||
} {
|
|
||||||
f.recordSuccess()
|
|
||||||
|
|
||||||
if got := f.getInterval(); got != want {
|
|
||||||
t.Errorf("interval after %d successes = %v, want %v",
|
|
||||||
i+1, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestFetchAndUpdateMarksAnASWithoutReferral looks up a private-use AS, which
|
|
||||||
// IANA refers to no RIR. The AS must not be looked up again until it is
|
|
||||||
// stale, and the lookup counts as neither a success nor a failure.
|
|
||||||
func TestFetchAndUpdateMarksAnASWithoutReferral(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
f, db := newFetcherWithWHOISAnswers(t,
|
|
||||||
map[string]string{ianaWHOISServer: ianaPrivateUseBlock})
|
|
||||||
|
|
||||||
f.fetchAndUpdate(t.Context(), testASNA)
|
|
||||||
|
|
||||||
asn, err := db.GetNextStaleASN(t.Context(), whoisStaleThreshold)
|
|
||||||
if !errors.Is(err, database.ErrNoStaleASN) {
|
|
||||||
t.Errorf("GetNextStaleASN = %d, %v; want ErrNoStaleASN", asn, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
got := f.GetStats()
|
|
||||||
if got.CurrentInterval != baseInterval || got.SuccessesLastHour != 0 ||
|
|
||||||
got.ErrorsLastHour != 0 {
|
|
||||||
t.Errorf("interval %v, %d successes, %d errors; want %v, 0, 0",
|
|
||||||
got.CurrentInterval, got.SuccessesLastHour, got.ErrorsLastHour,
|
|
||||||
baseInterval)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestASNFetcherStopReturnsByItsDeadline starts a fetcher, has it look up an
|
|
||||||
// AS whose WHOIS server does not answer until the test ends, and stops it with
|
|
||||||
// a short deadline while that lookup is in progress. Stop must return all the
|
|
||||||
// same instead of waiting for the lookup.
|
|
||||||
func TestASNFetcherStopReturnsByItsDeadline(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const stopDeadline = 100 * time.Millisecond
|
|
||||||
|
|
||||||
lookupStarted := make(chan struct{})
|
|
||||||
release := make(chan struct{})
|
|
||||||
|
|
||||||
f := NewASNFetcher(newMockStore(), logger.New().Logger)
|
|
||||||
f.whoisClient.DialContext = func(
|
|
||||||
context.Context, string, string,
|
|
||||||
) (net.Conn, error) {
|
|
||||||
close(lookupStarted)
|
|
||||||
<-release
|
|
||||||
|
|
||||||
return nil, syscall.ECONNREFUSED
|
|
||||||
}
|
|
||||||
|
|
||||||
f.Start(t.Context())
|
|
||||||
f.QueueImmediate(testASNA)
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-lookupStarted:
|
|
||||||
case <-time.After(stepTimeout):
|
|
||||||
t.Fatal("the fetcher did not start the lookup")
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(t.Context(), stopDeadline)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
if !returnsInTime(func() { f.Stop(ctx) }) {
|
|
||||||
t.Fatalf("Stop with a deadline of %s did not return while a lookup "+
|
|
||||||
"was in progress", stopDeadline)
|
|
||||||
}
|
|
||||||
|
|
||||||
close(release)
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-f.done:
|
|
||||||
case <-time.After(stepTimeout):
|
|
||||||
t.Fatal("the fetcher goroutine did not return after the lookup ended")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRecordKeepsOnlyTheStatsWindow records a success and a failure every
|
|
||||||
// second, by a clock the test sets, for three times statsWindow, and never
|
|
||||||
// asks for the stats. The fetcher must keep only the times within the last
|
|
||||||
// statsWindow.
|
|
||||||
func TestRecordKeepsOnlyTheStatsWindow(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const (
|
|
||||||
step = time.Second
|
|
||||||
want = int(statsWindow / step)
|
|
||||||
records = 3 * want
|
|
||||||
)
|
|
||||||
|
|
||||||
clock := time.Now()
|
|
||||||
|
|
||||||
f := NewASNFetcher(newMockStore(), logger.New().Logger)
|
|
||||||
f.now = func() time.Time { return clock }
|
|
||||||
|
|
||||||
for range records {
|
|
||||||
clock = clock.Add(step)
|
|
||||||
|
|
||||||
f.recordSuccess()
|
|
||||||
f.recordFailure()
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(f.successTimes) != want || len(f.errorTimes) != want {
|
|
||||||
t.Errorf("%d success and %d error times kept, want %d of each, "+
|
|
||||||
"those of the last %s", len(f.successTimes), len(f.errorTimes),
|
|
||||||
want, statsWindow)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,62 +0,0 @@
|
|||||||
package routewatch
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// flushLoop is the flush goroutine of the prefix, AS and peer handlers. It is
|
|
||||||
// the only goroutine that writes a handler's batch. flushBatch takes the batch
|
|
||||||
// and writes it.
|
|
||||||
//
|
|
||||||
// flushLoop calls flushBatch on each tick of a timer, every batchTimeout. It
|
|
||||||
// also calls it at once when batchFullCh wakes it. A full batch's write
|
|
||||||
// restarts the timer, so the next tick comes batchTimeout after that write
|
|
||||||
// began. When stopCh hands it Stop's context, it calls flushBatch once more,
|
|
||||||
// unless the stop deadline has passed. Then it closes done and returns.
|
|
||||||
func flushLoop(
|
|
||||||
batchTimeout time.Duration,
|
|
||||||
batchFullCh <-chan struct{},
|
|
||||||
stopCh <-chan context.Context,
|
|
||||||
done chan<- struct{},
|
|
||||||
flushBatch func(),
|
|
||||||
) {
|
|
||||||
defer close(done)
|
|
||||||
|
|
||||||
ticker := time.NewTicker(batchTimeout)
|
|
||||||
defer ticker.Stop()
|
|
||||||
|
|
||||||
for {
|
|
||||||
// Stop goes first. A write can run past the stop deadline while a full
|
|
||||||
// batch's wake-up or a tick waits beside Stop's context. The select
|
|
||||||
// below picks one of them at random, but past the deadline no flush
|
|
||||||
// may start.
|
|
||||||
select {
|
|
||||||
case ctx := <-stopCh:
|
|
||||||
// Final flush, unless the stop deadline has passed
|
|
||||||
if ctx.Err() == nil {
|
|
||||||
flushBatch()
|
|
||||||
}
|
|
||||||
|
|
||||||
return
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-ticker.C:
|
|
||||||
flushBatch()
|
|
||||||
case <-batchFullCh:
|
|
||||||
// Reset also drops a tick that waits in ticker.C, so no timed
|
|
||||||
// write follows this one at once.
|
|
||||||
ticker.Reset(batchTimeout)
|
|
||||||
flushBatch()
|
|
||||||
case ctx := <-stopCh:
|
|
||||||
// Final flush, unless the stop deadline has passed
|
|
||||||
if ctx.Err() == nil {
|
|
||||||
flushBatch()
|
|
||||||
}
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,891 +0,0 @@
|
|||||||
package routewatch
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"log/slog"
|
|
||||||
"slices"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"testing/synctest"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/database"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/ristypes"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/routetable"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
// stepTimeout is how long a test waits for a step it expects, such as
|
|
||||||
// HandleMessage returning, a write starting or Shutdown returning, before
|
|
||||||
// it fails. Only a step that never comes should reach it. On a loaded
|
|
||||||
// host, under the race detector, HandleMessage takes seconds to apply a
|
|
||||||
// full prefix batch. The helpers below bound every wait with it, Stop
|
|
||||||
// included. A test that hangs in one of them waits out at most two of
|
|
||||||
// these bounds, its step's and Stop's, and then fails with its own
|
|
||||||
// message. That leaves time before go test's 90-second timeout.
|
|
||||||
stepTimeout = 15 * time.Second
|
|
||||||
|
|
||||||
// stillWaitingAfter is how long a test watches a HandleMessage that must
|
|
||||||
// wait. One that does not wait returns well within it.
|
|
||||||
stillWaitingAfter = 100 * time.Millisecond
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestFlushLoopWritesOnEveryTick runs flushLoop in a synctest bubble, whose
|
|
||||||
// clock is fake: time.Sleep there moves the clock on and returns at once. Each
|
|
||||||
// write takes a tenth of the interval. Two timed writes must start one interval
|
|
||||||
// apart. A full batch's write, which starts as the second timed write ends,
|
|
||||||
// must be followed by a timed write one interval after it began.
|
|
||||||
func TestFlushLoopWritesOnEveryTick(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
const (
|
|
||||||
interval = time.Second
|
|
||||||
writeTime = 100 * time.Millisecond
|
|
||||||
)
|
|
||||||
|
|
||||||
writeStarts := make(chan time.Time)
|
|
||||||
batchFullCh := make(chan struct{}, 1)
|
|
||||||
stopCh := make(chan context.Context, 1)
|
|
||||||
done := make(chan struct{})
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
flushLoop(interval, batchFullCh, stopCh, done, func() {
|
|
||||||
writeStarts <- time.Now()
|
|
||||||
|
|
||||||
time.Sleep(writeTime)
|
|
||||||
})
|
|
||||||
}()
|
|
||||||
|
|
||||||
first := <-writeStarts
|
|
||||||
second := <-writeStarts
|
|
||||||
|
|
||||||
if gap := second.Sub(first); gap != interval {
|
|
||||||
t.Errorf("the second timed write started %s after the first, "+
|
|
||||||
"want %s", gap, interval)
|
|
||||||
}
|
|
||||||
|
|
||||||
batchFullCh <- struct{}{}
|
|
||||||
|
|
||||||
full := <-writeStarts
|
|
||||||
timed := <-writeStarts
|
|
||||||
|
|
||||||
if gap := timed.Sub(full); gap != interval {
|
|
||||||
t.Errorf("the timed write after a full batch's started %s after "+
|
|
||||||
"it, want %s", gap, interval)
|
|
||||||
}
|
|
||||||
|
|
||||||
stopCh <- t.Context()
|
|
||||||
|
|
||||||
<-writeStarts // Stop's last write
|
|
||||||
<-done
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// blockingStore is a mockStore whose batch writes for the AS, peer and prefix
|
|
||||||
// handlers and peering write wait until the test calls releaseWrites.
|
|
||||||
// writeStarted is closed when the first of them starts. It records what each
|
|
||||||
// batch write was given.
|
|
||||||
type blockingStore struct {
|
|
||||||
*mockStore
|
|
||||||
|
|
||||||
release chan struct{}
|
|
||||||
releaseOnce sync.Once
|
|
||||||
writeStarted chan struct{}
|
|
||||||
startOnce sync.Once
|
|
||||||
|
|
||||||
mu sync.Mutex
|
|
||||||
prefixes map[string]bool
|
|
||||||
asns map[int]bool
|
|
||||||
peers map[string]bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func newBlockingStore() *blockingStore {
|
|
||||||
return &blockingStore{
|
|
||||||
mockStore: newMockStore(),
|
|
||||||
release: make(chan struct{}),
|
|
||||||
writeStarted: make(chan struct{}),
|
|
||||||
prefixes: make(map[string]bool),
|
|
||||||
asns: make(map[int]bool),
|
|
||||||
peers: make(map[string]bool),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *blockingStore) UpdatePrefixesBatch(prefixes map[string]time.Time) error {
|
|
||||||
s.waitForRelease()
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
|
|
||||||
for prefix := range prefixes {
|
|
||||||
s.prefixes[prefix] = true
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *blockingStore) GetOrCreateASNBatch(asns map[int]database.SeenTimes) error {
|
|
||||||
s.waitForRelease()
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
|
|
||||||
for asn := range asns {
|
|
||||||
s.asns[asn] = true
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *blockingStore) UpdatePeerBatch(peers map[string]database.PeerUpdate) error {
|
|
||||||
s.waitForRelease()
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
|
|
||||||
for peerIP := range peers {
|
|
||||||
s.peers[peerIP] = true
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *blockingStore) RecordPeering(
|
|
||||||
asA, asB int, seen database.SeenTimes,
|
|
||||||
) error {
|
|
||||||
s.waitForRelease()
|
|
||||||
|
|
||||||
return s.mockStore.RecordPeering(asA, asB, seen)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *blockingStore) waitForRelease() {
|
|
||||||
s.startOnce.Do(func() { close(s.writeStarted) })
|
|
||||||
<-s.release
|
|
||||||
}
|
|
||||||
|
|
||||||
// releaseWrites lets the held writes, and all later ones, finish. A second call
|
|
||||||
// does nothing.
|
|
||||||
func (s *blockingStore) releaseWrites() {
|
|
||||||
s.releaseOnce.Do(func() { close(s.release) })
|
|
||||||
}
|
|
||||||
|
|
||||||
// stopLog holds what a handler's Stop logs it did not write.
|
|
||||||
//
|
|
||||||
//nolint:tagliatelle // field names are the keys the handlers log
|
|
||||||
type stopLog struct {
|
|
||||||
Msg string `json:"msg"`
|
|
||||||
Updates int `json:"updates"`
|
|
||||||
TableChanges int `json:"table_changes"`
|
|
||||||
Paths int `json:"paths"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// newErrorLogger returns a logger that writes only errors, as JSON lines, to
|
|
||||||
// logs.
|
|
||||||
func newErrorLogger(logs *bytes.Buffer) *logger.Logger {
|
|
||||||
return &logger.Logger{Logger: slog.New(slog.NewJSONHandler(logs,
|
|
||||||
&slog.HandlerOptions{Level: slog.LevelError}))}
|
|
||||||
}
|
|
||||||
|
|
||||||
// lastLogged decodes the last line in logs, which is the one Stop logs: a
|
|
||||||
// handler logs nothing after it.
|
|
||||||
func lastLogged(t *testing.T, logs *bytes.Buffer) stopLog {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
lines := bytes.Split(bytes.TrimSpace(logs.Bytes()), []byte("\n"))
|
|
||||||
|
|
||||||
var logged stopLog
|
|
||||||
|
|
||||||
err := json.Unmarshal(lines[len(lines)-1], &logged)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("want the error Stop logs, got %q: %v", logs.String(), err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return logged
|
|
||||||
}
|
|
||||||
|
|
||||||
// returnsInTime reports whether f returns within stepTimeout.
|
|
||||||
func returnsInTime(f func()) bool {
|
|
||||||
done := make(chan struct{})
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
f()
|
|
||||||
close(done)
|
|
||||||
}()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-done:
|
|
||||||
return true
|
|
||||||
case <-time.After(stepTimeout):
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// stopInTime stops h, with no deadline, and fails the test if Stop does not
|
|
||||||
// return within stepTimeout. It only marks the test failed, so it can be
|
|
||||||
// deferred.
|
|
||||||
func stopInTime(t *testing.T, h interface{ Stop(ctx context.Context) }) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
if !returnsInTime(func() { h.Stop(t.Context()) }) {
|
|
||||||
t.Errorf("Stop did not return within %s", stepTimeout)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// startBlockedWrite hands h the messages in fill and waits until h starts to
|
|
||||||
// write them to store. The prefix, AS and peer handlers start when fill fills
|
|
||||||
// their batch, and the peering handler on its timer. store holds the write
|
|
||||||
// until the test calls releaseWrites.
|
|
||||||
func startBlockedWrite(
|
|
||||||
t *testing.T,
|
|
||||||
store *blockingStore,
|
|
||||||
h interface {
|
|
||||||
HandleMessage(msg *ristypes.RISMessage)
|
|
||||||
},
|
|
||||||
fill []*ristypes.RISMessage,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
if !returnsInTime(func() {
|
|
||||||
for _, msg := range fill {
|
|
||||||
h.HandleMessage(msg)
|
|
||||||
}
|
|
||||||
}) {
|
|
||||||
t.Fatal("HandleMessage did not return after filling the batch: " +
|
|
||||||
"it waits for the database write")
|
|
||||||
}
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-store.writeStarted:
|
|
||||||
case <-time.After(stepTimeout):
|
|
||||||
t.Fatal("the handler did not start to write the messages")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// handleWhileAWriteIsBlocked starts a blocked write with the messages in fill
|
|
||||||
// and hands h the message extra while store holds the write. HandleMessage must
|
|
||||||
// return each time. Then it lets the writes finish and stops h, which writes
|
|
||||||
// what it holds.
|
|
||||||
func handleWhileAWriteIsBlocked(
|
|
||||||
t *testing.T,
|
|
||||||
store *blockingStore,
|
|
||||||
h interface {
|
|
||||||
HandleMessage(msg *ristypes.RISMessage)
|
|
||||||
Stop(ctx context.Context)
|
|
||||||
},
|
|
||||||
fill []*ristypes.RISMessage,
|
|
||||||
extra *ristypes.RISMessage,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
defer stopInTime(t, h)
|
|
||||||
defer store.releaseWrites()
|
|
||||||
|
|
||||||
startBlockedWrite(t, store, h, fill)
|
|
||||||
|
|
||||||
if !returnsInTime(func() { h.HandleMessage(extra) }) {
|
|
||||||
t.Fatal("HandleMessage did not return while a database write was blocked")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// handleAtMaxBatchSize starts a blocked write with the messages in fill, then,
|
|
||||||
// while store holds the write, hands h the messages in toMax, which bring its
|
|
||||||
// next batch to its maximum size, and the message extra. HandleMessage for
|
|
||||||
// extra must wait until the test lets the write finish, and return after. Then
|
|
||||||
// it stops h, which writes what it holds. The tests lower h's maximum batch
|
|
||||||
// size to one batch, so toMax is one batch: the real maximum of ten batches is
|
|
||||||
// slow to reach under the race detector.
|
|
||||||
func handleAtMaxBatchSize(
|
|
||||||
t *testing.T,
|
|
||||||
store *blockingStore,
|
|
||||||
h interface {
|
|
||||||
HandleMessage(msg *ristypes.RISMessage)
|
|
||||||
Stop(ctx context.Context)
|
|
||||||
},
|
|
||||||
fill, toMax []*ristypes.RISMessage,
|
|
||||||
extra *ristypes.RISMessage,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
defer stopInTime(t, h)
|
|
||||||
defer store.releaseWrites()
|
|
||||||
|
|
||||||
startBlockedWrite(t, store, h, fill)
|
|
||||||
|
|
||||||
if !returnsInTime(func() {
|
|
||||||
for _, msg := range toMax {
|
|
||||||
h.HandleMessage(msg)
|
|
||||||
}
|
|
||||||
}) {
|
|
||||||
t.Fatal("HandleMessage did not return while the batch grew to its " +
|
|
||||||
"maximum size")
|
|
||||||
}
|
|
||||||
|
|
||||||
returned := make(chan struct{})
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
h.HandleMessage(extra)
|
|
||||||
close(returned)
|
|
||||||
}()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-returned:
|
|
||||||
t.Fatal("HandleMessage returned while the batch was at its maximum " +
|
|
||||||
"size and a write was blocked")
|
|
||||||
case <-time.After(stillWaitingAfter):
|
|
||||||
}
|
|
||||||
|
|
||||||
store.releaseWrites()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-returned:
|
|
||||||
case <-time.After(stepTimeout):
|
|
||||||
t.Fatal("HandleMessage did not return after the database write finished")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// stopWhileAWriteIsBlocked starts a blocked write with the messages in fill,
|
|
||||||
// hands h the messages in extra, and stops h with a deadline that passes while
|
|
||||||
// store holds the write. Stop must return all the same. Then it lets the write
|
|
||||||
// finish and waits until done, closed when h's flush goroutine returns. Past
|
|
||||||
// the deadline that goroutine skips its last write, so extra is not written,
|
|
||||||
// and Stop logs as not written both extra and what the held write took.
|
|
||||||
func stopWhileAWriteIsBlocked(
|
|
||||||
t *testing.T,
|
|
||||||
store *blockingStore,
|
|
||||||
h interface {
|
|
||||||
HandleMessage(msg *ristypes.RISMessage)
|
|
||||||
Stop(ctx context.Context)
|
|
||||||
},
|
|
||||||
done <-chan struct{},
|
|
||||||
fill []*ristypes.RISMessage,
|
|
||||||
extra ...*ristypes.RISMessage,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
const stopDeadline = 100 * time.Millisecond
|
|
||||||
|
|
||||||
defer store.releaseWrites()
|
|
||||||
|
|
||||||
startBlockedWrite(t, store, h, fill)
|
|
||||||
|
|
||||||
if !returnsInTime(func() {
|
|
||||||
for _, msg := range extra {
|
|
||||||
h.HandleMessage(msg)
|
|
||||||
}
|
|
||||||
}) {
|
|
||||||
t.Fatal("HandleMessage did not return while a database write was blocked")
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(t.Context(), stopDeadline)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
if !returnsInTime(func() { h.Stop(ctx) }) {
|
|
||||||
t.Fatalf("Stop with a deadline of %s did not return while a write "+
|
|
||||||
"was blocked", stopDeadline)
|
|
||||||
}
|
|
||||||
|
|
||||||
store.releaseWrites()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-done:
|
|
||||||
case <-time.After(stepTimeout):
|
|
||||||
t.Fatal("the flush goroutine did not return after the write finished")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPrefixHandlerSkipsItsLastWriteAfterTheStopDeadline also checks the
|
|
||||||
// routing table's changes: the flush whose prefix write is held wrote those of
|
|
||||||
// the fill, so Stop must log the one of the extra prefix as not written. The
|
|
||||||
// flush timer must not fire. A flush it started before the fill would be the
|
|
||||||
// one held, and it would not have written the fill's change.
|
|
||||||
func TestPrefixHandlerSkipsItsLastWriteAfterTheStopDeadline(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const extraPrefix = "198.51.100.0/24"
|
|
||||||
|
|
||||||
var logs bytes.Buffer
|
|
||||||
|
|
||||||
store := newBlockingStore()
|
|
||||||
h := newPrefixHandler(store, newErrorLogger(&logs),
|
|
||||||
routetable.New(routetable.Params{}), longerThanAnyTest,
|
|
||||||
maxTableChangesPerWrite)
|
|
||||||
fill := slices.Repeat([]string{testV4Prefix}, prefixBatchSize)
|
|
||||||
stopWhileAWriteIsBlocked(t, store, h, h.done,
|
|
||||||
[]*ristypes.RISMessage{prefixAnnouncement(fill)},
|
|
||||||
prefixAnnouncement([]string{extraPrefix}))
|
|
||||||
|
|
||||||
store.mu.Lock()
|
|
||||||
defer store.mu.Unlock()
|
|
||||||
|
|
||||||
if store.prefixes[extraPrefix] {
|
|
||||||
t.Errorf("%s was written after the stop deadline", extraPrefix)
|
|
||||||
}
|
|
||||||
|
|
||||||
logged := lastLogged(t, &logs)
|
|
||||||
if logged.Updates != prefixBatchSize+1 || logged.TableChanges != 1 {
|
|
||||||
t.Errorf("Stop logged %d updates and %d table changes not written, "+
|
|
||||||
"want %d and 1", logged.Updates, logged.TableChanges,
|
|
||||||
prefixBatchSize+1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPrefixHandlerSkipsAFullBatchAfterTheStopDeadline hands the prefix handler
|
|
||||||
// a full batch while a write is held, so the batch's wake-up of the flush
|
|
||||||
// goroutine waits. The stop deadline passes meanwhile. When the write ends, the
|
|
||||||
// flush goroutine must stop instead of writing the batch.
|
|
||||||
func TestPrefixHandlerSkipsAFullBatchAfterTheStopDeadline(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const extraPrefix = "198.51.100.0/24"
|
|
||||||
|
|
||||||
store := newBlockingStore()
|
|
||||||
h := NewPrefixHandler(store, logger.New(), routetable.New(routetable.Params{}))
|
|
||||||
fill := slices.Repeat([]string{testV4Prefix}, prefixBatchSize)
|
|
||||||
full := slices.Repeat([]string{extraPrefix}, prefixBatchSize)
|
|
||||||
stopWhileAWriteIsBlocked(t, store, h, h.done,
|
|
||||||
[]*ristypes.RISMessage{prefixAnnouncement(fill)},
|
|
||||||
prefixAnnouncement(full))
|
|
||||||
|
|
||||||
store.mu.Lock()
|
|
||||||
defer store.mu.Unlock()
|
|
||||||
|
|
||||||
if store.prefixes[extraPrefix] {
|
|
||||||
t.Errorf("%s was written after the stop deadline", extraPrefix)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// tableWriteBlockingStore is a blockingStore whose writes of the routing
|
|
||||||
// table's changes also wait until the test calls releaseWrites.
|
|
||||||
type tableWriteBlockingStore struct {
|
|
||||||
*blockingStore
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s tableWriteBlockingStore) WriteRouteChanges(
|
|
||||||
changes []routetable.Change, newPeers map[string]uint,
|
|
||||||
) error {
|
|
||||||
s.waitForRelease()
|
|
||||||
|
|
||||||
return s.mockStore.WriteRouteChanges(changes, newPeers)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPrefixHandlerStopCountsTheTableChangesOfAWriteInProgress holds the write
|
|
||||||
// of the routing table's changes that a full batch starts, then hands the
|
|
||||||
// prefix handler one more prefix and stops it with a deadline that passes
|
|
||||||
// meanwhile. Stop must log as not written both the change the held write took
|
|
||||||
// and the one since.
|
|
||||||
func TestPrefixHandlerStopCountsTheTableChangesOfAWriteInProgress(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const extraPrefix = "198.51.100.0/24"
|
|
||||||
|
|
||||||
var logs bytes.Buffer
|
|
||||||
|
|
||||||
blocking := newBlockingStore()
|
|
||||||
h := NewPrefixHandler(tableWriteBlockingStore{blockingStore: blocking},
|
|
||||||
newErrorLogger(&logs), routetable.New(routetable.Params{}))
|
|
||||||
fill := slices.Repeat([]string{testV4Prefix}, prefixBatchSize)
|
|
||||||
stopWhileAWriteIsBlocked(t, blocking, h, h.done,
|
|
||||||
[]*ristypes.RISMessage{prefixAnnouncement(fill)},
|
|
||||||
prefixAnnouncement([]string{extraPrefix}))
|
|
||||||
|
|
||||||
if logged := lastLogged(t, &logs); logged.TableChanges != 2 {
|
|
||||||
t.Errorf("Stop logged %d table changes not written, want 2",
|
|
||||||
logged.TableChanges)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestASHandlerSkipsItsLastWriteAfterTheStopDeadline(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Each message gives an update for its origin AS and one for each AS
|
|
||||||
// in its path: fill asnBatchSize+1, extra 2.
|
|
||||||
const wantUpdates = asnBatchSize + 1 + 2
|
|
||||||
|
|
||||||
var logs bytes.Buffer
|
|
||||||
|
|
||||||
store := newBlockingStore()
|
|
||||||
h := newASHandler(store, newErrorLogger(&logs), longerThanAnyTest)
|
|
||||||
fill := ristypes.ASPath{Sequence: slices.Repeat([]int{testASNA}, asnBatchSize)}
|
|
||||||
stopWhileAWriteIsBlocked(t, store, h, h.done,
|
|
||||||
[]*ristypes.RISMessage{{Path: fill}},
|
|
||||||
&ristypes.RISMessage{Path: ristypes.ASPath{Sequence: []int{testASNB}}})
|
|
||||||
|
|
||||||
store.mu.Lock()
|
|
||||||
defer store.mu.Unlock()
|
|
||||||
|
|
||||||
if store.asns[testASNB] {
|
|
||||||
t.Errorf("AS%d was written after the stop deadline", testASNB)
|
|
||||||
}
|
|
||||||
|
|
||||||
if logged := lastLogged(t, &logs); logged.Updates != wantUpdates {
|
|
||||||
t.Errorf("Stop logged %d updates not written, want %d",
|
|
||||||
logged.Updates, wantUpdates)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestASHandlerSkipsAFullBatchAfterTheStopDeadline hands the AS handler a full
|
|
||||||
// batch while a write is held, so the batch's wake-up of the flush goroutine
|
|
||||||
// waits. The stop deadline passes meanwhile. When the write ends, the flush
|
|
||||||
// goroutine must stop instead of writing the batch.
|
|
||||||
func TestASHandlerSkipsAFullBatchAfterTheStopDeadline(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
store := newBlockingStore()
|
|
||||||
h := newASHandler(store, logger.New(), longerThanAnyTest)
|
|
||||||
fill := ristypes.ASPath{Sequence: slices.Repeat([]int{testASNA}, asnBatchSize)}
|
|
||||||
full := ristypes.ASPath{Sequence: slices.Repeat([]int{testASNB}, asnBatchSize)}
|
|
||||||
stopWhileAWriteIsBlocked(t, store, h, h.done,
|
|
||||||
[]*ristypes.RISMessage{{Path: fill}},
|
|
||||||
&ristypes.RISMessage{Path: full})
|
|
||||||
|
|
||||||
store.mu.Lock()
|
|
||||||
defer store.mu.Unlock()
|
|
||||||
|
|
||||||
if store.asns[testASNB] {
|
|
||||||
t.Errorf("AS%d was written after the stop deadline", testASNB)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPeerHandlerSkipsItsLastWriteAfterTheStopDeadline(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const otherPeerIP = "2001:db8:ffff::2"
|
|
||||||
|
|
||||||
var logs bytes.Buffer
|
|
||||||
|
|
||||||
store := newBlockingStore()
|
|
||||||
h := newPeerHandler(store, newErrorLogger(&logs), longerThanAnyTest)
|
|
||||||
stopWhileAWriteIsBlocked(t, store, h, h.done,
|
|
||||||
slices.Repeat([]*ristypes.RISMessage{{Peer: testPeerIP}}, peerBatchSize),
|
|
||||||
&ristypes.RISMessage{Peer: otherPeerIP})
|
|
||||||
|
|
||||||
store.mu.Lock()
|
|
||||||
defer store.mu.Unlock()
|
|
||||||
|
|
||||||
if store.peers[otherPeerIP] {
|
|
||||||
t.Errorf("peer %s was written after the stop deadline", otherPeerIP)
|
|
||||||
}
|
|
||||||
|
|
||||||
if logged := lastLogged(t, &logs); logged.Updates != peerBatchSize+1 {
|
|
||||||
t.Errorf("Stop logged %d updates not written, want %d",
|
|
||||||
logged.Updates, peerBatchSize+1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPeerHandlerSkipsAFullBatchAfterTheStopDeadline hands the peer handler a
|
|
||||||
// full batch while a write is held, so the batch's wake-up of the flush
|
|
||||||
// goroutine waits. The stop deadline passes meanwhile. When the write ends, the
|
|
||||||
// flush goroutine must stop instead of writing the batch.
|
|
||||||
func TestPeerHandlerSkipsAFullBatchAfterTheStopDeadline(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const otherPeerIP = "2001:db8:ffff::2"
|
|
||||||
|
|
||||||
store := newBlockingStore()
|
|
||||||
h := newPeerHandler(store, logger.New(), longerThanAnyTest)
|
|
||||||
stopWhileAWriteIsBlocked(t, store, h, h.done,
|
|
||||||
slices.Repeat([]*ristypes.RISMessage{{Peer: testPeerIP}}, peerBatchSize),
|
|
||||||
slices.Repeat([]*ristypes.RISMessage{{Peer: otherPeerIP}},
|
|
||||||
peerBatchSize)...)
|
|
||||||
|
|
||||||
store.mu.Lock()
|
|
||||||
defer store.mu.Unlock()
|
|
||||||
|
|
||||||
if store.peers[otherPeerIP] {
|
|
||||||
t.Errorf("peer %s was written after the stop deadline", otherPeerIP)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPeeringHandlerStopCountsTheWriteInProgress hands a running peering
|
|
||||||
// handler one AS path and stops it with a deadline that passes while store
|
|
||||||
// holds Stop's last write, of that path's peering. Stop must log that path as
|
|
||||||
// not written.
|
|
||||||
func TestPeeringHandlerStopCountsTheWriteInProgress(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const stopDeadline = 100 * time.Millisecond
|
|
||||||
|
|
||||||
var logs bytes.Buffer
|
|
||||||
|
|
||||||
store := newBlockingStore()
|
|
||||||
defer store.releaseWrites()
|
|
||||||
|
|
||||||
h := NewPeeringHandler(store, newErrorLogger(&logs))
|
|
||||||
handle(h, time.Now().UTC(), testASNA, testASNB)
|
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(t.Context(), stopDeadline)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
if !returnsInTime(func() { h.Stop(ctx) }) {
|
|
||||||
t.Fatalf("Stop with a deadline of %s did not return while a write "+
|
|
||||||
"was blocked", stopDeadline)
|
|
||||||
}
|
|
||||||
|
|
||||||
store.releaseWrites()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-h.done:
|
|
||||||
case <-time.After(stepTimeout):
|
|
||||||
t.Fatal("the processing goroutine did not return after the write " +
|
|
||||||
"finished")
|
|
||||||
}
|
|
||||||
|
|
||||||
if logged := lastLogged(t, &logs); logged.Paths != 1 {
|
|
||||||
t.Errorf("Stop logged %d paths not written, want 1", logged.Paths)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPeeringHandlerStartsNoWriteAfterTheStopDeadline has the peering handler's
|
|
||||||
// timer start a write that store holds, and hands the handler one more AS path
|
|
||||||
// meanwhile. The stop deadline passes, and a tick of the timer waits. When the
|
|
||||||
// write ends, the processing goroutine must stop instead of writing the path.
|
|
||||||
func TestPeeringHandlerStartsNoWriteAfterTheStopDeadline(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// The timer fires many times while Stop waits for its deadline.
|
|
||||||
const processInterval = 10 * time.Millisecond
|
|
||||||
|
|
||||||
store := newBlockingStore()
|
|
||||||
h := newPeeringHandler(store, logger.New(), processInterval, maxTrackedPaths)
|
|
||||||
stopWhileAWriteIsBlocked(t, store, h, h.done,
|
|
||||||
[]*ristypes.RISMessage{
|
|
||||||
{Path: ristypes.ASPath{Sequence: []int{testASNA, testASNB}}},
|
|
||||||
},
|
|
||||||
&ristypes.RISMessage{
|
|
||||||
Path: ristypes.ASPath{Sequence: []int{testASNC, testASND}},
|
|
||||||
})
|
|
||||||
|
|
||||||
store.mockStore.mu.Lock()
|
|
||||||
defer store.mockStore.mu.Unlock()
|
|
||||||
|
|
||||||
if store.Peerings[fmt.Sprintf("%d_%d", testASNC, testASND)] {
|
|
||||||
t.Errorf("the peering of AS%d and AS%d was written after the stop "+
|
|
||||||
"deadline", testASNC, testASND)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// prefixAnnouncement returns a message in which testPeerIP announces prefixes
|
|
||||||
// with origin AS testASNA.
|
|
||||||
func prefixAnnouncement(prefixes []string) *ristypes.RISMessage {
|
|
||||||
return &ristypes.RISMessage{
|
|
||||||
Peer: testPeerIP,
|
|
||||||
Path: ristypes.ASPath{Sequence: []int{testASNA}},
|
|
||||||
Announcements: []ristypes.RISAnnouncement{
|
|
||||||
{NextHop: testPeerIP, Prefixes: prefixes},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPrefixHandlerHandlesMessagesWhileAWriteIsBlocked(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const extraPrefix = "198.51.100.0/24"
|
|
||||||
|
|
||||||
store := newBlockingStore()
|
|
||||||
h := NewPrefixHandler(store, logger.New(), routetable.New(routetable.Params{}))
|
|
||||||
fill := slices.Repeat([]string{testV4Prefix}, prefixBatchSize)
|
|
||||||
handleWhileAWriteIsBlocked(t, store, h,
|
|
||||||
[]*ristypes.RISMessage{prefixAnnouncement(fill)},
|
|
||||||
prefixAnnouncement([]string{extraPrefix}))
|
|
||||||
|
|
||||||
store.mu.Lock()
|
|
||||||
defer store.mu.Unlock()
|
|
||||||
|
|
||||||
if !store.prefixes[extraPrefix] {
|
|
||||||
t.Errorf("Stop did not write %s, handled while a write was blocked",
|
|
||||||
extraPrefix)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestASHandlerHandlesMessagesWhileAWriteIsBlocked(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
store := newBlockingStore()
|
|
||||||
h := newASHandler(store, logger.New(), longerThanAnyTest)
|
|
||||||
fill := ristypes.ASPath{Sequence: slices.Repeat([]int{testASNA}, asnBatchSize)}
|
|
||||||
handleWhileAWriteIsBlocked(t, store, h,
|
|
||||||
[]*ristypes.RISMessage{{Path: fill}},
|
|
||||||
&ristypes.RISMessage{Path: ristypes.ASPath{Sequence: []int{testASNB}}})
|
|
||||||
|
|
||||||
store.mu.Lock()
|
|
||||||
defer store.mu.Unlock()
|
|
||||||
|
|
||||||
if !store.asns[testASNB] {
|
|
||||||
t.Errorf("Stop did not write AS%d, handled while a write was blocked",
|
|
||||||
testASNB)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPeerHandlerHandlesMessagesWhileAWriteIsBlocked(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const otherPeerIP = "2001:db8:ffff::2"
|
|
||||||
|
|
||||||
store := newBlockingStore()
|
|
||||||
h := newPeerHandler(store, logger.New(), longerThanAnyTest)
|
|
||||||
handleWhileAWriteIsBlocked(t, store, h,
|
|
||||||
slices.Repeat([]*ristypes.RISMessage{{Peer: testPeerIP}}, peerBatchSize),
|
|
||||||
&ristypes.RISMessage{Peer: otherPeerIP})
|
|
||||||
|
|
||||||
store.mu.Lock()
|
|
||||||
defer store.mu.Unlock()
|
|
||||||
|
|
||||||
if !store.peers[otherPeerIP] {
|
|
||||||
t.Errorf("Stop did not write peer %s, handled while a write was blocked",
|
|
||||||
otherPeerIP)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPrefixHandlerWaitsAtTheMaximumBatchSize(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const extraPrefix = "198.51.100.0/24"
|
|
||||||
|
|
||||||
store := newBlockingStore()
|
|
||||||
h := newPrefixHandler(store, logger.New(), routetable.New(routetable.Params{}),
|
|
||||||
longerThanAnyTest, maxTableChangesPerWrite)
|
|
||||||
h.batchMaxSize = prefixBatchSize
|
|
||||||
fill := slices.Repeat([]string{testV4Prefix}, prefixBatchSize)
|
|
||||||
toMax := slices.Repeat([]string{testV4Prefix}, prefixBatchSize)
|
|
||||||
handleAtMaxBatchSize(t, store, h,
|
|
||||||
[]*ristypes.RISMessage{prefixAnnouncement(fill)},
|
|
||||||
[]*ristypes.RISMessage{prefixAnnouncement(toMax)},
|
|
||||||
prefixAnnouncement([]string{extraPrefix}))
|
|
||||||
|
|
||||||
store.mu.Lock()
|
|
||||||
defer store.mu.Unlock()
|
|
||||||
|
|
||||||
if !store.prefixes[extraPrefix] {
|
|
||||||
t.Errorf("Stop did not write %s, handled after the batch reached its "+
|
|
||||||
"maximum size", extraPrefix)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestASHandlerWaitsAtTheMaximumBatchSize(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
store := newBlockingStore()
|
|
||||||
h := newASHandler(store, logger.New(), longerThanAnyTest)
|
|
||||||
h.batchMaxSize = asnBatchSize
|
|
||||||
fill := ristypes.ASPath{Sequence: slices.Repeat([]int{testASNA}, asnBatchSize)}
|
|
||||||
toMax := ristypes.ASPath{Sequence: slices.Repeat([]int{testASNA}, asnBatchSize)}
|
|
||||||
handleAtMaxBatchSize(t, store, h,
|
|
||||||
[]*ristypes.RISMessage{{Path: fill}},
|
|
||||||
[]*ristypes.RISMessage{{Path: toMax}},
|
|
||||||
&ristypes.RISMessage{Path: ristypes.ASPath{Sequence: []int{testASNB}}})
|
|
||||||
|
|
||||||
store.mu.Lock()
|
|
||||||
defer store.mu.Unlock()
|
|
||||||
|
|
||||||
if !store.asns[testASNB] {
|
|
||||||
t.Errorf("Stop did not write AS%d, handled after the batch reached its "+
|
|
||||||
"maximum size", testASNB)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPeerHandlerWaitsAtTheMaximumBatchSize must not have its flush timer fire.
|
|
||||||
// A timer flush during the fill would take only part of it. The batch would
|
|
||||||
// then reach its maximum size before the test's last message, and the test
|
|
||||||
// would fail.
|
|
||||||
func TestPeerHandlerWaitsAtTheMaximumBatchSize(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const otherPeerIP = "2001:db8:ffff::2"
|
|
||||||
|
|
||||||
store := newBlockingStore()
|
|
||||||
h := newPeerHandler(store, logger.New(), longerThanAnyTest)
|
|
||||||
h.batchMaxSize = peerBatchSize
|
|
||||||
message := []*ristypes.RISMessage{{Peer: testPeerIP}}
|
|
||||||
handleAtMaxBatchSize(t, store, h,
|
|
||||||
slices.Repeat(message, peerBatchSize),
|
|
||||||
slices.Repeat(message, peerBatchSize),
|
|
||||||
&ristypes.RISMessage{Peer: otherPeerIP})
|
|
||||||
|
|
||||||
store.mu.Lock()
|
|
||||||
defer store.mu.Unlock()
|
|
||||||
|
|
||||||
if !store.peers[otherPeerIP] {
|
|
||||||
t.Errorf("Stop did not write peer %s, handled after the batch reached "+
|
|
||||||
"its maximum size", otherPeerIP)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandlersWakeTheFlushGoroutineWhenTheBatchIsFull hands each handler, built
|
|
||||||
// without its flush goroutine so that nothing takes a wake-up, one update short
|
|
||||||
// of a full batch, and a second one exactly a full batch. Only the full batch
|
|
||||||
// leaves a wake-up in batchFullCh.
|
|
||||||
func TestHandlersWakeTheFlushGoroutineWhenTheBatchIsFull(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
batchSize int
|
|
||||||
// handle hands a new handler n updates and returns its batchFullCh.
|
|
||||||
handle func(n int) chan struct{}
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "prefix",
|
|
||||||
batchSize: prefixBatchSize,
|
|
||||||
handle: func(n int) chan struct{} {
|
|
||||||
h := &PrefixHandler{
|
|
||||||
batchMaxSize: prefixBatchMaxSize,
|
|
||||||
batchFullCh: make(chan struct{}, 1),
|
|
||||||
table: routetable.New(routetable.Params{}),
|
|
||||||
}
|
|
||||||
h.HandleMessage(prefixAnnouncement(
|
|
||||||
slices.Repeat([]string{testV4Prefix}, n)))
|
|
||||||
|
|
||||||
return h.batchFullCh
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "AS",
|
|
||||||
batchSize: asnBatchSize,
|
|
||||||
handle: func(n int) chan struct{} {
|
|
||||||
h := &ASHandler{
|
|
||||||
batchMaxSize: asnBatchMaxSize,
|
|
||||||
batchFullCh: make(chan struct{}, 1),
|
|
||||||
}
|
|
||||||
// The origin AS and each AS in the path are one update each.
|
|
||||||
h.HandleMessage(&ristypes.RISMessage{
|
|
||||||
Path: ristypes.ASPath{Sequence: slices.Repeat([]int{testASNA}, n-1)},
|
|
||||||
})
|
|
||||||
|
|
||||||
return h.batchFullCh
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "peer",
|
|
||||||
batchSize: peerBatchSize,
|
|
||||||
handle: func(n int) chan struct{} {
|
|
||||||
h := &PeerHandler{
|
|
||||||
batchMaxSize: peerBatchMaxSize,
|
|
||||||
batchFullCh: make(chan struct{}, 1),
|
|
||||||
}
|
|
||||||
for range n {
|
|
||||||
h.HandleMessage(&ristypes.RISMessage{Peer: testPeerIP})
|
|
||||||
}
|
|
||||||
|
|
||||||
return h.batchFullCh
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
got := len(tt.handle(tt.batchSize - 1))
|
|
||||||
if got != 0 {
|
|
||||||
t.Errorf("one update short of a full batch left %d wake-ups, "+
|
|
||||||
"want none", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
got = len(tt.handle(tt.batchSize))
|
|
||||||
if got != 1 {
|
|
||||||
t.Errorf("a full batch left %d wake-ups, want 1", got)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+26
-36
@@ -3,8 +3,10 @@ package routewatch
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"os"
|
"os"
|
||||||
|
"os/signal"
|
||||||
"runtime"
|
"runtime"
|
||||||
"strings"
|
"strings"
|
||||||
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
"git.eeqj.de/sneak/routewatch/internal/logger"
|
||||||
@@ -12,20 +14,12 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
// shutdownTimeout is how long Shutdown has after a stop signal. upaas,
|
// shutdownTimeout is the maximum time allowed for graceful shutdown
|
||||||
// like docker stop by default, kills the container 10 seconds after the
|
shutdownTimeout = 60 * time.Second
|
||||||
// stop signal (README "Running under upaas").
|
|
||||||
shutdownTimeout = 8 * time.Second
|
|
||||||
// stopTimeout is fx's limit on the stop hook, past which fx exits with
|
|
||||||
// status 1: shutdownTimeout and time for Shutdown to close the HTTP
|
|
||||||
// server after its deadline.
|
|
||||||
stopTimeout = 9 * time.Second
|
|
||||||
// debugInterval is how often to log debug stats
|
// debugInterval is how often to log debug stats
|
||||||
debugInterval = 60 * time.Second
|
debugInterval = 60 * time.Second
|
||||||
// bytesPerMB is bytes per megabyte
|
// bytesPerMB is bytes per megabyte
|
||||||
bytesPerMB = 1024 * 1024
|
bytesPerMB = 1024 * 1024
|
||||||
// failureExitCode is the exit status when RouteWatch stops on an error
|
|
||||||
failureExitCode = 1
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// logDebugStats logs goroutine count and memory usage
|
// logDebugStats logs goroutine count and memory usage
|
||||||
@@ -60,47 +54,43 @@ func logDebugStats(logger *logger.Logger) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// CLIEntry is the main entry point for the routewatch command-line interface.
|
// CLIEntry is the main entry point for the routewatch command-line interface.
|
||||||
// It initializes the application using the fx dependency injection framework
|
// It initializes the application using the fx dependency injection framework,
|
||||||
// and starts the RouteWatch service. It blocks until fx receives SIGINT or
|
// sets up signal handling for graceful shutdown, and starts the RouteWatch service.
|
||||||
// SIGTERM or RouteWatch stops on an error; fx then runs the stop hook and sets
|
// This function blocks until the application receives a shutdown signal or encounters
|
||||||
// the exit code.
|
// a fatal error.
|
||||||
func CLIEntry() {
|
func CLIEntry() {
|
||||||
app := fx.New(
|
app := fx.New(
|
||||||
fx.Options(getModule()...),
|
getModule(),
|
||||||
fx.StopTimeout(stopTimeout),
|
fx.StopTimeout(shutdownTimeout), // Allow 60 seconds for graceful shutdown
|
||||||
fx.Invoke(func(
|
fx.Invoke(func(lc fx.Lifecycle, rw *RouteWatch, logger *logger.Logger, shutdowner fx.Shutdowner) {
|
||||||
lc fx.Lifecycle,
|
|
||||||
rw *RouteWatch,
|
|
||||||
logger *logger.Logger,
|
|
||||||
shutdowner fx.Shutdowner,
|
|
||||||
) {
|
|
||||||
lc.Append(fx.Hook{
|
lc.Append(fx.Hook{
|
||||||
OnStart: func(ctx context.Context) error {
|
OnStart: func(ctx context.Context) error {
|
||||||
// Start debug stats logging
|
// Start debug stats logging
|
||||||
go logDebugStats(logger)
|
go logDebugStats(logger)
|
||||||
|
|
||||||
go func() {
|
// Handle shutdown signals
|
||||||
err := rw.Run(ctx)
|
sigCh := make(chan os.Signal, 1)
|
||||||
if err != nil {
|
signal.Notify(sigCh, os.Interrupt, syscall.SIGTERM)
|
||||||
logger.Error("RouteWatch error", "error", err)
|
|
||||||
|
|
||||||
err = shutdowner.Shutdown(fx.ExitCode(failureExitCode))
|
go func() {
|
||||||
if err != nil {
|
<-sigCh
|
||||||
logger.Error("Failed to shutdown gracefully",
|
logger.Info("Received shutdown signal")
|
||||||
"error", err)
|
if err := shutdowner.Shutdown(); err != nil {
|
||||||
|
logger.Error("Failed to shutdown gracefully", "error", err)
|
||||||
}
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
if err := rw.Run(ctx); err != nil {
|
||||||
|
logger.Error("RouteWatch error", "error", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
OnStop: func(ctx context.Context) error {
|
OnStop: func(_ context.Context) error {
|
||||||
logger.Info("Shutting down RouteWatch")
|
logger.Info("Shutting down RouteWatch")
|
||||||
|
rw.Shutdown()
|
||||||
ctx, cancel := context.WithTimeout(ctx, shutdownTimeout)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
rw.Shutdown(ctx)
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
|
// Package routewatch contains the database maintainer for background maintenance tasks.
|
||||||
package routewatch
|
package routewatch
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
@@ -35,18 +35,12 @@ const (
|
|||||||
analyzeTimeout = 5 * time.Minute
|
analyzeTimeout = 5 * time.Minute
|
||||||
)
|
)
|
||||||
|
|
||||||
// checkpointIncompleteMessage is logged, at debug level like a completed
|
|
||||||
// checkpoint, when a checkpoint leaves part of the WAL for a later one: while
|
|
||||||
// reads overlap, most do. The database warns when the WAL file stays over its
|
|
||||||
// size limit.
|
|
||||||
const checkpointIncompleteMessage = "WAL checkpoint incomplete"
|
|
||||||
|
|
||||||
// DBMaintainer handles background database maintenance tasks.
|
// DBMaintainer handles background database maintenance tasks.
|
||||||
type DBMaintainer struct {
|
type DBMaintainer struct {
|
||||||
db database.Store
|
db database.Store
|
||||||
logger *slog.Logger
|
logger *slog.Logger
|
||||||
stopCh chan struct{}
|
stopCh chan struct{}
|
||||||
done chan struct{} // closed when the maintenance goroutine returns
|
wg sync.WaitGroup
|
||||||
|
|
||||||
// Stats tracking
|
// Stats tracking
|
||||||
statsMu sync.Mutex
|
statsMu sync.Mutex
|
||||||
@@ -67,15 +61,13 @@ func NewDBMaintainer(db database.Store, logger *slog.Logger) *DBMaintainer {
|
|||||||
db: db,
|
db: db,
|
||||||
logger: logger.With("component", "db_maintainer"),
|
logger: logger.With("component", "db_maintainer"),
|
||||||
stopCh: make(chan struct{}),
|
stopCh: make(chan struct{}),
|
||||||
done: make(chan struct{}),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Start begins the background maintenance goroutine. It runs until Stop:
|
// Start begins the background maintenance goroutine.
|
||||||
// the end of ctx does not stop it.
|
func (m *DBMaintainer) Start() {
|
||||||
func (m *DBMaintainer) Start(ctx context.Context) {
|
m.wg.Add(1)
|
||||||
go m.run(context.WithoutCancel(ctx))
|
go m.run()
|
||||||
|
|
||||||
m.logger.Info("Database maintainer started",
|
m.logger.Info("Database maintainer started",
|
||||||
"checkpoint_interval", checkpointInterval,
|
"checkpoint_interval", checkpointInterval,
|
||||||
"vacuum_interval", vacuumInterval,
|
"vacuum_interval", vacuumInterval,
|
||||||
@@ -83,32 +75,23 @@ func (m *DBMaintainer) Start(ctx context.Context) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stop stops the maintainer and waits, until ctx ends, for a maintenance task
|
// Stop gracefully shuts down the maintainer.
|
||||||
// in progress to finish.
|
func (m *DBMaintainer) Stop() {
|
||||||
func (m *DBMaintainer) Stop(ctx context.Context) {
|
|
||||||
close(m.stopCh)
|
close(m.stopCh)
|
||||||
|
m.wg.Wait()
|
||||||
select {
|
|
||||||
case <-m.done:
|
|
||||||
m.logger.Info("Database maintainer stopped")
|
m.logger.Info("Database maintainer stopped")
|
||||||
case <-ctx.Done():
|
|
||||||
m.logger.Warn("Stop deadline passed before the database maintainer " +
|
|
||||||
"stopped")
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// run is the main background loop.
|
// run is the main background loop.
|
||||||
func (m *DBMaintainer) run(ctx context.Context) {
|
func (m *DBMaintainer) run() {
|
||||||
defer close(m.done)
|
defer m.wg.Done()
|
||||||
|
|
||||||
// Use different timers for each task
|
// Use different timers for each task
|
||||||
checkpointTimer := time.NewTimer(checkpointInterval)
|
checkpointTimer := time.NewTimer(checkpointInterval)
|
||||||
defer checkpointTimer.Stop()
|
|
||||||
|
|
||||||
vacuumTimer := time.NewTimer(vacuumInterval)
|
vacuumTimer := time.NewTimer(vacuumInterval)
|
||||||
defer vacuumTimer.Stop()
|
|
||||||
|
|
||||||
analyzeTimer := time.NewTimer(analyzeInterval)
|
analyzeTimer := time.NewTimer(analyzeInterval)
|
||||||
|
defer checkpointTimer.Stop()
|
||||||
|
defer vacuumTimer.Stop()
|
||||||
defer analyzeTimer.Stop()
|
defer analyzeTimer.Stop()
|
||||||
|
|
||||||
for {
|
for {
|
||||||
@@ -117,92 +100,90 @@ func (m *DBMaintainer) run(ctx context.Context) {
|
|||||||
return
|
return
|
||||||
|
|
||||||
case <-checkpointTimer.C:
|
case <-checkpointTimer.C:
|
||||||
m.runCheckpoint(ctx)
|
m.runCheckpoint()
|
||||||
checkpointTimer.Reset(checkpointInterval)
|
checkpointTimer.Reset(checkpointInterval)
|
||||||
|
|
||||||
case <-vacuumTimer.C:
|
case <-vacuumTimer.C:
|
||||||
m.runVacuum(ctx)
|
m.runVacuum()
|
||||||
vacuumTimer.Reset(vacuumInterval)
|
vacuumTimer.Reset(vacuumInterval)
|
||||||
|
|
||||||
case <-analyzeTimer.C:
|
case <-analyzeTimer.C:
|
||||||
m.runAnalyze(ctx)
|
m.runAnalyze()
|
||||||
analyzeTimer.Reset(analyzeInterval)
|
analyzeTimer.Reset(analyzeInterval)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// runCheckpoint performs a WAL checkpoint to keep the WAL file small.
|
// runCheckpoint performs a WAL checkpoint to keep the WAL file small.
|
||||||
func (m *DBMaintainer) runCheckpoint(ctx context.Context) {
|
func (m *DBMaintainer) runCheckpoint() {
|
||||||
duration, err := runWithTimeout(ctx, checkpointTimeout, m.db.Checkpoint)
|
ctx, cancel := context.WithTimeout(context.Background(), checkpointTimeout)
|
||||||
m.recordResult(&m.lastCheckpoint, &m.lastCheckpointError, &m.checkpointCount, err)
|
defer cancel()
|
||||||
|
|
||||||
switch {
|
startTime := time.Now()
|
||||||
case errors.Is(err, database.ErrCheckpointIncomplete):
|
|
||||||
m.logger.Debug(checkpointIncompleteMessage,
|
err := m.db.Checkpoint(ctx)
|
||||||
"error", err, "duration", duration)
|
|
||||||
case err != nil:
|
m.statsMu.Lock()
|
||||||
m.logger.Error("WAL checkpoint failed", "error", err, "duration", duration)
|
m.lastCheckpoint = time.Now()
|
||||||
default:
|
m.lastCheckpointError = err
|
||||||
m.logger.Debug("WAL checkpoint completed", "duration", duration)
|
if err == nil {
|
||||||
|
m.checkpointCount++
|
||||||
|
}
|
||||||
|
m.statsMu.Unlock()
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
m.logger.Error("WAL checkpoint failed", "error", err, "duration", time.Since(startTime))
|
||||||
|
} else {
|
||||||
|
m.logger.Debug("WAL checkpoint completed", "duration", time.Since(startTime))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// runVacuum performs an incremental vacuum operation on the database.
|
// runVacuum performs an incremental vacuum operation on the database.
|
||||||
func (m *DBMaintainer) runVacuum(ctx context.Context) {
|
func (m *DBMaintainer) runVacuum() {
|
||||||
m.logger.Debug("Running incremental vacuum")
|
ctx, cancel := context.WithTimeout(context.Background(), vacuumTimeout)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
duration, err := runWithTimeout(ctx, vacuumTimeout, m.db.Vacuum)
|
m.logger.Debug("Running incremental vacuum")
|
||||||
m.recordResult(&m.lastVacuum, &m.lastVacuumError, &m.vacuumCount, err)
|
startTime := time.Now()
|
||||||
|
|
||||||
|
err := m.db.Vacuum(ctx)
|
||||||
|
|
||||||
|
m.statsMu.Lock()
|
||||||
|
m.lastVacuum = time.Now()
|
||||||
|
m.lastVacuumError = err
|
||||||
|
if err == nil {
|
||||||
|
m.vacuumCount++
|
||||||
|
}
|
||||||
|
m.statsMu.Unlock()
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
m.logger.Error("Incremental vacuum failed", "error", err, "duration", duration)
|
m.logger.Error("Incremental vacuum failed", "error", err, "duration", time.Since(startTime))
|
||||||
} else {
|
} else {
|
||||||
m.logger.Debug("Incremental vacuum completed", "duration", duration)
|
m.logger.Debug("Incremental vacuum completed", "duration", time.Since(startTime))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// runAnalyze performs an ANALYZE operation on the database.
|
// runAnalyze performs an ANALYZE operation on the database.
|
||||||
func (m *DBMaintainer) runAnalyze(ctx context.Context) {
|
func (m *DBMaintainer) runAnalyze() {
|
||||||
m.logger.Info("Starting database ANALYZE")
|
ctx, cancel := context.WithTimeout(context.Background(), analyzeTimeout)
|
||||||
|
|
||||||
duration, err := runWithTimeout(ctx, analyzeTimeout, m.db.Analyze)
|
|
||||||
m.recordResult(&m.lastAnalyze, &m.lastAnalyzeError, &m.analyzeCount, err)
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
m.logger.Error("ANALYZE failed", "error", err, "duration", duration)
|
|
||||||
} else {
|
|
||||||
m.logger.Info("ANALYZE completed", "duration", duration)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// runWithTimeout runs operation with a context that ends after timeout and
|
|
||||||
// returns how long it took and its error.
|
|
||||||
func runWithTimeout(
|
|
||||||
ctx context.Context,
|
|
||||||
timeout time.Duration,
|
|
||||||
operation func(context.Context) error,
|
|
||||||
) (time.Duration, error) {
|
|
||||||
ctx, cancel := context.WithTimeout(ctx, timeout)
|
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
|
m.logger.Info("Starting database ANALYZE")
|
||||||
startTime := time.Now()
|
startTime := time.Now()
|
||||||
err := operation(ctx)
|
|
||||||
|
|
||||||
return time.Since(startTime), err
|
err := m.db.Analyze(ctx)
|
||||||
}
|
|
||||||
|
|
||||||
// recordResult stores when a maintenance task last ran and its error, and
|
|
||||||
// counts it when it succeeded.
|
|
||||||
func (m *DBMaintainer) recordResult(
|
|
||||||
lastRun *time.Time, lastErr *error, successCount *int, err error,
|
|
||||||
) {
|
|
||||||
m.statsMu.Lock()
|
m.statsMu.Lock()
|
||||||
defer m.statsMu.Unlock()
|
m.lastAnalyze = time.Now()
|
||||||
|
m.lastAnalyzeError = err
|
||||||
*lastRun = time.Now()
|
|
||||||
*lastErr = err
|
|
||||||
|
|
||||||
if err == nil {
|
if err == nil {
|
||||||
*successCount++
|
m.analyzeCount++
|
||||||
|
}
|
||||||
|
m.statsMu.Unlock()
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
m.logger.Error("ANALYZE failed", "error", err, "duration", time.Since(startTime))
|
||||||
|
} else {
|
||||||
|
m.logger.Info("ANALYZE completed", "duration", time.Since(startTime))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,120 +0,0 @@
|
|||||||
package routewatch
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"log/slog"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// incompleteCheckpointStore is a store whose every checkpoint leaves part of
|
|
||||||
// the WAL uncopied.
|
|
||||||
type incompleteCheckpointStore struct{ *mockStore }
|
|
||||||
|
|
||||||
// Checkpoint reports the checkpoint incomplete.
|
|
||||||
func (incompleteCheckpointStore) Checkpoint(context.Context) error {
|
|
||||||
return database.ErrCheckpointIncomplete
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestIncompleteCheckpointNotLoggedAsCompleted runs a checkpoint that leaves
|
|
||||||
// part of the WAL uncopied and checks that the maintainer logs it as
|
|
||||||
// incomplete, not as completed, and does not count it as a checkpoint that
|
|
||||||
// completed.
|
|
||||||
func TestIncompleteCheckpointNotLoggedAsCompleted(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var logs bytes.Buffer
|
|
||||||
|
|
||||||
logger := slog.New(slog.NewJSONHandler(&logs,
|
|
||||||
&slog.HandlerOptions{Level: slog.LevelDebug}))
|
|
||||||
m := NewDBMaintainer(incompleteCheckpointStore{newMockStore()}, logger)
|
|
||||||
|
|
||||||
m.runCheckpoint(t.Context())
|
|
||||||
|
|
||||||
var logged struct {
|
|
||||||
Msg string `json:"msg"`
|
|
||||||
}
|
|
||||||
|
|
||||||
err := json.Unmarshal(logs.Bytes(), &logged)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("want one log message, got %q: %v", logs.String(), err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if logged.Msg != checkpointIncompleteMessage {
|
|
||||||
t.Errorf("logged %q, want %q", logged.Msg, checkpointIncompleteMessage)
|
|
||||||
}
|
|
||||||
|
|
||||||
if m.checkpointCount != 0 {
|
|
||||||
t.Errorf("checkpointCount = %d after an incomplete checkpoint, want 0",
|
|
||||||
m.checkpointCount)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// heldCheckpointStore is a store whose checkpoint closes started and then
|
|
||||||
// waits until release is closed.
|
|
||||||
type heldCheckpointStore struct {
|
|
||||||
*mockStore
|
|
||||||
|
|
||||||
started chan struct{}
|
|
||||||
release chan struct{}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Checkpoint waits until release is closed.
|
|
||||||
func (s heldCheckpointStore) Checkpoint(context.Context) error {
|
|
||||||
close(s.started)
|
|
||||||
<-s.release
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestDBMaintainerStopReturnsByItsDeadline stops the maintainer with a short
|
|
||||||
// deadline while its goroutine runs a checkpoint that the store holds until
|
|
||||||
// the test ends. Stop must return all the same instead of waiting for the
|
|
||||||
// checkpoint.
|
|
||||||
func TestDBMaintainerStopReturnsByItsDeadline(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const stopDeadline = 100 * time.Millisecond
|
|
||||||
|
|
||||||
store := heldCheckpointStore{
|
|
||||||
mockStore: newMockStore(),
|
|
||||||
started: make(chan struct{}),
|
|
||||||
release: make(chan struct{}),
|
|
||||||
}
|
|
||||||
m := NewDBMaintainer(store, slog.Default())
|
|
||||||
|
|
||||||
// Start would run the first checkpoint only after checkpointInterval.
|
|
||||||
// This goroutine stands for the one Start runs and checkpoints at once.
|
|
||||||
go func() {
|
|
||||||
defer close(m.done)
|
|
||||||
|
|
||||||
m.runCheckpoint(t.Context())
|
|
||||||
}()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-store.started:
|
|
||||||
case <-time.After(stepTimeout):
|
|
||||||
t.Fatal("the maintainer did not start the checkpoint")
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(t.Context(), stopDeadline)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
if !returnsInTime(func() { m.Stop(ctx) }) {
|
|
||||||
t.Fatalf("Stop with a deadline of %s did not return while a "+
|
|
||||||
"checkpoint was in progress", stopDeadline)
|
|
||||||
}
|
|
||||||
|
|
||||||
close(store.release)
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-m.done:
|
|
||||||
case <-time.After(stepTimeout):
|
|
||||||
t.Fatal("the maintenance goroutine did not return after the " +
|
|
||||||
"checkpoint ended")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,8 +1,6 @@
|
|||||||
package routewatch
|
package routewatch
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"slices"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
"git.eeqj.de/sneak/routewatch/internal/logger"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/ristypes"
|
"git.eeqj.de/sneak/routewatch/internal/ristypes"
|
||||||
)
|
)
|
||||||
@@ -17,11 +15,10 @@ type SimpleHandler struct {
|
|||||||
callback func(*ristypes.RISMessage)
|
callback func(*ristypes.RISMessage)
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewSimpleHandler creates a new SimpleHandler that accepts specific message
|
// NewSimpleHandler creates a new SimpleHandler that accepts specific message types.
|
||||||
// types. The messageTypes parameter specifies which RIS message types this
|
// The messageTypes parameter specifies which RIS message types this handler will process.
|
||||||
// handler will process. If messageTypes is empty, the handler will accept all
|
// If messageTypes is empty, the handler will accept all message types.
|
||||||
// message types. The callback function is invoked for each message that
|
// The callback function is invoked for each message that passes the type filter.
|
||||||
// passes the type filter.
|
|
||||||
func NewSimpleHandler(
|
func NewSimpleHandler(
|
||||||
logger *logger.Logger,
|
logger *logger.Logger,
|
||||||
messageTypes []string,
|
messageTypes []string,
|
||||||
@@ -34,17 +31,22 @@ func NewSimpleHandler(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// WantsMessage returns true if this handler wants to process messages of the
|
// WantsMessage returns true if this handler wants to process messages of the given type.
|
||||||
// given type. It checks whether messageType is in the handler's configured
|
// It checks whether messageType is in the handler's configured list of accepted types.
|
||||||
// list of accepted types. If no specific types were configured (empty
|
// If no specific types were configured (empty messageTypes slice), it returns true for all types.
|
||||||
// messageTypes slice), it returns true for all types.
|
|
||||||
func (h *SimpleHandler) WantsMessage(messageType string) bool {
|
func (h *SimpleHandler) WantsMessage(messageType string) bool {
|
||||||
// If no specific types are set, accept all messages
|
// If no specific types are set, accept all messages
|
||||||
if len(h.messageTypes) == 0 {
|
if len(h.messageTypes) == 0 {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
return slices.Contains(h.messageTypes, messageType)
|
for _, t := range h.messageTypes {
|
||||||
|
if t == messageType {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// HandleMessage processes a RIS message by invoking the configured callback function.
|
// HandleMessage processes a RIS message by invoking the configured callback function.
|
||||||
|
|||||||
@@ -1,11 +1,9 @@
|
|||||||
package routewatch
|
package routewatch
|
||||||
|
|
||||||
// peerhandler.go provides batched peer tracking functionality for BGP route
|
// peerhandler.go provides batched peer tracking functionality for BGP route monitoring.
|
||||||
// monitoring. It tracks BGP peers from incoming UPDATE messages and maintains
|
// It tracks BGP peers from all incoming RIS messages and maintains peer state in the database.
|
||||||
// peer state in the database.
|
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"strconv"
|
"strconv"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
@@ -25,45 +23,24 @@ const (
|
|||||||
// peerBatchSize is the number of peer updates to batch together
|
// peerBatchSize is the number of peer updates to batch together
|
||||||
peerBatchSize = 10000
|
peerBatchSize = 10000
|
||||||
|
|
||||||
// peerBatchMaxSize is the most peer updates the batch holds while the
|
|
||||||
// flush goroutine writes the previous one: ten batches, about 10 MiB
|
|
||||||
// (README "Memory"). At this size HandleMessage waits until the flush
|
|
||||||
// goroutine takes the batch.
|
|
||||||
peerBatchMaxSize = 100000
|
|
||||||
|
|
||||||
// peerBatchTimeout is the maximum time to wait before flushing a batch
|
// peerBatchTimeout is the maximum time to wait before flushing a batch
|
||||||
peerBatchTimeout = 2 * time.Second
|
peerBatchTimeout = 2 * time.Second
|
||||||
)
|
)
|
||||||
|
|
||||||
// PeerHandler tracks BGP peers from UPDATE messages using batched operations.
|
// PeerHandler tracks BGP peers from all message types using batched operations.
|
||||||
// It maintains a queue of peer updates and periodically flushes them to the database
|
// It maintains a queue of peer updates and periodically flushes them to the database
|
||||||
// in batches to improve performance. The handler writes one row per peer IP
|
// in batches to improve performance. The handler deduplicates peer updates within
|
||||||
// address for each batch, with the earliest and the latest time of its updates.
|
// each batch, keeping only the most recent update for each peer IP address.
|
||||||
type PeerHandler struct {
|
type PeerHandler struct {
|
||||||
db database.Store
|
db database.Store
|
||||||
logger *logger.Logger
|
logger *logger.Logger
|
||||||
|
|
||||||
// batchMaxSize is peerBatchMaxSize. Only HandleMessage reads it, so a test
|
// Batching
|
||||||
// may lower it before it hands the handler a message.
|
|
||||||
batchMaxSize int
|
|
||||||
|
|
||||||
// Batching. mu guards peerBatch and writing only: the flush goroutine
|
|
||||||
// takes the batch under mu and writes it after releasing mu, so
|
|
||||||
// HandleMessage waits for a database write only while peerBatch is at
|
|
||||||
// batchMaxSize.
|
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
peerBatch []peerUpdate
|
peerBatch []peerUpdate
|
||||||
// writing is how many updates the flush goroutine took from peerBatch
|
lastFlush time.Time
|
||||||
// and has not finished writing.
|
stopCh chan struct{}
|
||||||
writing int
|
wg sync.WaitGroup
|
||||||
// batchTaken is signalled when the flush goroutine takes peerBatch; its L
|
|
||||||
// is &mu.
|
|
||||||
batchTaken sync.Cond
|
|
||||||
batchFullCh chan struct{} // wakes the flush goroutine when peerBatch is full
|
|
||||||
// stopCh hands Stop's context to the flush goroutine. It holds one, so
|
|
||||||
// Stop does not wait for a write in progress to hand it over.
|
|
||||||
stopCh chan context.Context
|
|
||||||
done chan struct{} // closed when the flush goroutine returns
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type peerUpdate struct {
|
type peerUpdate struct {
|
||||||
@@ -78,37 +55,26 @@ type peerUpdate struct {
|
|||||||
// periodically flushes accumulated peer updates to the database. The handler
|
// periodically flushes accumulated peer updates to the database. The handler
|
||||||
// should be stopped by calling Stop when it is no longer needed.
|
// should be stopped by calling Stop when it is no longer needed.
|
||||||
func NewPeerHandler(db database.Store, logger *logger.Logger) *PeerHandler {
|
func NewPeerHandler(db database.Store, logger *logger.Logger) *PeerHandler {
|
||||||
return newPeerHandler(db, logger, peerBatchTimeout)
|
|
||||||
}
|
|
||||||
|
|
||||||
// newPeerHandler is NewPeerHandler with the flush timer's interval,
|
|
||||||
// batchTimeout, in place of peerBatchTimeout. A test that the timer must not
|
|
||||||
// disturb gives an interval longer than the test runs.
|
|
||||||
func newPeerHandler(
|
|
||||||
db database.Store, logger *logger.Logger, batchTimeout time.Duration,
|
|
||||||
) *PeerHandler {
|
|
||||||
h := &PeerHandler{
|
h := &PeerHandler{
|
||||||
db: db,
|
db: db,
|
||||||
logger: logger,
|
logger: logger,
|
||||||
batchMaxSize: peerBatchMaxSize,
|
|
||||||
peerBatch: make([]peerUpdate, 0, peerBatchSize),
|
peerBatch: make([]peerUpdate, 0, peerBatchSize),
|
||||||
batchFullCh: make(chan struct{}, 1),
|
lastFlush: time.Now(),
|
||||||
stopCh: make(chan context.Context, 1),
|
stopCh: make(chan struct{}),
|
||||||
done: make(chan struct{}),
|
|
||||||
}
|
}
|
||||||
h.batchTaken.L = &h.mu
|
|
||||||
|
|
||||||
// Start the flush timer goroutine
|
// Start the flush timer goroutine
|
||||||
go flushLoop(batchTimeout, h.batchFullCh, h.stopCh, h.done, h.flushBatch)
|
h.wg.Add(1)
|
||||||
|
go h.flushLoop()
|
||||||
|
|
||||||
return h
|
return h
|
||||||
}
|
}
|
||||||
|
|
||||||
// WantsMessage returns true for UPDATE messages only. A peer whose session
|
// WantsMessage returns true for all message types since peer information
|
||||||
// never comes up sends STATE messages and nothing else, so a STATE message is
|
// is extracted from every RIS message regardless of type. This satisfies
|
||||||
// not a sign of a peer that carries routes.
|
// the MessageHandler interface.
|
||||||
func (h *PeerHandler) WantsMessage(messageType string) bool {
|
func (h *PeerHandler) WantsMessage(_ string) bool {
|
||||||
return messageType == ristypes.MessageTypeUpdate
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
// QueueCapacity returns the desired queue capacity for this handler.
|
// QueueCapacity returns the desired queue capacity for this handler.
|
||||||
@@ -121,15 +87,13 @@ func (h *PeerHandler) QueueCapacity() int {
|
|||||||
|
|
||||||
// HandleMessage processes a RIS message to track peer information.
|
// HandleMessage processes a RIS message to track peer information.
|
||||||
// It extracts the peer IP address and ASN from the message and adds
|
// It extracts the peer IP address and ASN from the message and adds
|
||||||
// the update to an internal batch. The flush goroutine writes the batch to the
|
// the update to an internal batch. When the batch reaches peerBatchSize
|
||||||
// database when it reaches peerBatchSize or the batch timeout expires.
|
// or the batch timeout expires, the batch is flushed to the database.
|
||||||
func (h *PeerHandler) HandleMessage(msg *ristypes.RISMessage) {
|
func (h *PeerHandler) HandleMessage(msg *ristypes.RISMessage) {
|
||||||
// Parse peer ASN from string
|
// Parse peer ASN from string
|
||||||
peerASN := 0
|
peerASN := 0
|
||||||
|
|
||||||
if msg.PeerASN != "" {
|
if msg.PeerASN != "" {
|
||||||
asn, err := strconv.Atoi(msg.PeerASN)
|
if asn, err := strconv.Atoi(msg.PeerASN); err == nil {
|
||||||
if err == nil {
|
|
||||||
peerASN = asn
|
peerASN = asn
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -137,13 +101,6 @@ func (h *PeerHandler) HandleMessage(msg *ristypes.RISMessage) {
|
|||||||
h.mu.Lock()
|
h.mu.Lock()
|
||||||
defer h.mu.Unlock()
|
defer h.mu.Unlock()
|
||||||
|
|
||||||
// While the batch is at its maximum size, wait until the flush goroutine
|
|
||||||
// takes it. Meanwhile the handler takes nothing off its queue, which
|
|
||||||
// fills, and the streamer drops messages for it.
|
|
||||||
for len(h.peerBatch) >= h.batchMaxSize {
|
|
||||||
h.batchTaken.Wait()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add to batch
|
// Add to batch
|
||||||
h.peerBatch = append(h.peerBatch, peerUpdate{
|
h.peerBatch = append(h.peerBatch, peerUpdate{
|
||||||
peerIP: msg.Peer,
|
peerIP: msg.Peer,
|
||||||
@@ -152,91 +109,74 @@ func (h *PeerHandler) HandleMessage(msg *ristypes.RISMessage) {
|
|||||||
timestamp: msg.ParsedTimestamp,
|
timestamp: msg.ParsedTimestamp,
|
||||||
})
|
})
|
||||||
|
|
||||||
// A full batch wakes the flush goroutine. If it is writing, the wake-up
|
// Check if we need to flush
|
||||||
// waits in batchFullCh and the batch keeps growing until the write ends
|
|
||||||
// or the batch reaches batchMaxSize.
|
|
||||||
if len(h.peerBatch) >= peerBatchSize {
|
if len(h.peerBatch) >= peerBatchSize {
|
||||||
select {
|
h.flushBatchLocked()
|
||||||
case h.batchFullCh <- struct{}{}:
|
|
||||||
default: // already woken
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stop stops the flush goroutine, which writes the updates the handler holds
|
// flushLoop runs in a goroutine and periodically flushes batches
|
||||||
// before it returns, and waits for it until ctx ends. Once ctx has ended the
|
func (h *PeerHandler) flushLoop() {
|
||||||
// flush goroutine skips that last write, and a write still running is lost
|
defer h.wg.Done()
|
||||||
// when the process exits, so Stop logs how many updates are not written.
|
ticker := time.NewTicker(peerBatchTimeout)
|
||||||
func (h *PeerHandler) Stop(ctx context.Context) {
|
defer ticker.Stop()
|
||||||
h.stopCh <- ctx
|
|
||||||
|
|
||||||
|
for {
|
||||||
select {
|
select {
|
||||||
case <-h.done:
|
case <-ticker.C:
|
||||||
case <-ctx.Done():
|
|
||||||
}
|
|
||||||
|
|
||||||
h.mu.Lock()
|
h.mu.Lock()
|
||||||
unwritten := h.writing + len(h.peerBatch)
|
if time.Since(h.lastFlush) >= peerBatchTimeout {
|
||||||
|
h.flushBatchLocked()
|
||||||
|
}
|
||||||
|
h.mu.Unlock()
|
||||||
|
case <-h.stopCh:
|
||||||
|
// Final flush
|
||||||
|
h.mu.Lock()
|
||||||
|
h.flushBatchLocked()
|
||||||
h.mu.Unlock()
|
h.mu.Unlock()
|
||||||
|
|
||||||
if unwritten > 0 {
|
return
|
||||||
h.logger.Error("Stop deadline passed before the peer handler wrote "+
|
}
|
||||||
"its updates", "updates", unwritten)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// takeBatch returns the pending updates and leaves an empty batch in their
|
// flushBatchLocked flushes the peer batch to the database (must be called with mutex held)
|
||||||
// place for HandleMessage to fill. They count in writing until flushBatch has
|
func (h *PeerHandler) flushBatchLocked() {
|
||||||
// written them.
|
|
||||||
func (h *PeerHandler) takeBatch() []peerUpdate {
|
|
||||||
h.mu.Lock()
|
|
||||||
defer h.mu.Unlock()
|
|
||||||
|
|
||||||
if len(h.peerBatch) == 0 {
|
if len(h.peerBatch) == 0 {
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
batch := h.peerBatch
|
|
||||||
h.writing = len(batch)
|
|
||||||
h.peerBatch = make([]peerUpdate, 0, peerBatchSize)
|
|
||||||
h.batchTaken.Broadcast()
|
|
||||||
|
|
||||||
return batch
|
|
||||||
}
|
|
||||||
|
|
||||||
// flushBatch takes the pending batch and writes it to the database, without
|
|
||||||
// holding the mutex during the write.
|
|
||||||
func (h *PeerHandler) flushBatch() {
|
|
||||||
batch := h.takeBatch()
|
|
||||||
if len(batch) == 0 {
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// The earliest and the latest time the batch saw each peer, with the peer
|
// Deduplicate by peer IP, keeping the latest update for each peer
|
||||||
// AS and message type of its latest update. The batch holds several
|
peerMap := make(map[string]peerUpdate)
|
||||||
// updates for a peer, so a new peer is stored first seen at the earliest of
|
for _, update := range h.peerBatch {
|
||||||
// them and last seen at the latest.
|
if existing, ok := peerMap[update.peerIP]; !ok || update.timestamp.After(existing.timestamp) {
|
||||||
peers := make(map[string]database.PeerUpdate)
|
peerMap[update.peerIP] = update
|
||||||
for _, update := range batch {
|
}
|
||||||
peer, ok := peers[update.peerIP]
|
|
||||||
if !ok || update.timestamp.After(peer.Seen.Last) {
|
|
||||||
peer.PeerIP = update.peerIP
|
|
||||||
peer.PeerASN = update.peerASN
|
|
||||||
peer.MessageType = update.messageType
|
|
||||||
}
|
}
|
||||||
|
|
||||||
peer.Seen.Add(update.timestamp)
|
// Convert to database format
|
||||||
peers[update.peerIP] = peer
|
dbPeerMap := make(map[string]database.PeerUpdate)
|
||||||
|
for peerIP, update := range peerMap {
|
||||||
|
dbPeerMap[peerIP] = database.PeerUpdate{
|
||||||
|
PeerIP: update.peerIP,
|
||||||
|
PeerASN: update.peerASN,
|
||||||
|
MessageType: update.messageType,
|
||||||
|
Timestamp: update.timestamp,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Process all peers in a single batch transaction
|
// Process all peers in a single batch transaction
|
||||||
err := h.db.UpdatePeerBatch(peers)
|
if err := h.db.UpdatePeerBatch(dbPeerMap); err != nil {
|
||||||
if err != nil {
|
h.logger.Error("Failed to process peer batch", "error", err, "count", len(dbPeerMap))
|
||||||
h.logger.Error("Failed to process peer batch",
|
|
||||||
"error", err, "count", len(peers))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
h.mu.Lock()
|
// Clear batch
|
||||||
h.writing = 0
|
h.peerBatch = h.peerBatch[:0]
|
||||||
h.mu.Unlock()
|
h.lastFlush = time.Now()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop gracefully stops the handler and flushes remaining batches
|
||||||
|
func (h *PeerHandler) Stop() {
|
||||||
|
close(h.stopCh)
|
||||||
|
h.wg.Wait()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,59 +0,0 @@
|
|||||||
package routewatch
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/database"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
|
||||||
)
|
|
||||||
|
|
||||||
// peerBatchStore keeps the peers of the last UpdatePeerBatch call. The
|
|
||||||
// database has no read of the peer table, so the peer handler's test checks
|
|
||||||
// what the handler writes.
|
|
||||||
type peerBatchStore struct {
|
|
||||||
*mockStore
|
|
||||||
|
|
||||||
peers map[string]database.PeerUpdate
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *peerBatchStore) UpdatePeerBatch(
|
|
||||||
peers map[string]database.PeerUpdate,
|
|
||||||
) error {
|
|
||||||
s.peers = peers
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPeerHandlerWritesTheEarliestAndLatestTimesOfANewPeer handles two updates
|
|
||||||
// from one peer, the second 5 seconds after the first, in one batch. The peer
|
|
||||||
// handler writes the peer first seen at the time of the first update and last
|
|
||||||
// seen at the time of the second.
|
|
||||||
func TestPeerHandlerWritesTheEarliestAndLatestTimesOfANewPeer(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
store := &peerBatchStore{mockStore: newMockStore()}
|
|
||||||
|
|
||||||
// Built without NewPeerHandler's flush goroutine, so the test flushes
|
|
||||||
// itself.
|
|
||||||
h := &PeerHandler{db: store, logger: logger.New(), batchMaxSize: peerBatchMaxSize}
|
|
||||||
|
|
||||||
first := peerAnnouncement(testPeerIP, testV4Prefix)
|
|
||||||
second := peerAnnouncement(testPeerIP, testV4Prefix)
|
|
||||||
second.ParsedTimestamp = first.ParsedTimestamp.Add(secondUpdateAfter)
|
|
||||||
|
|
||||||
h.HandleMessage(first)
|
|
||||||
h.HandleMessage(second)
|
|
||||||
h.flushBatch()
|
|
||||||
|
|
||||||
seen := store.peers[testPeerIP].Seen
|
|
||||||
|
|
||||||
if !seen.First.Equal(first.ParsedTimestamp) {
|
|
||||||
t.Errorf("first_seen of peer %s = %s, want %s", testPeerIP,
|
|
||||||
seen.First, first.ParsedTimestamp)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !seen.Last.Equal(second.ParsedTimestamp) {
|
|
||||||
t.Errorf("last_seen of peer %s = %s, want %s", testPeerIP,
|
|
||||||
seen.Last, second.ParsedTimestamp)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,7 +1,6 @@
|
|||||||
package routewatch
|
package routewatch
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
@@ -42,23 +41,12 @@ type PeeringHandler struct {
|
|||||||
db database.Store
|
db database.Store
|
||||||
logger *logger.Logger
|
logger *logger.Logger
|
||||||
|
|
||||||
// maxPaths is the most paths asPaths holds: maxTrackedPaths, except in
|
// In-memory AS path tracking
|
||||||
// tests that give a smaller one.
|
|
||||||
maxPaths int
|
|
||||||
|
|
||||||
// In-memory AS path tracking: each path, JSON-encoded, with the earliest
|
|
||||||
// and the latest time an update carried it.
|
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
asPaths map[string]database.SeenTimes
|
asPaths map[string]time.Time // key is JSON-encoded AS path
|
||||||
droppedPaths int // paths dropped since the last run because the map was full
|
droppedPaths int // paths dropped because the map was full
|
||||||
// writing is how many paths processPeerings took from asPaths and has
|
|
||||||
// not finished writing the peerings of.
|
|
||||||
writing int
|
|
||||||
|
|
||||||
// stopCh hands Stop's context to the processing goroutine. It holds one,
|
stopCh chan struct{}
|
||||||
// so Stop does not wait for a write in progress to hand it over.
|
|
||||||
stopCh chan context.Context
|
|
||||||
done chan struct{} // closed when the processing goroutine returns
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewPeeringHandler creates and initializes a new PeeringHandler with the
|
// NewPeeringHandler creates and initializes a new PeeringHandler with the
|
||||||
@@ -66,28 +54,15 @@ type PeeringHandler struct {
|
|||||||
// periodically processes accumulated AS paths into peering records. The
|
// periodically processes accumulated AS paths into peering records. The
|
||||||
// handler begins processing immediately upon creation.
|
// handler begins processing immediately upon creation.
|
||||||
func NewPeeringHandler(db database.Store, logger *logger.Logger) *PeeringHandler {
|
func NewPeeringHandler(db database.Store, logger *logger.Logger) *PeeringHandler {
|
||||||
return newPeeringHandler(db, logger, peeringProcessInterval, maxTrackedPaths)
|
|
||||||
}
|
|
||||||
|
|
||||||
// newPeeringHandler is NewPeeringHandler with the processing timer's interval,
|
|
||||||
// processInterval, in place of peeringProcessInterval, and the most paths it
|
|
||||||
// holds, maxPaths, in place of maxTrackedPaths. A test gives a short interval to
|
|
||||||
// have the timer start a write, and a small maxPaths to fill the map quickly.
|
|
||||||
func newPeeringHandler(
|
|
||||||
db database.Store, logger *logger.Logger, processInterval time.Duration,
|
|
||||||
maxPaths int,
|
|
||||||
) *PeeringHandler {
|
|
||||||
h := &PeeringHandler{
|
h := &PeeringHandler{
|
||||||
db: db,
|
db: db,
|
||||||
logger: logger,
|
logger: logger,
|
||||||
asPaths: make(map[string]database.SeenTimes),
|
asPaths: make(map[string]time.Time),
|
||||||
maxPaths: maxPaths,
|
stopCh: make(chan struct{}),
|
||||||
stopCh: make(chan context.Context, 1),
|
|
||||||
done: make(chan struct{}),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Start the periodic processing goroutine
|
// Start the periodic processing goroutine
|
||||||
go h.processLoop(processInterval)
|
go h.processLoop()
|
||||||
|
|
||||||
return h
|
return h
|
||||||
}
|
}
|
||||||
@@ -97,7 +72,7 @@ func newPeeringHandler(
|
|||||||
// the AS path information needed to extract peering relationships.
|
// the AS path information needed to extract peering relationships.
|
||||||
func (h *PeeringHandler) WantsMessage(messageType string) bool {
|
func (h *PeeringHandler) WantsMessage(messageType string) bool {
|
||||||
// We only care about UPDATE messages that have AS paths
|
// We only care about UPDATE messages that have AS paths
|
||||||
return messageType == ristypes.MessageTypeUpdate
|
return messageType == "UPDATE"
|
||||||
}
|
}
|
||||||
|
|
||||||
// QueueCapacity returns the buffer size for the handler's message queue.
|
// QueueCapacity returns the buffer size for the handler's message queue.
|
||||||
@@ -112,41 +87,51 @@ func (h *PeeringHandler) QueueCapacity() int {
|
|||||||
// than minPathLengthForPeering are ignored as they cannot contain valid
|
// than minPathLengthForPeering are ignored as they cannot contain valid
|
||||||
// peering information.
|
// peering information.
|
||||||
func (h *PeeringHandler) HandleMessage(msg *ristypes.RISMessage) {
|
func (h *PeeringHandler) HandleMessage(msg *ristypes.RISMessage) {
|
||||||
// Peerings are between neighbours in the path's sequence only: the
|
|
||||||
// members of an AS set that ends the path are not neighbours of each
|
|
||||||
// other or of the AS before them.
|
|
||||||
sequence := msg.Path.Sequence
|
|
||||||
|
|
||||||
// Skip if no AS path or only one AS
|
// Skip if no AS path or only one AS
|
||||||
if len(sequence) < minPathLengthForPeering {
|
if len(msg.Path) < minPathLengthForPeering {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
timestamp := msg.ParsedTimestamp
|
timestamp := msg.ParsedTimestamp
|
||||||
|
|
||||||
// Encode AS path as JSON for use as map key
|
// Encode AS path as JSON for use as map key
|
||||||
pathJSON, err := json.Marshal(sequence)
|
pathJSON, err := json.Marshal(msg.Path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logger.Error("Failed to encode AS path", "error", err)
|
h.logger.Error("Failed to encode AS path", "error", err)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
key := string(pathJSON)
|
key := string(pathJSON)
|
||||||
|
|
||||||
h.mu.Lock()
|
h.mu.Lock()
|
||||||
defer h.mu.Unlock()
|
if _, exists := h.asPaths[key]; exists {
|
||||||
|
// Already tracked: refresh its timestamp.
|
||||||
seen, exists := h.asPaths[key]
|
h.asPaths[key] = timestamp
|
||||||
if !exists && len(h.asPaths) >= h.maxPaths {
|
} else if len(h.asPaths) >= maxTrackedPaths {
|
||||||
// Map is full; drop this new path and count it.
|
// Map is full; drop this new path and count it.
|
||||||
h.droppedPaths++
|
h.droppedPaths++
|
||||||
|
} else {
|
||||||
|
h.asPaths[key] = timestamp
|
||||||
|
}
|
||||||
|
h.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// processLoop runs periodically to process AS paths into peerings
|
||||||
|
func (h *PeeringHandler) processLoop() {
|
||||||
|
ticker := time.NewTicker(peeringProcessInterval)
|
||||||
|
defer ticker.Stop()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ticker.C:
|
||||||
|
h.processPeerings()
|
||||||
|
case <-h.stopCh:
|
||||||
|
// Final processing
|
||||||
|
h.processPeerings()
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
}
|
||||||
seen.Add(timestamp)
|
|
||||||
h.asPaths[key] = seen
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ProcessPeeringsNow triggers immediate processing of all accumulated AS
|
// ProcessPeeringsNow triggers immediate processing of all accumulated AS
|
||||||
@@ -156,78 +141,15 @@ func (h *PeeringHandler) ProcessPeeringsNow() {
|
|||||||
h.processPeerings()
|
h.processPeerings()
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stop stops the processing goroutine, which writes the peerings of the AS
|
|
||||||
// paths the handler holds before it returns, and waits for it until ctx ends.
|
|
||||||
// Once ctx has ended the processing goroutine skips that last write, and a
|
|
||||||
// write still running is lost when the process exits, so Stop logs how many
|
|
||||||
// paths' peerings are not written.
|
|
||||||
func (h *PeeringHandler) Stop(ctx context.Context) {
|
|
||||||
h.stopCh <- ctx
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-h.done:
|
|
||||||
case <-ctx.Done():
|
|
||||||
}
|
|
||||||
|
|
||||||
h.mu.Lock()
|
|
||||||
unwritten := h.writing + len(h.asPaths)
|
|
||||||
h.mu.Unlock()
|
|
||||||
|
|
||||||
if unwritten > 0 {
|
|
||||||
h.logger.Error("Stop deadline passed before the peering handler wrote "+
|
|
||||||
"its peerings", "paths", unwritten)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// processLoop processes AS paths into peerings every processInterval, and once
|
|
||||||
// more on Stop.
|
|
||||||
func (h *PeeringHandler) processLoop(processInterval time.Duration) {
|
|
||||||
defer close(h.done)
|
|
||||||
|
|
||||||
ticker := time.NewTicker(processInterval)
|
|
||||||
defer ticker.Stop()
|
|
||||||
|
|
||||||
for {
|
|
||||||
// Stop goes first. A write can run past the stop deadline while a
|
|
||||||
// tick waits beside Stop's context. The select below picks one of
|
|
||||||
// them at random, but past the deadline no write may start.
|
|
||||||
select {
|
|
||||||
case ctx := <-h.stopCh:
|
|
||||||
// Final processing, unless the stop deadline has passed
|
|
||||||
if ctx.Err() == nil {
|
|
||||||
h.processPeerings()
|
|
||||||
}
|
|
||||||
|
|
||||||
return
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-ticker.C:
|
|
||||||
h.processPeerings()
|
|
||||||
case ctx := <-h.stopCh:
|
|
||||||
// Final processing, unless the stop deadline has passed
|
|
||||||
if ctx.Err() == nil {
|
|
||||||
h.processPeerings()
|
|
||||||
}
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// processPeerings extracts peerings from AS paths and writes to database
|
// processPeerings extracts peerings from AS paths and writes to database
|
||||||
func (h *PeeringHandler) processPeerings() {
|
func (h *PeeringHandler) processPeerings() {
|
||||||
// Take the accumulated paths and replace the map with a fresh empty one
|
// Take the accumulated paths and replace the map with a fresh empty one
|
||||||
// under the lock. Each path is processed exactly once and the memory is
|
// under the lock. Each path is processed exactly once and the memory is
|
||||||
// released, so the map never grows past a single interval's traffic. The
|
// released, so the map never grows past a single interval's traffic.
|
||||||
// count of dropped paths starts again with the new map.
|
|
||||||
h.mu.Lock()
|
h.mu.Lock()
|
||||||
paths := h.asPaths
|
paths := h.asPaths
|
||||||
h.asPaths = make(map[string]database.SeenTimes)
|
h.asPaths = make(map[string]time.Time)
|
||||||
h.writing = len(paths)
|
|
||||||
dropped := h.droppedPaths
|
dropped := h.droppedPaths
|
||||||
h.droppedPaths = 0
|
|
||||||
h.mu.Unlock()
|
h.mu.Unlock()
|
||||||
|
|
||||||
if len(paths) == 0 {
|
if len(paths) == 0 {
|
||||||
@@ -238,18 +160,11 @@ func (h *PeeringHandler) processPeerings() {
|
|||||||
type peeringKey struct {
|
type peeringKey struct {
|
||||||
low, high int
|
low, high int
|
||||||
}
|
}
|
||||||
|
peerings := make(map[peeringKey]time.Time)
|
||||||
|
|
||||||
// The earliest and the latest time an update carried each peering. Several
|
for pathJSON, timestamp := range paths {
|
||||||
// paths, each carried at several times, can hold a peering, so a new
|
|
||||||
// peering is stored first seen at the earliest of them and last seen at the
|
|
||||||
// latest.
|
|
||||||
peerings := make(map[peeringKey]database.SeenTimes)
|
|
||||||
|
|
||||||
for pathJSON, pathSeen := range paths {
|
|
||||||
var path []int
|
var path []int
|
||||||
|
if err := json.Unmarshal([]byte(pathJSON), &path); err != nil {
|
||||||
err := json.Unmarshal([]byte(pathJSON), &path)
|
|
||||||
if err != nil {
|
|
||||||
h.logger.Error("Failed to decode AS path", "error", err)
|
h.logger.Error("Failed to decode AS path", "error", err)
|
||||||
|
|
||||||
continue
|
continue
|
||||||
@@ -272,20 +187,18 @@ func (h *PeeringHandler) processPeerings() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
key := peeringKey{low: low, high: high}
|
key := peeringKey{low: low, high: high}
|
||||||
|
// Update timestamp if this is newer
|
||||||
seen := peerings[key]
|
if existing, ok := peerings[key]; !ok || timestamp.After(existing) {
|
||||||
seen.Add(pathSeen.First)
|
peerings[key] = timestamp
|
||||||
seen.Add(pathSeen.Last)
|
}
|
||||||
peerings[key] = seen
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Record peerings in database
|
// Record peerings in database
|
||||||
start := time.Now()
|
start := time.Now()
|
||||||
successCount := 0
|
successCount := 0
|
||||||
|
for key, ts := range peerings {
|
||||||
for key, seen := range peerings {
|
err := h.db.RecordPeering(key.low, key.high, ts)
|
||||||
err := h.db.RecordPeering(key.low, key.high, seen)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logger.Error("Failed to record peering",
|
h.logger.Error("Failed to record peering",
|
||||||
"as_a", key.low,
|
"as_a", key.low,
|
||||||
@@ -297,10 +210,6 @@ func (h *PeeringHandler) processPeerings() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
h.mu.Lock()
|
|
||||||
h.writing = 0
|
|
||||||
h.mu.Unlock()
|
|
||||||
|
|
||||||
h.logger.Info("Processed AS peerings",
|
h.logger.Info("Processed AS peerings",
|
||||||
"paths", len(paths),
|
"paths", len(paths),
|
||||||
"unique_peerings", len(peerings),
|
"unique_peerings", len(peerings),
|
||||||
@@ -309,3 +218,13 @@ func (h *PeeringHandler) processPeerings() {
|
|||||||
"duration", time.Since(start),
|
"duration", time.Since(start),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Stop gracefully shuts down the handler by signaling the background
|
||||||
|
// goroutine to stop and performing a final synchronous processing of
|
||||||
|
// any remaining AS paths. This ensures no peering data is lost during
|
||||||
|
// shutdown.
|
||||||
|
func (h *PeeringHandler) Stop() {
|
||||||
|
close(h.stopCh)
|
||||||
|
// Process any remaining peerings synchronously
|
||||||
|
h.processPeerings()
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,387 +0,0 @@
|
|||||||
package routewatch
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"maps"
|
|
||||||
"slices"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/database"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/ristypes"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
testASNA = 64500
|
|
||||||
testASNB = 64501
|
|
||||||
testASNC = 64502
|
|
||||||
testASND = 64503
|
|
||||||
|
|
||||||
// testMaxTrackedPaths is a limit of the path map that a test fills
|
|
||||||
// quickly, in place of maxTrackedPaths.
|
|
||||||
testMaxTrackedPaths = 10
|
|
||||||
)
|
|
||||||
|
|
||||||
// recordingStore wraps mockStore to count every RecordPeering call, so a
|
|
||||||
// test can tell how many times a peering was written across separate runs.
|
|
||||||
type recordingStore struct {
|
|
||||||
*mockStore
|
|
||||||
|
|
||||||
mu sync.Mutex
|
|
||||||
calls int
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *recordingStore) RecordPeering(
|
|
||||||
asA, asB int, seen database.SeenTimes,
|
|
||||||
) error {
|
|
||||||
r.mu.Lock()
|
|
||||||
r.calls++
|
|
||||||
r.mu.Unlock()
|
|
||||||
|
|
||||||
return r.mockStore.RecordPeering(asA, asB, seen)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *recordingStore) callCount() int {
|
|
||||||
r.mu.Lock()
|
|
||||||
defer r.mu.Unlock()
|
|
||||||
|
|
||||||
return r.calls
|
|
||||||
}
|
|
||||||
|
|
||||||
// newTestHandler builds a PeeringHandler without starting the periodic
|
|
||||||
// processing goroutine, so tests drive processing explicitly.
|
|
||||||
func newTestHandler(db database.Store) *PeeringHandler {
|
|
||||||
return &PeeringHandler{
|
|
||||||
db: db,
|
|
||||||
logger: logger.New(),
|
|
||||||
asPaths: make(map[string]database.SeenTimes),
|
|
||||||
maxPaths: maxTrackedPaths,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// snapshot returns how many paths the handler tracks and how many it dropped.
|
|
||||||
func snapshot(h *PeeringHandler) (int, int) {
|
|
||||||
h.mu.Lock()
|
|
||||||
defer h.mu.Unlock()
|
|
||||||
|
|
||||||
return len(h.asPaths), h.droppedPaths
|
|
||||||
}
|
|
||||||
|
|
||||||
func pathKey(t *testing.T, asns ...int) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
b, err := json.Marshal(asns)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to marshal path: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return string(b)
|
|
||||||
}
|
|
||||||
|
|
||||||
func handle(h *PeeringHandler, ts time.Time, asns ...int) {
|
|
||||||
h.HandleMessage(&ristypes.RISMessage{
|
|
||||||
Path: ristypes.ASPath{Sequence: asns},
|
|
||||||
ParsedTimestamp: ts,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// decodePath decodes an AS path from the JSON the feed writes for it, in which
|
|
||||||
// an AS set is an array inside the path: decodePath(t, 1, 2, []int{3, 4})
|
|
||||||
// decodes [1, 2, [3, 4]].
|
|
||||||
func decodePath(t *testing.T, elements ...any) ristypes.ASPath {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
data, err := json.Marshal(elements)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to marshal path: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var path ristypes.ASPath
|
|
||||||
|
|
||||||
err = json.Unmarshal(data, &path)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to decode path %s: %v", data, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return path
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPeeringHandlerLeavesATrailingASSetOutOfThePeerings handles AS paths that
|
|
||||||
// end in an AS set. The members of a set are not neighbours of each other or of
|
|
||||||
// the AS before the set, so a set of several members is in no peering; a set of
|
|
||||||
// one member counts as that AS.
|
|
||||||
func TestPeeringHandlerLeavesATrailingASSetOutOfThePeerings(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
path []any
|
|
||||||
// The peerings recorded, keyed as the mock store keys them.
|
|
||||||
wantPeerings []string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "a set of several members",
|
|
||||||
path: []any{testASNA, testASNB, []int{testASNC, testASND}},
|
|
||||||
wantPeerings: []string{fmt.Sprintf("%d_%d", testASNA, testASNB)},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a set of one member",
|
|
||||||
path: []any{testASNA, testASNB, []int{testASNC}},
|
|
||||||
wantPeerings: []string{
|
|
||||||
fmt.Sprintf("%d_%d", testASNA, testASNB),
|
|
||||||
fmt.Sprintf("%d_%d", testASNB, testASNC),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
store := newMockStore()
|
|
||||||
h := newTestHandler(store)
|
|
||||||
h.HandleMessage(&ristypes.RISMessage{
|
|
||||||
Path: decodePath(t, tt.path...),
|
|
||||||
ParsedTimestamp: time.Now().UTC(),
|
|
||||||
})
|
|
||||||
h.ProcessPeeringsNow()
|
|
||||||
|
|
||||||
got := slices.Sorted(maps.Keys(store.Peerings))
|
|
||||||
if !slices.Equal(got, tt.wantPeerings) {
|
|
||||||
t.Errorf("peerings recorded = %v, want %v", got, tt.wantPeerings)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPeeringHandlerRecordsNothingFromAPathWithASZero handles AS paths that
|
|
||||||
// hold AS 0. RFC 7607 makes such a path malformed, so no peering is recorded
|
|
||||||
// from it, not even one between two other ASes of the path.
|
|
||||||
func TestPeeringHandlerRecordsNothingFromAPathWithASZero(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
path []any
|
|
||||||
}{
|
|
||||||
{name: "AS 0 between two ASes", path: []any{testASNA, 0, testASNB}},
|
|
||||||
{
|
|
||||||
name: "AS 0 after two ASes",
|
|
||||||
path: []any{testASNA, testASNB, 0, testASNC},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
store := newMockStore()
|
|
||||||
h := newTestHandler(store)
|
|
||||||
h.HandleMessage(&ristypes.RISMessage{
|
|
||||||
Path: decodePath(t, tt.path...),
|
|
||||||
ParsedTimestamp: time.Now().UTC(),
|
|
||||||
})
|
|
||||||
h.ProcessPeeringsNow()
|
|
||||||
|
|
||||||
if len(store.Peerings) > 0 {
|
|
||||||
t.Errorf("peerings recorded = %v, want none",
|
|
||||||
slices.Sorted(maps.Keys(store.Peerings)))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPeeringHandlerProcessesEachRunAndEmpties verifies that a processing run
|
|
||||||
// empties the path map (the swap) and that a path seen again after a run is
|
|
||||||
// recorded in the next run too.
|
|
||||||
func TestPeeringHandlerProcessesEachRunAndEmpties(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
store := &recordingStore{mockStore: newMockStore()}
|
|
||||||
h := newTestHandler(store)
|
|
||||||
|
|
||||||
now := time.Now().UTC()
|
|
||||||
|
|
||||||
// Run 1: one path, one peering recorded, map emptied afterwards.
|
|
||||||
handle(h, now, testASNA, testASNB)
|
|
||||||
h.ProcessPeeringsNow()
|
|
||||||
|
|
||||||
if tracked, _ := snapshot(h); tracked != 0 {
|
|
||||||
t.Fatalf("map not empty after first run: %d paths remain", tracked)
|
|
||||||
}
|
|
||||||
|
|
||||||
if got := store.callCount(); got != 1 {
|
|
||||||
t.Fatalf("want 1 RecordPeering call after first run, got %d", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Run 2: the same path again is recorded again (RecordPeering upserts).
|
|
||||||
handle(h, now.Add(time.Second), testASNA, testASNB)
|
|
||||||
h.ProcessPeeringsNow()
|
|
||||||
|
|
||||||
if tracked, _ := snapshot(h); tracked != 0 {
|
|
||||||
t.Fatalf("map not empty after second run: %d paths remain", tracked)
|
|
||||||
}
|
|
||||||
|
|
||||||
if got := store.callCount(); got != 2 {
|
|
||||||
t.Fatalf("want 2 RecordPeering calls after second run, got %d", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPeeringHandlerStopWritesTheRemainingPaths hands a running handler one
|
|
||||||
// path and stops it. The peering must be in the database when Stop returns.
|
|
||||||
func TestPeeringHandlerStopWritesTheRemainingPaths(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
store := &recordingStore{mockStore: newMockStore()}
|
|
||||||
h := NewPeeringHandler(store, logger.New())
|
|
||||||
|
|
||||||
handle(h, time.Now().UTC(), testASNA, testASNB)
|
|
||||||
h.Stop(t.Context())
|
|
||||||
|
|
||||||
if got := store.callCount(); got != 1 {
|
|
||||||
t.Errorf("want 1 RecordPeering call when Stop returns, got %d", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPeeringHandlerCapDropsAndCounts verifies that a full map drops new paths
|
|
||||||
// and counts them, while a path already tracked is refreshed rather than
|
|
||||||
// dropped. The handler's limit is small, and its timer does not fire.
|
|
||||||
func TestPeeringHandlerCapDropsAndCounts(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
store := &recordingStore{mockStore: newMockStore()}
|
|
||||||
h := newPeeringHandler(store, logger.New(), longerThanAnyTest,
|
|
||||||
testMaxTrackedPaths)
|
|
||||||
|
|
||||||
defer stopInTime(t, h)
|
|
||||||
|
|
||||||
now := time.Now().UTC()
|
|
||||||
|
|
||||||
// Fill the map to exactly its limit, including one real path key so the
|
|
||||||
// "already tracked" branch can be exercised. The other paths are of one
|
|
||||||
// AS, so they carry no peering.
|
|
||||||
existing := pathKey(t, testASNA, testASNB)
|
|
||||||
|
|
||||||
h.mu.Lock()
|
|
||||||
h.asPaths[existing] = database.SeenTimes{First: now, Last: now}
|
|
||||||
|
|
||||||
for asn := 1; len(h.asPaths) < testMaxTrackedPaths; asn++ {
|
|
||||||
h.asPaths[pathKey(t, asn)] = database.SeenTimes{First: now, Last: now}
|
|
||||||
}
|
|
||||||
h.mu.Unlock()
|
|
||||||
|
|
||||||
// A new path is dropped and counted because the map is full.
|
|
||||||
handle(h, now.Add(time.Second), testASNA, testASNC)
|
|
||||||
|
|
||||||
tracked, dropped := snapshot(h)
|
|
||||||
if tracked != testMaxTrackedPaths {
|
|
||||||
t.Fatalf("want map size %d after drop, got %d", testMaxTrackedPaths,
|
|
||||||
tracked)
|
|
||||||
}
|
|
||||||
|
|
||||||
if dropped != 1 {
|
|
||||||
t.Fatalf("want dropped count 1, got %d", dropped)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A path already tracked is refreshed, not dropped.
|
|
||||||
refreshed := now.Add(2 * time.Second)
|
|
||||||
handle(h, refreshed, testASNA, testASNB)
|
|
||||||
|
|
||||||
tracked, dropped = snapshot(h)
|
|
||||||
if tracked != testMaxTrackedPaths {
|
|
||||||
t.Fatalf("want map size %d after refresh, got %d", testMaxTrackedPaths,
|
|
||||||
tracked)
|
|
||||||
}
|
|
||||||
|
|
||||||
if dropped != 1 {
|
|
||||||
t.Fatalf("want dropped count still 1 after refresh, got %d", dropped)
|
|
||||||
}
|
|
||||||
|
|
||||||
h.mu.Lock()
|
|
||||||
gotSeen := h.asPaths[existing]
|
|
||||||
h.mu.Unlock()
|
|
||||||
|
|
||||||
if !gotSeen.Last.Equal(refreshed) {
|
|
||||||
t.Fatalf("existing path last seen not refreshed: want %v, got %v",
|
|
||||||
refreshed, gotSeen.Last)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPeeringHandlerRunResetsTheDroppedCount fills the map, has the handler
|
|
||||||
// drop one path, and runs the processing. The count of dropped paths is the
|
|
||||||
// run's own, so it is 0 again after the run. The handler's limit is small, and
|
|
||||||
// its timer does not fire.
|
|
||||||
func TestPeeringHandlerRunResetsTheDroppedCount(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
h := newPeeringHandler(newMockStore(), logger.New(), longerThanAnyTest,
|
|
||||||
testMaxTrackedPaths)
|
|
||||||
|
|
||||||
defer stopInTime(t, h)
|
|
||||||
|
|
||||||
now := time.Now().UTC()
|
|
||||||
|
|
||||||
// Paths of one AS carry no peering, so the run records nothing.
|
|
||||||
h.mu.Lock()
|
|
||||||
for asn := 1; len(h.asPaths) < testMaxTrackedPaths; asn++ {
|
|
||||||
h.asPaths[pathKey(t, asn)] = database.SeenTimes{First: now, Last: now}
|
|
||||||
}
|
|
||||||
h.mu.Unlock()
|
|
||||||
|
|
||||||
handle(h, now, testASNA, testASNB)
|
|
||||||
|
|
||||||
if _, dropped := snapshot(h); dropped != 1 {
|
|
||||||
t.Fatalf("want dropped count 1 with the map full, got %d", dropped)
|
|
||||||
}
|
|
||||||
|
|
||||||
h.ProcessPeeringsNow()
|
|
||||||
|
|
||||||
if _, dropped := snapshot(h); dropped != 0 {
|
|
||||||
t.Errorf("want dropped count 0 after a run, got %d", dropped)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPeeringHandlerStoresTheEarliestAndLatestTimesOfANewPeering handles two
|
|
||||||
// updates whose path carries a peering the database does not hold yet, the
|
|
||||||
// second 5 seconds after the first, in one run into a real database. The
|
|
||||||
// peering is stored first seen at the time of the first update and last seen
|
|
||||||
// at the time of the second.
|
|
||||||
func TestPeeringHandlerStoresTheEarliestAndLatestTimesOfANewPeering(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := newTestDatabase(t)
|
|
||||||
h := newTestHandler(db)
|
|
||||||
|
|
||||||
first := peerAnnouncement(testPeerIP, testV4Prefix)
|
|
||||||
second := peerAnnouncement(testPeerIP, testV4Prefix)
|
|
||||||
second.ParsedTimestamp = first.ParsedTimestamp.Add(secondUpdateAfter)
|
|
||||||
|
|
||||||
h.HandleMessage(first)
|
|
||||||
h.HandleMessage(second)
|
|
||||||
h.ProcessPeeringsNow()
|
|
||||||
|
|
||||||
peers, err := db.GetASPeers(testASNA)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GetASPeers(%d): %v", testASNA, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(peers) != 1 || peers[0].ASN != testASNB {
|
|
||||||
t.Fatalf("peers of AS%d = %+v, want AS%d alone", testASNA, peers,
|
|
||||||
testASNB)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !peers[0].FirstSeen.Equal(first.ParsedTimestamp) {
|
|
||||||
t.Errorf("first_seen of peering AS%d-AS%d = %s, want %s", testASNA,
|
|
||||||
testASNB, peers[0].FirstSeen, first.ParsedTimestamp)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !peers[0].LastSeen.Equal(second.ParsedTimestamp) {
|
|
||||||
t.Errorf("last_seen of peering AS%d-AS%d = %s, want %s", testASNA,
|
|
||||||
testASNB, peers[0].LastSeen, second.ParsedTimestamp)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,163 @@
|
|||||||
|
package routewatch
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strconv"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.eeqj.de/sneak/routewatch/internal/database"
|
||||||
|
"git.eeqj.de/sneak/routewatch/internal/logger"
|
||||||
|
"git.eeqj.de/sneak/routewatch/internal/ristypes"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
testASNA = 64500
|
||||||
|
testASNB = 64501
|
||||||
|
testASNC = 64502
|
||||||
|
)
|
||||||
|
|
||||||
|
// recordingStore wraps mockStore to count every RecordPeering call, so a
|
||||||
|
// test can tell how many times a peering was written across separate runs.
|
||||||
|
type recordingStore struct {
|
||||||
|
*mockStore
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
calls int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *recordingStore) RecordPeering(asA, asB int, ts time.Time) error {
|
||||||
|
r.mu.Lock()
|
||||||
|
r.calls++
|
||||||
|
r.mu.Unlock()
|
||||||
|
|
||||||
|
return r.mockStore.RecordPeering(asA, asB, ts)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *recordingStore) callCount() int {
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
|
||||||
|
return r.calls
|
||||||
|
}
|
||||||
|
|
||||||
|
// newTestHandler builds a PeeringHandler without starting the periodic
|
||||||
|
// processing goroutine, so tests drive processing explicitly.
|
||||||
|
func newTestHandler(db database.Store) *PeeringHandler {
|
||||||
|
return &PeeringHandler{
|
||||||
|
db: db,
|
||||||
|
logger: logger.New(),
|
||||||
|
asPaths: make(map[string]time.Time),
|
||||||
|
stopCh: make(chan struct{}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func snapshot(h *PeeringHandler) (tracked, dropped int) {
|
||||||
|
h.mu.Lock()
|
||||||
|
defer h.mu.Unlock()
|
||||||
|
|
||||||
|
return len(h.asPaths), h.droppedPaths
|
||||||
|
}
|
||||||
|
|
||||||
|
func pathKey(t *testing.T, asns ...int) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
b, err := json.Marshal(ristypes.ASPath(asns))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to marshal path: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return string(b)
|
||||||
|
}
|
||||||
|
|
||||||
|
func handle(h *PeeringHandler, ts time.Time, asns ...int) {
|
||||||
|
h.HandleMessage(&ristypes.RISMessage{
|
||||||
|
Path: ristypes.ASPath(asns),
|
||||||
|
ParsedTimestamp: ts,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestPeeringHandlerProcessesEachRunAndEmpties verifies that a processing run
|
||||||
|
// empties the path map (the swap) and that a path seen again after a run is
|
||||||
|
// recorded in the next run too.
|
||||||
|
func TestPeeringHandlerProcessesEachRunAndEmpties(t *testing.T) {
|
||||||
|
store := &recordingStore{mockStore: newMockStore()}
|
||||||
|
h := newTestHandler(store)
|
||||||
|
|
||||||
|
now := time.Now().UTC()
|
||||||
|
|
||||||
|
// Run 1: one path, one peering recorded, map emptied afterwards.
|
||||||
|
handle(h, now, testASNA, testASNB)
|
||||||
|
h.ProcessPeeringsNow()
|
||||||
|
|
||||||
|
if tracked, _ := snapshot(h); tracked != 0 {
|
||||||
|
t.Fatalf("map not empty after first run: %d paths remain", tracked)
|
||||||
|
}
|
||||||
|
if got := store.callCount(); got != 1 {
|
||||||
|
t.Fatalf("want 1 RecordPeering call after first run, got %d", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run 2: the same path again is recorded again (RecordPeering upserts).
|
||||||
|
handle(h, now.Add(time.Second), testASNA, testASNB)
|
||||||
|
h.ProcessPeeringsNow()
|
||||||
|
|
||||||
|
if tracked, _ := snapshot(h); tracked != 0 {
|
||||||
|
t.Fatalf("map not empty after second run: %d paths remain", tracked)
|
||||||
|
}
|
||||||
|
if got := store.callCount(); got != 2 {
|
||||||
|
t.Fatalf("want 2 RecordPeering calls after second run, got %d", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestPeeringHandlerCapDropsAndCounts verifies that a full map drops new paths
|
||||||
|
// and counts them, while a path already tracked is refreshed rather than
|
||||||
|
// dropped.
|
||||||
|
func TestPeeringHandlerCapDropsAndCounts(t *testing.T) {
|
||||||
|
store := &recordingStore{mockStore: newMockStore()}
|
||||||
|
h := newTestHandler(store)
|
||||||
|
|
||||||
|
now := time.Now().UTC()
|
||||||
|
|
||||||
|
// Fill the map to exactly maxTrackedPaths, including one real path key so
|
||||||
|
// the "already tracked" branch can be exercised. The filler keys are never
|
||||||
|
// processed in this test, so their contents do not matter.
|
||||||
|
existing := pathKey(t, testASNA, testASNB)
|
||||||
|
|
||||||
|
h.mu.Lock()
|
||||||
|
h.asPaths[existing] = now
|
||||||
|
for i := 0; len(h.asPaths) < maxTrackedPaths; i++ {
|
||||||
|
h.asPaths[strconv.Itoa(i)] = now
|
||||||
|
}
|
||||||
|
h.mu.Unlock()
|
||||||
|
|
||||||
|
// A new path is dropped and counted because the map is full.
|
||||||
|
handle(h, now.Add(time.Second), testASNA, testASNC)
|
||||||
|
|
||||||
|
tracked, dropped := snapshot(h)
|
||||||
|
if tracked != maxTrackedPaths {
|
||||||
|
t.Fatalf("want map size %d after drop, got %d", maxTrackedPaths, tracked)
|
||||||
|
}
|
||||||
|
if dropped != 1 {
|
||||||
|
t.Fatalf("want dropped count 1, got %d", dropped)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A path already tracked is refreshed, not dropped.
|
||||||
|
refreshed := now.Add(2 * time.Second)
|
||||||
|
handle(h, refreshed, testASNA, testASNB)
|
||||||
|
|
||||||
|
tracked, dropped = snapshot(h)
|
||||||
|
if tracked != maxTrackedPaths {
|
||||||
|
t.Fatalf("want map size %d after refresh, got %d", maxTrackedPaths, tracked)
|
||||||
|
}
|
||||||
|
if dropped != 1 {
|
||||||
|
t.Fatalf("want dropped count still 1 after refresh, got %d", dropped)
|
||||||
|
}
|
||||||
|
|
||||||
|
h.mu.Lock()
|
||||||
|
gotTS := h.asPaths[existing]
|
||||||
|
h.mu.Unlock()
|
||||||
|
if !gotTS.Equal(refreshed) {
|
||||||
|
t.Fatalf("existing path timestamp not refreshed: want %v, got %v", refreshed, gotTS)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,9 +1,9 @@
|
|||||||
package routewatch
|
package routewatch
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"net"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"slices"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -11,93 +11,43 @@ import (
|
|||||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
"git.eeqj.de/sneak/routewatch/internal/logger"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/metrics"
|
"git.eeqj.de/sneak/routewatch/internal/metrics"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/ristypes"
|
"git.eeqj.de/sneak/routewatch/internal/ristypes"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/routetable"
|
"github.com/google/uuid"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
// prefixHandlerQueueSize is the queue capacity for prefix tracking
|
// prefixHandlerQueueSize is the queue capacity for prefix tracking
|
||||||
// operations, about 4 seconds of feed at peak. The streamer drops updates
|
// operations, about 4 seconds of feed at peak. The streamer drops rather
|
||||||
// rather than blocks when a queue is full (a STATE message is held until
|
// than blocks when a queue is full, so this bounds memory. Batches still
|
||||||
// there is room), so this bounds memory. Batches still flush on a timer
|
// flush on a timer (prefixBatchTimeout).
|
||||||
// (prefixBatchTimeout).
|
|
||||||
prefixHandlerQueueSize = 20000
|
prefixHandlerQueueSize = 20000
|
||||||
|
|
||||||
// prefixBatchSize is the number of prefix updates to batch together
|
// prefixBatchSize is the number of prefix updates to batch together
|
||||||
prefixBatchSize = 25000
|
prefixBatchSize = 25000
|
||||||
|
|
||||||
// prefixBatchMaxSize is the most prefix updates the batch holds while the
|
|
||||||
// flush goroutine writes the previous one: ten batches, about 15 MiB
|
|
||||||
// (README "Memory"). At this size HandleMessage waits until the flush
|
|
||||||
// goroutine takes the batch.
|
|
||||||
prefixBatchMaxSize = 250000
|
|
||||||
|
|
||||||
// prefixBatchTimeout is the maximum time to wait before flushing a batch
|
// prefixBatchTimeout is the maximum time to wait before flushing a batch
|
||||||
// DO NOT reduce this timeout - larger batches are more efficient
|
// DO NOT reduce this timeout - larger batches are more efficient
|
||||||
prefixBatchTimeout = 1 * time.Second
|
prefixBatchTimeout = 1 * time.Second
|
||||||
|
|
||||||
// maxTableChangesPerWrite is the most changed prefix and origin AS pairs
|
// IP version constants
|
||||||
// of the routing table that one flush writes; the rest wait for the next
|
ipv4Version = 4
|
||||||
// flushes. The change of one pair takes about 430 bytes of memory and 15
|
ipv6Version = 6
|
||||||
// microseconds to write. So a write holds about 9 MB and takes about a
|
|
||||||
// third of the one-second flush interval, which leaves the rest to the
|
|
||||||
// batch. The feed peaks at about 5,000 updates a second, each changing at
|
|
||||||
// most one pair. So the million pairs that a full-table peer's session
|
|
||||||
// going down changes are written in about a minute.
|
|
||||||
maxTableChangesPerWrite = 20000
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// PrefixHandler keeps the routing table in memory current from the feed. It
|
// PrefixHandler tracks BGP prefixes and maintains a live routing table in the database.
|
||||||
// applies each update to the table as it handles it. An announcement gives
|
// Routes are added on announcement and deleted on withdrawal.
|
||||||
// the peer a route to the prefix. A withdrawal removes the peer's route, as
|
|
||||||
// does an announcement whose AS path is empty or holds AS 0 anywhere. A peer
|
|
||||||
// whose BGP session goes down leaves every prefix.
|
|
||||||
//
|
|
||||||
// Each flush writes the table's changes to the database. It also stores each
|
|
||||||
// prefix the updates name with the time it was first seen.
|
|
||||||
type PrefixHandler struct {
|
type PrefixHandler struct {
|
||||||
db database.Store
|
db database.Store
|
||||||
logger *logger.Logger
|
logger *logger.Logger
|
||||||
metrics *metrics.Tracker
|
metrics *metrics.Tracker
|
||||||
table *routetable.Table
|
|
||||||
|
|
||||||
// batchMaxSize is prefixBatchMaxSize. Only HandleMessage reads it, so a
|
// Batching
|
||||||
// test may lower it before it hands the handler a message.
|
|
||||||
batchMaxSize int
|
|
||||||
// tableChangesPerWrite is maxTableChangesPerWrite, except in tests that
|
|
||||||
// give a smaller one.
|
|
||||||
tableChangesPerWrite int
|
|
||||||
|
|
||||||
// Batching. mu guards batch, writing and writingTableChanges only: the
|
|
||||||
// flush goroutine takes them under mu and writes them after releasing
|
|
||||||
// mu, so HandleMessage waits for a database write only while batch is at
|
|
||||||
// batchMaxSize.
|
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
batch []seenPrefix
|
batch []prefixUpdate
|
||||||
// writing is how many updates the flush goroutine took and has not
|
lastFlush time.Time
|
||||||
// finished writing.
|
stopCh chan struct{}
|
||||||
writing int
|
wg sync.WaitGroup
|
||||||
// writingTableChanges is how many changes the flush goroutine took from
|
|
||||||
// the routing table and has not finished writing.
|
|
||||||
writingTableChanges int
|
|
||||||
// batchTaken is signalled when the flush goroutine takes batch; its L is
|
|
||||||
// &mu.
|
|
||||||
batchTaken sync.Cond
|
|
||||||
batchFullCh chan struct{} // wakes the flush goroutine when batch is full
|
|
||||||
// stopCh hands Stop's context to the flush goroutine. It holds one, so
|
|
||||||
// Stop does not wait for a write in progress to hand it over.
|
|
||||||
stopCh chan context.Context
|
|
||||||
done chan struct{} // closed when the flush goroutine returns
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// seenPrefix is what the batch keeps of an update: the prefix it names and its
|
|
||||||
// time. A flush needs only these to store each new prefix's first-seen time.
|
|
||||||
type seenPrefix struct {
|
|
||||||
prefix string
|
|
||||||
timestamp time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// prefixUpdate is one prefix of an update message, as applyToTable applies it
|
|
||||||
// to the routing table.
|
|
||||||
type prefixUpdate struct {
|
type prefixUpdate struct {
|
||||||
prefix string
|
prefix string
|
||||||
originASN int
|
originASN int
|
||||||
@@ -105,56 +55,21 @@ type prefixUpdate struct {
|
|||||||
messageType string
|
messageType string
|
||||||
timestamp time.Time
|
timestamp time.Time
|
||||||
path []int
|
path []int
|
||||||
// endsInSet says the path ends in an AS set of several members, so it
|
|
||||||
// gives no origin.
|
|
||||||
endsInSet bool
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// announcesRoute says whether u gives the peer a route to the prefix: an
|
// NewPrefixHandler creates a new batched prefix tracking handler
|
||||||
// announcement whose path gives an origin, or ends in an AS set of several
|
func NewPrefixHandler(db database.Store, logger *logger.Logger) *PrefixHandler {
|
||||||
// members, which gives origin 0 (none). Any other update removes the peer's
|
|
||||||
// route for the prefix: a withdrawal, or an announcement whose path is empty
|
|
||||||
// or holds AS 0 anywhere (the decoder empties such a path). Such an
|
|
||||||
// announcement replaces the peer's route, as any announcement does, with none
|
|
||||||
// (RFC 7607 treats AS 0 in a path as a withdrawal).
|
|
||||||
func (u prefixUpdate) announcesRoute() bool {
|
|
||||||
return u.messageType == "announcement" && (u.originASN > 0 || u.endsInSet)
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewPrefixHandler creates a new batched prefix tracking handler, which applies
|
|
||||||
// the updates to table, the routing table read from db at startup.
|
|
||||||
func NewPrefixHandler(
|
|
||||||
db database.Store, logger *logger.Logger, table *routetable.Table,
|
|
||||||
) *PrefixHandler {
|
|
||||||
return newPrefixHandler(db, logger, table, prefixBatchTimeout,
|
|
||||||
maxTableChangesPerWrite)
|
|
||||||
}
|
|
||||||
|
|
||||||
// newPrefixHandler is NewPrefixHandler with the flush timer's interval,
|
|
||||||
// batchTimeout, in place of prefixBatchTimeout, and tableChangesPerWrite in
|
|
||||||
// place of maxTableChangesPerWrite. A test that the timer must not disturb
|
|
||||||
// gives an interval longer than the test runs. A test of writes split over
|
|
||||||
// several flushes gives a small tableChangesPerWrite, so that it exceeds it
|
|
||||||
// with few changes.
|
|
||||||
func newPrefixHandler(
|
|
||||||
db database.Store, logger *logger.Logger, table *routetable.Table,
|
|
||||||
batchTimeout time.Duration, tableChangesPerWrite int,
|
|
||||||
) *PrefixHandler {
|
|
||||||
h := &PrefixHandler{
|
h := &PrefixHandler{
|
||||||
db: db,
|
db: db,
|
||||||
logger: logger,
|
logger: logger,
|
||||||
table: table,
|
batch: make([]prefixUpdate, 0, prefixBatchSize),
|
||||||
batch: make([]seenPrefix, 0, prefixBatchSize),
|
lastFlush: time.Now(),
|
||||||
batchMaxSize: prefixBatchMaxSize,
|
stopCh: make(chan struct{}),
|
||||||
tableChangesPerWrite: tableChangesPerWrite,
|
|
||||||
batchFullCh: make(chan struct{}, 1),
|
|
||||||
stopCh: make(chan context.Context, 1),
|
|
||||||
done: make(chan struct{}),
|
|
||||||
}
|
}
|
||||||
h.batchTaken.L = &h.mu
|
|
||||||
|
|
||||||
// Start the flush timer goroutine
|
// Start the flush timer goroutine
|
||||||
go flushLoop(batchTimeout, h.batchFullCh, h.stopCh, h.done, h.flushBatch)
|
h.wg.Add(1)
|
||||||
|
go h.flushLoop()
|
||||||
|
|
||||||
return h
|
return h
|
||||||
}
|
}
|
||||||
@@ -164,13 +79,10 @@ func (h *PrefixHandler) SetMetricsTracker(metrics *metrics.Tracker) {
|
|||||||
h.metrics = metrics
|
h.metrics = metrics
|
||||||
}
|
}
|
||||||
|
|
||||||
// WantsMessage returns true if this handler wants to process messages of the
|
// WantsMessage returns true if this handler wants to process messages of the given type
|
||||||
// given type
|
|
||||||
func (h *PrefixHandler) WantsMessage(messageType string) bool {
|
func (h *PrefixHandler) WantsMessage(messageType string) bool {
|
||||||
// UPDATE messages change routes; a STATE message can say a peer's
|
// We only care about UPDATE messages for the routing table
|
||||||
// session went down, which removes all of its routes.
|
return messageType == "UPDATE"
|
||||||
return messageType == ristypes.MessageTypeUpdate ||
|
|
||||||
messageType == ristypes.MessageTypeState
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// QueueCapacity returns the desired queue capacity for this handler
|
// QueueCapacity returns the desired queue capacity for this handler
|
||||||
@@ -181,48 +93,30 @@ func (h *PrefixHandler) QueueCapacity() int {
|
|||||||
|
|
||||||
// HandleMessage processes a message to track prefix information
|
// HandleMessage processes a message to track prefix information
|
||||||
func (h *PrefixHandler) HandleMessage(msg *ristypes.RISMessage) {
|
func (h *PrefixHandler) HandleMessage(msg *ristypes.RISMessage) {
|
||||||
if msg.Type == ristypes.MessageTypeState {
|
|
||||||
if msg.State == ristypes.StateDown {
|
|
||||||
h.table.PeerDown(msg.Peer)
|
|
||||||
}
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Use the pre-parsed timestamp
|
// Use the pre-parsed timestamp
|
||||||
timestamp := msg.ParsedTimestamp
|
timestamp := msg.ParsedTimestamp
|
||||||
|
|
||||||
originASN := msg.Path.Origin()
|
// Get origin ASN from path (last element)
|
||||||
|
var originASN int
|
||||||
// The stored and shown AS path lists the members of an AS set that ends
|
if len(msg.Path) > 0 {
|
||||||
// the path after the rest of it.
|
originASN = msg.Path[len(msg.Path)-1]
|
||||||
path := slices.Concat(msg.Path.Sequence, msg.Path.Set)
|
}
|
||||||
|
|
||||||
h.mu.Lock()
|
h.mu.Lock()
|
||||||
defer h.mu.Unlock()
|
defer h.mu.Unlock()
|
||||||
|
|
||||||
// While the batch is at its maximum size, wait until the flush goroutine
|
// Withdrawals are queued before announcements, because the batch keeps
|
||||||
// takes it. Meanwhile the handler takes nothing off its queue, which
|
// each peer's last update for a prefix: a message that lists a prefix as
|
||||||
// fills, and the streamer drops messages for it.
|
// both withdrawn and announced keeps the route (RFC 4271 section 4.3).
|
||||||
for len(h.batch) >= h.batchMaxSize {
|
|
||||||
h.batchTaken.Wait()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Withdrawals are applied to the routing table before announcements, so
|
|
||||||
// a message that lists a prefix as both withdrawn and announced keeps the
|
|
||||||
// route (RFC 4271 section 4.3). A withdrawal names only the prefix and
|
|
||||||
// peer: the message's AS path, and so its origin AS, belongs to the
|
|
||||||
// announced prefixes.
|
|
||||||
for _, prefix := range msg.Withdrawals {
|
for _, prefix := range msg.Withdrawals {
|
||||||
update := prefixUpdate{
|
h.batch = append(h.batch, prefixUpdate{
|
||||||
prefix: canonicalPrefix(prefix),
|
prefix: canonicalPrefix(prefix),
|
||||||
|
originASN: originASN, // Use the originASN from path if available
|
||||||
peer: msg.Peer,
|
peer: msg.Peer,
|
||||||
messageType: "withdrawal",
|
messageType: "withdrawal",
|
||||||
timestamp: timestamp,
|
timestamp: timestamp,
|
||||||
}
|
path: msg.Path,
|
||||||
h.batch = append(h.batch, seenPrefix{update.prefix, update.timestamp})
|
})
|
||||||
h.applyToTable(update)
|
|
||||||
|
|
||||||
// Record withdrawal in metrics
|
// Record withdrawal in metrics
|
||||||
if h.metrics != nil {
|
if h.metrics != nil {
|
||||||
h.metrics.RecordWithdrawal()
|
h.metrics.RecordWithdrawal()
|
||||||
@@ -231,18 +125,14 @@ func (h *PrefixHandler) HandleMessage(msg *ristypes.RISMessage) {
|
|||||||
|
|
||||||
for _, announcement := range msg.Announcements {
|
for _, announcement := range msg.Announcements {
|
||||||
for _, prefix := range announcement.Prefixes {
|
for _, prefix := range announcement.Prefixes {
|
||||||
update := prefixUpdate{
|
h.batch = append(h.batch, prefixUpdate{
|
||||||
prefix: canonicalPrefix(prefix),
|
prefix: canonicalPrefix(prefix),
|
||||||
originASN: originASN,
|
originASN: originASN,
|
||||||
peer: msg.Peer,
|
peer: msg.Peer,
|
||||||
messageType: "announcement",
|
messageType: "announcement",
|
||||||
timestamp: timestamp,
|
timestamp: timestamp,
|
||||||
path: path,
|
path: msg.Path,
|
||||||
endsInSet: len(msg.Path.Set) > 0,
|
})
|
||||||
}
|
|
||||||
h.batch = append(h.batch, seenPrefix{update.prefix, update.timestamp})
|
|
||||||
h.applyToTable(update)
|
|
||||||
|
|
||||||
// Record announcement in metrics
|
// Record announcement in metrics
|
||||||
if h.metrics != nil {
|
if h.metrics != nil {
|
||||||
h.metrics.RecordAnnouncement()
|
h.metrics.RecordAnnouncement()
|
||||||
@@ -250,182 +140,138 @@ func (h *PrefixHandler) HandleMessage(msg *ristypes.RISMessage) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// A full batch wakes the flush goroutine. If it is writing, the wake-up
|
// Check if we need to flush
|
||||||
// waits in batchFullCh and the batch keeps growing until the write ends
|
|
||||||
// or the batch reaches batchMaxSize.
|
|
||||||
if len(h.batch) >= prefixBatchSize {
|
if len(h.batch) >= prefixBatchSize {
|
||||||
select {
|
h.flushBatchLocked()
|
||||||
case h.batchFullCh <- struct{}{}:
|
|
||||||
default: // already woken
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stop stops the flush goroutine, which makes one last flush before it returns,
|
// flushLoop runs in a goroutine and periodically flushes batches
|
||||||
// and waits for it until ctx ends. That flush writes at most
|
func (h *PrefixHandler) flushLoop() {
|
||||||
// tableChangesPerWrite of the routing table's changes, and the prefixes of
|
defer h.wg.Done()
|
||||||
// the updates the handler holds. Once ctx has ended the flush goroutine skips
|
ticker := time.NewTicker(prefixBatchTimeout)
|
||||||
// that last flush, and a write still running is lost when the process exits. A
|
defer ticker.Stop()
|
||||||
// failed write of the table's changes is not retried either. Stop logs how many
|
|
||||||
// updates and how many of the table's changes are not written.
|
|
||||||
func (h *PrefixHandler) Stop(ctx context.Context) {
|
|
||||||
h.stopCh <- ctx
|
|
||||||
|
|
||||||
|
for {
|
||||||
select {
|
select {
|
||||||
case <-h.done:
|
case <-ticker.C:
|
||||||
case <-ctx.Done():
|
|
||||||
}
|
|
||||||
|
|
||||||
h.mu.Lock()
|
h.mu.Lock()
|
||||||
unwritten := h.writing + len(h.batch)
|
if time.Since(h.lastFlush) >= prefixBatchTimeout {
|
||||||
tableChanges := h.writingTableChanges + h.table.ChangeCount()
|
h.flushBatchLocked()
|
||||||
|
}
|
||||||
|
h.mu.Unlock()
|
||||||
|
case <-h.stopCh:
|
||||||
|
// Final flush
|
||||||
|
h.mu.Lock()
|
||||||
|
h.flushBatchLocked()
|
||||||
h.mu.Unlock()
|
h.mu.Unlock()
|
||||||
|
|
||||||
if unwritten == 0 && tableChanges == 0 {
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if ctx.Err() != nil {
|
|
||||||
h.logger.Error("Stop deadline passed before the prefix handler wrote "+
|
|
||||||
"its updates", "updates", unwritten, "table_changes", tableChanges)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
}
|
||||||
|
|
||||||
h.logger.Error("The prefix handler stopped before it wrote everything",
|
|
||||||
"updates", unwritten, "table_changes", tableChanges)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// takeBatch returns the pending updates and leaves an empty batch in their
|
// flushBatchLocked flushes the prefix batch to the database (must be called with mutex held)
|
||||||
// place for HandleMessage to fill. The updates count in writing until
|
func (h *PrefixHandler) flushBatchLocked() {
|
||||||
// flushBatch has written them.
|
|
||||||
func (h *PrefixHandler) takeBatch() []seenPrefix {
|
|
||||||
h.mu.Lock()
|
|
||||||
defer h.mu.Unlock()
|
|
||||||
|
|
||||||
if len(h.batch) == 0 {
|
if len(h.batch) == 0 {
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
batch := h.batch
|
|
||||||
h.writing = len(batch)
|
|
||||||
h.batch = make([]seenPrefix, 0, prefixBatchSize)
|
|
||||||
h.batchTaken.Broadcast()
|
|
||||||
|
|
||||||
return batch
|
|
||||||
}
|
|
||||||
|
|
||||||
// flushBatch writes the routing table's changes, then takes the pending batch
|
|
||||||
// and stores the prefixes it names, without holding the mutex during the
|
|
||||||
// writes.
|
|
||||||
func (h *PrefixHandler) flushBatch() {
|
|
||||||
h.writeTableChanges()
|
|
||||||
|
|
||||||
batch := h.takeBatch()
|
|
||||||
if len(batch) == 0 {
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
startTime := time.Now()
|
startTime := time.Now()
|
||||||
|
batchSize := len(h.batch)
|
||||||
|
|
||||||
// Each prefix the batch names, announced or withdrawn, with the earliest
|
// For each prefix and peer, keep only the last update the feed delivered.
|
||||||
// timestamp among its updates: the prefix tables store only first_seen,
|
// Updates from different peers for the same prefix are all kept. Arrival
|
||||||
// written when the prefix is new and never changed after.
|
// order decides, not the timestamp: timestamps are whole seconds, so
|
||||||
|
// updates within one second carry the same one.
|
||||||
|
type prefixAndPeer struct{ prefix, peer string }
|
||||||
|
lastUpdates := make(map[prefixAndPeer]prefixUpdate)
|
||||||
|
for _, update := range h.batch {
|
||||||
|
lastUpdates[prefixAndPeer{update.prefix, update.peer}] = update
|
||||||
|
}
|
||||||
|
|
||||||
|
// Collect routes to upsert and delete
|
||||||
|
var routesToUpsert []*database.LiveRoute
|
||||||
|
var routesToDelete []database.LiveRouteDeletion
|
||||||
|
|
||||||
|
// Collect unique prefixes to update
|
||||||
prefixesToUpdate := make(map[string]time.Time)
|
prefixesToUpdate := make(map[string]time.Time)
|
||||||
|
|
||||||
for _, update := range batch {
|
for _, update := range lastUpdates {
|
||||||
seen, exists := prefixesToUpdate[update.prefix]
|
// Track prefix for both announcements and withdrawals
|
||||||
if !exists || update.timestamp.Before(seen) {
|
if _, exists := prefixesToUpdate[update.prefix]; !exists || update.timestamp.After(prefixesToUpdate[update.prefix]) {
|
||||||
prefixesToUpdate[update.prefix] = update.timestamp
|
prefixesToUpdate[update.prefix] = update.timestamp
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
err := h.db.UpdatePrefixesBatch(prefixesToUpdate)
|
if update.messageType == "announcement" && update.originASN > 0 {
|
||||||
|
// Create live route for batch upsert
|
||||||
|
route := h.createLiveRoute(update)
|
||||||
|
if route != nil {
|
||||||
|
routesToUpsert = append(routesToUpsert, route)
|
||||||
|
}
|
||||||
|
} else if update.messageType == "withdrawal" {
|
||||||
|
// Parse CIDR to get IP version
|
||||||
|
_, ipVersion, err := parseCIDR(update.prefix)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logger.Error("Failed to update prefix batch",
|
h.logger.Error("Failed to parse CIDR for withdrawal", "prefix", update.prefix, "error", err)
|
||||||
"error", err, "count", len(prefixesToUpdate))
|
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
h.mu.Lock()
|
// Create deletion record for batch delete
|
||||||
h.writing = 0
|
routesToDelete = append(routesToDelete, database.LiveRouteDeletion{
|
||||||
h.mu.Unlock()
|
Prefix: update.prefix,
|
||||||
|
OriginASN: update.originASN,
|
||||||
h.logger.Debug("Flushed prefix batch",
|
PeerIP: update.peer,
|
||||||
"batch_size", len(batch),
|
IPVersion: ipVersion,
|
||||||
"unique_prefixes", len(prefixesToUpdate),
|
|
||||||
"duration_ms", time.Since(startTime).Milliseconds(),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// applyToTable applies update to the routing table in memory: as an
|
|
||||||
// announcement when it gives the peer a route, otherwise as a withdrawal. A
|
|
||||||
// prefix that does not parse is logged and left out. Each announcement counts
|
|
||||||
// in the IPv4 or IPv6 route update rate.
|
|
||||||
func (h *PrefixHandler) applyToTable(update prefixUpdate) {
|
|
||||||
prefix, err := netip.ParsePrefix(update.prefix)
|
|
||||||
if err != nil {
|
|
||||||
h.logger.Error("Failed to parse prefix",
|
|
||||||
"prefix", update.prefix, "error", err)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if !update.announcesRoute() {
|
|
||||||
h.table.Withdraw(prefix, update.peer)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if h.metrics != nil {
|
|
||||||
if prefix.Addr().Is4() {
|
|
||||||
h.metrics.RecordIPv4Update()
|
|
||||||
} else {
|
|
||||||
h.metrics.RecordIPv6Update()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
h.table.Announce(prefix, update.originASN, routetable.Route{
|
|
||||||
Peer: update.peer,
|
|
||||||
ASPath: update.path,
|
|
||||||
Time: update.timestamp,
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// writeTableChanges writes at most tableChangesPerWrite changes of the routing
|
|
||||||
// table in memory, and every peer the table numbered since the last write.
|
|
||||||
// When the write fails, they go back to the table, so that a later write writes
|
|
||||||
// them again, each pair with its content then.
|
|
||||||
func (h *PrefixHandler) writeTableChanges() {
|
|
||||||
h.mu.Lock()
|
|
||||||
changes, newPeers := h.table.TakeChanges(h.tableChangesPerWrite)
|
|
||||||
h.writingTableChanges = len(changes)
|
|
||||||
h.mu.Unlock()
|
|
||||||
|
|
||||||
if len(changes) == 0 && len(newPeers) == 0 {
|
|
||||||
return
|
|
||||||
}
|
}
|
||||||
|
|
||||||
err := h.db.WriteRouteChanges(changes, newPeers)
|
// Process batch operations
|
||||||
|
successCount := 0
|
||||||
h.mu.Lock()
|
if len(routesToUpsert) > 0 {
|
||||||
defer h.mu.Unlock()
|
if err := h.db.UpsertLiveRouteBatch(routesToUpsert); err != nil {
|
||||||
|
h.logger.Error("Failed to upsert route batch", "error", err, "count", len(routesToUpsert))
|
||||||
h.writingTableChanges = 0
|
} else {
|
||||||
|
successCount += len(routesToUpsert)
|
||||||
if err != nil {
|
|
||||||
h.logger.Error("Failed to write the routing table's changes; the next "+
|
|
||||||
"flush retries", "changes", len(changes), "new_peers", len(newPeers),
|
|
||||||
"error", err)
|
|
||||||
|
|
||||||
h.table.PutBackChanges(changes, newPeers)
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(routesToDelete) > 0 {
|
||||||
|
if err := h.db.DeleteLiveRouteBatch(routesToDelete); err != nil {
|
||||||
|
h.logger.Error("Failed to delete route batch", "error", err, "count", len(routesToDelete))
|
||||||
|
} else {
|
||||||
|
successCount += len(routesToDelete)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update prefix tables
|
||||||
|
if len(prefixesToUpdate) > 0 {
|
||||||
|
if err := h.db.UpdatePrefixesBatch(prefixesToUpdate); err != nil {
|
||||||
|
h.logger.Error("Failed to update prefix batch", "error", err, "count", len(prefixesToUpdate))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
elapsed := time.Since(startTime)
|
||||||
|
h.logger.Debug("Flushed prefix batch",
|
||||||
|
"batch_size", batchSize,
|
||||||
|
"unique_prefixes", len(prefixesToUpdate),
|
||||||
|
"success", successCount,
|
||||||
|
"duration_ms", elapsed.Milliseconds(),
|
||||||
|
)
|
||||||
|
|
||||||
|
// Clear batch
|
||||||
|
h.batch = h.batch[:0]
|
||||||
|
h.lastFlush = time.Now()
|
||||||
}
|
}
|
||||||
|
|
||||||
// canonicalPrefix returns prefix in the text form net/netip prints, the form
|
// canonicalPrefix returns prefix in the text form net/netip prints, the form
|
||||||
// the prefix tables store and the IP lookup reads them in. The feed sends IPv6
|
// the IP lookup builds when it looks a prefix up. The feed sends IPv6
|
||||||
// announcements compressed ("2001:db8::/32") but IPv6 withdrawals uncompressed
|
// announcements compressed ("2001:db8::/32") but IPv6 withdrawals uncompressed
|
||||||
// ("2001:db8:0:0:0:0:0:0/32"). A prefix that does not parse is returned
|
// ("2001:db8:0:0:0:0:0:0/32"); stored as received, a withdrawal would not match
|
||||||
// unchanged, and applyToTable logs it.
|
// the route its announcement stored. A prefix that does not parse is returned
|
||||||
|
// unchanged, and the batch flush reports it.
|
||||||
func canonicalPrefix(prefix string) string {
|
func canonicalPrefix(prefix string) string {
|
||||||
p, err := netip.ParsePrefix(prefix)
|
p, err := netip.ParsePrefix(prefix)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -434,3 +280,201 @@ func canonicalPrefix(prefix string) string {
|
|||||||
|
|
||||||
return p.Masked().String()
|
return p.Masked().String()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// parseCIDR extracts the mask length and IP version from a prefix string
|
||||||
|
func parseCIDR(prefix string) (maskLength int, ipVersion int, err error) {
|
||||||
|
_, ipNet, err := net.ParseCIDR(prefix)
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
ones, _ := ipNet.Mask.Size()
|
||||||
|
if strings.Contains(prefix, ":") {
|
||||||
|
return ones, ipv6Version, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return ones, ipv4Version, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// processAnnouncement handles storing an announcement in the database
|
||||||
|
// nolint:unused // kept for potential future use
|
||||||
|
func (h *PrefixHandler) processAnnouncement(_ *database.Prefix, update prefixUpdate) {
|
||||||
|
// Parse CIDR to get mask length
|
||||||
|
maskLength, ipVersion, err := parseCIDR(update.prefix)
|
||||||
|
if err != nil {
|
||||||
|
h.logger.Error("Failed to parse CIDR",
|
||||||
|
"prefix", update.prefix,
|
||||||
|
"error", err,
|
||||||
|
)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Track route update metrics
|
||||||
|
if h.metrics != nil {
|
||||||
|
if ipVersion == ipv4Version {
|
||||||
|
h.metrics.RecordIPv4Update()
|
||||||
|
} else {
|
||||||
|
h.metrics.RecordIPv6Update()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create live route record
|
||||||
|
liveRoute := &database.LiveRoute{
|
||||||
|
ID: uuid.New(),
|
||||||
|
Prefix: update.prefix,
|
||||||
|
MaskLength: maskLength,
|
||||||
|
IPVersion: ipVersion,
|
||||||
|
OriginASN: update.originASN,
|
||||||
|
PeerIP: update.peer,
|
||||||
|
ASPath: update.path,
|
||||||
|
NextHop: update.peer, // Using peer as next hop
|
||||||
|
LastUpdated: update.timestamp,
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := h.db.UpsertLiveRoute(liveRoute); err != nil {
|
||||||
|
h.logger.Error("Failed to upsert live route",
|
||||||
|
"prefix", update.prefix,
|
||||||
|
"error", err,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// createLiveRoute creates a LiveRoute from a prefix update
|
||||||
|
func (h *PrefixHandler) createLiveRoute(update prefixUpdate) *database.LiveRoute {
|
||||||
|
// Parse CIDR to get mask length
|
||||||
|
maskLength, ipVersion, err := parseCIDR(update.prefix)
|
||||||
|
if err != nil {
|
||||||
|
h.logger.Error("Failed to parse CIDR",
|
||||||
|
"prefix", update.prefix,
|
||||||
|
"error", err,
|
||||||
|
)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Track route update metrics
|
||||||
|
if h.metrics != nil {
|
||||||
|
if ipVersion == ipv4Version {
|
||||||
|
h.metrics.RecordIPv4Update()
|
||||||
|
} else {
|
||||||
|
h.metrics.RecordIPv6Update()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create live route record
|
||||||
|
liveRoute := &database.LiveRoute{
|
||||||
|
ID: uuid.New(),
|
||||||
|
Prefix: update.prefix,
|
||||||
|
MaskLength: maskLength,
|
||||||
|
IPVersion: ipVersion,
|
||||||
|
OriginASN: update.originASN,
|
||||||
|
PeerIP: update.peer,
|
||||||
|
ASPath: update.path,
|
||||||
|
NextHop: update.peer, // Using peer as next hop
|
||||||
|
LastUpdated: update.timestamp,
|
||||||
|
}
|
||||||
|
|
||||||
|
return liveRoute
|
||||||
|
}
|
||||||
|
|
||||||
|
// processAnnouncementDirect handles storing an announcement directly without prefix table
|
||||||
|
// nolint:unused // kept for potential future use
|
||||||
|
func (h *PrefixHandler) processAnnouncementDirect(update prefixUpdate) {
|
||||||
|
// Parse CIDR to get mask length
|
||||||
|
maskLength, ipVersion, err := parseCIDR(update.prefix)
|
||||||
|
if err != nil {
|
||||||
|
h.logger.Error("Failed to parse CIDR",
|
||||||
|
"prefix", update.prefix,
|
||||||
|
"error", err,
|
||||||
|
)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Track route update metrics
|
||||||
|
if h.metrics != nil {
|
||||||
|
if ipVersion == ipv4Version {
|
||||||
|
h.metrics.RecordIPv4Update()
|
||||||
|
} else {
|
||||||
|
h.metrics.RecordIPv6Update()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create live route record
|
||||||
|
liveRoute := &database.LiveRoute{
|
||||||
|
ID: uuid.New(),
|
||||||
|
Prefix: update.prefix,
|
||||||
|
MaskLength: maskLength,
|
||||||
|
IPVersion: ipVersion,
|
||||||
|
OriginASN: update.originASN,
|
||||||
|
PeerIP: update.peer,
|
||||||
|
ASPath: update.path,
|
||||||
|
NextHop: update.peer, // Using peer as next hop
|
||||||
|
LastUpdated: update.timestamp,
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := h.db.UpsertLiveRoute(liveRoute); err != nil {
|
||||||
|
h.logger.Error("Failed to upsert live route",
|
||||||
|
"prefix", update.prefix,
|
||||||
|
"error", err,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// processWithdrawalDirect handles removing a route directly without prefix table
|
||||||
|
// nolint:unused // kept for potential future use
|
||||||
|
func (h *PrefixHandler) processWithdrawalDirect(update prefixUpdate) {
|
||||||
|
// For withdrawals, we need to delete the route from live_routes
|
||||||
|
if update.originASN > 0 {
|
||||||
|
if err := h.db.DeleteLiveRoute(update.prefix, update.originASN, update.peer); err != nil {
|
||||||
|
h.logger.Error("Failed to delete live route",
|
||||||
|
"prefix", update.prefix,
|
||||||
|
"origin_asn", update.originASN,
|
||||||
|
"peer", update.peer,
|
||||||
|
"error", err,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// If no origin ASN, just delete all routes for this prefix from this peer
|
||||||
|
if err := h.db.DeleteLiveRoute(update.prefix, 0, update.peer); err != nil {
|
||||||
|
h.logger.Error("Failed to delete live route (no origin ASN)",
|
||||||
|
"prefix", update.prefix,
|
||||||
|
"peer", update.peer,
|
||||||
|
"error", err,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// processWithdrawal handles removing a route from the live routing table
|
||||||
|
// nolint:unused // kept for potential future use
|
||||||
|
func (h *PrefixHandler) processWithdrawal(_ *database.Prefix, update prefixUpdate) {
|
||||||
|
// For withdrawals, we need to delete the route from live_routes
|
||||||
|
// Since we have the origin ASN from the update, we can delete the specific route
|
||||||
|
if update.originASN > 0 {
|
||||||
|
if err := h.db.DeleteLiveRoute(update.prefix, update.originASN, update.peer); err != nil {
|
||||||
|
h.logger.Error("Failed to delete live route",
|
||||||
|
"prefix", update.prefix,
|
||||||
|
"origin_asn", update.originASN,
|
||||||
|
"peer", update.peer,
|
||||||
|
"error", err,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// If no origin ASN, just delete all routes for this prefix from this peer
|
||||||
|
if err := h.db.DeleteLiveRoute(update.prefix, 0, update.peer); err != nil {
|
||||||
|
h.logger.Error("Failed to delete live route (no origin ASN)",
|
||||||
|
"prefix", update.prefix,
|
||||||
|
"peer", update.peer,
|
||||||
|
"error", err,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop gracefully stops the handler and flushes remaining batches
|
||||||
|
func (h *PrefixHandler) Stop() {
|
||||||
|
close(h.stopCh)
|
||||||
|
h.wg.Wait()
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,317 +0,0 @@
|
|||||||
package routewatch
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"net/netip"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/config"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/database"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/ristypes"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/routetable"
|
|
||||||
)
|
|
||||||
|
|
||||||
const testPeerIP = "2001:db8:ffff::1"
|
|
||||||
|
|
||||||
// An IPv4 and an IPv6 prefix the tests announce.
|
|
||||||
const (
|
|
||||||
testV4Prefix = "192.0.2.0/24"
|
|
||||||
testV6Prefix = "2001:db8:1::/48"
|
|
||||||
)
|
|
||||||
|
|
||||||
// secondUpdateAfter is how long after the first update the tests of first_seen
|
|
||||||
// and last_seen send the second, in the same batch.
|
|
||||||
const secondUpdateAfter = 5 * time.Second
|
|
||||||
|
|
||||||
// TestPrefixHandlerStoresPrefixesTheIPLookupFinds runs announcements and
|
|
||||||
// withdrawals through the prefix handler into a real database. The prefixes
|
|
||||||
// are written the way the feed sends them: IPv6 announcements compressed, IPv6
|
|
||||||
// withdrawals uncompressed. The IP lookup must find each announced prefix in
|
|
||||||
// the routing table, and its first-seen time in the prefix tables. The
|
|
||||||
// withdrawal must remove the peer's route and store no second form of the
|
|
||||||
// prefix.
|
|
||||||
func TestPrefixHandlerStoresPrefixesTheIPLookupFinds(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// The two prefixes the announcement names.
|
|
||||||
const wantPrefixes = 2
|
|
||||||
|
|
||||||
db := newTestDatabase(t)
|
|
||||||
h := prefixHandlerWithoutTimer(db)
|
|
||||||
|
|
||||||
ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
|
||||||
h.HandleMessage(&ristypes.RISMessage{
|
|
||||||
Peer: testPeerIP,
|
|
||||||
Path: ristypes.ASPath{Sequence: []int{testASNA, testASNB}},
|
|
||||||
ParsedTimestamp: ts,
|
|
||||||
Announcements: []ristypes.RISAnnouncement{{
|
|
||||||
NextHop: testPeerIP,
|
|
||||||
Prefixes: []string{testV6Prefix, testV4Prefix},
|
|
||||||
}},
|
|
||||||
})
|
|
||||||
h.flushBatch()
|
|
||||||
|
|
||||||
for ip, want := range map[string]string{
|
|
||||||
"2001:db8:1::1": testV6Prefix,
|
|
||||||
"192.0.2.1": testV4Prefix,
|
|
||||||
} {
|
|
||||||
summary, found := h.table.Lookup(netip.MustParseAddr(ip))
|
|
||||||
if !found || summary.Prefix.String() != want ||
|
|
||||||
summary.OriginASN != testASNB {
|
|
||||||
t.Errorf("Lookup(%s) after announcement = %+v, %t; want %s AS%d",
|
|
||||||
ip, summary, found, want, testASNB)
|
|
||||||
}
|
|
||||||
|
|
||||||
firstSeen, err := db.GetPrefixFirstSeen(t.Context(),
|
|
||||||
netip.MustParsePrefix(want))
|
|
||||||
if err != nil || !firstSeen.Equal(ts) {
|
|
||||||
t.Errorf("GetPrefixFirstSeen(%s) = %v, %v; want %v",
|
|
||||||
want, firstSeen, err, ts)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
h.HandleMessage(&ristypes.RISMessage{
|
|
||||||
Peer: testPeerIP,
|
|
||||||
ParsedTimestamp: ts.Add(time.Minute),
|
|
||||||
Withdrawals: []string{"2001:db8:1:0:0:0:0:0/48", testV4Prefix},
|
|
||||||
})
|
|
||||||
h.flushBatch()
|
|
||||||
|
|
||||||
for _, ip := range []string{"2001:db8:1::1", "192.0.2.1"} {
|
|
||||||
summary, found := h.table.Lookup(netip.MustParseAddr(ip))
|
|
||||||
if found {
|
|
||||||
t.Errorf("Lookup(%s) after withdrawal = %+v, want none", ip, summary)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
stats, err := db.GetStatsContext(t.Context())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GetStatsContext: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if stats.Prefixes != wantPrefixes {
|
|
||||||
t.Errorf("the prefix tables hold %d prefixes after the withdrawal, "+
|
|
||||||
"want %d", stats.Prefixes, wantPrefixes)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPrefixHandlerStoresANewPrefixFirstSeenAtItsEarliestUpdate announces a
|
|
||||||
// prefix the database does not hold yet twice in one batch, the second time 5
|
|
||||||
// seconds after the first, through the prefix handler into a real database. The
|
|
||||||
// prefix is stored first seen at the time of the first announcement, whether
|
|
||||||
// the two come from two peers or from one.
|
|
||||||
func TestPrefixHandlerStoresANewPrefixFirstSeenAtItsEarliestUpdate(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const otherPeerIP = "2001:db8:ffff::2"
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
firstPeer, secondPeer string
|
|
||||||
}{
|
|
||||||
{name: "two peers", firstPeer: testPeerIP, secondPeer: otherPeerIP},
|
|
||||||
{name: "one peer", firstPeer: testPeerIP, secondPeer: testPeerIP},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
first := peerAnnouncement(tt.firstPeer, testV4Prefix)
|
|
||||||
second := peerAnnouncement(tt.secondPeer, testV4Prefix)
|
|
||||||
second.ParsedTimestamp = first.ParsedTimestamp.Add(secondUpdateAfter)
|
|
||||||
db := handleBatches(t, [][]*ristypes.RISMessage{{first, second}})
|
|
||||||
|
|
||||||
firstSeen, err := db.GetPrefixFirstSeen(t.Context(),
|
|
||||||
netip.MustParsePrefix(testV4Prefix))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GetPrefixFirstSeen(%s): %v", testV4Prefix, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !firstSeen.Equal(first.ParsedTimestamp) {
|
|
||||||
t.Errorf("first_seen of %s = %s, want %s", testV4Prefix,
|
|
||||||
firstSeen, first.ParsedTimestamp)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPrefixHandlerDoesNotStoreAPrefixThatDoesNotParse runs an announcement of
|
|
||||||
// "not-a-prefix" and a withdrawal of "also/bad" through the prefix handler into
|
|
||||||
// a real database. Neither is stored in the prefix tables: the prefix count in
|
|
||||||
// the stats stays 0, both as kept in memory and as counted from the tables when
|
|
||||||
// the database is opened again.
|
|
||||||
func TestPrefixHandlerDoesNotStoreAPrefixThatDoesNotParse(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
cfg := &config.Config{StateDir: t.TempDir()}
|
|
||||||
|
|
||||||
db, err := database.New(cfg, logger.New())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to create database: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
h := prefixHandlerWithoutTimer(db)
|
|
||||||
|
|
||||||
withdrawal := &ristypes.RISMessage{
|
|
||||||
Type: ristypes.MessageTypeUpdate,
|
|
||||||
Peer: testPeerIP,
|
|
||||||
ParsedTimestamp: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC),
|
|
||||||
Withdrawals: []string{"also/bad"},
|
|
||||||
}
|
|
||||||
h.HandleMessage(peerAnnouncement(testPeerIP, "not-a-prefix"))
|
|
||||||
h.HandleMessage(withdrawal)
|
|
||||||
h.flushBatch()
|
|
||||||
|
|
||||||
checkNoPrefixes := func(when string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
stats, err := db.GetStatsContext(context.Background())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GetStatsContext %s: %v", when, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if stats.Prefixes != 0 {
|
|
||||||
t.Errorf("prefix count %s = %d, want 0", when, stats.Prefixes)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
checkNoPrefixes("after the flush")
|
|
||||||
|
|
||||||
err = db.Close()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Close: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
db, err = database.New(cfg, logger.New())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to open the database again: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Cleanup(func() { _ = db.Close() })
|
|
||||||
|
|
||||||
checkNoPrefixes("after opening the database again")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestStateMessagesGoToThePrefixHandlerNotThePeerHandler checks that STATE
|
|
||||||
// messages go to the prefix handler, which acts on a down, and not to the peer
|
|
||||||
// handler, which would record a peer whose session never comes up as active.
|
|
||||||
func TestStateMessagesGoToThePrefixHandlerNotThePeerHandler(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
if !(&PrefixHandler{}).WantsMessage(ristypes.MessageTypeState) {
|
|
||||||
t.Error("the prefix handler does not want STATE messages")
|
|
||||||
}
|
|
||||||
|
|
||||||
if (&PeerHandler{}).WantsMessage(ristypes.MessageTypeState) {
|
|
||||||
t.Error("the peer handler wants STATE messages")
|
|
||||||
}
|
|
||||||
|
|
||||||
if !(&PeerHandler{}).WantsMessage(ristypes.MessageTypeUpdate) {
|
|
||||||
t.Error("the peer handler does not want UPDATE messages")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPrefixHandlerStopLogsNothingWhenItWroteEverything stops a prefix
|
|
||||||
// handler, with no deadline, after a peer announced a prefix and its session
|
|
||||||
// went down. Its last flush writes the routing table's changes and the prefix,
|
|
||||||
// so Stop must log nothing.
|
|
||||||
func TestPrefixHandlerStopLogsNothingWhenItWroteEverything(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var logs bytes.Buffer
|
|
||||||
|
|
||||||
h := NewPrefixHandler(newTestDatabase(t), newErrorLogger(&logs),
|
|
||||||
routetable.New(routetable.Params{}))
|
|
||||||
h.HandleMessage(peerAnnouncement(testPeerIP, testV4Prefix))
|
|
||||||
h.HandleMessage(sessionDown())
|
|
||||||
h.Stop(t.Context())
|
|
||||||
|
|
||||||
if logs.Len() != 0 {
|
|
||||||
t.Errorf("Stop logged %q after it wrote everything", logs.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// longerThanAnyTest is a timer interval that no test lasts. A test gives it to
|
|
||||||
// newPrefixHandler, newASHandler, newPeerHandler or newPeeringHandler when the
|
|
||||||
// handler must write only when its batch is full, when the test says, or on
|
|
||||||
// Stop.
|
|
||||||
const longerThanAnyTest = time.Hour
|
|
||||||
|
|
||||||
// prefixHandlerWithoutTimer returns a prefix handler that writes to store and
|
|
||||||
// applies the updates to an empty routing table. It is built without
|
|
||||||
// NewPrefixHandler's flush goroutine, so the test flushes itself.
|
|
||||||
func prefixHandlerWithoutTimer(store database.Store) *PrefixHandler {
|
|
||||||
return &PrefixHandler{
|
|
||||||
db: store,
|
|
||||||
logger: logger.New(),
|
|
||||||
table: routetable.New(routetable.Params{}),
|
|
||||||
batchMaxSize: prefixBatchMaxSize,
|
|
||||||
tableChangesPerWrite: maxTableChangesPerWrite,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// newTestDatabase opens a database in a new temporary directory. It is closed
|
|
||||||
// when the test ends.
|
|
||||||
func newTestDatabase(t *testing.T) *database.Database {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to create database: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Cleanup(func() { _ = db.Close() })
|
|
||||||
|
|
||||||
return db
|
|
||||||
}
|
|
||||||
|
|
||||||
// handleBatches runs each batch of messages through a prefix handler into a
|
|
||||||
// new database, flushing after each batch, and returns the database.
|
|
||||||
func handleBatches(
|
|
||||||
t *testing.T, batches [][]*ristypes.RISMessage,
|
|
||||||
) *database.Database {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
db := newTestDatabase(t)
|
|
||||||
h := prefixHandlerWithoutTimer(db)
|
|
||||||
|
|
||||||
for _, batch := range batches {
|
|
||||||
for _, msg := range batch {
|
|
||||||
h.HandleMessage(msg)
|
|
||||||
}
|
|
||||||
|
|
||||||
h.flushBatch()
|
|
||||||
}
|
|
||||||
|
|
||||||
return db
|
|
||||||
}
|
|
||||||
|
|
||||||
// peerAnnouncement returns an UPDATE message in which peer announces prefixes
|
|
||||||
// with origin AS testASNB.
|
|
||||||
func peerAnnouncement(peer string, prefixes ...string) *ristypes.RISMessage {
|
|
||||||
return &ristypes.RISMessage{
|
|
||||||
Type: ristypes.MessageTypeUpdate,
|
|
||||||
Peer: peer,
|
|
||||||
Path: ristypes.ASPath{Sequence: []int{testASNA, testASNB}},
|
|
||||||
ParsedTimestamp: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC),
|
|
||||||
Announcements: []ristypes.RISAnnouncement{
|
|
||||||
{NextHop: peer, Prefixes: prefixes},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// sessionDown returns the STATE message that says the BGP session of the peer
|
|
||||||
// testPeerIP went down.
|
|
||||||
func sessionDown() *ristypes.RISMessage {
|
|
||||||
return &ristypes.RISMessage{
|
|
||||||
Type: ristypes.MessageTypeState,
|
|
||||||
Peer: testPeerIP,
|
|
||||||
State: ristypes.StateDown,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,616 +0,0 @@
|
|||||||
package routewatch
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"log/slog"
|
|
||||||
"net/netip"
|
|
||||||
"reflect"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/database"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/ristypes"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/routetable"
|
|
||||||
)
|
|
||||||
|
|
||||||
// More peers and prefixes for the tests of the routing table in memory.
|
|
||||||
const (
|
|
||||||
secondPeerIP = "2001:db8:ffff::2"
|
|
||||||
thirdPeerIP = "2001:db8:ffff::3"
|
|
||||||
fourthPeerIP = "192.0.2.4"
|
|
||||||
fifthPeerIP = "192.0.2.5"
|
|
||||||
|
|
||||||
otherV4Prefix = "198.51.100.0/24"
|
|
||||||
setV4Prefix = "203.0.113.0/24"
|
|
||||||
|
|
||||||
// firstManyPrefix is the first prefix announceManyPrefixes announces.
|
|
||||||
firstManyPrefix = "10.0.0.0/32"
|
|
||||||
|
|
||||||
// testMaxTableChangesPerWrite is a limit of one write of the routing
|
|
||||||
// table's changes that a test exceeds quickly, in place of
|
|
||||||
// maxTableChangesPerWrite.
|
|
||||||
testMaxTableChangesPerWrite = 10
|
|
||||||
)
|
|
||||||
|
|
||||||
// tableCase is one test of the routing table in memory: the messages handed to
|
|
||||||
// the prefix handler, and for each prefix the peers that carry it with each
|
|
||||||
// origin AS after them.
|
|
||||||
type tableCase struct {
|
|
||||||
name string
|
|
||||||
messages []*ristypes.RISMessage
|
|
||||||
want map[string]map[int][]string
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPrefixHandlerAppliesUpdatesToTheRoutingTable hands the prefix handler
|
|
||||||
// announcements, withdrawals and a peer's session going down.
|
|
||||||
func TestPrefixHandlerAppliesUpdatesToTheRoutingTable(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
announceAndWithdraw := peerAnnouncement(testPeerIP, testV4Prefix)
|
|
||||||
announceAndWithdraw.Withdrawals = []string{testV4Prefix}
|
|
||||||
|
|
||||||
checkTableCases(t, []tableCase{
|
|
||||||
{
|
|
||||||
name: "an announcement",
|
|
||||||
messages: []*ristypes.RISMessage{peerAnnouncement(testPeerIP, testV4Prefix)},
|
|
||||||
want: map[string]map[int][]string{testV4Prefix: {testASNB: {testPeerIP}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a withdrawal",
|
|
||||||
messages: []*ristypes.RISMessage{
|
|
||||||
peerAnnouncement(testPeerIP, testV4Prefix),
|
|
||||||
peerWithdrawal(testPeerIP, testV4Prefix),
|
|
||||||
},
|
|
||||||
want: map[string]map[int][]string{testV4Prefix: {}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
// RFC 4271 section 4.3: the announcement counts, not the withdrawal.
|
|
||||||
name: "a message that announces and withdraws the prefix",
|
|
||||||
messages: []*ristypes.RISMessage{announceAndWithdraw},
|
|
||||||
want: map[string]map[int][]string{testV4Prefix: {testASNB: {testPeerIP}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a peer's session going down",
|
|
||||||
messages: []*ristypes.RISMessage{
|
|
||||||
peerAnnouncement(testPeerIP, testV4Prefix, testV6Prefix),
|
|
||||||
peerAnnouncement(secondPeerIP, testV4Prefix),
|
|
||||||
sessionDown(),
|
|
||||||
},
|
|
||||||
want: map[string]map[int][]string{
|
|
||||||
testV4Prefix: {testASNB: {secondPeerIP}},
|
|
||||||
testV6Prefix: {},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "an announcement after the peer's session went down",
|
|
||||||
messages: []*ristypes.RISMessage{
|
|
||||||
peerAnnouncement(testPeerIP, testV4Prefix),
|
|
||||||
sessionDown(),
|
|
||||||
peerAnnouncement(testPeerIP, otherV4Prefix),
|
|
||||||
},
|
|
||||||
want: map[string]map[int][]string{
|
|
||||||
testV4Prefix: {},
|
|
||||||
otherV4Prefix: {testASNB: {testPeerIP}},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPrefixHandlerAppliesAPathWithoutAnOriginToTheRoutingTable hands the
|
|
||||||
// prefix handler announcements whose AS path gives no origin. One whose path
|
|
||||||
// is empty or holds AS 0 removes the peer's route, as a withdrawal does; one
|
|
||||||
// whose path ends in an AS set of several members is kept with origin 0.
|
|
||||||
func TestPrefixHandlerAppliesAPathWithoutAnOriginToTheRoutingTable(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
announceWithPath := func(path ristypes.ASPath) *ristypes.RISMessage {
|
|
||||||
msg := peerAnnouncement(testPeerIP, testV4Prefix)
|
|
||||||
msg.Path = path
|
|
||||||
|
|
||||||
return msg
|
|
||||||
}
|
|
||||||
|
|
||||||
checkTableCases(t, []tableCase{
|
|
||||||
{
|
|
||||||
name: "a path whose last AS is 0",
|
|
||||||
messages: []*ristypes.RISMessage{
|
|
||||||
peerAnnouncement(testPeerIP, testV4Prefix),
|
|
||||||
announceWithPath(ristypes.ASPath{Sequence: []int{testASNA, 0}}),
|
|
||||||
},
|
|
||||||
want: map[string]map[int][]string{testV4Prefix: {}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "an empty path",
|
|
||||||
messages: []*ristypes.RISMessage{
|
|
||||||
peerAnnouncement(testPeerIP, testV4Prefix),
|
|
||||||
announceWithPath(ristypes.ASPath{}),
|
|
||||||
},
|
|
||||||
want: map[string]map[int][]string{testV4Prefix: {}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a path with AS 0 before its end",
|
|
||||||
messages: []*ristypes.RISMessage{
|
|
||||||
peerAnnouncement(testPeerIP, testV4Prefix),
|
|
||||||
announceWithPath(decodePath(t, testASNA, 0, testASNB)),
|
|
||||||
},
|
|
||||||
want: map[string]map[int][]string{testV4Prefix: {}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a path that ends in an AS set",
|
|
||||||
messages: []*ristypes.RISMessage{
|
|
||||||
announceWithPath(decodePath(t, testASNA, []int{testASNC, testASND})),
|
|
||||||
},
|
|
||||||
want: map[string]map[int][]string{testV4Prefix: {0: {testPeerIP}}},
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPrefixHandlerKeepsThePathOfAPathEndingInASet announces a prefix with AS
|
|
||||||
// paths that end in an AS set. Any member of a set of several may be the
|
|
||||||
// origin, so the routing table keeps the route with origin 0 (none); a set of
|
|
||||||
// one member counts as that AS. The kept AS path lists the set's members after
|
|
||||||
// the rest of the path.
|
|
||||||
func TestPrefixHandlerKeepsThePathOfAPathEndingInASet(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
path []any
|
|
||||||
wantOrigin int
|
|
||||||
wantPath []int
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "a set of several members",
|
|
||||||
path: []any{testASNA, testASNB, []int{testASNC, testASND}},
|
|
||||||
wantPath: []int{testASNA, testASNB, testASNC, testASND},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a set of one member",
|
|
||||||
path: []any{testASNA, testASNB, []int{testASNC}},
|
|
||||||
wantOrigin: testASNC,
|
|
||||||
wantPath: []int{testASNA, testASNB, testASNC},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
msg := peerAnnouncement(testPeerIP, testV4Prefix)
|
|
||||||
msg.Path = decodePath(t, tt.path...)
|
|
||||||
|
|
||||||
h := prefixHandlerWithoutTimer(newMockStore())
|
|
||||||
h.HandleMessage(msg)
|
|
||||||
|
|
||||||
origins := h.table.Origins(netip.MustParsePrefix(testV4Prefix))
|
|
||||||
if len(origins) != 1 || origins[0].ASN != tt.wantOrigin ||
|
|
||||||
len(origins[0].Routes) != 1 ||
|
|
||||||
!slices.Equal(origins[0].Routes[0].ASPath, tt.wantPath) {
|
|
||||||
t.Errorf("Origins(%s) = %+v, want one route with origin AS%d "+
|
|
||||||
"and AS path %v", testV4Prefix, origins, tt.wantOrigin,
|
|
||||||
tt.wantPath)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkTableCases hands the messages of each case to a new prefix handler and
|
|
||||||
// checks its routing table right after, with no flush.
|
|
||||||
func checkTableCases(t *testing.T, tests []tableCase) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
h := prefixHandlerWithoutTimer(newMockStore())
|
|
||||||
for _, msg := range tt.messages {
|
|
||||||
h.HandleMessage(msg)
|
|
||||||
}
|
|
||||||
|
|
||||||
for prefix, want := range tt.want {
|
|
||||||
got := tableOrigins(h.table, prefix)
|
|
||||||
if !reflect.DeepEqual(got, want) {
|
|
||||||
t.Errorf("peers of %s by origin AS = %v, want %v",
|
|
||||||
prefix, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// tableWritesStore is a real database that hands each write of the routing
|
|
||||||
// table's changes to written once it is made.
|
|
||||||
type tableWritesStore struct {
|
|
||||||
*database.Database
|
|
||||||
|
|
||||||
written chan []routetable.Change
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s tableWritesStore) WriteRouteChanges(
|
|
||||||
changes []routetable.Change, newPeers map[string]uint,
|
|
||||||
) error {
|
|
||||||
err := s.Database.WriteRouteChanges(changes, newPeers)
|
|
||||||
s.written <- changes
|
|
||||||
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPrefixHandlerWritesTheTableChangesEachSecond hands a running prefix
|
|
||||||
// handler an announcement and then its withdrawal, each far short of a full
|
|
||||||
// batch. The flush timer must write the change of each to the database.
|
|
||||||
func TestPrefixHandlerWritesTheTableChangesEachSecond(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// The two writes the test waits for, and room for one more.
|
|
||||||
const writesBuffered = 3
|
|
||||||
|
|
||||||
store := tableWritesStore{
|
|
||||||
Database: newTestDatabase(t),
|
|
||||||
written: make(chan []routetable.Change, writesBuffered),
|
|
||||||
}
|
|
||||||
h := NewPrefixHandler(store, logger.New(), routetable.New(routetable.Params{}))
|
|
||||||
|
|
||||||
defer h.Stop(t.Context())
|
|
||||||
|
|
||||||
prefix := netip.MustParsePrefix(testV4Prefix)
|
|
||||||
|
|
||||||
for _, step := range []struct {
|
|
||||||
msg *ristypes.RISMessage
|
|
||||||
wantRemoved bool
|
|
||||||
}{
|
|
||||||
{msg: peerAnnouncement(testPeerIP, testV4Prefix)},
|
|
||||||
{msg: peerWithdrawal(testPeerIP, testV4Prefix), wantRemoved: true},
|
|
||||||
} {
|
|
||||||
h.HandleMessage(step.msg)
|
|
||||||
|
|
||||||
select {
|
|
||||||
case changes := <-store.written:
|
|
||||||
if len(changes) != 1 || changes[0].Prefix != prefix ||
|
|
||||||
changes[0].OriginASN != testASNB ||
|
|
||||||
changes[0].Removed != step.wantRemoved {
|
|
||||||
t.Fatalf("wrote %+v, want the pair of %s and AS%d with "+
|
|
||||||
"removed %t", changes, prefix, testASNB, step.wantRemoved)
|
|
||||||
}
|
|
||||||
case <-time.After(stepTimeout):
|
|
||||||
t.Fatalf("the routing table's change was not written within %s",
|
|
||||||
stepTimeout)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// errTestTableWrite is the error a test store's failing write of the routing
|
|
||||||
// table's changes returns.
|
|
||||||
var errTestTableWrite = errors.New("test: the write of the table's changes failed")
|
|
||||||
|
|
||||||
// firstTableWriteFailsStore is a real database whose first write of the
|
|
||||||
// routing table's changes fails, writing nothing. writes counts the writes
|
|
||||||
// asked for.
|
|
||||||
type firstTableWriteFailsStore struct {
|
|
||||||
*database.Database
|
|
||||||
|
|
||||||
writes int
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *firstTableWriteFailsStore) WriteRouteChanges(
|
|
||||||
changes []routetable.Change, newPeers map[string]uint,
|
|
||||||
) error {
|
|
||||||
s.writes++
|
|
||||||
if s.writes == 1 {
|
|
||||||
return errTestTableWrite
|
|
||||||
}
|
|
||||||
|
|
||||||
return s.Database.WriteRouteChanges(changes, newPeers)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPrefixHandlerWritesAFailedTableWriteAgain runs two flushes into a
|
|
||||||
// database whose first write of the routing table's changes fails. Before the
|
|
||||||
// first, two new peers announce a prefix each; before the second, one of them
|
|
||||||
// announces its prefix again with another path to the same origin AS. The
|
|
||||||
// second write must hold the changes and new peers of the first, with the later
|
|
||||||
// path, so the database holds the table that is in memory.
|
|
||||||
func TestPrefixHandlerWritesAFailedTableWriteAgain(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := newTestDatabase(t)
|
|
||||||
store := &firstTableWriteFailsStore{Database: db}
|
|
||||||
h := prefixHandlerWithoutTimer(store)
|
|
||||||
|
|
||||||
first := peerAnnouncement(testPeerIP, testV4Prefix)
|
|
||||||
h.HandleMessage(first)
|
|
||||||
h.HandleMessage(peerAnnouncement(secondPeerIP, otherV4Prefix))
|
|
||||||
h.flushBatch()
|
|
||||||
|
|
||||||
laterPath := []int{testASNC, testASNB}
|
|
||||||
later := peerAnnouncement(testPeerIP, testV4Prefix)
|
|
||||||
later.Path = ristypes.ASPath{Sequence: laterPath}
|
|
||||||
later.ParsedTimestamp = first.ParsedTimestamp.Add(secondUpdateAfter)
|
|
||||||
h.HandleMessage(later)
|
|
||||||
h.flushBatch()
|
|
||||||
|
|
||||||
if store.writes != 2 {
|
|
||||||
t.Fatalf("the store was asked for %d writes of the table's changes, "+
|
|
||||||
"want 2", store.writes)
|
|
||||||
}
|
|
||||||
|
|
||||||
read, err := db.ReadRouteTable(t.Context())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ReadRouteTable: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
checkSameTable(t, read, h.table, testV4Prefix, otherV4Prefix)
|
|
||||||
|
|
||||||
origins := read.Origins(netip.MustParsePrefix(testV4Prefix))
|
|
||||||
if len(origins) != 1 || len(origins[0].Routes) != 1 ||
|
|
||||||
!slices.Equal(origins[0].Routes[0].ASPath, laterPath) {
|
|
||||||
t.Errorf("Origins(%s) = %+v, want one route with the path %v",
|
|
||||||
testV4Prefix, origins, laterPath)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// tableWriteSizesStore is a real database that records how many changes and
|
|
||||||
// how many new peers each write of the routing table's changes holds.
|
|
||||||
type tableWriteSizesStore struct {
|
|
||||||
*database.Database
|
|
||||||
|
|
||||||
changes []int
|
|
||||||
newPeers []int
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *tableWriteSizesStore) WriteRouteChanges(
|
|
||||||
changes []routetable.Change, newPeers map[string]uint,
|
|
||||||
) error {
|
|
||||||
s.changes = append(s.changes, len(changes))
|
|
||||||
s.newPeers = append(s.newPeers, len(newPeers))
|
|
||||||
|
|
||||||
return s.Database.WriteRouteChanges(changes, newPeers)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPrefixHandlerWritesManyTableChangesOverSeveralFlushes has one new peer
|
|
||||||
// change one and a half times as many prefix and origin AS pairs as one write
|
|
||||||
// takes, then flushes three times. The first write must hold as many pairs as
|
|
||||||
// one write takes, and the peer. The second must hold the other pairs, and the
|
|
||||||
// third flush must write nothing. The database must then hold the table in
|
|
||||||
// memory. The handler's limit is small, and its timer does not fire.
|
|
||||||
func TestPrefixHandlerWritesManyTableChangesOverSeveralFlushes(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const (
|
|
||||||
pairs = testMaxTableChangesPerWrite * 3 / 2
|
|
||||||
flushes = 3
|
|
||||||
)
|
|
||||||
|
|
||||||
db := newTestDatabase(t)
|
|
||||||
store := &tableWriteSizesStore{Database: db}
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
announceManyPrefixes(table, pairs)
|
|
||||||
|
|
||||||
h := newPrefixHandler(store, logger.New(), table, longerThanAnyTest,
|
|
||||||
testMaxTableChangesPerWrite)
|
|
||||||
|
|
||||||
defer stopInTime(t, h)
|
|
||||||
|
|
||||||
for range flushes {
|
|
||||||
h.flushBatch()
|
|
||||||
}
|
|
||||||
|
|
||||||
wantChanges := []int{
|
|
||||||
testMaxTableChangesPerWrite, pairs - testMaxTableChangesPerWrite,
|
|
||||||
}
|
|
||||||
wantNewPeers := []int{1, 0}
|
|
||||||
|
|
||||||
if !slices.Equal(store.changes, wantChanges) ||
|
|
||||||
!slices.Equal(store.newPeers, wantNewPeers) {
|
|
||||||
t.Errorf("the writes held %v pairs and %v new peers, want %v and %v",
|
|
||||||
store.changes, store.newPeers, wantChanges, wantNewPeers)
|
|
||||||
}
|
|
||||||
|
|
||||||
read, err := db.ReadRouteTable(t.Context())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ReadRouteTable: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
checkSameTable(t, read, h.table, firstManyPrefix)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPrefixHandlerStopLogsTheTableChangesLeftUnwritten stops a prefix handler,
|
|
||||||
// with no deadline, whose routing table holds more changed pairs than one write
|
|
||||||
// takes. Its last flush writes one write's worth, so Stop must log the pairs
|
|
||||||
// that no write took, and must not say that the stop deadline passed. The
|
|
||||||
// flush timer must not fire, or its writes could leave no pairs for Stop to
|
|
||||||
// log. The handler's limit of one write is small.
|
|
||||||
func TestPrefixHandlerStopLogsTheTableChangesLeftUnwritten(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const pairs = testMaxTableChangesPerWrite + 1
|
|
||||||
|
|
||||||
var logs bytes.Buffer
|
|
||||||
|
|
||||||
store := &tableWriteSizesStore{Database: newTestDatabase(t)}
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
announceManyPrefixes(table, pairs)
|
|
||||||
|
|
||||||
h := newPrefixHandler(store, newErrorLogger(&logs), table, longerThanAnyTest,
|
|
||||||
testMaxTableChangesPerWrite)
|
|
||||||
h.Stop(t.Context())
|
|
||||||
|
|
||||||
written := 0
|
|
||||||
for _, n := range store.changes {
|
|
||||||
written += n
|
|
||||||
}
|
|
||||||
|
|
||||||
logged := lastLogged(t, &logs)
|
|
||||||
if strings.Contains(logged.Msg, "deadline") {
|
|
||||||
t.Errorf("Stop logged %q although its deadline had not passed",
|
|
||||||
logged.Msg)
|
|
||||||
}
|
|
||||||
|
|
||||||
if logged.TableChanges != pairs-written {
|
|
||||||
t.Errorf("Stop logged %d table changes not written, want %d",
|
|
||||||
logged.TableChanges, pairs-written)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// readLog holds what readRouteTable logs.
|
|
||||||
//
|
|
||||||
//nolint:tagliatelle // field names are the keys readRouteTable logs
|
|
||||||
type readLog struct {
|
|
||||||
Msg string `json:"msg"`
|
|
||||||
Pairs int `json:"pairs"`
|
|
||||||
Peers int `json:"peers"`
|
|
||||||
DurationMs *int64 `json:"duration_ms"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestARestartReadsTheRoutingTableBack runs announcements, a withdrawal and a
|
|
||||||
// peer's session going down through a prefix handler into a database, then
|
|
||||||
// reads the table back as startup does. The table read must answer as the one
|
|
||||||
// in memory, and the log must give its pairs and peers. A new handler on the
|
|
||||||
// table read then takes later updates and writes them as it stops: the
|
|
||||||
// database must then hold what the first table holds after the same updates.
|
|
||||||
func TestARestartReadsTheRoutingTableBack(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
prefixes := []string{testV4Prefix, testV6Prefix, otherV4Prefix, setV4Prefix}
|
|
||||||
|
|
||||||
db := newTestDatabase(t)
|
|
||||||
h := prefixHandlerWithoutTimer(db)
|
|
||||||
|
|
||||||
viaOtherOrigin := peerAnnouncement(thirdPeerIP, testV4Prefix)
|
|
||||||
viaOtherOrigin.Path = ristypes.ASPath{Sequence: []int{testASNA, testASNC}}
|
|
||||||
endsInSet := peerAnnouncement(secondPeerIP, setV4Prefix)
|
|
||||||
endsInSet.Path = decodePath(t, testASNA, []int{testASNC, testASND})
|
|
||||||
|
|
||||||
for _, msg := range []*ristypes.RISMessage{
|
|
||||||
peerAnnouncement(testPeerIP, testV4Prefix, testV6Prefix),
|
|
||||||
peerAnnouncement(secondPeerIP, testV4Prefix),
|
|
||||||
viaOtherOrigin,
|
|
||||||
endsInSet,
|
|
||||||
peerAnnouncement(thirdPeerIP, otherV4Prefix),
|
|
||||||
peerWithdrawal(thirdPeerIP, otherV4Prefix),
|
|
||||||
peerAnnouncement(fourthPeerIP, otherV4Prefix),
|
|
||||||
sessionDownOf(fourthPeerIP),
|
|
||||||
} {
|
|
||||||
h.HandleMessage(msg)
|
|
||||||
}
|
|
||||||
|
|
||||||
h.flushBatch()
|
|
||||||
|
|
||||||
var logs bytes.Buffer
|
|
||||||
|
|
||||||
table, err := readRouteTable(db,
|
|
||||||
&logger.Logger{Logger: slog.New(slog.NewJSONHandler(&logs, nil))})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("readRouteTable: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
checkSameTable(t, table, h.table, prefixes...)
|
|
||||||
|
|
||||||
// testV4Prefix with two origin ASes, testV6Prefix and setV4Prefix; four
|
|
||||||
// peers, the last of which carries nothing.
|
|
||||||
const wantPairs, wantPeers = 4, 4
|
|
||||||
|
|
||||||
var logged readLog
|
|
||||||
|
|
||||||
err = json.Unmarshal(bytes.TrimSpace(logs.Bytes()), &logged)
|
|
||||||
if err != nil || logged.Pairs != wantPairs || logged.Peers != wantPeers ||
|
|
||||||
logged.DurationMs == nil {
|
|
||||||
t.Errorf("readRouteTable logged %q, want %d pairs, %d peers and the "+
|
|
||||||
"duration", logs.String(), wantPairs, wantPeers)
|
|
||||||
}
|
|
||||||
|
|
||||||
restarted := NewPrefixHandler(db, logger.New(), table)
|
|
||||||
|
|
||||||
for _, msg := range []*ristypes.RISMessage{
|
|
||||||
peerWithdrawal(testPeerIP, testV6Prefix),
|
|
||||||
sessionDownOf(secondPeerIP),
|
|
||||||
peerAnnouncement(fourthPeerIP, otherV4Prefix),
|
|
||||||
peerAnnouncement(fifthPeerIP, setV4Prefix),
|
|
||||||
} {
|
|
||||||
// The first handler only applies the updates to its table.
|
|
||||||
h.HandleMessage(msg)
|
|
||||||
restarted.HandleMessage(msg)
|
|
||||||
}
|
|
||||||
|
|
||||||
restarted.Stop(t.Context())
|
|
||||||
|
|
||||||
read, err := db.ReadRouteTable(t.Context())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ReadRouteTable after the restart: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
checkSameTable(t, read, h.table, prefixes...)
|
|
||||||
}
|
|
||||||
|
|
||||||
// peerWithdrawal returns an UPDATE message in which peer withdraws prefixes.
|
|
||||||
func peerWithdrawal(peer string, prefixes ...string) *ristypes.RISMessage {
|
|
||||||
return &ristypes.RISMessage{
|
|
||||||
Type: ristypes.MessageTypeUpdate,
|
|
||||||
Peer: peer,
|
|
||||||
ParsedTimestamp: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC),
|
|
||||||
Withdrawals: prefixes,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// announceManyPrefixes has testPeerIP announce n prefixes in table, with the
|
|
||||||
// path of peerAnnouncement: the /32 of each address from firstManyPrefix on.
|
|
||||||
func announceManyPrefixes(table *routetable.Table, n int) {
|
|
||||||
addr := netip.MustParsePrefix(firstManyPrefix).Addr()
|
|
||||||
route := routetable.Route{
|
|
||||||
Peer: testPeerIP,
|
|
||||||
ASPath: []int{testASNA, testASNB},
|
|
||||||
Time: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC),
|
|
||||||
}
|
|
||||||
|
|
||||||
for range n {
|
|
||||||
table.Announce(netip.PrefixFrom(addr, addr.BitLen()), testASNB, route)
|
|
||||||
addr = addr.Next()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// sessionDownOf returns the STATE message that says the BGP session of peer
|
|
||||||
// went down.
|
|
||||||
func sessionDownOf(peer string) *ristypes.RISMessage {
|
|
||||||
return &ristypes.RISMessage{
|
|
||||||
Type: ristypes.MessageTypeState,
|
|
||||||
Peer: peer,
|
|
||||||
State: ristypes.StateDown,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// tableOrigins returns, for each origin AS of prefix in table, the peers that
|
|
||||||
// carry the prefix with it.
|
|
||||||
func tableOrigins(table *routetable.Table, prefix string) map[int][]string {
|
|
||||||
origins := make(map[int][]string)
|
|
||||||
for _, o := range table.Origins(netip.MustParsePrefix(prefix)) {
|
|
||||||
origins[o.ASN] = o.Peers
|
|
||||||
}
|
|
||||||
|
|
||||||
return origins
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkSameTable checks that got answers as want does for each of prefixes, and
|
|
||||||
// that the two tables have the same counts.
|
|
||||||
func checkSameTable(
|
|
||||||
t *testing.T, got, want *routetable.Table, prefixes ...string,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
for _, text := range prefixes {
|
|
||||||
prefix := netip.MustParsePrefix(text)
|
|
||||||
|
|
||||||
gotOrigins, wantOrigins := got.Origins(prefix), want.Origins(prefix)
|
|
||||||
if !reflect.DeepEqual(gotOrigins, wantOrigins) {
|
|
||||||
t.Errorf("Origins(%s) = %+v, want %+v", text, gotOrigins, wantOrigins)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
gotCounts, wantCounts := got.Counts(), want.Counts()
|
|
||||||
if !reflect.DeepEqual(gotCounts, wantCounts) {
|
|
||||||
t.Errorf("Counts() = %+v, want %+v", gotCounts, wantCounts)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,213 @@
|
|||||||
|
package routewatch
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"slices"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.eeqj.de/sneak/routewatch/internal/config"
|
||||||
|
"git.eeqj.de/sneak/routewatch/internal/database"
|
||||||
|
"git.eeqj.de/sneak/routewatch/internal/logger"
|
||||||
|
"git.eeqj.de/sneak/routewatch/internal/ristypes"
|
||||||
|
)
|
||||||
|
|
||||||
|
const testPeerIP = "2001:db8:ffff::1"
|
||||||
|
|
||||||
|
// TestPrefixHandlerStoresPrefixesTheIPLookupFinds runs announcements and
|
||||||
|
// withdrawals through the prefix handler into a real database and looks the
|
||||||
|
// addresses up. The prefixes are written the way the feed sends them: IPv6
|
||||||
|
// announcements compressed, IPv6 withdrawals uncompressed. The withdrawal must
|
||||||
|
// remove the route the announcement stored, and the IP lookup must find the
|
||||||
|
// stored prefix.
|
||||||
|
func TestPrefixHandlerStoresPrefixesTheIPLookupFinds(t *testing.T) {
|
||||||
|
db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to create database: %v", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = db.Close() }()
|
||||||
|
|
||||||
|
// Built without NewPrefixHandler's flush timer, so the test flushes itself.
|
||||||
|
h := &PrefixHandler{db: db, logger: logger.New()}
|
||||||
|
flush := func() {
|
||||||
|
h.mu.Lock()
|
||||||
|
defer h.mu.Unlock()
|
||||||
|
h.flushBatchLocked()
|
||||||
|
}
|
||||||
|
|
||||||
|
ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
||||||
|
h.HandleMessage(&ristypes.RISMessage{
|
||||||
|
Peer: testPeerIP,
|
||||||
|
Path: ristypes.ASPath{testASNA, testASNB},
|
||||||
|
ParsedTimestamp: ts,
|
||||||
|
Announcements: []ristypes.RISAnnouncement{{
|
||||||
|
NextHop: testPeerIP,
|
||||||
|
Prefixes: []string{"2001:db8:1::/48", "192.0.2.0/24"},
|
||||||
|
}},
|
||||||
|
})
|
||||||
|
flush()
|
||||||
|
|
||||||
|
for ip, want := range map[string]string{
|
||||||
|
"2001:db8:1::1": "2001:db8:1::/48",
|
||||||
|
"192.0.2.1": "192.0.2.0/24",
|
||||||
|
} {
|
||||||
|
info, err := db.GetIPInfo(ip)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetIPInfo(%s) after announcement: %v", ip, err)
|
||||||
|
}
|
||||||
|
if info.Netblock != want || info.ASN != testASNB {
|
||||||
|
t.Errorf("GetIPInfo(%s) = %s AS%d, want %s AS%d", ip, info.Netblock, info.ASN, want, testASNB)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
h.HandleMessage(&ristypes.RISMessage{
|
||||||
|
Peer: testPeerIP,
|
||||||
|
ParsedTimestamp: ts.Add(time.Minute),
|
||||||
|
Withdrawals: []string{"2001:db8:1:0:0:0:0:0/48", "192.0.2.0/24"},
|
||||||
|
})
|
||||||
|
flush()
|
||||||
|
|
||||||
|
for _, ip := range []string{"2001:db8:1::1", "192.0.2.1"} {
|
||||||
|
if info, err := db.GetIPInfo(ip); !errors.Is(err, database.ErrNoRoute) {
|
||||||
|
t.Errorf("GetIPInfo(%s) after withdrawal = %+v, %v; want ErrNoRoute", ip, info, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestPrefixHandlerAppliesEachPeersLastUpdateInABatch runs updates for one
|
||||||
|
// prefix through the prefix handler into a real database. All updates carry the
|
||||||
|
// same timestamp, as updates within one second of the feed do. Within a batch,
|
||||||
|
// the last update each peer sent for the prefix decides whether that peer has a
|
||||||
|
// live route for it, and updates from different peers are all applied. The
|
||||||
|
// in-memory route count and prefix distribution must match the route table.
|
||||||
|
func TestPrefixHandlerAppliesEachPeersLastUpdateInABatch(t *testing.T) {
|
||||||
|
const (
|
||||||
|
prefix = "198.51.100.0/24"
|
||||||
|
prefixMaskLength = 24
|
||||||
|
otherPeerIP = "2001:db8:ffff::2"
|
||||||
|
)
|
||||||
|
|
||||||
|
ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
||||||
|
announce := func(peer string) *ristypes.RISMessage {
|
||||||
|
return &ristypes.RISMessage{
|
||||||
|
Peer: peer,
|
||||||
|
Path: ristypes.ASPath{testASNA, testASNB},
|
||||||
|
ParsedTimestamp: ts,
|
||||||
|
Announcements: []ristypes.RISAnnouncement{{NextHop: peer, Prefixes: []string{prefix}}},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
withdraw := func(peer string) *ristypes.RISMessage {
|
||||||
|
return &ristypes.RISMessage{Peer: peer, ParsedTimestamp: ts, Withdrawals: []string{prefix}}
|
||||||
|
}
|
||||||
|
announceAndWithdraw := func(peer string) *ristypes.RISMessage {
|
||||||
|
msg := announce(peer)
|
||||||
|
msg.Withdrawals = []string{prefix}
|
||||||
|
|
||||||
|
return msg
|
||||||
|
}
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
// Each inner list is handled and then flushed as one batch.
|
||||||
|
batches [][]*ristypes.RISMessage
|
||||||
|
// The peers left with a live route for the prefix, by peer IP.
|
||||||
|
wantPeers []string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "two peers announce the prefix",
|
||||||
|
batches: [][]*ristypes.RISMessage{{announce(testPeerIP), announce(otherPeerIP)}},
|
||||||
|
wantPeers: []string{testPeerIP, otherPeerIP},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "two peers withdraw the prefix",
|
||||||
|
batches: [][]*ristypes.RISMessage{
|
||||||
|
{announce(testPeerIP)},
|
||||||
|
{announce(otherPeerIP)},
|
||||||
|
{withdraw(testPeerIP), withdraw(otherPeerIP)},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "a peer announces and then withdraws the prefix",
|
||||||
|
batches: [][]*ristypes.RISMessage{{announce(testPeerIP), withdraw(testPeerIP)}},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "a peer withdraws and then re-announces the prefix",
|
||||||
|
batches: [][]*ristypes.RISMessage{
|
||||||
|
{announce(testPeerIP)},
|
||||||
|
{withdraw(testPeerIP), announce(testPeerIP)},
|
||||||
|
},
|
||||||
|
wantPeers: []string{testPeerIP},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// RFC 4271 section 4.3: the announcement counts, not the withdrawal.
|
||||||
|
name: "one message announces and withdraws the prefix",
|
||||||
|
batches: [][]*ristypes.RISMessage{
|
||||||
|
{announce(testPeerIP)},
|
||||||
|
{announceAndWithdraw(testPeerIP)},
|
||||||
|
},
|
||||||
|
wantPeers: []string{testPeerIP},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to create database: %v", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = db.Close() }()
|
||||||
|
|
||||||
|
// Built without NewPrefixHandler's flush timer, so the test flushes itself.
|
||||||
|
h := &PrefixHandler{db: db, logger: logger.New()}
|
||||||
|
for _, batch := range tt.batches {
|
||||||
|
for _, msg := range batch {
|
||||||
|
h.HandleMessage(msg)
|
||||||
|
}
|
||||||
|
h.mu.Lock()
|
||||||
|
h.flushBatchLocked()
|
||||||
|
h.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
routes, err := db.GetPrefixDetailsContext(ctx, prefix)
|
||||||
|
if err != nil && !errors.Is(err, database.ErrNoRoute) {
|
||||||
|
t.Fatalf("GetPrefixDetailsContext: %v", err)
|
||||||
|
}
|
||||||
|
var gotPeers []string
|
||||||
|
for _, route := range routes {
|
||||||
|
gotPeers = append(gotPeers, route.PeerIP)
|
||||||
|
}
|
||||||
|
if !slices.Equal(gotPeers, tt.wantPeers) {
|
||||||
|
t.Errorf("peers with a live route for %s = %v, want %v", prefix, gotPeers, tt.wantPeers)
|
||||||
|
}
|
||||||
|
|
||||||
|
stats, err := db.GetStatsContext(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetStatsContext: %v", err)
|
||||||
|
}
|
||||||
|
tableRoutesV4, _, err := db.GetLiveRouteCountsContext(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetLiveRouteCountsContext: %v", err)
|
||||||
|
}
|
||||||
|
if stats.IPv4Routes != len(tt.wantPeers) || tableRoutesV4 != len(tt.wantPeers) {
|
||||||
|
t.Errorf("IPv4 live routes: %d in memory, %d in the table, want %d",
|
||||||
|
stats.IPv4Routes, tableRoutesV4, len(tt.wantPeers))
|
||||||
|
}
|
||||||
|
|
||||||
|
var wantDistribution []database.PrefixDistribution
|
||||||
|
if len(tt.wantPeers) > 0 {
|
||||||
|
wantDistribution = []database.PrefixDistribution{{MaskLength: prefixMaskLength, Count: 1}}
|
||||||
|
}
|
||||||
|
queryDistributionV4, _, err := db.GetPrefixDistributionContext(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetPrefixDistributionContext: %v", err)
|
||||||
|
}
|
||||||
|
if !slices.Equal(stats.IPv4PrefixDistribution, wantDistribution) ||
|
||||||
|
!slices.Equal(queryDistributionV4, wantDistribution) {
|
||||||
|
t.Errorf("IPv4 prefix distribution: %v in memory, %v from the table, want %v",
|
||||||
|
stats.IPv4PrefixDistribution, queryDistributionV4, wantDistribution)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
+635
-1158
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,413 +0,0 @@
|
|||||||
package server
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"net/netip"
|
|
||||||
"reflect"
|
|
||||||
"slices"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/database"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/ristypes"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/routetable"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The origin ASes in the routing table that newTwoOriginTable fills.
|
|
||||||
const (
|
|
||||||
mostPeersASN = 64500
|
|
||||||
fewerPeersASN = 64501
|
|
||||||
v6OriginASN = 64502
|
|
||||||
)
|
|
||||||
|
|
||||||
// noKeptRoutePrefix is the prefix of newTwoOriginTable whose origin keeps no
|
|
||||||
// route.
|
|
||||||
const noKeptRoutePrefix = "198.51.100.0/24"
|
|
||||||
|
|
||||||
// routeTimeFormat is how the pages write a route's time.
|
|
||||||
const routeTimeFormat = "2006-01-02 15:04:05"
|
|
||||||
|
|
||||||
// newTwoOriginTable returns a routing table that holds:
|
|
||||||
// - testPrefixV4 with two origin ASes. Peers 1, 2 and 3 carry it with
|
|
||||||
// mostPeersASN, which keeps the routes of peers 3 and 2. Peer 4 carries
|
|
||||||
// it with fewerPeersASN, in the prefix's newest route.
|
|
||||||
// - noKeptRoutePrefix with mostPeersASN. Only peer 7 carries it, and its
|
|
||||||
// route is not kept: peers 5 and 6, whose routes were, withdrew it.
|
|
||||||
// - testPrefixV6 with v6OriginASN.
|
|
||||||
func newTwoOriginTable() *routetable.Table {
|
|
||||||
table := emptyTable()
|
|
||||||
|
|
||||||
for n := 1; n <= 3; n++ {
|
|
||||||
announce(table, testPrefixV4, mostPeersASN, testPeer(n), testRouteTime(n))
|
|
||||||
}
|
|
||||||
|
|
||||||
announce(table, testPrefixV4, fewerPeersASN, testPeer(4), testRouteTime(5))
|
|
||||||
|
|
||||||
announce(table, noKeptRoutePrefix, mostPeersASN, testPeer(5), testRouteTime(4))
|
|
||||||
announce(table, noKeptRoutePrefix, mostPeersASN, testPeer(6), testRouteTime(6))
|
|
||||||
announce(table, noKeptRoutePrefix, mostPeersASN, testPeer(7), testRouteTime(0))
|
|
||||||
withdraw(table, noKeptRoutePrefix, testPeer(5))
|
|
||||||
withdraw(table, noKeptRoutePrefix, testPeer(6))
|
|
||||||
|
|
||||||
announce(table, testPrefixV6, v6OriginASN, testPeerV6, testRouteTime(0))
|
|
||||||
|
|
||||||
return table
|
|
||||||
}
|
|
||||||
|
|
||||||
// get sends a GET for path through s's router and fails the test unless it
|
|
||||||
// answers 200.
|
|
||||||
func get(t *testing.T, s *Server, path string) *httptest.ResponseRecorder {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
rec := httptest.NewRecorder()
|
|
||||||
s.router.ServeHTTP(rec, httptest.NewRequestWithContext(
|
|
||||||
t.Context(), http.MethodGet, path, nil))
|
|
||||||
|
|
||||||
if rec.Code != http.StatusOK {
|
|
||||||
t.Fatalf("%s: status %d, want %d; body %s",
|
|
||||||
path, rec.Code, http.StatusOK, rec.Body)
|
|
||||||
}
|
|
||||||
|
|
||||||
return rec
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestIPLookupGivesTheOriginMostPeersCarry looks up an address in a prefix
|
|
||||||
// with two origin ASes, through each IP lookup route. The answer names the
|
|
||||||
// origin AS that more peers carry, counts every peer, lists both origin ASes,
|
|
||||||
// and gives the time of the prefix's newest kept route. The prefix's
|
|
||||||
// first-seen time and the AS's WHOIS details come from the database. A WHOIS
|
|
||||||
// detail the database does not hold is null.
|
|
||||||
func TestIPLookupGivesTheOriginMostPeersCarry(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := newServerWithTable(t, newTwoOriginTable())
|
|
||||||
s.resolver = &fakeResolver{}
|
|
||||||
|
|
||||||
firstSeen := testRouteTime(-1)
|
|
||||||
|
|
||||||
err := s.db.UpdatePrefixesBatch(map[string]time.Time{testPrefixV4: firstSeen})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("UpdatePrefixesBatch: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = s.db.GetOrCreateASN(mostPeersASN, firstSeen)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GetOrCreateASN: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
whois := &database.ASNWHOISUpdate{
|
|
||||||
ASN: mostPeersASN, ASName: "TEST-AS", CountryCode: "DE",
|
|
||||||
}
|
|
||||||
|
|
||||||
err = s.db.UpdateASNWHOIS(t.Context(), whois)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("UpdateASNWHOIS: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The details the database does not hold, such as org_name, are null.
|
|
||||||
want := IPInfo{
|
|
||||||
Netblock: new(testPrefixV4),
|
|
||||||
MaskLength: new(netip.MustParsePrefix(testPrefixV4).Bits()),
|
|
||||||
IPVersion: new(ipVersionV4),
|
|
||||||
NumPeers: new(4),
|
|
||||||
ASN: new(mostPeersASN),
|
|
||||||
NumOrigins: new(2),
|
|
||||||
OriginASNs: []int{mostPeersASN, fewerPeersASN},
|
|
||||||
ASName: &whois.ASName,
|
|
||||||
CountryCode: &whois.CountryCode,
|
|
||||||
FirstSeen: &firstSeen,
|
|
||||||
LastSeen: new(testRouteTime(5)),
|
|
||||||
}
|
|
||||||
|
|
||||||
wantJSON, err := json.Marshal(want)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("encoding %+v: %v", want, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ownAddressPath looks up the request's own address, which httptest
|
|
||||||
// gives as addrInTestPrefixV4.
|
|
||||||
for _, path := range []string{
|
|
||||||
"/api/v1/ip/" + addrInTestPrefixV4, "/ip/" + addrInTestPrefixV4,
|
|
||||||
ownAddressPath,
|
|
||||||
} {
|
|
||||||
var body struct {
|
|
||||||
Data IPLookupResponse `json:"data"`
|
|
||||||
}
|
|
||||||
|
|
||||||
answer := get(t, s, path).Body.Bytes()
|
|
||||||
|
|
||||||
err := json.Unmarshal(answer, &body)
|
|
||||||
if err != nil || len(body.Data.Results) != 1 {
|
|
||||||
t.Fatalf("%s: answer %s (%v), want one result", path, answer, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !reflect.DeepEqual(body.Data.Results[0].IPInfo, want) {
|
|
||||||
t.Errorf("%s: answer %s, want the fields %s", path, answer, wantJSON)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestIPLookupAsksForNoWHOISLookupOfOriginZero checks that the IP lookup asks
|
|
||||||
// for no WHOIS lookup of AS 0, which means no known origin. It asks for one of
|
|
||||||
// an AS whose WHOIS details were never fetched or are stale, and not of one
|
|
||||||
// whose details are fresh.
|
|
||||||
func TestIPLookupAsksForNoWHOISLookupOfOriginZero(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fresh := time.Now().Add(-time.Hour)
|
|
||||||
stale := time.Now().Add(-2 * whoisStaleThreshold)
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
asInfo database.ASN
|
|
||||||
want bool
|
|
||||||
}{
|
|
||||||
{"no known origin", database.ASN{ASN: ristypes.NoOrigin}, false},
|
|
||||||
{"details never fetched", database.ASN{ASN: mostPeersASN}, true},
|
|
||||||
{"stale details", database.ASN{ASN: mostPeersASN, WHOISUpdatedAt: &stale}, true},
|
|
||||||
{"fresh details", database.ASN{ASN: mostPeersASN, WHOISUpdatedAt: &fresh}, false},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
if got := needsWHOISRefresh(&tt.asInfo); got != tt.want {
|
|
||||||
t.Errorf("%s: needsWHOISRefresh = %t, want %t", tt.name, got, tt.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// prefixRoute is one route of the prefix API's answer. A field the answer
|
|
||||||
// leaves out stays nil.
|
|
||||||
//
|
|
||||||
//nolint:tagliatelle // field names are the existing JSON API
|
|
||||||
type prefixRoute struct {
|
|
||||||
OriginASN int `json:"origin_asn"`
|
|
||||||
PeerIP string `json:"peer_ip"`
|
|
||||||
ASPath []int `json:"as_path"`
|
|
||||||
NextHop *string `json:"next_hop"`
|
|
||||||
LastUpdated *time.Time `json:"last_updated"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPrefixAnswersListEveryPeer asks the prefix API and page for a prefix
|
|
||||||
// with two origin ASes, and for one whose origin keeps no route. Each answer
|
|
||||||
// lists every peer that carries the prefix, by origin AS. Only the kept routes
|
|
||||||
// have an AS path, a next hop and a time, newest first. The other peers follow
|
|
||||||
// without them.
|
|
||||||
func TestPrefixAnswersListEveryPeer(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := newServerWithTable(t, newTwoOriginTable())
|
|
||||||
|
|
||||||
// keptAt[peer] is the time of the peer's kept route; a peer not in it is
|
|
||||||
// listed without a route.
|
|
||||||
keptAt := map[string]time.Time{
|
|
||||||
testPeer(3): testRouteTime(3),
|
|
||||||
testPeer(2): testRouteTime(2),
|
|
||||||
testPeer(4): testRouteTime(5),
|
|
||||||
}
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
prefix string
|
|
||||||
wantPeers []string
|
|
||||||
wantOrigins []int
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
testPrefixV4,
|
|
||||||
[]string{testPeer(3), testPeer(2), testPeer(1), testPeer(4)},
|
|
||||||
[]int{mostPeersASN, mostPeersASN, mostPeersASN, fewerPeersASN},
|
|
||||||
},
|
|
||||||
{noKeptRoutePrefix, []string{testPeer(7)}, []int{mostPeersASN}},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
//nolint:tagliatelle // field names are the existing JSON API
|
|
||||||
var body struct {
|
|
||||||
Data struct {
|
|
||||||
Routes []prefixRoute `json:"routes"`
|
|
||||||
PeerCount int `json:"peer_count"`
|
|
||||||
OriginCount int `json:"origin_count"`
|
|
||||||
} `json:"data"`
|
|
||||||
}
|
|
||||||
|
|
||||||
path := "/api/v1/prefix/" + tt.prefix
|
|
||||||
|
|
||||||
err := json.Unmarshal(get(t, s, path).Body.Bytes(), &body)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("%s: decoding the answer: %v", path, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
wantOriginCount := len(slices.Compact(slices.Clone(tt.wantOrigins)))
|
|
||||||
if body.Data.PeerCount != len(tt.wantPeers) ||
|
|
||||||
body.Data.OriginCount != wantOriginCount {
|
|
||||||
t.Errorf("%s: %d peers and %d origins, want %d and %d", path,
|
|
||||||
body.Data.PeerCount, body.Data.OriginCount, len(tt.wantPeers),
|
|
||||||
wantOriginCount)
|
|
||||||
}
|
|
||||||
|
|
||||||
checkPrefixRoutes(t, path, body.Data.Routes, tt.wantPeers, tt.wantOrigins,
|
|
||||||
keptAt)
|
|
||||||
checkPrefixPage(t, s, tt.prefix, tt.wantPeers, keptAt)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkPrefixRoutes checks that routes come from wantPeers, in that order,
|
|
||||||
// with wantOrigins as their origin ASes. The route of a peer in keptAt must
|
|
||||||
// have an AS path, the peer as next hop, and the time keptAt gives. Any other
|
|
||||||
// route must have none of the three.
|
|
||||||
func checkPrefixRoutes(
|
|
||||||
t *testing.T, path string, routes []prefixRoute, wantPeers []string,
|
|
||||||
wantOrigins []int, keptAt map[string]time.Time,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
if len(routes) != len(wantPeers) {
|
|
||||||
t.Fatalf("%s: routes %+v, want one each from %v", path, routes, wantPeers)
|
|
||||||
}
|
|
||||||
|
|
||||||
for i, route := range routes {
|
|
||||||
if route.PeerIP != wantPeers[i] || route.OriginASN != wantOrigins[i] {
|
|
||||||
t.Errorf("%s: route %d from %s with AS%d, want from %s with AS%d",
|
|
||||||
path, i, route.PeerIP, route.OriginASN, wantPeers[i], wantOrigins[i])
|
|
||||||
}
|
|
||||||
|
|
||||||
at, kept := keptAt[route.PeerIP]
|
|
||||||
if !kept {
|
|
||||||
if route.ASPath != nil || route.NextHop != nil || route.LastUpdated != nil {
|
|
||||||
t.Errorf("%s: route from %s is not kept but has %+v",
|
|
||||||
path, route.PeerIP, route)
|
|
||||||
}
|
|
||||||
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if route.ASPath == nil || route.NextHop == nil ||
|
|
||||||
*route.NextHop != route.PeerIP || route.LastUpdated == nil ||
|
|
||||||
!route.LastUpdated.Equal(at) {
|
|
||||||
t.Errorf("%s: kept route from %s is %+v, want its AS path, the peer "+
|
|
||||||
"as next hop and the time %v", path, route.PeerIP, route, at)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkPrefixPage checks that the page of prefix shows each of wantPeers, the
|
|
||||||
// time of each kept route among them, and no time that was never set.
|
|
||||||
func checkPrefixPage(
|
|
||||||
t *testing.T, s *Server, prefix string, wantPeers []string,
|
|
||||||
keptAt map[string]time.Time,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
path := "/prefix/" + prefix
|
|
||||||
page := get(t, s, path).Body.String()
|
|
||||||
|
|
||||||
for _, peer := range wantPeers {
|
|
||||||
want := `<td class="peer-ip">` + peer + `</td>`
|
|
||||||
if !strings.Contains(page, want) {
|
|
||||||
t.Errorf("%s does not list the peer %s", path, peer)
|
|
||||||
}
|
|
||||||
|
|
||||||
at, kept := keptAt[peer]
|
|
||||||
if kept && !strings.Contains(page, at.Format(routeTimeFormat)) {
|
|
||||||
t.Errorf("%s does not show the time %v of the route from %s",
|
|
||||||
path, at, peer)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if strings.Contains(page, zeroDate) {
|
|
||||||
t.Errorf("%s shows the date %s", path, zeroDate)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestASAnswersGiveEachPrefixItsNewestKeptRoute asks the AS API and page for
|
|
||||||
// mostPeersASN. Its prefixes are testPrefixV4, whose newest kept route with
|
|
||||||
// that origin AS is peer 3's, and noKeptRoutePrefix, which keeps no route. The
|
|
||||||
// first has that route's time. The second is listed without a time.
|
|
||||||
func TestASAnswersGiveEachPrefixItsNewestKeptRoute(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := newServerWithTable(t, newTwoOriginTable())
|
|
||||||
path := "/api/v1/as/" + strconv.Itoa(mostPeersASN)
|
|
||||||
|
|
||||||
//nolint:tagliatelle // field names are the existing JSON API
|
|
||||||
var body struct {
|
|
||||||
Data struct {
|
|
||||||
IPv4Prefixes []struct {
|
|
||||||
Prefix string `json:"prefix"`
|
|
||||||
LastUpdated *time.Time `json:"last_updated"`
|
|
||||||
} `json:"ipv4_prefixes"`
|
|
||||||
} `json:"data"`
|
|
||||||
}
|
|
||||||
|
|
||||||
err := json.Unmarshal(get(t, s, path).Body.Bytes(), &body)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("%s: decoding the answer: %v", path, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
prefixes := body.Data.IPv4Prefixes
|
|
||||||
if len(prefixes) != 2 || prefixes[0].Prefix != testPrefixV4 ||
|
|
||||||
prefixes[0].LastUpdated == nil ||
|
|
||||||
!prefixes[0].LastUpdated.Equal(testRouteTime(3)) ||
|
|
||||||
prefixes[1].Prefix != noKeptRoutePrefix || prefixes[1].LastUpdated != nil {
|
|
||||||
t.Errorf("%s: prefixes %+v, want %s updated %v and %s without a time",
|
|
||||||
path, prefixes, testPrefixV4, testRouteTime(3), noKeptRoutePrefix)
|
|
||||||
}
|
|
||||||
|
|
||||||
path = "/as/" + strconv.Itoa(mostPeersASN)
|
|
||||||
page := get(t, s, path).Body.String()
|
|
||||||
|
|
||||||
for _, want := range []string{
|
|
||||||
testPrefixV4, noKeptRoutePrefix, testRouteTime(3).Format(routeTimeFormat),
|
|
||||||
} {
|
|
||||||
if !strings.Contains(page, want) {
|
|
||||||
t.Errorf("%s does not show %s", path, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if strings.Contains(page, zeroDate) {
|
|
||||||
t.Errorf("%s shows the date %s", path, zeroDate)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPrefixLengthPagesShowTheOriginMostPeersCarry asks for the IPv4 /24
|
|
||||||
// page and the IPv6 /32 page. The first lists both /24 prefixes with
|
|
||||||
// mostPeersASN, the origin AS more peers carry testPrefixV4 with, and gives
|
|
||||||
// noKeptRoutePrefix no age. The second lists testPrefixV6.
|
|
||||||
func TestPrefixLengthPagesShowTheOriginMostPeersCarry(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := newServerWithTable(t, newTwoOriginTable())
|
|
||||||
|
|
||||||
page := get(t, s, prefixLengthPagePath).Body.String()
|
|
||||||
|
|
||||||
for _, want := range []string{
|
|
||||||
">" + testPrefixV4 + "<", ">" + noKeptRoutePrefix + "<",
|
|
||||||
`href="/as/` + strconv.Itoa(mostPeersASN) + `"`,
|
|
||||||
} {
|
|
||||||
if !strings.Contains(page, want) {
|
|
||||||
t.Errorf("%s does not show %s", prefixLengthPagePath, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if strings.Contains(page, `href="/as/`+strconv.Itoa(fewerPeersASN)+`"`) {
|
|
||||||
t.Errorf("%s links to AS%d, which fewer peers carry",
|
|
||||||
prefixLengthPagePath, fewerPeersASN)
|
|
||||||
}
|
|
||||||
|
|
||||||
const noAge = `<td class="age"></td>`
|
|
||||||
if strings.Count(page, noAge) != 1 {
|
|
||||||
t.Errorf("%s gives %d prefixes no age, want 1 (%s)",
|
|
||||||
prefixLengthPagePath, strings.Count(page, noAge), noKeptRoutePrefix)
|
|
||||||
}
|
|
||||||
|
|
||||||
const v6PagePath = "/prefixlength6/32"
|
|
||||||
|
|
||||||
page = get(t, s, v6PagePath).Body.String()
|
|
||||||
if !strings.Contains(page, ">"+testPrefixV6+"<") ||
|
|
||||||
!strings.Contains(page, `href="/as/`+strconv.Itoa(v6OriginASN)+`"`) {
|
|
||||||
t.Errorf("%s does not show %s with AS%d", v6PagePath, testPrefixV6,
|
|
||||||
v6OriginASN)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,402 @@
|
|||||||
|
package server
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"runtime"
|
||||||
|
"slices"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.eeqj.de/sneak/routewatch/internal/config"
|
||||||
|
"git.eeqj.de/sneak/routewatch/internal/database"
|
||||||
|
"git.eeqj.de/sneak/routewatch/internal/logger"
|
||||||
|
"git.eeqj.de/sneak/routewatch/internal/metrics"
|
||||||
|
"git.eeqj.de/sneak/routewatch/internal/streamer"
|
||||||
|
"github.com/google/uuid"
|
||||||
|
)
|
||||||
|
|
||||||
|
// blockingStatsDB embeds database.Store (left nil) and overrides only
|
||||||
|
// GetStatsContext, which blocks until release is closed. The stats handlers
|
||||||
|
// call it in a goroutine; every other Store method is unused on the timeout
|
||||||
|
// path and would panic if called.
|
||||||
|
type blockingStatsDB struct {
|
||||||
|
database.Store
|
||||||
|
release chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d blockingStatsDB) GetStatsContext(_ context.Context) (database.Stats, error) {
|
||||||
|
<-d.release
|
||||||
|
|
||||||
|
return database.Stats{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestStatsHandlersDoNotLeakOnTimeout drives each stats handler repeatedly with
|
||||||
|
// a request whose context times out before the database responds, then releases
|
||||||
|
// the blocked queries and asserts the goroutine count returns to its starting
|
||||||
|
// value. Before the fix the per-request goroutine sent on an unbuffered channel
|
||||||
|
// that nothing received once the timeout won, so it blocked forever and every
|
||||||
|
// poll leaked one goroutine.
|
||||||
|
func TestStatsHandlersDoNotLeakOnTimeout(t *testing.T) {
|
||||||
|
release := make(chan struct{})
|
||||||
|
db := blockingStatsDB{release: release}
|
||||||
|
s := New(db, streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{})
|
||||||
|
|
||||||
|
handlers := map[string]http.HandlerFunc{
|
||||||
|
"status.json": s.handleStatusJSON(),
|
||||||
|
"stats": s.handleStats(),
|
||||||
|
}
|
||||||
|
|
||||||
|
baseline := settledGoroutineCount()
|
||||||
|
|
||||||
|
const (
|
||||||
|
iterations = 20
|
||||||
|
requestTimeout = 50 * time.Millisecond
|
||||||
|
)
|
||||||
|
for _, handler := range handlers {
|
||||||
|
for range iterations {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), requestTimeout)
|
||||||
|
req := httptest.NewRequest(http.MethodGet, "/", nil).WithContext(ctx)
|
||||||
|
handler(httptest.NewRecorder(), req)
|
||||||
|
cancel()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Let the blocked queries finish; with buffered channels each goroutine's
|
||||||
|
// send now succeeds and the goroutine exits.
|
||||||
|
close(release)
|
||||||
|
|
||||||
|
if !waitForGoroutines(baseline) {
|
||||||
|
t.Fatalf("goroutines did not return to baseline %d, got %d",
|
||||||
|
baseline, runtime.NumGoroutine())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestStatsHandlersAnswerFromMemory checks that both stats handlers answer 200
|
||||||
|
// with the live route counts and the prefix distribution while the database is
|
||||||
|
// closed, so that any query would fail: the request path reads them from
|
||||||
|
// memory. The prefix distribution query it used to run read every live route
|
||||||
|
// and, on a large database, took the whole 4-second deadline, so
|
||||||
|
// /api/v1/stats answered 500 (https://git.eeqj.de/sneak/routewatch/issues/30).
|
||||||
|
// The oldest and newest route times still come from one-row lookups at the ends
|
||||||
|
// of an index; with the database closed they are left out of the answer.
|
||||||
|
func TestStatsHandlersAnswerFromMemory(t *testing.T) {
|
||||||
|
db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("database.New: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
||||||
|
if err := db.UpsertLiveRouteBatch([]*database.LiveRoute{
|
||||||
|
{
|
||||||
|
ID: uuid.New(), Prefix: "198.51.100.0/24", MaskLength: 24, IPVersion: 4,
|
||||||
|
OriginASN: 64500, PeerIP: "192.0.2.1", ASPath: []int{64500}, NextHop: "192.0.2.1",
|
||||||
|
LastUpdated: ts,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ID: uuid.New(), Prefix: "2001:db8::/32", MaskLength: 32, IPVersion: 6,
|
||||||
|
OriginASN: 64501, PeerIP: "2001:db8::1", ASPath: []int{64501}, NextHop: "2001:db8::1",
|
||||||
|
LastUpdated: ts,
|
||||||
|
},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("UpsertLiveRouteBatch: %v", err)
|
||||||
|
}
|
||||||
|
if err := db.Close(); err != nil {
|
||||||
|
t.Fatalf("Close: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
s := New(db, streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{})
|
||||||
|
handlers := map[string]http.HandlerFunc{
|
||||||
|
"status.json": s.handleStatusJSON(),
|
||||||
|
"stats": s.handleStats(),
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, handler := range handlers {
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
handler(rec, httptest.NewRequest(http.MethodGet, "/", nil))
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Errorf("%s: status %d, want %d; body %s", name, rec.Code, http.StatusOK, rec.Body)
|
||||||
|
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
var body struct {
|
||||||
|
Data struct {
|
||||||
|
IPv4Routes int `json:"ipv4_routes"`
|
||||||
|
IPv6Routes int `json:"ipv6_routes"`
|
||||||
|
IPv4PrefixDistribution []database.PrefixDistribution `json:"ipv4_prefix_distribution"`
|
||||||
|
IPv6PrefixDistribution []database.PrefixDistribution `json:"ipv6_prefix_distribution"`
|
||||||
|
} `json:"data"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
|
||||||
|
t.Fatalf("%s: decoding the answer: %v", name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if body.Data.IPv4Routes != 1 || body.Data.IPv6Routes != 1 {
|
||||||
|
t.Errorf("%s: routes = (v4 %d, v6 %d), want (1, 1)", name, body.Data.IPv4Routes, body.Data.IPv6Routes)
|
||||||
|
}
|
||||||
|
wantV4 := []database.PrefixDistribution{{MaskLength: 24, Count: 1}}
|
||||||
|
if !slices.Equal(body.Data.IPv4PrefixDistribution, wantV4) {
|
||||||
|
t.Errorf("%s: IPv4 distribution = %v, want %v", name, body.Data.IPv4PrefixDistribution, wantV4)
|
||||||
|
}
|
||||||
|
wantV6 := []database.PrefixDistribution{{MaskLength: 32, Count: 1}}
|
||||||
|
if !slices.Equal(body.Data.IPv6PrefixDistribution, wantV6) {
|
||||||
|
t.Errorf("%s: IPv6 distribution = %v, want %v", name, body.Data.IPv6PrefixDistribution, wantV6)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestPrefixDetailHandlersParsePrefix checks that both prefix pages find a
|
||||||
|
// stored route whatever valid text form the URL writes its prefix in, and
|
||||||
|
// answer 400 for a prefix that does not parse. The tables store the form
|
||||||
|
// net/netip prints. Before the fix the URL text was looked up as typed, so
|
||||||
|
// "2001:DB8:0:0::/32" was not found although "2001:db8::/32" was stored, and
|
||||||
|
// "foo/24" answered 500 (https://git.eeqj.de/sneak/routewatch/issues/71).
|
||||||
|
func TestPrefixDetailHandlersParsePrefix(t *testing.T) {
|
||||||
|
db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("database.New: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = db.Close() })
|
||||||
|
|
||||||
|
ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
||||||
|
if err := db.UpsertLiveRouteBatch([]*database.LiveRoute{
|
||||||
|
{
|
||||||
|
ID: uuid.New(), Prefix: "192.0.2.0/24", MaskLength: 24, IPVersion: 4,
|
||||||
|
OriginASN: 64500, PeerIP: "198.51.100.1", ASPath: []int{64500}, NextHop: "198.51.100.1",
|
||||||
|
LastUpdated: ts,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ID: uuid.New(), Prefix: "2001:db8::/32", MaskLength: 32, IPVersion: 6,
|
||||||
|
OriginASN: 64501, PeerIP: "2001:db8::1", ASPath: []int{64501}, NextHop: "2001:db8::1",
|
||||||
|
LastUpdated: ts,
|
||||||
|
},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("UpsertLiveRouteBatch: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
s := New(db, streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{})
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
path string
|
||||||
|
want int
|
||||||
|
}{
|
||||||
|
{"/api/v1/prefix/foo/24", http.StatusBadRequest},
|
||||||
|
{"/prefix/foo/24", http.StatusBadRequest},
|
||||||
|
{"/api/v1/prefix/2001:DB8:0:0::/32", http.StatusOK},
|
||||||
|
{"/prefix/2001:DB8:0:0::/32", http.StatusOK},
|
||||||
|
{"/api/v1/prefix/192.0.2.1/24", http.StatusOK},
|
||||||
|
{"/prefix/192.0.2.1/24", http.StatusOK},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
s.router.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, tt.path, nil))
|
||||||
|
if rec.Code != tt.want {
|
||||||
|
t.Errorf("%s: status %d, want %d; body %s", tt.path, rec.Code, tt.want, rec.Body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// shortRequestDeadline is the deadline the tests below give a request. The
|
||||||
|
// server's 30-second limit and the stats handlers' 4-second limit inherit it,
|
||||||
|
// so the tests do not wait for either.
|
||||||
|
const shortRequestDeadline = 50 * time.Millisecond
|
||||||
|
|
||||||
|
// TestRequestPastTimeLimitAnswers504 sends a request through the server's
|
||||||
|
// router to a page and to a JSON route whose handlers wait for the time limit
|
||||||
|
// to pass and return without answering. The page answers 504; the JSON route
|
||||||
|
// answers 504 with the JSON error body. Before the fix both answered 408,
|
||||||
|
// written by the time limit middleware from its own goroutine while the
|
||||||
|
// handler could still be writing.
|
||||||
|
func TestRequestPastTimeLimitAnswers504(t *testing.T) {
|
||||||
|
s := New(nil, streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{})
|
||||||
|
|
||||||
|
waitForTimeLimit := func(_ http.ResponseWriter, r *http.Request) {
|
||||||
|
<-r.Context().Done()
|
||||||
|
}
|
||||||
|
// Registered the way setupRoutes registers pages and JSON routes.
|
||||||
|
s.router.Get("/slow-page", waitForTimeLimit)
|
||||||
|
s.router.Get("/slow.json", JSONValidationMiddleware(http.HandlerFunc(waitForTimeLimit)).ServeHTTP)
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
path string
|
||||||
|
wantJSONBody bool
|
||||||
|
}{
|
||||||
|
{path: "/slow-page"},
|
||||||
|
{path: "/slow.json", wantJSONBody: true},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), shortRequestDeadline)
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
s.router.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, tt.path, nil).WithContext(ctx))
|
||||||
|
cancel()
|
||||||
|
|
||||||
|
if rec.Code != http.StatusGatewayTimeout {
|
||||||
|
t.Errorf("%s: status %d, want %d; body %s", tt.path, rec.Code, http.StatusGatewayTimeout, rec.Body)
|
||||||
|
}
|
||||||
|
if tt.wantJSONBody {
|
||||||
|
checkJSONErrorCode(t, tt.path, rec, http.StatusGatewayTimeout)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestStatsHandlersAnswer504WhenDatabaseReadRunsOut calls each stats handler
|
||||||
|
// with a store that does not answer before the handler's 4-second limit runs
|
||||||
|
// out, and checks that the handler itself answers 504 with the JSON error
|
||||||
|
// body. Before the fix status.json answered 408, and /api/v1/stats wrote
|
||||||
|
// nothing, which the JSON check on its route turned into 500.
|
||||||
|
func TestStatsHandlersAnswer504WhenDatabaseReadRunsOut(t *testing.T) {
|
||||||
|
release := make(chan struct{})
|
||||||
|
defer close(release)
|
||||||
|
db := blockingStatsDB{release: release}
|
||||||
|
s := New(db, streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{})
|
||||||
|
|
||||||
|
handlers := map[string]http.HandlerFunc{
|
||||||
|
"status.json": s.handleStatusJSON(),
|
||||||
|
"stats": s.handleStats(),
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, handler := range handlers {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), shortRequestDeadline)
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
handler(rec, httptest.NewRequest(http.MethodGet, "/", nil).WithContext(ctx))
|
||||||
|
cancel()
|
||||||
|
|
||||||
|
if rec.Code != http.StatusGatewayTimeout {
|
||||||
|
t.Errorf("%s: status %d, want %d; body %s", name, rec.Code, http.StatusGatewayTimeout, rec.Body)
|
||||||
|
}
|
||||||
|
checkJSONErrorCode(t, name, rec, http.StatusGatewayTimeout)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// slowDB embeds database.Store (left nil) and overrides the reads the routes
|
||||||
|
// in TestRoutesAnswer504WhenDatabaseReadIsCutOff make. Each read waits until
|
||||||
|
// its context ends and returns the context's error, as a slow database read
|
||||||
|
// cut off by a time limit does.
|
||||||
|
type slowDB struct {
|
||||||
|
database.Store
|
||||||
|
}
|
||||||
|
|
||||||
|
func (slowDB) GetStatsContext(ctx context.Context) (database.Stats, error) {
|
||||||
|
<-ctx.Done()
|
||||||
|
|
||||||
|
return database.Stats{}, ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (slowDB) GetIPInfoContext(ctx context.Context, _ string) (*database.IPInfo, error) {
|
||||||
|
<-ctx.Done()
|
||||||
|
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (slowDB) GetASDetailsContext(ctx context.Context, _ int) (*database.ASN, []database.LiveRoute, error) {
|
||||||
|
<-ctx.Done()
|
||||||
|
|
||||||
|
return nil, nil, ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (slowDB) GetPrefixDetailsContext(ctx context.Context, _ string) ([]database.LiveRoute, error) {
|
||||||
|
<-ctx.Done()
|
||||||
|
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (slowDB) GetRandomPrefixesByLengthContext(ctx context.Context, _, _, _ int) ([]database.LiveRoute, error) {
|
||||||
|
<-ctx.Done()
|
||||||
|
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRoutesAnswer504WhenDatabaseReadIsCutOff sends a request through the
|
||||||
|
// server's router for each route that reads the database, with a store whose
|
||||||
|
// reads fail when the request's time limit passes. Each route answers 504, the
|
||||||
|
// JSON routes with the JSON error body. Before the fix they answered 408 or,
|
||||||
|
// when the handler wrote first, its own error: 500, or 404 on the IP lookups.
|
||||||
|
// The health check is left out: it answers 503 when its own 2-second ping runs
|
||||||
|
// out.
|
||||||
|
func TestRoutesAnswer504WhenDatabaseReadIsCutOff(t *testing.T) {
|
||||||
|
s := New(slowDB{}, streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{})
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
path string
|
||||||
|
wantJSONBody bool
|
||||||
|
}{
|
||||||
|
{path: "/as/64500"},
|
||||||
|
{path: "/prefix/192.0.2.0/24"},
|
||||||
|
{path: "/prefixlength/24"},
|
||||||
|
{path: "/prefixlength6/48"},
|
||||||
|
{path: "/status.json", wantJSONBody: true},
|
||||||
|
{path: "/ip/192.0.2.1", wantJSONBody: true},
|
||||||
|
{path: "/api/v1/stats", wantJSONBody: true},
|
||||||
|
{path: "/api/v1/ip/192.0.2.1", wantJSONBody: true},
|
||||||
|
{path: "/api/v1/as/64500", wantJSONBody: true},
|
||||||
|
{path: "/api/v1/prefix/192.0.2.0/24", wantJSONBody: true},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), shortRequestDeadline)
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
s.router.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, tt.path, nil).WithContext(ctx))
|
||||||
|
cancel()
|
||||||
|
|
||||||
|
if rec.Code != http.StatusGatewayTimeout {
|
||||||
|
t.Errorf("%s: status %d, want %d; body %s", tt.path, rec.Code, http.StatusGatewayTimeout, rec.Body)
|
||||||
|
}
|
||||||
|
if tt.wantJSONBody {
|
||||||
|
checkJSONErrorCode(t, tt.path, rec, http.StatusGatewayTimeout)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkJSONErrorCode checks that rec holds the JSON error body with the given
|
||||||
|
// code.
|
||||||
|
func checkJSONErrorCode(t *testing.T, name string, rec *httptest.ResponseRecorder, want int) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
var body struct {
|
||||||
|
Status string `json:"status"`
|
||||||
|
Error struct {
|
||||||
|
Code int `json:"code"`
|
||||||
|
} `json:"error"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
|
||||||
|
t.Errorf("%s: decoding the answer %q: %v", name, rec.Body, err)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if body.Status != "error" || body.Error.Code != want {
|
||||||
|
t.Errorf("%s: answer %s, want status \"error\" and error code %d", name, rec.Body, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// settledGoroutineCount lets transient goroutines finish, then reports the
|
||||||
|
// current count.
|
||||||
|
func settledGoroutineCount() int {
|
||||||
|
prev := runtime.NumGoroutine()
|
||||||
|
for range 20 {
|
||||||
|
time.Sleep(10 * time.Millisecond)
|
||||||
|
cur := runtime.NumGoroutine()
|
||||||
|
if cur == prev {
|
||||||
|
return cur
|
||||||
|
}
|
||||||
|
prev = cur
|
||||||
|
}
|
||||||
|
|
||||||
|
return prev
|
||||||
|
}
|
||||||
|
|
||||||
|
// waitForGoroutines waits until the goroutine count drops to target or below.
|
||||||
|
func waitForGoroutines(target int) bool {
|
||||||
|
for range 100 {
|
||||||
|
if runtime.NumGoroutine() <= target {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
time.Sleep(10 * time.Millisecond)
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
@@ -1,206 +0,0 @@
|
|||||||
package server
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"html/template"
|
|
||||||
"net/http"
|
|
||||||
"slices"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/database"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/routetable"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/templates"
|
|
||||||
asinfo "git.eeqj.de/sneak/routewatch/pkg/asinfo"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
// homeCardsRefreshInterval is how often the server works out the home
|
|
||||||
// page's cards again. Some of their figures take a walk over the whole
|
|
||||||
// routing table, so no request works them out.
|
|
||||||
homeCardsRefreshInterval = time.Minute
|
|
||||||
|
|
||||||
// homeCardsDatabaseTimeout is how long the database read behind the
|
|
||||||
// cards of the newest and the oldest AS may take.
|
|
||||||
homeCardsDatabaseTimeout = 4 * time.Second
|
|
||||||
|
|
||||||
// maxListedEntries is how many entries the bogon and multiple-origin
|
|
||||||
// pages list at most.
|
|
||||||
maxListedEntries = 1000
|
|
||||||
)
|
|
||||||
|
|
||||||
// homeCards is what the home page's cards show. The bogon and multiple-origin
|
|
||||||
// pages list the entries their cards count. The server works it out every
|
|
||||||
// homeCardsRefreshInterval and serves this copy in between.
|
|
||||||
type homeCards struct {
|
|
||||||
routetable.Overview
|
|
||||||
|
|
||||||
// NewestAS and OldestAS are the ASes the asns table last saw most
|
|
||||||
// recently and longest ago. They are nil when the database did not
|
|
||||||
// answer or holds no AS.
|
|
||||||
NewestAS *database.ASN
|
|
||||||
OldestAS *database.ASN
|
|
||||||
|
|
||||||
// WellKnownNetworks are the networks of wellKnownASNs, newest update
|
|
||||||
// first.
|
|
||||||
WellKnownNetworks []wellKnownNetwork
|
|
||||||
}
|
|
||||||
|
|
||||||
// wellKnownNetwork is one network on the home page's card of well-known
|
|
||||||
// networks.
|
|
||||||
type wellKnownNetwork struct {
|
|
||||||
ASN int
|
|
||||||
// Name is its description in the AS list built into RouteWatch.
|
|
||||||
Name string
|
|
||||||
// Prefixes is how many prefixes it originates.
|
|
||||||
Prefixes int
|
|
||||||
// Newest is the time of its newest kept route, or nil when it keeps none.
|
|
||||||
Newest *time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// listPageData is what the bogon and multiple-origin pages are rendered with.
|
|
||||||
type listPageData struct {
|
|
||||||
// CSPNonce is the nonce the page's style block carries.
|
|
||||||
CSPNonce string
|
|
||||||
// Cards is the copy of the home page's cards, or nil before the server
|
|
||||||
// has first worked them out.
|
|
||||||
Cards *homeCards
|
|
||||||
}
|
|
||||||
|
|
||||||
// The AS numbers of the networks the home page lists as well known.
|
|
||||||
const (
|
|
||||||
appleASN = 714
|
|
||||||
googleASN = 15169
|
|
||||||
amazonASN = 16509
|
|
||||||
microsoftASN = 8075
|
|
||||||
metaASN = 32934
|
|
||||||
cloudflareASN = 13335
|
|
||||||
akamaiASN = 20940
|
|
||||||
netflixASN = 2906
|
|
||||||
dodNetworkInfoCtrASN = 721
|
|
||||||
cogentASN = 174
|
|
||||||
hurricaneElectricASN = 6939
|
|
||||||
level3ASN = 3356
|
|
||||||
arelionASN = 1299
|
|
||||||
nttASN = 2914
|
|
||||||
mitASN = 3
|
|
||||||
)
|
|
||||||
|
|
||||||
// wellKnownASNs returns the AS numbers of the networks the home page lists as
|
|
||||||
// well known: large companies, large transit networks, the US Department of
|
|
||||||
// Defense's Network Information Center and a university.
|
|
||||||
func wellKnownASNs() []int {
|
|
||||||
return []int{
|
|
||||||
appleASN, googleASN, amazonASN, microsoftASN, metaASN, cloudflareASN,
|
|
||||||
akamaiASN, netflixASN, dodNetworkInfoCtrASN, cogentASN,
|
|
||||||
hurricaneElectricASN, level3ASN, arelionASN, nttASN, mitASN,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// keepHomeCardsFresh works out the home page's cards now, and again every
|
|
||||||
// interval until ctx ends.
|
|
||||||
func (s *Server) keepHomeCardsFresh(ctx context.Context, interval time.Duration) {
|
|
||||||
ticker := time.NewTicker(interval)
|
|
||||||
defer ticker.Stop()
|
|
||||||
|
|
||||||
for {
|
|
||||||
s.refreshHomeCards(ctx)
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case <-ticker.C:
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// refreshHomeCards works out the home page's cards and keeps them for the
|
|
||||||
// pages to serve. When the database does not answer, the failure is logged
|
|
||||||
// and the cards of the newest and the oldest AS stay empty.
|
|
||||||
func (s *Server) refreshHomeCards(ctx context.Context) {
|
|
||||||
cards := &homeCards{
|
|
||||||
Overview: s.table.Overview(maxListedEntries),
|
|
||||||
WellKnownNetworks: s.wellKnownNetworks(),
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(ctx, homeCardsDatabaseTimeout)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
newest, oldest, err := s.db.GetLastSeenASes(ctx)
|
|
||||||
if err != nil {
|
|
||||||
s.logger.Warn("Failed to read the newest and the oldest AS", "error", err)
|
|
||||||
} else {
|
|
||||||
cards.NewestAS, cards.OldestAS = newest, oldest
|
|
||||||
}
|
|
||||||
|
|
||||||
s.cards.Store(cards)
|
|
||||||
}
|
|
||||||
|
|
||||||
// wellKnownNetworks returns the networks of wellKnownASNs, each with how many
|
|
||||||
// prefixes it originates and the time of its newest kept route. They come
|
|
||||||
// newest first, and those that keep no route last.
|
|
||||||
func (s *Server) wellKnownNetworks() []wellKnownNetwork {
|
|
||||||
asns := wellKnownASNs()
|
|
||||||
networks := make([]wellKnownNetwork, 0, len(asns))
|
|
||||||
|
|
||||||
for _, asn := range asns {
|
|
||||||
originated := s.table.OriginatedPrefixes(asn)
|
|
||||||
network := wellKnownNetwork{ASN: asn, Prefixes: len(originated)}
|
|
||||||
|
|
||||||
if info, ok := asinfo.Get(asn); ok {
|
|
||||||
network.Name = info.Description
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, p := range originated {
|
|
||||||
if p.Newest != nil &&
|
|
||||||
(network.Newest == nil || p.Newest.After(*network.Newest)) {
|
|
||||||
network.Newest = p.Newest
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
networks = append(networks, network)
|
|
||||||
}
|
|
||||||
|
|
||||||
slices.SortStableFunc(networks, func(a, b wellKnownNetwork) int {
|
|
||||||
switch {
|
|
||||||
case a.Newest == nil && b.Newest == nil:
|
|
||||||
return 0
|
|
||||||
case a.Newest == nil:
|
|
||||||
return 1
|
|
||||||
case b.Newest == nil:
|
|
||||||
return -1
|
|
||||||
}
|
|
||||||
|
|
||||||
return b.Newest.Compare(*a.Newest)
|
|
||||||
})
|
|
||||||
|
|
||||||
return networks
|
|
||||||
}
|
|
||||||
|
|
||||||
// handleBogons returns a handler that serves the page listing the bogon
|
|
||||||
// announcements.
|
|
||||||
func (s *Server) handleBogons() http.HandlerFunc {
|
|
||||||
return s.listPageHandler(templates.BogonsTemplate())
|
|
||||||
}
|
|
||||||
|
|
||||||
// handleMultipleOrigins returns a handler that serves the page listing the
|
|
||||||
// prefixes announced by more than one origin AS.
|
|
||||||
func (s *Server) handleMultipleOrigins() http.HandlerFunc {
|
|
||||||
return s.listPageHandler(templates.MultipleOriginsTemplate())
|
|
||||||
}
|
|
||||||
|
|
||||||
// listPageHandler returns a handler that renders tmpl, a page that lists
|
|
||||||
// entries the home page's cards count, from the copy of the cards.
|
|
||||||
func (s *Server) listPageHandler(tmpl *template.Template) http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
|
||||||
|
|
||||||
err := tmpl.Execute(w, listPageData{
|
|
||||||
CSPNonce: cspNonce(r.Context()),
|
|
||||||
Cards: s.cards.Load(),
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
s.logger.Error("Failed to render a list page", "error", err)
|
|
||||||
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,419 +0,0 @@
|
|||||||
package server
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"net/netip"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"testing/synctest"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/database"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/routetable"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The ASes of the routing table that newCardsTable fills. Each card test
|
|
||||||
// finds one of them on its card.
|
|
||||||
const (
|
|
||||||
twoPrefixASN = 13335 // originates two prefixes, the most
|
|
||||||
newestRouteASN = 15169 // the newest route
|
|
||||||
oldestRouteASN = 3 // the oldest route
|
|
||||||
secondOrigin = 16509 // a second origin AS of twoPrefixASN's prefix
|
|
||||||
privateUseASN = 64512 // reserved, so its route is a bogon announcement
|
|
||||||
)
|
|
||||||
|
|
||||||
// Paths of the pages that list what two of the home page's cards count.
|
|
||||||
const (
|
|
||||||
bogonsPath = "/bogons"
|
|
||||||
multipleOriginsPath = "/multiple-origins"
|
|
||||||
)
|
|
||||||
|
|
||||||
// cardsNotReadyText is what the pages say before the cards are first worked
|
|
||||||
// out.
|
|
||||||
const cardsNotReadyText = "being worked out"
|
|
||||||
|
|
||||||
// newCardsTable returns a routing table in which:
|
|
||||||
// - twoPrefixASN originates 1.0.0.0/24 and 1.0.4.0/22;
|
|
||||||
// - secondOrigin originates 1.0.0.0/24 too, which so has two origin ASes;
|
|
||||||
// - newestRouteASN's 8.8.8.0/24 has the newest route;
|
|
||||||
// - oldestRouteASN's 18.0.0.0/8 has the oldest;
|
|
||||||
// - privateUseASN originates 10.0.0.0/8, carried by two peers: a bogon
|
|
||||||
// announcement for both its prefix and its AS.
|
|
||||||
//
|
|
||||||
// The last four ASes originate one prefix each.
|
|
||||||
func newCardsTable() *routetable.Table {
|
|
||||||
table := emptyTable()
|
|
||||||
announce(table, "1.0.0.0/24", twoPrefixASN, testPeer(1), testRouteTime(10))
|
|
||||||
announce(table, "1.0.4.0/22", twoPrefixASN, testPeer(1), testRouteTime(5))
|
|
||||||
announce(table, "1.0.0.0/24", secondOrigin, testPeer(3), testRouteTime(12))
|
|
||||||
announce(table, "8.8.8.0/24", newestRouteASN, testPeer(2), testRouteTime(20))
|
|
||||||
announce(table, "18.0.0.0/8", oldestRouteASN, testPeer(2), testRouteTime(1))
|
|
||||||
announce(table, "10.0.0.0/8", privateUseASN, testPeer(1), testRouteTime(15))
|
|
||||||
announce(table, "10.0.0.0/8", privateUseASN, testPeer(2), testRouteTime(15))
|
|
||||||
|
|
||||||
return table
|
|
||||||
}
|
|
||||||
|
|
||||||
// homeCard returns the HTML of the home page's card with heading, up to the
|
|
||||||
// end of the card.
|
|
||||||
func homeCard(t *testing.T, page, heading string) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
_, card, found := strings.Cut(page, "<h2>"+heading+"</h2>")
|
|
||||||
if !found {
|
|
||||||
t.Fatalf("the home page has no card %q", heading)
|
|
||||||
}
|
|
||||||
|
|
||||||
card, _, _ = strings.Cut(card, "</div>")
|
|
||||||
|
|
||||||
return card
|
|
||||||
}
|
|
||||||
|
|
||||||
// asLink returns the HTML attribute of a link to the AS page of asn.
|
|
||||||
func asLink(asn int) string {
|
|
||||||
return `href="/as/` + strconv.Itoa(asn) + `"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHomePageCardsShowTheTableFigures fills a routing table and the asns
|
|
||||||
// table, works out the cards, and checks that each card shows its figure.
|
|
||||||
func TestHomePageCardsShowTheTableFigures(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const (
|
|
||||||
newestSeenASN = 6939
|
|
||||||
oldestSeenASN = 174
|
|
||||||
)
|
|
||||||
|
|
||||||
s := newServerWithTable(t, newCardsTable())
|
|
||||||
|
|
||||||
err := s.db.GetOrCreateASNBatch(map[int]database.SeenTimes{
|
|
||||||
newestSeenASN: {First: testRouteTime(0), Last: testRouteTime(30)},
|
|
||||||
oldestSeenASN: {First: testRouteTime(-30), Last: testRouteTime(-30)},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GetOrCreateASNBatch: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
s.refreshHomeCards(t.Context())
|
|
||||||
|
|
||||||
page := get(t, s, "/").Body.String()
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
heading string
|
|
||||||
want []string
|
|
||||||
}{
|
|
||||||
{"Newest Updated Route", []string{
|
|
||||||
"8.8.8.0/24", asLink(newestRouteASN),
|
|
||||||
testRouteTime(20).Format(routeTimeFormat),
|
|
||||||
}},
|
|
||||||
{"Oldest Updated Route", []string{
|
|
||||||
"18.0.0.0/8", asLink(oldestRouteASN),
|
|
||||||
testRouteTime(1).Format(routeTimeFormat),
|
|
||||||
}},
|
|
||||||
{"Newest Updated AS", []string{
|
|
||||||
asLink(newestSeenASN), testRouteTime(30).Format(routeTimeFormat),
|
|
||||||
}},
|
|
||||||
{"Oldest Updated AS", []string{
|
|
||||||
asLink(oldestSeenASN), testRouteTime(-30).Format(routeTimeFormat),
|
|
||||||
}},
|
|
||||||
{"AS with the Most Prefixes", []string{
|
|
||||||
asLink(twoPrefixASN), "Prefixes it originates: 2",
|
|
||||||
}},
|
|
||||||
{"ASes with the Fewest Prefixes", []string{
|
|
||||||
"ASes that originate a single prefix: 4", asLink(newestRouteASN),
|
|
||||||
"8.8.8.0/24",
|
|
||||||
}},
|
|
||||||
{"Bogon Announcements", []string{`href="/bogons">1<`}},
|
|
||||||
{"Prefixes with Several Origins", []string{`href="/multiple-origins">1<`}},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range tests {
|
|
||||||
card := homeCard(t, page, tc.heading)
|
|
||||||
for _, want := range tc.want {
|
|
||||||
if !strings.Contains(card, want) {
|
|
||||||
t.Errorf("card %q does not hold %s: %s", tc.heading, want, card)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The well-known networks come newest update first, and those without
|
|
||||||
// a route after them.
|
|
||||||
networks := homeCard(t, page, "Well-Known Networks")
|
|
||||||
order := []int{newestRouteASN, secondOrigin, twoPrefixASN, oldestRouteASN, 714}
|
|
||||||
last := -1
|
|
||||||
|
|
||||||
for _, asn := range order {
|
|
||||||
i := strings.Index(networks, asLink(asn))
|
|
||||||
if i <= last {
|
|
||||||
t.Fatalf("well-known networks: AS%d is not after the ASes before "+
|
|
||||||
"it in %v: %s", asn, order, networks)
|
|
||||||
}
|
|
||||||
|
|
||||||
last = i
|
|
||||||
}
|
|
||||||
|
|
||||||
if !strings.Contains(networks, "<td>2</td>") {
|
|
||||||
t.Errorf("well-known networks: no network with 2 prefixes: %s", networks)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestListPagesListTheirEntries checks the rows of the bogon and multiple-origin
|
|
||||||
// pages for the table of newCardsTable.
|
|
||||||
func TestListPagesListTheirEntries(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := newServerWithTable(t, newCardsTable())
|
|
||||||
s.refreshHomeCards(t.Context())
|
|
||||||
|
|
||||||
pages := map[string][]string{
|
|
||||||
bogonsPath: {
|
|
||||||
"Bogon announcements: 1.", "10.0.0.0/8", asLink(privateUseASN),
|
|
||||||
"<td>2</td>",
|
|
||||||
"Private-Use (10.0.0.0/8); Private-Use (AS64512-AS65534)",
|
|
||||||
},
|
|
||||||
multipleOriginsPath: {
|
|
||||||
"Prefixes announced by more than one origin AS: 1.", "1.0.0.0/24",
|
|
||||||
asLink(twoPrefixASN), asLink(secondOrigin), ">1 peer<",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for path, wants := range pages {
|
|
||||||
page := get(t, s, path).Body.String()
|
|
||||||
for _, want := range wants {
|
|
||||||
if !strings.Contains(page, want) {
|
|
||||||
t.Errorf("%s does not hold %s", path, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestListPagesListAtMostMaxListedEntries fills a table with more prefixes than
|
|
||||||
// the list pages list, each announced by two origin ASes in reserved address
|
|
||||||
// space. Each page lists maxListedEntries of them and gives how many there are
|
|
||||||
// in all.
|
|
||||||
func TestListPagesListAtMostMaxListedEntries(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const prefixCount = maxListedEntries + 1
|
|
||||||
|
|
||||||
table := emptyTable()
|
|
||||||
addr := netip.MustParseAddr("10.0.0.0")
|
|
||||||
|
|
||||||
for range prefixCount {
|
|
||||||
prefix := netip.PrefixFrom(addr, addr.BitLen()).String()
|
|
||||||
announce(table, prefix, twoPrefixASN, testPeer(1), testRouteTime(0))
|
|
||||||
announce(table, prefix, secondOrigin, testPeer(2), testRouteTime(0))
|
|
||||||
|
|
||||||
addr = addr.Next()
|
|
||||||
}
|
|
||||||
|
|
||||||
s := newServerWithTable(t, table)
|
|
||||||
s.refreshHomeCards(t.Context())
|
|
||||||
|
|
||||||
listed := "The first " + strconv.Itoa(maxListedEntries) + " are listed"
|
|
||||||
tests := []struct {
|
|
||||||
path string
|
|
||||||
total string
|
|
||||||
}{
|
|
||||||
{bogonsPath, "Bogon announcements: " + strconv.Itoa(2*prefixCount) + "."},
|
|
||||||
{multipleOriginsPath, "Prefixes announced by more than one origin AS: " +
|
|
||||||
strconv.Itoa(prefixCount) + "."},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range tests {
|
|
||||||
page := get(t, s, tc.path).Body.String()
|
|
||||||
|
|
||||||
rows := strings.Count(page, `class="prefix-link"`)
|
|
||||||
if rows != maxListedEntries {
|
|
||||||
t.Errorf("%s lists %d entries, want %d", tc.path, rows,
|
|
||||||
maxListedEntries)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !strings.Contains(page, tc.total) || !strings.Contains(page, listed) {
|
|
||||||
t.Errorf("%s does not say %q and %q", tc.path, tc.total, listed)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPagesServeTheCopyOfTheCards checks that the pages show the cards as they
|
|
||||||
// were last worked out, not as the routing table is now, and that they say so
|
|
||||||
// before the cards are first worked out.
|
|
||||||
func TestPagesServeTheCopyOfTheCards(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
table := emptyTable()
|
|
||||||
announce(table, "1.0.0.0/24", twoPrefixASN, testPeer(1), testRouteTime(0))
|
|
||||||
|
|
||||||
s := newServerWithTable(t, table)
|
|
||||||
|
|
||||||
for _, path := range []string{"/", bogonsPath, multipleOriginsPath} {
|
|
||||||
if !strings.Contains(get(t, s, path).Body.String(), cardsNotReadyText) {
|
|
||||||
t.Errorf("%s does not say the cards are %s", path, cardsNotReadyText)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
s.refreshHomeCards(t.Context())
|
|
||||||
announce(table, "8.8.8.0/24", newestRouteASN, testPeer(2), testRouteTime(1))
|
|
||||||
|
|
||||||
card := homeCard(t, get(t, s, "/").Body.String(), "Newest Updated Route")
|
|
||||||
if !strings.Contains(card, "1.0.0.0/24") {
|
|
||||||
t.Errorf("before the cards are worked out again, the newest route "+
|
|
||||||
"is not the one they were worked out with: %s", card)
|
|
||||||
}
|
|
||||||
|
|
||||||
s.refreshHomeCards(t.Context())
|
|
||||||
|
|
||||||
card = homeCard(t, get(t, s, "/").Body.String(), "Newest Updated Route")
|
|
||||||
if !strings.Contains(card, "8.8.8.0/24") {
|
|
||||||
t.Errorf("after the cards are worked out again, the newest route is "+
|
|
||||||
"not the table's: %s", card)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestKeepHomeCardsFreshWorksThemOutEveryInterval runs keepHomeCardsFresh with
|
|
||||||
// a short interval. It works the cards out at once, again after the table
|
|
||||||
// changes, and stops when its context ends.
|
|
||||||
func TestKeepHomeCardsFreshWorksThemOutEveryInterval(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const (
|
|
||||||
interval = 10 * time.Millisecond
|
|
||||||
maxWait = 5 * time.Second
|
|
||||||
)
|
|
||||||
|
|
||||||
table := emptyTable()
|
|
||||||
announce(table, "1.0.0.0/24", twoPrefixASN, testPeer(1), testRouteTime(0))
|
|
||||||
|
|
||||||
s := newServerWithTable(t, table)
|
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(t.Context())
|
|
||||||
stopped := make(chan struct{})
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
s.keepHomeCardsFresh(ctx, interval)
|
|
||||||
close(stopped)
|
|
||||||
}()
|
|
||||||
|
|
||||||
newestRouteASNIs := func(asn int) bool {
|
|
||||||
cards := s.cards.Load()
|
|
||||||
|
|
||||||
return cards != nil && cards.NewestRoute != nil &&
|
|
||||||
cards.NewestRoute.OriginASN == asn
|
|
||||||
}
|
|
||||||
|
|
||||||
waitUntil(t, maxWait, func() bool { return newestRouteASNIs(twoPrefixASN) })
|
|
||||||
|
|
||||||
announce(table, "8.8.8.0/24", newestRouteASN, testPeer(2), testRouteTime(1))
|
|
||||||
waitUntil(t, maxWait, func() bool { return newestRouteASNIs(newestRouteASN) })
|
|
||||||
|
|
||||||
cancel()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-stopped:
|
|
||||||
case <-time.After(maxWait):
|
|
||||||
t.Fatal("keepHomeCardsFresh did not stop when its context ended")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestStartedServerWorksOutTheCards starts a server and waits until its home
|
|
||||||
// page shows the cards.
|
|
||||||
func TestStartedServerWorksOutTheCards(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const maxWait = 5 * time.Second
|
|
||||||
|
|
||||||
s := newServerWithTable(t, newCardsTable())
|
|
||||||
|
|
||||||
err := s.Start(t.Context())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Start: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() { _ = s.Stop(context.Background()) }()
|
|
||||||
|
|
||||||
waitUntil(t, maxWait, func() bool {
|
|
||||||
return !strings.Contains(get(t, s, "/").Body.String(), cardsNotReadyText)
|
|
||||||
})
|
|
||||||
|
|
||||||
card := homeCard(t, get(t, s, "/").Body.String(), "Newest Updated Route")
|
|
||||||
if !strings.Contains(card, "8.8.8.0/24") {
|
|
||||||
t.Errorf("the newest route is not the table's: %s", card)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestStartedServerWorksOutTheCardsUntilStop starts a server with a context
|
|
||||||
// that ends as soon as Start returns. It runs in a synctest bubble, whose clock
|
|
||||||
// is fake: time.Sleep there moves the clock on and returns at once. The server
|
|
||||||
// must work the cards out again a minute later, and no more after Stop. Each
|
|
||||||
// time it works them out, it keeps a new copy. Before the fix the end of
|
|
||||||
// Start's context stopped the goroutine that works them out.
|
|
||||||
func TestStartedServerWorksOutTheCardsUntilStop(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// The server is made outside the bubble. The streamer's metrics start a
|
|
||||||
// goroutine that never returns, and a bubble waits for each of its
|
|
||||||
// goroutines to return.
|
|
||||||
s := newServerWithTable(t, newCardsTable())
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
ctx, cancel := context.WithCancel(t.Context())
|
|
||||||
|
|
||||||
err := s.Start(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Start: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cancel()
|
|
||||||
|
|
||||||
// The bubble's clock moves only while each of its goroutines waits
|
|
||||||
// on the bubble, and the goroutine that serves HTTP waits on the
|
|
||||||
// network. The test sends no request, so it closes the HTTP server.
|
|
||||||
_ = s.srv.Close()
|
|
||||||
|
|
||||||
synctest.Wait()
|
|
||||||
|
|
||||||
started := s.cards.Load()
|
|
||||||
if started == nil {
|
|
||||||
t.Fatal("the server did not work out the cards when it started")
|
|
||||||
}
|
|
||||||
|
|
||||||
time.Sleep(homeCardsRefreshInterval)
|
|
||||||
synctest.Wait()
|
|
||||||
|
|
||||||
if s.cards.Load() == started {
|
|
||||||
t.Error("the server did not work out the cards again a minute " +
|
|
||||||
"after its start context ended")
|
|
||||||
}
|
|
||||||
|
|
||||||
err = s.Stop(t.Context())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Stop: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
stopped := s.cards.Load()
|
|
||||||
|
|
||||||
time.Sleep(homeCardsRefreshInterval)
|
|
||||||
synctest.Wait()
|
|
||||||
|
|
||||||
if s.cards.Load() != stopped {
|
|
||||||
t.Error("the server worked out the cards again after Stop")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// waitUntil fails the test unless done reports true within maxWait.
|
|
||||||
func waitUntil(t *testing.T, maxWait time.Duration, done func() bool) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
const pollInterval = 5 * time.Millisecond
|
|
||||||
|
|
||||||
deadline := time.Now().Add(maxWait)
|
|
||||||
for !done() {
|
|
||||||
if time.Now().After(deadline) {
|
|
||||||
t.Fatalf("the cards did not change within %s", maxWait)
|
|
||||||
}
|
|
||||||
|
|
||||||
time.Sleep(pollInterval)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+26
-183
@@ -3,26 +3,18 @@ package server
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"crypto/rand"
|
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/netip"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/go-chi/chi/v5"
|
|
||||||
"github.com/go-chi/chi/v5/middleware"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// responseWriter wraps http.ResponseWriter to capture the response
|
// responseWriter wraps http.ResponseWriter to capture the response
|
||||||
type responseWriter struct {
|
type responseWriter struct {
|
||||||
http.ResponseWriter
|
http.ResponseWriter
|
||||||
|
|
||||||
body *bytes.Buffer
|
body *bytes.Buffer
|
||||||
statusCode int
|
statusCode int
|
||||||
written bool
|
written bool
|
||||||
@@ -54,15 +46,6 @@ func (rw *responseWriter) Header() http.Header {
|
|||||||
return rw.ResponseWriter.Header()
|
return rw.ResponseWriter.Header()
|
||||||
}
|
}
|
||||||
|
|
||||||
// The @meta object JSONResponseMiddleware adds to a JSON answer: its key, and
|
|
||||||
// the keys of its fields.
|
|
||||||
const (
|
|
||||||
jsonKeyMeta = "@meta"
|
|
||||||
metaKeyTimeZone = "time_zone"
|
|
||||||
metaKeyAPIVersion = "api_version"
|
|
||||||
metaKeyExecutionTimeMs = "execution_time_ms"
|
|
||||||
)
|
|
||||||
|
|
||||||
// JSONResponseMiddleware is an HTTP middleware that wraps all JSON responses
|
// JSONResponseMiddleware is an HTTP middleware that wraps all JSON responses
|
||||||
// with a @meta field containing execution metadata. The metadata includes the
|
// with a @meta field containing execution metadata. The metadata includes the
|
||||||
// time zone (always UTC), API version, and request execution time in milliseconds.
|
// time zone (always UTC), API version, and request execution time in milliseconds.
|
||||||
@@ -104,10 +87,8 @@ func JSONResponseMiddleware(next http.Handler) http.Handler {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Parse the original response
|
// Parse the original response
|
||||||
var originalResponse map[string]any
|
var originalResponse map[string]interface{}
|
||||||
|
if err := json.Unmarshal(rw.body.Bytes(), &originalResponse); err != nil {
|
||||||
err := json.Unmarshal(rw.body.Bytes(), &originalResponse)
|
|
||||||
if err != nil {
|
|
||||||
// If we can't parse it, just write original
|
// If we can't parse it, just write original
|
||||||
w.WriteHeader(rw.statusCode)
|
w.WriteHeader(rw.statusCode)
|
||||||
_, _ = w.Write(rw.body.Bytes())
|
_, _ = w.Write(rw.body.Bytes())
|
||||||
@@ -116,10 +97,10 @@ func JSONResponseMiddleware(next http.Handler) http.Handler {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Add @meta field
|
// Add @meta field
|
||||||
originalResponse[jsonKeyMeta] = map[string]any{
|
originalResponse["@meta"] = map[string]interface{}{
|
||||||
metaKeyTimeZone: "UTC",
|
"time_zone": "UTC",
|
||||||
metaKeyAPIVersion: 1,
|
"api_version": 1,
|
||||||
metaKeyExecutionTimeMs: execTime.Milliseconds(),
|
"execution_time": fmt.Sprintf("%d ms", execTime.Milliseconds()),
|
||||||
}
|
}
|
||||||
|
|
||||||
// Write the enhanced response
|
// Write the enhanced response
|
||||||
@@ -132,17 +113,9 @@ func JSONResponseMiddleware(next http.Handler) http.Handler {
|
|||||||
// requestTimeoutMessage is the error message sent with 504 Gateway Timeout.
|
// requestTimeoutMessage is the error message sent with 504 Gateway Timeout.
|
||||||
const requestTimeoutMessage = "Request timeout"
|
const requestTimeoutMessage = "Request timeout"
|
||||||
|
|
||||||
// internalErrorMessage is the error message sent with 500 Internal Server
|
// deadlinePassed reports whether ctx ended because its deadline passed. When
|
||||||
// Error, whatever went wrong; the handler logs the detail.
|
// a handler's database read or other lookup fails, it uses this to tell a
|
||||||
const internalErrorMessage = "Internal server error"
|
// request cut off by its time limit, which answers 504, from other failures.
|
||||||
|
|
||||||
// deadlinePassed reports whether ctx ended because its deadline passed.
|
|
||||||
//
|
|
||||||
// A handler calls it right after each database read or host name lookup,
|
|
||||||
// before it looks at what came back, error or not, and answers 504 if it
|
|
||||||
// reports true. A read cut off by the time limit can return an error, or part
|
|
||||||
// of its rows or none of them without one, so only this check tells a cut-off
|
|
||||||
// read from a complete one.
|
|
||||||
func deadlinePassed(ctx context.Context) bool {
|
func deadlinePassed(ctx context.Context) bool {
|
||||||
return errors.Is(ctx.Err(), context.DeadlineExceeded)
|
return errors.Is(ctx.Err(), context.DeadlineExceeded)
|
||||||
}
|
}
|
||||||
@@ -152,11 +125,9 @@ func deadlinePassed(ctx context.Context) bool {
|
|||||||
//
|
//
|
||||||
// If a response is not valid JSON or is empty when JSON is expected, the middleware
|
// If a response is not valid JSON or is empty when JSON is expected, the middleware
|
||||||
// returns a properly formatted JSON error response. If the request's time limit
|
// returns a properly formatted JSON error response. If the request's time limit
|
||||||
// has passed, it answers 504 with the error message "Request timeout". A 4xx
|
// has passed, it answers 504 with the error message "Request timeout". For other
|
||||||
// keeps its status, with the status's standard text as the message. For other
|
// errors, it returns "Internal server error" with status 500 if the original
|
||||||
// errors, it returns "Internal server error", with status 500 if the original
|
// status was 200.
|
||||||
// status was 200. A panic in the handler answers 500 with "Internal server
|
|
||||||
// error" too.
|
|
||||||
//
|
//
|
||||||
// Non-JSON responses are passed through unchanged.
|
// Non-JSON responses are passed through unchanged.
|
||||||
func JSONValidationMiddleware(next http.Handler) http.Handler {
|
func JSONValidationMiddleware(next http.Handler) http.Handler {
|
||||||
@@ -168,9 +139,8 @@ func JSONValidationMiddleware(next http.Handler) http.Handler {
|
|||||||
statusCode: http.StatusOK,
|
statusCode: http.StatusOK,
|
||||||
}
|
}
|
||||||
|
|
||||||
// Serve the request. Recoverer logs a panic in the handler and turns it
|
// Serve the request
|
||||||
// into a 500 without a body, which gets the JSON error body below.
|
next.ServeHTTP(rw, r)
|
||||||
middleware.Recoverer(next).ServeHTTP(rw, r)
|
|
||||||
|
|
||||||
// Check if it's meant to be a JSON response
|
// Check if it's meant to be a JSON response
|
||||||
contentType := rw.Header().Get("Content-Type")
|
contentType := rw.Header().Get("Content-Type")
|
||||||
@@ -186,10 +156,8 @@ func JSONValidationMiddleware(next http.Handler) http.Handler {
|
|||||||
|
|
||||||
// For JSON responses, validate the JSON
|
// For JSON responses, validate the JSON
|
||||||
if rw.body.Len() > 0 {
|
if rw.body.Len() > 0 {
|
||||||
var testParse any
|
var testParse interface{}
|
||||||
|
if err := json.Unmarshal(rw.body.Bytes(), &testParse); err == nil {
|
||||||
err := json.Unmarshal(rw.body.Bytes(), &testParse)
|
|
||||||
if err == nil {
|
|
||||||
// Valid JSON, write it out
|
// Valid JSON, write it out
|
||||||
w.Header().Set("Content-Type", "application/json")
|
w.Header().Set("Content-Type", "application/json")
|
||||||
w.WriteHeader(rw.statusCode)
|
w.WriteHeader(rw.statusCode)
|
||||||
@@ -208,28 +176,21 @@ func JSONValidationMiddleware(next http.Handler) http.Handler {
|
|||||||
if statusCode == http.StatusOK {
|
if statusCode == http.StatusOK {
|
||||||
statusCode = http.StatusInternalServerError
|
statusCode = http.StatusInternalServerError
|
||||||
}
|
}
|
||||||
|
|
||||||
if deadlinePassed(r.Context()) {
|
if deadlinePassed(r.Context()) {
|
||||||
statusCode = http.StatusGatewayTimeout
|
statusCode = http.StatusGatewayTimeout
|
||||||
}
|
}
|
||||||
|
|
||||||
w.WriteHeader(statusCode)
|
w.WriteHeader(statusCode)
|
||||||
|
|
||||||
errorMsg := internalErrorMessage
|
errorMsg := "Internal server error"
|
||||||
|
if statusCode == http.StatusGatewayTimeout {
|
||||||
switch {
|
|
||||||
case statusCode == http.StatusGatewayTimeout:
|
|
||||||
errorMsg = requestTimeoutMessage
|
errorMsg = requestTimeoutMessage
|
||||||
case statusCode >= http.StatusBadRequest &&
|
|
||||||
statusCode < http.StatusInternalServerError:
|
|
||||||
errorMsg = http.StatusText(statusCode)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
response := map[string]any{
|
response := map[string]interface{}{
|
||||||
jsonKeyStatus: jsonStatusError,
|
"status": "error",
|
||||||
jsonKeyError: map[string]any{
|
"error": map[string]interface{}{
|
||||||
jsonKeyMsg: errorMsg,
|
"msg": errorMsg,
|
||||||
jsonKeyCode: statusCode,
|
"code": statusCode,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
_ = json.NewEncoder(w).Encode(response)
|
_ = json.NewEncoder(w).Encode(response)
|
||||||
@@ -239,7 +200,6 @@ func JSONValidationMiddleware(next http.Handler) http.Handler {
|
|||||||
// statusWriter wraps http.ResponseWriter to capture the status code
|
// statusWriter wraps http.ResponseWriter to capture the status code
|
||||||
type statusWriter struct {
|
type statusWriter struct {
|
||||||
http.ResponseWriter
|
http.ResponseWriter
|
||||||
|
|
||||||
statusCode int
|
statusCode int
|
||||||
written bool
|
written bool
|
||||||
}
|
}
|
||||||
@@ -249,7 +209,6 @@ func (sw *statusWriter) WriteHeader(statusCode int) {
|
|||||||
sw.statusCode = statusCode
|
sw.statusCode = statusCode
|
||||||
sw.written = true
|
sw.written = true
|
||||||
}
|
}
|
||||||
|
|
||||||
sw.ResponseWriter.WriteHeader(statusCode)
|
sw.ResponseWriter.WriteHeader(statusCode)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -289,12 +248,11 @@ func RequestLoggerMiddleware(logger *slog.Logger) func(http.Handler) http.Handle
|
|||||||
// Slow query threshold (1 second)
|
// Slow query threshold (1 second)
|
||||||
const slowQueryThreshold = 1 * time.Second
|
const slowQueryThreshold = 1 * time.Second
|
||||||
|
|
||||||
switch {
|
if sw.statusCode >= http.StatusInternalServerError {
|
||||||
case sw.statusCode >= http.StatusInternalServerError:
|
|
||||||
logLevel = slog.LevelError
|
logLevel = slog.LevelError
|
||||||
case sw.statusCode >= http.StatusBadRequest:
|
} else if sw.statusCode >= http.StatusBadRequest {
|
||||||
logLevel = slog.LevelWarn
|
logLevel = slog.LevelWarn
|
||||||
case duration >= slowQueryThreshold:
|
} else if duration >= slowQueryThreshold {
|
||||||
logLevel = slog.LevelWarn
|
logLevel = slog.LevelWarn
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -309,118 +267,3 @@ func RequestLoggerMiddleware(logger *slog.Logger) func(http.Handler) http.Handle
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// DecodedPathMiddleware makes the router match routes against the decoded
|
|
||||||
// path, so that the handlers read their URL parameters decoded. Without it,
|
|
||||||
// chi matches against the escaped path whenever the request has one and hands
|
|
||||||
// the handlers their parameters still escaped: /ip/2001%3Adb8%3A%3A1 would
|
|
||||||
// look up "2001%3Adb8%3A%3A1".
|
|
||||||
func DecodedPathMiddleware(next http.Handler) http.Handler {
|
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
chi.RouteContext(r.Context()).RoutePath = r.URL.Path
|
|
||||||
|
|
||||||
next.ServeHTTP(w, r)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// The values of the headers SecurityHeadersMiddleware sets.
|
|
||||||
const (
|
|
||||||
// strictTransportSecurity tells browsers to reach this host and its
|
|
||||||
// subdomains only over HTTPS for a year. The server listens on plain
|
|
||||||
// HTTP behind a TLS-terminating proxy, which passes the header on.
|
|
||||||
strictTransportSecurity = "max-age=31536000; includeSubDomains"
|
|
||||||
|
|
||||||
// contentSecurityPolicy lets a page load content only from this server,
|
|
||||||
// and run or apply an inline script or style block only when the block
|
|
||||||
// carries the nonce made for that answer, which replaces %[1]s. No other
|
|
||||||
// site may frame the pages.
|
|
||||||
contentSecurityPolicy = "default-src 'self'; " +
|
|
||||||
"script-src 'nonce-%[1]s'; style-src 'nonce-%[1]s'; " +
|
|
||||||
"frame-ancestors 'none'; base-uri 'none'; form-action 'self'"
|
|
||||||
|
|
||||||
frameOptions = "DENY"
|
|
||||||
contentTypeOptions = "nosniff"
|
|
||||||
referrerPolicy = "strict-origin-when-cross-origin"
|
|
||||||
|
|
||||||
// permissionsPolicy turns off browser features the pages do not use.
|
|
||||||
permissionsPolicy = "accelerometer=(), camera=(), geolocation=(), " +
|
|
||||||
"gyroscope=(), magnetometer=(), microphone=(), payment=(), usb=()"
|
|
||||||
)
|
|
||||||
|
|
||||||
// cspNonceKey is the request context key of the nonce SecurityHeadersMiddleware
|
|
||||||
// made for the answer.
|
|
||||||
type cspNonceKey struct{}
|
|
||||||
|
|
||||||
// SecurityHeadersMiddleware sets the security headers REPO_POLICIES.md requires
|
|
||||||
// on every answer. Each answer's Content-Security-Policy names a nonce of its
|
|
||||||
// own; a page reads it with cspNonce and writes it on its script and style
|
|
||||||
// blocks.
|
|
||||||
func SecurityHeadersMiddleware(next http.Handler) http.Handler {
|
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
nonce := rand.Text()
|
|
||||||
|
|
||||||
header := w.Header()
|
|
||||||
header.Set("Strict-Transport-Security", strictTransportSecurity)
|
|
||||||
header.Set("Content-Security-Policy",
|
|
||||||
fmt.Sprintf(contentSecurityPolicy, nonce))
|
|
||||||
header.Set("X-Frame-Options", frameOptions)
|
|
||||||
header.Set("X-Content-Type-Options", contentTypeOptions)
|
|
||||||
header.Set("Referrer-Policy", referrerPolicy)
|
|
||||||
header.Set("Permissions-Policy", permissionsPolicy)
|
|
||||||
|
|
||||||
ctx := context.WithValue(r.Context(), cspNonceKey{}, nonce)
|
|
||||||
next.ServeHTTP(w, r.WithContext(ctx))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// cspNonce returns the nonce SecurityHeadersMiddleware made for the answer to
|
|
||||||
// the request with ctx.
|
|
||||||
func cspNonce(ctx context.Context) string {
|
|
||||||
nonce, _ := ctx.Value(cspNonceKey{}).(string)
|
|
||||||
|
|
||||||
return nonce
|
|
||||||
}
|
|
||||||
|
|
||||||
// RealIPMiddleware gives a request from one of trustedProxies the address of
|
|
||||||
// the client the proxies forwarded it for as its RemoteAddr: the last
|
|
||||||
// X-Forwarded-For entry that is not a trusted proxy. Each proxy appends the
|
|
||||||
// address it took the request from, so the entries left of that one are
|
|
||||||
// whatever the client sent. A request from any other peer keeps the peer's
|
|
||||||
// address, whatever headers it carries.
|
|
||||||
func RealIPMiddleware(trustedProxies []netip.Prefix) func(http.Handler) http.Handler {
|
|
||||||
isTrusted := func(addr netip.Addr) bool {
|
|
||||||
return slices.ContainsFunc(trustedProxies, func(proxy netip.Prefix) bool {
|
|
||||||
return proxy.Contains(addr.Unmap())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return func(next http.Handler) http.Handler {
|
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
peer, err := netip.ParseAddrPort(r.RemoteAddr)
|
|
||||||
if err != nil || !isTrusted(peer.Addr()) {
|
|
||||||
next.ServeHTTP(w, r)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// A header sent on several lines is one list.
|
|
||||||
entries := strings.Split(
|
|
||||||
strings.Join(r.Header.Values("X-Forwarded-For"), ","), ",")
|
|
||||||
|
|
||||||
for _, entry := range slices.Backward(entries) {
|
|
||||||
addr, err := netip.ParseAddr(strings.TrimSpace(entry))
|
|
||||||
if err != nil {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
|
|
||||||
r.RemoteAddr = addr.String()
|
|
||||||
|
|
||||||
if !isTrusted(addr) {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
next.ServeHTTP(w, r)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,597 +0,0 @@
|
|||||||
package server
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"io"
|
|
||||||
"log/slog"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"net/netip"
|
|
||||||
"os"
|
|
||||||
"regexp"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/config"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/database"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/metrics"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/streamer"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The addresses the forwarding tests use: the client a proxy names in
|
|
||||||
// X-Forwarded-For, the range of trusted proxies, a peer inside it, and a peer
|
|
||||||
// outside it, which lies in testPrefixV4 so that an IP lookup of it finds a
|
|
||||||
// route. reverseProxyPeer is a reverse proxy at a private address, as Docker
|
|
||||||
// gives one.
|
|
||||||
const (
|
|
||||||
forwardedClient = "203.0.113.9"
|
|
||||||
trustedProxyRange = "10.0.0.0/8"
|
|
||||||
trustedPeer = "10.0.0.2:1234"
|
|
||||||
untrustedPeerAddr = "192.0.2.1"
|
|
||||||
untrustedPeer = untrustedPeerAddr + ":1234"
|
|
||||||
reverseProxyPeer = "172.19.0.14:1234"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestResponsesCarrySecurityHeaders checks that a page, a JSON route and a
|
|
||||||
// path no route matches each answer with the security headers REPO_POLICIES.md
|
|
||||||
// requires, and that the Content-Security-Policy allows no inline content
|
|
||||||
// without a nonce (https://git.eeqj.de/sneak/routewatch/issues/108).
|
|
||||||
func TestResponsesCarrySecurityHeaders(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := newServerWithTestRoutes(t)
|
|
||||||
|
|
||||||
headers := []string{
|
|
||||||
"Strict-Transport-Security",
|
|
||||||
"Content-Security-Policy",
|
|
||||||
"X-Frame-Options",
|
|
||||||
"X-Content-Type-Options",
|
|
||||||
"Referrer-Policy",
|
|
||||||
"Permissions-Policy",
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, path := range []string{"/", statsPath, "/no-such-page"} {
|
|
||||||
rec := httptest.NewRecorder()
|
|
||||||
s.router.ServeHTTP(rec, httptest.NewRequestWithContext(
|
|
||||||
t.Context(), http.MethodGet, path, nil))
|
|
||||||
|
|
||||||
for _, name := range headers {
|
|
||||||
if rec.Header().Get(name) == "" {
|
|
||||||
t.Errorf("%s: no %s header", path, name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
policy := rec.Header().Get("Content-Security-Policy")
|
|
||||||
if strings.Contains(policy, "'unsafe-") {
|
|
||||||
t.Errorf("%s: Content-Security-Policy %q allows unsafe content",
|
|
||||||
path, policy)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestJSONValidationKeeps4xxWithoutBody wraps a handler that answers 400
|
|
||||||
// without a body in JSONValidationMiddleware and checks that the JSON error
|
|
||||||
// body keeps the 400 and gives the status's own text. Before the fix it said
|
|
||||||
// "Internal server error"
|
|
||||||
// (https://git.eeqj.de/sneak/routewatch/issues/111).
|
|
||||||
func TestJSONValidationKeeps4xxWithoutBody(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
handler := JSONValidationMiddleware(http.HandlerFunc(
|
|
||||||
func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
w.WriteHeader(http.StatusBadRequest)
|
|
||||||
}))
|
|
||||||
|
|
||||||
rec := httptest.NewRecorder()
|
|
||||||
handler.ServeHTTP(rec, httptest.NewRequestWithContext(
|
|
||||||
t.Context(), http.MethodGet, "/", nil))
|
|
||||||
|
|
||||||
const name = "400 without a body"
|
|
||||||
|
|
||||||
if rec.Code != http.StatusBadRequest {
|
|
||||||
t.Errorf("%s: status %d, want %d; body %s",
|
|
||||||
name, rec.Code, http.StatusBadRequest, rec.Body)
|
|
||||||
}
|
|
||||||
|
|
||||||
checkJSONErrorCode(t, name, rec, http.StatusBadRequest)
|
|
||||||
checkJSONErrorMessage(t, name, rec, http.StatusText(http.StatusBadRequest))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestJSONAnswersPutNoUnitInAString sends a request to each JSON route and
|
|
||||||
// checks each answer. Its @meta.execution_time_ms is a number, and none of its
|
|
||||||
// strings is a number with a unit, such as "12 ms", "1.1 GB" or "1h18m34s"
|
|
||||||
// (https://git.eeqj.de/sneak/routewatch/issues/199). The health check answers
|
|
||||||
// 503, since the stream is not connected, and the last path matches no route.
|
|
||||||
func TestJSONAnswersPutNoUnitInAString(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
numberWithUnit := regexp.MustCompile(
|
|
||||||
`^([0-9]+(\.[0-9]+)? ?(ns|µs|ms|s|m|h|B|kB|MB|GB|TB))+$`)
|
|
||||||
|
|
||||||
s := newServerWithTestRoutes(t)
|
|
||||||
s.resolver = &fakeResolver{}
|
|
||||||
s.SetASNFetcher(fixedIntervalASNFetcher{interval: time.Second})
|
|
||||||
|
|
||||||
paths := []string{
|
|
||||||
statusJSONPath,
|
|
||||||
statsPath,
|
|
||||||
healthCheckPath,
|
|
||||||
"/api/v1/ip/" + untrustedPeerAddr,
|
|
||||||
"/ip/" + untrustedPeerAddr,
|
|
||||||
asJSONPath,
|
|
||||||
"/api/v1/prefix/" + testPrefixV4,
|
|
||||||
"/api/v1/search/as?q=google",
|
|
||||||
"/api/v1/no-such-route",
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, path := range paths {
|
|
||||||
rec := httptest.NewRecorder()
|
|
||||||
s.router.ServeHTTP(rec, httptest.NewRequestWithContext(
|
|
||||||
t.Context(), http.MethodGet, path, nil))
|
|
||||||
|
|
||||||
var answer map[string]any
|
|
||||||
|
|
||||||
err := json.Unmarshal(rec.Body.Bytes(), &answer)
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("%s: decoding the answer: %v; answer %s", path, err, rec.Body)
|
|
||||||
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
meta, _ := answer["@meta"].(map[string]any)
|
|
||||||
if _, ok := meta["execution_time_ms"].(float64); !ok {
|
|
||||||
t.Errorf("%s: @meta.execution_time_ms is not a number: %s",
|
|
||||||
path, rec.Body)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, value := range stringsIn(answer) {
|
|
||||||
if numberWithUnit.MatchString(value) {
|
|
||||||
t.Errorf("%s: %q is a number with a unit in a string", path, value)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestJSONAnswersCarryMeta sends requests to every JSON route through the
|
|
||||||
// server's router, and checks that each answer carries @meta with its three
|
|
||||||
// fields. The answers include successes, and errors that the handlers, the
|
|
||||||
// router and the middleware write. Before the fix a panic in a handler, and a
|
|
||||||
// method /status.json or the health check does not take, answered with an
|
|
||||||
// empty body (https://git.eeqj.de/sneak/routewatch/issues/201).
|
|
||||||
func TestJSONAnswersCarryMeta(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// JSON routes whose handlers write nothing or panic, so that the JSON
|
|
||||||
// check on the route writes the error answer itself.
|
|
||||||
const (
|
|
||||||
writesNothingPath = "/writes-nothing.json"
|
|
||||||
panicsPath = "/panics.json"
|
|
||||||
)
|
|
||||||
|
|
||||||
s := newServerWithTestRoutes(t)
|
|
||||||
s.resolver = &fakeResolver{}
|
|
||||||
s.streamer.GetMetricsTracker().SetConnected(true)
|
|
||||||
s.router.Get(writesNothingPath, JSONValidationMiddleware(http.HandlerFunc(
|
|
||||||
func(http.ResponseWriter, *http.Request) {})).ServeHTTP)
|
|
||||||
s.router.Get(panicsPath, JSONValidationMiddleware(http.HandlerFunc(
|
|
||||||
func(http.ResponseWriter, *http.Request) { panic(errDiskIO) })).ServeHTTP)
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
method string
|
|
||||||
path string
|
|
||||||
wantCode int
|
|
||||||
}{
|
|
||||||
// Answers that succeed
|
|
||||||
{http.MethodGet, statusJSONPath, http.StatusOK},
|
|
||||||
{http.MethodGet, healthCheckPath, http.StatusOK},
|
|
||||||
{http.MethodGet, "/ip", http.StatusOK},
|
|
||||||
{http.MethodGet, ownAddressPath, http.StatusOK},
|
|
||||||
{http.MethodGet, "/ip/" + addrInTestPrefixV4, http.StatusOK},
|
|
||||||
{http.MethodGet, statsPath, http.StatusOK},
|
|
||||||
{http.MethodGet, "/api/v1/ip/" + addrInTestPrefixV4, http.StatusOK},
|
|
||||||
{http.MethodGet, asJSONPath, http.StatusOK},
|
|
||||||
{http.MethodGet, "/api/v1/prefix/" + testPrefixV4, http.StatusOK},
|
|
||||||
{http.MethodGet, "/api/v1/search/as?q=google", http.StatusOK},
|
|
||||||
// Errors the handlers write
|
|
||||||
{http.MethodGet, "/ip/no-such-host.example", http.StatusBadRequest},
|
|
||||||
{http.MethodGet, "/api/v1/ip/no-such-host.example", http.StatusBadRequest},
|
|
||||||
{http.MethodGet, "/api/v1/as/foo", http.StatusBadRequest},
|
|
||||||
{http.MethodGet, "/api/v1/as/0", http.StatusNotFound},
|
|
||||||
{http.MethodGet, "/api/v1/prefix/192.0.2.0/33", http.StatusBadRequest},
|
|
||||||
{http.MethodGet, "/api/v1/prefix/198.51.100.0/24", http.StatusNotFound},
|
|
||||||
{http.MethodGet, "/api/v1/search/as", http.StatusBadRequest},
|
|
||||||
// Errors the router and the middleware write
|
|
||||||
{http.MethodGet, "/api/v1/no-such-route", http.StatusNotFound},
|
|
||||||
{http.MethodGet, "/ip/192.0.2.1/nothing", http.StatusNotFound},
|
|
||||||
{http.MethodPost, statsPath, http.StatusMethodNotAllowed},
|
|
||||||
{http.MethodPost, ownAddressPath, http.StatusMethodNotAllowed},
|
|
||||||
{http.MethodPost, statusJSONPath, http.StatusMethodNotAllowed},
|
|
||||||
{http.MethodPut, healthCheckPath, http.StatusMethodNotAllowed},
|
|
||||||
{http.MethodGet, writesNothingPath, http.StatusInternalServerError},
|
|
||||||
{http.MethodGet, panicsPath, http.StatusInternalServerError},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
checkAnswerCarriesMeta(t, s, httptest.NewRequestWithContext(
|
|
||||||
t.Context(), tt.method, tt.path, nil), tt.wantCode)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A request whose time limit has already passed answers 504. The handlers
|
|
||||||
// write the answers of the first five routes, and the JSON check on the
|
|
||||||
// route writes the last.
|
|
||||||
pastTimeLimit, cancel := context.WithDeadline(t.Context(), time.Now())
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
for _, path := range []string{
|
|
||||||
statusJSONPath, statsPath, "/ip/" + addrInTestPrefixV4,
|
|
||||||
"/api/v1/ip/" + addrInTestPrefixV4, asJSONPath, writesNothingPath,
|
|
||||||
} {
|
|
||||||
checkAnswerCarriesMeta(t, s, httptest.NewRequestWithContext(
|
|
||||||
pastTimeLimit, http.MethodGet, path, nil), http.StatusGatewayTimeout)
|
|
||||||
}
|
|
||||||
|
|
||||||
// With a store whose reads fail, the health check answers 503, and the
|
|
||||||
// handlers of the other routes 500.
|
|
||||||
failing := New(failingDB{}, newTestTable(),
|
|
||||||
streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{})
|
|
||||||
failing.resolver = &fakeResolver{}
|
|
||||||
|
|
||||||
checkAnswerCarriesMeta(t, failing, httptest.NewRequestWithContext(
|
|
||||||
t.Context(), http.MethodGet, healthCheckPath, nil),
|
|
||||||
http.StatusServiceUnavailable)
|
|
||||||
|
|
||||||
for _, path := range []string{
|
|
||||||
statusJSONPath, statsPath, asJSONPath,
|
|
||||||
"/api/v1/ip/" + addrInTestPrefixV4, "/ip/" + addrInTestPrefixV4,
|
|
||||||
} {
|
|
||||||
checkAnswerCarriesMeta(t, failing, httptest.NewRequestWithContext(
|
|
||||||
t.Context(), http.MethodGet, path, nil), http.StatusInternalServerError)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkAnswerCarriesMeta serves req through the server's router, and checks
|
|
||||||
// that it answers wantCode with @meta and its three fields.
|
|
||||||
func checkAnswerCarriesMeta(
|
|
||||||
t *testing.T, s *Server, req *http.Request, wantCode int,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
// The values README "API v1" gives the two fields that never change.
|
|
||||||
const (
|
|
||||||
wantTimeZone = "UTC"
|
|
||||||
wantAPIVersion = 1
|
|
||||||
)
|
|
||||||
|
|
||||||
name := req.Method + " " + req.URL.String()
|
|
||||||
|
|
||||||
rec := httptest.NewRecorder()
|
|
||||||
s.router.ServeHTTP(rec, req)
|
|
||||||
|
|
||||||
if rec.Code != wantCode {
|
|
||||||
t.Errorf("%s: status %d, want %d; body %s", name, rec.Code, wantCode, rec.Body)
|
|
||||||
}
|
|
||||||
|
|
||||||
var answer map[string]any
|
|
||||||
|
|
||||||
err := json.Unmarshal(rec.Body.Bytes(), &answer)
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("%s: decoding the answer %q: %v", name, rec.Body, err)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
meta, _ := answer[jsonKeyMeta].(map[string]any)
|
|
||||||
_, timeIsNumber := meta[metaKeyExecutionTimeMs].(float64)
|
|
||||||
|
|
||||||
if meta[metaKeyTimeZone] != wantTimeZone ||
|
|
||||||
meta[metaKeyAPIVersion] != float64(wantAPIVersion) || !timeIsNumber {
|
|
||||||
t.Errorf("%s: @meta %v, want time_zone %q, api_version %d and a "+
|
|
||||||
"number in execution_time_ms", name, meta, wantTimeZone, wantAPIVersion)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// stringsIn returns every string in a decoded JSON value, at any depth.
|
|
||||||
func stringsIn(value any) []string {
|
|
||||||
var found []string
|
|
||||||
|
|
||||||
switch v := value.(type) {
|
|
||||||
case string:
|
|
||||||
found = append(found, v)
|
|
||||||
case []any:
|
|
||||||
for _, item := range v {
|
|
||||||
found = append(found, stringsIn(item)...)
|
|
||||||
}
|
|
||||||
case map[string]any:
|
|
||||||
for _, item := range v {
|
|
||||||
found = append(found, stringsIn(item)...)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return found
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPagesCarryTheCSPNonce checks that every script and style block of each
|
|
||||||
// page carries the nonce its Content-Security-Policy header names, so that the
|
|
||||||
// browser runs and applies them, that each answer has a nonce of its own, and
|
|
||||||
// that no page uses a style attribute, which the policy does not allow.
|
|
||||||
func TestPagesCarryTheCSPNonce(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := newServerWithTestRoutes(t)
|
|
||||||
|
|
||||||
_, err := s.db.GetOrCreateASN(64500, time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GetOrCreateASN: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
paths := []string{
|
|
||||||
"/", "/status", asPagePath, "/prefix/" + testPrefixV4,
|
|
||||||
prefixLengthPagePath, "/prefixlength6/32",
|
|
||||||
}
|
|
||||||
seen := make(map[string]bool)
|
|
||||||
|
|
||||||
for _, path := range paths {
|
|
||||||
rec := httptest.NewRecorder()
|
|
||||||
s.router.ServeHTTP(rec, httptest.NewRequestWithContext(
|
|
||||||
t.Context(), http.MethodGet, path, nil))
|
|
||||||
|
|
||||||
if rec.Code != http.StatusOK {
|
|
||||||
t.Errorf("%s: status %d, want %d; body %s",
|
|
||||||
path, rec.Code, http.StatusOK, rec.Body)
|
|
||||||
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
policy := rec.Header().Get("Content-Security-Policy")
|
|
||||||
|
|
||||||
_, nonce, found := strings.Cut(policy, "'nonce-")
|
|
||||||
if !found {
|
|
||||||
t.Errorf("%s: Content-Security-Policy %q names no nonce", path, policy)
|
|
||||||
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
nonce, _, _ = strings.Cut(nonce, "'")
|
|
||||||
if seen[nonce] {
|
|
||||||
t.Errorf("%s: nonce %q was already used for another answer", path, nonce)
|
|
||||||
}
|
|
||||||
|
|
||||||
seen[nonce] = true
|
|
||||||
page := rec.Body.String()
|
|
||||||
|
|
||||||
for _, tag := range []string{"<script", "<style"} {
|
|
||||||
withNonce := tag + ` nonce="` + nonce + `"`
|
|
||||||
if strings.Count(page, tag) != strings.Count(page, withNonce) {
|
|
||||||
t.Errorf("%s: a %s tag does not carry the nonce %q", path, tag, nonce)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if strings.Contains(page, "style=") {
|
|
||||||
t.Errorf("%s: the page uses a style attribute", path)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestForwardedForFromUntrustedPeerIsIgnored asks /ip/ for the caller's own
|
|
||||||
// address from a peer that is not a trusted proxy, with forwarding headers
|
|
||||||
// naming another address, and checks that the lookup is for the TCP peer.
|
|
||||||
// Before the fix any client could choose the address it was looked up and
|
|
||||||
// logged as (https://git.eeqj.de/sneak/routewatch/issues/108).
|
|
||||||
func TestForwardedForFromUntrustedPeerIsIgnored(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := newServerWithTestRoutes(t)
|
|
||||||
|
|
||||||
req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/ip/", nil)
|
|
||||||
req.RemoteAddr = untrustedPeer
|
|
||||||
req.Header.Set("X-Forwarded-For", forwardedClient)
|
|
||||||
req.Header.Set("X-Real-IP", forwardedClient)
|
|
||||||
req.Header.Set("True-Client-IP", forwardedClient)
|
|
||||||
|
|
||||||
rec := httptest.NewRecorder()
|
|
||||||
s.router.ServeHTTP(rec, req)
|
|
||||||
|
|
||||||
var body struct {
|
|
||||||
Data IPLookupResponse `json:"data"`
|
|
||||||
}
|
|
||||||
|
|
||||||
err := json.Unmarshal(rec.Body.Bytes(), &body)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("decoding the answer %q: %v", rec.Body, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if body.Data.Query != untrustedPeerAddr {
|
|
||||||
t.Errorf("looked up %q, want the TCP peer %s; answer %s",
|
|
||||||
body.Data.Query, untrustedPeerAddr, rec.Body)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestDefaultTrustedProxiesGiveTheForwardedClient builds the server from the
|
|
||||||
// configuration with TRUSTED_PROXIES unset. A request from a reverse proxy at
|
|
||||||
// a private address must be looked up by /ip/, shown on the home page's card
|
|
||||||
// and logged as the client its X-Forwarded-For names. A direct request from a
|
|
||||||
// public address with a forged X-Forwarded-For keeps its own address. Before
|
|
||||||
// the fix no proxy was trusted unless TRUSTED_PROXIES named it, so /ip/ gave
|
|
||||||
// the proxy's address (https://git.eeqj.de/sneak/routewatch/issues/198).
|
|
||||||
func TestDefaultTrustedProxiesGiveTheForwardedClient(t *testing.T) {
|
|
||||||
t.Setenv("PORT", "")
|
|
||||||
t.Setenv("XDG_DATA_HOME", "")
|
|
||||||
|
|
||||||
// t.Setenv puts the variable back when the test ends.
|
|
||||||
t.Setenv("TRUSTED_PROXIES", "")
|
|
||||||
|
|
||||||
err := os.Unsetenv("TRUSTED_PROXIES")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg, err := config.New()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("config.New: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var requestLog bytes.Buffer
|
|
||||||
|
|
||||||
s := newServerWithConfig(t, cfg, &requestLog)
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
peer string
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{"through a reverse proxy", reverseProxyPeer, forwardedClient},
|
|
||||||
{"direct from a public address", untrustedPeer, untrustedPeerAddr},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
requestLog.Reset()
|
|
||||||
|
|
||||||
get := func(path string) *httptest.ResponseRecorder {
|
|
||||||
req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, path, nil)
|
|
||||||
req.RemoteAddr = tt.peer
|
|
||||||
req.Header.Set("X-Forwarded-For", forwardedClient)
|
|
||||||
|
|
||||||
rec := httptest.NewRecorder()
|
|
||||||
s.router.ServeHTTP(rec, req)
|
|
||||||
|
|
||||||
return rec
|
|
||||||
}
|
|
||||||
|
|
||||||
var answer struct {
|
|
||||||
Data IPLookupResponse `json:"data"`
|
|
||||||
}
|
|
||||||
|
|
||||||
rec := get("/ip/")
|
|
||||||
|
|
||||||
err := json.Unmarshal(rec.Body.Bytes(), &answer)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("%s: decoding the answer %q: %v", tt.name, rec.Body, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if answer.Data.Query != tt.want {
|
|
||||||
t.Errorf("%s: /ip/ looked up %q, want %q", tt.name, answer.Data.Query, tt.want)
|
|
||||||
}
|
|
||||||
|
|
||||||
page := get("/").Body.String()
|
|
||||||
if !strings.Contains(page, `data-client-ip="`+tt.want+`"`) {
|
|
||||||
t.Errorf("%s: the home page's card does not hold %s", tt.name, tt.want)
|
|
||||||
}
|
|
||||||
|
|
||||||
logged := loggedAddresses(requestLog.String())
|
|
||||||
if len(logged) == 0 {
|
|
||||||
t.Errorf("%s: the request log names no address", tt.name)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, addr := range logged {
|
|
||||||
if addr != tt.want {
|
|
||||||
t.Errorf("%s: the request log gives %s, want %s", tt.name, addr, tt.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// newServerWithConfig returns a server built with cfg, on a new database, that
|
|
||||||
// reads the routes of newTestTable and logs its requests as JSON to
|
|
||||||
// requestLog.
|
|
||||||
func newServerWithConfig(
|
|
||||||
t *testing.T, cfg *config.Config, requestLog io.Writer,
|
|
||||||
) *Server {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
cfg.StateDir = t.TempDir()
|
|
||||||
|
|
||||||
db, err := database.New(cfg, logger.New())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("database.New: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Cleanup(func() { _ = db.Close() })
|
|
||||||
|
|
||||||
s := New(db, newTestTable(), streamer.New(logger.New(), metrics.New()),
|
|
||||||
&logger.Logger{Logger: slog.New(slog.NewJSONHandler(requestLog, nil))}, cfg)
|
|
||||||
s.resolver = &fakeResolver{}
|
|
||||||
|
|
||||||
return s
|
|
||||||
}
|
|
||||||
|
|
||||||
// loggedAddresses returns the address each line of the JSON request log
|
|
||||||
// names, without the port a direct request is logged with.
|
|
||||||
func loggedAddresses(requestLog string) []string {
|
|
||||||
var addrs []string
|
|
||||||
|
|
||||||
for line := range strings.Lines(requestLog) {
|
|
||||||
//nolint:tagliatelle // the field name is the key the request log uses
|
|
||||||
var entry struct {
|
|
||||||
RemoteAddr string `json:"remote_addr"`
|
|
||||||
}
|
|
||||||
|
|
||||||
err := json.Unmarshal([]byte(line), &entry)
|
|
||||||
if err != nil || entry.RemoteAddr == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
addr, _, err := net.SplitHostPort(entry.RemoteAddr)
|
|
||||||
if err != nil {
|
|
||||||
addr = entry.RemoteAddr
|
|
||||||
}
|
|
||||||
|
|
||||||
addrs = append(addrs, addr)
|
|
||||||
}
|
|
||||||
|
|
||||||
return addrs
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRealIPMiddlewareTrustsOnlyConfiguredProxies checks which address
|
|
||||||
// RealIPMiddleware gives a request: the TCP peer, unless the peer is a trusted
|
|
||||||
// proxy; then the last X-Forwarded-For entry that is not a trusted proxy,
|
|
||||||
// since the entries left of it are whatever the client sent.
|
|
||||||
func TestRealIPMiddlewareTrustsOnlyConfiguredProxies(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
trusted := []netip.Prefix{netip.MustParsePrefix(trustedProxyRange)}
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
peer string
|
|
||||||
forwardedFor string
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{"untrusted peer", untrustedPeer, forwardedClient, untrustedPeer},
|
|
||||||
{"trusted peer", trustedPeer, forwardedClient, forwardedClient},
|
|
||||||
{
|
|
||||||
"entries the client wrote", trustedPeer,
|
|
||||||
"198.51.100.7, " + forwardedClient, forwardedClient,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"two trusted proxies", trustedPeer,
|
|
||||||
forwardedClient + ", 10.0.0.3", forwardedClient,
|
|
||||||
},
|
|
||||||
{"no header", trustedPeer, "", trustedPeer},
|
|
||||||
{"entry not an address", trustedPeer, "client", trustedPeer},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
var got string
|
|
||||||
|
|
||||||
handler := RealIPMiddleware(trusted)(http.HandlerFunc(
|
|
||||||
func(_ http.ResponseWriter, r *http.Request) { got = r.RemoteAddr }))
|
|
||||||
|
|
||||||
req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil)
|
|
||||||
req.RemoteAddr = tt.peer
|
|
||||||
|
|
||||||
if tt.forwardedFor != "" {
|
|
||||||
req.Header.Set("X-Forwarded-For", tt.forwardedFor)
|
|
||||||
}
|
|
||||||
|
|
||||||
handler.ServeHTTP(httptest.NewRecorder(), req)
|
|
||||||
|
|
||||||
if got != tt.want {
|
|
||||||
t.Errorf("%s: RemoteAddr %q, want %q", tt.name, got, tt.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,74 +0,0 @@
|
|||||||
package server
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
// ErrInvalidIP says that an IP address is malformed. The IP lookup returns
|
|
||||||
// it.
|
|
||||||
ErrInvalidIP = errors.New("invalid IP address")
|
|
||||||
// ErrNoRoute says that the routing table has no route for an address, a
|
|
||||||
// prefix or an AS. The lookups return it.
|
|
||||||
ErrNoRoute = errors.New("no route found")
|
|
||||||
)
|
|
||||||
|
|
||||||
// LiveRoute represents a route in the live routing table. JSON leaves out an
|
|
||||||
// empty AS path, next hop or time: the routing table in memory keeps them only
|
|
||||||
// for the two newest routes of each prefix and origin AS.
|
|
||||||
//
|
|
||||||
//nolint:tagliatelle // field names are the existing JSON API
|
|
||||||
type LiveRoute struct {
|
|
||||||
Prefix string `json:"prefix"`
|
|
||||||
MaskLength int `json:"mask_length"`
|
|
||||||
IPVersion int `json:"ip_version"`
|
|
||||||
OriginASN int `json:"origin_asn"`
|
|
||||||
PeerIP string `json:"peer_ip"`
|
|
||||||
ASPath []int `json:"as_path,omitempty"`
|
|
||||||
NextHop string `json:"next_hop,omitempty"`
|
|
||||||
LastUpdated time.Time `json:"last_updated,omitzero"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// PrefixDistribution represents the distribution of prefixes by mask length
|
|
||||||
//
|
|
||||||
//nolint:tagliatelle // field names are the existing JSON API
|
|
||||||
type PrefixDistribution struct {
|
|
||||||
MaskLength int `json:"mask_length"`
|
|
||||||
Count int `json:"count"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// IPInfo is what the IP lookup knows about the route of an address and that
|
|
||||||
// route's origin AS. A field is nil, null in JSON, when its value is not known.
|
|
||||||
// Every field is nil for an address with no route.
|
|
||||||
//
|
|
||||||
//nolint:tagliatelle // field names are the existing JSON API
|
|
||||||
type IPInfo struct {
|
|
||||||
// Netblock is the most specific prefix that holds the address.
|
|
||||||
Netblock *string `json:"netblock"`
|
|
||||||
MaskLength *int `json:"mask_length"`
|
|
||||||
IPVersion *int `json:"ip_version"`
|
|
||||||
NumPeers *int `json:"num_peers"`
|
|
||||||
// ASN is the origin AS that the most peers carry the netblock with.
|
|
||||||
ASN *int `json:"asn"`
|
|
||||||
// NumOrigins and OriginASNs are how many origin ASes peers carry the
|
|
||||||
// netblock with, and which, in AS number order.
|
|
||||||
NumOrigins *int `json:"num_origins"`
|
|
||||||
OriginASNs []int `json:"origin_asns"`
|
|
||||||
// The details the database holds about the AS in ASN.
|
|
||||||
ASName *string `json:"as_name"`
|
|
||||||
Handle *string `json:"handle"`
|
|
||||||
Description *string `json:"description"`
|
|
||||||
OrgName *string `json:"org_name"`
|
|
||||||
OrgID *string `json:"org_id"`
|
|
||||||
Address *string `json:"address"`
|
|
||||||
CountryCode *string `json:"country_code"`
|
|
||||||
AbuseEmail *string `json:"abuse_email"`
|
|
||||||
RIR *string `json:"rir"`
|
|
||||||
// FirstSeen is when RouteWatch first saw the netblock. LastSeen is the
|
|
||||||
// time of the netblock's newest kept route.
|
|
||||||
FirstSeen *time.Time `json:"first_seen"`
|
|
||||||
LastSeen *time.Time `json:"last_seen"`
|
|
||||||
// Indicates if WHOIS data needs refresh (not serialized)
|
|
||||||
NeedsWHOISRefresh bool `json:"-"`
|
|
||||||
}
|
|
||||||
@@ -1,46 +1,20 @@
|
|||||||
package server
|
package server
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"net/http"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/go-chi/chi/v5"
|
"github.com/go-chi/chi/v5"
|
||||||
"github.com/go-chi/chi/v5/middleware"
|
"github.com/go-chi/chi/v5/middleware"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The error messages the JSON routes answer with when no route matches the
|
|
||||||
// path, and when the route does not take the method.
|
|
||||||
const (
|
|
||||||
notFoundMessage = "Not found"
|
|
||||||
methodNotAllowedMessage = "Method not allowed"
|
|
||||||
|
|
||||||
// jsonRoutesAllow is the Allow header of their 405 answer: every one of
|
|
||||||
// them takes GET and HEAD, and no other method.
|
|
||||||
jsonRoutesAllow = "GET, HEAD"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The paths of the status figures and the health check, the two JSON routes
|
|
||||||
// outside /api/v1 and /ip.
|
|
||||||
const (
|
|
||||||
statusJSONPath = "/status.json"
|
|
||||||
healthCheckPath = "/.well-known/healthcheck.json"
|
|
||||||
)
|
|
||||||
|
|
||||||
// setupRoutes configures the HTTP routes
|
// setupRoutes configures the HTTP routes
|
||||||
func (s *Server) setupRoutes() {
|
func (s *Server) setupRoutes() {
|
||||||
r := chi.NewRouter()
|
r := chi.NewRouter()
|
||||||
|
|
||||||
// Middleware
|
// Middleware
|
||||||
r.Use(DecodedPathMiddleware)
|
|
||||||
// GetHead serves HEAD with the GET handler. It looks the route up by the
|
|
||||||
// path DecodedPathMiddleware sets, so it comes after it.
|
|
||||||
r.Use(middleware.GetHead)
|
|
||||||
r.Use(SecurityHeadersMiddleware)
|
|
||||||
r.Use(middleware.RequestID)
|
r.Use(middleware.RequestID)
|
||||||
r.Use(RealIPMiddleware(s.trustedProxies))
|
r.Use(middleware.RealIP)
|
||||||
r.Use(RequestLoggerMiddleware(s.logger.Logger)) // Structured request logging
|
r.Use(RequestLoggerMiddleware(s.logger.Logger)) // Structured request logging
|
||||||
// Recoverer answers a panic on a page. JSONValidationMiddleware recovers a
|
|
||||||
// panic on a JSON route itself, so that the answer is JSON with @meta.
|
|
||||||
r.Use(middleware.Recoverer)
|
r.Use(middleware.Recoverer)
|
||||||
r.Use(JSONResponseMiddleware)
|
r.Use(JSONResponseMiddleware)
|
||||||
// Timeout must come after JSONResponseMiddleware, which sends its answer
|
// Timeout must come after JSONResponseMiddleware, which sends its answer
|
||||||
@@ -52,6 +26,8 @@ func (s *Server) setupRoutes() {
|
|||||||
// Routes
|
// Routes
|
||||||
r.Get("/", s.handleIndex())
|
r.Get("/", s.handleIndex())
|
||||||
r.Get("/status", s.handleStatusHTML())
|
r.Get("/status", s.handleStatusHTML())
|
||||||
|
r.Get("/status.json", JSONValidationMiddleware(s.handleStatusJSON()).ServeHTTP)
|
||||||
|
r.Get("/.well-known/healthcheck.json", JSONValidationMiddleware(s.handleHealthCheck()).ServeHTTP)
|
||||||
|
|
||||||
// AS and prefix detail pages
|
// AS and prefix detail pages
|
||||||
r.Get("/as/{asn}", s.handleASDetail())
|
r.Get("/as/{asn}", s.handleASDetail())
|
||||||
@@ -59,58 +35,20 @@ func (s *Server) setupRoutes() {
|
|||||||
r.Get("/prefixlength/{length}", s.handlePrefixLength())
|
r.Get("/prefixlength/{length}", s.handlePrefixLength())
|
||||||
r.Get("/prefixlength6/{length}", s.handlePrefixLength6())
|
r.Get("/prefixlength6/{length}", s.handlePrefixLength6())
|
||||||
|
|
||||||
// The pages that list what two of the home page's cards count
|
|
||||||
r.Get("/bogons", s.handleBogons())
|
|
||||||
r.Get("/multiple-origins", s.handleMultipleOrigins())
|
|
||||||
|
|
||||||
// The JSON routes answer a path or method they do not take with the JSON
|
|
||||||
// error body, as they answer everything else.
|
|
||||||
jsonNotFound := func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
s.writeJSONError(w, http.StatusNotFound, notFoundMessage)
|
|
||||||
}
|
|
||||||
jsonMethodNotAllowed := func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
// chi's own 405 answer sets Allow; one set in its place must too.
|
|
||||||
w.Header().Set("Allow", jsonRoutesAllow)
|
|
||||||
s.writeJSONError(w, http.StatusMethodNotAllowed, methodNotAllowedMessage)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The status figures and the health check. chi sets one 405 answer for a
|
|
||||||
// whole router. These two routes are on the pages' router, whose 405 stays
|
|
||||||
// chi's own. So each method they do not take is routed to the JSON 405
|
|
||||||
// here.
|
|
||||||
r.Get(statusJSONPath, JSONValidationMiddleware(s.handleStatusJSON()).ServeHTTP)
|
|
||||||
r.Get(healthCheckPath, JSONValidationMiddleware(s.handleHealthCheck()).ServeHTTP)
|
|
||||||
|
|
||||||
for _, method := range []string{
|
|
||||||
http.MethodPost, http.MethodPut, http.MethodPatch, http.MethodDelete,
|
|
||||||
http.MethodOptions, http.MethodTrace, http.MethodConnect,
|
|
||||||
} {
|
|
||||||
r.MethodFunc(method, statusJSONPath, jsonMethodNotAllowed)
|
|
||||||
r.MethodFunc(method, healthCheckPath, jsonMethodNotAllowed)
|
|
||||||
}
|
|
||||||
|
|
||||||
// IP info JSON endpoints (replaces old /ip redirect)
|
// IP info JSON endpoints (replaces old /ip redirect)
|
||||||
r.Route("/ip", func(r chi.Router) {
|
r.Route("/ip", func(r chi.Router) {
|
||||||
r.Use(JSONValidationMiddleware)
|
r.Use(JSONValidationMiddleware)
|
||||||
r.NotFound(jsonNotFound)
|
|
||||||
r.MethodNotAllowed(jsonMethodNotAllowed)
|
|
||||||
r.Get("/", s.handleIPInfo()) // Client IP
|
r.Get("/", s.handleIPInfo()) // Client IP
|
||||||
// GetHead does not reach this route: mounting the group registers
|
|
||||||
// /ip and /ip/ for every method, so HEAD arrives here as HEAD.
|
|
||||||
r.Head("/", s.handleIPInfo())
|
|
||||||
r.Get("/{addr}", s.handleIPInfo()) // Specified IP
|
r.Get("/{addr}", s.handleIPInfo()) // Specified IP
|
||||||
})
|
})
|
||||||
|
|
||||||
// API routes
|
// API routes
|
||||||
r.Route("/api/v1", func(r chi.Router) {
|
r.Route("/api/v1", func(r chi.Router) {
|
||||||
r.Use(JSONValidationMiddleware)
|
r.Use(JSONValidationMiddleware)
|
||||||
r.NotFound(jsonNotFound)
|
|
||||||
r.MethodNotAllowed(jsonMethodNotAllowed)
|
|
||||||
r.Get("/stats", s.handleStats())
|
r.Get("/stats", s.handleStats())
|
||||||
r.Get("/ip/{ip}", s.handleIPLookup())
|
r.Get("/ip/{ip}", s.handleIPLookup())
|
||||||
r.Get("/as/{asn}", s.handleASDetailJSON())
|
r.Get("/as/{asn}", s.handleASDetailJSON())
|
||||||
r.Get("/prefix/{prefix}/{len}", s.handlePrefixDetailJSON())
|
r.Get("/prefix/{prefix}/{len}", s.handlePrefixDetailJSON())
|
||||||
r.Get("/search/as", s.handleASSearch())
|
|
||||||
})
|
})
|
||||||
|
|
||||||
s.router = r
|
s.router = r
|
||||||
|
|||||||
@@ -1,111 +0,0 @@
|
|||||||
package server
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestRoutesAnswerHEAD sends HEAD for a page and JSON routes through the
|
|
||||||
// server's router and checks that each answers as GET does. /ip and /ip/, the
|
|
||||||
// lookup of the caller's own address, are in the list because chi's GetHead
|
|
||||||
// does not reach them. Before the fix every route answered HEAD with 405
|
|
||||||
// (https://git.eeqj.de/sneak/routewatch/issues/111).
|
|
||||||
func TestRoutesAnswerHEAD(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := newServerWithTestRoutes(t)
|
|
||||||
|
|
||||||
for _, path := range []string{"/", statusJSONPath, statsPath, "/ip", "/ip/"} {
|
|
||||||
rec := httptest.NewRecorder()
|
|
||||||
s.router.ServeHTTP(rec, httptest.NewRequestWithContext(
|
|
||||||
t.Context(), http.MethodHead, path, nil))
|
|
||||||
|
|
||||||
if rec.Code != http.StatusOK {
|
|
||||||
t.Errorf("HEAD %s: status %d, want %d; body %s",
|
|
||||||
path, rec.Code, http.StatusOK, rec.Body)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestJSONRoutesAnswerUnmatchedRequestsWithJSONErrors sends a method the JSON
|
|
||||||
// routes do not take, and a path none of them matches, to each group of JSON
|
|
||||||
// routes through the server's router. It also sends a method they do not take
|
|
||||||
// to /status.json and the health check. Each must answer with the JSON error
|
|
||||||
// body, and a 405 must name the methods the routes take in its Allow header.
|
|
||||||
// Before the fix a 405 said "Internal server error" and a 404 was chi's plain
|
|
||||||
// text answer (https://git.eeqj.de/sneak/routewatch/issues/111).
|
|
||||||
// /status.json and the health check gave chi's empty 405
|
|
||||||
// (https://git.eeqj.de/sneak/routewatch/issues/201).
|
|
||||||
func TestJSONRoutesAnswerUnmatchedRequestsWithJSONErrors(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := newServerWithTestRoutes(t)
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
method string
|
|
||||||
path string
|
|
||||||
wantCode int
|
|
||||||
wantMsg string
|
|
||||||
}{
|
|
||||||
{http.MethodPost, statsPath, http.StatusMethodNotAllowed, methodNotAllowedMessage},
|
|
||||||
{http.MethodPost, "/ip/", http.StatusMethodNotAllowed, methodNotAllowedMessage},
|
|
||||||
{
|
|
||||||
http.MethodDelete, statusJSONPath,
|
|
||||||
http.StatusMethodNotAllowed, methodNotAllowedMessage,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
http.MethodOptions, healthCheckPath,
|
|
||||||
http.StatusMethodNotAllowed, methodNotAllowedMessage,
|
|
||||||
},
|
|
||||||
{http.MethodGet, "/api/v1/nothing", http.StatusNotFound, notFoundMessage},
|
|
||||||
{http.MethodGet, "/ip/192.0.2.1/nothing", http.StatusNotFound, notFoundMessage},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
name := tt.method + " " + tt.path
|
|
||||||
|
|
||||||
rec := httptest.NewRecorder()
|
|
||||||
s.router.ServeHTTP(rec, httptest.NewRequestWithContext(
|
|
||||||
t.Context(), tt.method, tt.path, nil))
|
|
||||||
|
|
||||||
if rec.Code != tt.wantCode {
|
|
||||||
t.Errorf("%s: status %d, want %d; body %s",
|
|
||||||
name, rec.Code, tt.wantCode, rec.Body)
|
|
||||||
}
|
|
||||||
|
|
||||||
checkJSONErrorCode(t, name, rec, tt.wantCode)
|
|
||||||
checkJSONErrorMessage(t, name, rec, tt.wantMsg)
|
|
||||||
|
|
||||||
allow := rec.Header().Get("Allow")
|
|
||||||
if tt.wantCode == http.StatusMethodNotAllowed && allow != jsonRoutesAllow {
|
|
||||||
t.Errorf("%s: Allow %q, want %q", name, allow, jsonRoutesAllow)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkJSONErrorMessage checks that rec holds the JSON error body with the
|
|
||||||
// given message.
|
|
||||||
func checkJSONErrorMessage(
|
|
||||||
t *testing.T, name string, rec *httptest.ResponseRecorder, want string,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var body struct {
|
|
||||||
Error struct {
|
|
||||||
Msg string `json:"msg"`
|
|
||||||
} `json:"error"`
|
|
||||||
}
|
|
||||||
|
|
||||||
err := json.Unmarshal(rec.Body.Bytes(), &body)
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("%s: decoding the answer %q: %v", name, rec.Body, err)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if body.Error.Msg != want {
|
|
||||||
t.Errorf("%s: error message %q, want %q", name, body.Error.Msg, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+11
-119
@@ -1,32 +1,15 @@
|
|||||||
// Package server serves RouteWatch's web pages and JSON API:
|
// Package server provides HTTP endpoints for status monitoring and statistics
|
||||||
// - the home page, the status figures and the health check
|
|
||||||
// - the pages that list the bogon announcements and the prefixes announced
|
|
||||||
// by more than one origin AS
|
|
||||||
// - the AS, prefix and prefix length pages, and the JSON answers behind them
|
|
||||||
// - the IP lookups
|
|
||||||
// - the AS name search
|
|
||||||
//
|
|
||||||
// Every route it shows comes from the routing table in memory. AS names,
|
|
||||||
// WHOIS details and the times a prefix and an AS were first seen come from
|
|
||||||
// the database.
|
|
||||||
package server
|
package server
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/netip"
|
|
||||||
"strconv"
|
"strconv"
|
||||||
"sync/atomic"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/blocklist"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/config"
|
"git.eeqj.de/sneak/routewatch/internal/config"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/database"
|
"git.eeqj.de/sneak/routewatch/internal/database"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
"git.eeqj.de/sneak/routewatch/internal/logger"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/routetable"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/streamer"
|
"git.eeqj.de/sneak/routewatch/internal/streamer"
|
||||||
"github.com/go-chi/chi/v5"
|
"github.com/go-chi/chi/v5"
|
||||||
)
|
)
|
||||||
@@ -45,68 +28,24 @@ type ASNFetcher interface {
|
|||||||
GetStats() ASNFetcherStats
|
GetStats() ASNFetcherStats
|
||||||
}
|
}
|
||||||
|
|
||||||
// Blocklists tells which public blocklists list an address, from copies it
|
|
||||||
// keeps in memory, as *blocklist.Blocklists does.
|
|
||||||
type Blocklists interface {
|
|
||||||
Lookup(addr netip.Addr) []blocklist.Listing
|
|
||||||
ShortestInterval() time.Duration
|
|
||||||
}
|
|
||||||
|
|
||||||
// dnsResolver looks up the addresses of host names and the names of addresses
|
|
||||||
// in DNS, as *net.Resolver does.
|
|
||||||
type dnsResolver interface {
|
|
||||||
LookupHost(ctx context.Context, host string) ([]string, error)
|
|
||||||
LookupAddr(ctx context.Context, addr string) ([]string, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Server provides HTTP endpoints for status monitoring
|
// Server provides HTTP endpoints for status monitoring
|
||||||
type Server struct {
|
type Server struct {
|
||||||
router *chi.Mux
|
router *chi.Mux
|
||||||
db database.Store
|
db database.Store
|
||||||
table *routetable.Table
|
|
||||||
streamer *streamer.Streamer
|
streamer *streamer.Streamer
|
||||||
logger *logger.Logger
|
logger *logger.Logger
|
||||||
port int
|
port int
|
||||||
srv *http.Server
|
srv *http.Server
|
||||||
asnFetcher ASNFetcher
|
asnFetcher ASNFetcher
|
||||||
blocklists Blocklists
|
|
||||||
|
|
||||||
// trustedProxies are the reverse proxies whose X-Forwarded-For header
|
|
||||||
// RealIPMiddleware believes
|
|
||||||
trustedProxies []netip.Prefix
|
|
||||||
|
|
||||||
// resolver makes the IP lookup's DNS lookups: net.DefaultResolver, or a
|
|
||||||
// fake in tests
|
|
||||||
resolver dnsResolver
|
|
||||||
|
|
||||||
// cards is the copy of the home page's cards that the pages serve, nil
|
|
||||||
// until the server first works them out
|
|
||||||
cards atomic.Pointer[homeCards]
|
|
||||||
|
|
||||||
// stopCards ends the goroutine that works out the cards, and
|
|
||||||
// cardsStopped is closed once it has returned. Both are nil until Start
|
|
||||||
// starts the goroutine.
|
|
||||||
stopCards context.CancelFunc
|
|
||||||
cardsStopped chan struct{}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// New creates a new HTTP server that reads routes from table, the routing
|
// New creates a new HTTP server
|
||||||
// table that the prefix handler keeps current.
|
func New(db database.Store, streamer *streamer.Streamer, logger *logger.Logger, cfg *config.Config) *Server {
|
||||||
func New(
|
|
||||||
db database.Store,
|
|
||||||
table *routetable.Table,
|
|
||||||
streamer *streamer.Streamer,
|
|
||||||
logger *logger.Logger,
|
|
||||||
cfg *config.Config,
|
|
||||||
) *Server {
|
|
||||||
s := &Server{
|
s := &Server{
|
||||||
db: db,
|
db: db,
|
||||||
table: table,
|
|
||||||
streamer: streamer,
|
streamer: streamer,
|
||||||
logger: logger,
|
logger: logger,
|
||||||
port: cfg.Port,
|
port: cfg.Port,
|
||||||
trustedProxies: cfg.TrustedProxies,
|
|
||||||
resolver: net.DefaultResolver,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
s.setupRoutes()
|
s.setupRoutes()
|
||||||
@@ -114,14 +53,9 @@ func New(
|
|||||||
return s
|
return s
|
||||||
}
|
}
|
||||||
|
|
||||||
// Start listens on the configured port and serves HTTP in the background.
|
// Start starts the HTTP server
|
||||||
// It returns an error if it cannot listen, for example because the port is
|
func (s *Server) Start() error {
|
||||||
// already in use. ctx bounds opening the port, not serving. Once listening,
|
|
||||||
// the server works out the home page's cards every minute until Stop: the end
|
|
||||||
// of ctx does not stop it.
|
|
||||||
func (s *Server) Start(ctx context.Context) error {
|
|
||||||
const (
|
const (
|
||||||
network = "tcp"
|
|
||||||
readHeaderTimeout = 40 * time.Second
|
readHeaderTimeout = 40 * time.Second
|
||||||
readTimeout = 60 * time.Second
|
readTimeout = 60 * time.Second
|
||||||
writeTimeout = 60 * time.Second
|
writeTimeout = 60 * time.Second
|
||||||
@@ -139,38 +73,18 @@ func (s *Server) Start(ctx context.Context) error {
|
|||||||
|
|
||||||
s.logger.Info("Starting HTTP server", "port", s.port, "addr", s.srv.Addr)
|
s.logger.Info("Starting HTTP server", "port", s.port, "addr", s.srv.Addr)
|
||||||
|
|
||||||
var listenConfig net.ListenConfig
|
// Start in goroutine but log when actually listening
|
||||||
|
|
||||||
listener, err := listenConfig.Listen(ctx, network, s.srv.Addr)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("starting HTTP server: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
s.logger.Info("HTTP server listening", "addr", listener.Addr().String())
|
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
err := s.srv.Serve(listener)
|
s.logger.Info("HTTP server listening", "addr", s.srv.Addr)
|
||||||
if err != nil && !errors.Is(err, http.ErrServerClosed) {
|
if err := s.srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||||
s.logger.Error("HTTP server error", "error", err)
|
s.logger.Error("HTTP server error", "error", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
ctx, s.stopCards = context.WithCancel(context.WithoutCancel(ctx))
|
|
||||||
s.cardsStopped = make(chan struct{})
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
defer close(s.cardsStopped)
|
|
||||||
|
|
||||||
s.keepHomeCardsFresh(ctx, homeCardsRefreshInterval)
|
|
||||||
}()
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stop stops the goroutine that works out the home page's cards, and waits
|
// Stop gracefully stops the HTTP server
|
||||||
// until it has returned or ctx ends. It then stops the HTTP server. It waits,
|
|
||||||
// until ctx ends, for the requests in progress to finish, and then closes the
|
|
||||||
// connections of those still running and returns ctx's error.
|
|
||||||
func (s *Server) Stop(ctx context.Context) error {
|
func (s *Server) Stop(ctx context.Context) error {
|
||||||
if s.srv == nil {
|
if s.srv == nil {
|
||||||
return nil
|
return nil
|
||||||
@@ -178,32 +92,10 @@ func (s *Server) Stop(ctx context.Context) error {
|
|||||||
|
|
||||||
s.logger.Info("Stopping HTTP server")
|
s.logger.Info("Stopping HTTP server")
|
||||||
|
|
||||||
// Start returns before it starts the goroutine when it cannot listen.
|
return s.srv.Shutdown(ctx)
|
||||||
if s.stopCards != nil {
|
|
||||||
s.stopCards()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-s.cardsStopped:
|
|
||||||
case <-ctx.Done():
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
err := s.srv.Shutdown(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return errors.Join(err, s.srv.Close())
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetASNFetcher sets the ASN WHOIS fetcher for on-demand lookups. Call it
|
// SetASNFetcher sets the ASN WHOIS fetcher for on-demand lookups.
|
||||||
// before Start: the handlers read the fetcher without a lock.
|
|
||||||
func (s *Server) SetASNFetcher(fetcher ASNFetcher) {
|
func (s *Server) SetASNFetcher(fetcher ASNFetcher) {
|
||||||
s.asnFetcher = fetcher
|
s.asnFetcher = fetcher
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetBlocklists sets the blocklists the IP lookups report. Call it before
|
|
||||||
// Start: the handlers read the blocklists without a lock.
|
|
||||||
func (s *Server) SetBlocklists(blocklists Blocklists) {
|
|
||||||
s.blocklists = blocklists
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,92 +0,0 @@
|
|||||||
package server
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestStopClosesTheConnectionOfARequestStillRunning serves a request whose
|
|
||||||
// handler does not return until the test ends, and stops the server with a
|
|
||||||
// deadline that passes while the request runs. The client must then see its
|
|
||||||
// connection closed. Before the fix Stop returned at the deadline and left the
|
|
||||||
// connection open until the handler returned.
|
|
||||||
func TestStopClosesTheConnectionOfARequestStillRunning(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const (
|
|
||||||
stopDeadline = 100 * time.Millisecond
|
|
||||||
// waitTimeout is how long the test waits for the request to reach
|
|
||||||
// the handler, and then for the client to see its connection closed.
|
|
||||||
waitTimeout = 2 * time.Second
|
|
||||||
// readHeaderTimeout bounds the test server's read of the request
|
|
||||||
// header, as every http.Server must.
|
|
||||||
readHeaderTimeout = time.Second
|
|
||||||
)
|
|
||||||
|
|
||||||
release := make(chan struct{})
|
|
||||||
defer close(release)
|
|
||||||
|
|
||||||
started := make(chan struct{})
|
|
||||||
|
|
||||||
var listenConfig net.ListenConfig
|
|
||||||
|
|
||||||
listener, err := listenConfig.Listen(t.Context(), "tcp", "127.0.0.1:0")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to open the test listener: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
s := &Server{logger: logger.New(), srv: &http.Server{
|
|
||||||
Handler: http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
|
|
||||||
close(started)
|
|
||||||
<-release
|
|
||||||
}),
|
|
||||||
ReadHeaderTimeout: readHeaderTimeout,
|
|
||||||
}}
|
|
||||||
|
|
||||||
go func() { _ = s.srv.Serve(listener) }()
|
|
||||||
|
|
||||||
answered := make(chan error, 1)
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet,
|
|
||||||
"http://"+listener.Addr().String()+"/", nil)
|
|
||||||
if err != nil {
|
|
||||||
answered <- err
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, err := http.DefaultClient.Do(req)
|
|
||||||
if err == nil {
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
answered <- err
|
|
||||||
}()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-started:
|
|
||||||
case <-time.After(waitTimeout):
|
|
||||||
t.Fatal("the request did not reach the handler")
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(t.Context(), stopDeadline)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
_ = s.Stop(ctx)
|
|
||||||
|
|
||||||
select {
|
|
||||||
case err := <-answered:
|
|
||||||
if err == nil {
|
|
||||||
t.Error("the request still running at the deadline was answered")
|
|
||||||
}
|
|
||||||
case <-time.After(waitTimeout):
|
|
||||||
t.Fatal("the connection of the request still running at the " +
|
|
||||||
"deadline was open after Stop returned")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,47 +0,0 @@
|
|||||||
package server_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"net"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/config"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/metrics"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/server"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/streamer"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestStartFailsWhenPortIsHeld holds a port with a listener of its own and
|
|
||||||
// checks that Start on that port returns an error. Before the fix Start
|
|
||||||
// listened in a goroutine, only logged the failure and returned nil, so the
|
|
||||||
// daemon kept running without its web server.
|
|
||||||
func TestStartFailsWhenPortIsHeld(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var listenConfig net.ListenConfig
|
|
||||||
|
|
||||||
held, err := listenConfig.Listen(t.Context(), "tcp", ":0")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to open the test listener: %v", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = held.Close() }()
|
|
||||||
|
|
||||||
addr, ok := held.Addr().(*net.TCPAddr)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("test listener address %v is not a TCP address", held.Addr())
|
|
||||||
}
|
|
||||||
|
|
||||||
port := addr.Port
|
|
||||||
log := logger.New()
|
|
||||||
s := server.New(nil, nil, streamer.New(log, metrics.New()), log,
|
|
||||||
&config.Config{Port: port})
|
|
||||||
|
|
||||||
defer func() { _ = s.Stop(context.Background()) }()
|
|
||||||
|
|
||||||
err = s.Start(t.Context())
|
|
||||||
if err == nil {
|
|
||||||
t.Fatalf("Start returned nil while port %d was held by another listener",
|
|
||||||
port)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+121
-332
@@ -1,6 +1,5 @@
|
|||||||
// Package streamer implements an HTTP client that connects to the RIPE RIS Live
|
// Package streamer implements an HTTP client that connects to the RIPE RIS Live streaming API,
|
||||||
// streaming API, parses BGP UPDATE messages from the JSON stream, and
|
// parses BGP UPDATE messages from the JSON stream, and dispatches them to registered handlers.
|
||||||
// dispatches them to registered handlers.
|
|
||||||
package streamer
|
package streamer
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -8,7 +7,6 @@ import (
|
|||||||
"compress/gzip"
|
"compress/gzip"
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"math"
|
"math"
|
||||||
@@ -26,20 +24,20 @@ import (
|
|||||||
// countingReader wraps an io.Reader and counts bytes read
|
// countingReader wraps an io.Reader and counts bytes read
|
||||||
type countingReader struct {
|
type countingReader struct {
|
||||||
reader io.Reader
|
reader io.Reader
|
||||||
count atomic.Int64
|
count int64
|
||||||
}
|
}
|
||||||
|
|
||||||
// Read implements io.Reader and counts bytes
|
// Read implements io.Reader and counts bytes
|
||||||
func (c *countingReader) Read(p []byte) (int, error) {
|
func (c *countingReader) Read(p []byte) (int, error) {
|
||||||
n, err := c.reader.Read(p)
|
n, err := c.reader.Read(p)
|
||||||
c.count.Add(int64(n))
|
atomic.AddInt64(&c.count, int64(n))
|
||||||
|
|
||||||
return n, err
|
return n, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Count returns the total bytes read
|
// Count returns the total bytes read
|
||||||
func (c *countingReader) Count() int64 {
|
func (c *countingReader) Count() int64 {
|
||||||
return c.count.Load()
|
return atomic.LoadInt64(&c.count)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Configuration constants for the RIS Live streamer.
|
// Configuration constants for the RIS Live streamer.
|
||||||
@@ -50,13 +48,9 @@ const (
|
|||||||
metricsUpdateRate = time.Second
|
metricsUpdateRate = time.Second
|
||||||
minBackoffDelay = 5 * time.Second
|
minBackoffDelay = 5 * time.Second
|
||||||
maxBackoffDelay = 320 * time.Second
|
maxBackoffDelay = 320 * time.Second
|
||||||
// Reconnect when no line arrives for this long
|
|
||||||
streamIdleTimeout = 60 * time.Second
|
|
||||||
metricsLogInterval = 10 * time.Second
|
metricsLogInterval = 10 * time.Second
|
||||||
bytesPerKB = 1024
|
bytesPerKB = 1024
|
||||||
bytesPerMB = 1024 * 1024
|
bytesPerMB = 1024 * 1024
|
||||||
// Well above an extended BGP message (RFC 8654) in hex plus its prefixes
|
|
||||||
maxLineSize = bytesPerMB
|
|
||||||
maxConcurrentHandlers = 800 // Maximum number of concurrent message handlers
|
maxConcurrentHandlers = 800 // Maximum number of concurrent message handlers
|
||||||
|
|
||||||
// Backpressure constants
|
// Backpressure constants
|
||||||
@@ -64,26 +58,11 @@ const (
|
|||||||
backpressureSlope = 2.0 // Slope for linear drop probability increase
|
backpressureSlope = 2.0 // Slope for linear drop probability increase
|
||||||
)
|
)
|
||||||
|
|
||||||
// messageTypeNotification is the type of a BGP NOTIFICATION message, which ends
|
|
||||||
// the session with the peer.
|
|
||||||
const messageTypeNotification = "NOTIFICATION"
|
|
||||||
|
|
||||||
var (
|
|
||||||
errAlreadyRunning = errors.New("streamer already running")
|
|
||||||
errUnexpectedStatus = errors.New("unexpected status code")
|
|
||||||
// errStreamEnded is returned by stream when RIS Live ends the response.
|
|
||||||
errStreamEnded = errors.New("RIS Live ended the stream")
|
|
||||||
// errStreamIdle is returned by stream when no line arrived for the idle
|
|
||||||
// timeout.
|
|
||||||
errStreamIdle = errors.New("no line received")
|
|
||||||
)
|
|
||||||
|
|
||||||
// MessageHandler defines the interface for processing RIS messages.
|
// MessageHandler defines the interface for processing RIS messages.
|
||||||
// Implementations must specify which message types they want to receive,
|
// Implementations must specify which message types they want to receive,
|
||||||
// how to process messages, and their desired queue capacity.
|
// how to process messages, and their desired queue capacity.
|
||||||
type MessageHandler interface {
|
type MessageHandler interface {
|
||||||
// WantsMessage returns true if this handler wants to process messages of
|
// WantsMessage returns true if this handler wants to process messages of the given type.
|
||||||
// the given type.
|
|
||||||
WantsMessage(messageType string) bool
|
WantsMessage(messageType string) bool
|
||||||
|
|
||||||
// HandleMessage processes a RIS message. This method is called from a dedicated
|
// HandleMessage processes a RIS message. This method is called from a dedicated
|
||||||
@@ -94,8 +73,7 @@ type MessageHandler interface {
|
|||||||
// QueueCapacity returns the desired queue capacity for this handler.
|
// QueueCapacity returns the desired queue capacity for this handler.
|
||||||
// Handlers that process quickly can have larger queues to buffer bursts.
|
// Handlers that process quickly can have larger queues to buffer bursts.
|
||||||
// When the queue fills up, messages will be dropped according to the
|
// When the queue fills up, messages will be dropped according to the
|
||||||
// backpressure algorithm, except STATE messages, which are held until
|
// backpressure algorithm.
|
||||||
// there is room.
|
|
||||||
QueueCapacity() int
|
QueueCapacity() int
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -106,12 +84,12 @@ type RawMessageHandler func(line string)
|
|||||||
// handlerMetrics tracks performance metrics for a handler
|
// handlerMetrics tracks performance metrics for a handler
|
||||||
type handlerMetrics struct {
|
type handlerMetrics struct {
|
||||||
processedCount uint64 // Total messages processed
|
processedCount uint64 // Total messages processed
|
||||||
droppedCount atomic.Uint64 // Total messages dropped; not under mu
|
droppedCount uint64 // Total messages dropped
|
||||||
totalTime time.Duration // Total processing time (for average calculation)
|
totalTime time.Duration // Total processing time (for average calculation)
|
||||||
minTime time.Duration // Minimum processing time
|
minTime time.Duration // Minimum processing time
|
||||||
maxTime time.Duration // Maximum processing time
|
maxTime time.Duration // Maximum processing time
|
||||||
queueHighWaterMark int // Maximum queue length seen
|
queueHighWaterMark int // Maximum queue length seen
|
||||||
mu sync.Mutex // Protects the other metrics
|
mu sync.Mutex // Protects the metrics
|
||||||
}
|
}
|
||||||
|
|
||||||
// handlerInfo wraps a handler with its queue and metrics
|
// handlerInfo wraps a handler with its queue and metrics
|
||||||
@@ -119,9 +97,6 @@ type handlerInfo struct {
|
|||||||
handler MessageHandler
|
handler MessageHandler
|
||||||
queue chan *ristypes.RISMessage
|
queue chan *ristypes.RISMessage
|
||||||
metrics handlerMetrics
|
metrics handlerMetrics
|
||||||
// heldState holds, in feed order, the STATE messages that found queue
|
|
||||||
// full (see queueMessage). Only the read loop uses it, and then Stop.
|
|
||||||
heldState []*ristypes.RISMessage
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Streamer manages a connection to the RIPE RIS Live streaming API for receiving
|
// Streamer manages a connection to the RIPE RIS Live streaming API for receiving
|
||||||
@@ -132,18 +107,13 @@ type Streamer struct {
|
|||||||
logger *logger.Logger
|
logger *logger.Logger
|
||||||
client *http.Client
|
client *http.Client
|
||||||
url string
|
url string
|
||||||
// Drop the connection when no line arrives for this long
|
|
||||||
idleTimeout time.Duration
|
|
||||||
// First wait before reconnecting; doubles up to maxBackoffDelay
|
|
||||||
minBackoff time.Duration
|
|
||||||
handlers []*handlerInfo
|
handlers []*handlerInfo
|
||||||
rawHandler RawMessageHandler
|
rawHandler RawMessageHandler
|
||||||
mu sync.RWMutex
|
mu sync.RWMutex
|
||||||
cancel context.CancelFunc
|
cancel context.CancelFunc
|
||||||
running bool
|
running bool
|
||||||
workers sync.WaitGroup // the handler workers; Stop waits for them
|
|
||||||
metrics *metrics.Tracker
|
metrics *metrics.Tracker
|
||||||
totalDropped atomic.Uint64 // Total dropped messages across all handlers
|
totalDropped uint64 // Total dropped messages across all handlers
|
||||||
random *rand.Rand // Random number generator for backpressure drops
|
random *rand.Rand // Random number generator for backpressure drops
|
||||||
bgpPeers map[string]bool // Track active BGP peers by peer IP
|
bgpPeers map[string]bool // Track active BGP peers by peer IP
|
||||||
bgpPeersMu sync.RWMutex // Protects bgpPeers map
|
bgpPeersMu sync.RWMutex // Protects bgpPeers map
|
||||||
@@ -151,14 +121,11 @@ type Streamer struct {
|
|||||||
|
|
||||||
// New creates a new Streamer instance configured to connect to the RIS Live API.
|
// New creates a new Streamer instance configured to connect to the RIS Live API.
|
||||||
// The logger is used for structured logging of connection events and errors.
|
// The logger is used for structured logging of connection events and errors.
|
||||||
// The metrics tracker is used to record message counts, bytes received, and
|
// The metrics tracker is used to record message counts, bytes received, and connection status.
|
||||||
// connection status.
|
|
||||||
func New(logger *logger.Logger, metrics *metrics.Tracker) *Streamer {
|
func New(logger *logger.Logger, metrics *metrics.Tracker) *Streamer {
|
||||||
return &Streamer{
|
return &Streamer{
|
||||||
logger: logger,
|
logger: logger,
|
||||||
url: risLiveURL,
|
url: risLiveURL,
|
||||||
idleTimeout: streamIdleTimeout,
|
|
||||||
minBackoff: minBackoffDelay,
|
|
||||||
client: &http.Client{
|
client: &http.Client{
|
||||||
Timeout: 0, // No timeout for streaming
|
Timeout: 0, // No timeout for streaming
|
||||||
Transport: &http.Transport{
|
Transport: &http.Transport{
|
||||||
@@ -186,8 +153,7 @@ func (s *Streamer) RegisterHandler(handler MessageHandler) {
|
|||||||
handler: handler,
|
handler: handler,
|
||||||
queue: make(chan *ristypes.RISMessage, handler.QueueCapacity()),
|
queue: make(chan *ristypes.RISMessage, handler.QueueCapacity()),
|
||||||
metrics: handlerMetrics{
|
metrics: handlerMetrics{
|
||||||
// Initialize to max so first value sets the floor
|
minTime: time.Duration(math.MaxInt64), // Initialize to max so first value sets the floor
|
||||||
minTime: time.Duration(math.MaxInt64),
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -195,49 +161,37 @@ func (s *Streamer) RegisterHandler(handler MessageHandler) {
|
|||||||
|
|
||||||
// If we're already running, start a worker for this handler
|
// If we're already running, start a worker for this handler
|
||||||
if s.running {
|
if s.running {
|
||||||
s.workers.Add(1)
|
|
||||||
|
|
||||||
go s.runHandlerWorker(info)
|
go s.runHandlerWorker(info)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetURL makes the streamer read from url instead of RIS Live. Call it before
|
|
||||||
// Start. Tests use it to read from a local server.
|
|
||||||
func (s *Streamer) SetURL(url string) {
|
|
||||||
s.url = url
|
|
||||||
}
|
|
||||||
|
|
||||||
// RegisterRawHandler sets a callback to receive raw JSON lines from the stream
|
// RegisterRawHandler sets a callback to receive raw JSON lines from the stream
|
||||||
// before they are parsed. Only one raw handler can be registered at a time;
|
// before they are parsed. Only one raw handler can be registered at a time;
|
||||||
// subsequent calls will replace the previous handler.
|
// subsequent calls will replace the previous handler.
|
||||||
func (s *Streamer) RegisterRawHandler(handler RawMessageHandler) {
|
func (s *Streamer) RegisterRawHandler(handler RawMessageHandler) {
|
||||||
s.mu.Lock()
|
s.mu.Lock()
|
||||||
defer s.mu.Unlock()
|
defer s.mu.Unlock()
|
||||||
|
|
||||||
s.rawHandler = handler
|
s.rawHandler = handler
|
||||||
}
|
}
|
||||||
|
|
||||||
// Start begins streaming BGP updates from the RIS Live API in a background goroutine.
|
// Start begins streaming BGP updates from the RIS Live API in a background goroutine.
|
||||||
// It starts worker goroutines for each registered handler and manages automatic
|
// It starts worker goroutines for each registered handler and manages automatic
|
||||||
// reconnection with exponential backoff on connection failures.
|
// reconnection with exponential backoff on connection failures.
|
||||||
// Returns an error if the streamer is already running. It streams until Stop:
|
// Returns an error if the streamer is already running.
|
||||||
// the end of ctx does not stop it.
|
func (s *Streamer) Start() error {
|
||||||
func (s *Streamer) Start(ctx context.Context) error {
|
|
||||||
s.mu.Lock()
|
s.mu.Lock()
|
||||||
defer s.mu.Unlock()
|
defer s.mu.Unlock()
|
||||||
|
|
||||||
if s.running {
|
if s.running {
|
||||||
return errAlreadyRunning
|
return fmt.Errorf("streamer already running")
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.WithoutCancel(ctx))
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
s.cancel = cancel
|
s.cancel = cancel
|
||||||
s.running = true
|
s.running = true
|
||||||
|
|
||||||
// Start workers for each handler
|
// Start workers for each handler
|
||||||
for _, info := range s.handlers {
|
for _, info := range s.handlers {
|
||||||
s.workers.Add(1)
|
|
||||||
|
|
||||||
go s.runHandlerWorker(info)
|
go s.runHandlerWorker(info)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -251,12 +205,10 @@ func (s *Streamer) Start(ctx context.Context) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stop ends the stream and stops the handler workers. It returns once each
|
// Stop halts the streaming connection and shuts down all handler workers.
|
||||||
// worker has handed its handler every message left on its queue and then the
|
// It cancels the streaming context, closes all handler queues, and updates
|
||||||
// STATE messages held for it (see queueMessage), or once ctx ends, and then
|
// the connection status in metrics. This method is safe to call multiple times.
|
||||||
// logs how many messages were not handed over. This method is safe to call
|
func (s *Streamer) Stop() {
|
||||||
// multiple times.
|
|
||||||
func (s *Streamer) Stop(ctx context.Context) {
|
|
||||||
s.mu.Lock()
|
s.mu.Lock()
|
||||||
if s.cancel == nil {
|
if s.cancel == nil {
|
||||||
// Not started, or already stopped: closing the queues again would panic.
|
// Not started, or already stopped: closing the queues again would panic.
|
||||||
@@ -264,61 +216,44 @@ func (s *Streamer) Stop(ctx context.Context) {
|
|||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Once the stream's context is cancelled the read loop queues and holds
|
|
||||||
// nothing more (see dispatchMessage).
|
|
||||||
s.cancel()
|
s.cancel()
|
||||||
s.cancel = nil
|
s.cancel = nil
|
||||||
|
// Close all handler queues to signal workers to stop
|
||||||
notHandedOver := 0
|
|
||||||
|
|
||||||
for _, info := range s.handlers {
|
for _, info := range s.handlers {
|
||||||
// The held messages came after everything on the queue. Each waits
|
|
||||||
// for the worker, which does not take s.mu, to make room, until ctx
|
|
||||||
// ends.
|
|
||||||
for len(info.heldState) > 0 && ctx.Err() == nil {
|
|
||||||
select {
|
|
||||||
case info.queue <- info.heldState[0]:
|
|
||||||
info.heldState = info.heldState[1:]
|
|
||||||
case <-ctx.Done():
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
notHandedOver += len(info.heldState)
|
|
||||||
info.heldState = nil
|
|
||||||
|
|
||||||
// The worker returns once it has emptied the closed queue.
|
|
||||||
close(info.queue)
|
close(info.queue)
|
||||||
}
|
}
|
||||||
|
|
||||||
handlers := s.handlers
|
|
||||||
s.running = false
|
s.running = false
|
||||||
s.mu.Unlock()
|
s.mu.Unlock()
|
||||||
s.metrics.SetConnected(false)
|
s.metrics.SetConnected(false)
|
||||||
|
}
|
||||||
|
|
||||||
workersDone := make(chan struct{})
|
// runHandlerWorker processes messages for a specific handler
|
||||||
|
func (s *Streamer) runHandlerWorker(info *handlerInfo) {
|
||||||
|
for msg := range info.queue {
|
||||||
|
start := time.Now()
|
||||||
|
info.handler.HandleMessage(msg)
|
||||||
|
elapsed := time.Since(start)
|
||||||
|
|
||||||
go func() {
|
// Update metrics
|
||||||
s.workers.Wait()
|
info.metrics.mu.Lock()
|
||||||
close(workersDone)
|
info.metrics.processedCount++
|
||||||
}()
|
info.metrics.totalTime += elapsed
|
||||||
|
|
||||||
select {
|
// Update min time
|
||||||
case <-workersDone:
|
if elapsed < info.metrics.minTime {
|
||||||
case <-ctx.Done():
|
info.metrics.minTime = elapsed
|
||||||
for _, info := range handlers {
|
|
||||||
notHandedOver += len(info.queue)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if notHandedOver > 0 {
|
// Update max time
|
||||||
s.logger.Error("Stop deadline passed before the handlers took every "+
|
if elapsed > info.metrics.maxTime {
|
||||||
"message", "messages", notHandedOver)
|
info.metrics.maxTime = elapsed
|
||||||
|
}
|
||||||
|
info.metrics.mu.Unlock()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// IsRunning reports whether the streamer is currently connected and processing
|
// IsRunning reports whether the streamer is currently connected and processing messages.
|
||||||
// messages. This is safe to call concurrently from multiple goroutines.
|
// This is safe to call concurrently from multiple goroutines.
|
||||||
func (s *Streamer) IsRunning() bool {
|
func (s *Streamer) IsRunning() bool {
|
||||||
s.mu.RLock()
|
s.mu.RLock()
|
||||||
defer s.mu.RUnlock()
|
defer s.mu.RUnlock()
|
||||||
@@ -371,7 +306,7 @@ func (s *Streamer) GetHandlerStats() []HandlerStats {
|
|||||||
QueueCapacity: cap(info.queue),
|
QueueCapacity: cap(info.queue),
|
||||||
QueueHighWaterMark: info.metrics.queueHighWaterMark,
|
QueueHighWaterMark: info.metrics.queueHighWaterMark,
|
||||||
ProcessedCount: info.metrics.processedCount,
|
ProcessedCount: info.metrics.processedCount,
|
||||||
DroppedCount: info.metrics.droppedCount.Load(),
|
DroppedCount: info.metrics.droppedCount,
|
||||||
MinProcessTime: info.metrics.minTime,
|
MinProcessTime: info.metrics.minTime,
|
||||||
MaxProcessTime: info.metrics.maxTime,
|
MaxProcessTime: info.metrics.maxTime,
|
||||||
}
|
}
|
||||||
@@ -379,15 +314,12 @@ func (s *Streamer) GetHandlerStats() []HandlerStats {
|
|||||||
// Calculate average time
|
// Calculate average time
|
||||||
if info.metrics.processedCount > 0 {
|
if info.metrics.processedCount > 0 {
|
||||||
processedCount := info.metrics.processedCount
|
processedCount := info.metrics.processedCount
|
||||||
|
|
||||||
const maxInt64 = 1<<63 - 1
|
const maxInt64 = 1<<63 - 1
|
||||||
|
|
||||||
if processedCount > maxInt64 {
|
if processedCount > maxInt64 {
|
||||||
processedCount = maxInt64
|
processedCount = maxInt64
|
||||||
}
|
}
|
||||||
|
//nolint:gosec // processedCount is explicitly bounded above
|
||||||
hs.AvgProcessTime = info.metrics.totalTime /
|
hs.AvgProcessTime = info.metrics.totalTime / time.Duration(processedCount)
|
||||||
time.Duration(processedCount)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
info.metrics.mu.Unlock()
|
info.metrics.mu.Unlock()
|
||||||
@@ -398,40 +330,11 @@ func (s *Streamer) GetHandlerStats() []HandlerStats {
|
|||||||
return stats
|
return stats
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetDroppedMessages returns the total number of messages dropped across all
|
// GetDroppedMessages returns the total number of messages dropped across all handlers
|
||||||
// handlers due to queue overflow or backpressure. This counter is monotonically
|
// due to queue overflow or backpressure. This counter is monotonically increasing
|
||||||
// increasing and is safe to call concurrently.
|
// and is safe to call concurrently.
|
||||||
func (s *Streamer) GetDroppedMessages() uint64 {
|
func (s *Streamer) GetDroppedMessages() uint64 {
|
||||||
return s.totalDropped.Load()
|
return atomic.LoadUint64(&s.totalDropped)
|
||||||
}
|
|
||||||
|
|
||||||
// runHandlerWorker processes messages for a specific handler
|
|
||||||
func (s *Streamer) runHandlerWorker(info *handlerInfo) {
|
|
||||||
defer s.workers.Done()
|
|
||||||
|
|
||||||
for msg := range info.queue {
|
|
||||||
start := time.Now()
|
|
||||||
|
|
||||||
info.handler.HandleMessage(msg)
|
|
||||||
|
|
||||||
elapsed := time.Since(start)
|
|
||||||
|
|
||||||
// Update metrics
|
|
||||||
info.metrics.mu.Lock()
|
|
||||||
info.metrics.processedCount++
|
|
||||||
info.metrics.totalTime += elapsed
|
|
||||||
|
|
||||||
// Update min time
|
|
||||||
if elapsed < info.metrics.minTime {
|
|
||||||
info.metrics.minTime = elapsed
|
|
||||||
}
|
|
||||||
|
|
||||||
// Update max time
|
|
||||||
if elapsed > info.metrics.maxTime {
|
|
||||||
info.metrics.maxTime = elapsed
|
|
||||||
}
|
|
||||||
info.metrics.mu.Unlock()
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// logMetrics logs the current streaming statistics
|
// logMetrics logs the current streaming statistics
|
||||||
@@ -440,8 +343,7 @@ func (s *Streamer) logMetrics() {
|
|||||||
uptime := time.Since(metrics.ConnectedSince)
|
uptime := time.Since(metrics.ConnectedSince)
|
||||||
|
|
||||||
const bitsPerMegabit = 1000000
|
const bitsPerMegabit = 1000000
|
||||||
|
totalDropped := atomic.LoadUint64(&s.totalDropped)
|
||||||
totalDropped := s.totalDropped.Load()
|
|
||||||
|
|
||||||
s.logger.Info(
|
s.logger.Info(
|
||||||
"Stream statistics",
|
"Stream statistics",
|
||||||
@@ -467,19 +369,16 @@ func (s *Streamer) logMetrics() {
|
|||||||
|
|
||||||
// Log per-handler statistics
|
// Log per-handler statistics
|
||||||
s.mu.RLock()
|
s.mu.RLock()
|
||||||
|
|
||||||
for i, info := range s.handlers {
|
for i, info := range s.handlers {
|
||||||
info.metrics.mu.Lock()
|
info.metrics.mu.Lock()
|
||||||
if info.metrics.processedCount > 0 {
|
if info.metrics.processedCount > 0 {
|
||||||
// Safe conversion: processedCount is bounded by maxInt64
|
// Safe conversion: processedCount is bounded by maxInt64
|
||||||
processedCount := info.metrics.processedCount
|
processedCount := info.metrics.processedCount
|
||||||
|
|
||||||
const maxInt64 = 1<<63 - 1
|
const maxInt64 = 1<<63 - 1
|
||||||
|
|
||||||
if processedCount > maxInt64 {
|
if processedCount > maxInt64 {
|
||||||
processedCount = maxInt64
|
processedCount = maxInt64
|
||||||
}
|
}
|
||||||
|
//nolint:gosec // processedCount is explicitly bounded above
|
||||||
avgTime := info.metrics.totalTime / time.Duration(processedCount)
|
avgTime := info.metrics.totalTime / time.Duration(processedCount)
|
||||||
s.logger.Info(
|
s.logger.Info(
|
||||||
"Handler statistics",
|
"Handler statistics",
|
||||||
@@ -488,7 +387,7 @@ func (s *Streamer) logMetrics() {
|
|||||||
"queue_len", len(info.queue),
|
"queue_len", len(info.queue),
|
||||||
"queue_cap", cap(info.queue),
|
"queue_cap", cap(info.queue),
|
||||||
"processed", info.metrics.processedCount,
|
"processed", info.metrics.processedCount,
|
||||||
"dropped", info.metrics.droppedCount.Load(),
|
"dropped", info.metrics.droppedCount,
|
||||||
"avg_time", avgTime,
|
"avg_time", avgTime,
|
||||||
"min_time", info.metrics.minTime,
|
"min_time", info.metrics.minTime,
|
||||||
"max_time", info.metrics.maxTime,
|
"max_time", info.metrics.maxTime,
|
||||||
@@ -496,7 +395,6 @@ func (s *Streamer) logMetrics() {
|
|||||||
}
|
}
|
||||||
info.metrics.mu.Unlock()
|
info.metrics.mu.Unlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
s.mu.RUnlock()
|
s.mu.RUnlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -505,10 +403,9 @@ func (s *Streamer) updateMetrics(messageBytes int) {
|
|||||||
s.metrics.RecordMessage(int64(messageBytes))
|
s.metrics.RecordMessage(int64(messageBytes))
|
||||||
}
|
}
|
||||||
|
|
||||||
// streamWithReconnect handles streaming with automatic reconnection and
|
// streamWithReconnect handles streaming with automatic reconnection and exponential backoff
|
||||||
// exponential backoff
|
|
||||||
func (s *Streamer) streamWithReconnect(ctx context.Context) {
|
func (s *Streamer) streamWithReconnect(ctx context.Context) {
|
||||||
backoffDelay := s.minBackoff
|
backoffDelay := minBackoffDelay
|
||||||
consecutiveFailures := 0
|
consecutiveFailures := 0
|
||||||
|
|
||||||
for {
|
for {
|
||||||
@@ -525,9 +422,8 @@ func (s *Streamer) streamWithReconnect(ctx context.Context) {
|
|||||||
err := s.stream(ctx)
|
err := s.stream(ctx)
|
||||||
streamDuration := time.Since(startTime)
|
streamDuration := time.Since(startTime)
|
||||||
|
|
||||||
// Only Stop ends the streamer. A stream that ends any other way,
|
if err == nil {
|
||||||
// with or without an error, is a disconnect and is reconnected.
|
// Clean exit (context cancelled)
|
||||||
if ctx.Err() != nil {
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -538,14 +434,17 @@ func (s *Streamer) streamWithReconnect(ctx context.Context) {
|
|||||||
"stream_duration", streamDuration)
|
"stream_duration", streamDuration)
|
||||||
s.metrics.SetConnected(false)
|
s.metrics.SetConnected(false)
|
||||||
|
|
||||||
// A stream that lasted more than 30 seconds resets the backoff. That
|
// Check if context is cancelled
|
||||||
// includes a stream dropped after streamIdleTimeout with no line, so
|
if ctx.Err() != nil {
|
||||||
// a silent feed is retried after the first backoff delay every time.
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// If we streamed for more than 30 seconds, reset the backoff
|
||||||
|
// This indicates we had a successful connection that received data
|
||||||
if streamDuration > 30*time.Second {
|
if streamDuration > 30*time.Second {
|
||||||
s.logger.Info("Resetting backoff delay due to successful connection",
|
s.logger.Info("Resetting backoff delay due to successful connection",
|
||||||
"stream_duration", streamDuration)
|
"stream_duration", streamDuration)
|
||||||
|
backoffDelay = minBackoffDelay
|
||||||
backoffDelay = s.minBackoff
|
|
||||||
consecutiveFailures = 0
|
consecutiveFailures = 0
|
||||||
} else {
|
} else {
|
||||||
// Increment consecutive failures
|
// Increment consecutive failures
|
||||||
@@ -578,13 +477,7 @@ func (s *Streamer) stream(ctx context.Context) error {
|
|||||||
connCtx, connCancel := context.WithCancel(ctx)
|
connCtx, connCancel := context.WithCancel(ctx)
|
||||||
defer connCancel()
|
defer connCancel()
|
||||||
|
|
||||||
// The idle timer drops the connection, by cancelling connCtx, when no
|
req, err := http.NewRequestWithContext(ctx, "GET", s.url, nil)
|
||||||
// line arrives for s.idleTimeout. Each line read starts it over. It
|
|
||||||
// also covers the wait for the server to answer.
|
|
||||||
idleTimer := time.AfterFunc(s.idleTimeout, connCancel)
|
|
||||||
defer idleTimer.Stop()
|
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(connCtx, http.MethodGet, s.url, nil)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to create request: %w", err)
|
return fmt.Errorf("failed to create request: %w", err)
|
||||||
}
|
}
|
||||||
@@ -597,14 +490,13 @@ func (s *Streamer) stream(ctx context.Context) error {
|
|||||||
return fmt.Errorf("failed to connect to RIS Live: %w", err)
|
return fmt.Errorf("failed to connect to RIS Live: %w", err)
|
||||||
}
|
}
|
||||||
defer func() {
|
defer func() {
|
||||||
err := resp.Body.Close()
|
if err := resp.Body.Close(); err != nil {
|
||||||
if err != nil {
|
|
||||||
s.logger.Error("Failed to close response body", "error", err)
|
s.logger.Error("Failed to close response body", "error", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
if resp.StatusCode != http.StatusOK {
|
||||||
return fmt.Errorf("%w: %d", errUnexpectedStatus, resp.StatusCode)
|
return fmt.Errorf("unexpected status code: %d", resp.StatusCode)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Wrap body with counting reader to track actual wire bytes
|
// Wrap body with counting reader to track actual wire bytes
|
||||||
@@ -618,9 +510,7 @@ func (s *Streamer) stream(ctx context.Context) error {
|
|||||||
return fmt.Errorf("failed to create gzip reader: %w", err)
|
return fmt.Errorf("failed to create gzip reader: %w", err)
|
||||||
}
|
}
|
||||||
defer func() { _ = gzReader.Close() }()
|
defer func() { _ = gzReader.Close() }()
|
||||||
|
|
||||||
reader = gzReader
|
reader = gzReader
|
||||||
|
|
||||||
s.logger.Info("Connected to RIS Live stream", "compression", "gzip")
|
s.logger.Info("Connected to RIS Live stream", "compression", "gzip")
|
||||||
} else {
|
} else {
|
||||||
s.logger.Info("Connected to RIS Live stream", "compression", "none")
|
s.logger.Info("Connected to RIS Live stream", "compression", "none")
|
||||||
@@ -628,6 +518,9 @@ func (s *Streamer) stream(ctx context.Context) error {
|
|||||||
|
|
||||||
s.metrics.SetConnected(true)
|
s.metrics.SetConnected(true)
|
||||||
|
|
||||||
|
// Track wire bytes for metrics updates
|
||||||
|
var lastWireBytes int64
|
||||||
|
|
||||||
// Start metrics logging goroutine
|
// Start metrics logging goroutine
|
||||||
metricsTicker := time.NewTicker(metricsLogInterval)
|
metricsTicker := time.NewTicker(metricsLogInterval)
|
||||||
defer metricsTicker.Stop()
|
defer metricsTicker.Stop()
|
||||||
@@ -644,50 +537,28 @@ func (s *Streamer) stream(ctx context.Context) error {
|
|||||||
}()
|
}()
|
||||||
|
|
||||||
// Wire byte update ticker - update metrics with actual wire bytes periodically
|
// Wire byte update ticker - update metrics with actual wire bytes periodically
|
||||||
go s.recordWireBytes(connCtx, wireCounter)
|
|
||||||
|
|
||||||
return s.readStream(ctx, connCtx, reader, idleTimer)
|
|
||||||
}
|
|
||||||
|
|
||||||
// recordWireBytes adds the bytes wireCounter has counted to the metrics once a
|
|
||||||
// second until ctx is cancelled.
|
|
||||||
func (s *Streamer) recordWireBytes(
|
|
||||||
ctx context.Context, wireCounter *countingReader,
|
|
||||||
) {
|
|
||||||
wireUpdateTicker := time.NewTicker(time.Second)
|
wireUpdateTicker := time.NewTicker(time.Second)
|
||||||
defer wireUpdateTicker.Stop()
|
defer wireUpdateTicker.Stop()
|
||||||
|
|
||||||
var lastWireBytes int64
|
go func() {
|
||||||
|
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-wireUpdateTicker.C:
|
case <-wireUpdateTicker.C:
|
||||||
currentBytes := wireCounter.Count()
|
currentBytes := wireCounter.Count()
|
||||||
|
|
||||||
delta := currentBytes - lastWireBytes
|
delta := currentBytes - lastWireBytes
|
||||||
if delta > 0 {
|
if delta > 0 {
|
||||||
s.metrics.RecordWireBytes(delta)
|
s.metrics.RecordWireBytes(delta)
|
||||||
|
|
||||||
lastWireBytes = currentBytes
|
lastWireBytes = currentBytes
|
||||||
}
|
}
|
||||||
case <-ctx.Done():
|
case <-connCtx.Done():
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}()
|
||||||
|
|
||||||
// readStream hands each line of the stream to processLine until the stream
|
|
||||||
// ends, a line fails, or ctx is cancelled. Each line restarts idleTimer, which
|
|
||||||
// cancels connCtx when it fires.
|
|
||||||
func (s *Streamer) readStream(
|
|
||||||
ctx, connCtx context.Context, reader io.Reader, idleTimer *time.Timer,
|
|
||||||
) error {
|
|
||||||
scanner := bufio.NewScanner(reader)
|
scanner := bufio.NewScanner(reader)
|
||||||
scanner.Buffer(nil, maxLineSize)
|
|
||||||
|
|
||||||
for scanner.Scan() {
|
for scanner.Scan() {
|
||||||
idleTimer.Reset(s.idleTimeout)
|
|
||||||
|
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
s.logger.Info("Stream stopped by context")
|
s.logger.Info("Stream stopped by context")
|
||||||
@@ -701,32 +572,6 @@ func (s *Streamer) readStream(
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
err := s.processLine(ctx, line)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if ctx.Err() != nil {
|
|
||||||
return ctx.Err()
|
|
||||||
}
|
|
||||||
|
|
||||||
if connCtx.Err() != nil {
|
|
||||||
return fmt.Errorf("%w for %s", errStreamIdle, s.idleTimeout)
|
|
||||||
}
|
|
||||||
|
|
||||||
err := scanner.Err()
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("scanner error: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return errStreamEnded
|
|
||||||
}
|
|
||||||
|
|
||||||
// processLine passes one line of the stream to the raw handler, parses it,
|
|
||||||
// updates the peer set (updatePeers), and dispatches an UPDATE or STATE
|
|
||||||
// message to the handlers. An error ends the connection.
|
|
||||||
func (s *Streamer) processLine(ctx context.Context, line []byte) error {
|
|
||||||
// Update metrics with decompressed message size
|
// Update metrics with decompressed message size
|
||||||
s.updateMetrics(len(line))
|
s.updateMetrics(len(line))
|
||||||
|
|
||||||
@@ -742,9 +587,7 @@ func (s *Streamer) processLine(ctx context.Context, line []byte) error {
|
|||||||
|
|
||||||
// Parse the message first
|
// Parse the message first
|
||||||
var wrapper ristypes.RISLiveMessage
|
var wrapper ristypes.RISLiveMessage
|
||||||
|
if err := json.Unmarshal(line, &wrapper); err != nil {
|
||||||
err := json.Unmarshal(line, &wrapper)
|
|
||||||
if err != nil {
|
|
||||||
// Log the error and return to trigger reconnection
|
// Log the error and return to trigger reconnection
|
||||||
s.logger.Error("Failed to parse JSON",
|
s.logger.Error("Failed to parse JSON",
|
||||||
"error", err,
|
"error", err,
|
||||||
@@ -761,7 +604,7 @@ func (s *Streamer) processLine(ctx context.Context, line []byte) error {
|
|||||||
"line", string(line),
|
"line", string(line),
|
||||||
)
|
)
|
||||||
|
|
||||||
return nil
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get the actual message
|
// Get the actual message
|
||||||
@@ -770,100 +613,84 @@ func (s *Streamer) processLine(ctx context.Context, line []byte) error {
|
|||||||
// Parse the timestamp
|
// Parse the timestamp
|
||||||
msg.ParsedTimestamp = time.Unix(int64(msg.Timestamp), 0).UTC()
|
msg.ParsedTimestamp = time.Unix(int64(msg.Timestamp), 0).UTC()
|
||||||
|
|
||||||
peerCount := s.updatePeers(&msg)
|
|
||||||
|
|
||||||
// Process based on message type
|
// Process based on message type
|
||||||
switch msg.Type {
|
switch msg.Type {
|
||||||
case ristypes.MessageTypeUpdate:
|
case "UPDATE":
|
||||||
// Process BGP UPDATE messages
|
// Process BGP UPDATE messages
|
||||||
// Will be dispatched to handlers
|
// Will be dispatched to handlers
|
||||||
case "RIS_PEER_STATE":
|
case "RIS_PEER_STATE":
|
||||||
// RIS peer state messages - silently ignore
|
// RIS peer state messages - silently ignore
|
||||||
return nil
|
continue
|
||||||
case "KEEPALIVE":
|
case "KEEPALIVE":
|
||||||
// BGP keepalive messages - silently process
|
// BGP keepalive messages - silently process
|
||||||
return nil
|
continue
|
||||||
case "OPEN":
|
case "OPEN":
|
||||||
|
// BGP open messages - track peer as active
|
||||||
|
s.bgpPeersMu.Lock()
|
||||||
|
s.bgpPeers[msg.Peer] = true
|
||||||
|
peerCount := len(s.bgpPeers)
|
||||||
|
s.bgpPeersMu.Unlock()
|
||||||
|
s.metrics.SetBGPPeerCount(peerCount)
|
||||||
|
|
||||||
s.logger.Info("BGP session opened",
|
s.logger.Info("BGP session opened",
|
||||||
"peer", msg.Peer,
|
"peer", msg.Peer,
|
||||||
"peer_asn", msg.PeerASN,
|
"peer_asn", msg.PeerASN,
|
||||||
"total_peers", peerCount,
|
"total_peers", peerCount,
|
||||||
)
|
)
|
||||||
|
|
||||||
return nil
|
continue
|
||||||
case messageTypeNotification:
|
case "NOTIFICATION":
|
||||||
|
// BGP notification messages (session closed)
|
||||||
|
s.bgpPeersMu.Lock()
|
||||||
|
delete(s.bgpPeers, msg.Peer)
|
||||||
|
peerCount := len(s.bgpPeers)
|
||||||
|
s.bgpPeersMu.Unlock()
|
||||||
|
s.metrics.SetBGPPeerCount(peerCount)
|
||||||
|
|
||||||
s.logger.Warn("BGP notification",
|
s.logger.Warn("BGP notification",
|
||||||
"peer", msg.Peer,
|
"peer", msg.Peer,
|
||||||
"peer_asn", msg.PeerASN,
|
"peer_asn", msg.PeerASN,
|
||||||
"total_peers", peerCount,
|
"total_peers", peerCount,
|
||||||
)
|
)
|
||||||
|
|
||||||
return nil
|
continue
|
||||||
case ristypes.MessageTypeState:
|
case "STATE":
|
||||||
// The BGP session with the peer changed state, for example went
|
// Peer state changes - silently ignore
|
||||||
// down. Dispatched to the handlers that want it.
|
continue
|
||||||
default:
|
default:
|
||||||
s.logger.Warn("Unknown message type, skipping",
|
s.logger.Warn("Unknown message type, skipping",
|
||||||
"type", msg.Type,
|
"type", msg.Type,
|
||||||
)
|
)
|
||||||
|
|
||||||
return nil
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
return s.dispatchMessage(ctx, &msg)
|
// Dispatch to interested handlers. Stop cancels ctx and closes the
|
||||||
}
|
// queues under the write lock, so if ctx is cancelled here, under the
|
||||||
|
// read lock, the queues are closed and must not be sent to.
|
||||||
// updatePeers adds the peer msg came from to the peer set, or removes it if
|
|
||||||
// msg ends the peer's session: a NOTIFICATION, or a STATE message whose state
|
|
||||||
// is down. It records the size of the set as the BGP peer count and returns
|
|
||||||
// it.
|
|
||||||
func (s *Streamer) updatePeers(msg *ristypes.RISMessage) int {
|
|
||||||
sessionEnded := msg.Type == messageTypeNotification ||
|
|
||||||
(msg.Type == ristypes.MessageTypeState && msg.State == ristypes.StateDown)
|
|
||||||
|
|
||||||
s.bgpPeersMu.Lock()
|
|
||||||
|
|
||||||
if sessionEnded {
|
|
||||||
delete(s.bgpPeers, msg.Peer)
|
|
||||||
} else {
|
|
||||||
s.bgpPeers[msg.Peer] = true
|
|
||||||
}
|
|
||||||
|
|
||||||
peerCount := len(s.bgpPeers)
|
|
||||||
s.bgpPeersMu.Unlock()
|
|
||||||
s.metrics.SetBGPPeerCount(peerCount)
|
|
||||||
|
|
||||||
return peerCount
|
|
||||||
}
|
|
||||||
|
|
||||||
// dispatchMessage queues msg for each handler that wants it, unless
|
|
||||||
// queueMessage drops it for that handler.
|
|
||||||
func (s *Streamer) dispatchMessage(
|
|
||||||
ctx context.Context, msg *ristypes.RISMessage,
|
|
||||||
) error {
|
|
||||||
// Stop cancels ctx and closes the queues under the write lock, so if ctx
|
|
||||||
// is cancelled here, under the read lock, the queues are closed and must
|
|
||||||
// not be sent to.
|
|
||||||
s.mu.RLock()
|
s.mu.RLock()
|
||||||
|
|
||||||
if ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
s.mu.RUnlock()
|
s.mu.RUnlock()
|
||||||
|
|
||||||
return ctx.Err()
|
return ctx.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, info := range s.handlers {
|
for _, info := range s.handlers {
|
||||||
if !info.handler.WantsMessage(msg.Type) {
|
if !info.handler.WantsMessage(msg.Type) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
if !s.queueMessage(info, msg) {
|
// Check if we should drop due to backpressure
|
||||||
info.metrics.droppedCount.Add(1)
|
if s.shouldDropForBackpressure(info) {
|
||||||
s.totalDropped.Add(1)
|
atomic.AddUint64(&info.metrics.droppedCount, 1)
|
||||||
|
atomic.AddUint64(&s.totalDropped, 1)
|
||||||
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Try to queue the message
|
||||||
|
select {
|
||||||
|
case info.queue <- &msg:
|
||||||
|
// Message queued successfully
|
||||||
// Update high water mark if needed
|
// Update high water mark if needed
|
||||||
queueLen := len(info.queue)
|
queueLen := len(info.queue)
|
||||||
info.metrics.mu.Lock()
|
info.metrics.mu.Lock()
|
||||||
@@ -871,61 +698,23 @@ func (s *Streamer) dispatchMessage(
|
|||||||
info.metrics.queueHighWaterMark = queueLen
|
info.metrics.queueHighWaterMark = queueLen
|
||||||
}
|
}
|
||||||
info.metrics.mu.Unlock()
|
info.metrics.mu.Unlock()
|
||||||
|
default:
|
||||||
|
// Queue is full, drop the message
|
||||||
|
atomic.AddUint64(&info.metrics.droppedCount, 1)
|
||||||
|
atomic.AddUint64(&s.totalDropped, 1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
s.mu.RUnlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
s.mu.RUnlock()
|
if err := scanner.Err(); err != nil {
|
||||||
|
return fmt.Errorf("scanner error: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// queueMessage puts msg on the handler's queue and reports false if it dropped
|
// shouldDropForBackpressure determines if a message should be dropped based on queue utilization
|
||||||
// msg instead. It never waits for room, so a full queue does not hold up the
|
|
||||||
// read loop.
|
|
||||||
//
|
|
||||||
// A message is dropped while the queue is filling up (shouldDropForBackpressure)
|
|
||||||
// or full, except a STATE message: one whose state is down is the only notice
|
|
||||||
// that all of a peer's routes are gone. A STATE message skips the random drop,
|
|
||||||
// and one that finds the queue full is held in info.heldState. Held messages
|
|
||||||
// go into the queue, oldest first, before anything else for the handler, as
|
|
||||||
// soon as there is room, or when Stop is called; while any is still held, the
|
|
||||||
// handler's other messages are dropped, so none of them overtakes a held down.
|
|
||||||
//
|
|
||||||
// The held list stays small: a STATE message is a few hundred bytes and the
|
|
||||||
// feed sends about ten a second, so a queue full for a minute holds about 600.
|
|
||||||
func (s *Streamer) queueMessage(
|
|
||||||
info *handlerInfo, msg *ristypes.RISMessage,
|
|
||||||
) bool {
|
|
||||||
for len(info.heldState) > 0 && trySend(info.queue, info.heldState[0]) {
|
|
||||||
info.heldState = info.heldState[1:]
|
|
||||||
}
|
|
||||||
|
|
||||||
if msg.Type == ristypes.MessageTypeState {
|
|
||||||
if len(info.heldState) > 0 || !trySend(info.queue, msg) {
|
|
||||||
info.heldState = append(info.heldState, msg)
|
|
||||||
}
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(info.heldState) > 0 || s.shouldDropForBackpressure(info) {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
return trySend(info.queue, msg)
|
|
||||||
}
|
|
||||||
|
|
||||||
// trySend puts msg on queue if there is room and reports whether it did.
|
|
||||||
func trySend(queue chan *ristypes.RISMessage, msg *ristypes.RISMessage) bool {
|
|
||||||
select {
|
|
||||||
case queue <- msg:
|
|
||||||
return true
|
|
||||||
default:
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// shouldDropForBackpressure determines if a message should be dropped based on
|
|
||||||
// queue utilization
|
|
||||||
func (s *Streamer) shouldDropForBackpressure(info *handlerInfo) bool {
|
func (s *Streamer) shouldDropForBackpressure(info *handlerInfo) bool {
|
||||||
// Calculate queue utilization
|
// Calculate queue utilization
|
||||||
queueLen := len(info.queue)
|
queueLen := len(info.queue)
|
||||||
|
|||||||
@@ -1,850 +0,0 @@
|
|||||||
package streamer
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"io"
|
|
||||||
"log/slog"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"runtime"
|
|
||||||
"strings"
|
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/metrics"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/ristypes"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestNewStreamer(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
logger := logger.New()
|
|
||||||
metricsTracker := metrics.New()
|
|
||||||
s := New(logger, metricsTracker)
|
|
||||||
|
|
||||||
if s == nil {
|
|
||||||
t.Fatal("New() returned nil")
|
|
||||||
}
|
|
||||||
|
|
||||||
if s.logger != logger {
|
|
||||||
t.Error("logger not set correctly")
|
|
||||||
}
|
|
||||||
|
|
||||||
if s.client == nil {
|
|
||||||
t.Error("HTTP client not initialized")
|
|
||||||
}
|
|
||||||
|
|
||||||
if s.handlers == nil {
|
|
||||||
t.Error("handlers slice not initialized")
|
|
||||||
}
|
|
||||||
|
|
||||||
if s.metrics != metricsTracker {
|
|
||||||
t.Error("metrics tracker not set correctly")
|
|
||||||
}
|
|
||||||
|
|
||||||
if s.idleTimeout != streamIdleTimeout {
|
|
||||||
t.Errorf("idleTimeout = %s, want %s", s.idleTimeout, streamIdleTimeout)
|
|
||||||
}
|
|
||||||
|
|
||||||
if s.minBackoff != minBackoffDelay {
|
|
||||||
t.Errorf("minBackoff = %s, want %s", s.minBackoff, minBackoffDelay)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestStreamDoesNotLeakTickersAcrossReconnects drives many short-lived
|
|
||||||
// connections (each stream call is one reconnect cycle) and asserts the
|
|
||||||
// goroutine count returns to its starting value. Each connection starts two
|
|
||||||
// ticker goroutines; before the fix they lived until the streamer's lifetime
|
|
||||||
// context was cancelled, so every reconnect leaked two.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // counts goroutines, so no other test may run alongside
|
|
||||||
func TestStreamDoesNotLeakTickersAcrossReconnects(t *testing.T) {
|
|
||||||
// The handler returns immediately, so the response body is empty and each
|
|
||||||
// stream call ends at once, standing in for a dropped connection.
|
|
||||||
srv := httptest.NewServer(http.HandlerFunc(
|
|
||||||
func(_ http.ResponseWriter, _ *http.Request) {}))
|
|
||||||
defer srv.Close()
|
|
||||||
|
|
||||||
s := New(logger.New(), metrics.New())
|
|
||||||
s.url = srv.URL
|
|
||||||
|
|
||||||
// One warm-up connection so any persistent HTTP transport goroutine exists
|
|
||||||
// before we take the baseline.
|
|
||||||
err := s.stream(context.Background())
|
|
||||||
if !errors.Is(err, errStreamEnded) {
|
|
||||||
t.Fatalf("warm-up stream returned %v, want %v", err, errStreamEnded)
|
|
||||||
}
|
|
||||||
|
|
||||||
s.client.CloseIdleConnections()
|
|
||||||
|
|
||||||
baseline := settledGoroutineCount()
|
|
||||||
|
|
||||||
const reconnects = 20
|
|
||||||
for range reconnects {
|
|
||||||
err := s.stream(context.Background())
|
|
||||||
if !errors.Is(err, errStreamEnded) {
|
|
||||||
t.Fatalf("stream returned %v, want %v", err, errStreamEnded)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
s.client.CloseIdleConnections()
|
|
||||||
|
|
||||||
if !waitForGoroutines(baseline) {
|
|
||||||
t.Fatalf("goroutines did not return to baseline %d after %d reconnects, got %d",
|
|
||||||
baseline, reconnects, runtime.NumGoroutine())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// updateLine is one UPDATE message as RIS Live sends it.
|
|
||||||
const updateLine = `{"type":"ris_message","data":{"type":"UPDATE",` +
|
|
||||||
`"peer":"192.0.2.1","peer_asn":"64496","timestamp":1700000000}}` + "\n"
|
|
||||||
|
|
||||||
// updateHandler wants UPDATE messages. For each one it handles it sends on
|
|
||||||
// handled if there is room, and otherwise does nothing; a nil handled is
|
|
||||||
// never sent on.
|
|
||||||
type updateHandler struct {
|
|
||||||
handled chan struct{}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (updateHandler) WantsMessage(messageType string) bool {
|
|
||||||
return messageType == "UPDATE"
|
|
||||||
}
|
|
||||||
|
|
||||||
func (updateHandler) QueueCapacity() int { return 10 }
|
|
||||||
|
|
||||||
func (h updateHandler) HandleMessage(*ristypes.RISMessage) {
|
|
||||||
select {
|
|
||||||
case h.handled <- struct{}{}:
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// flush sends what w has buffered to the client. The writers httptest servers
|
|
||||||
// hand to handlers can always do that.
|
|
||||||
func flush(w http.ResponseWriter) {
|
|
||||||
flusher, ok := w.(http.Flusher)
|
|
||||||
if ok {
|
|
||||||
flusher.Flush()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// stateDownLine is the STATE message RIS Live sends when the collector's BGP
|
|
||||||
// session with the peer stateDownPeer goes down.
|
|
||||||
const (
|
|
||||||
stateDownPeer = "192.0.2.1"
|
|
||||||
stateDownLine = `{"type":"ris_message","data":{"type":"STATE",` +
|
|
||||||
`"peer":"` + stateDownPeer + `",` +
|
|
||||||
`"peer_asn":"64496","timestamp":1700000000,"state":"down"}}` + "\n"
|
|
||||||
)
|
|
||||||
|
|
||||||
// stateHandler wants STATE messages only. Tests read what it is handed from
|
|
||||||
// its queue, so HandleMessage does nothing.
|
|
||||||
type stateHandler struct{}
|
|
||||||
|
|
||||||
func (stateHandler) WantsMessage(messageType string) bool {
|
|
||||||
return messageType == ristypes.MessageTypeState
|
|
||||||
}
|
|
||||||
|
|
||||||
func (stateHandler) QueueCapacity() int { return 10 }
|
|
||||||
|
|
||||||
func (stateHandler) HandleMessage(*ristypes.RISMessage) {}
|
|
||||||
|
|
||||||
// TestStreamHandsStateMessagesToHandlersThatWantThem has the server send one
|
|
||||||
// STATE message whose state is down. The streamer must queue it, with its
|
|
||||||
// peer and state, for a handler that wants STATE messages.
|
|
||||||
func TestStreamHandsStateMessagesToHandlersThatWantThem(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
srv := httptest.NewServer(http.HandlerFunc(
|
|
||||||
func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
_, _ = io.WriteString(w, stateDownLine)
|
|
||||||
}))
|
|
||||||
defer srv.Close()
|
|
||||||
|
|
||||||
s := New(logger.New(), metrics.New())
|
|
||||||
s.url = srv.URL
|
|
||||||
s.RegisterHandler(stateHandler{})
|
|
||||||
|
|
||||||
err := s.stream(context.Background())
|
|
||||||
if !errors.Is(err, errStreamEnded) {
|
|
||||||
t.Fatalf("stream returned %v, want %v", err, errStreamEnded)
|
|
||||||
}
|
|
||||||
|
|
||||||
select {
|
|
||||||
case msg := <-s.handlers[0].queue:
|
|
||||||
if msg.Type != ristypes.MessageTypeState || msg.Peer != stateDownPeer ||
|
|
||||||
msg.State != ristypes.StateDown {
|
|
||||||
t.Errorf("queued message has type %q, peer %q, state %q; "+
|
|
||||||
"want %q, %q, %q", msg.Type, msg.Peer, msg.State,
|
|
||||||
ristypes.MessageTypeState, stateDownPeer, ristypes.StateDown)
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
t.Fatal("the STATE message was not queued for the handler that wants it")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHeldStateMessagesKeepTheirPlaceInTheQueue hands queueMessage two STATE
|
|
||||||
// messages and two updates while the queue is full, making room in between.
|
|
||||||
// Neither STATE message may be dropped: each must reach the queue after
|
|
||||||
// everything queued before it and before anything handed over after it. The
|
|
||||||
// update handed over while a STATE message is held is dropped.
|
|
||||||
func TestHeldStateMessagesKeepTheirPlaceInTheQueue(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := New(logger.New(), metrics.New())
|
|
||||||
s.RegisterHandler(stateHandler{})
|
|
||||||
|
|
||||||
info := s.handlers[0]
|
|
||||||
|
|
||||||
before := &ristypes.RISMessage{Type: ristypes.MessageTypeUpdate}
|
|
||||||
for len(info.queue) < cap(info.queue) {
|
|
||||||
info.queue <- before
|
|
||||||
}
|
|
||||||
|
|
||||||
firstDown := &ristypes.RISMessage{
|
|
||||||
Type: ristypes.MessageTypeState, State: ristypes.StateDown,
|
|
||||||
}
|
|
||||||
secondDown := &ristypes.RISMessage{
|
|
||||||
Type: ristypes.MessageTypeState, State: ristypes.StateDown,
|
|
||||||
}
|
|
||||||
updateWhileHeld := &ristypes.RISMessage{Type: ristypes.MessageTypeUpdate}
|
|
||||||
after := &ristypes.RISMessage{Type: ristypes.MessageTypeUpdate}
|
|
||||||
|
|
||||||
if !s.queueMessage(info, firstDown) {
|
|
||||||
t.Fatal("the first STATE message was dropped on a full queue")
|
|
||||||
}
|
|
||||||
|
|
||||||
<-info.queue // the handler makes room for one message
|
|
||||||
|
|
||||||
if !s.queueMessage(info, secondDown) {
|
|
||||||
t.Fatal("the second STATE message was dropped")
|
|
||||||
}
|
|
||||||
|
|
||||||
if s.queueMessage(info, updateWhileHeld) {
|
|
||||||
t.Fatal("an update was queued while a STATE message was held")
|
|
||||||
}
|
|
||||||
|
|
||||||
// The handler takes every message queued before the first STATE message.
|
|
||||||
for range cap(info.queue) - 1 {
|
|
||||||
if <-info.queue != before {
|
|
||||||
t.Fatal("a message overtook those queued before the first STATE message")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if !s.queueMessage(info, after) {
|
|
||||||
t.Fatal("an update was dropped on a queue with room")
|
|
||||||
}
|
|
||||||
|
|
||||||
if <-info.queue != firstDown {
|
|
||||||
t.Fatal("the first STATE message did not come next")
|
|
||||||
}
|
|
||||||
|
|
||||||
if <-info.queue != secondDown {
|
|
||||||
t.Fatal("the second STATE message did not come after the first")
|
|
||||||
}
|
|
||||||
|
|
||||||
if <-info.queue != after {
|
|
||||||
t.Fatal("the update handed over after the STATE messages did not come " +
|
|
||||||
"after them")
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(info.queue) != 0 {
|
|
||||||
t.Errorf("%d messages left on the queue, want none", len(info.queue))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// takeAll takes the messages off queue, as a handler's worker would, and sends
|
|
||||||
// them on the returned channel once queue is closed.
|
|
||||||
func takeAll(queue chan *ristypes.RISMessage) chan []*ristypes.RISMessage {
|
|
||||||
taken := make(chan []*ristypes.RISMessage, 1)
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
var msgs []*ristypes.RISMessage
|
|
||||||
|
|
||||||
for msg := range queue {
|
|
||||||
msgs = append(msgs, msg)
|
|
||||||
}
|
|
||||||
|
|
||||||
taken <- msgs
|
|
||||||
}()
|
|
||||||
|
|
||||||
return taken
|
|
||||||
}
|
|
||||||
|
|
||||||
// newStreamerWithFullStateQueue returns a streamer reading from url with one
|
|
||||||
// handler, a stateHandler whose queue is full.
|
|
||||||
func newStreamerWithFullStateQueue(url string) *Streamer {
|
|
||||||
s := New(logger.New(), metrics.New())
|
|
||||||
s.url = url
|
|
||||||
s.RegisterHandler(stateHandler{})
|
|
||||||
|
|
||||||
queue := s.handlers[0].queue
|
|
||||||
for len(queue) < cap(queue) {
|
|
||||||
queue <- &ristypes.RISMessage{Type: ristypes.MessageTypeState}
|
|
||||||
}
|
|
||||||
|
|
||||||
return s
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestOtherHandlersKeepReceivingWhileAStateMessageIsHeld has the server send a
|
|
||||||
// STATE message whose state is down, then an UPDATE, while the queue of the
|
|
||||||
// handler that wants STATE messages is full. The STATE message must be held
|
|
||||||
// for that handler, and the read loop must go on to queue the UPDATE for the
|
|
||||||
// handler that wants it.
|
|
||||||
func TestOtherHandlersKeepReceivingWhileAStateMessageIsHeld(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
srv := httptest.NewServer(http.HandlerFunc(
|
|
||||||
func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
_, _ = io.WriteString(w, stateDownLine+updateLine)
|
|
||||||
}))
|
|
||||||
defer srv.Close()
|
|
||||||
|
|
||||||
s := newStreamerWithFullStateQueue(srv.URL)
|
|
||||||
s.RegisterHandler(updateHandler{})
|
|
||||||
|
|
||||||
err := s.stream(context.Background())
|
|
||||||
if !errors.Is(err, errStreamEnded) {
|
|
||||||
t.Fatalf("stream returned %v, want %v", err, errStreamEnded)
|
|
||||||
}
|
|
||||||
|
|
||||||
held := s.handlers[0].heldState
|
|
||||||
if len(held) != 1 || held[0].Peer != stateDownPeer ||
|
|
||||||
held[0].State != ristypes.StateDown {
|
|
||||||
t.Errorf("held %d STATE messages, want the down from %s alone",
|
|
||||||
len(held), stateDownPeer)
|
|
||||||
}
|
|
||||||
|
|
||||||
got := len(s.handlers[1].queue)
|
|
||||||
if got != 1 {
|
|
||||||
t.Errorf("%d messages queued for the handler that wants UPDATE "+
|
|
||||||
"messages, want 1", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestStopQueuesTheHeldStateMessages has the server send two STATE messages
|
|
||||||
// and an UPDATE and hold the response open while the queue of the handler that
|
|
||||||
// wants STATE messages is full, so both are held. Stop must put both on the
|
|
||||||
// queue, after the messages already on it, before it closes the queue, and
|
|
||||||
// must end the stream.
|
|
||||||
func TestStopQueuesTheHeldStateMessages(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const (
|
|
||||||
held = 2
|
|
||||||
lines = held + 1
|
|
||||||
maxWait = 5 * time.Second
|
|
||||||
)
|
|
||||||
|
|
||||||
srv := httptest.NewServer(http.HandlerFunc(
|
|
||||||
func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
_, _ = io.WriteString(w, stateDownLine+stateDownLine+updateLine)
|
|
||||||
|
|
||||||
flush(w)
|
|
||||||
<-r.Context().Done()
|
|
||||||
}))
|
|
||||||
defer srv.Close()
|
|
||||||
|
|
||||||
s := newStreamerWithFullStateQueue(srv.URL)
|
|
||||||
// The raw handler sees each line before the read loop hands it to the
|
|
||||||
// queues, so once it has seen the UPDATE, both STATE messages are held.
|
|
||||||
linesSeen := make(chan struct{}, lines)
|
|
||||||
|
|
||||||
s.RegisterRawHandler(func(string) { linesSeen <- struct{}{} })
|
|
||||||
|
|
||||||
// As in TestStopBeforeMessageReachesQueues, setting cancel as Start does
|
|
||||||
// lets Stop cancel the stream run here.
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
s.cancel = cancel
|
|
||||||
streamErr := make(chan error, 1)
|
|
||||||
|
|
||||||
go func() { streamErr <- s.stream(ctx) }()
|
|
||||||
|
|
||||||
for range lines {
|
|
||||||
select {
|
|
||||||
case <-linesSeen:
|
|
||||||
case <-time.After(maxWait):
|
|
||||||
t.Fatalf("the read loop did not read %d lines within %s",
|
|
||||||
lines, maxWait)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// No worker runs here: the test takes from the full queue in its place.
|
|
||||||
queue := s.handlers[0].queue
|
|
||||||
taken := takeAll(queue)
|
|
||||||
stopped := make(chan struct{})
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
s.Stop(t.Context())
|
|
||||||
close(stopped)
|
|
||||||
}()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-stopped:
|
|
||||||
case <-time.After(maxWait):
|
|
||||||
t.Fatalf("Stop did not return within %s while a STATE message was held",
|
|
||||||
maxWait)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stop has closed the queue, so takeAll is done or about to be.
|
|
||||||
msgs := <-taken
|
|
||||||
if len(msgs) != cap(queue)+held {
|
|
||||||
t.Fatalf("%d messages came off the queue, want the %d on it and the "+
|
|
||||||
"%d held", len(msgs), cap(queue), held)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, msg := range msgs[cap(queue):] {
|
|
||||||
if msg.Peer != stateDownPeer || msg.State != ristypes.StateDown {
|
|
||||||
t.Errorf("message after those on the queue has peer %q, state %q; "+
|
|
||||||
"want the down from %s", msg.Peer, msg.State, stateDownPeer)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
select {
|
|
||||||
case err := <-streamErr:
|
|
||||||
if !errors.Is(err, context.Canceled) {
|
|
||||||
t.Errorf("stream returned %v, want %v", err, context.Canceled)
|
|
||||||
}
|
|
||||||
case <-time.After(maxWait):
|
|
||||||
t.Fatalf("stream did not end within %s of Stop", maxWait)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestStopReturnsByItsDeadline stops a streamer whose handler has a full queue
|
|
||||||
// and a STATE message held for it, and a worker that stays busy until the test
|
|
||||||
// ends, as one does whose handler waits for a database write. Stop must return
|
|
||||||
// within its deadline plus a small margin instead of waiting for room on the
|
|
||||||
// queue and for the worker, and log as not handed over the held message and
|
|
||||||
// every message on the queue.
|
|
||||||
func TestStopReturnsByItsDeadline(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const (
|
|
||||||
deadline = 100 * time.Millisecond
|
|
||||||
margin = 2 * time.Second
|
|
||||||
)
|
|
||||||
|
|
||||||
var logs bytes.Buffer
|
|
||||||
|
|
||||||
s := newStreamerWithFullStateQueue("")
|
|
||||||
s.logger = &logger.Logger{Logger: slog.New(slog.NewJSONHandler(&logs,
|
|
||||||
&slog.HandlerOptions{Level: slog.LevelError}))}
|
|
||||||
s.handlers[0].heldState = []*ristypes.RISMessage{
|
|
||||||
{Type: ristypes.MessageTypeState},
|
|
||||||
}
|
|
||||||
wantNotHandedOver := 1 + len(s.handlers[0].queue)
|
|
||||||
// Stop cancels the stream Start would have started; there is none here.
|
|
||||||
s.cancel = func() {}
|
|
||||||
|
|
||||||
release := make(chan struct{})
|
|
||||||
defer close(release)
|
|
||||||
|
|
||||||
s.workers.Go(func() { <-release })
|
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(t.Context(), deadline)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
stopped := make(chan struct{})
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
s.Stop(ctx)
|
|
||||||
close(stopped)
|
|
||||||
}()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-stopped:
|
|
||||||
case <-time.After(deadline + margin):
|
|
||||||
t.Fatalf("Stop did not return within %s of its deadline", margin)
|
|
||||||
}
|
|
||||||
|
|
||||||
var logged struct {
|
|
||||||
Messages int `json:"messages"`
|
|
||||||
}
|
|
||||||
|
|
||||||
err := json.Unmarshal(logs.Bytes(), &logged)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("want the error Stop logs, got %q: %v", logs.String(), err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if logged.Messages != wantNotHandedOver {
|
|
||||||
t.Errorf("Stop logged %d messages not handed over, want %d",
|
|
||||||
logged.Messages, wantNotHandedOver)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestStopBeforeMessageReachesQueues stops the streamer after the read loop
|
|
||||||
// has checked for cancellation but before it hands the message to the handler
|
|
||||||
// queues. That is the gap a stop from another goroutine can land in, and it
|
|
||||||
// used to end in "send on closed channel". The raw handler runs in that gap on
|
|
||||||
// the read loop itself, so calling Stop from it hits the gap every time.
|
|
||||||
func TestStopBeforeMessageReachesQueues(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
srv := httptest.NewServer(http.HandlerFunc(
|
|
||||||
func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
_, _ = io.WriteString(w, updateLine)
|
|
||||||
}))
|
|
||||||
defer srv.Close()
|
|
||||||
|
|
||||||
s := New(logger.New(), metrics.New())
|
|
||||||
s.url = srv.URL
|
|
||||||
s.RegisterHandler(updateHandler{})
|
|
||||||
s.RegisterRawHandler(func(string) { s.Stop(t.Context()) })
|
|
||||||
|
|
||||||
// Start would run the stream in the background, where the test cannot
|
|
||||||
// wait for it. Setting cancel as Start does lets Stop cancel the stream
|
|
||||||
// run here instead.
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
s.cancel = cancel
|
|
||||||
|
|
||||||
err := s.stream(ctx)
|
|
||||||
if !errors.Is(err, context.Canceled) {
|
|
||||||
t.Fatalf("stream returned %v, want %v", err, context.Canceled)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A second Stop must not close the queues again.
|
|
||||||
s.Stop(t.Context())
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestReconnectWhenStreamEnds has the server end the first response
|
|
||||||
// normally, as when RIS Live restarts. The streamer must treat that as a
|
|
||||||
// disconnect and connect again.
|
|
||||||
func TestReconnectWhenStreamEnds(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
endResponse := func(http.ResponseWriter, *http.Request) {}
|
|
||||||
checkReconnect(t, endResponse, streamIdleTimeout)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestReconnectWhenStreamGoesIdle has the server hold the first response
|
|
||||||
// open without sending anything. The streamer must drop it once the idle
|
|
||||||
// timeout passes and connect again.
|
|
||||||
func TestReconnectWhenStreamGoesIdle(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const idleTimeout = 50 * time.Millisecond
|
|
||||||
|
|
||||||
holdResponse := func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
flush(w)
|
|
||||||
<-r.Context().Done()
|
|
||||||
}
|
|
||||||
checkReconnect(t, holdResponse, idleTimeout)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestReconnectWhenServerNeverAnswers has the server accept the first
|
|
||||||
// request and never answer it, not even with headers. The streamer must give
|
|
||||||
// up on it once the idle timeout passes and connect again.
|
|
||||||
func TestReconnectWhenServerNeverAnswers(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const idleTimeout = 50 * time.Millisecond
|
|
||||||
|
|
||||||
neverAnswer := func(_ http.ResponseWriter, r *http.Request) {
|
|
||||||
<-r.Context().Done()
|
|
||||||
}
|
|
||||||
checkReconnect(t, neverAnswer, idleTimeout)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestStreamNotDroppedWhileLinesArrive has the server send lines well within
|
|
||||||
// the idle timeout for several times the timeout, then end the response. Each
|
|
||||||
// line must start the idle timeout over, so the stream lasts until the server
|
|
||||||
// ends it instead of being dropped as idle.
|
|
||||||
func TestStreamNotDroppedWhileLinesArrive(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const (
|
|
||||||
idleTimeout = 100 * time.Millisecond
|
|
||||||
lineInterval = idleTimeout / 10
|
|
||||||
sendFor = 5 * idleTimeout
|
|
||||||
)
|
|
||||||
|
|
||||||
srv := httptest.NewServer(http.HandlerFunc(
|
|
||||||
func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
deadline := time.Now().Add(sendFor)
|
|
||||||
for time.Now().Before(deadline) {
|
|
||||||
_, _ = io.WriteString(w, updateLine)
|
|
||||||
|
|
||||||
flush(w)
|
|
||||||
time.Sleep(lineInterval)
|
|
||||||
}
|
|
||||||
}))
|
|
||||||
defer srv.Close()
|
|
||||||
|
|
||||||
s := New(logger.New(), metrics.New())
|
|
||||||
s.url = srv.URL
|
|
||||||
s.idleTimeout = idleTimeout
|
|
||||||
|
|
||||||
err := s.stream(context.Background())
|
|
||||||
if !errors.Is(err, errStreamEnded) {
|
|
||||||
t.Fatalf("stream returned %v, want %v", err, errStreamEnded)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestStreamReadsLongLine has the server send one UPDATE line of about
|
|
||||||
// 200 KiB, padded with raw hex the way RIS Live writes an extended BGP
|
|
||||||
// message, and then end the response. The line is longer than a
|
|
||||||
// bufio.Scanner reads by default. The streamer must hand the message to the
|
|
||||||
// handler's queue and read on until the server ends the stream, instead of
|
|
||||||
// dropping the connection at the long line.
|
|
||||||
func TestStreamReadsLongLine(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const rawHexLength = 200 * bytesPerKB
|
|
||||||
|
|
||||||
longLine := `{"type":"ris_message","data":{"type":"UPDATE","peer":"192.0.2.1",` +
|
|
||||||
`"peer_asn":"64496","timestamp":1700000000,"raw":"` +
|
|
||||||
strings.Repeat("0", rawHexLength) + `"}}` + "\n"
|
|
||||||
|
|
||||||
srv := httptest.NewServer(http.HandlerFunc(
|
|
||||||
func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
_, _ = io.WriteString(w, longLine)
|
|
||||||
}))
|
|
||||||
defer srv.Close()
|
|
||||||
|
|
||||||
s := New(logger.New(), metrics.New())
|
|
||||||
s.url = srv.URL
|
|
||||||
s.RegisterHandler(updateHandler{})
|
|
||||||
|
|
||||||
err := s.stream(context.Background())
|
|
||||||
if !errors.Is(err, errStreamEnded) {
|
|
||||||
t.Fatalf("stream returned %v, want %v", err, errStreamEnded)
|
|
||||||
}
|
|
||||||
|
|
||||||
queued := len(s.handlers[0].queue)
|
|
||||||
if queued != 1 {
|
|
||||||
t.Errorf("handler queue holds %d messages, want 1", queued)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// blockedHandler wants UPDATE messages, has room on its queue for one, and
|
|
||||||
// does not return from HandleMessage until release is closed.
|
|
||||||
type blockedHandler struct {
|
|
||||||
release chan struct{}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (blockedHandler) WantsMessage(messageType string) bool {
|
|
||||||
return messageType == ristypes.MessageTypeUpdate
|
|
||||||
}
|
|
||||||
|
|
||||||
func (blockedHandler) QueueCapacity() int { return 1 }
|
|
||||||
|
|
||||||
func (h blockedHandler) HandleMessage(*ristypes.RISMessage) { <-h.release }
|
|
||||||
|
|
||||||
// TestHandlerStatsReadWhileMessagesAreDropped has the server send a few UPDATE
|
|
||||||
// lines to a blockedHandler, so the read loop drops all but the one the
|
|
||||||
// handler holds and the one on its queue, while the test reads the handler's
|
|
||||||
// statistics. The drop count must reach the number dropped, and the race
|
|
||||||
// detector must find the count read and written safely.
|
|
||||||
func TestHandlerStatsReadWhileMessagesAreDropped(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const (
|
|
||||||
lines = 5
|
|
||||||
wantDropped = lines - 2
|
|
||||||
maxWait = 5 * time.Second
|
|
||||||
)
|
|
||||||
|
|
||||||
srv := httptest.NewServer(http.HandlerFunc(
|
|
||||||
func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
_, _ = io.WriteString(w, strings.Repeat(updateLine, lines))
|
|
||||||
|
|
||||||
flush(w)
|
|
||||||
<-r.Context().Done()
|
|
||||||
}))
|
|
||||||
defer srv.Close()
|
|
||||||
|
|
||||||
s := New(logger.New(), metrics.New())
|
|
||||||
s.url = srv.URL
|
|
||||||
handler := blockedHandler{release: make(chan struct{})}
|
|
||||||
s.RegisterHandler(handler)
|
|
||||||
|
|
||||||
err := s.Start(t.Context())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Start returned error: %v", err)
|
|
||||||
}
|
|
||||||
defer s.Stop(t.Context())
|
|
||||||
defer close(handler.release)
|
|
||||||
|
|
||||||
// The test polls instead of waiting for a signal from the read loop: a
|
|
||||||
// signal would order the drops before the reads here, and the race
|
|
||||||
// detector would have nothing to report.
|
|
||||||
deadline := time.Now().Add(maxWait)
|
|
||||||
|
|
||||||
for s.GetHandlerStats()[0].DroppedCount < wantDropped {
|
|
||||||
if time.Now().After(deadline) {
|
|
||||||
t.Fatalf("fewer than %d messages counted as dropped within %s",
|
|
||||||
wantDropped, maxWait)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPeerCountCountsEveryPeerHeardFrom has the server send an UPDATE from
|
|
||||||
// stateDownPeer and a KEEPALIVE from another peer, with no OPEN from either,
|
|
||||||
// and then a message that ends stateDownPeer's session: a NOTIFICATION, or a
|
|
||||||
// STATE message whose state is down. The peer count must be 2 after the first
|
|
||||||
// two messages and 1 after the third.
|
|
||||||
func TestPeerCountCountsEveryPeerHeardFrom(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const (
|
|
||||||
otherPeer = "192.0.2.2"
|
|
||||||
peerUpdateLine = `{"type":"ris_message","data":{"type":"UPDATE",` +
|
|
||||||
`"peer":"` + stateDownPeer + `",` +
|
|
||||||
`"peer_asn":"64496","timestamp":1700000000}}` + "\n"
|
|
||||||
otherPeerKeepaliveLine = `{"type":"ris_message","data":{"type":"KEEPALIVE",` +
|
|
||||||
`"peer":"` + otherPeer + `",` +
|
|
||||||
`"peer_asn":"64497","timestamp":1700000000}}` + "\n"
|
|
||||||
peerNotificationLine = `{"type":"ris_message","data":{"type":"NOTIFICATION",` +
|
|
||||||
`"peer":"` + stateDownPeer + `",` +
|
|
||||||
`"peer_asn":"64496","timestamp":1700000000}}` + "\n"
|
|
||||||
bothPeers = 2
|
|
||||||
)
|
|
||||||
|
|
||||||
sessionEndLines := map[string]string{
|
|
||||||
"NOTIFICATION": peerNotificationLine,
|
|
||||||
"STATE down": stateDownLine,
|
|
||||||
}
|
|
||||||
|
|
||||||
for name, sessionEndLine := range sessionEndLines {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
tracker := metrics.New()
|
|
||||||
s := New(logger.New(), tracker)
|
|
||||||
|
|
||||||
streamLines(t, s, peerUpdateLine+otherPeerKeepaliveLine)
|
|
||||||
|
|
||||||
got := tracker.GetBGPPeerCount()
|
|
||||||
if got != bothPeers {
|
|
||||||
t.Fatalf("peer count is %d after messages from two peers, want %d",
|
|
||||||
got, bothPeers)
|
|
||||||
}
|
|
||||||
|
|
||||||
streamLines(t, s, sessionEndLine)
|
|
||||||
|
|
||||||
got = tracker.GetBGPPeerCount()
|
|
||||||
if got != 1 {
|
|
||||||
t.Errorf("peer count is %d after one of two peers' sessions "+
|
|
||||||
"ended, want 1", got)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// streamLines has a server send lines as its whole response and runs one
|
|
||||||
// stream of s against it.
|
|
||||||
func streamLines(t *testing.T, s *Streamer, lines string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
srv := httptest.NewServer(http.HandlerFunc(
|
|
||||||
func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
_, _ = io.WriteString(w, lines)
|
|
||||||
}))
|
|
||||||
defer srv.Close()
|
|
||||||
|
|
||||||
s.url = srv.URL
|
|
||||||
|
|
||||||
err := s.stream(context.Background())
|
|
||||||
if !errors.Is(err, errStreamEnded) {
|
|
||||||
t.Fatalf("stream returned %v, want %v", err, errStreamEnded)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkReconnect runs a streamer against a fake RIS Live server. The first
|
|
||||||
// response is served by firstResponse; every later one carries one UPDATE
|
|
||||||
// line and then stays open. The streamer must come back for a second
|
|
||||||
// response, must not be marked connected when it does, and must handle the
|
|
||||||
// UPDATE in it.
|
|
||||||
func checkReconnect(
|
|
||||||
t *testing.T, firstResponse http.HandlerFunc, idleTimeout time.Duration,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
const maxWait = 5 * time.Second
|
|
||||||
|
|
||||||
tracker := metrics.New()
|
|
||||||
connectedAtReconnect := make(chan bool, 1)
|
|
||||||
|
|
||||||
var requests atomic.Int32
|
|
||||||
|
|
||||||
srv := httptest.NewServer(http.HandlerFunc(
|
|
||||||
func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if requests.Add(1) == 1 {
|
|
||||||
firstResponse(w, r)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
select {
|
|
||||||
case connectedAtReconnect <- tracker.IsConnected():
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
_, _ = io.WriteString(w, updateLine)
|
|
||||||
|
|
||||||
flush(w)
|
|
||||||
<-r.Context().Done()
|
|
||||||
}))
|
|
||||||
defer srv.Close()
|
|
||||||
|
|
||||||
s := New(logger.New(), tracker)
|
|
||||||
s.url = srv.URL
|
|
||||||
s.idleTimeout = idleTimeout
|
|
||||||
s.minBackoff = time.Millisecond
|
|
||||||
handler := updateHandler{handled: make(chan struct{}, 1)}
|
|
||||||
s.RegisterHandler(handler)
|
|
||||||
|
|
||||||
err := s.Start(t.Context())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Start returned error: %v", err)
|
|
||||||
}
|
|
||||||
defer s.Stop(t.Context())
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-handler.handled:
|
|
||||||
case <-time.After(maxWait):
|
|
||||||
t.Fatalf("no message from a second response handled within %s", maxWait)
|
|
||||||
}
|
|
||||||
|
|
||||||
if <-connectedAtReconnect {
|
|
||||||
t.Error("streamer was still marked connected when it came back " +
|
|
||||||
"for a second response")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// settledGoroutineCount lets transient goroutines finish, then reports the
|
|
||||||
// current count.
|
|
||||||
func settledGoroutineCount() int {
|
|
||||||
prev := runtime.NumGoroutine()
|
|
||||||
|
|
||||||
for range 20 {
|
|
||||||
time.Sleep(10 * time.Millisecond)
|
|
||||||
|
|
||||||
cur := runtime.NumGoroutine()
|
|
||||||
if cur == prev {
|
|
||||||
return cur
|
|
||||||
}
|
|
||||||
|
|
||||||
prev = cur
|
|
||||||
}
|
|
||||||
|
|
||||||
return prev
|
|
||||||
}
|
|
||||||
|
|
||||||
// waitForGoroutines waits until the goroutine count drops to target or below.
|
|
||||||
func waitForGoroutines(target int) bool {
|
|
||||||
for range 100 {
|
|
||||||
if runtime.NumGoroutine() <= target {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
time.Sleep(10 * time.Millisecond)
|
|
||||||
}
|
|
||||||
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,146 @@
|
|||||||
|
package streamer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"runtime"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.eeqj.de/sneak/routewatch/internal/logger"
|
||||||
|
"git.eeqj.de/sneak/routewatch/internal/metrics"
|
||||||
|
"git.eeqj.de/sneak/routewatch/internal/ristypes"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNewStreamer(t *testing.T) {
|
||||||
|
logger := logger.New()
|
||||||
|
metricsTracker := metrics.New()
|
||||||
|
s := New(logger, metricsTracker)
|
||||||
|
|
||||||
|
if s == nil {
|
||||||
|
t.Fatal("New() returned nil")
|
||||||
|
}
|
||||||
|
|
||||||
|
if s.logger != logger {
|
||||||
|
t.Error("logger not set correctly")
|
||||||
|
}
|
||||||
|
|
||||||
|
if s.client == nil {
|
||||||
|
t.Error("HTTP client not initialized")
|
||||||
|
}
|
||||||
|
|
||||||
|
if s.handlers == nil {
|
||||||
|
t.Error("handlers slice not initialized")
|
||||||
|
}
|
||||||
|
|
||||||
|
if s.metrics != metricsTracker {
|
||||||
|
t.Error("metrics tracker not set correctly")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestStreamDoesNotLeakTickersAcrossReconnects drives many short-lived
|
||||||
|
// connections (each stream call is one reconnect cycle) and asserts the
|
||||||
|
// goroutine count returns to its starting value. Each connection starts two
|
||||||
|
// ticker goroutines; before the fix they lived until the streamer's lifetime
|
||||||
|
// context was cancelled, so every reconnect leaked two.
|
||||||
|
func TestStreamDoesNotLeakTickersAcrossReconnects(t *testing.T) {
|
||||||
|
// The handler returns immediately, so the response body is empty and each
|
||||||
|
// stream call ends at once, standing in for a dropped connection.
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
s := New(logger.New(), metrics.New())
|
||||||
|
s.url = srv.URL
|
||||||
|
|
||||||
|
// One warm-up connection so any persistent HTTP transport goroutine exists
|
||||||
|
// before we take the baseline.
|
||||||
|
if err := s.stream(context.Background()); err != nil {
|
||||||
|
t.Fatalf("warm-up stream returned error: %v", err)
|
||||||
|
}
|
||||||
|
s.client.CloseIdleConnections()
|
||||||
|
|
||||||
|
baseline := settledGoroutineCount()
|
||||||
|
|
||||||
|
const reconnects = 20
|
||||||
|
for range reconnects {
|
||||||
|
if err := s.stream(context.Background()); err != nil {
|
||||||
|
t.Fatalf("stream returned error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
s.client.CloseIdleConnections()
|
||||||
|
|
||||||
|
if !waitForGoroutines(baseline) {
|
||||||
|
t.Fatalf("goroutines did not return to baseline %d after %d reconnects, got %d",
|
||||||
|
baseline, reconnects, runtime.NumGoroutine())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// updateHandler wants UPDATE messages and does nothing with them.
|
||||||
|
type updateHandler struct{}
|
||||||
|
|
||||||
|
func (updateHandler) WantsMessage(messageType string) bool { return messageType == "UPDATE" }
|
||||||
|
func (updateHandler) HandleMessage(*ristypes.RISMessage) {}
|
||||||
|
func (updateHandler) QueueCapacity() int { return 10 }
|
||||||
|
|
||||||
|
// TestStopBeforeMessageReachesQueues stops the streamer after the read loop
|
||||||
|
// has checked for cancellation but before it hands the message to the handler
|
||||||
|
// queues. That is the gap a stop from another goroutine can land in, and it
|
||||||
|
// used to end in "send on closed channel". The raw handler runs in that gap on
|
||||||
|
// the read loop itself, so calling Stop from it hits the gap every time.
|
||||||
|
func TestStopBeforeMessageReachesQueues(t *testing.T) {
|
||||||
|
const line = `{"type":"ris_message","data":{"type":"UPDATE","peer":"192.0.2.1",` +
|
||||||
|
`"peer_asn":"64496","timestamp":1700000000}}` + "\n"
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
_, _ = io.WriteString(w, line)
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
s := New(logger.New(), metrics.New())
|
||||||
|
s.url = srv.URL
|
||||||
|
s.RegisterHandler(updateHandler{})
|
||||||
|
s.RegisterRawHandler(func(string) { s.Stop() })
|
||||||
|
|
||||||
|
// Start would run the stream in the background, where the test cannot
|
||||||
|
// wait for it. Setting cancel as Start does lets Stop cancel the stream
|
||||||
|
// run here instead.
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
s.cancel = cancel
|
||||||
|
|
||||||
|
if err := s.stream(ctx); !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("stream returned %v, want %v", err, context.Canceled)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A second Stop must not close the queues again.
|
||||||
|
s.Stop()
|
||||||
|
}
|
||||||
|
|
||||||
|
// settledGoroutineCount lets transient goroutines finish, then reports the
|
||||||
|
// current count.
|
||||||
|
func settledGoroutineCount() int {
|
||||||
|
prev := runtime.NumGoroutine()
|
||||||
|
for range 20 {
|
||||||
|
time.Sleep(10 * time.Millisecond)
|
||||||
|
cur := runtime.NumGoroutine()
|
||||||
|
if cur == prev {
|
||||||
|
return cur
|
||||||
|
}
|
||||||
|
prev = cur
|
||||||
|
}
|
||||||
|
|
||||||
|
return prev
|
||||||
|
}
|
||||||
|
|
||||||
|
// waitForGoroutines waits until the goroutine count drops to target or below.
|
||||||
|
func waitForGoroutines(target int) bool {
|
||||||
|
for range 100 {
|
||||||
|
if runtime.NumGoroutine() <= target {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
time.Sleep(10 * time.Millisecond)
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
@@ -4,7 +4,7 @@
|
|||||||
<meta charset="UTF-8">
|
<meta charset="UTF-8">
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
<title>AS{{.ASN.ASN}} - {{.ASN.Handle}} - RouteWatch</title>
|
<title>AS{{.ASN.ASN}} - {{.ASN.Handle}} - RouteWatch</title>
|
||||||
<style nonce="{{.CSPNonce}}">
|
<style>
|
||||||
* {
|
* {
|
||||||
box-sizing: border-box;
|
box-sizing: border-box;
|
||||||
}
|
}
|
||||||
@@ -126,26 +126,6 @@
|
|||||||
margin: 0;
|
margin: 0;
|
||||||
color: #2c3e50;
|
color: #2c3e50;
|
||||||
}
|
}
|
||||||
/* A click on a prefix list's heading folds or unfolds the list. Not
|
|
||||||
every browser shows its own marker on a flex element, so the page
|
|
||||||
hides it and draws a triangle before the heading. The triangle
|
|
||||||
points down while the list is open. */
|
|
||||||
summary.prefix-header {
|
|
||||||
cursor: pointer;
|
|
||||||
list-style: none;
|
|
||||||
}
|
|
||||||
summary.prefix-header::-webkit-details-marker {
|
|
||||||
display: none;
|
|
||||||
}
|
|
||||||
summary.prefix-header h2::before {
|
|
||||||
content: "\25B8";
|
|
||||||
display: inline-block;
|
|
||||||
margin-right: 10px;
|
|
||||||
transition: transform 0.2s;
|
|
||||||
}
|
|
||||||
details[open] > summary.prefix-header h2::before {
|
|
||||||
transform: rotate(90deg);
|
|
||||||
}
|
|
||||||
.prefix-count {
|
.prefix-count {
|
||||||
background: #e74c3c;
|
background: #e74c3c;
|
||||||
color: white;
|
color: white;
|
||||||
@@ -253,20 +233,18 @@
|
|||||||
<div class="info-label">Peer ASNs</div>
|
<div class="info-label">Peer ASNs</div>
|
||||||
<div class="info-value">{{.PeerCount}}</div>
|
<div class="info-value">{{.PeerCount}}</div>
|
||||||
</div>
|
</div>
|
||||||
{{if not .ASN.FirstSeen.IsZero}}
|
|
||||||
<div class="info-card">
|
<div class="info-card">
|
||||||
<div class="info-label">First Seen</div>
|
<div class="info-label">First Seen</div>
|
||||||
<div class="info-value">{{.ASN.FirstSeen.Format "2006-01-02"}}</div>
|
<div class="info-value">{{.ASN.FirstSeen.Format "2006-01-02"}}</div>
|
||||||
</div>
|
</div>
|
||||||
{{end}}
|
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{{if .IPv4Prefixes}}
|
{{if .IPv4Prefixes}}
|
||||||
<details class="prefix-section" open>
|
<div class="prefix-section">
|
||||||
<summary class="prefix-header">
|
<div class="prefix-header">
|
||||||
<h2>IPv4 Prefixes</h2>
|
<h2>IPv4 Prefixes</h2>
|
||||||
<span class="prefix-count">{{.IPv4Count}}</span>
|
<span class="prefix-count">{{.IPv4Count}}</span>
|
||||||
</summary>
|
</div>
|
||||||
<table class="prefix-table">
|
<table class="prefix-table">
|
||||||
<thead>
|
<thead>
|
||||||
<tr>
|
<tr>
|
||||||
@@ -281,21 +259,21 @@
|
|||||||
<tr>
|
<tr>
|
||||||
<td><a href="{{.Prefix | prefixURL}}" class="prefix-link">{{.Prefix}}</a></td>
|
<td><a href="{{.Prefix | prefixURL}}" class="prefix-link">{{.Prefix}}</a></td>
|
||||||
<td>/{{.MaskLength}}</td>
|
<td>/{{.MaskLength}}</td>
|
||||||
<td>{{if not .LastUpdated.IsZero}}{{.LastUpdated.Format "2006-01-02 15:04:05"}}{{end}}</td>
|
<td>{{.LastUpdated.Format "2006-01-02 15:04:05"}}</td>
|
||||||
<td class="age">{{if not .LastUpdated.IsZero}}{{.LastUpdated | timeSince}}{{end}}</td>
|
<td class="age">{{.LastUpdated | timeSince}}</td>
|
||||||
</tr>
|
</tr>
|
||||||
{{end}}
|
{{end}}
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
</details>
|
</div>
|
||||||
{{end}}
|
{{end}}
|
||||||
|
|
||||||
{{if .IPv6Prefixes}}
|
{{if .IPv6Prefixes}}
|
||||||
<details class="prefix-section" open>
|
<div class="prefix-section">
|
||||||
<summary class="prefix-header">
|
<div class="prefix-header">
|
||||||
<h2>IPv6 Prefixes</h2>
|
<h2>IPv6 Prefixes</h2>
|
||||||
<span class="prefix-count">{{.IPv6Count}}</span>
|
<span class="prefix-count">{{.IPv6Count}}</span>
|
||||||
</summary>
|
</div>
|
||||||
<table class="prefix-table">
|
<table class="prefix-table">
|
||||||
<thead>
|
<thead>
|
||||||
<tr>
|
<tr>
|
||||||
@@ -310,13 +288,13 @@
|
|||||||
<tr>
|
<tr>
|
||||||
<td><a href="{{.Prefix | prefixURL}}" class="prefix-link">{{.Prefix}}</a></td>
|
<td><a href="{{.Prefix | prefixURL}}" class="prefix-link">{{.Prefix}}</a></td>
|
||||||
<td>/{{.MaskLength}}</td>
|
<td>/{{.MaskLength}}</td>
|
||||||
<td>{{if not .LastUpdated.IsZero}}{{.LastUpdated.Format "2006-01-02 15:04:05"}}{{end}}</td>
|
<td>{{.LastUpdated.Format "2006-01-02 15:04:05"}}</td>
|
||||||
<td class="age">{{if not .LastUpdated.IsZero}}{{.LastUpdated | timeSince}}{{end}}</td>
|
<td class="age">{{.LastUpdated | timeSince}}</td>
|
||||||
</tr>
|
</tr>
|
||||||
{{end}}
|
{{end}}
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
</details>
|
</div>
|
||||||
{{end}}
|
{{end}}
|
||||||
|
|
||||||
{{if eq .TotalCount 0}}
|
{{if eq .TotalCount 0}}
|
||||||
|
|||||||
@@ -1,170 +0,0 @@
|
|||||||
<!DOCTYPE html>
|
|
||||||
<html lang="en">
|
|
||||||
<head>
|
|
||||||
<meta charset="UTF-8">
|
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
||||||
<title>Bogon Announcements - RouteWatch</title>
|
|
||||||
<style nonce="{{.CSPNonce}}">
|
|
||||||
* {
|
|
||||||
box-sizing: border-box;
|
|
||||||
}
|
|
||||||
body {
|
|
||||||
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
|
|
||||||
margin: 0;
|
|
||||||
padding: 0;
|
|
||||||
background: #f5f5f5;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Navbar styles */
|
|
||||||
.navbar {
|
|
||||||
background: #2c3e50;
|
|
||||||
padding: 0 20px;
|
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
justify-content: space-between;
|
|
||||||
height: 56px;
|
|
||||||
box-shadow: 0 2px 4px rgba(0,0,0,0.1);
|
|
||||||
}
|
|
||||||
.navbar-brand {
|
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
gap: 8px;
|
|
||||||
font-size: 18px;
|
|
||||||
font-weight: 600;
|
|
||||||
}
|
|
||||||
.navbar-brand a {
|
|
||||||
color: white;
|
|
||||||
text-decoration: none;
|
|
||||||
}
|
|
||||||
.navbar-brand a:hover {
|
|
||||||
color: #ecf0f1;
|
|
||||||
}
|
|
||||||
.navbar-brand .by {
|
|
||||||
font-weight: normal;
|
|
||||||
color: #95a5a6;
|
|
||||||
font-size: 14px;
|
|
||||||
}
|
|
||||||
.navbar-brand .author {
|
|
||||||
color: #3498db;
|
|
||||||
font-weight: normal;
|
|
||||||
}
|
|
||||||
.navbar-brand .author:hover {
|
|
||||||
text-decoration: underline;
|
|
||||||
}
|
|
||||||
.navbar-links {
|
|
||||||
display: flex;
|
|
||||||
gap: 20px;
|
|
||||||
}
|
|
||||||
.navbar-links a {
|
|
||||||
color: #ecf0f1;
|
|
||||||
text-decoration: none;
|
|
||||||
font-size: 14px;
|
|
||||||
padding: 8px 12px;
|
|
||||||
border-radius: 4px;
|
|
||||||
transition: background-color 0.2s;
|
|
||||||
}
|
|
||||||
.navbar-links a:hover {
|
|
||||||
background: rgba(255,255,255,0.1);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Main content */
|
|
||||||
.main-content {
|
|
||||||
max-width: 1200px;
|
|
||||||
margin: 0 auto;
|
|
||||||
padding: 20px;
|
|
||||||
}
|
|
||||||
h1 {
|
|
||||||
color: #333;
|
|
||||||
margin-bottom: 10px;
|
|
||||||
}
|
|
||||||
.subtitle {
|
|
||||||
color: #666;
|
|
||||||
margin-bottom: 30px;
|
|
||||||
}
|
|
||||||
table {
|
|
||||||
width: 100%;
|
|
||||||
border-collapse: collapse;
|
|
||||||
background: white;
|
|
||||||
box-shadow: 0 2px 4px rgba(0,0,0,0.1);
|
|
||||||
border-radius: 8px;
|
|
||||||
overflow: hidden;
|
|
||||||
}
|
|
||||||
th {
|
|
||||||
background: #f8f9fa;
|
|
||||||
padding: 12px;
|
|
||||||
text-align: left;
|
|
||||||
font-weight: 600;
|
|
||||||
color: #333;
|
|
||||||
border-bottom: 2px solid #dee2e6;
|
|
||||||
}
|
|
||||||
td {
|
|
||||||
padding: 12px;
|
|
||||||
border-bottom: 1px solid #eee;
|
|
||||||
}
|
|
||||||
tr:last-child td {
|
|
||||||
border-bottom: none;
|
|
||||||
}
|
|
||||||
tr:hover {
|
|
||||||
background: #f8f9fa;
|
|
||||||
}
|
|
||||||
a {
|
|
||||||
color: #0066cc;
|
|
||||||
text-decoration: none;
|
|
||||||
}
|
|
||||||
a:hover {
|
|
||||||
text-decoration: underline;
|
|
||||||
}
|
|
||||||
.prefix-link {
|
|
||||||
font-family: 'SF Mono', Monaco, 'Cascadia Mono', 'Roboto Mono', Consolas, 'Courier New', monospace;
|
|
||||||
}
|
|
||||||
</style>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<nav class="navbar">
|
|
||||||
<div class="navbar-brand">
|
|
||||||
<a href="/">routewatch</a>
|
|
||||||
<span class="by">by</span>
|
|
||||||
<a href="https://sneak.berlin" class="author">@sneak</a>
|
|
||||||
</div>
|
|
||||||
<div class="navbar-links">
|
|
||||||
<a href="/">Home</a>
|
|
||||||
<a href="/status">Status</a>
|
|
||||||
</div>
|
|
||||||
</nav>
|
|
||||||
|
|
||||||
<main class="main-content">
|
|
||||||
<h1>Bogon Announcements</h1>
|
|
||||||
{{ with .Cards }}
|
|
||||||
<p class="subtitle">
|
|
||||||
Bogon announcements: {{ .BogonCount }}.
|
|
||||||
A bogon announcement is a route that announces a reserved prefix, or that comes from a reserved AS number.
|
|
||||||
{{ if lt (len .Bogons) .BogonCount }}The first {{ len .Bogons }} are listed, in prefix order.{{ end }}
|
|
||||||
The list is worked out once a minute.
|
|
||||||
</p>
|
|
||||||
|
|
||||||
<table>
|
|
||||||
<thead>
|
|
||||||
<tr>
|
|
||||||
<th>Prefix</th>
|
|
||||||
<th>Origin AS</th>
|
|
||||||
<th>Peers</th>
|
|
||||||
<th>Why it is a bogon</th>
|
|
||||||
</tr>
|
|
||||||
</thead>
|
|
||||||
<tbody>
|
|
||||||
{{ range .Bogons }}
|
|
||||||
<tr>
|
|
||||||
<td><a href="{{ prefixURL .Prefix.String }}" class="prefix-link">{{ .Prefix }}</a></td>
|
|
||||||
<td>{{ if .OriginASN }}<a href="/as/{{ .OriginASN }}">AS{{ .OriginASN }}</a>{{ else }}No known origin{{ end }}</td>
|
|
||||||
<td>{{ .Peers }}</td>
|
|
||||||
<td>{{ range $i, $reason := .Reasons }}{{ if $i }}; {{ end }}{{ $reason }}{{ end }}</td>
|
|
||||||
</tr>
|
|
||||||
{{ end }}
|
|
||||||
</tbody>
|
|
||||||
</table>
|
|
||||||
{{ else }}
|
|
||||||
<p class="subtitle">The list is being worked out. Reload the page in a minute.</p>
|
|
||||||
{{ end }}
|
|
||||||
</main>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
+31
-363
@@ -4,7 +4,7 @@
|
|||||||
<meta charset="UTF-8">
|
<meta charset="UTF-8">
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
<title>RouteWatch - BGP Route Monitor</title>
|
<title>RouteWatch - BGP Route Monitor</title>
|
||||||
<style nonce="{{.CSPNonce}}">
|
<style>
|
||||||
* {
|
* {
|
||||||
box-sizing: border-box;
|
box-sizing: border-box;
|
||||||
}
|
}
|
||||||
@@ -107,15 +107,9 @@
|
|||||||
.stat-card.connected .stat-value {
|
.stat-card.connected .stat-value {
|
||||||
color: #27ae60;
|
color: #27ae60;
|
||||||
}
|
}
|
||||||
.stat-card.disconnected .stat-value,
|
.stat-card.disconnected .stat-value {
|
||||||
.stale {
|
|
||||||
color: #e74c3c;
|
color: #e74c3c;
|
||||||
}
|
}
|
||||||
/* The card has no room beside "Connected", so the spinner and "stale" go under it */
|
|
||||||
#freshness {
|
|
||||||
display: block;
|
|
||||||
font-size: 20px;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Search section */
|
/* Search section */
|
||||||
.search-section {
|
.search-section {
|
||||||
@@ -173,45 +167,6 @@
|
|||||||
color: #95a5a6;
|
color: #95a5a6;
|
||||||
margin-top: 8px;
|
margin-top: 8px;
|
||||||
}
|
}
|
||||||
#asname-results p {
|
|
||||||
color: #7f8c8d;
|
|
||||||
margin-top: 12px;
|
|
||||||
}
|
|
||||||
#asname-results .search-message {
|
|
||||||
font-size: 13px;
|
|
||||||
}
|
|
||||||
#asname-results .error {
|
|
||||||
color: #c00;
|
|
||||||
}
|
|
||||||
.asname-list {
|
|
||||||
list-style: none;
|
|
||||||
margin: 12px 0 0 0;
|
|
||||||
padding: 0;
|
|
||||||
max-height: 400px;
|
|
||||||
overflow-y: auto;
|
|
||||||
}
|
|
||||||
.asname-list li {
|
|
||||||
border-bottom: 1px solid #eee;
|
|
||||||
}
|
|
||||||
.asname-list a {
|
|
||||||
display: block;
|
|
||||||
padding: 8px 4px;
|
|
||||||
color: #2c3e50;
|
|
||||||
text-decoration: none;
|
|
||||||
font-size: 14px;
|
|
||||||
}
|
|
||||||
.asname-list a:hover {
|
|
||||||
background: #f5f5f5;
|
|
||||||
}
|
|
||||||
.asname-list .asn {
|
|
||||||
font-weight: 600;
|
|
||||||
color: #3498db;
|
|
||||||
}
|
|
||||||
.asname-list .description {
|
|
||||||
display: block;
|
|
||||||
color: #7f8c8d;
|
|
||||||
font-size: 12px;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* IP Lookup result */
|
/* IP Lookup result */
|
||||||
.ip-result {
|
.ip-result {
|
||||||
@@ -239,8 +194,7 @@
|
|||||||
cursor: pointer;
|
cursor: pointer;
|
||||||
font-size: 12px;
|
font-size: 12px;
|
||||||
}
|
}
|
||||||
.ip-result pre,
|
.ip-result pre {
|
||||||
#ip-api pre {
|
|
||||||
background: #2c3e50;
|
background: #2c3e50;
|
||||||
color: #ecf0f1;
|
color: #ecf0f1;
|
||||||
padding: 16px;
|
padding: 16px;
|
||||||
@@ -252,76 +206,18 @@
|
|||||||
max-height: 400px;
|
max-height: 400px;
|
||||||
overflow-y: auto;
|
overflow-y: auto;
|
||||||
}
|
}
|
||||||
.ip-result .error,
|
.ip-result .error {
|
||||||
#ip-api .error {
|
|
||||||
background: #fee;
|
background: #fee;
|
||||||
color: #c00;
|
color: #c00;
|
||||||
padding: 12px;
|
padding: 12px;
|
||||||
border-radius: 6px;
|
border-radius: 6px;
|
||||||
font-size: 14px;
|
font-size: 14px;
|
||||||
}
|
}
|
||||||
.ip-result .loading,
|
.ip-result .loading {
|
||||||
#ip-api .loading {
|
|
||||||
color: #7f8c8d;
|
color: #7f8c8d;
|
||||||
font-style: italic;
|
font-style: italic;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* IP information API card: the command, then its answer */
|
|
||||||
#ip-api p {
|
|
||||||
margin: 0;
|
|
||||||
color: #7f8c8d;
|
|
||||||
font-size: 14px;
|
|
||||||
}
|
|
||||||
#ip-api pre {
|
|
||||||
margin-top: 12px;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Cards with figures of the routing table, worked out once a minute */
|
|
||||||
#table-cards p,
|
|
||||||
#well-known p {
|
|
||||||
margin: 0 0 8px 0;
|
|
||||||
font-size: 14px;
|
|
||||||
}
|
|
||||||
#table-cards a,
|
|
||||||
#well-known a {
|
|
||||||
color: #3498db;
|
|
||||||
text-decoration: none;
|
|
||||||
}
|
|
||||||
#table-cards a:hover,
|
|
||||||
#well-known a:hover {
|
|
||||||
text-decoration: underline;
|
|
||||||
}
|
|
||||||
#table-cards .card-figure {
|
|
||||||
font-size: 32px;
|
|
||||||
font-weight: bold;
|
|
||||||
}
|
|
||||||
#table-cards .card-detail,
|
|
||||||
#table-cards .card-time {
|
|
||||||
color: #7f8c8d;
|
|
||||||
font-size: 12px;
|
|
||||||
}
|
|
||||||
#table-cards .prefix {
|
|
||||||
font-family: 'SF Mono', Monaco, 'Cascadia Mono', 'Roboto Mono', Consolas, 'Courier New', monospace;
|
|
||||||
}
|
|
||||||
#well-known {
|
|
||||||
margin-bottom: 40px;
|
|
||||||
}
|
|
||||||
#well-known table {
|
|
||||||
width: 100%;
|
|
||||||
border-collapse: collapse;
|
|
||||||
font-size: 14px;
|
|
||||||
}
|
|
||||||
#well-known th,
|
|
||||||
#well-known td {
|
|
||||||
text-align: left;
|
|
||||||
padding: 8px 4px;
|
|
||||||
border-bottom: 1px solid #eee;
|
|
||||||
}
|
|
||||||
#well-known th {
|
|
||||||
color: #7f8c8d;
|
|
||||||
font-weight: 600;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Footer */
|
/* Footer */
|
||||||
.footer {
|
.footer {
|
||||||
margin-top: 40px;
|
margin-top: 40px;
|
||||||
@@ -362,10 +258,7 @@
|
|||||||
<main class="main-content">
|
<main class="main-content">
|
||||||
<div class="stats-grid">
|
<div class="stats-grid">
|
||||||
<div class="stat-card" id="status-card">
|
<div class="stat-card" id="status-card">
|
||||||
<div class="stat-value">
|
<div class="stat-value" id="stat-status">-</div>
|
||||||
<span id="stat-status">-</span>
|
|
||||||
<span id="freshness"></span>
|
|
||||||
</div>
|
|
||||||
<div class="stat-label">Status</div>
|
<div class="stat-label">Status</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="stat-card">
|
<div class="stat-card">
|
||||||
@@ -406,7 +299,7 @@
|
|||||||
<input type="text" id="asname-input" placeholder="e.g., Google, Cloudflare" autocomplete="off">
|
<input type="text" id="asname-input" placeholder="e.g., Google, Cloudflare" autocomplete="off">
|
||||||
<button type="submit">Search</button>
|
<button type="submit">Search</button>
|
||||||
</form>
|
</form>
|
||||||
<p class="search-hint">Search for autonomous systems by organization name or handle</p>
|
<p class="search-hint">Search for autonomous systems by organization name</p>
|
||||||
<div id="asname-results"></div>
|
<div id="asname-results"></div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
@@ -426,116 +319,6 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{{ with .Cards }}
|
|
||||||
<div class="search-section" id="table-cards">
|
|
||||||
<div class="search-card">
|
|
||||||
<h2>Newest Updated Route</h2>
|
|
||||||
{{ with .NewestRoute }}
|
|
||||||
<p><a href="{{ prefixURL .Prefix.String }}" class="prefix">{{ .Prefix }}</a> from {{ if .OriginASN }}<a href="/as/{{ .OriginASN }}">AS{{ .OriginASN }}</a>{{ else }}no known origin{{ end }}</p>
|
|
||||||
<p class="card-time">{{ .Time.UTC.Format "2006-01-02 15:04:05" }} UTC, {{ timeSince .Time }}</p>
|
|
||||||
{{ else }}
|
|
||||||
<p>The routing table keeps no route yet.</p>
|
|
||||||
{{ end }}
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="search-card">
|
|
||||||
<h2>Oldest Updated Route</h2>
|
|
||||||
{{ with .OldestRoute }}
|
|
||||||
<p><a href="{{ prefixURL .Prefix.String }}" class="prefix">{{ .Prefix }}</a> from {{ if .OriginASN }}<a href="/as/{{ .OriginASN }}">AS{{ .OriginASN }}</a>{{ else }}no known origin{{ end }}</p>
|
|
||||||
<p class="card-time">{{ .Time.UTC.Format "2006-01-02 15:04:05" }} UTC, {{ timeSince .Time }}</p>
|
|
||||||
{{ else }}
|
|
||||||
<p>The routing table keeps no route yet.</p>
|
|
||||||
{{ end }}
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="search-card">
|
|
||||||
<h2>Newest Updated AS</h2>
|
|
||||||
{{ with .NewestAS }}
|
|
||||||
<p><a href="/as/{{ .ASN }}">AS{{ .ASN }}</a> {{ .Handle }}</p>
|
|
||||||
<p class="card-detail">{{ .Description }}</p>
|
|
||||||
<p class="card-time">Last seen {{ .LastSeen.UTC.Format "2006-01-02 15:04:05" }} UTC, {{ timeSince .LastSeen }}</p>
|
|
||||||
{{ else }}
|
|
||||||
<p>No AS has been seen yet.</p>
|
|
||||||
{{ end }}
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="search-card">
|
|
||||||
<h2>Oldest Updated AS</h2>
|
|
||||||
{{ with .OldestAS }}
|
|
||||||
<p><a href="/as/{{ .ASN }}">AS{{ .ASN }}</a> {{ .Handle }}</p>
|
|
||||||
<p class="card-detail">{{ .Description }}</p>
|
|
||||||
<p class="card-time">Last seen {{ .LastSeen.UTC.Format "2006-01-02 15:04:05" }} UTC, {{ timeSince .LastSeen }}</p>
|
|
||||||
{{ else }}
|
|
||||||
<p>No AS has been seen yet.</p>
|
|
||||||
{{ end }}
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="search-card">
|
|
||||||
<h2>AS with the Most Prefixes</h2>
|
|
||||||
{{ if .MostPrefixes }}
|
|
||||||
<p><a href="/as/{{ .MostPrefixesASN }}">AS{{ .MostPrefixesASN }}</a></p>
|
|
||||||
<p>Prefixes it originates: {{ .MostPrefixes }}</p>
|
|
||||||
{{ else }}
|
|
||||||
<p>No AS originates a prefix yet.</p>
|
|
||||||
{{ end }}
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="search-card">
|
|
||||||
<h2>ASes with the Fewest Prefixes</h2>
|
|
||||||
<p>ASes that originate a single prefix: {{ .SinglePrefixASes }}</p>
|
|
||||||
{{ with .NewestSinglePrefix }}
|
|
||||||
<p>The most recently updated of them: <a href="/as/{{ .OriginASN }}">AS{{ .OriginASN }}</a>, with <a href="{{ prefixURL .Prefix.String }}" class="prefix">{{ .Prefix }}</a></p>
|
|
||||||
<p class="card-time">{{ .Time.UTC.Format "2006-01-02 15:04:05" }} UTC, {{ timeSince .Time }}</p>
|
|
||||||
{{ end }}
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="search-card">
|
|
||||||
<h2>Bogon Announcements</h2>
|
|
||||||
<p class="card-figure"><a href="/bogons">{{ .BogonCount }}</a></p>
|
|
||||||
<p>Routes that announce a reserved prefix, or that come from a reserved AS number</p>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="search-card">
|
|
||||||
<h2>Prefixes with Several Origins</h2>
|
|
||||||
<p class="card-figure"><a href="/multiple-origins">{{ .MultipleOriginCount }}</a></p>
|
|
||||||
<p>Prefixes announced by more than one origin AS</p>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="search-card" id="well-known">
|
|
||||||
<h2>Well-Known Networks</h2>
|
|
||||||
<table>
|
|
||||||
<thead>
|
|
||||||
<tr>
|
|
||||||
<th>Network</th>
|
|
||||||
<th>Prefixes</th>
|
|
||||||
<th>Last Update</th>
|
|
||||||
</tr>
|
|
||||||
</thead>
|
|
||||||
<tbody>
|
|
||||||
{{ range .WellKnownNetworks }}
|
|
||||||
<tr>
|
|
||||||
<td><a href="/as/{{ .ASN }}">AS{{ .ASN }}</a> {{ .Name }}</td>
|
|
||||||
<td>{{ .Prefixes }}</td>
|
|
||||||
<td>{{ with .Newest }}{{ timeSince . }}{{ else }}no route{{ end }}</td>
|
|
||||||
</tr>
|
|
||||||
{{ end }}
|
|
||||||
</tbody>
|
|
||||||
</table>
|
|
||||||
</div>
|
|
||||||
{{ else }}
|
|
||||||
<div class="search-card" id="well-known">
|
|
||||||
<p>The figures of the routing table are being worked out. Reload the page in a minute.</p>
|
|
||||||
</div>
|
|
||||||
{{ end }}
|
|
||||||
|
|
||||||
<div class="search-card" id="ip-api">
|
|
||||||
<h2>IP Information API</h2>
|
|
||||||
<p>The API answers what RouteWatch knows about an IP address, such as its AS number and country. <code>/ip/</code> with no address answers for the caller's own address.</p>
|
|
||||||
<pre id="ip-api-command" data-client-ip="{{.ClientIP}}"></pre>
|
|
||||||
<pre id="ip-api-answer" class="loading">Loading...</pre>
|
|
||||||
</div>
|
|
||||||
</main>
|
</main>
|
||||||
|
|
||||||
<footer class="footer">
|
<footer class="footer">
|
||||||
@@ -546,45 +329,7 @@
|
|||||||
<span><a href="{{appGitCommitURL}}">{{appGitRevision}}</a></span>
|
<span><a href="{{appGitCommitURL}}">{{appGitRevision}}</a></span>
|
||||||
</footer>
|
</footer>
|
||||||
|
|
||||||
<script nonce="{{.CSPNonce}}">
|
<script>
|
||||||
// The stats are fetched every REFRESH_PERIOD_MS. Next to the status, a
|
|
||||||
// spinner turns one frame each time an answer counts more messages
|
|
||||||
// than the one before. After STALE_AFTER_REFRESHES refreshes without
|
|
||||||
// such an answer, the spinner gives way to "stale" and the time since.
|
|
||||||
const REFRESH_PERIOD_MS = 5000;
|
|
||||||
const STALE_AFTER_REFRESHES = 3;
|
|
||||||
const MS_PER_SECOND = 1000;
|
|
||||||
const SPINNER_FRAMES = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];
|
|
||||||
const SPINNER_COLOURS = ['#e74c3c', '#e67e22', '#f1c40f', '#27ae60', '#3498db', '#9b59b6'];
|
|
||||||
|
|
||||||
let spinnerFrame = 0;
|
|
||||||
let lastMessageCount = null;
|
|
||||||
let lastFreshAt = Date.now();
|
|
||||||
|
|
||||||
// Turn the spinner when the message count has risen since the last answer.
|
|
||||||
function noteMessageCount(count) {
|
|
||||||
if (lastMessageCount !== null && count > lastMessageCount) {
|
|
||||||
spinnerFrame = (spinnerFrame + 1) % SPINNER_FRAMES.length;
|
|
||||||
lastFreshAt = Date.now();
|
|
||||||
}
|
|
||||||
lastMessageCount = count;
|
|
||||||
showFreshness();
|
|
||||||
}
|
|
||||||
|
|
||||||
function showFreshness() {
|
|
||||||
const freshness = document.getElementById('freshness');
|
|
||||||
const sinceFreshMs = Date.now() - lastFreshAt;
|
|
||||||
const stale = sinceFreshMs >= STALE_AFTER_REFRESHES * REFRESH_PERIOD_MS;
|
|
||||||
freshness.classList.toggle('stale', stale);
|
|
||||||
if (stale) {
|
|
||||||
freshness.textContent = 'stale for ' + Math.floor(sinceFreshMs / MS_PER_SECOND) + 's';
|
|
||||||
freshness.style.color = '';
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
freshness.textContent = SPINNER_FRAMES[spinnerFrame];
|
|
||||||
freshness.style.color = SPINNER_COLOURS[spinnerFrame % SPINNER_COLOURS.length];
|
|
||||||
}
|
|
||||||
|
|
||||||
function formatNumber(num) {
|
function formatNumber(num) {
|
||||||
if (num >= 1000000) {
|
if (num >= 1000000) {
|
||||||
return (num / 1000000).toFixed(1) + 'M';
|
return (num / 1000000).toFixed(1) + 'M';
|
||||||
@@ -606,7 +351,6 @@
|
|||||||
const statusEl = document.getElementById('stat-status');
|
const statusEl = document.getElementById('stat-status');
|
||||||
statusEl.textContent = data.connected ? 'Connected' : 'Disconnected';
|
statusEl.textContent = data.connected ? 'Connected' : 'Disconnected';
|
||||||
statusCard.className = 'stat-card ' + (data.connected ? 'connected' : 'disconnected');
|
statusCard.className = 'stat-card ' + (data.connected ? 'connected' : 'disconnected');
|
||||||
noteMessageCount(data.total_messages);
|
|
||||||
|
|
||||||
document.getElementById('stat-routes').textContent = formatNumber(data.live_routes);
|
document.getElementById('stat-routes').textContent = formatNumber(data.live_routes);
|
||||||
document.getElementById('stat-asns').textContent = formatNumber(data.asns);
|
document.getElementById('stat-asns').textContent = formatNumber(data.asns);
|
||||||
@@ -617,7 +361,7 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
const totalUpdates = data.ipv4_updates_per_sec + data.ipv6_updates_per_sec;
|
const totalUpdates = data.ipv4_updates_per_sec + data.ipv6_updates_per_sec;
|
||||||
document.getElementById('stat-updates').textContent = Math.round(totalUpdates).toLocaleString();
|
document.getElementById('stat-updates').textContent = totalUpdates.toFixed(1);
|
||||||
})
|
})
|
||||||
.catch(() => {
|
.catch(() => {
|
||||||
document.getElementById('stat-status').textContent = 'Error';
|
document.getElementById('stat-status').textContent = 'Error';
|
||||||
@@ -636,86 +380,27 @@
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
// AS name search. The handles and descriptions come from outside data,
|
// AS name search
|
||||||
// so the results are built with textContent, never innerHTML.
|
|
||||||
const asNameResults = document.getElementById('asname-results');
|
|
||||||
|
|
||||||
// Replace the search results with one line of text.
|
|
||||||
function showASNameMessage(text, className) {
|
|
||||||
const line = document.createElement('p');
|
|
||||||
line.className = className;
|
|
||||||
line.textContent = text;
|
|
||||||
asNameResults.replaceChildren(line);
|
|
||||||
}
|
|
||||||
|
|
||||||
// List the matches of a search answer, each as a link to its AS page,
|
|
||||||
// and say how many more matched than the answer holds.
|
|
||||||
function showASNameMatches(data) {
|
|
||||||
if (data.total === 0) {
|
|
||||||
showASNameMessage('No AS matches "' + data.query + '".', 'search-message');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const list = document.createElement('ul');
|
|
||||||
list.className = 'asname-list';
|
|
||||||
for (const match of data.results) {
|
|
||||||
const asn = document.createElement('span');
|
|
||||||
asn.className = 'asn';
|
|
||||||
asn.textContent = 'AS' + match.asn;
|
|
||||||
|
|
||||||
const description = document.createElement('span');
|
|
||||||
description.className = 'description';
|
|
||||||
description.textContent = match.description;
|
|
||||||
|
|
||||||
const link = document.createElement('a');
|
|
||||||
link.href = '/as/' + match.asn;
|
|
||||||
link.append(asn, ' ' + match.handle, description);
|
|
||||||
|
|
||||||
const item = document.createElement('li');
|
|
||||||
item.append(link);
|
|
||||||
list.append(item);
|
|
||||||
}
|
|
||||||
asNameResults.replaceChildren(list);
|
|
||||||
|
|
||||||
const more = data.total - data.results.length;
|
|
||||||
if (more > 0) {
|
|
||||||
const line = document.createElement('p');
|
|
||||||
line.className = 'search-message';
|
|
||||||
line.textContent = more.toLocaleString() + ' more matched. Narrow the search to see them.';
|
|
||||||
asNameResults.append(line);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
document.getElementById('asname-form').addEventListener('submit', function(e) {
|
document.getElementById('asname-form').addEventListener('submit', function(e) {
|
||||||
e.preventDefault();
|
e.preventDefault();
|
||||||
const query = document.getElementById('asname-input').value.trim();
|
const query = document.getElementById('asname-input').value.trim();
|
||||||
if (!query) return;
|
if (!query) return;
|
||||||
|
|
||||||
// A number, or AS and a number, opens that AS's page
|
const resultsDiv = document.getElementById('asname-results');
|
||||||
const asn = query.replace(/^AS/i, '');
|
resultsDiv.innerHTML = '<p class="loading" style="color: #7f8c8d; margin-top: 12px;">Searching...</p>';
|
||||||
if (/^\d+$/.test(asn)) {
|
|
||||||
window.location.href = '/as/' + asn;
|
// Use a simple client-side search against the asinfo data
|
||||||
|
// For now, redirect to AS page if it looks like an ASN
|
||||||
|
if (/^\d+$/.test(query)) {
|
||||||
|
window.location.href = '/as/' + query;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
showASNameMessage('Searching...', 'loading');
|
// Show a message that server-side search is coming
|
||||||
|
resultsDiv.innerHTML = '<p style="color: #7f8c8d; margin-top: 12px; font-size: 13px;">AS name search coming soon. For now, try an AS number.</p>';
|
||||||
fetch('/api/v1/search/as?q=' + encodeURIComponent(query))
|
|
||||||
.then(response => response.json())
|
|
||||||
.then(response => {
|
|
||||||
if (response.status === 'error') {
|
|
||||||
showASNameMessage('Error: ' + response.error.msg, 'error');
|
|
||||||
} else {
|
|
||||||
showASNameMatches(response.data);
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.catch(error => {
|
|
||||||
showASNameMessage('Error: ' + error.message, 'error');
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
|
|
||||||
// IP lookup: show the whole JSON answer, its @meta and an error answer
|
// IP lookup
|
||||||
// included, as the IP information API card does.
|
|
||||||
document.getElementById('ip-form').addEventListener('submit', function(e) {
|
document.getElementById('ip-form').addEventListener('submit', function(e) {
|
||||||
e.preventDefault();
|
e.preventDefault();
|
||||||
const ip = document.getElementById('ip-input').value.trim();
|
const ip = document.getElementById('ip-input').value.trim();
|
||||||
@@ -731,11 +416,18 @@
|
|||||||
|
|
||||||
fetch('/ip/' + encodeURIComponent(ip))
|
fetch('/ip/' + encodeURIComponent(ip))
|
||||||
.then(response => response.json())
|
.then(response => response.json())
|
||||||
.then(answer => {
|
.then(response => {
|
||||||
|
contentEl.classList.remove('loading');
|
||||||
|
if (response.status === 'error') {
|
||||||
|
contentEl.className = 'error';
|
||||||
|
contentEl.textContent = 'Error: ' + response.error.msg;
|
||||||
|
} else {
|
||||||
contentEl.className = '';
|
contentEl.className = '';
|
||||||
contentEl.textContent = JSON.stringify(answer, null, 2);
|
contentEl.textContent = JSON.stringify(response.data, null, 2);
|
||||||
|
}
|
||||||
})
|
})
|
||||||
.catch(error => {
|
.catch(error => {
|
||||||
|
contentEl.classList.remove('loading');
|
||||||
contentEl.className = 'error';
|
contentEl.className = 'error';
|
||||||
contentEl.textContent = 'Error: ' + error.message;
|
contentEl.textContent = 'Error: ' + error.message;
|
||||||
});
|
});
|
||||||
@@ -747,33 +439,9 @@
|
|||||||
document.getElementById('ip-input').value = '';
|
document.getElementById('ip-input').value = '';
|
||||||
});
|
});
|
||||||
|
|
||||||
// IP information API card: show the curl command that asks /ip/ about
|
// Initial load and refresh stats every 5 seconds
|
||||||
// the visitor's own address, then make that same request and show the
|
|
||||||
// JSON it answers, an error answer included. The answer holds outside
|
|
||||||
// data, so it is set with textContent, never innerHTML.
|
|
||||||
const ipAPICommand = document.getElementById('ip-api-command');
|
|
||||||
const ipAPIAnswer = document.getElementById('ip-api-answer');
|
|
||||||
const ipAPIURL = window.location.origin + '/ip/' + ipAPICommand.dataset.clientIp;
|
|
||||||
|
|
||||||
ipAPICommand.textContent = 'curl ' + ipAPIURL;
|
|
||||||
|
|
||||||
fetch(ipAPIURL)
|
|
||||||
.then(response => response.json())
|
|
||||||
.then(answer => {
|
|
||||||
ipAPIAnswer.className = '';
|
|
||||||
ipAPIAnswer.textContent = JSON.stringify(answer, null, 2);
|
|
||||||
})
|
|
||||||
.catch(error => {
|
|
||||||
ipAPIAnswer.className = 'error';
|
|
||||||
ipAPIAnswer.textContent = 'Error: ' + error.message;
|
|
||||||
});
|
|
||||||
|
|
||||||
// Initial load and refresh stats every REFRESH_PERIOD_MS. The freshness
|
|
||||||
// check runs every second on its own, so a request that never returns
|
|
||||||
// still turns the spinner into "stale".
|
|
||||||
updateStats();
|
updateStats();
|
||||||
setInterval(updateStats, REFRESH_PERIOD_MS);
|
setInterval(updateStats, 5000);
|
||||||
setInterval(showFreshness, MS_PER_SECOND);
|
|
||||||
</script>
|
</script>
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user