Report which public blocklists list an address in the IP lookup (closes #158)
check / check (push) Canceled after 0s
check / check (push) Canceled after 0s
A new package, internal/blocklist, downloads seven public lists on a schedule: DShield, Feodo Tracker, FireHOL abusers and level 3, Spamhaus DROP for IPv4 and IPv6, and the Tor exit list. It keeps the last good copy of each in memory and in the state directory's blocklists directory, and loads the copies at startup. Each list's interval follows its publisher's page, cited in the code. Each IP lookup result carries blocklists, one entry per list the address is on. A successful lookup answer carries Cache-Control, private for the caller's own address. The address sets come from go4.org/netipx. Model: opus-5-5
This commit is contained in:
@@ -0,0 +1,172 @@
|
||||
// Package blocklist downloads public IP blocklists on a schedule, keeps the
|
||||
// last good copy of each on disk and in memory, and answers which lists
|
||||
// contain an address.
|
||||
package blocklist
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"net/netip"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go4.org/netipx"
|
||||
)
|
||||
|
||||
// Format is how a list's file is written.
|
||||
type Format int
|
||||
|
||||
const (
|
||||
// FormatLines has one address or network per line, such as 192.0.2.1 or
|
||||
// 192.0.2.0/24. Lines that start with # are comments.
|
||||
FormatLines Format = iota
|
||||
|
||||
// FormatSpamhausJSON has one JSON object per line, with a network in its
|
||||
// cidr field. The last line holds the file's metadata and no network.
|
||||
FormatSpamhausJSON
|
||||
|
||||
// FormatDShield has one network per line, as tab-separated fields that
|
||||
// start with the network's first and last address. Lines that start
|
||||
// with # are comments.
|
||||
FormatDShield
|
||||
)
|
||||
|
||||
// List is one public blocklist.
|
||||
type List struct {
|
||||
// Name names the list in lookup answers, and names its copy on disk.
|
||||
Name string
|
||||
|
||||
// Description says in one line what being on the list means.
|
||||
Description string
|
||||
|
||||
// URL is where the publisher serves the whole list.
|
||||
URL string
|
||||
|
||||
// Interval is how long a copy is used before the list is downloaded
|
||||
// again.
|
||||
Interval time.Duration
|
||||
|
||||
// Format is how the list's file is written.
|
||||
Format Format
|
||||
}
|
||||
|
||||
// Listing says that a list contains an address.
|
||||
type Listing struct {
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
|
||||
// Updated is when routewatch last downloaded the list, or last heard from
|
||||
// its publisher that its copy was still current.
|
||||
Updated time.Time `json:"updated"`
|
||||
}
|
||||
|
||||
// listCopy is a list's last good copy in memory.
|
||||
type listCopy struct {
|
||||
addresses *netipx.IPSet
|
||||
updated time.Time
|
||||
}
|
||||
|
||||
// Params is what New needs.
|
||||
type Params struct {
|
||||
// Dir is the directory that holds the copies on disk.
|
||||
Dir string
|
||||
|
||||
// Lists are the lists to download and look addresses up in.
|
||||
Lists []List
|
||||
|
||||
Logger *slog.Logger
|
||||
}
|
||||
|
||||
// Blocklists keeps a copy of each list and answers which lists contain an
|
||||
// address. Start loads the copies from disk and starts downloading the lists.
|
||||
type Blocklists struct {
|
||||
dir string
|
||||
lists []List
|
||||
logger *slog.Logger
|
||||
|
||||
// copies holds each list's last good copy, by list name. A list with no
|
||||
// good copy yet has none.
|
||||
mu sync.RWMutex
|
||||
copies map[string]listCopy
|
||||
|
||||
// stop ends the goroutine Start runs, and done is closed when it returns.
|
||||
stop context.CancelFunc
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
// New returns a Blocklists with no copies. Start loads them.
|
||||
func New(params Params) *Blocklists {
|
||||
return &Blocklists{
|
||||
dir: params.Dir,
|
||||
lists: params.Lists,
|
||||
logger: params.Logger.With("component", "blocklists"),
|
||||
copies: make(map[string]listCopy),
|
||||
done: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
// Lookup returns a Listing for each list whose copy contains addr, in the
|
||||
// order of the lists given to New. It makes no outside request. When no list
|
||||
// contains addr, it returns an empty list, not nil.
|
||||
func (b *Blocklists) Lookup(addr netip.Addr) []Listing {
|
||||
b.mu.RLock()
|
||||
defer b.mu.RUnlock()
|
||||
|
||||
listings := []Listing{}
|
||||
|
||||
for _, list := range b.lists {
|
||||
c, ok := b.copies[list.Name]
|
||||
if ok && c.addresses.Contains(addr.Unmap()) {
|
||||
listings = append(listings, Listing{
|
||||
Name: list.Name,
|
||||
Description: list.Description,
|
||||
Updated: c.updated,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return listings
|
||||
}
|
||||
|
||||
// ShortestInterval returns the shortest Interval of the lists. That is the
|
||||
// soonest a copy can change, and so how long a lookup answer stays current.
|
||||
func (b *Blocklists) ShortestInterval() time.Duration {
|
||||
var shortest time.Duration
|
||||
|
||||
for i, list := range b.lists {
|
||||
if i == 0 || list.Interval < shortest {
|
||||
shortest = list.Interval
|
||||
}
|
||||
}
|
||||
|
||||
return shortest
|
||||
}
|
||||
|
||||
// updated returns when the named list's copy was last downloaded or found
|
||||
// current, or the zero time when the list has no copy.
|
||||
func (b *Blocklists) updated(name string) time.Time {
|
||||
b.mu.RLock()
|
||||
defer b.mu.RUnlock()
|
||||
|
||||
return b.copies[name].updated
|
||||
}
|
||||
|
||||
// setCopy makes c the named list's copy.
|
||||
func (b *Blocklists) setCopy(name string, c listCopy) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
|
||||
b.copies[name] = c
|
||||
}
|
||||
|
||||
// setUpdated records that the named list's copy was found current at t.
|
||||
func (b *Blocklists) setUpdated(name string, t time.Time) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
|
||||
c, ok := b.copies[name]
|
||||
if ok {
|
||||
c.updated = t
|
||||
b.copies[name] = c
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,386 @@
|
||||
package blocklist
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"io/fs"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/netip"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The test list's name and description, and the address its good version
|
||||
// lists.
|
||||
const (
|
||||
testListName = "test_list"
|
||||
testListDescription = "On the test list"
|
||||
listedAddr = "192.0.2.1"
|
||||
goodListBody = "# A test list\n" + listedAddr + "\n"
|
||||
)
|
||||
|
||||
// maxWait is how long a test waits for the download goroutine.
|
||||
const maxWait = 10 * time.Second
|
||||
|
||||
// publisher stands in for a list's publisher. It answers every request with
|
||||
// its status and body, and keeps the headers of each request.
|
||||
type publisher struct {
|
||||
mu sync.Mutex
|
||||
status int
|
||||
body string
|
||||
headers []http.Header
|
||||
}
|
||||
|
||||
// newPublisher starts a publisher that answers 200 with body, and returns it
|
||||
// and its URL.
|
||||
func newPublisher(t *testing.T, body string) (*publisher, string) {
|
||||
t.Helper()
|
||||
|
||||
p := &publisher{status: http.StatusOK, body: body}
|
||||
srv := httptest.NewServer(p)
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
return p, srv.URL
|
||||
}
|
||||
|
||||
func (p *publisher) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
p.headers = append(p.headers, r.Header.Clone())
|
||||
w.WriteHeader(p.status)
|
||||
_, _ = io.WriteString(w, p.body)
|
||||
}
|
||||
|
||||
// answer makes the publisher answer later requests with status and body.
|
||||
func (p *publisher) answer(status int, body string) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
p.status, p.body = status, body
|
||||
}
|
||||
|
||||
// requests returns the headers of the requests the publisher has had.
|
||||
func (p *publisher) requests() []http.Header {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
return append([]http.Header(nil), p.headers...)
|
||||
}
|
||||
|
||||
// testList returns a list served at url, written in format.
|
||||
func testList(url string, format Format) List {
|
||||
return List{
|
||||
Name: testListName,
|
||||
Description: testListDescription,
|
||||
URL: url,
|
||||
Interval: time.Hour,
|
||||
Format: format,
|
||||
}
|
||||
}
|
||||
|
||||
func newTestBlocklists(dir string, list List) *Blocklists {
|
||||
return New(Params{
|
||||
Dir: dir,
|
||||
Lists: []List{list},
|
||||
Logger: slog.New(slog.DiscardHandler),
|
||||
})
|
||||
}
|
||||
|
||||
// lookup looks addr up in b.
|
||||
func lookup(t *testing.T, b *Blocklists, addr string) []Listing {
|
||||
t.Helper()
|
||||
|
||||
return b.Lookup(netip.MustParseAddr(addr))
|
||||
}
|
||||
|
||||
// TestLookupFindsListedAddresses downloads a list in each format and looks
|
||||
// up addresses on it, inside its networks, and off it, in IPv4 and IPv6.
|
||||
// A listed address has one listing that names the list; any other has an
|
||||
// empty list of listings.
|
||||
func TestLookupFindsListedAddresses(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
format Format
|
||||
body string
|
||||
listed []string
|
||||
unlisted []string
|
||||
}{
|
||||
{
|
||||
name: "single addresses",
|
||||
format: FormatLines,
|
||||
body: "# comment\n192.0.2.1\n\n2001:db8::1\n",
|
||||
listed: []string{"192.0.2.1", "::ffff:192.0.2.1", "2001:db8::1"},
|
||||
unlisted: []string{"192.0.2.2", "2001:db8::2"},
|
||||
},
|
||||
{
|
||||
name: "networks",
|
||||
format: FormatLines,
|
||||
body: "# comment\n198.51.100.0/24\n2001:db8:1::/48\n",
|
||||
listed: []string{"198.51.100.0", "198.51.100.255", "2001:db8:1::5"},
|
||||
unlisted: []string{"198.51.101.0", "2001:db8:2::1"},
|
||||
},
|
||||
{
|
||||
name: "Spamhaus networks",
|
||||
format: FormatSpamhausJSON,
|
||||
body: `{"cidr":"203.0.113.0/25","sblid":"SBL1","rir":"arin"}` + "\n" +
|
||||
`{"cidr":"2001:db8:3::/48","sblid":"SBL2","rir":"ripencc"}` + "\n" +
|
||||
`{"type":"metadata","timestamp":1793074442,"records":2}` + "\n",
|
||||
listed: []string{"203.0.113.1", "2001:db8:3::1"},
|
||||
unlisted: []string{"203.0.113.200", "2001:db8:4::1"},
|
||||
},
|
||||
{
|
||||
name: "DShield networks",
|
||||
format: FormatDShield,
|
||||
body: "# DShield.org Recommended Block List\n" +
|
||||
"192.0.2.0\t192.0.2.255\t24\t327\tEXAMPLE\tUS\tNone\n",
|
||||
listed: []string{"192.0.2.0", "192.0.2.77"},
|
||||
unlisted: []string{"192.0.3.1", "2001:db8::1"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, url := newPublisher(t, tt.body)
|
||||
b := newTestBlocklists(t.TempDir(), testList(url, tt.format))
|
||||
|
||||
err := b.download(t.Context(), b.lists[0])
|
||||
if err != nil {
|
||||
t.Fatalf("download: %v", err)
|
||||
}
|
||||
|
||||
for _, addr := range tt.listed {
|
||||
listings := lookup(t, b, addr)
|
||||
if len(listings) != 1 || listings[0].Name != testListName ||
|
||||
listings[0].Description != testListDescription ||
|
||||
listings[0].Updated.IsZero() {
|
||||
t.Errorf("%s: listings %+v, want one for %s", addr,
|
||||
listings, testListName)
|
||||
}
|
||||
}
|
||||
|
||||
for _, addr := range tt.unlisted {
|
||||
listings := lookup(t, b, addr)
|
||||
if listings == nil || len(listings) != 0 {
|
||||
t.Errorf("%s: listings %#v, want an empty list", addr, listings)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestFailedDownloadKeepsThePreviousCopy downloads a good version of a list,
|
||||
// then one that fails. The good copy stays in memory and on disk, and no
|
||||
// part of the failed download is left beside it.
|
||||
func TestFailedDownloadKeepsThePreviousCopy(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
status int
|
||||
body string
|
||||
}{
|
||||
{"server error", http.StatusInternalServerError, "unavailable"},
|
||||
{"not a list", http.StatusOK, "<!DOCTYPE html>\n<html></html>\n"},
|
||||
{
|
||||
"over the size limit", http.StatusOK,
|
||||
strings.Repeat(listedAddr+"\n", maxDownloadSize/len(listedAddr)),
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
p, url := newPublisher(t, goodListBody)
|
||||
b := newTestBlocklists(dir, testList(url, FormatLines))
|
||||
|
||||
err := b.download(t.Context(), b.lists[0])
|
||||
if err != nil {
|
||||
t.Fatalf("first download: %v", err)
|
||||
}
|
||||
|
||||
before := lookup(t, b, listedAddr)
|
||||
|
||||
p.answer(tt.status, tt.body)
|
||||
|
||||
err = b.download(t.Context(), b.lists[0])
|
||||
if err == nil {
|
||||
t.Fatal("second download returned no error")
|
||||
}
|
||||
|
||||
after := lookup(t, b, listedAddr)
|
||||
if len(after) != 1 || len(before) != 1 ||
|
||||
!after[0].Updated.Equal(before[0].Updated) {
|
||||
t.Errorf("listings %+v after the failure, want %+v", after, before)
|
||||
}
|
||||
|
||||
copies := os.DirFS(dir)
|
||||
|
||||
onDisk, err := fs.ReadFile(copies, testListName)
|
||||
if err != nil || string(onDisk) != goodListBody {
|
||||
t.Errorf("copy on disk %q (%v), want %q", onDisk, err, goodListBody)
|
||||
}
|
||||
|
||||
_, err = fs.Stat(copies, testListName+downloadSuffix)
|
||||
if !errors.Is(err, fs.ErrNotExist) {
|
||||
t.Errorf("the failed download's file is still there (%v)", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestRestartLoadsCopiesFromDisk downloads a list, then starts a new
|
||||
// Blocklists on the same directory, as a restart does. The new one answers
|
||||
// from the copy on disk at once, and does not download the list again while
|
||||
// the copy is younger than the list's interval.
|
||||
func TestRestartLoadsCopiesFromDisk(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
p, url := newPublisher(t, goodListBody)
|
||||
list := testList(url, FormatLines)
|
||||
|
||||
first := newTestBlocklists(dir, list)
|
||||
|
||||
err := first.download(t.Context(), list)
|
||||
if err != nil {
|
||||
t.Fatalf("download: %v", err)
|
||||
}
|
||||
|
||||
downloaded := lookup(t, first, listedAddr)[0].Updated
|
||||
|
||||
second := newTestBlocklists(dir, list)
|
||||
second.Start(t.Context())
|
||||
second.Stop(t.Context())
|
||||
|
||||
listings := lookup(t, second, listedAddr)
|
||||
if len(listings) != 1 ||
|
||||
listings[0].Updated.Sub(downloaded).Abs() > time.Second {
|
||||
t.Errorf("after the restart, listings %+v, want one updated at %s",
|
||||
listings, downloaded)
|
||||
}
|
||||
|
||||
if got := len(p.requests()); got != 1 {
|
||||
t.Errorf("the publisher had %d requests, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestStartRefreshesAnOldCopy starts on a copy older than its list's
|
||||
// interval. The list is downloaded at once, with routewatch's User-Agent and
|
||||
// the copy's time in If-Modified-Since. The publisher answers that the copy
|
||||
// is still current, so the copy is kept and marked as updated now.
|
||||
func TestStartRefreshesAnOldCopy(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
p, url := newPublisher(t, goodListBody)
|
||||
list := testList(url, FormatLines)
|
||||
|
||||
err := newTestBlocklists(dir, list).download(t.Context(), list)
|
||||
if err != nil {
|
||||
t.Fatalf("download: %v", err)
|
||||
}
|
||||
|
||||
old := time.Now().Add(-2 * list.Interval).UTC().Truncate(time.Second)
|
||||
|
||||
err = os.Chtimes(filepath.Join(dir, testListName), old, old)
|
||||
if err != nil {
|
||||
t.Fatalf("Chtimes: %v", err)
|
||||
}
|
||||
|
||||
p.answer(http.StatusNotModified, "")
|
||||
|
||||
b := newTestBlocklists(dir, list)
|
||||
b.Start(t.Context())
|
||||
|
||||
deadline := time.Now().Add(maxWait)
|
||||
for !lookup(t, b, listedAddr)[0].Updated.After(old) {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("the copy was not marked updated within %s", maxWait)
|
||||
}
|
||||
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
|
||||
b.Stop(t.Context())
|
||||
|
||||
requests := p.requests()
|
||||
last := requests[len(requests)-1]
|
||||
|
||||
if got := last.Get(ifModifiedSinceHeader); got != old.Format(http.TimeFormat) {
|
||||
t.Errorf("If-Modified-Since %q, want %q", got,
|
||||
old.Format(http.TimeFormat))
|
||||
}
|
||||
|
||||
if got := last.Get(userAgentHeader); got != userAgent {
|
||||
t.Errorf("User-Agent %q, want %q", got, userAgent)
|
||||
}
|
||||
}
|
||||
|
||||
// TestStopCancelsADownloadInProgress stops while the publisher has not
|
||||
// answered. Stop cancels the download and returns once the download
|
||||
// goroutine has stopped.
|
||||
func TestStopCancelsADownloadInProgress(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
arrived := make(chan struct{})
|
||||
|
||||
var once sync.Once
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(
|
||||
func(_ http.ResponseWriter, r *http.Request) {
|
||||
once.Do(func() { close(arrived) })
|
||||
<-r.Context().Done()
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
b := newTestBlocklists(t.TempDir(), testList(srv.URL, FormatLines))
|
||||
b.Start(t.Context())
|
||||
|
||||
select {
|
||||
case <-arrived:
|
||||
case <-time.After(maxWait):
|
||||
t.Fatalf("no download began within %s", maxWait)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(t.Context(), maxWait)
|
||||
defer cancel()
|
||||
|
||||
b.Stop(ctx)
|
||||
|
||||
select {
|
||||
case <-b.done:
|
||||
default:
|
||||
t.Error("Stop returned before the download goroutine stopped")
|
||||
}
|
||||
}
|
||||
|
||||
// TestPublicListsAreComplete checks that every public list has what a
|
||||
// download and a lookup answer need, and a unique name.
|
||||
func TestPublicListsAreComplete(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
names := make(map[string]bool)
|
||||
|
||||
for _, list := range Lists() {
|
||||
if list.Name == "" || list.Description == "" || list.URL == "" ||
|
||||
list.Interval <= 0 || names[list.Name] {
|
||||
t.Errorf("list %+v is incomplete or has a name used before", list)
|
||||
}
|
||||
|
||||
names[list.Name] = true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,284 @@
|
||||
package blocklist
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"os"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"go4.org/netipx"
|
||||
)
|
||||
|
||||
// Limits of one download.
|
||||
const (
|
||||
// downloadTimeout is how long one download may take.
|
||||
downloadTimeout = time.Minute
|
||||
|
||||
// maxDownloadSize is the size of the largest list file accepted, in
|
||||
// bytes. The largest list, FireHOL abusers, is about 2.5 MiB.
|
||||
maxDownloadSize = 16 << 20
|
||||
)
|
||||
|
||||
// The request headers a download sets.
|
||||
const (
|
||||
userAgentHeader = "User-Agent"
|
||||
ifModifiedSinceHeader = "If-Modified-Since"
|
||||
|
||||
// userAgent names routewatch to the publishers.
|
||||
userAgent = "routewatch (+https://git.eeqj.de/sneak/routewatch)"
|
||||
)
|
||||
|
||||
const (
|
||||
// downloadSuffix ends the name of the file a download is written to,
|
||||
// beside the list's copy, until it has parsed.
|
||||
downloadSuffix = ".download"
|
||||
|
||||
// dirPermissions are those of the directory that holds the copies.
|
||||
dirPermissions = 0o750
|
||||
)
|
||||
|
||||
var (
|
||||
errUnexpectedStatus = errors.New("unexpected HTTP status")
|
||||
errTooLarge = errors.New("list is over the size limit")
|
||||
)
|
||||
|
||||
// Start loads each list's copy from disk. It then starts the goroutine that
|
||||
// downloads each list once its copy is older than its Interval, and a list
|
||||
// with no copy at once. The goroutine runs until Stop: the end of ctx does
|
||||
// not stop it.
|
||||
func (b *Blocklists) Start(ctx context.Context) {
|
||||
err := os.MkdirAll(b.dir, dirPermissions)
|
||||
if err != nil {
|
||||
b.logger.Error("Failed to create the blocklists directory",
|
||||
"dir", b.dir, "error", err)
|
||||
}
|
||||
|
||||
nextDownload := make([]time.Time, len(b.lists))
|
||||
|
||||
for i, list := range b.lists {
|
||||
loadErr := b.loadCopy(list)
|
||||
if loadErr != nil && !errors.Is(loadErr, fs.ErrNotExist) {
|
||||
b.logger.Error("Failed to load a blocklist copy",
|
||||
"list", list.Name, "error", loadErr)
|
||||
}
|
||||
|
||||
// A list with no copy has the zero time, so it is due at once.
|
||||
nextDownload[i] = b.updated(list.Name).Add(list.Interval)
|
||||
}
|
||||
|
||||
ctx, b.stop = context.WithCancel(context.WithoutCancel(ctx))
|
||||
|
||||
go func() {
|
||||
b.run(ctx, nextDownload)
|
||||
}()
|
||||
|
||||
b.logger.Info("Blocklist downloads started", "dir", b.dir)
|
||||
}
|
||||
|
||||
// Stop stops the downloads, cancelling one in progress, and waits until the
|
||||
// goroutine has returned or ctx ends. The copies in memory stay, so Lookup
|
||||
// still answers.
|
||||
func (b *Blocklists) Stop(ctx context.Context) {
|
||||
b.stop()
|
||||
|
||||
select {
|
||||
case <-b.done:
|
||||
b.logger.Info("Blocklist downloads stopped")
|
||||
case <-ctx.Done():
|
||||
b.logger.Warn("Stop deadline passed before the blocklist downloads " +
|
||||
"stopped")
|
||||
}
|
||||
}
|
||||
|
||||
// run downloads each list when nextDownload says, until ctx ends. After each
|
||||
// try, whether it worked or not, the list's next download is an Interval
|
||||
// later.
|
||||
func (b *Blocklists) run(ctx context.Context, nextDownload []time.Time) {
|
||||
defer close(b.done)
|
||||
|
||||
if len(nextDownload) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
for {
|
||||
soonest := slices.MinFunc(nextDownload, time.Time.Compare)
|
||||
timer := time.NewTimer(time.Until(soonest))
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
|
||||
return
|
||||
case <-timer.C:
|
||||
}
|
||||
|
||||
for i, list := range b.lists {
|
||||
if time.Now().Before(nextDownload[i]) {
|
||||
continue
|
||||
}
|
||||
|
||||
err := b.download(ctx, list)
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
b.logger.Error("Blocklist download failed; keeping the "+
|
||||
"previous copy", "list", list.Name, "error", err)
|
||||
}
|
||||
|
||||
nextDownload[i] = time.Now().Add(list.Interval)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// download asks the publisher for list. A new version that parses becomes
|
||||
// the list's copy, on disk and in memory. An answer that the copy is still
|
||||
// current marks the copy as updated now. Anything else leaves the copy as it
|
||||
// is and returns an error.
|
||||
func (b *Blocklists) download(ctx context.Context, list List) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, downloadTimeout)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, list.URL, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
req.Header.Set(userAgentHeader, userAgent)
|
||||
|
||||
updated := b.updated(list.Name)
|
||||
if !updated.IsZero() {
|
||||
req.Header.Set(ifModifiedSinceHeader, updated.Format(http.TimeFormat))
|
||||
}
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
switch resp.StatusCode {
|
||||
case http.StatusOK:
|
||||
return b.replaceCopy(list, resp.Body)
|
||||
case http.StatusNotModified:
|
||||
return b.markCurrent(list)
|
||||
default:
|
||||
return fmt.Errorf("%w: %s", errUnexpectedStatus, resp.Status)
|
||||
}
|
||||
}
|
||||
|
||||
// replaceCopy writes a new version of list, read from body, beside the list's
|
||||
// copy. Only when it parses does it replace the copy, on disk and in memory.
|
||||
func (b *Blocklists) replaceCopy(list List, body io.Reader) error {
|
||||
root, err := os.OpenRoot(b.dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() { _ = root.Close() }()
|
||||
|
||||
downloadName := list.Name + downloadSuffix
|
||||
|
||||
addresses, err := writeAndParse(root, downloadName, body, list.Format)
|
||||
if err != nil {
|
||||
_ = root.Remove(downloadName)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
err = root.Rename(downloadName, list.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
b.setCopy(list.Name, listCopy{addresses: addresses, updated: time.Now().UTC()})
|
||||
b.logger.Info("Downloaded a new version of a blocklist", "list", list.Name)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeAndParse writes body to the file name in root, up to maxDownloadSize
|
||||
// bytes, and parses the file as format.
|
||||
func writeAndParse(
|
||||
root *os.Root, name string, body io.Reader, format Format,
|
||||
) (*netipx.IPSet, error) {
|
||||
f, err := root.Create(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer func() { _ = f.Close() }()
|
||||
|
||||
written, err := io.Copy(f, io.LimitReader(body, maxDownloadSize+1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if written > maxDownloadSize {
|
||||
return nil, errTooLarge
|
||||
}
|
||||
|
||||
_, err = f.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return parse(f, format)
|
||||
}
|
||||
|
||||
// markCurrent records that the publisher says the list's copy is still
|
||||
// current: the copy, on disk and in memory, is marked as updated now.
|
||||
func (b *Blocklists) markCurrent(list List) error {
|
||||
root, err := os.OpenRoot(b.dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() { _ = root.Close() }()
|
||||
|
||||
now := time.Now().UTC()
|
||||
|
||||
err = root.Chtimes(list.Name, now, now)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
b.setUpdated(list.Name, now)
|
||||
b.logger.Debug("Blocklist copy is still current", "list", list.Name)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// loadCopy makes the list's copy on disk its copy in memory. The file's
|
||||
// modification time is when the copy was updated.
|
||||
func (b *Blocklists) loadCopy(list List) error {
|
||||
f, err := os.OpenInRoot(b.dir, list.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() { _ = f.Close() }()
|
||||
|
||||
info, err := f.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
addresses, err := parse(f, list.Format)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
b.setCopy(list.Name, listCopy{
|
||||
addresses: addresses, updated: info.ModTime().UTC(),
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package blocklist
|
||||
|
||||
import "time"
|
||||
|
||||
// How often each list is downloaded. No list is downloaded more often than
|
||||
// its publisher updates it or allows.
|
||||
const (
|
||||
// DShield asks that its feeds be downloaded at most once an hour:
|
||||
// https://isc.sans.edu/feeds_doc.html
|
||||
dshieldInterval = time.Hour
|
||||
|
||||
// abuse.ch makes the list every 5 minutes and recommends a download at
|
||||
// least every 15: https://feodotracker.abuse.ch/blocklist/
|
||||
feodoTrackerInterval = 15 * time.Minute
|
||||
|
||||
// FireHOL's page for each list gives how often the list changes on
|
||||
// average: about every 5 hours for abusers and every 3 hours for level
|
||||
// 3. FireHOL sets no download limit.
|
||||
// https://iplists.firehol.org/?ipset=firehol_abusers_30d
|
||||
// https://iplists.firehol.org/?ipset=firehol_level3
|
||||
fireHOLAbusersInterval = 5 * time.Hour
|
||||
fireHOLLevel3Interval = 3 * time.Hour
|
||||
|
||||
// Spamhaus asks that DROP be downloaded at most once a day:
|
||||
// https://www.spamhaus.org/blocklists/do-not-route-or-peer/
|
||||
spamhausDROPInterval = 24 * time.Hour
|
||||
|
||||
// Tor publishes a new exit list every hour and sets no download limit:
|
||||
// https://collector.torproject.org/recent/exit-lists/
|
||||
torExitsInterval = time.Hour
|
||||
)
|
||||
|
||||
// Lists returns the public blocklists routewatch downloads. Each can be
|
||||
// downloaded whole without an account or a key. Spamhaus merged its EDROP
|
||||
// list into DROP, so DROP covers both.
|
||||
func Lists() []List {
|
||||
return []List{
|
||||
{
|
||||
Name: "dshield",
|
||||
Description: "In one of the 20 networks that DShield saw scan " +
|
||||
"the most targets in the last three days",
|
||||
URL: "https://feeds.dshield.org/block.txt",
|
||||
Interval: dshieldInterval,
|
||||
Format: FormatDShield,
|
||||
},
|
||||
{
|
||||
Name: "feodo_tracker",
|
||||
Description: "A botnet command-and-control server",
|
||||
URL: "https://feodotracker.abuse.ch/downloads/ipblocklist.txt",
|
||||
Interval: feodoTrackerInterval,
|
||||
Format: FormatLines,
|
||||
},
|
||||
{
|
||||
Name: "firehol_abusers_30d",
|
||||
Description: "Reported for comment, form or forum spam in the " +
|
||||
"last 30 days",
|
||||
URL: "https://iplists.firehol.org/files/firehol_abusers_30d.netset",
|
||||
Interval: fireHOLAbusersInterval,
|
||||
Format: FormatLines,
|
||||
},
|
||||
{
|
||||
Name: "firehol_level3",
|
||||
Description: "Reported for attacks, spyware or viruses in the " +
|
||||
"last 30 days",
|
||||
URL: "https://iplists.firehol.org/files/firehol_level3.netset",
|
||||
Interval: fireHOLLevel3Interval,
|
||||
Format: FormatLines,
|
||||
},
|
||||
{
|
||||
Name: "spamhaus_drop",
|
||||
Description: "In an IPv4 network that Spamhaus says spammers " +
|
||||
"or cyber criminals have hijacked or lease",
|
||||
URL: "https://www.spamhaus.org/drop/drop_v4.json",
|
||||
Interval: spamhausDROPInterval,
|
||||
Format: FormatSpamhausJSON,
|
||||
},
|
||||
{
|
||||
Name: "spamhaus_dropv6",
|
||||
Description: "In an IPv6 network that Spamhaus says spammers " +
|
||||
"or cyber criminals have hijacked or lease",
|
||||
URL: "https://www.spamhaus.org/drop/drop_v6.json",
|
||||
Interval: spamhausDROPInterval,
|
||||
Format: FormatSpamhausJSON,
|
||||
},
|
||||
{
|
||||
Name: "tor_exits",
|
||||
Description: "A Tor exit node, where traffic from the Tor network leaves",
|
||||
URL: "https://check.torproject.org/torbulkexitlist",
|
||||
Interval: torExitsInterval,
|
||||
Format: FormatLines,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
package blocklist
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/netip"
|
||||
"strings"
|
||||
|
||||
"go4.org/netipx"
|
||||
)
|
||||
|
||||
const (
|
||||
// commentPrefix starts a comment line in a list file.
|
||||
commentPrefix = "#"
|
||||
|
||||
// networkSeparator separates the address and the mask length of a
|
||||
// network, as in 192.0.2.0/24.
|
||||
networkSeparator = "/"
|
||||
|
||||
// dshieldFieldSeparator separates the fields of a DShield line.
|
||||
dshieldFieldSeparator = "\t"
|
||||
|
||||
// spamhausMetadataType is the type of the line that ends a Spamhaus
|
||||
// file. That line holds the file's metadata and no network.
|
||||
spamhausMetadataType = "metadata"
|
||||
)
|
||||
|
||||
var (
|
||||
errUnknownFormat = errors.New("unknown list format")
|
||||
errNoNetwork = errors.New("no network on the line")
|
||||
errBadRange = errors.New("first and last address make no range")
|
||||
)
|
||||
|
||||
// parse reads a list file written in format and returns the addresses it
|
||||
// holds. A line it cannot read fails the whole file, so a file that is not
|
||||
// the list, such as an error page, never replaces a good copy.
|
||||
func parse(r io.Reader, format Format) (*netipx.IPSet, error) {
|
||||
var builder netipx.IPSetBuilder
|
||||
|
||||
scanner := bufio.NewScanner(r)
|
||||
|
||||
for lineNumber := 1; scanner.Scan(); lineNumber++ {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" || strings.HasPrefix(line, commentPrefix) {
|
||||
continue
|
||||
}
|
||||
|
||||
err := addLine(&builder, line, format)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("line %d: %w", lineNumber, err)
|
||||
}
|
||||
}
|
||||
|
||||
err := scanner.Err()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return builder.IPSet()
|
||||
}
|
||||
|
||||
// addLine adds the address or network on one line of a list file to builder.
|
||||
func addLine(builder *netipx.IPSetBuilder, line string, format Format) error {
|
||||
switch format {
|
||||
case FormatLines:
|
||||
return addAddressOrNetwork(builder, line)
|
||||
case FormatSpamhausJSON:
|
||||
return addSpamhausNetwork(builder, line)
|
||||
case FormatDShield:
|
||||
return addDShieldNetwork(builder, line)
|
||||
default:
|
||||
return errUnknownFormat
|
||||
}
|
||||
}
|
||||
|
||||
// addAddressOrNetwork adds an address, such as 192.0.2.1, or a network, such
|
||||
// as 192.0.2.0/24, to builder.
|
||||
func addAddressOrNetwork(builder *netipx.IPSetBuilder, text string) error {
|
||||
if strings.Contains(text, networkSeparator) {
|
||||
network, err := netip.ParsePrefix(text)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
builder.AddPrefix(network)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
addr, err := netip.ParseAddr(text)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
builder.Add(addr)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// addSpamhausNetwork adds the network on a line of a Spamhaus file to
|
||||
// builder. The metadata line adds nothing.
|
||||
func addSpamhausNetwork(builder *netipx.IPSetBuilder, line string) error {
|
||||
var entry struct {
|
||||
CIDR string `json:"cidr"`
|
||||
Type string `json:"type"`
|
||||
}
|
||||
|
||||
err := json.Unmarshal([]byte(line), &entry)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if entry.Type == spamhausMetadataType {
|
||||
return nil
|
||||
}
|
||||
|
||||
network, err := netip.ParsePrefix(entry.CIDR)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
builder.AddPrefix(network)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// addDShieldNetwork adds the network on a line of a DShield file to builder.
|
||||
// The line's first two fields are the network's first and last address.
|
||||
func addDShieldNetwork(builder *netipx.IPSetBuilder, line string) error {
|
||||
firstField, rest, found := strings.Cut(line, dshieldFieldSeparator)
|
||||
if !found {
|
||||
return errNoNetwork
|
||||
}
|
||||
|
||||
lastField, _, _ := strings.Cut(rest, dshieldFieldSeparator)
|
||||
|
||||
first, err := netip.ParseAddr(firstField)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
last, err := netip.ParseAddr(lastField)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
network := netipx.IPRangeFrom(first, last)
|
||||
if !network.IsValid() {
|
||||
return errBadRange
|
||||
}
|
||||
|
||||
builder.AddRange(network)
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -8,9 +8,11 @@ package routewatch
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/routewatch/internal/blocklist"
|
||||
"git.eeqj.de/sneak/routewatch/internal/config"
|
||||
"git.eeqj.de/sneak/routewatch/internal/database"
|
||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
||||
@@ -52,8 +54,13 @@ type RouteWatch struct {
|
||||
peeringHandler *PeeringHandler
|
||||
asnFetcher *ASNFetcher
|
||||
dbMaintainer *DBMaintainer
|
||||
blocklists *blocklist.Blocklists
|
||||
}
|
||||
|
||||
// blocklistsDirName is the directory in the state directory that holds the
|
||||
// blocklist copies.
|
||||
const blocklistsDirName = "blocklists"
|
||||
|
||||
// New creates a new RouteWatch instance
|
||||
func New(deps Dependencies) *RouteWatch {
|
||||
rw := &RouteWatch{
|
||||
@@ -122,6 +129,18 @@ func (rw *RouteWatch) Run(ctx context.Context) error {
|
||||
rw.asnFetcher.Start(ctx)
|
||||
rw.server.SetASNFetcher(rw.asnFetcher)
|
||||
|
||||
// Load the blocklist copies from disk and start downloading the lists.
|
||||
// The HTTP handlers read the server's blocklists without a lock, so they
|
||||
// are set before the handlers run.
|
||||
blocklists := blocklist.New(blocklist.Params{
|
||||
Dir: filepath.Join(rw.config.StateDir, blocklistsDirName),
|
||||
Lists: blocklist.Lists(),
|
||||
Logger: rw.logger.Logger,
|
||||
})
|
||||
blocklists.Start(ctx)
|
||||
rw.blocklists = blocklists
|
||||
rw.server.SetBlocklists(blocklists)
|
||||
|
||||
// Start HTTP server
|
||||
err = rw.server.Start(ctx)
|
||||
if err != nil {
|
||||
@@ -139,11 +158,12 @@ func (rw *RouteWatch) Run(ctx context.Context) error {
|
||||
}
|
||||
|
||||
// Shutdown stops all services in order: the streamer, the handlers, which
|
||||
// write what they hold, the WHOIS fetcher, the database maintainer and the
|
||||
// HTTP server, and last the database, which it checkpoints and closes. Each
|
||||
// stage waits at most until ctx ends, so Shutdown returns soon after: a
|
||||
// handler logs what it could not write by then, and the HTTP server closes
|
||||
// the connections of requests still running.
|
||||
// write what they hold, the WHOIS fetcher, the database maintainer, the
|
||||
// blocklist downloads and the HTTP server, and last the database, which it
|
||||
// checkpoints and closes. Each stage waits at most until ctx ends, so
|
||||
// Shutdown returns soon after: a handler logs what it could not write by
|
||||
// then, and the HTTP server closes the connections of requests still
|
||||
// running.
|
||||
func (rw *RouteWatch) Shutdown(ctx context.Context) {
|
||||
rw.mu.Lock()
|
||||
if rw.shutdown {
|
||||
@@ -190,6 +210,10 @@ func (rw *RouteWatch) Shutdown(ctx context.Context) {
|
||||
rw.dbMaintainer.Stop(ctx)
|
||||
}
|
||||
|
||||
if rw.blocklists != nil {
|
||||
rw.blocklists.Stop(ctx)
|
||||
}
|
||||
|
||||
err := rw.server.Stop(ctx)
|
||||
if err != nil {
|
||||
rw.logger.Error("Stopped the HTTP server before every request finished",
|
||||
|
||||
@@ -466,9 +466,9 @@ func TestRouteWatchLiveFeed(t *testing.T) {
|
||||
// Create streamer
|
||||
s := streamer.New(logger, metricsTracker)
|
||||
|
||||
// Create test config with empty state dir (no snapshot loading)
|
||||
// The state directory holds the blocklist copies Run downloads.
|
||||
cfg := &config.Config{
|
||||
StateDir: "",
|
||||
StateDir: t.TempDir(),
|
||||
MaxRuntime: 5 * time.Second,
|
||||
EnableBatchedDatabaseWrites: true,
|
||||
}
|
||||
|
||||
+69
-13
@@ -15,6 +15,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/routewatch/internal/blocklist"
|
||||
"git.eeqj.de/sneak/routewatch/internal/database"
|
||||
"git.eeqj.de/sneak/routewatch/internal/metrics"
|
||||
"git.eeqj.de/sneak/routewatch/internal/ristypes"
|
||||
@@ -45,6 +46,15 @@ const (
|
||||
ptrLookupTimeout = 2 * time.Second
|
||||
)
|
||||
|
||||
// The Cache-Control header of an IP lookup answer, such as
|
||||
// "public, max-age=900".
|
||||
const (
|
||||
cacheControlHeader = "Cache-Control"
|
||||
cacheControlPublic = "public"
|
||||
cacheControlPrivate = "private"
|
||||
cacheControlMaxAge = ", max-age="
|
||||
)
|
||||
|
||||
// Keys of the JSON envelope the handlers and middleware write, and the status
|
||||
// value of an error answer.
|
||||
const (
|
||||
@@ -693,9 +703,18 @@ func (s *Server) handleIPLookup() http.HandlerFunc {
|
||||
|
||||
// IPLookupResponse is the standard response for IP/hostname lookups.
|
||||
type IPLookupResponse struct {
|
||||
Query string `json:"query"`
|
||||
Results []*database.IPInfo `json:"results"`
|
||||
Errors []string `json:"errors,omitempty"`
|
||||
Query string `json:"query"`
|
||||
Results []IPLookupResult `json:"results"`
|
||||
Errors []string `json:"errors,omitempty"`
|
||||
}
|
||||
|
||||
// IPLookupResult is the answer for one address: its route and AS from the
|
||||
// database, and the blocklists that list it.
|
||||
type IPLookupResult struct {
|
||||
*database.IPInfo
|
||||
|
||||
// Blocklists has one entry for each blocklist the address is on.
|
||||
Blocklists []blocklist.Listing `json:"blocklists"`
|
||||
}
|
||||
|
||||
// handleIPInfo returns a handler that provides comprehensive IP information.
|
||||
@@ -704,7 +723,7 @@ type IPLookupResponse struct {
|
||||
// Always returns the same response structure with PTR records for each IP.
|
||||
func (s *Server) handleIPInfo() http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
target := lookupTarget(r)
|
||||
target, ownAddress := lookupTarget(r)
|
||||
if target == "" {
|
||||
s.writeJSONError(w, http.StatusBadRequest, "Could not determine IP address")
|
||||
|
||||
@@ -712,7 +731,7 @@ func (s *Server) handleIPInfo() http.HandlerFunc {
|
||||
}
|
||||
|
||||
ctx := r.Context()
|
||||
response := IPLookupResponse{Query: target, Results: make([]*database.IPInfo, 0)}
|
||||
response := IPLookupResponse{Query: target, Results: make([]IPLookupResult, 0)}
|
||||
|
||||
// Collect all IPs to look up
|
||||
ipsToLookup, resolveErrors, err := s.resolveLookupTargets(ctx, target)
|
||||
@@ -772,7 +791,10 @@ func (s *Server) handleIPInfo() http.HandlerFunc {
|
||||
ipInfo.PTR = ptrs
|
||||
}
|
||||
|
||||
response.Results = append(response.Results, ipInfo)
|
||||
response.Results = append(response.Results, IPLookupResult{
|
||||
IPInfo: ipInfo,
|
||||
Blocklists: s.blocklistListings(ip),
|
||||
})
|
||||
|
||||
if ipInfo.NeedsWHOISRefresh {
|
||||
refreshASNs[ipInfo.ASN] = true
|
||||
@@ -782,10 +804,40 @@ func (s *Server) handleIPInfo() http.HandlerFunc {
|
||||
// Queue WHOIS refresh for stale ASNs (non-blocking)
|
||||
s.queueWHOISRefresh(refreshASNs)
|
||||
|
||||
s.writeIPLookupAnswer(w, response)
|
||||
s.writeIPLookupAnswer(w, response, ownAddress)
|
||||
}
|
||||
}
|
||||
|
||||
// blocklistListings returns the blocklists that list ip. It returns an empty
|
||||
// list, never nil, so that an answer's blocklists is [] rather than null.
|
||||
func (s *Server) blocklistListings(ip string) []blocklist.Listing {
|
||||
addr, err := netip.ParseAddr(ip)
|
||||
if s.blocklists == nil || err != nil {
|
||||
return []blocklist.Listing{}
|
||||
}
|
||||
|
||||
return s.blocklists.Lookup(addr)
|
||||
}
|
||||
|
||||
// setIPLookupCacheControl lets caches keep an IP lookup answer for the
|
||||
// shortest blocklist interval, the soonest a blocklist copy can change. The
|
||||
// answer about the caller's own address is private, so that a shared cache
|
||||
// does not give it to another caller. Without blocklists it sets nothing.
|
||||
func (s *Server) setIPLookupCacheControl(w http.ResponseWriter, ownAddress bool) {
|
||||
if s.blocklists == nil {
|
||||
return
|
||||
}
|
||||
|
||||
visibility := cacheControlPublic
|
||||
if ownAddress {
|
||||
visibility = cacheControlPrivate
|
||||
}
|
||||
|
||||
maxAgeSeconds := int(s.blocklists.ShortestInterval().Seconds())
|
||||
w.Header().Set(cacheControlHeader,
|
||||
visibility+cacheControlMaxAge+strconv.Itoa(maxAgeSeconds))
|
||||
}
|
||||
|
||||
// noTargetsMessage returns the message of the 400 an IP lookup answers when
|
||||
// no address is left to look up, followed by the first of resolveErrors, which
|
||||
// says why, when there is one.
|
||||
@@ -800,9 +852,10 @@ func noTargetsMessage(resolveErrors []string) string {
|
||||
}
|
||||
|
||||
// writeIPLookupAnswer sends response, or 404 with its first error when no
|
||||
// address had a route.
|
||||
// address had a route. ownAddress says the request asked about the caller's
|
||||
// own address.
|
||||
func (s *Server) writeIPLookupAnswer(
|
||||
w http.ResponseWriter, response IPLookupResponse,
|
||||
w http.ResponseWriter, response IPLookupResponse, ownAddress bool,
|
||||
) {
|
||||
if len(response.Results) == 0 && len(response.Errors) > 0 {
|
||||
s.writeJSONError(w, http.StatusNotFound, "No routes found: "+response.Errors[0])
|
||||
@@ -810,6 +863,8 @@ func (s *Server) writeIPLookupAnswer(
|
||||
return
|
||||
}
|
||||
|
||||
s.setIPLookupCacheControl(w, ownAddress)
|
||||
|
||||
err := writeJSONSuccess(w, response)
|
||||
if err != nil {
|
||||
s.logger.Error("Failed to encode IP lookup response", "error", err)
|
||||
@@ -817,8 +872,9 @@ func (s *Server) writeIPLookupAnswer(
|
||||
}
|
||||
|
||||
// lookupTarget returns what an IP lookup request asks for: the ip or addr URL
|
||||
// parameter, or else the client's own address.
|
||||
func lookupTarget(r *http.Request) string {
|
||||
// parameter, or else the client's own address. The bool is true for the
|
||||
// client's own address.
|
||||
func lookupTarget(r *http.Request) (string, bool) {
|
||||
target := chi.URLParam(r, "ip")
|
||||
if target == "" {
|
||||
target = chi.URLParam(r, "addr")
|
||||
@@ -826,10 +882,10 @@ func lookupTarget(r *http.Request) string {
|
||||
|
||||
if target == "" {
|
||||
// Use client IP (RealIP middleware has already processed this)
|
||||
target = extractClientIP(r)
|
||||
return extractClientIP(r), true
|
||||
}
|
||||
|
||||
return target
|
||||
return target, false
|
||||
}
|
||||
|
||||
// queueWHOISRefresh asks the WHOIS fetcher, when there is one, to refresh
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"html"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/netip"
|
||||
@@ -16,6 +17,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/routewatch/internal/blocklist"
|
||||
"git.eeqj.de/sneak/routewatch/internal/config"
|
||||
"git.eeqj.de/sneak/routewatch/internal/database"
|
||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
||||
@@ -1246,6 +1248,129 @@ func TestIPLookupKeepsRouteWhenPTRLookupDoesNotAnswer(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// addrInTestPrefixV6 is an address in testPrefixV6, which
|
||||
// newServerWithTestRoutes stores a route for.
|
||||
const addrInTestPrefixV6 = "2001:db8::1"
|
||||
|
||||
// testBlocklistInterval is fakeBlocklists' shortest interval.
|
||||
const testBlocklistInterval = 15 * time.Minute
|
||||
|
||||
// ownAddressPath is the IP lookup of the caller's own address.
|
||||
const ownAddressPath = "/ip/"
|
||||
|
||||
// fakeBlocklists has one list, which lists addrInTestPrefixV4 and no other
|
||||
// address.
|
||||
type fakeBlocklists struct{}
|
||||
|
||||
func testListing() blocklist.Listing {
|
||||
return blocklist.Listing{
|
||||
Name: "test_list",
|
||||
Description: "On the test list",
|
||||
Updated: time.Date(2026, 10, 27, 4, 0, 0, 0, time.UTC),
|
||||
}
|
||||
}
|
||||
|
||||
func (fakeBlocklists) Lookup(addr netip.Addr) []blocklist.Listing {
|
||||
if addr == netip.MustParseAddr(addrInTestPrefixV4) {
|
||||
return []blocklist.Listing{testListing()}
|
||||
}
|
||||
|
||||
return []blocklist.Listing{}
|
||||
}
|
||||
|
||||
func (fakeBlocklists) ShortestInterval() time.Duration {
|
||||
return testBlocklistInterval
|
||||
}
|
||||
|
||||
// TestIPLookupReportsBlocklists looks up two routed addresses, one of them on
|
||||
// a blocklist. The listed address's result names the list, says what it means
|
||||
// and when it was updated. The other's result has an empty list.
|
||||
func TestIPLookupReportsBlocklists(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s := newServerWithTestRoutes(t)
|
||||
s.resolver = &fakeResolver{}
|
||||
s.SetBlocklists(fakeBlocklists{})
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
s.router.ServeHTTP(rec, httptest.NewRequestWithContext(t.Context(),
|
||||
http.MethodGet,
|
||||
"/api/v1/ip/"+addrInTestPrefixV4+","+addrInTestPrefixV6, nil))
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status %d, want %d; body %s", rec.Code, http.StatusOK, rec.Body)
|
||||
}
|
||||
|
||||
var body struct {
|
||||
Data IPLookupResponse `json:"data"`
|
||||
}
|
||||
|
||||
err := json.Unmarshal(rec.Body.Bytes(), &body)
|
||||
if err != nil {
|
||||
t.Fatalf("decoding the answer %q: %v", rec.Body, err)
|
||||
}
|
||||
|
||||
results := body.Data.Results
|
||||
if len(results) != 2 {
|
||||
t.Fatalf("answer %s, want two results", rec.Body)
|
||||
}
|
||||
|
||||
want := testListing()
|
||||
|
||||
listed := results[0].Blocklists
|
||||
if len(listed) != 1 || listed[0].Name != want.Name ||
|
||||
listed[0].Description != want.Description ||
|
||||
!listed[0].Updated.Equal(want.Updated) {
|
||||
t.Errorf("%s: blocklists %+v, want [%+v]", results[0].IP, listed, want)
|
||||
}
|
||||
|
||||
if !strings.Contains(rec.Body.String(), `"blocklists":[]`) {
|
||||
t.Errorf("answer %s, want an empty blocklists list for %s",
|
||||
rec.Body, addrInTestPrefixV6)
|
||||
}
|
||||
}
|
||||
|
||||
// TestIPLookupAnswersMayBeCached checks the Cache-Control header of the IP
|
||||
// lookup answers. Caches may keep them for the blocklists' shortest interval,
|
||||
// and only the caller's own browser may keep the answer about the caller's
|
||||
// own address.
|
||||
func TestIPLookupAnswersMayBeCached(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s := newServerWithTestRoutes(t)
|
||||
s.resolver = &fakeResolver{}
|
||||
s.SetBlocklists(fakeBlocklists{})
|
||||
|
||||
tests := []struct {
|
||||
path string
|
||||
want string
|
||||
}{
|
||||
{"/api/v1/ip/" + addrInTestPrefixV4, "public, max-age=900"},
|
||||
{ownAddressPath + addrInTestPrefixV4, "public, max-age=900"},
|
||||
{ownAddressPath, "private, max-age=900"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
req := httptest.NewRequestWithContext(t.Context(), http.MethodGet,
|
||||
tt.path, nil)
|
||||
req.RemoteAddr = net.JoinHostPort(addrInTestPrefixV4, "1234")
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
s.router.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Errorf("%s: status %d, want %d; body %s",
|
||||
tt.path, rec.Code, http.StatusOK, rec.Body)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
if got := rec.Header().Get(cacheControlHeader); got != tt.want {
|
||||
t.Errorf("%s: Cache-Control %q, want %q", tt.path, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The host name TestIPLookupHidesPrivateAddresses resolves to a private
|
||||
// address, and that address.
|
||||
const (
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/routewatch/internal/blocklist"
|
||||
"git.eeqj.de/sneak/routewatch/internal/config"
|
||||
"git.eeqj.de/sneak/routewatch/internal/database"
|
||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
||||
@@ -36,6 +37,13 @@ type ASNFetcher interface {
|
||||
GetStats() ASNFetcherStats
|
||||
}
|
||||
|
||||
// Blocklists tells which public blocklists list an address, from copies it
|
||||
// keeps in memory, as *blocklist.Blocklists does.
|
||||
type Blocklists interface {
|
||||
Lookup(addr netip.Addr) []blocklist.Listing
|
||||
ShortestInterval() time.Duration
|
||||
}
|
||||
|
||||
// dnsResolver looks up the addresses of host names and the names of addresses
|
||||
// in DNS, as *net.Resolver does.
|
||||
type dnsResolver interface {
|
||||
@@ -52,6 +60,7 @@ type Server struct {
|
||||
port int
|
||||
srv *http.Server
|
||||
asnFetcher ASNFetcher
|
||||
blocklists Blocklists
|
||||
|
||||
// trustedProxies are the reverse proxies whose X-Forwarded-For header
|
||||
// RealIPMiddleware believes
|
||||
@@ -148,3 +157,9 @@ func (s *Server) Stop(ctx context.Context) error {
|
||||
func (s *Server) SetASNFetcher(fetcher ASNFetcher) {
|
||||
s.asnFetcher = fetcher
|
||||
}
|
||||
|
||||
// SetBlocklists sets the blocklists the IP lookups report. Call it before
|
||||
// Start: the handlers read the blocklists without a lock.
|
||||
func (s *Server) SetBlocklists(blocklists Blocklists) {
|
||||
s.blocklists = blocklists
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user