routewatch/internal/iplist/lists.go

package iplist

import (
	"slices"
	"time"
)

// TorExitsList is the name of the list of Tor exit nodes.
const TorExitsList = "tor_exits"

// How often each list is downloaded. No list is downloaded more often than
// its publisher updates it or allows.
const (
	// DShield asks that its feeds be downloaded at most once an hour:
	// https://isc.sans.edu/feeds_doc.html
	dshieldInterval = time.Hour

	// abuse.ch makes the list every 5 minutes and recommends a download at
	// least every 15: https://feodotracker.abuse.ch/blocklist/
	feodoTrackerInterval = 15 * time.Minute

	// FireHOL's page for each list gives how often the list changes on
	// average: about every 5 hours for abusers and every 3 hours for level
	// 3. FireHOL sets no download limit.
	// https://iplists.firehol.org/?ipset=firehol_abusers_30d
	// https://iplists.firehol.org/?ipset=firehol_level3
	fireHOLAbusersInterval = 5 * time.Hour
	fireHOLLevel3Interval  = 3 * time.Hour

	// Spamhaus asks that DROP be downloaded at most once a day:
	// https://www.spamhaus.org/blocklists/do-not-route-or-peer/
	spamhausDROPInterval = 24 * time.Hour

	// Onionoo's relay data follows Tor's hourly network consensus, and
	// Onionoo sets no download limit:
	// https://metrics.torproject.org/onionoo.html
	torExitsInterval = time.Hour

	// A VPN service's servers change slowly, so once a day is enough.
	vpnListInterval = 24 * time.Hour
)

// Lists returns the public IP lists routewatch downloads. Each can be
// downloaded whole without an account or a key. Spamhaus merged its EDROP
// list into DROP, so DROP covers both.
func Lists() []List {
	return slices.Concat(nonVPNLists(), vpnLists())
}

// nonVPNLists returns the public lists that are not lists of VPN servers.
func nonVPNLists() []List {
	return []List{
		{
			Name: "dshield",
			Description: "In one of the 20 networks that DShield saw scan " +
				"the most targets in the last three days",
			URL:      "https://feeds.dshield.org/block.txt",
			Interval: dshieldInterval,
			Format:   FormatDShield,
		},
		{
			Name:        "feodo_tracker",
			Description: "A botnet command-and-control server",
			URL:         "https://feodotracker.abuse.ch/downloads/ipblocklist.txt",
			Interval:    feodoTrackerInterval,
			Format:      FormatLines,
		},
		{
			Name: "firehol_abusers_30d",
			Description: "Reported for comment, form or forum spam in the " +
				"last 30 days",
			URL:      "https://iplists.firehol.org/files/firehol_abusers_30d.netset",
			Interval: fireHOLAbusersInterval,
			Format:   FormatLines,
		},
		{
			Name: "firehol_level3",
			Description: "Reported for attacks, spyware or viruses in the " +
				"last 30 days",
			URL:      "https://iplists.firehol.org/files/firehol_level3.netset",
			Interval: fireHOLLevel3Interval,
			Format:   FormatLines,
		},
		{
			Name: "spamhaus_drop",
			Description: "In an IPv4 network that Spamhaus says spammers " +
				"or cyber criminals have hijacked or lease",
			URL:      "https://www.spamhaus.org/drop/drop_v4.json",
			Interval: spamhausDROPInterval,
			Format:   FormatSpamhausJSON,
		},
		{
			Name: "spamhaus_dropv6",
			Description: "In an IPv6 network that Spamhaus says spammers " +
				"or cyber criminals have hijacked or lease",
			URL:      "https://www.spamhaus.org/drop/drop_v6.json",
			Interval: spamhausDROPInterval,
			Format:   FormatSpamhausJSON,
		},
		{
			Name:        TorExitsList,
			Description: "A Tor exit node, where traffic from the Tor network leaves",
			URL: "https://onionoo.torproject.org/details?" +
				"flag=Exit&running=true&fields=exit_addresses,or_addresses",
			Interval: torExitsInterval,
			Format:   FormatOnionooJSON,
		},
	}
}

// vpnLists returns the public lists of VPN servers. Their names are what
// vpn_providers reports.
func vpnLists() []List {
	return []List{
		{
			Name:        "ivpn",
			Description: "A server of the IVPN service",
			URL:         "https://api.ivpn.net/v5/servers.json",
			Interval:    vpnListInterval,
			Format:      FormatIVPNJSON,
			VPN:         true,
		},
		{
			Name:        "mullvad",
			Description: "A server of the Mullvad VPN service",
			URL:         "https://api.mullvad.net/www/relays/all/",
			Interval:    vpnListInterval,
			Format:      FormatMullvadJSON,
			VPN:         true,
		},
		{
			Name:        "nordvpn",
			Description: "A server of the NordVPN service",
			URL: "https://api.nordvpn.com/v1/servers?limit=0" +
				"&fields[servers.station]&fields[servers.ipv6_station]" +
				"&fields[servers.hostname]",
			Interval: vpnListInterval,
			Format:   FormatNordVPNJSON,
			VPN:      true,
		},
		{
			Name:        "pia",
			Description: "A server of the Private Internet Access VPN service",
			URL:         "https://serverlist.piaservers.net/vpninfo/servers/v6",
			Interval:    vpnListInterval,
			Format:      FormatPIAJSON,
			VPN:         true,
		},
		{
			Name: "x4bnet",
			Description: "In an IPv4 network of a VPN service, according to " +
				"X4BNet's community list",
			URL: "https://raw.githubusercontent.com/X4BNet/lists_vpn/main/" +
				"output/vpn/ipv4.txt",
			Interval: vpnListInterval,
			Format:   FormatLines,
			VPN:      true,
		},
		{
			Name: "x4bnet_v6",
			Description: "In an IPv6 network of a VPN service, according to " +
				"X4BNet's community list",
			URL: "https://raw.githubusercontent.com/X4BNet/lists_vpn/main/" +
				"output/vpn/ipv6.txt",
			Interval: vpnListInterval,
			Format:   FormatLines,
			VPN:      true,
		},
	}
}