external
/
mailinabox
mirror of https://github.com/mail-in-a-box/mailinabox.git
1
0
Fork 0
Code Issues Releases Wiki Activity

add daily ip blacklist check

This commit is contained in:
KiekerJan 2021-05-18 13:02:05 +02:00
parent c9364b7d20
commit 1af0c58623
2 changed files with 448 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
management/daily_tasks.sh
View File

@ -9,6 +9,8 @@ export LC_ALL=en_US.UTF-8
export LANG=en_US.UTF-8
export LC_TYPE=en_US.UTF-8
source /etc/mailinabox.conf
# On Mondays, i.e. once a week, send the administrator a report of total emails
# sent and received so the admin might notice server abuse.
if [ `date "+%u"` -eq 1 ]; then
@ -25,3 +27,6 @@ management/ssl_certificates.py -q 2>&1 | management/email_administrator.py "TLS
# Run status checks and email the administrator if anything changed.
management/status_checks.py --show-changes 2>&1 | management/email_administrator.py "Status Checks Change Notice"
# Check blacklists
tools/check-dnsbl.py $PUBLIC_IP $PUBLIC_IPV6 2>&1 | management/email_administrator.py "Blacklist Check Result"

443
tools/check-dnsbl.py Executable file
View File

@ -0,0 +1,443 @@
#!/usr/bin/env python3
# 2016, Georg Sauthoff <mail@georg.so>, GPLv3+
import argparse
import csv
# require dnspython >= 1.15
# because of: https://github.com/rthalley/dnspython/issues/206
import dns.resolver
import dns.reversename
import logging
import re
import sys
import time
default_blacklists = [
('zen.spamhaus.org' , 'Spamhaus SBL, XBL and PBL' ),
('dnsbl.sorbs.net' , 'SORBS aggregated' ),
('safe.dnsbl.sorbs.net' , "'safe' subset of SORBS aggregated"),
('ix.dnsbl.manitu.net' , 'Heise iX NiX Spam' ),
('truncate.gbudb.net' , 'Exclusively Spam/Malware' ),
('dnsbl-1.uceprotect.net' , 'Trapserver Cluster' ),
('cbl.abuseat.org' , 'Net of traps' ),
('dnsbl.cobion.com' , 'used in IBM products' ),
('psbl.surriel.com' , 'passive list, easy to unlist' ),
('db.wpbl.info' , 'Weighted private' ),
('bl.spamcop.net' , 'Based on spamcop users' ),
('dyna.spamrats.com' , 'Dynamic IP addresses' ),
('spam.spamrats.com' , 'Manual submissions' ),
('auth.spamrats.com' , 'Suspicious authentications' ),
('dnsbl.inps.de' , 'automated and reported' ),
('bl.blocklist.de' , 'fail2ban reports etc.' ),
('all.s5h.net' , 'traps' ),
('rbl.realtimeblacklist.com' , 'lists ip ranges' ),
('b.barracudacentral.org' , 'traps' ),
('hostkarma.junkemailfilter.com', 'Autotected Virus Senders' ),
('ubl.unsubscore.com' , 'Collected Opt-Out Addresses' ),
('0spam.fusionzero.com' , 'Spam Trap' ),
('bl.nordspam.com' , 'NordSpam IP addresses' ),
('rbl.nordspam.com' , 'NordSpam Domain list ' ),
('combined.mail.abusix.zone' , 'Abusix aggregated' ),
('black.dnsbl.brukalai.lt' , 'Brukalai.lt junk mail' ),
('light.dnsbl.brukalai.lt' , 'Brukalai.lt abuse' ),
]
# blacklists disabled by default because they return mostly garbage
garbage_blacklists = [
# The spfbl.net operator doesn't publish clear criteria that lead to a
# blacklisting.
# When an IP address is blacklisted the operator can't name a specific
# reason for the blacklisting. The blacklisting details page just names
# overly generic reasons like:
# 'This IP was flagged due to misconfiguration of the e-mail service or
# the suspicion that there is no MTA at it.'
# When contacting the operator's support, they can't back up such
# claims.
# There are additions of IP addresses to the spfbl.net blacklist that
# have a properly configured MTA running and that aren't listed in any
# other blacklist. Likely, those additions are caused by a bug in the
# spfbl.net update process. But their support is uninterested in
# improving that process. Instead they want to externalize maintenance
# work by asking listed parties to waste some time on their manual
# delisting process.
# Suspiciously, you can even whitelist your listed address via
# transferring $ 1.50 via PayPal. Go figure.
# Thus, the value of querying this blacklist is utterly low as
# you get false-positive results, very likely.
('dnsbl.spfbl.net' , 'Reputation Database' ),
]
# See also:
# https://en.wikipedia.org/wiki/DNSBL
# https://tools.ietf.org/html/rfc5782
# https://en.wikipedia.org/wiki/Comparison_of_DNS_blacklists
# some lists provide detailed stats, i.e. the actual listed addresses
# useful for testing
log_format = '%(asctime)s - %(levelname)-8s - %(message)s [%(name)s]'
log_date_format = '%Y-%m-%d %H:%M:%S'
## Simple Setup
# Note that the basicConfig() call is a NOP in Jupyter
# because Jupyter calls it before
logging.basicConfig(format=log_format, datefmt=log_date_format, level=logging.WARNING)
log = logging.getLogger(__name__)
def mk_arg_parser():
p = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description = 'Check if mailservers are in any blacklist (DNSBL)',
epilog='''Don't panic if a server is listed in some blacklist.
See also https://en.wikipedia.org/wiki/Comparison_of_DNS_blacklists for the
mechanics and policies of the different lists.
2016, Georg Sauthoff <mail@georg.so>, GPLv3+''')
p.add_argument('dests', metavar='DESTINATION', nargs='+',
help = 'servers, a MX lookup is done if it is a domain')
p.add_argument('--bl', action='append', default=[],
help='add another blacklist')
p.add_argument('--bl-file', help='read more DNSBL from a CSV file')
p.add_argument('--clear', action='store_true',
help='clear default list of DNSBL')
# https://blog.cloudflare.com/dns-resolver-1-1-1-1/
p.add_argument('--cloudflare', action='store_true',
help="use Cloudflare's public DNS nameservers")
p.add_argument('--debug', action='store_true',
help='print debug log messages')
# cf. https://en.wikipedia.org/wiki/Google_Public_DNS
p.add_argument('--google', action='store_true',
help="use Google's public DNS nameservers")
p.add_argument('--rev', action='store_true', default=True,
help='check reverse DNS record for each domain (default: on)')
p.add_argument('--mx', action='store_true', default=True,
help='try to folow MX entries')
p.add_argument('--no-mx', dest='mx', action='store_false',
help='ignore any MX records')
p.add_argument('--no-rev', action='store_false', dest='rev',
help='disable reverse DNS checking')
p.add_argument('--ns', action='append', default=[],
help='use one or more alternate nameserverse')
# cf. https://en.wikipedia.org/wiki/OpenDNS
p.add_argument('--opendns', action='store_true',
help="use Cisco's public DNS nameservers")
# cf. https://quad9.net/faq/
p.add_argument('--quad9', action='store_true',
help="use Quad9's public DNS nameservers (i.e. the filtering ones)")
p.add_argument('--retries', type=int, default=5,
help='Number of retries if request times out (default: 5)')
p.add_argument('--with-garbage', action='store_true',
help=('also include low-quality blacklists that are maintained'
' by clueless operators and thus easily return false-positives'))
return p
def parse_args(*a):
p = mk_arg_parser()
args = p.parse_args(*a)
args.bls = default_blacklists
if args.clear:
args.bls = []
for bl in args.bl:
args.bls.append((bl, ''))
if args.bl_file:
args.bls = args.bls + read_csv_bl(args.bl_file)
if args.with_garbage:
args.bls.extend(garbage_blacklists)
if args.google:
args.ns = args.ns + ['8.8.8.8', '2001:4860:4860::8888', '8.8.4.4', '2001:4860:4860::8844']
if args.opendns:
args.ns = args.ns + ['208.67.222.222', '2620:0:ccc::2', '208.67.220.220', '2620:0:ccd::2']
if args.cloudflare:
args.ns += ['1.1.1.1', '2606:4700:4700::1111', '1.0.0.1', '2606:4700:4700::1001']
if args.quad9:
args.ns += ['9.9.9.9', '2620:fe::fe', '149.112.112.112', '2620:fe::9']
if args.ns:
dns.resolver.default_resolver = dns.resolver.Resolver(configure=False)
dns.resolver.default_resolver.nameservers = args.ns
if args.debug:
l = logging.getLogger() # root logger
l.setLevel(logging.DEBUG)
return args
def read_csv_bl(filename):
with open(filename, newline='') as f:
reader = csv.reader(f)
xs = [ row for row in reader
if len(row) > 0 and not row[0].startswith('#') ]
return xs
v4_ex = re.compile('^[.0-9]+$')
v6_ex = re.compile('^[:0-9a-fA-F]+$')
def get_addrs(dest, mx=True):
if v4_ex.match(dest) or v6_ex.match(dest):
return [ (dest, None) ]
domains = [ dest ]
if mx:
try:
r = dns.resolver.resolve(dest, 'mx')
domains = [ answer.exchange for answer in r ]
log.debug('destinatin {} has MXs: {}'
.format(dest, ', '.join([str(d) for d in domains])))
except dns.resolver.NoAnswer:
pass
addrs = []
for domain in domains:
for t in ['a', 'aaaa']:
try:
r = dns.resolver.resolve(domain, t)
except dns.resolver.NoAnswer:
continue
xs = [ ( answer.address, domain ) for answer in r ]
addrs = addrs + xs
log.debug('domain {} has addresses: {}'
.format(domain, ', '.join([x[0] for x in xs])))
if not addrs:
raise ValueError("There isn't any a/aaaa DNS record for {}".format(domain))
return addrs
def check_dnsbl(addr, bl):
rev = dns.reversename.from_address(addr)
domain = str(rev.split(3)[0]) + '.' + bl
try:
r = dns.resolver.resolve(domain, 'a')
except (dns.resolver.NXDOMAIN, dns.resolver.NoNameservers, dns.resolver.NoAnswer):
return 0
address = list(r)[0].address
try:
r = dns.resolver.resolve(domain, 'txt')
txt = list(r)[0].to_text()
except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN):
txt = ''
log.error('OMG, {} is listed in DNSBL {}: {} ({})'.format(
addr, bl, address, txt))
return 1
def check_rdns(addrs):
errs = 0
for (addr, domain) in addrs:
log.debug('Check if there is a reverse DNS record that maps address {} to {}'
.format(addr, domain))
try:
r = dns.resolver.resolve(dns.reversename.from_address(addr), 'ptr')
a = list(r)[0]
target = str(a.target).lower()
source = str(domain).lower()
log.debug('Reserve DNS record for {} points to {}'.format(addr, target))
if domain and source + '.' != target and source != target:
log.error('domain {} resolves to {}, but the reverse record resolves to {}'.
format(domain, addr, target))
errs = errs + 1
except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
log.error('There is no reverse DNS record for {}'.format(addr))
errs = errs + 1
return errs
return errs
def run(args):
log.debug('Checking {} DNS blacklists'.format(args.bls.__len__()))
errs = 0
for dest in args.dests:
addrs = get_addrs(dest, mx=args.mx)
if args.rev:
errs = errs + check_rdns(addrs)
old_errs = errs
ls = [ ( (x[0], x[1], y) for x in addrs for y in args.bls) ]
i = 0
while ls:
ms = []
for addr, domain, bl in ls[0]:
log.debug('Checking if address {} (via {}) is listed in {} ({})'
.format(addr, dest, bl[0], bl[1]))
try:
errs = errs + check_dnsbl(addr, bl[0])
except dns.exception.Timeout as e:
m = 'Resolving {}/{} in {} timed out: {}'.format(
addr, domain, bl[0], e)
if i >= args.retries:
log.warn(m)
else:
log.debug(m)
ms.append( (addr, domain, bl) )
ls.pop(0)
if ms and i + 1 < args.retries:
ls.append(ms)
log.debug('({}) Retrying {} timed-out entries'.format(i, len(ms)))
time.sleep(23+i*23)
i = i + 1
if old_errs < errs:
log.error('{} is listed in {} blacklists'.format(dest, errs - old_errs))
return 0 if errs == 0 else 1
def main(*a):
args = parse_args(*a)
return run(args)
if __name__ == '__main__':
if 'IPython' in sys.modules:
# do something different when running inside a Jupyter notebook
pass
else:
sys.exit(main())
##### Scratch area:
#
#
## In[ ]:
#
#check_rdns([('89.238.75.224', 'georg.so')])
#
#
## In[ ]:
#
#r = dns.resolver.resolve(dns.reversename.from_address('89.238.75.224'), 'ptr')
#a = list(r)[0]
#a.target.to_text()
#
#
## In[ ]:
#
#tr = dns.resolver.default_resolver
#
#
## In[ ]:
#
#dns.resolver.default_resolver = dns.resolver.Resolver(configure=False)
## some DNSBLs might block public DNS servers (because of the volume) such that
## false-negatives are generated with them
## e.g. Google's Public DNS
#dns.resolver.default_resolver.nameservers = ['8.8.8.8', '2001:4860:4860::8888', '8.8.4.4', '2001:4860:4860::8844']
#
#
## In[ ]:
#
#dns.resolver.default_resolver = dns.resolver.Resolver(configure=False)
## OpenDNS
#dns.resolver.default_resolver.nameservers = ['208.67.222.222', '2620:0:ccc::2', '208.67.220.220', '2620:0:ccd::2']
#
#
## In[ ]:
#
#tr.nameservers
#
#
## In[ ]:
#
#dns.resolver.default_resolver = tr
#
#
## In[ ]:
#
#dns.__version__
#
#
## In[ ]:
#
## as of 2016-11, listed
#r = dns.resolver.resolve('39.227.103.116.zen.spamhaus.org', 'txt')
#answer = list(r)[0]
#answer.to_text()
#
#
## In[ ]:
#
#check_dnsbl('116.103.227.39', 'zen.spamhaus.org')
#
#
## In[ ]:
#
## as of 2016-11, not listed
#check_dnsbl('217.146.132.159', 'zen.spamhaus.org')
#
#
## In[ ]:
#
#get_addrs('georg.so')
#
#
## In[ ]:
#
#parse_args(['georg.so'])
#
#
## In[ ]:
#
#a = dns.resolver.resolve('georg.so', 'MX')
#
#
## In[ ]:
#
#print(dns.resolver.Resolver.query.__doc__)
#
#
## In[ ]:
#
#[ str(x.exchange) for x in a ]
#
#
## In[ ]:
#
#[ x.exchange for x in a]
#dns.resolver.resolve(list(a)[0].exchange, 'a')
#
#
## In[ ]:
#
#r = dns.reversename.from_address('89.238.75.224')
#str(r.split(3)[0])
#
#
## In[ ]:
#
## should throw NoAnswer
#a = dns.resolver.resolve('escher.lru.li', 'mx')
##b = list(a)
#a
#
#
## In[ ]:
#
#a = dns.resolver.resolve('georg.so', 'a')
#b = list(a)[0]
#b.address
#dns.reversename.from_address(b.address)
#
#
## In[ ]:
#
## should throw NXDOMAIN
#rs = str(r.split(3)[0])
#dns.resolver.resolve(rs + '.zen.spamhaus.org', 'A' )
#
#
## In[ ]:
#
#s = dns.reversename.from_address('2a00:1828:2000:164::12')
#str(s.split(3)[0])