add daily ip blacklist check

management/daily_tasks.sh

@@ -9,6 +9,8 @@ export LC_ALL=en_US.UTF-8
 export LANG=en_US.UTF-8
 export LC_TYPE=en_US.UTF-8
 
+source /etc/mailinabox.conf
+
 # On Mondays, i.e. once a week, send the administrator a report of total emails
 # sent and received so the admin might notice server abuse.
 if [ `date "+%u"` -eq 1 ]; then
@@ -25,3 +27,6 @@ management/ssl_certificates.py -q  2>&1 | management/email_administrator.py "TLS
 
 # Run status checks and email the administrator if anything changed.
 management/status_checks.py --show-changes  2>&1 | management/email_administrator.py "Status Checks Change Notice"
+
+# Check blacklists
+tools/check-dnsbl.py $PUBLIC_IP $PUBLIC_IPV6 2>&1 | management/email_administrator.py "Blacklist Check Result"

tools/check-dnsbl.py

@@ -0,0 +1,443 @@
+#!/usr/bin/env python3
+
+
+# 2016, Georg Sauthoff <mail@georg.so>, GPLv3+
+
+import argparse
+import csv
+# require dnspython >= 1.15
+# because of: https://github.com/rthalley/dnspython/issues/206
+import dns.resolver
+import dns.reversename
+import logging
+import re
+import sys
+import time
+
+
+default_blacklists = [
+        ('zen.spamhaus.org'             , 'Spamhaus SBL, XBL and PBL'        ),
+        ('dnsbl.sorbs.net'              , 'SORBS aggregated'                 ),
+        ('safe.dnsbl.sorbs.net'         , "'safe' subset of SORBS aggregated"),
+        ('ix.dnsbl.manitu.net'          , 'Heise iX NiX Spam'                ),
+        ('truncate.gbudb.net'           , 'Exclusively Spam/Malware'         ),
+        ('dnsbl-1.uceprotect.net'       , 'Trapserver Cluster'               ),
+        ('cbl.abuseat.org'              , 'Net of traps'                     ),
+        ('dnsbl.cobion.com'             , 'used in IBM products'             ),
+        ('psbl.surriel.com'             , 'passive list, easy to unlist'     ),
+        ('db.wpbl.info'                 , 'Weighted private'                 ),
+        ('bl.spamcop.net'               , 'Based on spamcop users'           ),
+        ('dyna.spamrats.com'            , 'Dynamic IP addresses'             ),
+        ('spam.spamrats.com'            , 'Manual submissions'               ),
+        ('auth.spamrats.com'            , 'Suspicious authentications'       ),
+        ('dnsbl.inps.de'                , 'automated and reported'           ),
+        ('bl.blocklist.de'              , 'fail2ban reports etc.'            ),
+        ('all.s5h.net'                  , 'traps'                            ),
+        ('rbl.realtimeblacklist.com'    , 'lists ip ranges'                  ),
+        ('b.barracudacentral.org'       , 'traps'                            ),
+        ('hostkarma.junkemailfilter.com', 'Autotected Virus Senders'         ),
+        ('ubl.unsubscore.com'           , 'Collected Opt-Out Addresses'      ),
+        ('0spam.fusionzero.com'         , 'Spam Trap'                        ),
+        ('bl.nordspam.com'              , 'NordSpam IP addresses'            ),
+        ('rbl.nordspam.com'             , 'NordSpam Domain list '            ),
+        ('combined.mail.abusix.zone'    , 'Abusix aggregated'                ),
+        ('black.dnsbl.brukalai.lt'      , 'Brukalai.lt junk mail'            ),
+        ('light.dnsbl.brukalai.lt'      , 'Brukalai.lt abuse'                ),
+        ]
+
+# blacklists disabled by default because they return mostly garbage
+garbage_blacklists = [
+        # The spfbl.net operator doesn't publish clear criteria that lead to a
+        # blacklisting.
+        # When an IP address is blacklisted the operator can't name a specific
+        # reason for the blacklisting. The blacklisting details page just names
+        # overly generic reasons like:
+        # 'This IP was flagged due to misconfiguration of the e-mail service or
+        # the suspicion that there is no MTA at it.'
+        # When contacting the operator's support, they can't back up such
+        # claims.
+        # There are additions of IP addresses to the spfbl.net blacklist that
+        # have a properly configured MTA running and that aren't listed in any
+        # other blacklist. Likely, those additions are caused by a bug in the
+        # spfbl.net update process. But their support is uninterested in
+        # improving that process. Instead they want to externalize maintenance
+        # work by asking listed parties to waste some time on their manual
+        # delisting process.
+        # Suspiciously, you can even whitelist your listed address via
+        # transferring $ 1.50 via PayPal. Go figure.
+        # Thus, the value of querying this blacklist is utterly low as
+        # you get false-positive results, very likely.
+        ('dnsbl.spfbl.net'              , 'Reputation Database'              ),
+        ]
+
+
+# See also:
+# https://en.wikipedia.org/wiki/DNSBL
+# https://tools.ietf.org/html/rfc5782
+# https://en.wikipedia.org/wiki/Comparison_of_DNS_blacklists
+
+# some lists provide detailed stats, i.e. the actual listed addresses
+# useful for testing
+
+
+log_format      = '%(asctime)s - %(levelname)-8s - %(message)s [%(name)s]'
+log_date_format = '%Y-%m-%d %H:%M:%S'
+
+## Simple Setup
+
+# Note that the basicConfig() call is a NOP in Jupyter
+# because Jupyter calls it before
+logging.basicConfig(format=log_format, datefmt=log_date_format, level=logging.WARNING)
+log = logging.getLogger(__name__)
+
+
+
+def mk_arg_parser():
+    p = argparse.ArgumentParser(
+            formatter_class=argparse.RawDescriptionHelpFormatter,
+            description = 'Check if mailservers are in any blacklist (DNSBL)',
+            epilog='''Don't panic if a server is listed in some blacklist.
+See also https://en.wikipedia.org/wiki/Comparison_of_DNS_blacklists for the
+mechanics and policies of the different lists.
+
+2016, Georg Sauthoff <mail@georg.so>, GPLv3+''')
+    p.add_argument('dests', metavar='DESTINATION', nargs='+',
+            help = 'servers, a MX lookup is done if it is a domain')
+    p.add_argument('--bl', action='append', default=[],
+            help='add another blacklist')
+    p.add_argument('--bl-file', help='read more DNSBL from a CSV file')
+    p.add_argument('--clear', action='store_true',
+            help='clear default list of DNSBL')
+    # https://blog.cloudflare.com/dns-resolver-1-1-1-1/
+    p.add_argument('--cloudflare', action='store_true',
+            help="use Cloudflare's public DNS nameservers")
+    p.add_argument('--debug', action='store_true',
+            help='print debug log messages')
+    # cf. https://en.wikipedia.org/wiki/Google_Public_DNS
+    p.add_argument('--google', action='store_true',
+            help="use Google's public DNS nameservers")
+    p.add_argument('--rev', action='store_true', default=True,
+            help='check reverse DNS record for each domain (default: on)')
+    p.add_argument('--mx', action='store_true', default=True,
+            help='try to folow MX entries')
+    p.add_argument('--no-mx', dest='mx', action='store_false',
+            help='ignore any MX records')
+    p.add_argument('--no-rev', action='store_false', dest='rev',
+            help='disable reverse DNS checking')
+    p.add_argument('--ns', action='append', default=[],
+            help='use one or more alternate nameserverse')
+    # cf. https://en.wikipedia.org/wiki/OpenDNS
+    p.add_argument('--opendns', action='store_true',
+            help="use Cisco's public DNS nameservers")
+    # cf. https://quad9.net/faq/
+    p.add_argument('--quad9', action='store_true',
+            help="use Quad9's public DNS nameservers (i.e. the filtering ones)")
+    p.add_argument('--retries', type=int, default=5,
+            help='Number of retries if request times out (default: 5)')
+    p.add_argument('--with-garbage', action='store_true',
+            help=('also include low-quality blacklists that are maintained'
+            ' by clueless operators and thus easily return false-positives'))
+    return p
+
+
+
+def parse_args(*a):
+    p = mk_arg_parser()
+    args = p.parse_args(*a)
+    args.bls = default_blacklists
+    if args.clear:
+        args.bls = []
+    for bl in args.bl:
+        args.bls.append((bl, ''))
+    if args.bl_file:
+        args.bls = args.bls + read_csv_bl(args.bl_file)
+    if args.with_garbage:
+        args.bls.extend(garbage_blacklists)
+    if args.google:
+        args.ns = args.ns + ['8.8.8.8', '2001:4860:4860::8888', '8.8.4.4', '2001:4860:4860::8844']
+    if args.opendns:
+        args.ns = args.ns + ['208.67.222.222', '2620:0:ccc::2', '208.67.220.220', '2620:0:ccd::2']
+    if args.cloudflare:
+        args.ns += ['1.1.1.1', '2606:4700:4700::1111', '1.0.0.1', '2606:4700:4700::1001']
+    if args.quad9:
+        args.ns += ['9.9.9.9', '2620:fe::fe', '149.112.112.112', '2620:fe::9']
+    if args.ns:
+        dns.resolver.default_resolver = dns.resolver.Resolver(configure=False)
+        dns.resolver.default_resolver.nameservers = args.ns
+    if args.debug:
+        l = logging.getLogger() # root logger
+        l.setLevel(logging.DEBUG)
+    return args
+
+
+
+def read_csv_bl(filename):
+    with open(filename, newline='') as f:
+        reader = csv.reader(f)
+        xs = [ row for row in reader
+                if len(row) > 0 and not row[0].startswith('#') ]
+        return xs
+
+
+
+v4_ex = re.compile('^[.0-9]+$')
+v6_ex = re.compile('^[:0-9a-fA-F]+$')
+
+def get_addrs(dest, mx=True):
+    if v4_ex.match(dest) or v6_ex.match(dest):
+        return [ (dest, None) ]
+    domains = [ dest ]
+    if mx:
+        try:
+            r = dns.resolver.resolve(dest, 'mx')
+            domains = [ answer.exchange for answer in r ]
+            log.debug('destinatin {} has MXs: {}'
+                      .format(dest, ', '.join([str(d) for d in domains])))
+        except dns.resolver.NoAnswer:
+            pass
+    addrs = []
+    for domain in domains:
+        for t in ['a', 'aaaa']:
+            try:
+                r = dns.resolver.resolve(domain, t)
+            except dns.resolver.NoAnswer:
+                continue
+            xs = [ ( answer.address, domain ) for answer in r ]
+            addrs = addrs + xs
+            log.debug('domain {} has addresses: {}'
+                      .format(domain, ', '.join([x[0] for x in xs])))
+    if not addrs:
+        raise ValueError("There isn't any a/aaaa DNS record for {}".format(domain))
+    return addrs
+
+
+
+def check_dnsbl(addr, bl):
+    rev = dns.reversename.from_address(addr)
+    domain = str(rev.split(3)[0]) + '.' + bl
+    try:
+        r = dns.resolver.resolve(domain, 'a')
+    except (dns.resolver.NXDOMAIN, dns.resolver.NoNameservers, dns.resolver.NoAnswer):
+        return 0
+    address = list(r)[0].address
+    try:
+        r = dns.resolver.resolve(domain, 'txt')
+        txt = list(r)[0].to_text()
+    except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN):
+        txt = ''
+    log.error('OMG, {} is listed in DNSBL {}: {} ({})'.format(
+        addr, bl, address, txt))
+    return 1
+
+
+
+def check_rdns(addrs):
+    errs = 0
+    for (addr, domain) in addrs:
+        log.debug('Check if there is a reverse DNS record that maps address {} to {}'
+                  .format(addr, domain))
+        try:
+            r = dns.resolver.resolve(dns.reversename.from_address(addr), 'ptr')
+            a = list(r)[0]
+            target = str(a.target).lower()
+            source = str(domain).lower()
+            log.debug('Reserve DNS record for {} points to {}'.format(addr, target))
+            if domain and source + '.' != target and source != target:
+                log.error('domain {} resolves to {}, but the reverse record resolves to {}'.
+                         format(domain, addr, target))
+                errs = errs + 1
+        except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
+            log.error('There is no reverse DNS record for {}'.format(addr))
+            errs = errs + 1
+            return errs
+    return errs
+
+
+
+def run(args):
+    log.debug('Checking {} DNS blacklists'.format(args.bls.__len__()))
+    errs = 0
+    for dest in args.dests:
+        addrs = get_addrs(dest, mx=args.mx)
+        if args.rev:
+            errs = errs + check_rdns(addrs)
+        old_errs = errs
+        ls = [ ( (x[0], x[1], y) for x in addrs for y in args.bls) ]
+        i = 0
+        while ls:
+            ms = []
+            for addr, domain, bl in ls[0]:
+                log.debug('Checking if address {} (via {}) is listed in {} ({})'
+                          .format(addr, dest, bl[0], bl[1]))
+                try:
+                    errs = errs + check_dnsbl(addr, bl[0])
+                except dns.exception.Timeout as e:
+                    m = 'Resolving  {}/{} in {} timed out: {}'.format(
+                        addr, domain, bl[0], e)
+                    if i >= args.retries:
+                        log.warn(m)
+                    else:
+                        log.debug(m)
+                        ms.append( (addr, domain, bl) )
+            ls.pop(0)
+            if ms and i + 1 < args.retries:
+                ls.append(ms)
+                log.debug('({}) Retrying {} timed-out entries'.format(i, len(ms)))
+                time.sleep(23+i*23)
+            i = i + 1
+        if old_errs < errs:
+            log.error('{} is listed in {} blacklists'.format(dest, errs - old_errs))
+    return 0 if errs == 0 else 1
+
+
+
+def main(*a):
+    args = parse_args(*a)
+    return run(args)
+
+
+
+if __name__ == '__main__':
+  if 'IPython' in sys.modules:
+    # do something different when running inside a Jupyter notebook
+    pass
+  else:
+    sys.exit(main())
+
+
+
+##### Scratch area:
+#
+#
+## In[ ]:
+#
+#check_rdns([('89.238.75.224', 'georg.so')])
+#
+#
+## In[ ]:
+#
+#r = dns.resolver.resolve(dns.reversename.from_address('89.238.75.224'), 'ptr')
+#a = list(r)[0]
+#a.target.to_text()
+#
+#
+## In[ ]:
+#
+#tr = dns.resolver.default_resolver 
+#
+#
+## In[ ]:
+#
+#dns.resolver.default_resolver = dns.resolver.Resolver(configure=False)
+## some DNSBLs might block public DNS servers (because of the volume) such that
+## false-negatives are generated with them
+## e.g. Google's Public DNS
+#dns.resolver.default_resolver.nameservers = ['8.8.8.8', '2001:4860:4860::8888', '8.8.4.4', '2001:4860:4860::8844']
+#
+#
+## In[ ]:
+#
+#dns.resolver.default_resolver = dns.resolver.Resolver(configure=False)
+## OpenDNS
+#dns.resolver.default_resolver.nameservers = ['208.67.222.222', '2620:0:ccc::2', '208.67.220.220', '2620:0:ccd::2']
+#
+#
+## In[ ]:
+#
+#tr.nameservers
+#
+#
+## In[ ]:
+#
+#dns.resolver.default_resolver = tr
+#
+#
+## In[ ]:
+#
+#dns.__version__
+#
+#
+## In[ ]:
+#
+## as of 2016-11, listed
+#r = dns.resolver.resolve('39.227.103.116.zen.spamhaus.org', 'txt')
+#answer = list(r)[0]
+#answer.to_text()
+#
+#
+## In[ ]:
+#
+#check_dnsbl('116.103.227.39', 'zen.spamhaus.org')
+#
+#
+## In[ ]:
+#
+## as of 2016-11, not listed
+#check_dnsbl('217.146.132.159', 'zen.spamhaus.org')
+#
+#
+## In[ ]:
+#
+#get_addrs('georg.so')
+#
+#
+## In[ ]:
+#
+#parse_args(['georg.so'])
+#
+#
+## In[ ]:
+#
+#a = dns.resolver.resolve('georg.so', 'MX')
+#
+#
+## In[ ]:
+#
+#print(dns.resolver.Resolver.query.__doc__)
+#
+#
+## In[ ]:
+#
+#[ str(x.exchange) for x in a ]
+#
+#
+## In[ ]:
+#
+#[ x.exchange for x in a]
+#dns.resolver.resolve(list(a)[0].exchange, 'a')
+#
+#
+## In[ ]:
+#
+#r = dns.reversename.from_address('89.238.75.224')
+#str(r.split(3)[0])
+#
+#
+## In[ ]:
+#
+## should throw NoAnswer
+#a = dns.resolver.resolve('escher.lru.li', 'mx')
+##b = list(a)
+#a
+#
+#
+## In[ ]:
+#
+#a = dns.resolver.resolve('georg.so', 'a')
+#b = list(a)[0]
+#b.address
+#dns.reversename.from_address(b.address)
+#
+#
+## In[ ]:
+#
+## should throw NXDOMAIN
+#rs = str(r.split(3)[0])
+#dns.resolver.resolve(rs + '.zen.spamhaus.org', 'A' )
+#
+#
+## In[ ]:
+#
+#s = dns.reversename.from_address('2a00:1828:2000:164::12')
+#str(s.split(3)[0])
+