Merge 86a31cd978 into 7ec662c83f

management/auth.py

@@ -1,4 +1,7 @@
-import base64, os, os.path, hmac
+import base64
+import os
+import os.path
+import hmac
 
 from flask import make_response
 
@@ -8,6 +11,7 @@ from mailconfig import get_mail_password, get_mail_user_privileges
 DEFAULT_KEY_PATH = '/var/lib/mailinabox/api.key'
 DEFAULT_AUTH_REALM = 'Mail-in-a-Box Management Server'
 
+
 class KeyAuthService:
     """Generate an API key for authenticating clients
 
@@ -116,7 +120,8 @@ class KeyAuthService:
         # (This call should never fail on a valid user. But if it did fail, it would
         # return a tuple of an error message and an HTTP status code.)
         privs = get_mail_user_privileges(email, env)
-		if isinstance(privs, tuple): raise Exception("Error getting privileges.")
+        if isinstance(privs, tuple):
+            raise Exception("Error getting privileges.")
 
         # Return a list of privileges.
         return privs

management/backup.py

@@ -9,8 +9,15 @@
 #    backup/secret_key.txt) to STORAGE_ROOT/backup/encrypted.
 # 5) STORAGE_ROOT/backup/after-backup is executd if it exists.
 
-import os, os.path, shutil, glob, re, datetime
+import os
-import dateutil.parser, dateutil.relativedelta, dateutil.tz
+import os.path
+import shutil
+import glob
+import re
+import datetime
+import dateutil.parser
+import dateutil.relativedelta
+import dateutil.tz
 
 from utils import exclusive_process, load_environment, shell
 
@@ -18,6 +25,7 @@ from utils import exclusive_process, load_environment, shell
 # that depends on it is this many days old.
 keep_backups_for_days = 3
 
+
 def backup_status(env):
     # What is the current status of backups?
     # Loop through all of the files in STORAGE_ROOT/backup/duplicity to
@@ -25,23 +33,32 @@ def backup_status(env):
     # see how large the storage is.
 
     now = datetime.datetime.now(dateutil.tz.tzlocal())
+
     def reldate(date, ref, clip):
-		if ref < date: return clip
+        if ref < date:
+            return clip
         rd = dateutil.relativedelta.relativedelta(ref, date)
-		if rd.months > 1: return "%d months, %d days" % (rd.months, rd.days)
+        if rd.months > 1:
-		if rd.months == 1: return "%d month, %d days" % (rd.months, rd.days)
+            return "%d months, %d days" % (rd.months, rd.days)
-		if rd.days >= 7: return "%d days" % rd.days
+        if rd.months == 1:
-		if rd.days > 1: return "%d days, %d hours" % (rd.days, rd.hours)
+            return "%d month, %d days" % (rd.months, rd.days)
-		if rd.days == 1: return "%d day, %d hours" % (rd.days, rd.hours)
+        if rd.days >= 7:
+            return "%d days" % rd.days
+        if rd.days > 1:
+            return "%d days, %d hours" % (rd.days, rd.hours)
+        if rd.days == 1:
+            return "%d day, %d hours" % (rd.days, rd.hours)
         return "%d hours, %d minutes" % (rd.hours, rd.minutes)
 
     backups = {}
     basedir = os.path.join(env['STORAGE_ROOT'], 'backup/duplicity/')
     encdir = os.path.join(env['STORAGE_ROOT'], 'backup/encrypted/')
-	os.makedirs(basedir, exist_ok=True) # os.listdir fails if directory does not exist
+    # os.listdir fails if directory does not exist
+    os.makedirs(basedir, exist_ok=True)
     for fn in os.listdir(basedir):
         m = re.match(r"duplicity-(full|full-signatures|(inc|new-signatures)\.(?P<incbase>\d+T\d+Z)\.to)\.(?P<date>\d+T\d+Z)\.", fn)
-		if not m: raise ValueError(fn)
+        if not m:
+            raise ValueError(fn)
 
         key = m.group("date")
         if key not in backups:
@@ -93,6 +110,7 @@ def backup_status(env):
         "backups": backups,
     }
 
+
 def should_force_full(env):
     # Force a full backup when the total size of the increments
     # since the last full backup is greater than half the size
@@ -112,6 +130,7 @@ def should_force_full(env):
         # (I love for/else blocks. Here it's just to show off.)
         return True
 
+
 def perform_backup(full_backup):
     env = load_environment()
 
@@ -174,7 +193,8 @@ def perform_backup(full_backup):
     os.makedirs(backup_encrypted_dir, exist_ok=True)
     for fn in os.listdir(backup_duplicity_dir):
         fn2 = os.path.join(backup_encrypted_dir, fn) + ".enc"
-		if os.path.exists(fn2): continue
+        if os.path.exists(fn2):
+            continue
 
         # Encrypt the backup using the backup private key.
         shell('check_call', [
@@ -194,7 +214,8 @@ def perform_backup(full_backup):
     # Remove encrypted backups that are no longer needed.
     for fn in os.listdir(backup_encrypted_dir):
         fn2 = os.path.join(backup_duplicity_dir, fn.replace(".enc", ""))
-		if os.path.exists(fn2): continue
+        if os.path.exists(fn2):
+            continue
         os.unlink(os.path.join(backup_encrypted_dir, fn))
 
     # Execute a post-backup script that does the copying to a remote server.

management/daemon.py

@@ -1,12 +1,16 @@
 #!/usr/bin/python3
 
-import os, os.path, re, json
+import os
+import os.path
+import re
+import json
 
 from functools import wraps
 
 from flask import Flask, request, render_template, abort, Response
 
-import auth, utils
+import auth
+import utils
 from mailconfig import get_mail_users, get_mail_users_ex, get_admins, add_mail_user, set_mail_password, remove_mail_user
 from mailconfig import get_mail_user_privileges, add_remove_mail_user_privilege
 from mailconfig import get_mail_aliases, get_mail_aliases_ex, get_mail_domains, add_mail_alias, remove_mail_alias
@@ -30,6 +34,7 @@ except OSError:
 
 app = Flask(__name__, template_folder=os.path.abspath(os.path.join(os.path.dirname(me), "templates")))
 
+
 # Decorator to protect views that require a user with 'admin' privileges.
 def authorized_personnel_only(viewfunc):
     @wraps(viewfunc)
@@ -75,10 +80,12 @@ def authorized_personnel_only(viewfunc):
 
     return newview
 
+
 @app.errorhandler(401)
 def unauthorized(error):
     return auth_service.make_unauthorized_response()
 
+
 def json_response(data):
     return Response(json.dumps(data), status=200, mimetype='application/json')
 
@@ -86,17 +93,20 @@ def json_response(data):
 
 # Control Panel (unauthenticated views)
 
+
 @app.route('/')
 def index():
     # Render the control panel. This route does not require user authentication
     # so it must be safe!
     no_admins_exist = (len(get_admins(env)) == 0)
-	return render_template('index.html',
+    return render_template(
+        'index.html',
         hostname=env['PRIMARY_HOSTNAME'],
         storage_root=env['STORAGE_ROOT'],
         no_admins_exist=no_admins_exist,
     )
 
+
 @app.route('/me')
 def me():
     # Is the caller authorized?
@@ -123,6 +133,7 @@ def me():
 
 # MAIL
 
+
 @app.route('/mail/users')
 @authorized_personnel_only
 def mail_users():
@@ -131,6 +142,7 @@ def mail_users():
     else:
         return "".join(x+"\n" for x in get_mail_users(env))
 
+
 @app.route('/mail/users/add', methods=['POST'])
 @authorized_personnel_only
 def mail_users_add():
@@ -139,6 +151,7 @@ def mail_users_add():
     except ValueError as e:
         return (str(e), 400)
 
+
 @app.route('/mail/users/password', methods=['POST'])
 @authorized_personnel_only
 def mail_users_password():
@@ -147,6 +160,7 @@ def mail_users_password():
     except ValueError as e:
         return (str(e), 400)
 
+
 @app.route('/mail/users/remove', methods=['POST'])
 @authorized_personnel_only
 def mail_users_remove():
@@ -157,14 +171,18 @@ def mail_users_remove():
 @authorized_personnel_only
 def mail_user_privs():
     privs = get_mail_user_privileges(request.args.get('email', ''), env)
-	if isinstance(privs, tuple): return privs # error
+    # error
+    if isinstance(privs, tuple):
+        return privs
     return "\n".join(privs)
 
+
 @app.route('/mail/users/privileges/add', methods=['POST'])
 @authorized_personnel_only
 def mail_user_privs_add():
     return add_remove_mail_user_privilege(request.form.get('email', ''), request.form.get('privilege', ''), "add", env)
 
+
 @app.route('/mail/users/privileges/remove', methods=['POST'])
 @authorized_personnel_only
 def mail_user_privs_remove():
@@ -179,6 +197,7 @@ def mail_aliases():
     else:
         return "".join(x+"\t"+y+"\n" for x, y in get_mail_aliases(env))
 
+
 @app.route('/mail/aliases/add', methods=['POST'])
 @authorized_personnel_only
 def mail_aliases_add():
@@ -189,11 +208,13 @@ def mail_aliases_add():
         update_if_exists=(request.form.get('update_if_exists', '') == '1')
         )
 
+
 @app.route('/mail/aliases/remove', methods=['POST'])
 @authorized_personnel_only
 def mail_aliases_remove():
     return remove_mail_alias(request.form.get('source', ''), env)
 
+
 @app.route('/mail/domains')
 @authorized_personnel_only
 def mail_domains():
@@ -201,12 +222,14 @@ def mail_domains():
 
 # DNS
 
+
 @app.route('/dns/zones')
 @authorized_personnel_only
 def dns_zones():
     from dns_update import get_dns_zones
     return json_response([z[0] for z in get_dns_zones(env)])
 
+
 @app.route('/dns/update', methods=['POST'])
 @authorized_personnel_only
 def dns_update():
@@ -216,12 +239,14 @@ def dns_update():
     except Exception as e:
         return (str(e), 500)
 
+
 @app.route('/dns/secondary-nameserver')
 @authorized_personnel_only
 def dns_get_secondary_nameserver():
     from dns_update import get_custom_dns_config
     return json_response({"hostname": get_custom_dns_config(env).get("_secondary_nameserver")})
 
+
 @app.route('/dns/secondary-nameserver', methods=['POST'])
 @authorized_personnel_only
 def dns_set_secondary_nameserver():
@@ -231,6 +256,7 @@ def dns_set_secondary_nameserver():
     except ValueError as e:
         return (str(e), 400)
 
+
 @app.route('/dns/set')
 @authorized_personnel_only
 def dns_get_records():
@@ -243,6 +269,7 @@ def dns_get_records():
         "value": r[2],
         } for r in records])
 
+
 @app.route('/dns/set/<qname>', methods=['POST'])
 @app.route('/dns/set/<qname>/<rtype>', methods=['POST'])
 @app.route('/dns/set/<qname>/<rtype>/<value>', methods=['POST'])
@@ -256,7 +283,8 @@ def dns_set_record(qname, rtype="A", value=None):
         if value is None:
             value = request.form.get("value")
         if value is None:
-			value = request.environ.get("HTTP_X_FORWARDED_FOR") # normally REMOTE_ADDR but we're behind nginx as a reverse proxy
+            # normally REMOTE_ADDR but we're behind nginx as a reverse proxy
+            value = request.environ.get("HTTP_X_FORWARDED_FOR")
         if value == '' or value == '__delete__':
             # request deletion
             value = None
@@ -266,6 +294,7 @@ def dns_set_record(qname, rtype="A", value=None):
     except ValueError as e:
         return (str(e), 400)
 
+
 @app.route('/dns/dump')
 @authorized_personnel_only
 def dns_get_dump():
@@ -274,6 +303,7 @@ def dns_get_dump():
 
 # SSL
 
+
 @app.route('/ssl/csr/<domain>', methods=['POST'])
 @authorized_personnel_only
 def ssl_get_csr(domain):
@@ -281,6 +311,7 @@ def ssl_get_csr(domain):
     ssl_key, ssl_certificate, ssl_via = get_domain_ssl_files(domain, env)
     return create_csr(domain, ssl_key, env)
 
+
 @app.route('/ssl/install', methods=['POST'])
 @authorized_personnel_only
 def ssl_install_cert():
@@ -292,12 +323,14 @@ def ssl_install_cert():
 
 # WEB
 
+
 @app.route('/web/domains')
 @authorized_personnel_only
 def web_get_domains():
     from web_update import get_web_domains_info
     return json_response(get_web_domains_info(env))
 
+
 @app.route('/web/update', methods=['POST'])
 @authorized_personnel_only
 def web_update():
@@ -306,27 +339,36 @@ def web_update():
 
 # System
 
+
 @app.route('/system/status', methods=["POST"])
 @authorized_personnel_only
 def system_status():
     from status_checks import run_checks
+
     class WebOutput:
         def __init__(self):
             self.items = []
+
         def add_heading(self, heading):
             self.items.append({"type": "heading", "text": heading, "extra": []})
+
         def print_ok(self, message):
             self.items.append({"type": "ok", "text": message, "extra": []})
+
         def print_error(self, message):
             self.items.append({"type": "error", "text": message, "extra": []})
+
         def print_warning(self, message):
             self.items.append({"type": "warning", "text": message, "extra": []})
+
         def print_line(self, message, monospace=False):
             self.items[-1]["extra"].append({"text": message, "monospace": monospace})
+
     output = WebOutput()
     run_checks(env, output, pool)
     return json_response(output.items)
 
+
 @app.route('/system/updates')
 @authorized_personnel_only
 def show_updates():
@@ -336,6 +378,7 @@ def show_updates():
         % (p["package"], p["version"])
         for p in list_apt_updates())
 
+
 @app.route('/system/update-packages', methods=["POST"])
 @authorized_personnel_only
 def do_updates():
@@ -344,6 +387,7 @@ def do_updates():
         "DEBIAN_FRONTEND": "noninteractive"
     })
 
+
 @app.route('/system/backup/status')
 @authorized_personnel_only
 def backup_status():
@@ -353,8 +397,10 @@ def backup_status():
 # APP
 
 if __name__ == '__main__':
-	if "DEBUG" in os.environ: app.debug = True
+    if "DEBUG" in os.environ:
-	if "APIKEY" in os.environ: auth_service.key = os.environ["APIKEY"]
+        app.debug = True
+    if "APIKEY" in os.environ:
+        auth_service.key = os.environ["APIKEY"]
 
     if not app.debug:
         app.logger.addHandler(utils.create_syslog_handler())
@@ -369,4 +415,3 @@ if __name__ == '__main__':
 
     # Start the application server. Listens on 127.0.0.1 (IPv4 only).
     app.run(port=10222)
-

management/dns_update.py

@@ -4,7 +4,13 @@
 # and mail aliases and restarts nsd.
 ########################################################################
 
-import os, os.path, urllib.parse, datetime, re, hashlib, base64
+import os
+import os.path
+import urllib.parse
+import datetime
+import re
+import hashlib
+import base64
 import ipaddress
 import rtyaml
 import dns.resolver
@@ -12,6 +18,7 @@ import dns.resolver
 from mailconfig import get_mail_domains
 from utils import shell, load_env_vars_from_file, safe_domain_name, sort_domains
 
+
 def get_dns_domains(env):
     # Add all domain names in use by email users and mail aliases and ensure
     # PRIMARY_HOSTNAME is in the list.
@@ -20,6 +27,7 @@ def get_dns_domains(env):
     domains.add(env['PRIMARY_HOSTNAME'])
     return domains
 
+
 def get_dns_zones(env):
     # What domains should we create DNS zones for? Never create a zone for
     # a domain & a subdomain of that domain.
@@ -50,17 +58,20 @@ def get_dns_zones(env):
 
     return zonefiles
 
+
 def get_custom_dns_config(env):
     try:
         return rtyaml.load(open(os.path.join(env['STORAGE_ROOT'], 'dns/custom.yaml')))
     except:
         return {}
 
+
 def write_custom_dns_config(config, env):
     config_yaml = rtyaml.dump(config)
     with open(os.path.join(env['STORAGE_ROOT'], 'dns/custom.yaml'), "w") as f:
         f.write(config_yaml)
 
+
 def do_dns_update(env, force=False):
     # What domains (and their zone filenames) should we build?
     domains = get_dns_domains(env)
@@ -137,6 +148,7 @@ def do_dns_update(env, force=False):
 
 ########################################################################
 
+
 def build_zone(domain, all_domains, additional_records, env, is_zone=True):
     records = []
 
@@ -156,7 +168,6 @@ def build_zone(domain, all_domains, additional_records, env, is_zone=True):
         secondary_ns = additional_records.get("_secondary_nameserver", "ns2." + env["PRIMARY_HOSTNAME"])
         records.append((None,  "NS", secondary_ns+'.', False))
 
-
     # In PRIMARY_HOSTNAME...
     if domain == env["PRIMARY_HOSTNAME"]:
         # Define ns1 and ns2.
@@ -171,7 +182,8 @@ def build_zone(domain, all_domains, additional_records, env, is_zone=True):
         # Set the A/AAAA records. Do this early for the PRIMARY_HOSTNAME so that the user cannot override them
         # and we can provide different explanatory text.
         records.append((None, "A", env["PUBLIC_IP"], "Required. Sets the IP address of the box."))
-		if env.get("PUBLIC_IPV6"): records.append((None, "AAAA", env["PUBLIC_IPV6"], "Required. Sets the IPv6 address of the box."))
+        if env.get("PUBLIC_IPV6"):
+            records.append((None, "AAAA", env["PUBLIC_IPV6"], "Required. Sets the IPv6 address of the box."))
 
         # Add a DANE TLSA record for SMTP.
         records.append(("_25._tcp", "TLSA", build_tlsa_record(env), "Recommended when DNSSEC is enabled. Advertises to mail servers connecting to the box that mandatory encryption should be used."))
@@ -194,7 +206,7 @@ def build_zone(domain, all_domains, additional_records, env, is_zone=True):
         subdomain_qname = subdomain[0:-len("." + domain)]
         subzone = build_zone(subdomain, [], additional_records, env, is_zone=False)
         for child_qname, child_rtype, child_value, child_explanation in subzone:
-			if child_qname == None:
+            if child_qname is None:
                 child_qname = subdomain_qname
             else:
                 child_qname += "." + subdomain_qname
@@ -208,7 +220,8 @@ def build_zone(domain, all_domains, additional_records, env, is_zone=True):
 
     # The user may set other records that don't conflict with our settings.
     for qname, rtype, value in get_custom_records(domain, additional_records, env):
-		if has_rec(qname, rtype): continue
+        if has_rec(qname, rtype):
+            continue
         records.append((qname, rtype, value, "(Set by user.)"))
 
     # Add defaults if not overridden by the user's custom settings (and not otherwise configured).
@@ -220,8 +233,12 @@ def build_zone(domain, all_domains, additional_records, env, is_zone=True):
         ("www", "AAAA", env.get('PUBLIC_IPV6'), "Optional. Sets the IPv6 address that www.%s resolves to, e.g. for web hosting." % domain),
     ]
     for qname, rtype, value, explanation in defaults:
-		if value is None or value.strip() == "": continue # skip IPV6 if not set
+        # skip IPV6 if not set
-		if not is_zone and qname == "www": continue # don't create any default 'www' subdomains on what are themselves subdomains
+        if value is None or value.strip() == "":
+            continue
+        # don't create any default 'www' subdomains on what are themselves subdomains
+        if not is_zone and qname == "www":
+            continue
         # Set the default record, but not if:
         # (1) there is not a user-set record of the same type already
         # (2) there is not a CNAME record already, since you can't set both and who knows what takes precedence
@@ -248,7 +265,6 @@ def build_zone(domain, all_domains, additional_records, env, is_zone=True):
         if not has_rec(dmarc_qname, "TXT", prefix="v=DMARC1; "):
             records.append((dmarc_qname, "TXT", 'v=DMARC1; p=reject', "Prevents unauthorized use of this domain name for outbound mail by requiring a valid DKIM signature."))
 
-
     # Sort the records. The None records *must* go first in the nsd zone file. Otherwise it doesn't matter.
     records.sort(key=lambda rec: list(reversed(rec[0].split(".")) if rec[0] is not None else ""))
 
@@ -256,15 +272,18 @@ def build_zone(domain, all_domains, additional_records, env, is_zone=True):
 
 ########################################################################
 
+
 def get_custom_records(domain, additional_records, env):
     for qname, value in additional_records.items():
         # We don't count the secondary nameserver config (if present) as a record - that would just be
         # confusing to users. Instead it is accessed/manipulated directly via (get/set)_custom_dns_config.
-		if qname == "_secondary_nameserver": continue
+        if qname == "_secondary_nameserver":
+                continue
 
         # Is this record for the domain or one of its subdomains?
         # If `domain` is None, return records for all domains.
-		if domain is not None and qname != domain and not qname.endswith("." + domain): continue
+        if domain is not None and qname != domain and not qname.endswith("." + domain):
+            continue
 
         # Turn the fully qualified domain name in the YAML file into
         # our short form (None => domain, or a relative QNAME) if
@@ -296,12 +315,15 @@ def get_custom_records(domain, additional_records, env):
             if rtype == "A" and value2 == "local":
                 value2 = env["PUBLIC_IP"]
             if rtype == "AAAA" and value2 == "local":
-				if "PUBLIC_IPV6" not in env: continue # no IPv6 address is available so don't set anything
+                # no IPv6 address is available so don't set anything
+                if "PUBLIC_IPV6" not in env:
+                    continue
                 value2 = env["PUBLIC_IPV6"]
             yield (qname, rtype, value2)
 
 ########################################################################
 
+
 def build_tlsa_record(env):
     # A DANE TLSA record in DNS specifies that connections on a port
     # must use TLS and the certificate must match a particular certificate.
@@ -325,6 +347,7 @@ def build_tlsa_record(env):
     # 1: The certificate is SHA256'd here.
     return "3 0 1 " + certhash
 
+
 def build_sshfp_records():
     # The SSHFP record is a way for us to embed this server's SSH public
     # key fingerprint into the DNS so that remote hosts have an out-of-band
@@ -349,7 +372,8 @@ def build_sshfp_records():
     # to the zone file (that trigger bumping the serial number).
     keys = shell("check_output", ["ssh-keyscan", "localhost"])
     for key in sorted(keys.split("\n")):
-		if key.strip() == "" or key[0] == "#": continue
+        if key.strip() == "" or key[0] == "#":
+            continue
         try:
             host, keytype, pubkey = key.split(" ")
             yield "%d %d ( %s )" % (
@@ -364,6 +388,7 @@ def build_sshfp_records():
 
 ########################################################################
 
+
 def write_nsd_zone(domain, zonefile, records, env, force):
     # On the $ORIGIN line, there's typically a ';' comment at the end explaining
     # what the $ORIGIN line does. Any further data after the domain confuses
@@ -375,7 +400,6 @@ def write_nsd_zone(domain, zonefile, records, env, force):
     # For the refresh through TTL fields, a good reference is:
     # http://www.peerwisdom.org/2013/05/15/dns-understanding-the-soa-record/
 
-
     zone = """
 $ORIGIN {domain}.
 $TTL 1800           ; default time to live
@@ -472,10 +496,12 @@ $TTL 1800           ; default time to live
     with open(zonefile, "w") as f:
         f.write(zone)
 
-	return True # file is updated
+    # file is updated
+    return True
 
 ########################################################################
 
+
 def write_nsd_conf(zonefiles, additional_records, env):
     # Basic header.
     nsdconf = """
@@ -494,7 +520,8 @@ server:
     # might have other network interfaces for e.g. tunnelling, we have
     # to be specific about the network interfaces that nsd binds to.
     for ipaddr in (env.get("PRIVATE_IP", "") + " " + env.get("PRIVATE_IPV6", "")).split(" "):
-		if ipaddr == "": continue
+        if ipaddr == "":
+            continue
         nsdconf += "  ip-address: %s\n" % ipaddr
 
     # Append the zones.
@@ -517,7 +544,6 @@ zone:
     provide-xfr: %s NOKEY
 """ % (ipaddr, ipaddr)
 
-
     # Check if the nsd.conf is changing. If it isn't changing,
     # return False to flag that no change was made.
     with open("/etc/nsd/nsd.conf") as f:
@@ -531,9 +557,9 @@ zone:
 
 ########################################################################
 
+
 def dnssec_choose_algo(domain, env):
-	if '.' in domain and domain.rsplit('.')[-1] in \
+    if '.' in domain and domain.rsplit('.')[-1] in ("email", "guide", "fund"):
-		("email", "guide", "fund"):
         # At GoDaddy, RSASHA256 is the only algorithm supported
         # for .email and .guide.
         # A variety of algorithms are supported for .fund. This
@@ -544,6 +570,7 @@ def dnssec_choose_algo(domain, env):
     # on existing users. We'll probably want to migrate to SHA256 later.
     return "RSASHA1-NSEC3-SHA1"
 
+
 def sign_zone(domain, zonefile, env):
     algo = dnssec_choose_algo(domain, env)
     dnssec_keys = load_env_vars_from_file(os.path.join(env['STORAGE_ROOT'], 'dns/dnssec/%s.conf' % algo))
@@ -562,27 +589,34 @@ def sign_zone(domain, zonefile, env):
     # we (root) can read.
     files_to_kill = []
     for key in ("KSK", "ZSK"):
-		if dnssec_keys.get(key, "").strip() == "": raise Exception("DNSSEC is not properly set up.")
+        if dnssec_keys.get(key, "").strip() == "":
+            raise Exception("DNSSEC is not properly set up.")
         oldkeyfn = os.path.join(env['STORAGE_ROOT'], 'dns/dnssec/' + dnssec_keys[key])
         newkeyfn = '/tmp/' + dnssec_keys[key].replace("_domain_", domain)
         dnssec_keys[key] = newkeyfn
         for ext in (".private", ".key"):
-			if not os.path.exists(oldkeyfn + ext): raise Exception("DNSSEC is not properly set up.")
+            if not os.path.exists(oldkeyfn + ext):
+                raise Exception("DNSSEC is not properly set up.")
             with open(oldkeyfn + ext, "r") as fr:
                 keydata = fr.read()
-			keydata = keydata.replace("_domain_", domain) # trick ldns-signkey into letting our generic key be used by this zone
+            # trick ldns-signkey into letting our generic key be used by this zone
+            keydata = keydata.replace("_domain_", domain)
             fn = newkeyfn + ext
-			prev_umask = os.umask(0o77) # ensure written file is not world-readable
+            # ensure written file is not world-readable
+            prev_umask = os.umask(0o77)
             try:
                 with open(fn, "w") as fw:
                     fw.write(keydata)
             finally:
-				os.umask(prev_umask) # other files we write should be world-readable
+                # other files we write should be world-readable
+                os.umask(prev_umask)
             files_to_kill.append(fn)
 
     # Do the signing.
     expiry_date = (datetime.datetime.now() + datetime.timedelta(days=30)).strftime("%Y%m%d")
-	shell('check_call', ["/usr/bin/ldns-signzone",
+    shell('check_call', [
+        "/usr/bin/ldns-signzone",
+
         # expire the zone after 30 days
         "-e", expiry_date,
 
@@ -607,7 +641,8 @@ def sign_zone(domain, zonefile, env):
     # actually be deployed. Preferebly the first.
     with open("/etc/nsd/zones/" + zonefile + ".ds", "w") as f:
         for digest_type in ('2', '1'):
-			rr_ds = shell('check_output', ["/usr/bin/ldns-key2ds",
+            rr_ds = shell('check_output', [
+                "/usr/bin/ldns-key2ds",
                 "-n",  # output to stdout
                 "-" + digest_type,  # 1=SHA1, 2=SHA256
                 dnssec_keys["KSK"] + ".key"
@@ -620,6 +655,7 @@ def sign_zone(domain, zonefile, env):
 
 ########################################################################
 
+
 def write_opendkim_tables(domains, env):
     # Append a record to OpenDKIM's KeyTable and SigningTable for each domain
     # that we send mail from (zones and all subdomains).
@@ -638,8 +674,7 @@ def write_opendkim_tables(domains, env):
         # Elsewhere we set the DMARC policy for each domain such that mail claiming
         # to be From: the domain must be signed with a DKIM key on the same domain.
         # So we must have a separate KeyTable entry for each domain.
-		"SigningTable":
+        "SigningTable": "".join(
-			"".join(
             "*@{domain} {domain}\n".format(domain=domain)
             for domain in domains
         ),
@@ -647,8 +682,7 @@ def write_opendkim_tables(domains, env):
         # The KeyTable specifies the signing domain, the DKIM selector, and the
         # path to the private key to use for signing some mail. Per DMARC, the
         # signing domain must match the sender's From: domain.
-		"KeyTable":
+        "KeyTable": "".join(
-			"".join(
             "{domain} {domain}:mail:{key_file}\n".format(domain=domain, key_file=opendkim_key_file)
             for domain in domains
         ),
@@ -673,6 +707,7 @@ def write_opendkim_tables(domains, env):
 
 ########################################################################
 
+
 def set_custom_dns_record(qname, rtype, value, env):
     # validate qname
     for zone, fn in get_dns_zones(env):
@@ -689,8 +724,10 @@ def set_custom_dns_record(qname, rtype, value, env):
     if value is not None:
         if rtype in ("A", "AAAA"):
             v = ipaddress.ip_address(value)
-			if rtype == "A" and not isinstance(v, ipaddress.IPv4Address): raise ValueError("That's an IPv6 address.")
+            if rtype == "A" and not isinstance(v, ipaddress.IPv4Address):
-			if rtype == "AAAA" and not isinstance(v, ipaddress.IPv6Address): raise ValueError("That's an IPv4 address.")
+                raise ValueError("That's an IPv6 address.")
+            if rtype == "AAAA" and not isinstance(v, ipaddress.IPv6Address):
+                raise ValueError("That's an IPv4 address.")
         elif rtype in ("CNAME", "TXT", "SRV"):
             # anything goes
             pass
@@ -754,6 +791,7 @@ def set_custom_dns_record(qname, rtype, value, env):
 
 ########################################################################
 
+
 def set_secondary_dns(hostname, env):
     config = get_custom_dns_config(env)
 
@@ -786,16 +824,21 @@ def justtestingdotemail(domain, records):
     # Ideally if dns4e.com supported NS records we would just have it
     # delegate DNS to us, but instead we will populate the whole zone.
 
-	import subprocess, json, urllib.parse
+    import subprocess
+    import json
+    import urllib.parse
 
     if not domain.endswith(".justtesting.email"):
         return
 
     for subdomain, querytype, value, explanation in records:
-		if querytype in ("NS",): continue
+        if querytype in ("NS",):
-		if subdomain in ("www", "ns1", "ns2"): continue # don't do unnecessary things
+            continue
+        # don't do unnecessary things
+        if subdomain in ("www", "ns1", "ns2"):
+            continue
 
-		if subdomain == None:
+        if subdomain is None:
             subdomain = domain
         else:
             subdomain = subdomain + "." + domain
@@ -821,6 +864,7 @@ def justtestingdotemail(domain, records):
 
 ########################################################################
 
+
 def build_recommended_dns(env):
     ret = []
     domains = get_dns_domains(env)
@@ -837,7 +881,7 @@ def build_recommended_dns(env):
 
         # expand qnames
         for i in range(len(records)):
-			if records[i][0] == None:
+            if records[i][0] is None:
                 qname = domain
             else:
                 qname = records[i][0] + "." + domain

management/mail_log.py

@@ -1,11 +1,13 @@
 #!/usr/bin/python3
 
-import re, os.path
+import re
+import os.path
 import dateutil.parser
 
 import mailconfig
 import utils
 
+
 def scan_mail_log(logger, env):
     collector = {
         "other-services": set(),
@@ -17,7 +19,8 @@ def scan_mail_log(logger, env):
     collector["real_mail_addresses"] = set(mailconfig.get_mail_users(env)) | set(alias[0] for alias in mailconfig.get_mail_aliases(env))
 
     for fn in ('/var/log/mail.log.1', '/var/log/mail.log'):
-		if not os.path.exists(fn): continue
+        if not os.path.exists(fn):
+            continue
         with open(fn, 'rb') as log:
             for line in log:
                 line = line.decode("utf8", errors='replace')
@@ -49,9 +52,11 @@ def scan_mail_log(logger, env):
         logger.add_heading("Other")
         logger.print_block("Unrecognized services in the log: " + ", ".join(collector["other-services"]))
 
+
 def scan_mail_log_line(line, collector):
     m = re.match(r"(\S+ \d+ \d+:\d+:\d+) (\S+) (\S+?)(\[\d+\])?: (.*)", line)
-	if not m: return
+    if not m:
+        return
 
     date, system, service, pid, log = m.groups()
     date = dateutil.parser.parse(date)
@@ -66,14 +71,16 @@ def scan_mail_log_line(line, collector):
         scan_postfix_smtpd_line(date, log, collector)
 
     elif service in ("postfix/qmgr", "postfix/pickup", "postfix/cleanup",
-			"postfix/scache", "spampd", "postfix/anvil", "postfix/master",
+                     "postfix/scache", "spampd", "postfix/anvil",
-			"opendkim", "postfix/lmtp", "postfix/tlsmgr"):
+                     "postfix/master", "opendkim", "postfix/lmtp",
+                     "postfix/tlsmgr"):
         # nothing to look at
         pass
 
     else:
         collector["other-services"].add(service)
 
+
 def scan_dovecot_line(date, log, collector):
     m = re.match("imap-login: Login: user=<(.*?)>, method=PLAIN, rip=(.*?),", log)
     if m:
@@ -81,6 +88,7 @@ def scan_dovecot_line(date, log, collector):
         if ip != "127.0.0.1":  # local login from webmail/zpush
             collector["imap-logins"].setdefault(login, {})[ip] = date
 
+
 def scan_postgrey_line(date, log, collector):
     m = re.match("action=(greylist|pass), reason=(.*?), (?:delay=\d+, )?client_name=(.*), client_address=(.*), sender=(.*), recipient=(.*)", log)
     if m:
@@ -91,6 +99,7 @@ def scan_postgrey_line(date, log, collector):
         elif action == "pass" and reason == "triplet found" and key in collector["postgrey"].get(recipient, {}):
             collector["postgrey"][recipient][key] = (collector["postgrey"][recipient][key][0], date)
 
+
 def scan_postfix_smtpd_line(date, log, collector):
     m = re.match("NOQUEUE: reject: RCPT from .*?: (.*?); from=<(.*?)> to=<(.*?)>", log)
     if m:

management/mailconfig.py

@@ -1,8 +1,13 @@
 #!/usr/bin/python3
 
-import subprocess, shutil, os, sqlite3, re
+import subprocess
+import shutil
+import os
+import sqlite3
+import re
 import utils
 
+
 def validate_email(email, mode=None):
     # There are a lot of characters permitted in email addresses, but
     # Dovecot's sqlite driver seems to get confused if there are any
@@ -10,7 +15,8 @@ def validate_email(email, mode=None):
     # the mailbox path name is based on the email address, the address
     # shouldn't be absurdly long and must not have a forward slash.
 
-	if len(email) > 255: return False
+    if len(email) > 255:
+        return False
 
     if mode == 'user':
         # For Dovecot's benefit, only allow basic characters.
@@ -40,7 +46,8 @@ def validate_email(email, mode=None):
 
     # Check the regular expression.
     m = re.match(ADDR_SPEC, email)
-	if not m: return False
+    if not m:
+        return False
 
     # Check that the domain part is IDNA-encodable.
     localpart, domainpart = m.groups()
@@ -51,6 +58,7 @@ def validate_email(email, mode=None):
 
     return True
 
+
 def sanitize_idn_email_address(email):
     # Convert an IDNA-encoded email address (domain part) into Unicode
     # before storing in our database. Chrome may IDNA-ize <input type="email">
@@ -65,6 +73,7 @@ def sanitize_idn_email_address(email):
         # leave unchanged.
         return email
 
+
 def open_database(env, with_connection=False):
     conn = sqlite3.connect(env["STORAGE_ROOT"] + "/mail/users.sqlite")
     if not with_connection:
@@ -72,6 +81,7 @@ def open_database(env, with_connection=False):
     else:
         return conn, conn.cursor()
 
+
 def get_mail_users(env):
     # Returns a flat, sorted list of all user accounts.
     c = open_database(env)
@@ -79,6 +89,7 @@ def get_mail_users(env):
     users = [row[0] for row in c.fetchall()]
     return utils.sort_email_addresses(users, env)
 
+
 def get_mail_users_ex(env, with_archived=False, with_slow_info=False):
     # Returns a complex data structure of all user accounts, optionally
     # including archived (status="inactive") accounts.
@@ -134,7 +145,8 @@ def get_mail_users_ex(env, with_archived=False, with_slow_info=False):
             for user in os.listdir(os.path.join(root, domain)):
                 email = user + "@" + domain
                 mbox = os.path.join(root, domain, user)
-				if email in active_accounts: continue
+                if email in active_accounts:
+                    continue
                 user = {
                     "email": email,
                     "privileges": "",
@@ -165,6 +177,7 @@ def get_mail_users_ex(env, with_archived=False, with_slow_info=False):
 
     return domains
 
+
 def get_admins(env):
     # Returns a set of users with admin privileges.
     users = set()
@@ -174,16 +187,19 @@ def get_admins(env):
                 users.add(user["email"])
     return users
 
+
 def get_mail_aliases(env):
     # Returns a sorted list of tuples of (alias, forward-to string).
     c = open_database(env)
     c.execute('SELECT source, destination FROM aliases')
-	aliases = { row[0]: row[1] for row in c.fetchall() } # make dict
+    # make dict
+    aliases = {row[0]: row[1] for row in c.fetchall()}
 
     # put in a canonical order: sort by domain, then by email address lexicographically
     aliases = [(source, aliases[source]) for source in utils.sort_email_addresses(aliases.keys(), env)]
     return aliases
 
+
 def get_mail_aliases_ex(env):
     # Returns a complex data structure of all mail aliases, similar
     # to get_mail_users_ex.
@@ -230,6 +246,7 @@ def get_mail_aliases_ex(env):
         domain["aliases"].sort(key=lambda alias: (alias["required"], alias["source"]))
     return domains
 
+
 def get_mail_alias_map(env):
     aliases = {}
     for alias, targets in get_mail_aliases(env):
@@ -238,6 +255,7 @@ def get_mail_alias_map(env):
             aliases.setdefault(em, []).append(alias)
     return aliases
 
+
 def evaluate_mail_alias_map(email, aliases, env):
     ret = set()
     for alias in aliases.get(email.lower(), []):
@@ -245,15 +263,18 @@ def evaluate_mail_alias_map(email, aliases,  env):
         ret |= evaluate_mail_alias_map(alias, aliases, env)
     return ret
 
+
 def get_domain(emailaddr):
     return emailaddr.split('@', 1)[1]
 
+
 def get_mail_domains(env, filter_aliases=lambda alias: True):
     return set(
-		   [get_domain(addr) for addr in get_mail_users(env)]
+        [get_domain(addr) for addr in get_mail_users(env)] +
-		 + [get_domain(source) for source, target in get_mail_aliases(env) if filter_aliases((source, target)) ]
+        [get_domain(source) for source, target in get_mail_aliases(env) if filter_aliases((source, target))]
     )
 
+
 def add_mail_user(email, pw, privs, env):
     # accept IDNA domain names but normalize to Unicode before going into database
     email = sanitize_idn_email_address(email)
@@ -273,7 +294,8 @@ def add_mail_user(email, pw, privs, env):
         privs = privs.split("\n")
         for p in privs:
             validation = validate_privilege(p)
-			if validation: return validation
+            if validation:
+                return validation
 
     # get the database
     conn, c = open_database(env, with_connection=True)
@@ -311,6 +333,7 @@ def add_mail_user(email, pw, privs, env):
     # Update things in case any new domains are added.
     return kick(env, "mail user added")
 
+
 def set_mail_password(email, pw, env):
     # accept IDNA domain names but normalize to Unicode before going into database
     email = sanitize_idn_email_address(email)
@@ -329,12 +352,14 @@ def set_mail_password(email, pw, env):
     conn.commit()
     return "OK"
 
+
 def hash_password(pw):
     # Turn the plain password into a Dovecot-format hashed password, meaning
     # something like "{SCHEME}hashedpassworddata".
     # http://wiki2.dovecot.org/Authentication/PasswordSchemes
     return utils.shell('check_output', ["/usr/bin/doveadm", "pw", "-s", "SHA512-CRYPT", "-p", pw]).strip()
 
+
 def get_mail_password(email, env):
     # Gets the hashed password for a user. Passwords are stored in Dovecot's
     # password format, with a prefixed scheme.
@@ -347,6 +372,7 @@ def get_mail_password(email, env):
         raise ValueError("That's not a user (%s)." % email)
     return rows[0][0]
 
+
 def remove_mail_user(email, env):
     # accept IDNA domain names but normalize to Unicode before going into database
     email = sanitize_idn_email_address(email)
@@ -361,9 +387,11 @@ def remove_mail_user(email, env):
     # Update things in case any domains are removed.
     return kick(env, "mail user removed")
 
+
 def parse_privs(value):
     return [p for p in value.split("\n") if p.strip() != ""]
 
+
 def get_mail_user_privileges(email, env):
     # accept IDNA domain names but normalize to Unicode before going into database
     email = sanitize_idn_email_address(email)
@@ -376,22 +404,27 @@ def get_mail_user_privileges(email, env):
         return ("That's not a user (%s)." % email, 400)
     return parse_privs(rows[0][0])
 
+
 def validate_privilege(priv):
     if "\n" in priv or priv.strip() == "":
         return ("That's not a valid privilege (%s)." % priv, 400)
     return None
 
+
 def add_remove_mail_user_privilege(email, priv, action, env):
     # accept IDNA domain names but normalize to Unicode before going into database
     email = sanitize_idn_email_address(email)
 
     # validate
     validation = validate_privilege(priv)
-	if validation: return validation
+    if validation:
+        return validation
 
     # get existing privs, but may fail
     privs = get_mail_user_privileges(email, env)
-	if isinstance(privs, tuple): return privs # error
+    # error
+    if isinstance(privs, tuple):
+        return privs
 
     # update privs set
     if action == "add":
@@ -411,6 +444,7 @@ def add_remove_mail_user_privilege(email, priv, action, env):
 
     return "OK"
 
+
 def add_mail_alias(source, destination, env, update_if_exists=False, do_kick=True):
     # accept IDNA domain names but normalize to Unicode before going into database
     source = sanitize_idn_email_address(source)
@@ -434,8 +468,10 @@ def add_mail_alias(source, destination, env, update_if_exists=False, do_kick=Tru
         for line in destination.split("\n"):
             for email in line.split(","):
                 email = email.strip()
-				email = sanitize_idn_email_address(email) # Unicode => IDNA
+                # Unicode => IDNA
-				if email == "": continue
+                email = sanitize_idn_email_address(email)
+                if email == "":
+                    continue
                 if not validate_email(email):
                     return ("Invalid destination email address (%s)." % email, 400)
                 dests.append(email)
@@ -461,6 +497,7 @@ def add_mail_alias(source, destination, env, update_if_exists=False, do_kick=Tru
         # Update things in case any new domains are added.
         return kick(env, return_status)
 
+
 def remove_mail_alias(source, env, do_kick=True):
     # accept IDNA domain names but normalize to Unicode before going into database
     source = sanitize_idn_email_address(source)
@@ -476,9 +513,11 @@ def remove_mail_alias(source, env, do_kick=True):
         # Update things in case any domains are removed.
         return kick(env, "alias removed")
 
+
 def get_system_administrator(env):
     return "administrator@" + env['PRIMARY_HOSTNAME']
 
+
 def get_required_aliases(env):
     # These are the aliases that must exist.
     aliases = set()
@@ -489,6 +528,7 @@ def get_required_aliases(env):
     # Get a list of domains we serve mail for, except ones for which the only
     # email on that domain is a postmaster/admin alias to the administrator
     # or a wildcard alias (since it will forward postmaster/admin).
+    # JMT: no clean way to PEP8 wrap this
     real_mail_domains = get_mail_domains(env,
         filter_aliases=lambda alias: 
             ((not alias[0].startswith("postmaster@") and not alias[0].startswith("admin@")) or alias[1] != get_system_administrator(env))
@@ -505,6 +545,7 @@ def get_required_aliases(env):
 
     return aliases
 
+
 def kick(env, mail_result=None):
     results = []
 
@@ -533,7 +574,6 @@ def kick(env, mail_result=None):
         add_mail_alias(source, administrator, env, do_kick=False)
         results.append("added alias %s (=> %s)\n" % (source, administrator))
 
-
     for alias in required_aliases:
         ensure_admin_alias_exists(alias)
 
@@ -541,9 +581,7 @@ def kick(env, mail_result=None):
     # longer have any other email addresses for.
     for source, target in existing_aliases:
         user, domain = source.split("@")
-		if user in ("postmaster", "admin") \
+        if user in ("postmaster", "admin") and source not in required_aliases and target == get_system_administrator(env):
-			and source not in required_aliases \
-			and target == get_system_administrator(env):
             remove_mail_alias(source, env, do_kick=False)
             results.append("removed alias %s (was to %s; domain no longer used for email)\n" % (source, target))
 
@@ -557,6 +595,7 @@ def kick(env, mail_result=None):
 
     return "".join(s for s in results if s != "")
 
+
 def validate_password(pw):
     # validate password
     if pw.strip() == "":

management/status_checks.py

@@ -6,10 +6,17 @@
 
 __ALL__ = ['check_certificate']
 
-import os, os.path, re, subprocess, datetime, multiprocessing.pool
+import os
+import os.path
+import re
+import subprocess
+import datetime
+import multiprocessing.pool
 
-import dns.reversename, dns.resolver
+import dns.reversename
-import dateutil.parser, dateutil.tz
+import dns.resolver
+import dateutil.parser
+import dateutil.tz
 
 from dns_update import get_dns_zones, build_tlsa_record, get_custom_dns_config
 from web_update import get_web_domains, get_domain_ssl_files
@@ -17,6 +24,7 @@ from mailconfig import get_mail_domains, get_mail_aliases
 
 from utils import shell, sort_domains, load_env_vars_from_file
 
+
 def run_checks(env, output, pool):
     # run systems checks
     output.add_heading("System")
@@ -40,6 +48,7 @@ def run_checks(env, output, pool):
     run_network_checks(env, output)
     run_domain_checks(env, output, pool)
 
+
 def get_ssh_port():
     # Returns ssh port
     output = shell('check_output', ['sshd', '-T'])
@@ -51,6 +60,7 @@ def get_ssh_port():
         if e == "port":
             returnNext = True
 
+
 def run_services_checks(env, output, pool):
     # Check that system services are running.
 
@@ -90,6 +100,7 @@ def run_services_checks(env, output, pool):
 
     return not fatal
 
+
 def check_service(i, service, env):
     import socket
     output = BufferedOutput()
@@ -126,19 +137,21 @@ def check_service(i, service, env):
             output.print_line(shell('check_output', ['nginx', '-t'], capture_stderr=True, trap=True)[1].strip())
 
         # Flag if local DNS is not running.
-		if service["port"] == 53 and service["public"] == False:
+        if service["port"] == 53 and service["public"] is False:
             fatal = True
     finally:
         s.close()
 
     return (i, running, fatal, output)
 
+
 def run_system_checks(env, output):
     check_ssh_password(env, output)
     check_software_updates(env, output)
     check_system_aliases(env, output)
     check_free_disk_space(env, output)
 
+
 def check_ssh_password(env, output):
     # Check that SSH login with password is disabled. The openssh-server
     # package may not be installed so check that before trying to access
@@ -146,8 +159,7 @@ def check_ssh_password(env, output):
     if not os.path.exists("/etc/ssh/sshd_config"):
         return
     sshd = open("/etc/ssh/sshd_config").read()
-	if re.search("\nPasswordAuthentication\s+yes", sshd) \
+    if re.search("\nPasswordAuthentication\s+yes", sshd) or not re.search("\nPasswordAuthentication\s+no", sshd):
-		or not re.search("\nPasswordAuthentication\s+no", sshd):
         output.print_error("""The SSH server on this machine permits password-based login. A more secure
             way to log in is using a public key. Add your SSH public key to $HOME/.ssh/authorized_keys, check
             that you can log in without a password, set the option 'PasswordAuthentication no' in
@@ -155,6 +167,7 @@ def check_ssh_password(env, output):
     else:
         output.print_ok("SSH disallows password-based login.")
 
+
 def check_software_updates(env, output):
     # Check for any software package updates.
     pkgs = list_apt_updates(apt_update=False)
@@ -167,11 +180,13 @@ def check_software_updates(env, output):
         for p in pkgs:
             output.print_line("%s (%s)" % (p["package"], p["version"]))
 
+
 def check_system_aliases(env, output):
     # Check that the administrator alias exists since that's where all
     # admin email is automatically directed.
     check_alias_exists("administrator@" + env['PRIMARY_HOSTNAME'], env, output)
 
+
 def check_free_disk_space(env, output):
     # Check free disk space.
     st = os.statvfs(env['STORAGE_ROOT'])
@@ -185,6 +200,7 @@ def check_free_disk_space(env, output):
     else:
         output.print_error(disk_msg)
 
+
 def run_network_checks(env, output):
     # Also see setup/network-checks.sh.
 
@@ -215,6 +231,7 @@ def run_network_checks(env, output):
             which may prevent recipients from receiving your email. See http://www.spamhaus.org/query/ip/%s."""
                            % (env['PUBLIC_IP'], zen, env['PUBLIC_IP']))
 
+
 def run_domain_checks(env, output, pool):
     # Get the list of domains we handle mail for.
     mail_domains = get_mail_domains(env)
@@ -236,10 +253,12 @@ def run_domain_checks(env, output, pool):
     args = ((domain, env, dns_domains, dns_zonefiles, mail_domains, web_domains)
             for domain in domains_to_check)
     ret = pool.starmap(run_domain_checks_on_domain, args, chunksize=1)
-	ret = dict(ret) # (domain, output) => { domain: output }
+    # (domain, output) => { domain: output }
+    ret = dict(ret)
     for domain in sort_domains(ret, env):
         ret[domain].playback(output)
 
+
 def run_domain_checks_on_domain(domain, env, dns_domains, dns_zonefiles, mail_domains, web_domains):
     output = BufferedOutput()
 
@@ -262,6 +281,7 @@ def run_domain_checks_on_domain(domain, env, dns_domains, dns_zonefiles, mail_do
 
     return (domain, output)
 
+
 def check_primary_hostname_dns(domain, env, output, dns_domains, dns_zonefiles):
     # If a DS record is set on the zone containing this domain, check DNSSEC now.
     for zone in dns_domains:
@@ -300,8 +320,7 @@ def check_primary_hostname_dns(domain, env, output, dns_domains, dns_zonefiles):
     if existing_rdns == domain:
         output.print_ok("Reverse DNS is set correctly at ISP. [%s => %s]" % (env['PUBLIC_IP'], env['PRIMARY_HOSTNAME']))
     else:
-		output.print_error("""Your box's reverse DNS is currently %s, but it should be %s. Your ISP or cloud provider will have instructions
+        output.print_error("""Your box's reverse DNS is currently %s, but it should be %s. Your ISP or cloud provider will have instructions on setting up reverse DNS for your box at %s.""" % (existing_rdns, domain, env['PUBLIC_IP']))
-			on setting up reverse DNS for your box at %s.""" % (existing_rdns, domain, env['PUBLIC_IP']) )
 
     # Check the TLSA record.
     tlsa_qname = "_25._tcp." + domain
@@ -319,6 +338,7 @@ def check_primary_hostname_dns(domain, env, output, dns_domains, dns_zonefiles):
     # Check that the hostmaster@ email address exists.
     check_alias_exists("hostmaster@" + domain, env, output)
 
+
 def check_alias_exists(alias, env, output):
     mail_alises = dict(get_mail_aliases(env))
     if alias in mail_alises:
@@ -326,6 +346,7 @@ def check_alias_exists(alias, env, output):
     else:
         output.print_error("""You must add a mail alias for %s and direct email to you or another administrator.""" % alias)
 
+
 def check_dns_zone(domain, env, output, dns_zonefiles):
     # If a DS record is set at the registrar, check DNSSEC first because it will affect the NS query.
     # If it is not set, we suggest it last.
@@ -351,6 +372,7 @@ def check_dns_zone(domain, env, output, dns_zonefiles):
             control panel to set the nameservers to %s."""
                            % (existing_ns, correct_ns))
 
+
 def check_dns_zone_suggestions(domain, env, output, dns_zonefiles):
     # Since DNSSEC is optional, if a DS record is NOT set at the registrar suggest it.
     # (If it was set, we did the check earlier.)
@@ -377,13 +399,16 @@ def check_dnssec(domain, env, output, dns_zonefiles, is_checking_primary=False):
     # Query public DNS for the DS record at the registrar.
     ds = query_dns(domain, "DS", nxdomain=None)
     ds_looks_valid = ds and len(ds.split(" ")) == 4
-	if ds_looks_valid: ds = ds.split(" ")
+    if ds_looks_valid:
+        ds = ds.split(" ")
     if ds_looks_valid and ds[0] == ds_keytag and ds[1] == ds_alg and ds[3] == digests.get(ds[2]):
-		if is_checking_primary: return
+        if is_checking_primary:
+            return
         output.print_ok("DNSSEC 'DS' record is set correctly at registrar.")
     else:
-		if ds == None:
+        if ds is None:
-			if is_checking_primary: return
+            if is_checking_primary:
+                return
             output.print_error("""This domain's DNSSEC DS record is not set. The DS record is optional. The DS record activates DNSSEC.
                 To set a DS record, you must follow the instructions provided by your domain name registrar and provide to them this information:""")
         else:
@@ -398,8 +423,8 @@ def check_dnssec(domain, env, output, dns_zonefiles, is_checking_primary=False):
         output.print_line("Key Tag: " + ds_keytag + ("" if not ds_looks_valid or ds[0] == ds_keytag else " (Got '%s')" % ds[0]))
         output.print_line("Key Flags: KSK")
         output.print_line(
-			  ("Algorithm: %s / %s" % (ds_alg, alg_name_map[ds_alg]))
+            ("Algorithm: %s / %s" % (ds_alg, alg_name_map[ds_alg])) +
-			+ ("" if not ds_looks_valid or ds[1] == ds_alg else " (Got '%s')" % ds[1]))
+            ("" if not ds_looks_valid or ds[1] == ds_alg else " (Got '%s')" % ds[1]))
             # see http://www.iana.org/assignments/dns-sec-alg-numbers/dns-sec-alg-numbers.xhtml
         output.print_line("Digest Type: 2 / SHA-256")
             # http://www.ietf.org/assignments/ds-rr-types/ds-rr-types.xml
@@ -413,6 +438,7 @@ def check_dnssec(domain, env, output, dns_zonefiles, is_checking_primary=False):
         output.print_line("" + ds_correct[0])
         output.print_line("")
 
+
 def check_mail_domain(domain, env, output):
     # Check the MX record.
 
@@ -422,7 +448,7 @@ def check_mail_domain(domain, env, output):
     if mx == expected_mx:
         output.print_ok("Domain's email is directed to this domain. [%s => %s]" % (domain, mx))
 
-	elif mx == None:
+    elif mx is None:
         # A missing MX record is okay on the primary hostname because
         # the primary hostname's A record (the MX fallback) is... itself,
         # which is what we want the MX to be.
@@ -435,7 +461,7 @@ def check_mail_domain(domain, env, output):
         else:
             domain_a = query_dns(domain, "A", nxdomain=None)
             primary_a = query_dns(env['PRIMARY_HOSTNAME'], "A", nxdomain=None)
-			if domain_a != None and domain_a == primary_a:
+            if domain_a is not None and domain_a == primary_a:
                 output.print_ok("Domain's email is directed to this domain. [%s has no MX record but its A record is OK]" % (domain,))
             else:
                 output.print_error("""This domain's DNS MX record is not set. It should be '%s'. Mail will not
@@ -463,6 +489,7 @@ def check_mail_domain(domain, env, output):
             which may prevent recipients from receiving your mail.
             See http://www.spamhaus.org/dbl/ and http://www.spamhaus.org/query/domain/%s.""" % (dbl, domain))
 
+
 def check_web_domain(domain, env, output):
     # See if the domain's A record resolves to our PUBLIC_IP. This is already checked
     # for PRIMARY_HOSTNAME, for which it is required for mail specifically. For it and
@@ -481,6 +508,7 @@ def check_web_domain(domain, env, output):
     # website for also needs a signed certificate.
     check_ssl_cert(domain, env, output)
 
+
 def query_dns(qname, rtype, nxdomain='[Not Set]'):
     # Make the qname absolute by appending a period. Without this, dns.resolver.query
     # will fall back a failed lookup to a second query with this machine's hostname
@@ -506,11 +534,13 @@ def query_dns(qname, rtype, nxdomain='[Not Set]'):
     # can compare to a well known order.
     return "; ".join(sorted(str(r).rstrip('.') for r in response))
 
+
 def check_ssl_cert(domain, env, output):
     # Check that SSL certificate is signed.
 
     # Skip the check if the A record is not pointed here.
-	if query_dns(domain, "A", None) not in (env['PUBLIC_IP'], None): return
+    if query_dns(domain, "A", None) not in (env['PUBLIC_IP'], None):
+        return
 
     # Where is the SSL stored?
     ssl_key, ssl_certificate, ssl_via = get_domain_ssl_files(domain, env)
@@ -560,6 +590,7 @@ def check_ssl_cert(domain, env, output):
             output.print_line(cert_status_details)
             output.print_line("")
 
+
 def check_certificate(domain, ssl_certificate, ssl_private_key):
     # Use openssl verify to check the status of a certificate.
 
@@ -640,7 +671,7 @@ def check_certificate(domain, ssl_certificate, ssl_private_key):
 
     cert = open(ssl_certificate).read()
     m = re.match(r'(-*BEGIN CERTIFICATE-*.*?-*END CERTIFICATE-*)(.*)', cert, re.S)
-	if m == None:
+    if m is None:
         return ("The certificate file is an invalid PEM certificate.", None)
     mycert, chaincerts = m.groups()
 
@@ -679,6 +710,8 @@ def check_certificate(domain, ssl_certificate, ssl_private_key):
         return ("OK", expiry_info)
 
 _apt_updates = None
+
+
 def list_apt_updates(apt_update=True):
     # See if we have this information cached recently.
     # Keep the information for 8 hours.
@@ -743,7 +776,8 @@ class ConsoleOutput:
                 print()
                 print("   ", end="")
                 linelen = 0
-			if linelen == 0 and w.strip() == "": continue
+            if linelen == 0 and w.strip() == "":
+                continue
             print(w, end="")
             linelen += len(w)
         print()
@@ -752,17 +786,21 @@ class ConsoleOutput:
         for line in message.split("\n"):
             self.print_block(line)
 
+
 class BufferedOutput:
     # Record all of the instance method calls so we can play them back later.
     def __init__(self):
         self.buf = []
+
     def __getattr__(self, attr):
         if attr not in ("add_heading", "print_ok", "print_error", "print_warning", "print_block", "print_line"):
             raise AttributeError
         # Return a function that just records the call & arguments to our buffer.
+
         def w(*args, **kwargs):
             self.buf.append((attr, args, kwargs))
         return w
+
     def playback(self, output):
         for attr, args, kwargs in self.buf:
             getattr(output, attr)(*args, **kwargs)
@@ -787,5 +825,3 @@ if __name__ == "__main__":
         if cert_status != "OK":
             sys.exit(1)
         sys.exit(0)
-
-

management/utils.py

@@ -2,27 +2,33 @@ import os.path
 
 CONF_DIR = os.path.join(os.path.dirname(__file__), "../conf")
 
+
 def load_environment():
     # Load settings from /etc/mailinabox.conf.
     return load_env_vars_from_file("/etc/mailinabox.conf")
 
+
 def load_env_vars_from_file(fn):
     # Load settings from a KEY=VALUE file.
     import collections
     env = collections.OrderedDict()
-    for line in open(fn): env.setdefault(*line.strip().split("=", 1))
+    for line in open(fn):
+        env.setdefault(*line.strip().split("=", 1))
     return env
 
+
 def save_environment(env):
     with open("/etc/mailinabox.conf", "w") as f:
         for k, v in env.items():
             f.write("%s=%s\n" % (k, v))
 
+
 def safe_domain_name(name):
     # Sanitize a domain name so it is safe to use as a file name on disk.
     import urllib.parse
     return urllib.parse.quote(name, safe='')
 
+
 def sort_domains(domain_names, env):
     # Put domain names in a nice sorted order. For web_update, PRIMARY_HOSTNAME
     # must appear first so it becomes the nginx default server.
@@ -51,6 +57,7 @@ def sort_domains(domain_names, env):
 
     return groups[0] + groups[1] + groups[2]
 
+
 def sort_email_addresses(email_addresses, env):
     email_addresses = set(email_addresses)
     domains = set(email.split("@", 1)[1] for email in email_addresses if "@" in email)
@@ -59,13 +66,17 @@ def sort_email_addresses(email_addresses, env):
         domain_emails = set(email for email in email_addresses if email.endswith("@" + domain))
         ret.extend(sorted(domain_emails))
         email_addresses -= domain_emails
-    ret.extend(sorted(email_addresses)) # whatever is left
+    # whatever is left
+    ret.extend(sorted(email_addresses))
     return ret
 
+
 def exclusive_process(name):
     # Ensure that a process named `name` does not execute multiple
     # times concurrently.
-    import os, sys, atexit
+    import os
+    import sys
+    import atexit
     pidfile = '/var/run/mailinabox-%s.pid' % name
     mypid = os.getpid()
 
@@ -95,7 +106,8 @@ def exclusive_process(name):
                 try:
                     existing_pid = int(f.read().strip())
                 except ValueError:
-                    pass # No valid integer in the file.
+                    # No valid integer in the file.
+                    pass
 
                 # Check if the pid in it is valid.
                 if existing_pid:
@@ -118,20 +130,26 @@ def clear_my_pid(pidfile):
 def is_pid_valid(pid):
     """Checks whether a pid is a valid process ID of a currently running process."""
     # adapted from http://stackoverflow.com/questions/568271/how-to-check-if-there-exists-a-process-with-a-given-pid
-    import os, errno
+    import os
-    if pid <= 0: raise ValueError('Invalid PID.')
+    import errno
+    if pid <= 0:
+        raise ValueError('Invalid PID.')
     try:
         os.kill(pid, 0)
     except OSError as err:
-        if err.errno == errno.ESRCH: # No such process
+        # No such process
+        if err.errno == errno.ESRCH:
             return False
-        elif err.errno == errno.EPERM: # Not permitted to send signal
+        # Not permitted to send signal
+        elif err.errno == errno.EPERM:
             return True
-        else: # EINVAL
+        # EINVAL
+        else:
             raise
     else:
         return True
 
+
 def shell(method, cmd_args, env={}, capture_stderr=False, return_bytes=False, trap=False, input=None):
     # A safe way to execute processes.
     # Some processes like apt-get require being given a sane PATH.
@@ -154,18 +172,21 @@ def shell(method, cmd_args, env={}, capture_stderr=False, return_bytes=False, tr
         except subprocess.CalledProcessError as e:
             ret = e.output
             code = e.returncode
-    if not return_bytes and isinstance(ret, bytes): ret = ret.decode("utf8")
+    if not return_bytes and isinstance(ret, bytes):
+        ret = ret.decode("utf8")
     if not trap:
         return ret
     else:
         return code, ret
 
+
 def create_syslog_handler():
     import logging.handlers
     handler = logging.handlers.SysLogHandler(address='/dev/log')
     handler.setLevel(logging.WARNING)
     return handler
 
+
 def du(path):
     # Computes the size of all files in the path, like the `du` command.
     # Based on http://stackoverflow.com/a/17936789. Takes into account

management/web_update.py

@@ -2,12 +2,18 @@
 # domains for which a mail account has been set up.
 ########################################################################
 
-import os, os.path, shutil, re, tempfile, rtyaml
+import os
+import os.path
+import shutil
+import re
+import tempfile
+import rtyaml
 
 from mailconfig import get_mail_domains
 from dns_update import get_custom_dns_config, do_dns_update
 from utils import shell, safe_domain_name, sort_domains
 
+
 def get_web_domains(env):
     # What domains should we serve websites for?
     domains = set()
@@ -25,11 +31,9 @@ def get_web_domains(env):
     # IP address than this box. Remove those domains from our list.
     dns = get_custom_dns_config(env)
     for domain, value in dns.items():
-		if domain not in domains: continue
+        if domain not in domains:
-		if (isinstance(value, str) and (value != "local")) \
+            continue
-		  or (isinstance(value, dict) and ("CNAME" in value)) \
+        if (isinstance(value, str) and (value != "local")) or (isinstance(value, dict) and ("CNAME" in value)) or (isinstance(value, dict) and ("A" in value) and (value["A"] != "local")) or (isinstance(value, dict) and ("AAAA" in value) and (value["AAAA"] != "local")):
-		  or (isinstance(value, dict) and ("A" in value) and (value["A"] != "local")) \
-		  or (isinstance(value, dict) and ("AAAA" in value) and (value["AAAA"] != "local")):
             domains.remove(domain)
 
     # Sort the list. Put PRIMARY_HOSTNAME first so it becomes the
@@ -38,6 +42,7 @@ def get_web_domains(env):
 
     return domains
 
+
 def do_web_update(env, ok_status="web updated\n"):
     # Build an nginx configuration file.
     nginx_conf = open(os.path.join(os.path.dirname(__file__), "../conf/nginx-top.conf")).read()
@@ -67,6 +72,7 @@ def do_web_update(env, ok_status="web updated\n"):
 
     return ok_status
 
+
 def make_domain_config(domain, template, template_for_primaryhost, env):
     # How will we configure this domain.
 
@@ -128,13 +134,16 @@ def make_domain_config(domain, template, template_for_primaryhost, env):
 
     return nginx_conf
 
+
 def get_web_root(domain, env, test_exists=True):
     # Try STORAGE_ROOT/web/domain_name if it exists, but fall back to STORAGE_ROOT/web/default.
     for test_domain in (domain, 'default'):
         root = os.path.join(env["STORAGE_ROOT"], "www", safe_domain_name(test_domain))
-		if os.path.exists(root) or not test_exists: break
+        if os.path.exists(root) or not test_exists:
+            break
     return root
 
+
 def get_domain_ssl_files(domain, env, allow_shared_cert=True):
     # What SSL private key will we use? Allow the user to override this, but
     # in many cases using the same private key for all domains would be fine.
@@ -175,6 +184,7 @@ def get_domain_ssl_files(domain, env, allow_shared_cert=True):
 
     return ssl_key, ssl_certificate, ssl_via
 
+
 def ensure_ssl_certificate_exists(domain, ssl_key, ssl_certificate, env):
     # For domains besides PRIMARY_HOSTNAME, generate a self-signed certificate if
     # a certificate doesn't already exist. See setup/mail.sh for documentation.
@@ -197,7 +207,8 @@ def ensure_ssl_certificate_exists(domain, ssl_key, ssl_certificate, env):
     # Start with a CSR written to a temporary file.
     with tempfile.NamedTemporaryFile(mode="w") as csr_fp:
         csr_fp.write(create_csr(domain, ssl_key, env))
-		csr_fp.flush() # since we won't close until after running 'openssl x509', since close triggers delete.
+        # since we won't close until after running 'openssl x509', since close triggers delete.
+        csr_fp.flush()
 
         # And then make the certificate.
         shell("check_call", [
@@ -207,6 +218,7 @@ def ensure_ssl_certificate_exists(domain, ssl_key, ssl_certificate, env):
             "-signkey", ssl_key,
             "-out", ssl_certificate])
 
+
 def create_csr(domain, ssl_key, env):
     return shell("check_output", [
         "openssl", "req", "-new",
@@ -215,13 +227,15 @@ def create_csr(domain, ssl_key, env):
         "-sha256",
         "-subj", "/C=%s/ST=/L=/O=/CN=%s" % (env["CSR_COUNTRY"], domain.encode("idna").decode("ascii"))])
 
+
 def install_cert(domain, ssl_cert, ssl_chain, env):
     if domain not in get_web_domains(env):
         return "Invalid domain name."
 
     # Write the combined cert+chain to a temporary path and validate that it is OK.
     # The certificate always goes above the chain.
-	import tempfile, os
+    import tempfile
+    import os
     fd, fn = tempfile.mkstemp('.pem')
     os.write(fd, (ssl_cert + '\n' + ssl_chain).encode("ascii"))
     os.close(fd)
@@ -258,6 +272,7 @@ def install_cert(domain, ssl_cert, ssl_chain, env):
     ret.append(do_web_update(env, ok_status=""))
     return "\n".join(r for r in ret if r.strip() != "")
 
+
 def get_web_domains_info(env):
     # load custom settings so we can tell what domains have a redirect or proxy set up on '/',
     # which means static hosting is not happening
@@ -265,6 +280,7 @@ def get_web_domains_info(env):
     nginx_conf_custom_fn = os.path.join(env["STORAGE_ROOT"], "www/custom.yaml")
     if os.path.exists(nginx_conf_custom_fn):
         custom_settings = rtyaml.load(open(nginx_conf_custom_fn))
+
     def has_root_proxy_or_redirect(domain):
         return custom_settings.get(domain, {}).get('redirects', {}).get('/') or custom_settings.get(domain, {}).get('proxies', {}).get('/')
 

setup/migrate.py

@@ -5,11 +5,17 @@
 # We have to be careful here that any dependencies are already installed in the previous
 # version since this script runs before all other aspects of the setup script.
 
-import sys, os, os.path, glob, re, shutil
+import sys
+import os
+import os.path
+import glob
+import re
+import shutil
 
 sys.path.insert(0, 'management')
 from utils import load_environment, save_environment, shell
 
+
 def migration_1(env):
     # Re-arrange where we store SSL certificates. There was a typo also.
 
@@ -26,8 +32,12 @@ def migration_1(env):
         if m:
             # get the new name for the file
             domain_name, file_type = m.groups()
-			if file_type == "certifiate.pem": file_type = "ssl_certificate.pem" # typo
+            # typo
-			if file_type == "cert_sign_req.csr": file_type = "certificate_signing_request.csr" # nicer
+            if file_type == "certifiate.pem":
+                file_type = "ssl_certificate.pem"
+            # nicer
+            if file_type == "cert_sign_req.csr":
+                file_type = "certificate_signing_request.csr"
             move_file(sslfn, domain_name, file_type)
 
     # Move the old domains directory if it is now empty.
@@ -36,6 +46,7 @@ def migration_1(env):
     except:
         pass
 
+
 def migration_2(env):
     # Delete the .dovecot_sieve script everywhere. This was formerly a copy of our spam -> Spam
     # script. We now install it as a global script, and we use managesieve, so the old file is
@@ -45,21 +56,25 @@ def migration_2(env):
     for fn in glob.glob(os.path.join(env["STORAGE_ROOT"], 'mail/mailboxes/*/*/.dovecot.svbin')):
         os.unlink(fn)
 
+
 def migration_3(env):
     # Move the migration ID from /etc/mailinabox.conf to $STORAGE_ROOT/mailinabox.version
     # so that the ID stays with the data files that it describes the format of. The writing
     # of the file will be handled by the main function.
     pass
 
+
 def migration_4(env):
     # Add a new column to the mail users table where we can store administrative privileges.
     db = os.path.join(env["STORAGE_ROOT"], 'mail/users.sqlite')
     shell("check_call", ["sqlite3", db, "ALTER TABLE users ADD privileges TEXT NOT NULL DEFAULT ''"])
 
+
 def migration_5(env):
     # The secret key for encrypting backups was world readable. Fix here.
     os.chmod(os.path.join(env["STORAGE_ROOT"], 'backup/secret_key.txt'), 0o600)
 
+
 def migration_6(env):
     # We now will generate multiple DNSSEC keys for different algorithms, since TLDs may
     # not support them all. .email only supports RSA/SHA-256. Rename the keys.conf file
@@ -67,6 +82,7 @@ def migration_6(env):
     basepath = os.path.join(env["STORAGE_ROOT"], 'dns/dnssec')
     shutil.move(os.path.join(basepath, 'keys.conf'), os.path.join(basepath, 'RSASHA1-NSEC3-SHA1.conf'))
 
+
 def get_current_migration():
     ver = 0
     while True:
@@ -76,6 +92,7 @@ def get_current_migration():
             return ver
         ver = next_ver
 
+
 def run_migrations():
     if not os.access("/etc/mailinabox.conf", os.W_OK, effective_ids=True):
         print("This script must be run as root.", file=sys.stderr)
@@ -135,4 +152,3 @@ if __name__ == "__main__":
     elif sys.argv[-1] == "--migrate":
         # Perform migrations.
         run_migrations()
-

tests/test_dns.py

@@ -7,8 +7,11 @@
 # where ipaddr is the IP address of your Mail-in-a-Box
 # and hostname is the domain name to check the DNS for.
 
-import sys, re, difflib
+import sys
-import dns.reversename, dns.resolver
+import re
+import difflib
+import dns.reversename
+import dns.resolver
 
 if len(sys.argv) < 3:
     print("Usage: tests/dns.py ipaddress hostname [primary hostname]")
@@ -19,6 +22,7 @@ primary_hostname = hostname
 if len(sys.argv) == 4:
     primary_hostname = sys.argv[3]
 
+
 def test(server, description):
     tests = [
         (hostname, "A", ipaddr),
@@ -34,6 +38,7 @@ def test(server, description):
     ]
     return test2(tests, server, description)
 
+
 def test_ptr(server, description):
     ipaddr_rev = dns.reversename.from_address(ipaddr)
     tests = [
@@ -41,6 +46,7 @@ def test_ptr(server, description):
     ]
     return test2(tests, server, description)
 
+
 def test2(tests, server, description):
     first = True
     resolver = dns.resolver.get_default_resolver()
@@ -58,15 +64,18 @@ def test2(tests, server, description):
             # difference is between the two exceptions
             response = ["[no value]"]
         response = ";".join(str(r) for r in response)
-		response = re.sub(r"(\"p=).*(\")", r"\1__KEY__\2", response) # normalize DKIM key
+        # normalize DKIM key
-		response = response.replace("\"\" ", "") # normalize TXT records (DNSSEC signing inserts empty text string components)
+        response = re.sub(r"(\"p=).*(\")", r"\1__KEY__\2", response)
+        # normalize TXT records (DNSSEC signing inserts empty text
+        # string components)
+        response = response.replace("\"\" ", "")
 
         # is it right?
         if response == expected_answer:
             #print(server, ":", qname, rtype, "?", response)
             continue
 
-		# show prolem
+        # show problem
         if first:
             print("Incorrect DNS Response from", description)
             print()
@@ -74,7 +83,8 @@ def test2(tests, server, description):
             first = False
 
         print((qname + "/" + rtype).ljust(20), response.ljust(12), expected_answer, sep='\t')
-	return first # success
+    # success
+    return first
 
 # Test the response from the machine itself.
 if not test(ipaddr, "Mail-in-a-Box"):

tests/test_mail.py

@@ -1,8 +1,14 @@
 #!/usr/bin/env python3
 # Tests sending and receiving mail by sending a test message to yourself.
 
-import sys, imaplib, smtplib, uuid, time
+import sys
-import socket, dns.reversename, dns.resolver
+import imaplib
+import smtplib
+import uuid
+import time
+import socket
+import dns.reversename
+import dns.resolver
 
 if len(sys.argv) < 3:
     print("Usage: tests/mail.py hostname emailaddress password")
@@ -48,6 +54,7 @@ server.starttls()
 # Verify that the EHLO name matches the server's reverse DNS.
 ipaddr = socket.gethostbyname(host)  # IPv4 only!
 reverse_ip = dns.reversename.from_address(ipaddr)  # e.g. "1.0.0.127.in-addr.arpa."
+
 try:
     reverse_dns = dns.resolver.query(reverse_ip, 'PTR')[0].target.to_text(omit_final_dot=True)  # => hostname
 except dns.resolver.NXDOMAIN:

tests/test_smtp_server.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
-import smtplib, sys
+import smtplib
+import sys
 
 if len(sys.argv) < 3:
         print("Usage: tests/smtp_server.py host email.to email.from")
@@ -16,4 +17,3 @@ server = smtplib.SMTP(host, 25)
 server.set_debuglevel(1)
 server.sendmail(fromaddr, [toaddr], msg)
 server.quit()
-

tools/editconf.py

@@ -20,7 +20,8 @@
 # NAME VAL
 #   UE
 
-import sys, re
+import sys
+import re
 
 # sanity check
 if len(sys.argv) < 3:
@@ -74,18 +75,20 @@ while len(input_lines) > 0:
         # Check that this line contain this setting from the command-line arguments.
         name, val = settings[i].split("=", 1)
         m = re.match(
-			   "(\s*)"
+            "(\s*)" +
-			 + "(" + re.escape(comment_char) + "\s*)?"
+            "(" + re.escape(comment_char) + "\s*)?" +
-			 + re.escape(name) + delimiter_re + "(.*?)\s*$",
+            re.escape(name) + delimiter_re + "(.*?)\s*$",
             line, re.S)
-		if not m: continue
+        if not m:
+            continue
         indent, is_comment, existing_val = m.groups()
 
         # If this is already the setting, do nothing.
         if is_comment is None and existing_val == val:
             # It may be that we've already inserted this setting higher
             # in the file so check for that first.
-			if i in found: break
+            if i in found:
+                break
             buf += line
             found.add(i)
             break

tools/mail.py

@@ -1,6 +1,11 @@
 #!/usr/bin/python3
 
-import sys, getpass, urllib.request, urllib.error, json
+import sys
+import getpass
+import urllib.request
+import urllib.error
+import json
+
 
 def mgmt(cmd, data=None, is_json=False):
     # The base URL for the management daemon. (Listens on IPv4 only.)
@@ -24,9 +29,11 @@ def mgmt(cmd, data=None, is_json=False):
             print(e, file=sys.stderr)
         sys.exit(1)
     resp = response.read().decode('utf8')
-	if is_json: resp = json.loads(resp)
+    if is_json:
+        resp = json.loads(resp)
     return resp
 
+
 def read_password():
     first = getpass.getpass('password: ')
     second = getpass.getpass(' (again): ')
@@ -36,6 +43,7 @@ def read_password():
         second = getpass.getpass(' (again): ')
     return first
 
+
 def setup_key_auth(mgmt_uri):
     key = open('/var/lib/mailinabox/api.key').read().strip()
 
@@ -70,7 +78,8 @@ elif sys.argv[1] == "user" and len(sys.argv) == 2:
     users = mgmt("/mail/users?format=json", is_json=True)
     for domain in users:
         for user in domain["users"]:
-			if user['status'] == 'inactive': continue
+            if user['status'] == 'inactive':
+                continue
             print(user['email'], end='')
             if "admin" in user['privileges']:
                 print("*", end='')
@@ -121,4 +130,3 @@ elif sys.argv[1] == "alias" and sys.argv[2] == "remove" and len(sys.argv) == 4:
 else:
     print("Invalid command-line arguments.")
     sys.exit(1)
-

tools/parse-nginx-log-bootstrap-accesses.py

@@ -4,7 +4,11 @@
 # access log to see how many people are installing Mail-in-a-Box each day, by
 # looking at accesses to the bootstrap.sh script.
 
-import re, glob, gzip, os.path, json
+import re
+import glob
+import gzip
+import os.path
+import json
 import dateutil.parser
 
 outfn = "/home/user-data/www/mailinabox.email/install-stats.json"

tools/readable_bash.py

@@ -3,10 +3,12 @@
 # Generate documentation for how this machine works by
 # parsing our bash scripts!
 
-import cgi, re
+import cgi
+import re
 import markdown
 from modgrammar import *
 
+
 def generate_documentation():
     print("""<!DOCTYPE html>
 <html>
@@ -151,11 +153,14 @@ def generate_documentation():
 </html>
 """)
 
+
 class HashBang(Grammar):
     grammar = (L('#!'), REST_OF_LINE, EOL)
+
     def value(self):
         return ""
 
+
 def strip_indent(s):
     s = s.replace("\t", "    ")
     lines = s.split("\n")
@@ -167,8 +172,10 @@ def strip_indent(s):
     lines = [line[min_indent:] for line in lines]
     return "\n".join(lines)
 
+
 class Comment(Grammar):
     grammar = ONE_OR_MORE(ZERO_OR_MORE(SPACE), L('#'), REST_OF_LINE, EOL)
+
     def value(self):
         if self.string.replace("#", "").strip() == "":
             return "\n"
@@ -179,35 +186,46 @@ class Comment(Grammar):
 
 FILENAME = WORD('a-z0-9-/.')
 
+
 class Source(Grammar):
     grammar = ((L('.') | L('source')), L(' '), FILENAME, Comment | EOL)
+
     def filename(self):
         return self[2].string.strip()
+
     def value(self):
         return BashScript.parse(self.filename())
 
+
 class CatEOF(Grammar):
     grammar = (ZERO_OR_MORE(SPACE), L('cat '), L('>') | L('>>'), L(' '), ANY_EXCEPT(WHITESPACE), L(" <<"), OPTIONAL(SPACE), L("EOF"), EOL, REPEAT(ANY, greedy=False), EOL, L("EOF"), EOL)
+
     def value(self):
         content = self[9].string
-		content = re.sub(r"\\([$])", r"\1", content) # un-escape bash-escaped characters
+        # un-escape bash-escaped characters
+        content = re.sub(r"\\([$])", r"\1", content)
         return "<div class='write-to'><div class='filename'>%s <span>(%s)</span></div><pre>%s</pre></div>\n" \
             % (self[4].string,
                "overwrite" if ">>" not in self[2].string else "append to",
                cgi.escape(content))
 
+
 class HideOutput(Grammar):
     grammar = (L("hide_output "), REF("BashElement"))
+
     def value(self):
         return self[1].value()
 
+
 class EchoLine(Grammar):
     grammar = (OPTIONAL(SPACE), L("echo "), REST_OF_LINE, EOL)
+
     def value(self):
         if "|" in self.string or ">" in self.string:
             return "<pre class='shell'><div>" + recode_bash(self.string.strip()) + "</div></pre>\n"
         return ""
 
+
 class EditConf(Grammar):
     grammar = (
         L('tools/editconf.py '),
@@ -221,61 +239,86 @@ class EditConf(Grammar):
         OPTIONAL(SPACE),
         EOL
         )
+
     def value(self):
         conffile = self[1]
         options = []
         eq = "="
-		if self[3] and "-s" in self[3].string: eq = " "
+        if self[3] and "-s" in self[3].string:
+            eq = " "
         for opt in re.split("\s+", self[4].string):
             k, v = opt.split("=", 1)
-			v = re.sub(r"\n+", "", fixup_tokens(v)) # not sure why newlines are getting doubled
+            # not sure why newlines are getting doubled
+            v = re.sub(r"\n+", "", fixup_tokens(v))
             options.append("%s%s%s" % (k, eq, v))
         return "<div class='write-to'><div class='filename'>" + self[1].string + " <span>(change settings)</span></div><pre>" + "\n".join(cgi.escape(s) for s in options) + "</pre></div>\n"
 
+
 class CaptureOutput(Grammar):
     grammar = OPTIONAL(SPACE), WORD("A-Za-z_"), L('=$('), REST_OF_LINE, L(")"), OPTIONAL(L(';')), EOL
+
     def value(self):
         cmd = self[3].string
         cmd = cmd.replace("; ", "\n")
         return "<div class='write-to'><div class='filename'>$" + self[1].string + "=</div><pre>" + cgi.escape(cmd) + "</pre></div>\n"
 
+
 class SedReplace(Grammar):
     grammar = OPTIONAL(SPACE), L('sed -i "s/'), OPTIONAL(L('^')), ONE_OR_MORE(WORD("-A-Za-z0-9 #=\\{};.*$_!()")), L('/'), ONE_OR_MORE(WORD("-A-Za-z0-9 #=\\{};.*$_!()")), L('/"'), SPACE, FILENAME, EOL
+
     def value(self):
         return "<div class='write-to'><div class='filename'>edit<br>" + self[8].string + "</div><p>replace</p><pre>" + cgi.escape(self[3].string.replace(".*", ". . .")) + "</pre><p>with</p><pre>" + cgi.escape(self[5].string.replace("\\n", "\n").replace("\\t", "\t")) + "</pre></div>\n"
 
+
 class EchoPipe(Grammar):
     grammar = OPTIONAL(SPACE), L("echo "), REST_OF_LINE, L(' | '), REST_OF_LINE, EOL
+
     def value(self):
         text = " ".join("\"%s\"" % s for s in self[2].string.split(" "))
         return "<pre class='shell'><div>echo " + recode_bash(text) + " \<br> | " + recode_bash(self[4].string) + "</div></pre>\n"
 
+
 def shell_line(bash):
     return "<pre class='shell'><div>" + recode_bash(bash.strip()) + "</div></pre>\n"
 
+
 class AptGet(Grammar):
     grammar = (ZERO_OR_MORE(SPACE), L("apt_install "), REST_OF_LINE, EOL)
+
     def value(self):
         return shell_line("apt-get install -y " + re.sub(r"\s+", " ", self[2].string))
+
+
 class UfwAllow(Grammar):
     grammar = (ZERO_OR_MORE(SPACE), L("ufw_allow "), REST_OF_LINE, EOL)
+
     def value(self):
         return shell_line("ufw allow " + self[2].string)
+
+
 class RestartService(Grammar):
     grammar = (ZERO_OR_MORE(SPACE), L("restart_service "), REST_OF_LINE, EOL)
+
     def value(self):
         return shell_line("service " + self[2].string + " restart")
 
+
 class OtherLine(Grammar):
     grammar = (REST_OF_LINE, EOL)
+
     def value(self):
-		if self.string.strip() == "": return ""
+        if self.string.strip() == "":
-		if "source setup/functions.sh" in self.string: return ""
+            return ""
-		if "source /etc/mailinabox.conf" in self.string: return ""
+        if "source setup/functions.sh" in self.string:
+            return ""
+        if "source /etc/mailinabox.conf" in self.string:
+            return ""
         return "<pre class='shell'><div>" + recode_bash(self.string.strip()) + "</div></pre>\n"
 
+
 class BashElement(Grammar):
     grammar = Comment | CatEOF | EchoPipe | EchoLine | HideOutput | EditConf | SedReplace | AptGet | UfwAllow | RestartService | OtherLine
+
     def value(self):
         return self[0].value()
 
@@ -292,6 +335,7 @@ bash_escapes = {
     "t": "\uE021",
 }
 
+
 def quasitokenize(bashscript):
     # Make a parse of bash easier by making the tokenization easy.
     newscript = ""
@@ -366,6 +410,7 @@ def quasitokenize(bashscript):
 
     return newscript
 
+
 def recode_bash(s):
     def requote(tok):
         tok = tok.replace("\\", "\\\\")
@@ -380,6 +425,7 @@ def recode_bash(s):
         return tok
     return cgi.escape(" ".join(requote(tok) for tok in s.split(" ")))
 
+
 def fixup_tokens(s):
     for c, enc in bash_special_characters1.items():
         s = s.replace(enc, c)
@@ -389,14 +435,17 @@ def fixup_tokens(s):
         s = s.replace(c, "\\" + esc)
     return s
 
+
 class BashScript(Grammar):
     grammar = (OPTIONAL(HashBang), REPEAT(BashElement))
+
     def value(self):
         return [line.value() for line in self[1]]
 
     @staticmethod
     def parse(fn):
-		if fn in ("setup/functions.sh", "/etc/mailinabox.conf"): return ""
+        if fn in ("setup/functions.sh", "/etc/mailinabox.conf"):
+            return ""
         string = open(fn).read()
 
         # tokenize
@@ -463,6 +512,7 @@ class BashScript(Grammar):
 
         return v
 
+
 def wrap_lines(text, cols=60):
     ret = ""
     words = re.split("(\s+)", text)
@@ -472,7 +522,8 @@ def wrap_lines(text, cols=60):
             ret += " \\\n"
             ret += "   "
             linelen = 0
-		if linelen == 0 and w.strip() == "": continue
+        if linelen == 0 and w.strip() == "":
+            continue
         ret += w
         linelen += len(w)
     return ret