Merge remote-tracking branch 'upstream/main' into merge-upstream

# Conflicts:
#	.gitignore
#	management/auth.py
#	management/daemon.py
#	management/mail_log.py
#	management/mailconfig.py
#	management/mfa.py
#	management/ssl_certificates.py
#	management/status_checks.py
#	management/utils.py
#	management/web_update.py
#	setup/mail-postfix.sh
#	setup/migrate.py
#	setup/preflight.sh
#	setup/webmail.sh
#	tests/test_mail.py
#	tools/editconf.py

.github/workflows/commit-tests.yml

@@ -22,13 +22,14 @@ jobs:
           path: tests/out/**/screenshot.png
           if-no-files-found: ignore
           retention-days: 5
-    
+
   # install upstream miab, then migrate to miabldap
   upgrade-from-upstream:
     runs-on: ubuntu-22.04
     env:
       PRIMARY_HOSTNAME: box2.abc.com
-      UPSTREAM_TAG: main
+      # TODO: change UPSTREAM_TAG to 'main' once upstream is installable
+      UPSTREAM_TAG: v67
     steps:
       - uses: actions/checkout@v2
       - name: setup

.gitignore

@@ -6,3 +6,4 @@ externals/
 api/docs/api-docs.html
 downloads/
 MAINTAIN/
+*.code-workspace

conf/fail2ban/filter.d/miab-munin.conf

@@ -3,5 +3,5 @@
 before = common.conf
 
 [Definition]
-failregex=<HOST> - .*GET /admin/munin/.* HTTP/1.1\" 401.*
+failregex=<HOST> - .*GET /admin/munin/.* HTTP/\d+\.\d+\" 401.*
 ignoreregex =

management/auth.py

@@ -8,7 +8,7 @@
 ##### details.
 #####
 
-import base64, os, os.path, hmac, json, secrets
+import base64, hmac, json, secrets
 from datetime import timedelta
 
 from expiringdict import ExpiringDict
@@ -32,7 +32,7 @@ class AuthService:
 	def init_system_api_key(self):
 		"""Write an API key to a local file so local processes can use the API"""
 
-		with open(self.key_path, 'r') as file:
+		with open(self.key_path, encoding='utf-8') as file:
 			self.key = file.read()
 
 	def authenticate(self, request, env, login_only=False, logout=False):
@@ -58,11 +58,13 @@ class AuthService:
 			return username, password
 
 		username, password = parse_http_authorization_basic(request.headers.get('Authorization', ''))
-		if username in (None, ""):
+		if username in {None, ""}:
-			raise ValueError("Authorization header invalid.")
+			msg = "Authorization header invalid."
+			raise ValueError(msg)
 
 		if username.strip() == "" and password.strip() == "":
-			raise ValueError("No email address, password, session key, or API key provided.")
+			msg = "No email address, password, session key, or API key provided."
+			raise ValueError(msg)
 
 		# If user passed the system API key, grant administrative privs. This key
 		# is not associated with a user.
@@ -82,7 +84,8 @@ class AuthService:
 
 		# If no password was given, but a username was given, we're missing some information.
 		elif password.strip() == "":
-			raise ValueError("Enter a password.")
+			msg = "Enter a password."
+			raise ValueError(msg)
 
 		else:
 			# The user is trying to log in with a username and a password
@@ -109,7 +112,8 @@ class AuthService:
 		# Authenticate.
 		if not validate_login(email, pw, env):
 			# Login failed.
-			raise ValueError("Incorrect email address or password.")
+			msg = "Incorrect email address or password."
+			raise ValueError(msg)
 
 		# If MFA is enabled, check that MFA passes.
 		status, hints = validate_auth_mfa(email, request, env)

management/backup.py

@@ -17,7 +17,7 @@
 # 4) The stopped services are restarted.
 # 5) STORAGE_ROOT/backup/after-backup is executed if it exists.
 
-import os, os.path, shutil, glob, re, datetime, sys
+import os, os.path, re, datetime, sys
 import dateutil.parser, dateutil.relativedelta, dateutil.tz
 import rtyaml
 from exclusiveprocess import Lock
@@ -69,7 +69,7 @@ def backup_status(env):
 		"--archive-dir", backup_cache_dir,
 		"--gpg-options", "'--cipher-algo=AES256'",
 		"--log-fd", "1",
-		] + get_duplicity_additional_args(env) + [
+		*get_duplicity_additional_args(env),
 		get_duplicity_target_url(config)
 		],
 		get_duplicity_env_vars(env),
@@ -79,7 +79,7 @@ def backup_status(env):
 		# destination for the backups or the last backup job terminated unexpectedly.
 		raise Exception("Something is wrong with the backup: " + collection_status)
 	for line in collection_status.split('\n'):
-		if line.startswith(" full") or line.startswith(" inc"):
+		if line.startswith((" full", " inc")):
 			backup = parse_line(line)
 			backups[backup["date"]] = backup
 
@@ -195,7 +195,7 @@ def get_passphrase(env):
 	# only needs to be 43 base64-characters to match AES256's key
 	# length of 32 bytes.
 	backup_root = os.path.join(env["STORAGE_ROOT"], 'backup')
-	with open(os.path.join(backup_root, 'secret_key.txt')) as f:
+	with open(os.path.join(backup_root, 'secret_key.txt'), encoding="utf-8") as f:
 		passphrase = f.readline().strip()
 	if len(passphrase) < 43: raise Exception("secret_key.txt's first line is too short!")
 
@@ -236,7 +236,7 @@ def get_duplicity_additional_args(env):
 			port = 22
 		if port is None:
 			port = 22
-						
+
 		return [
 			f"--ssh-options='-i /root/.ssh/id_rsa_miab -p {port}'",
 			f"--rsync-options='-e \"/usr/bin/ssh -oStrictHostKeyChecking=no -oBatchMode=yes -p {port} -i /root/.ssh/id_rsa_miab\"'",
@@ -267,8 +267,7 @@ def get_duplicity_env_vars(env):
 	return env
 
 def get_target_type(config):
-	protocol = config["target"].split(":")[0]
+	return config["target"].split(":")[0]
-	return protocol
 
 def nuke_local_files(backup_dir, backup_cache_dir, config, env):
 	# the files must be removed manually, duplicity won't do
@@ -351,8 +350,8 @@ def perform_backup(full_backup):
 			"--exclude", backup_root,
 			"--volsize", "250",
 			"--gpg-options", "'--cipher-algo=AES256'",
-			"--allow-source-mismatch"
+			"--allow-source-mismatch",
-			] + get_duplicity_additional_args(env) + [
+			*get_duplicity_additional_args(env),
 			env["STORAGE_ROOT"],
 			get_duplicity_target_url(config),
 			],
@@ -375,7 +374,7 @@ def perform_backup(full_backup):
 		"--verbosity", "error",
 		"--archive-dir", backup_cache_dir,
 		"--force",
-		] + get_duplicity_additional_args(env) + [
+		*get_duplicity_additional_args(env),
 		get_duplicity_target_url(config)
 		],
 		get_duplicity_env_vars(env))
@@ -391,7 +390,7 @@ def perform_backup(full_backup):
 		"--verbosity", "error",
 		"--archive-dir", backup_cache_dir,
 		"--force",
-		] + get_duplicity_additional_args(env) + [
+		*get_duplicity_additional_args(env),
 		get_duplicity_target_url(config)
 		],
 		get_duplicity_env_vars(env))
@@ -430,7 +429,7 @@ def run_duplicity_verification():
 		"--compare-data",
 		"--archive-dir", backup_cache_dir,
 		"--exclude", backup_root,
-		] + get_duplicity_additional_args(env) + [
+		*get_duplicity_additional_args(env),
 		get_duplicity_target_url(config),
 		env["STORAGE_ROOT"],
 	], get_duplicity_env_vars(env))
@@ -443,9 +442,9 @@ def run_duplicity_restore(args):
 		"/usr/bin/duplicity",
 		"restore",
 		"--archive-dir", backup_cache_dir,
-		] + get_duplicity_additional_args(env) + [
+		*get_duplicity_additional_args(env),
-		get_duplicity_target_url(config)
+		get_duplicity_target_url(config),
-		] + args,
+		*args],
 		get_duplicity_env_vars(env))
 
 def print_duplicity_command():
@@ -457,7 +456,7 @@ def print_duplicity_command():
 		print(f"export {k}={shlex.quote(v)}")
 	print("duplicity", "{command}", shlex.join([
 		"--archive-dir", backup_cache_dir,
-		] + get_duplicity_additional_args(env) + [
+		*get_duplicity_additional_args(env),
 		get_duplicity_target_url(config)
 		]))
 
@@ -513,21 +512,22 @@ def list_target_files(config):
 			if 'Permission denied (publickey).' in listing:
 				reason = "Invalid user or check you correctly copied the SSH key."
 			elif 'No such file or directory' in listing:
-				reason = "Provided path {} is invalid.".format(target_path)
+				reason = f"Provided path {target_path} is invalid."
 			elif 'Network is unreachable' in listing:
-				reason = "The IP address {} is unreachable.".format(target.hostname)
+				reason = f"The IP address {target.hostname} is unreachable."
 			elif 'Could not resolve hostname' in listing:
-				reason = "The hostname {} cannot be resolved.".format(target.hostname)
+				reason = f"The hostname {target.hostname} cannot be resolved."
 			else:
-				reason = "Unknown error." \
+				reason = ("Unknown error."
-						"Please check running 'management/backup.py --verify'" \
+						"Please check running 'management/backup.py --verify'"
-						"from mailinabox sources to debug the issue."
+						"from mailinabox sources to debug the issue.")
-			raise ValueError("Connection to rsync host failed: {}".format(reason))
+			msg = f"Connection to rsync host failed: {reason}"
+			raise ValueError(msg)
 
 	elif target.scheme == "s3":
 		import boto3.s3
 		from botocore.exceptions import ClientError
-		
+
 		# separate bucket from path in target
 		bucket = target.path[1:].split('/')[0]
 		path = '/'.join(target.path[1:].split('/')[1:]) + '/'
@@ -537,7 +537,8 @@ def list_target_files(config):
 			path = ''
 
 		if bucket == "":
-			raise ValueError("Enter an S3 bucket name.")
+			msg = "Enter an S3 bucket name."
+			raise ValueError(msg)
 
 		# connect to the region & bucket
 		try:
@@ -555,7 +556,7 @@ def list_target_files(config):
 		from b2sdk.v1.exception import NonExistentBucket
 		info = InMemoryAccountInfo()
 		b2_api = B2Api(info)
-		
+
 		# Extract information from target
 		b2_application_keyid = target.netloc[:target.netloc.index(':')]
 		b2_application_key = urllib.parse.unquote(target.netloc[target.netloc.index(':')+1:target.netloc.index('@')])
@@ -564,8 +565,9 @@ def list_target_files(config):
 		try:
 			b2_api.authorize_account("production", b2_application_keyid, b2_application_key)
 			bucket = b2_api.get_bucket_by_name(b2_bucket)
-		except NonExistentBucket as e:
+		except NonExistentBucket:
-			raise ValueError("B2 Bucket does not exist. Please double check your information!")
+			msg = "B2 Bucket does not exist. Please double check your information!"
+			raise ValueError(msg)
 		return [(key.file_name, key.size) for key, _ in bucket.ls()]
 
 	else:
@@ -586,7 +588,7 @@ def backup_set_custom(env, target, target_user, target_pass, min_age):
 
 	# Validate.
 	try:
-		if config["target"] not in ("off", "local"):
+		if config["target"] not in {"off", "local"}:
 			# these aren't supported by the following function, which expects a full url in the target key,
 			# which is what is there except when loading the config prior to saving
 			list_target_files(config)
@@ -608,9 +610,9 @@ def get_backup_config(env, for_save=False, for_ui=False):
 
 	# Merge in anything written to custom.yaml.
 	try:
-		with open(os.path.join(backup_root, 'custom.yaml'), 'r') as f:
+		with open(os.path.join(backup_root, 'custom.yaml'), encoding="utf-8") as f:
 			custom_config = rtyaml.load(f)
-		if not isinstance(custom_config, dict): raise ValueError() # caught below
+		if not isinstance(custom_config, dict): raise ValueError # caught below
 		config.update(custom_config)
 	except:
 		pass
@@ -634,18 +636,17 @@ def get_backup_config(env, for_save=False, for_ui=False):
 		config["target"] = "file://" + config["file_target_directory"]
 	ssh_pub_key = os.path.join('/root', '.ssh', 'id_rsa_miab.pub')
 	if os.path.exists(ssh_pub_key):
-		with open(ssh_pub_key, 'r') as f:
+		with open(ssh_pub_key, encoding="utf-8") as f:
 			config["ssh_pub_key"] = f.read()
 
 	return config
 
 def write_backup_config(env, newconfig):
 	backup_root = os.path.join(env["STORAGE_ROOT"], 'backup')
-	with open(os.path.join(backup_root, 'custom.yaml'), "w") as f:
+	with open(os.path.join(backup_root, 'custom.yaml'), "w", encoding="utf-8") as f:
 		f.write(rtyaml.dump(newconfig))
 
 if __name__ == "__main__":
-	import sys
 	if sys.argv[-1] == "--verify":
 		# Run duplicity's verification command to check a) the backup files
 		# are readable, and b) report if they are up to date.
@@ -654,7 +655,7 @@ if __name__ == "__main__":
 	elif sys.argv[-1] == "--list":
 		# List the saved backup files.
 		for fn, size in list_target_files(get_backup_config(load_environment())):
-			print("{}\t{}".format(fn, size))
+			print(f"{fn}\t{size}")
 
 	elif sys.argv[-1] == "--status":
 		# Show backup status.

management/cli.py

@@ -15,7 +15,8 @@
 # root API key. This file is readable only by root, so this
 # tool can only be used as root.
 
-import sys, getpass, urllib.request, urllib.error, json, re, csv
+import sys, getpass, urllib.request, urllib.error, json, csv
+import contextlib
 
 def mgmt(cmd, data=None, is_json=False):
 	# The base URL for the management daemon. (Listens on IPv4 only.)
@@ -28,10 +29,8 @@ def mgmt(cmd, data=None, is_json=False):
 		response = urllib.request.urlopen(req)
 	except urllib.error.HTTPError as e:
 		if e.code == 401:
-			try:
+			with contextlib.suppress(Exception):
 				print(e.read().decode("utf8"))
-			except:
-				pass
 			print("The management daemon refused access. The API key file may be out of sync. Try 'service mailinabox restart'.", file=sys.stderr)
 		elif hasattr(e, 'read'):
 			print(e.read().decode('utf8'), file=sys.stderr)
@@ -56,7 +55,7 @@ def read_password():
     return first
 
 def setup_key_auth(mgmt_uri):
-	with open('/var/lib/mailinabox/api.key', 'r') as f:
+	with open('/var/lib/mailinabox/api.key', encoding='utf-8') as f:
 		key = f.read().strip()
 
 	auth_handler = urllib.request.HTTPBasicAuthHandler()
@@ -100,12 +99,9 @@ elif sys.argv[1] == "user" and len(sys.argv) == 2:
 				print("*", end='')
 			print()
 
-elif sys.argv[1] == "user" and sys.argv[2] in ("add", "password"):
+elif sys.argv[1] == "user" and sys.argv[2] in {"add", "password"}:
 	if len(sys.argv) < 5:
-		if len(sys.argv) < 4:
+		email = input('email: ') if len(sys.argv) < 4 else sys.argv[3]
-			email = input("email: ")
-		else:
-			email = sys.argv[3]
 		pw = read_password()
 	else:
 		email, pw = sys.argv[3:5]
@@ -118,11 +114,8 @@ elif sys.argv[1] == "user" and sys.argv[2] in ("add", "password"):
 elif sys.argv[1] == "user" and sys.argv[2] == "remove" and len(sys.argv) == 4:
 	print(mgmt("/mail/users/remove", { "email": sys.argv[3] }))
 
-elif sys.argv[1] == "user" and sys.argv[2] in ("make-admin", "remove-admin") and len(sys.argv) == 4:
+elif sys.argv[1] == "user" and sys.argv[2] in {"make-admin", "remove-admin"} and len(sys.argv) == 4:
-	if sys.argv[2] == "make-admin":
+	action = 'add' if sys.argv[2] == 'make-admin' else 'remove'
-		action = "add"
-	else:
-		action = "remove"
 	print(mgmt("/mail/users/privileges/" + action, { "email": sys.argv[3], "privilege": "admin" }))
 
 elif sys.argv[1] == "user" and sys.argv[2] == "admins":
@@ -141,7 +134,7 @@ elif sys.argv[1] == "user" and len(sys.argv) == 5 and sys.argv[2:4] == ["mfa", "
 	for mfa in status["enabled_mfa"]:
 		W.writerow([mfa["id"], mfa["type"], mfa["label"]])
 
-elif sys.argv[1] == "user" and len(sys.argv) in (5, 6) and sys.argv[2:4] == ["mfa", "disable"]:
+elif sys.argv[1] == "user" and len(sys.argv) in {5, 6} and sys.argv[2:4] == ["mfa", "disable"]:
 	# Disable MFA (all or a particular device) for a user.
 	print(mgmt("/mfa/disable", { "user": sys.argv[4], "mfa-id": sys.argv[5] if len(sys.argv) == 6 else None }))
 

management/daemon.py

@@ -20,11 +20,11 @@
 # service mailinabox start # when done debugging, start it up again
 
 import os, os.path, re, json, time
-import multiprocessing.pool, subprocess
+import multiprocessing.pool
 
 from functools import wraps
 
-from flask import Flask, request, render_template, abort, Response, send_from_directory, make_response
+from flask import Flask, request, render_template, Response, send_from_directory, make_response
 
 import auth, utils
 from mailconfig import get_mail_users, get_mail_users_ex, get_admins, add_mail_user, set_mail_password, set_mail_display_name, remove_mail_user
@@ -32,6 +32,7 @@ from mailconfig import get_mail_user_privileges, add_remove_mail_user_privilege
 from mailconfig import get_mail_aliases, get_mail_aliases_ex, get_mail_domains, add_mail_alias, remove_mail_alias
 from mfa import get_public_mfa_state, enable_mfa, disable_mfa
 import mfa_totp
+import contextlib
 
 env = utils.load_environment()
 
@@ -39,14 +40,12 @@ auth_service = auth.AuthService()
 
 # We may deploy via a symbolic link, which confuses flask's template finding.
 me = __file__
-try:
+with contextlib.suppress(OSError):
 	me = os.readlink(__file__)
-except OSError:
-	pass
 
 # for generating CSRs we need a list of country codes
 csr_country_codes = []
-with open(os.path.join(os.path.dirname(me), "csr_country_codes.tsv")) as f:
+with open(os.path.join(os.path.dirname(me), "csr_country_codes.tsv"), encoding="utf-8") as f:
 	for line in f:
 		if line.strip() == "" or line.startswith("#"): continue
 		code, name = line.strip().split("\t")[0:2]
@@ -90,7 +89,7 @@ def authorized_personnel_only(viewfunc):
 		# Not authorized. Return a 401 (send auth) and a prompt to authorize by default.
 		status = 401
 		headers = {
-			'WWW-Authenticate': 'Basic realm="{0}"'.format(auth_service.auth_realm),
+			'WWW-Authenticate': f'Basic realm="{auth_service.auth_realm}"',
 			'X-Reason': error,
 		}
 
@@ -100,7 +99,7 @@ def authorized_personnel_only(viewfunc):
 			status = 403
 			headers = None
 
-		if request.headers.get('Accept') in (None, "", "*/*"):
+		if request.headers.get('Accept') in {None, "", "*/*"}:
 			# Return plain text output.
 			return Response(error+"\n", status=status, mimetype='text/plain', headers=headers)
 		else:
@@ -174,7 +173,7 @@ def login():
 		"api_key": auth_service.create_session_key(email, env, type='login'),
 	}
 
-	app.logger.info("New login session created for {}".format(email))
+	app.logger.info(f"New login session created for {email}")
 
 	# Return.
 	return json_response(resp)
@@ -183,8 +182,8 @@ def login():
 def logout():
 	try:
 		email, _ = auth_service.authenticate(request, env, logout=True)
-		app.logger.info("{} logged out".format(email))
+		app.logger.info(f"{email} logged out")
-	except ValueError as e:
+	except ValueError:
 		pass
 	finally:
 		return json_response({ "status": "ok" })
@@ -374,9 +373,9 @@ def dns_set_record(qname, rtype="A"):
 			# Get the existing records matching the qname and rtype.
 			return dns_get_records(qname, rtype)
 
-		elif request.method in ("POST", "PUT"):
+		elif request.method in {"POST", "PUT"}:
 			# There is a default value for A/AAAA records.
-			if rtype in ("A", "AAAA") and value == "":
+			if rtype in {"A", "AAAA"} and value == "":
 				value = request.environ.get("HTTP_X_FORWARDED_FOR") # normally REMOTE_ADDR but we're behind nginx as a reverse proxy
 
 			# Cannot add empty records.
@@ -438,7 +437,7 @@ def ssl_get_status():
 		{
 			"domain": d["domain"],
 			"status": d["ssl_certificate"][0],
-			"text": d["ssl_certificate"][1] + ((" " + cant_provision[d["domain"]] if d["domain"] in cant_provision else ""))
+			"text": d["ssl_certificate"][1] + (" " + cant_provision[d["domain"]] if d["domain"] in cant_provision else "")
 		} for d in domains_status ]
 
 	# Warn the user about domain names not hosted here because of other settings.
@@ -510,7 +509,7 @@ def totp_post_enable():
 	secret = request.form.get('secret')
 	token = request.form.get('token')
 	label = request.form.get('label')
-	if type(token) != str:
+	if not isinstance(token, str):
 		return ("Bad Input", 400)
 	try:
 		mfa_totp.validate_secret(secret)
@@ -606,8 +605,7 @@ def system_status():
 def show_updates():
 	from status_checks import list_apt_updates
 	return "".join(
-		"%s (%s)\n"
+		"{} ({})\n".format(p["package"], p["version"])
-		% (p["package"], p["version"])
 		for p in list_apt_updates())
 
 @app.route('/system/update-packages', methods=["POST"])
@@ -806,14 +804,11 @@ def log_failed_login(request):
 	# During setup we call the management interface directly to determine the user
 	# status. So we can't always use X-Forwarded-For because during setup that header
 	# will not be present.
-	if request.headers.getlist("X-Forwarded-For"):
+	ip = request.headers.getlist("X-Forwarded-For")[0] if request.headers.getlist("X-Forwarded-For") else request.remote_addr
-		ip = request.headers.getlist("X-Forwarded-For")[0]
-	else:
-		ip = request.remote_addr
 
 	# We need to add a timestamp to the log message, otherwise /dev/log will eat the "duplicate"
 	# message.
-	app.logger.warning( "Mail-in-a-Box Management Daemon: Failed login attempt from ip %s - timestamp %s" % (ip, time.time()))
+	app.logger.warning( f"Mail-in-a-Box Management Daemon: Failed login attempt from ip {ip} - timestamp {time.time()}")
 
 
 # APP

management/dns_update.py

@@ -13,7 +13,7 @@
 # and mail aliases and restarts nsd.
 ########################################################################
 
-import sys, os, os.path, urllib.parse, datetime, re, hashlib, base64
+import sys, os, os.path, datetime, re, hashlib, base64
 import ipaddress
 import rtyaml
 import dns.resolver
@@ -21,12 +21,13 @@ import hooks
 
 from utils import shell, load_env_vars_from_file, safe_domain_name, sort_domains
 from ssl_certificates import get_ssl_certificates, check_certificate
+import contextlib
 
 # From https://stackoverflow.com/questions/3026957/how-to-validate-a-domain-name-using-regex-php/16491074#16491074
 # This regular expression matches domain names according to RFCs, it also accepts fqdn with an leading dot,
 # underscores, as well as asteriks which are allowed in domain names but not hostnames (i.e. allowed in
 # DNS but not in URLs), which are common in certain record types like for DKIM.
-DOMAIN_RE = "^(?!\-)(?:[*][.])?(?:[a-zA-Z\d\-_]{0,62}[a-zA-Z\d_]\.){1,126}(?!\d+)[a-zA-Z\d_]{1,63}(\.?)$"
+DOMAIN_RE = r"^(?!\-)(?:[*][.])?(?:[a-zA-Z\d\-_]{0,62}[a-zA-Z\d_]\.){1,126}(?!\d+)[a-zA-Z\d_]{1,63}(\.?)$"
 
 def get_dns_domains(env):
 	# Add all domain names in use by email users and mail aliases, any
@@ -48,7 +49,7 @@ def get_dns_zones(env):
 	# Exclude domains that are subdomains of other domains we know. Proceed
 	# by looking at shorter domains first.
 	zone_domains = set()
-	for domain in sorted(domains, key=lambda d : len(d)):
+	for domain in sorted(domains, key=len):
 		for d in zone_domains:
 			if domain.endswith("." + d):
 				# We found a parent domain already in the list.
@@ -58,9 +59,7 @@ def get_dns_zones(env):
 			zone_domains.add(domain)
 
 	# Make a nice and safe filename for each domain.
-	zonefiles = []
+	zonefiles = [[domain, safe_domain_name(domain) + ".txt"] for domain in zone_domains]
-	for domain in zone_domains:
-		zonefiles.append([domain, safe_domain_name(domain) + ".txt"])
 
 	# Sort the list so that the order is nice and so that nsd.conf has a
 	# stable order so we don't rewrite the file & restart the service
@@ -204,8 +203,7 @@ def build_zone(domain, domain_properties, additional_records, env, is_zone=True)
 		# User may provide one or more additional nameservers
 		secondary_ns_list = get_secondary_dns(additional_records, mode="NS") \
 			or ["ns2." + env["PRIMARY_HOSTNAME"]]
-		for secondary_ns in secondary_ns_list:
+		records.extend((None,  "NS", secondary_ns+'.', False) for secondary_ns in secondary_ns_list)
-			records.append((None,  "NS", secondary_ns+'.', False))
 
 
 	# In PRIMARY_HOSTNAME...
@@ -222,8 +220,7 @@ def build_zone(domain, domain_properties, additional_records, env, is_zone=True)
 		records.append(("_443._tcp", "TLSA", build_tlsa_record(env), "Optional. When DNSSEC is enabled, provides out-of-band HTTPS certificate validation for a few web clients that support it."))
 
 		# Add a SSHFP records to help SSH key validation. One per available SSH key on this system.
-		for value in build_sshfp_records():
+		records.extend((None, "SSHFP", value, "Optional. Provides an out-of-band method for verifying an SSH key before connecting. Use 'VerifyHostKeyDNS yes' (or 'VerifyHostKeyDNS ask') when connecting with ssh.") for value in build_sshfp_records())
-			records.append((None, "SSHFP", value, "Optional. Provides an out-of-band method for verifying an SSH key before connecting. Use 'VerifyHostKeyDNS yes' (or 'VerifyHostKeyDNS ask') when connecting with ssh."))
 
 	# Add DNS records for any subdomains of this domain. We should not have a zone for
 	# both a domain and one of its subdomains.
@@ -233,7 +230,7 @@ def build_zone(domain, domain_properties, additional_records, env, is_zone=True)
 			subdomain_qname = subdomain[0:-len("." + domain)]
 			subzone = build_zone(subdomain, domain_properties, additional_records, env, is_zone=False)
 			for child_qname, child_rtype, child_value, child_explanation in subzone:
-				if child_qname == None:
+				if child_qname is None:
 					child_qname = subdomain_qname
 				else:
 					child_qname += "." + subdomain_qname
@@ -241,10 +238,7 @@ def build_zone(domain, domain_properties, additional_records, env, is_zone=True)
 
 	has_rec_base = list(records) # clone current state
 	def has_rec(qname, rtype, prefix=None):
-		for rec in has_rec_base:
+		return any(rec[0] == qname and rec[1] == rtype and (prefix is None or rec[2].startswith(prefix)) for rec in has_rec_base)
-			if rec[0] == qname and rec[1] == rtype and (prefix is None or rec[2].startswith(prefix)):
-				return True
-		return False
 
 	# The user may set other records that don't conflict with our settings.
 	# Don't put any TXT records above this line, or it'll prevent any custom TXT records.
@@ -272,7 +266,7 @@ def build_zone(domain, domain_properties, additional_records, env, is_zone=True)
 	has_rec_base = list(records)
 	a_expl = "Required. May have a different value. Sets the IP address that %s resolves to for web hosting and other services besides mail. The A record must be present but its value does not affect mail delivery." % domain
 	if domain_properties[domain]["auto"]:
-		if domain.startswith("ns1.") or domain.startswith("ns2."): a_expl = False # omit from 'External DNS' page since this only applies if box is its own DNS server
+		if domain.startswith(("ns1.", "ns2.")): a_expl = False # omit from 'External DNS' page since this only applies if box is its own DNS server
 		if domain.startswith("www."): a_expl = "Optional. Sets the IP address that %s resolves to so that the box can provide a redirect to the parent domain." % domain
 		if domain.startswith("mta-sts."): a_expl = "Optional. MTA-STS Policy Host serving /.well-known/mta-sts.txt."
 		if domain.startswith("autoconfig."): a_expl = "Provides email configuration autodiscovery support for Thunderbird Autoconfig."
@@ -308,7 +302,7 @@ def build_zone(domain, domain_properties, additional_records, env, is_zone=True)
 		# Append the DKIM TXT record to the zone as generated by OpenDKIM.
 		# Skip if the user has set a DKIM record already.
 		opendkim_record_file = os.path.join(env['STORAGE_ROOT'], 'mail/dkim/mail.txt')
-		with open(opendkim_record_file) as orf:
+		with open(opendkim_record_file, encoding="utf-8") as orf:
 			m = re.match(r'(\S+)\s+IN\s+TXT\s+\( ((?:"[^"]+"\s+)+)\)', orf.read(), re.S)
 			val = "".join(re.findall(r'"([^"]+)"', m.group(2)))
 			if not has_rec(m.group(1), "TXT", prefix="v=DKIM1; "):
@@ -374,8 +368,8 @@ def build_zone(domain, domain_properties, additional_records, env, is_zone=True)
 	# non-mail domain and also may include qnames from custom DNS records.
 	# Do this once at the end of generating a zone.
 	if is_zone:
-		qnames_with_a = set(qname for (qname, rtype, value, explanation) in records if rtype in ("A", "AAAA"))
+		qnames_with_a = {qname for (qname, rtype, value, explanation) in records if rtype in {"A", "AAAA"}}
-		qnames_with_mx = set(qname for (qname, rtype, value, explanation) in records if rtype == "MX")
+		qnames_with_mx = {qname for (qname, rtype, value, explanation) in records if rtype == "MX"}
 		for qname in qnames_with_a - qnames_with_mx:
 			# Mark this domain as not sending mail with hard-fail SPF and DMARC records.
 			d = (qname+"." if qname else "") + domain
@@ -471,14 +465,12 @@ def build_sshfp_records():
 	# specify that port to sshkeyscan.
 
 	port = 22
-	with open('/etc/ssh/sshd_config', 'r') as f:
+	with open('/etc/ssh/sshd_config', encoding="utf-8") as f:
 		for line in f:
 			s = line.rstrip().split()
 			if len(s) == 2 and s[0] == 'Port':
-				try:
+				with contextlib.suppress(ValueError):
 					port = int(s[1])
-				except ValueError:
-					pass
 				break
 
 	keys = shell("check_output", ["ssh-keyscan", "-4", "-t", "rsa,dsa,ecdsa,ed25519", "-p", str(port), "localhost"])
@@ -487,7 +479,7 @@ def build_sshfp_records():
 	for key in keys:
 		if key.strip() == "" or key[0] == "#": continue
 		try:
-			host, keytype, pubkey = key.split(" ")
+			_host, keytype, pubkey = key.split(" ")
 			yield "%d %d ( %s )" % (
 				algorithm_number[keytype],
 				2, # specifies we are using SHA-256 on next line
@@ -532,7 +524,7 @@ $TTL 86400          ; default time to live
 	zone = zone.format(domain=domain, primary_domain=env["PRIMARY_HOSTNAME"])
 
 	# Add records.
-	for subdomain, querytype, value, explanation in records:
+	for subdomain, querytype, value, _explanation in records:
 		if subdomain:
 			zone += subdomain
 		zone += "\tIN\t" + querytype + "\t"
@@ -550,7 +542,7 @@ $TTL 86400          ; default time to live
 		zone += value + "\n"
 
 	# Append a stable hash of DNSSEC signing keys in a comment.
-	zone += "\n; DNSSEC signing keys hash: {}\n".format(hash_dnssec_keys(domain, env))
+	zone += f"\n; DNSSEC signing keys hash: {hash_dnssec_keys(domain, env)}\n"
 
 	# DNSSEC requires re-signing a zone periodically. That requires
 	# bumping the serial number even if no other records have changed.
@@ -566,7 +558,7 @@ $TTL 86400          ; default time to live
 		# We've signed the domain. Check if we are close to the expiration
 		# time of the signature. If so, we'll force a bump of the serial
 		# number so we can re-sign it.
-		with open(zonefile + ".signed") as f:
+		with open(zonefile + ".signed", encoding="utf-8") as f:
 			signed_zone = f.read()
 		expiration_times = re.findall(r"\sRRSIG\s+SOA\s+\d+\s+\d+\s\d+\s+(\d{14})", signed_zone)
 		if len(expiration_times) == 0:
@@ -585,7 +577,7 @@ $TTL 86400          ; default time to live
 	if os.path.exists(zonefile):
 		# If the zone already exists, is different, and has a later serial number,
 		# increment the number.
-		with open(zonefile) as f:
+		with open(zonefile, encoding="utf-8") as f:
 			existing_zone = f.read()
 			m = re.search(r"(\d+)\s*;\s*serial number", existing_zone)
 			if m:
@@ -609,7 +601,7 @@ $TTL 86400          ; default time to live
 	zone = zone.replace("__SERIAL__", serial)
 
 	# Write the zone file.
-	with open(zonefile, "w") as f:
+	with open(zonefile, "w", encoding="utf-8") as f:
 		f.write(zone)
 
 	return True # file is updated
@@ -622,7 +614,7 @@ def get_dns_zonefile(zone, env):
 		raise ValueError("%s is not a domain name that corresponds to a zone." % zone)
 
 	nsd_zonefile = "/etc/nsd/zones/" + fn
-	with open(nsd_zonefile, "r") as f:
+	with open(nsd_zonefile, encoding="utf-8") as f:
 		return f.read()
 
 ########################################################################
@@ -634,11 +626,11 @@ def write_nsd_conf(zonefiles, additional_records, env):
 
 	# Append the zones.
 	for domain, zonefile in zonefiles:
-		nsdconf += """
+		nsdconf += f"""
 zone:
-	name: %s
+	name: {domain}
-	zonefile: %s
+	zonefile: {zonefile}
-""" % (domain, zonefile)
+"""
 
 		# If custom secondary nameservers have been set, allow zone transfers
 		# and, if not a subnet, notifies to them.
@@ -650,13 +642,13 @@ zone:
 	# Check if the file is changing. If it isn't changing,
 	# return False to flag that no change was made.
 	if os.path.exists(nsd_conf_file):
-		with open(nsd_conf_file) as f:
+		with open(nsd_conf_file, encoding="utf-8") as f:
 			if f.read() == nsdconf:
 				return False
 
 	# Write out new contents and return True to signal that
 	# configuration changed.
-	with open(nsd_conf_file, "w") as f:
+	with open(nsd_conf_file, "w", encoding="utf-8") as f:
 		f.write(nsdconf)
 	return True
 
@@ -690,9 +682,8 @@ def hash_dnssec_keys(domain, env):
 	keydata = []
 	for keytype, keyfn in sorted(find_dnssec_signing_keys(domain, env)):
 		oldkeyfn = os.path.join(env['STORAGE_ROOT'], 'dns/dnssec', keyfn + ".private")
-		keydata.append(keytype)
+		keydata.extend((keytype, keyfn))
-		keydata.append(keyfn)
+		with open(oldkeyfn, encoding="utf-8") as fr:
-		with open(oldkeyfn, "r") as fr:
 			keydata.append( fr.read() )
 	keydata = "".join(keydata).encode("utf8")
 	return hashlib.sha1(keydata).hexdigest()
@@ -720,12 +711,12 @@ def sign_zone(domain, zonefile, env):
 			# Use os.umask and open().write() to securely create a copy that only
 			# we (root) can read.
 			oldkeyfn = os.path.join(env['STORAGE_ROOT'], 'dns/dnssec', keyfn + ext)
-			with open(oldkeyfn, "r") as fr:
+			with open(oldkeyfn, encoding="utf-8") as fr:
 				keydata = fr.read()
 			keydata = keydata.replace("_domain_", domain)
 			prev_umask = os.umask(0o77) # ensure written file is not world-readable
 			try:
-				with open(newkeyfn + ext, "w") as fw:
+				with open(newkeyfn + ext, "w", encoding="utf-8") as fw:
 					fw.write(keydata)
 			finally:
 				os.umask(prev_umask) # other files we write should be world-readable
@@ -759,7 +750,7 @@ def sign_zone(domain, zonefile, env):
 	# be used, so we'll pre-generate all for each key. One DS record per line. Only one
 	# needs to actually be deployed at the registrar. We'll select the preferred one
 	# in the status checks.
-	with open("/etc/nsd/zones/" + zonefile + ".ds", "w") as f:
+	with open("/etc/nsd/zones/" + zonefile + ".ds", "w", encoding="utf-8") as f:
 		for key in ksk_keys:
 			for digest_type in ('1', '2', '4'):
 				rr_ds = shell('check_output', ["/usr/bin/ldns-key2ds",
@@ -796,7 +787,7 @@ def write_opendkim_tables(domains, env):
 		# So we must have a separate KeyTable entry for each domain.
 		"SigningTable":
 			"".join(
-				"*@{domain} {domain}\n".format(domain=domain)
+				f"*@{domain} {domain}\n"
 				for domain in domains
 			),
 
@@ -805,7 +796,7 @@ def write_opendkim_tables(domains, env):
 		# signing domain must match the sender's From: domain.
 		"KeyTable":
 			"".join(
-				"{domain} {domain}:mail:{key_file}\n".format(domain=domain, key_file=opendkim_key_file)
+				f"{domain} {domain}:mail:{opendkim_key_file}\n"
 				for domain in domains
 			),
 	}
@@ -814,12 +805,12 @@ def write_opendkim_tables(domains, env):
 	for filename, content in config.items():
 		# Don't write the file if it doesn't need an update.
 		if os.path.exists("/etc/opendkim/" + filename):
-			with open("/etc/opendkim/" + filename) as f:
+			with open("/etc/opendkim/" + filename, encoding="utf-8") as f:
 				if f.read() == content:
 					continue
 
 		# The contents needs to change.
-		with open("/etc/opendkim/" + filename, "w") as f:
+		with open("/etc/opendkim/" + filename, "w", encoding="utf-8") as f:
 			f.write(content)
 		did_update = True
 
@@ -831,9 +822,9 @@ def write_opendkim_tables(domains, env):
 
 def get_custom_dns_config(env, only_real_records=False):
 	try:
-		with open(os.path.join(env['STORAGE_ROOT'], 'dns/custom.yaml'), 'r') as f:
+		with open(os.path.join(env['STORAGE_ROOT'], 'dns/custom.yaml'), encoding="utf-8") as f:
 			custom_dns = rtyaml.load(f)
-		if not isinstance(custom_dns, dict): raise ValueError() # caught below
+		if not isinstance(custom_dns, dict): raise ValueError # caught below
 	except:
 		return [ ]
 
@@ -851,7 +842,7 @@ def get_custom_dns_config(env, only_real_records=False):
 
 		# No other type of data is allowed.
 		else:
-			raise ValueError()
+			raise ValueError
 
 		for rtype, value2 in values:
 			if isinstance(value2, str):
@@ -861,7 +852,7 @@ def get_custom_dns_config(env, only_real_records=False):
 					yield (qname, rtype, value3)
 			# No other type of data is allowed.
 			else:
-				raise ValueError()
+				raise ValueError
 
 def filter_custom_records(domain, custom_dns_iter):
 	for qname, rtype, value in custom_dns_iter:
@@ -877,10 +868,7 @@ def filter_custom_records(domain, custom_dns_iter):
 		# our short form (None => domain, or a relative QNAME) if
 		# domain is not None.
 		if domain is not None:
-			if qname == domain:
+			qname = None if qname == domain else qname[0:len(qname) - len("." + domain)]
-				qname = None
-			else:
-				qname = qname[0:len(qname)-len("." + domain)]
 
 		yield (qname, rtype, value)
 
@@ -916,12 +904,12 @@ def write_custom_dns_config(config, env):
 
 	# Write.
 	config_yaml = rtyaml.dump(dns)
-	with open(os.path.join(env['STORAGE_ROOT'], 'dns/custom.yaml'), "w") as f:
+	with open(os.path.join(env['STORAGE_ROOT'], 'dns/custom.yaml'), "w", encoding="utf-8") as f:
 		f.write(config_yaml)
 
 def set_custom_dns_record(qname, rtype, value, action, env):
 	# validate qname
-	for zone, fn in get_dns_zones(env):
+	for zone, _fn in get_dns_zones(env):
 		# It must match a zone apex or be a subdomain of a zone
 		# that we are otherwise hosting.
 		if qname == zone or qname.endswith("."+zone):
@@ -935,24 +923,27 @@ def set_custom_dns_record(qname, rtype, value, action, env):
 	rtype = rtype.upper()
 	if value is not None and qname != "_secondary_nameserver":
 		if not re.search(DOMAIN_RE, qname):
-			raise ValueError("Invalid name.")
+			msg = "Invalid name."
+			raise ValueError(msg)
 
-		if rtype in ("A", "AAAA"):
+		if rtype in {"A", "AAAA"}:
 			if value != "local": # "local" is a special flag for us
 				v = ipaddress.ip_address(value) # raises a ValueError if there's a problem
 				if rtype == "A" and not isinstance(v, ipaddress.IPv4Address): raise ValueError("That's an IPv6 address.")
 				if rtype == "AAAA" and not isinstance(v, ipaddress.IPv6Address): raise ValueError("That's an IPv4 address.")
-		elif rtype in ("CNAME", "NS"):
+		elif rtype in {"CNAME", "NS"}:
 			if rtype == "NS" and qname == zone:
-				raise ValueError("NS records can only be set for subdomains.")
+				msg = "NS records can only be set for subdomains."
+				raise ValueError(msg)
 
 			# ensure value has a trailing dot
 			if not value.endswith("."):
 				value = value + "."
 
 			if not re.search(DOMAIN_RE, value):
-				raise ValueError("Invalid value.")
+				msg = "Invalid value."
-		elif rtype in ("CNAME", "TXT", "SRV", "MX", "SSHFP", "CAA"):
+				raise ValueError(msg)
+		elif rtype in {"CNAME", "TXT", "SRV", "MX", "SSHFP", "CAA"}:
 			# anything goes
 			pass
 		else:
@@ -985,7 +976,7 @@ def set_custom_dns_record(qname, rtype, value, action, env):
 				# Drop this record.
 				made_change = True
 				continue
-			if value == None and (_qname, _rtype) == (qname, rtype):
+			if value is None and (_qname, _rtype) == (qname, rtype):
 				# Drop all qname-rtype records.
 				made_change = True
 				continue
@@ -995,7 +986,7 @@ def set_custom_dns_record(qname, rtype, value, action, env):
 		# Preserve this record.
 		newconfig.append((_qname, _rtype, _value))
 
-	if action in ("add", "set") and needs_add and value is not None:
+	if action in {"add", "set"} and needs_add and value is not None:
 		newconfig.append((qname, rtype, value))
 		made_change = True
 
@@ -1012,11 +1003,11 @@ def get_secondary_dns(custom_dns, mode=None):
 	resolver.lifetime = 10
 
 	values = []
-	for qname, rtype, value in custom_dns:
+	for qname, _rtype, value in custom_dns:
 		if qname != '_secondary_nameserver': continue
 		for hostname in value.split(" "):
 			hostname = hostname.strip()
-			if mode == None:
+			if mode is None:
 				# Just return the setting.
 				values.append(hostname)
 				continue
@@ -1057,24 +1048,24 @@ def set_secondary_dns(hostnames, env):
 		resolver = dns.resolver.get_default_resolver()
 		resolver.timeout = 5
 		resolver.lifetime = 5
-		
+
 		for item in hostnames:
 			if not item.startswith("xfr:"):
 				# Resolve hostname.
 				try:
-					response = resolver.resolve(item, "A")
+					resolver.resolve(item, "A")
 				except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer, dns.resolver.Timeout):
 					try:
-						response = resolver.resolve(item, "AAAA")
+						resolver.resolve(item, "AAAA")
 					except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer, dns.resolver.Timeout):
 						raise ValueError("Could not resolve the IP address of %s." % item)
 			else:
 				# Validate IP address.
 				try:
 					if "/" in item[4:]:
-						v = ipaddress.ip_network(item[4:]) # raises a ValueError if there's a problem
+						ipaddress.ip_network(item[4:]) # raises a ValueError if there's a problem
 					else:
-						v = ipaddress.ip_address(item[4:]) # raises a ValueError if there's a problem
+						ipaddress.ip_address(item[4:]) # raises a ValueError if there's a problem
 				except ValueError:
 					raise ValueError("'%s' is not an IPv4 or IPv6 address or subnet." % item[4:])
 
@@ -1092,13 +1083,12 @@ def get_custom_dns_records(custom_dns, qname, rtype):
 	for qname1, rtype1, value in custom_dns:
 		if qname1 == qname and rtype1 == rtype:
 			yield value
-	return None
 
 ########################################################################
 
 def build_recommended_dns(env):
 	ret = []
-	for (domain, zonefile, records) in build_zones(env):
+	for (domain, _zonefile, records) in build_zones(env):
 		# remove records that we don't display
 		records = [r for r in records if r[3] is not False]
 
@@ -1107,10 +1097,7 @@ def build_recommended_dns(env):
 
 		# expand qnames
 		for i in range(len(records)):
-			if records[i][0] == None:
+			qname = domain if records[i][0] is None else records[i][0] + "." + domain
-				qname = domain
-			else:
-				qname = records[i][0] + "." + domain
 
 			records[i] = {
 				"qname": qname,
@@ -1129,7 +1116,7 @@ if __name__ == "__main__":
 	if sys.argv[-1] == "--lint":
 		write_custom_dns_config(get_custom_dns_config(env), env)
 	else:
-		for zone, records in build_recommended_dns(env):
+		for _zone, records in build_recommended_dns(env):
 			for record in records:
 				print("; " + record['explanation'])
 				print(record['qname'], record['rtype'], record['value'], sep="\t")

management/email_administrator.py

@@ -46,11 +46,11 @@ msg = MIMEMultipart('alternative')
 # In Python 3.6:
 #msg = Message()
 
-msg['From'] = "\"%s\" <%s>" % (env['PRIMARY_HOSTNAME'], admin_addr)
+msg['From'] = '"{}" <{}>'.format(env['PRIMARY_HOSTNAME'], admin_addr)
 msg['To'] = admin_addr
-msg['Subject'] = "[%s] %s" % (env['PRIMARY_HOSTNAME'], subject)
+msg['Subject'] = "[{}] {}".format(env['PRIMARY_HOSTNAME'], subject)
 
-content_html = '<html><body><pre style="overflow-x: scroll; white-space: pre;">{}</pre></body></html>'.format(html.escape(content))
+content_html = f'<html><body><pre style="overflow-x: scroll; white-space: pre;">{html.escape(content)}</pre></body></html>'
 
 msg.attach(MIMEText(content, 'plain'))
 msg.attach(MIMEText(content_html, 'html'))

management/mail_log.py

@@ -131,8 +131,7 @@ def scan_mail_log(env):
     except ImportError:
         pass
 
-    print("Scanning logs from {:%Y-%m-%d %H:%M:%S} to {:%Y-%m-%d %H:%M:%S}".format(
+    print(f"Scanning logs from {START_DATE:%Y-%m-%d %H:%M:%S} to {END_DATE:%Y-%m-%d %H:%M:%S}"
-        START_DATE, END_DATE)
     )
 
     # Scan the lines in the log files until the date goes out of range
@@ -238,7 +237,7 @@ def scan_mail_log(env):
             ],
             sub_data=[
                 ("Protocol and Source", [[
-                    "{} {}: {} times".format(protocol_name, host, count)
+                    f"{protocol_name} {host}: {count} times"
                     for (protocol_name, host), count
                     in sorted(u["totals_by_protocol_and_host"].items(), key=lambda kv:-kv[1])
                   ] for u in data.values()])
@@ -314,8 +313,7 @@ def scan_mail_log(env):
                 for date, sender, message in user_data["blocked"]:
                     if len(sender) > 64:
                         sender = sender[:32] + "…" + sender[-32:]
-                    user_rejects.append("%s - %s " % (date, sender))
+                    user_rejects.extend((f'{date} - {sender} ', '  %s' % message))
-                    user_rejects.append("  %s" % message)
                 rejects.append(user_rejects)
 
         print_user_table(
@@ -333,7 +331,7 @@ def scan_mail_log(env):
     if collector["other-services"] and VERBOSE and False:
         print_header("Other services")
         print("The following unkown services were found in the log file.")
-        print(" ", *sorted(list(collector["other-services"])), sep='\n│ ')
+        print(" ", *sorted(collector["other-services"]), sep='\n│ ')
 
 
 def scan_mail_log_line(line, collector):
@@ -344,7 +342,7 @@ def scan_mail_log_line(line, collector):
     if not m:
         return True
 
-    date, system, service, log = m.groups()
+    date, _system, service, log = m.groups()
     collector["scan_count"] += 1
 
     # print()
@@ -355,7 +353,7 @@ def scan_mail_log_line(line, collector):
 
     # Replaced the dateutil parser for a less clever way of parser that is roughly 4 times faster.
     # date = dateutil.parser.parse(date)
-    
+
     # strptime fails on Feb 29 with ValueError: day is out of range for month if correct year is not provided.
     # See https://bugs.python.org/issue26460
     date = datetime.datetime.strptime(str(NOW.year) + ' ' + date, '%Y %b %d %H:%M:%S')
@@ -387,9 +385,9 @@ def scan_mail_log_line(line, collector):
     elif service == "postfix/smtpd":
         if SCAN_BLOCKED:
             scan_postfix_smtpd_line(date, log, collector)
-    elif service in ("postfix/qmgr", "postfix/pickup", "postfix/cleanup", "postfix/scache",
+    elif service in {"postfix/qmgr", "postfix/pickup", "postfix/cleanup", "postfix/scache",
                      "spampd", "postfix/anvil", "postfix/master", "opendkim", "postfix/lmtp",
-                     "postfix/tlsmgr", "anvil"):
+                     "postfix/tlsmgr", "anvil"}:
         # nothing to look at
         return True
     else:
@@ -403,7 +401,7 @@ def scan_mail_log_line(line, collector):
 def scan_postgrey_line(date, log, collector):
     """ Scan a postgrey log line and extract interesting data """
 
-    m = re.match("action=(greylist|pass), reason=(.*?), (?:delay=\d+, )?client_name=(.*), "
+    m = re.match(r"action=(greylist|pass), reason=(.*?), (?:delay=\d+, )?client_name=(.*), "
                  "client_address=(.*), sender=(.*), recipient=(.*)",
                  log)
 
@@ -446,36 +444,35 @@ def scan_postfix_smtpd_line(date, log, collector):
             return
 
         # only log mail to known recipients
-        if user_match(user):
+        if user_match(user) and (collector["known_addresses"] is None or user in collector["known_addresses"]):
-            if collector["known_addresses"] is None or user in collector["known_addresses"]:
+            data = collector["rejected"].get(
-                data = collector["rejected"].get(
+                user,
-                    user,
+                {
-                    {
+                    "blocked": [],
-                        "blocked": [],
+                    "earliest": None,
-                        "earliest": None,
+                    "latest": None,
-                        "latest": None,
+                }
-                    }
+            )
-                )
+            # simplify this one
-                # simplify this one
+            m = re.search(
+                r"Client host \[(.*?)\] blocked using zen.spamhaus.org; (.*)", message
+            )
+            if m:
+                message = "ip blocked: " + m.group(2)
+            else:
+                # simplify this one too
                 m = re.search(
-                    r"Client host \[(.*?)\] blocked using zen.spamhaus.org; (.*)", message
+                    r"Sender address \[.*@(.*)\] blocked using dbl.spamhaus.org; (.*)", message
                 )
                 if m:
-                    message = "ip blocked: " + m.group(2)
+                    message = "domain blocked: " + m.group(2)
-                else:
-                    # simplify this one too
-                    m = re.search(
-                        r"Sender address \[.*@(.*)\] blocked using dbl.spamhaus.org; (.*)", message
-                    )
-                    if m:
-                        message = "domain blocked: " + m.group(2)
 
-                if data["earliest"] is None:
+            if data["earliest"] is None:
-                    data["earliest"] = date
+                data["earliest"] = date
-                data["latest"] = date
+            data["latest"] = date
-                data["blocked"].append((date, sender, message))
+            data["blocked"].append((date, sender, message))
 
-                collector["rejected"][user] = data
+            collector["rejected"][user] = data
 
 
 def scan_dovecot_login_line(date, log, collector, protocol_name):
@@ -511,7 +508,7 @@ def add_login(user, date, protocol_name, host, collector):
             data["totals_by_protocol"][protocol_name] += 1
             data["totals_by_protocol_and_host"][(protocol_name, host)] += 1
 
-            if host not in ("127.0.0.1", "::1") or True:
+            if host not in {"127.0.0.1", "::1"} or True:
                 data["activity-by-hour"][protocol_name][date.hour] += 1
 
             collector["logins"][user] = data
@@ -525,7 +522,7 @@ def scan_postfix_lmtp_line(date, log, collector):
 
     """
 
-    m = re.match("([A-Z0-9]+): to=<(\S+)>, .* Saved", log)
+    m = re.match(r"([A-Z0-9]+): to=<(\S+)>, .* Saved", log)
 
     if m:
         _, user = m.groups()
@@ -561,12 +558,12 @@ def scan_postfix_submission_line(date, log, collector):
     """
 
     # Match both the 'plain' and 'login' sasl methods, since both authentication methods are
-    # allowed by Dovecot. Exclude trailing comma after the username when additional fields 
+    # allowed by Dovecot. Exclude trailing comma after the username when additional fields
 	# follow after.
-    m = re.match("([A-Z0-9]+): client=(\S+), sasl_method=(PLAIN|LOGIN), sasl_username=(\S+)(?<!,)", log)
+    m = re.match(r"([A-Z0-9]+): client=(\S+), sasl_method=(PLAIN|LOGIN), sasl_username=(\S+)(?<!,)", log)
 
     if m:
-        _, client, method, user = m.groups()
+        _, client, _method, user = m.groups()
 
         if user_match(user):
             # Get the user data, or create it if the user is new
@@ -599,7 +596,7 @@ def scan_postfix_submission_line(date, log, collector):
 def readline(filename):
     """ A generator that returns the lines of a file
     """
-    with open(filename, errors='replace') as file:
+    with open(filename, errors='replace', encoding='utf-8') as file:
         while True:
           line = file.readline()
           if not line:
@@ -633,10 +630,7 @@ def print_time_table(labels, data, do_print=True):
     data.insert(0, [str(h) for h in range(24)])
 
     temp = "│ {:<%d} " % max(len(l) for l in labels)
-    lines = []
+    lines = [temp.format(label) for label in labels]
-
-    for label in labels:
-        lines.append(temp.format(label))
 
     for h in range(24):
         max_len = max(len(str(d[h])) for d in data)
@@ -650,6 +644,7 @@ def print_time_table(labels, data, do_print=True):
 
     if do_print:
         print("\n".join(lines))
+        return None
     else:
         return lines
 
@@ -683,7 +678,7 @@ def print_user_table(users, data=None, sub_data=None, activity=None, latest=None
                 col_str = str_temp.format(d[row][:31] + "…" if len(d[row]) > 32 else d[row])
                 col_left[col] = True
             elif isinstance(d[row], datetime.datetime):
-                col_str = "{:<20}".format(str(d[row]))
+                col_str = f"{d[row]!s:<20}"
                 col_left[col] = True
             else:
                 temp = "{:>%s}" % max(5, len(l) + 1, len(str(d[row])) + 1)
@@ -695,7 +690,7 @@ def print_user_table(users, data=None, sub_data=None, activity=None, latest=None
                 data_accum[col] += d[row]
 
         try:
-            if None not in [latest, earliest]:
+            if None not in {latest, earliest}:
                 vert_pos = len(line)
                 e = earliest[row]
                 l = latest[row]
@@ -723,10 +718,7 @@ def print_user_table(users, data=None, sub_data=None, activity=None, latest=None
                 if sub_data is not None:
                     for l, d in sub_data:
                         if d[row]:
-                            lines.append("┬")
+                            lines.extend(('┬', '│ %s' % l, '├─%s─' % (len(l) * '─'), '│'))
-                            lines.append("│ %s" % l)
-                            lines.append("├─%s─" % (len(l) * "─"))
-                            lines.append("│")
                             max_len = 0
                             for v in list(d[row]):
                                 lines.append("│ %s" % v)
@@ -751,7 +743,7 @@ def print_user_table(users, data=None, sub_data=None, activity=None, latest=None
         else:
             header += l.rjust(max(5, len(l) + 1, col_widths[col]))
 
-    if None not in (latest, earliest):
+    if None not in {latest, earliest}:
         header += " │ timespan   "
 
     lines.insert(0, header.rstrip())
@@ -776,7 +768,7 @@ def print_user_table(users, data=None, sub_data=None, activity=None, latest=None
         footer += temp.format(data_accum[row])
 
     try:
-        if None not in [latest, earliest]:
+        if None not in {latest, earliest}:
             max_l = max(latest)
             min_e = min(earliest)
             timespan = relativedelta(max_l, min_e)
@@ -855,7 +847,7 @@ if __name__ == "__main__":
         END_DATE = args.enddate
         if args.timespan == 'today':
             args.timespan = 'day'
-        print("Setting end date to {}".format(END_DATE))
+        print(f"Setting end date to {END_DATE}")
 
     START_DATE = END_DATE - TIME_DELTAS[args.timespan]
 

management/mailconfig.py

@@ -97,7 +97,7 @@ def validate_email(email, mode=None):
 			email_domain = get_domain(email)
 		except IndexError:
 			raise EmailNotValidError(email)
-		
+
 		test_env = (
 			email_domain.endswith(".local") and
 			email_domain == socket.getfqdn()
@@ -161,10 +161,7 @@ def prettify_idn_email_address(email):
 
 def is_dcv_address(email):
 	email = email.lower()
-	for localpart in ("admin", "administrator", "postmaster", "hostmaster", "webmaster", "abuse"):
+	return any(email.startswith((localpart + "@", localpart + "+")) for localpart in ("admin", "administrator", "postmaster", "hostmaster", "webmaster", "abuse"))
-		if email.startswith(localpart+"@") or email.startswith(localpart+"+"):
-			return True
-	return False
 
 def utf8_from_idna(domain_idna):
 	try:
@@ -183,7 +180,7 @@ def find_mail_user(env, email, attributes=None, conn=None):
 	# email is the users email address
 	# attributes are a list of attributes to return eg ["mail","maildrop"]
 	# conn is a ldap database connection, if not specified a new one
-	# is established	
+	# is established
 	#
 	# The ldap record for the user is returned or None if not found.
 	if not conn: conn = open_database(env)
@@ -196,7 +193,7 @@ def find_mail_user(env, email, attributes=None, conn=None):
 		raise LookupError("Detected more than one user with the same email address (%s): %s" % (email, ";".join(dns)))
 	else:
 		return response.next()
-	
+
 def find_mail_alias(env, email_idna, attributes=None, conn=None, auto=None):
 	# Find the alias with the given address and return the ldap
 	# records for it and the associated permitted senders (if one).
@@ -216,7 +213,7 @@ def find_mail_alias(env, email_idna, attributes=None, conn=None, auto=None):
 	# A tuple having the two ldap records for the alias and it's
 	# permitted senders (alias, permitted_senders) is returned. If
 	# either is not found, the corresponding tuple value will be None.
-	# 
+	#
 	if not conn: conn = open_database(env)
 	# get alias
 	q = [
@@ -245,7 +242,7 @@ def find_mail_alias(env, email_idna, attributes=None, conn=None, auto=None):
 		raise LookupError("Detected more than one permitted senders group with the same email address (%s): %s" % (email_idna, ";".join(dns)))
 	permitted_senders = response.next()
 	return (alias, permitted_senders)
-	
+
 
 def primary_address(mail):
 	# return the first IDNA-encoded email address
@@ -285,7 +282,7 @@ def get_mail_users(env, as_map=False, map_by="maildrop"):
 		users = [ primary_address(rec['mail']).lower() for rec in pager ]
 		return utils.sort_email_addresses(users, env)
 
-	
+
 def get_mail_users_ex(env, with_archived=False):
 	# Returns a complex data structure of all user accounts, optionally
 	# including archived (status="inactive") accounts.
@@ -395,7 +392,7 @@ def get_mail_aliases(env, as_map=False, map_by="primary_address"):
 	c = open_database(env)
 	# get all permitted senders
 	pager = c.paged_search(env.LDAP_PERMITTED_SENDERS_BASE, "(objectClass=mailGroup)", attributes=["mail", "member"])
-	
+
 	# make a dict of permitted senders, key=mail(lowercase) value=members
 	permitted_senders = { }
 	for rec in pager:
@@ -409,7 +406,7 @@ def get_mail_aliases(env, as_map=False, map_by="primary_address"):
 		attributes=[
 			'mail','member','mailMember','description','namedProperty'
 		])
-	
+
 	# make a dict of aliases
 	# key=email(lowercase), value=(email, forward-tos, permitted-senders, auto).
 	aliases = {}
@@ -421,7 +418,7 @@ def get_mail_aliases(env, as_map=False, map_by="primary_address"):
 
 		for fwd_to in alias['mailMember']:
 			forward_tos.append(fwd_to)
-				
+
 		# chase down permitted senders' email addresses
 		allowed_senders = []
 		primary_email_lc = primary_address(alias['mail']).lower()
@@ -457,10 +454,10 @@ def get_mail_aliases(env, as_map=False, map_by="primary_address"):
 			xas = ",".join(alias["permitted_senders"])
 			list.append( (address, xft, None if xas == "" else xas, alias["auto"]) )
 		return list
-	
+
 	else:
 		return aliases
-			
+
 
 def get_mail_aliases_ex(env):
 	# Returns a complex data structure of all mail aliases, similar
@@ -486,24 +483,24 @@ def get_mail_aliases_ex(env):
 
 	aliases=get_mail_aliases(env, as_map=True, map_by="primary_address")
 	domains = {}
-	
+
 	for mail in aliases:
 		alias=aliases[mail]
 		address=primary_address(alias['mail']).lower()
-		
+
 		# get alias info
 		forwards_to=alias["forward_tos"]
 		permitted_senders=alias["permitted_senders"]
 		description=alias["description"]
 		auto=alias["auto"]
-		
+
 		# skip auto domain maps since these are not informative in the control panel's aliases list
 		if auto and address.startswith("@"): continue
-		
+
 		domain = get_domain(address)
-		
+
 		# add to list
-		if not domain in domains:
+		if domain not in domains:
 			domains[domain] = {
 				"domain": domain,
 				"aliases": [],
@@ -608,7 +605,7 @@ def get_mail_domains(env, as_map=False, category=None, users_only=False):
 			del domains[domain_idna]
 
 	return domains
-	
+
 
 
 def add_mail_domain(env, domain_idna, validate=True):
@@ -634,9 +631,9 @@ def add_mail_domain(env, domain_idna, validate=True):
 		if conn.wait(id).count() == 0:
 			# no mail users are using that domain!
 			return False
-		
+
 	dn = 'dc=%s,%s' % (domain_idna, env.LDAP_DOMAINS_BASE)
-	domain_utf8 = utf8_from_idna(domain_idna)	
+	domain_utf8 = utf8_from_idna(domain_idna)
 	try:
 		response = conn.wait( conn.add(dn, [ 'domain', 'mailDomain' ], {
 			"dcIntl": domain_utf8,
@@ -654,7 +651,7 @@ def add_mail_domain(env, domain_idna, validate=True):
 			pass
 		return False
 
-	
+
 def remove_mail_domain(env, domain_idna, validate=True):
 	# Remove the specified domain from the list of domains that we
 	# handle mail for. The domain must be IDNA encoded.
@@ -674,11 +671,11 @@ def remove_mail_domain(env, domain_idna, validate=True):
 			# there is one or more user or alias with that domain
 			log.debug("remove_mail_domain: %s: has users and/or aliases", domain_idna)
 			return False
-		
+
 	id = conn.search(env.LDAP_DOMAINS_BASE,
 					 "(&(objectClass=domain)(dc=%s))" % domain_idna,
 					 attributes=['businessCategory'])
-	
+
 	existing = conn.wait(id).next()
 	if existing is None:
 		# the domain doesn't exist!
@@ -713,7 +710,7 @@ def add_mail_user(email, pw, privs, display_name, env):
 	# http-status).
 	#
 	# If successful, the string "OK" is returned.
-	
+
 	# validate email
 	if email.strip() == "":
 		return ("No email address provided.", 400)
@@ -750,7 +747,7 @@ def add_mail_user(email, pw, privs, display_name, env):
 	id=conn.search(env.LDAP_ALIASES_BASE, "(&(objectClass=mailGroup)(mail=%s))" % email)
 	if conn.wait(id).count() > 0:
 		return ("An alias exists with that address.", 400)
-	
+
 	## Generate a unique id for uid
 	#uid = '%s' % uuid.uuid4()
 	# use a sha-1 hash of maildrop for uid
@@ -768,7 +765,7 @@ def add_mail_user(email, pw, privs, display_name, env):
 
 	# get the utf8 version if an idna domain was given
 	email_utf8 = email_name + "@" + get_domain(email, as_unicode=True)
-	
+
 	# compile user's attributes
 	# for historical reasons, make the email address lowercase
 	attrs = {
@@ -790,7 +787,7 @@ def add_mail_user(email, pw, privs, display_name, env):
 
 	# set the password - the ldap server will hash it
 	conn.extend.standard.modify_password(user=dn, new_password=pw)
-	
+
 	# tell postfix the domain is local, if needed
 	return_status = "mail user added"
 	domain_idna = get_domain(email, as_unicode=False)
@@ -802,12 +799,12 @@ def add_mail_user(email, pw, privs, display_name, env):
 			if isinstance(result, tuple):
 				# error occurred
 				return result
-			elif result != '':
+			elif result:
 				return_status += "\n" + result
-	
+
 	# convert alias's mailMember to member
 	convert_mailMember(env, conn, dn, email)
-	
+
 	# Update things in case any new domains are added.
 	if domain_added:
 		return kick(env, return_status)
@@ -818,7 +815,7 @@ def set_mail_password(email, pw, env):
 	# validate that the password is acceptable
 	validate_password(pw)
 
-	# find the user				  
+	# find the user
 	conn = open_database(env)
 	user = find_mail_user(env, email, ['shadowLastChange'], conn)
 	if user is None:
@@ -836,7 +833,7 @@ def set_mail_display_name(email, display_name, env):
 	# validate arguments
 	if not display_name or display_name.strip() == "":
 		return ("Display name may not be empty!", 400)
-	
+
 	# find the user
 	conn = open_database(env)
 	user = find_mail_user(env, email, ['cn', 'sn'], conn)
@@ -846,7 +843,7 @@ def set_mail_display_name(email, display_name, env):
 	# update cn and sn
 	sn = display_name[display_name.strip().find(' ')+1:]
 	conn.modify_record(user, {'cn': display_name.strip(), 'sn': sn})
-	
+
 	return "OK"
 
 def validate_login(email, pw, env):
@@ -889,12 +886,12 @@ def remove_mail_user(email_idna, env):
 	#
 	# If successful, the string "OK" is returned.
 	conn = open_database(env)
-	
+
 	# find the user
 	user = find_mail_user(env, email_idna, conn=conn)
 	if user is None:
 		return ("That's not a user (%s)." % email_idna, 400)
-	
+
 	# delete the user
 	conn.wait( conn.delete(user['dn']) )
 
@@ -909,7 +906,7 @@ def remove_mail_user(email_idna, env):
 			if isinstance(result, tuple):
 				# error occurred
 				return result
-			elif result != '':
+			elif result:
 				return_status += "\n" + result
 
 	# Update things in case any domains are removed.
@@ -929,11 +926,11 @@ def get_mail_user_privileges(email, env, empty_on_error=False):
 	except LookupError as e:
 		if empty_on_error: return []
 		raise e
-	
+
 	if user is None:
 		if empty_on_error: return []
 		return ("That's not a user (%s)." % email, 400)
-	
+
 	return user['mailaccess']
 
 def validate_privilege(priv):
@@ -951,7 +948,7 @@ def add_remove_mail_user_privilege(email, priv, action, env):
 	# http-status).
 	#
 	# If successful, the string "OK" is returned.
-	
+
 	# validate
 	validation = validate_privilege(priv)
 	if validation: return validation
@@ -960,7 +957,7 @@ def add_remove_mail_user_privilege(email, priv, action, env):
 	user = find_mail_user(env, email, attributes=['mailaccess'])
 	if user is None:
 		return ("That's not a user (%s)." % email, 400)
-		
+
 	privs = user['mailaccess'].copy()
 
 	# update privs set
@@ -969,7 +966,7 @@ def add_remove_mail_user_privilege(email, priv, action, env):
 		if priv not in privs:
 			privs.append(priv)
 			changed = True
-			
+
 	elif action == "remove":
 		if priv in privs:
 			privs.remove(priv)
@@ -981,7 +978,7 @@ def add_remove_mail_user_privilege(email, priv, action, env):
 	if changed:
 		conn = open_database(env)
 		conn.modify_record( user, {'mailaccess': privs} )
-		
+
 	return "OK"
 
 
@@ -1010,7 +1007,7 @@ def add_required_aliases(env, conn, domain_idna):
 			verbose_result=True
 		))
 		log.debug("add_required_alias: %s: %r", email_utf8, results[-1])
-		
+
 	return results
 
 def remove_required_aliases(env, conn, domain_idna):
@@ -1027,7 +1024,7 @@ def remove_required_aliases(env, conn, domain_idna):
 			ignore_if_not_exists=True
 		))
 		log.debug("remove_required_alias: %s: %r", email_utf8, results[-1])
-		
+
 	return results
 
 
@@ -1056,7 +1053,7 @@ def convert_mailMember(env, conn, dn, mail):
 		except ldap3.core.exceptions.LDAPAttributeOrValueExistsResult:
 			pass
 
-		
+
 def add_mail_alias(address_utf8, description, forwards_to, permitted_senders, env, auto=False, update_if_exists=False, do_kick=True, verbose_result=False):
 	# Add a new alias group with permitted senders.
 	#
@@ -1170,7 +1167,7 @@ def add_mail_alias(address_utf8, description, forwards_to, permitted_senders, en
 			vfwd_tos_local.append(dn)
 		else:
 			vfwd_tos_remote.append(fwd_to["email_idna"])
-			
+
 	# save to db
 
 	conn = open_database(env)
@@ -1187,7 +1184,7 @@ def add_mail_alias(address_utf8, description, forwards_to, permitted_senders, en
 		return ("Alias already exists (%s)." % address, 400)
 	if existing_alias and update_if_exists == 'ignore':
 		return ""
-	
+
 	cn="%s" % uuid.uuid4()
 	dn="cn=%s,%s" % (cn, env.LDAP_ALIASES_BASE)
 	if not description:
@@ -1202,12 +1199,12 @@ def add_mail_alias(address_utf8, description, forwards_to, permitted_senders, en
 				description = "Catch-all for %s" % address
 			else:
 				description ="Mail alias %s" % address
-		
+
 		# when updating, ensure the description has a value because
 		# the ldap schema does not allow an empty field
 		else:
 			description=" "
-	
+
 	attrs = {
 		"mail": address if address == address_utf8.lower() else [ address, address_utf8 ],
 		"description": description,
@@ -1222,7 +1219,7 @@ def add_mail_alias(address_utf8, description, forwards_to, permitted_senders, en
 		attributes,
 		[ 'mailGroup', 'namedProperties' ],
 		attrs)
-	
+
 	if op == 'modify':
 		return_status = "alias updated"
 	else:
@@ -1231,9 +1228,9 @@ def add_mail_alias(address_utf8, description, forwards_to, permitted_senders, en
 
 	if verbose_result:
 		return_status += ": " + address_utf8
-		
+
 	# add or modify permitted-senders group
-	
+
 	cn = '%s' % uuid.uuid4()
 	dn = "cn=%s,%s" % (cn, env.LDAP_PERMITTED_SENDERS_BASE)
 	attrs = {
@@ -1261,22 +1258,22 @@ def add_mail_alias(address_utf8, description, forwards_to, permitted_senders, en
 		remove_mail_domain(env, domain_idna, validate=False)
 	elif count_vfwd > 0:
 		domain_added = add_mail_domain(env,	domain_idna, validate=False)
-	
+
 	if domain_added:
 		results = add_required_aliases(env,	conn, domain_idna)
 		for result in results:
 			if isinstance(result, tuple):
 				# error occurred
 				return result
-			elif result != '':
+			elif result:
 				return_status += "\n" + result
-	
+
 	if do_kick and domain_added:
 		# Update things in case any new domains are added.
 		return kick(env, return_status)
 	else:
 		return return_status
-	
+
 
 def remove_mail_alias(address_utf8, env, do_kick=True, auto=None, ignore_if_not_exists=False, verbose_result=False):
 	# Remove an alias group and it's associated permitted senders
@@ -1292,7 +1289,7 @@ def remove_mail_alias(address_utf8, env, do_kick=True, auto=None, ignore_if_not_
 	# http-status).
 	#
 	# If successful, the string "OK" is returned.
-	
+
 	# convert Unicode domain to IDNA
 	address = sanitize_idn_email_address(address_utf8)
 
@@ -1322,18 +1319,19 @@ def remove_mail_alias(address_utf8, env, do_kick=True, auto=None, ignore_if_not_
 			if isinstance(result, tuple):
 				# error occurred
 				return result
-			elif result != '':
+			elif result:
 				return_status += "\n" + result
 
 	if do_kick and domain_removed:
 		# Update things in case any domains are removed.
 		return kick(env, return_status)
 	else:
-		return return_status
+		None
+
 
 def add_auto_aliases(aliases, env):
 	conn, c = open_database(env, with_connection=True)
-	c.execute("DELETE FROM auto_aliases");
+	c.execute("DELETE FROM auto_aliases")
 	for source, destination in aliases.items():
 		c.execute("INSERT INTO auto_aliases (source, destination) VALUES (?, ?)", (source, destination))
 	conn.commit()
@@ -1344,7 +1342,7 @@ def get_system_administrator(env):
 # def get_required_aliases(env):
 # 	# These are the aliases that must exist.
 # 	# Returns a set of email addresses.
-	
+
 # 	aliases = set()
 
 # 	# The system administrator alias is required.
@@ -1396,9 +1394,11 @@ def kick(env, mail_result=None):
 def validate_password(pw):
 	# validate password
 	if pw.strip() == "":
-		raise ValueError("No password provided.")
+		msg = "No password provided."
+		raise ValueError(msg)
 	if len(pw) < 8:
-		raise ValueError("Passwords must be at least eight characters.")
+		msg = "Passwords must be at least eight characters."
+		raise ValueError(msg)
 
 if __name__ == "__main__":
 	import sys

management/mfa.py

@@ -22,8 +22,8 @@ def strip_order_prefix(rec, attributes):
 	sorted in the record making the prefix superfluous.
 
 	For example, the function will change:
-       totpSecret: {0}secret1 
+       totpSecret: {0}secret1
-       totpSecret: {1}secret2 
+       totpSecret: {1}secret2
     to:
 	   totpSecret: secret1
        totpSecret: secret2
@@ -32,16 +32,16 @@ def strip_order_prefix(rec, attributes):
 	'''
 	for attr in attributes:
 		# ignore attribute that doesn't exist
-		if not attr in rec: continue		
+		if not attr in rec: continue
 		# ..as well as None values and empty list
 		if not rec[attr]: continue
-		
+
 		newvals = []
 		for val in rec[attr]:
 			i = val.find('}')
 			newvals.append(val[i+1:])
 		rec[attr] = newvals
-		
+
 def get_mfa_user(email, env, conn=None):
 	'''get the ldap record for the user along with all MFA-related
 	attributes
@@ -49,7 +49,7 @@ def get_mfa_user(email, env, conn=None):
 	'''
 	user = find_mail_user(env, email, ['objectClass','totpSecret','totpMruToken','totpMruTokenTime','totpLabel'], conn)
 	if not user:
-		raise ValueError("User does not exist.")	
+		raise ValueError("User does not exist.")
 	strip_order_prefix(user, ['totpSecret','totpMruToken','totpMruTokenTime','totpLabel'])
 	return user
 
@@ -104,7 +104,8 @@ multiple mfa schemes enabled of the same type.
 	if type == "totp":
 		mfa_totp.enable(user, secret, token, label, env)
 	else:
-		raise ValueError("Invalid MFA type.")
+		msg = "Invalid MFA type."
+		raise ValueError(msg)
 
 def disable_mfa(email, mfa_id, env):
 	'''disable a specific MFA scheme. `mfa_id` identifies the specific
@@ -121,7 +122,7 @@ def disable_mfa(email, mfa_id, env):
 		return mfa_totp.disable(user, mfa_id, env)
 	else:
 		return False
-			
+
 def validate_auth_mfa(email, request, env):
 	# Validates that a login request satisfies any MFA modes
 	# that have been enabled for the user's account. Returns

management/mfa_totp.py

@@ -42,7 +42,7 @@ def time_ns():
 		return time.time_ns()
 	else:
 		return int(time.time() * 1000000000)
-	
+
 def get_state(user):
 	state_list = []
 
@@ -62,7 +62,8 @@ def enable(user, secret, token, label, env):
 	# Sanity check with the provide current token.
 	totp = pyotp.TOTP(secret)
 	if not totp.verify(token, valid_window=1):
-		raise ValueError("Invalid token.")
+		msg = "Invalid token."
+		raise ValueError(msg)
 
 	mods = {
 		"totpSecret": user['totpSecret'].copy() + [secret],
@@ -72,7 +73,7 @@ def enable(user, secret, token, label, env):
 	}
 	if 'totpUser' not in user['objectClass']:
 		 mods['objectClass'] = user['objectClass'].copy() + ['totpUser']
-	
+
 	conn = open_database(env)
 	conn.modify_record(user, mods)
 
@@ -107,13 +108,13 @@ def disable(user, id, env):
 			"totpSecret": None,
 			"totpLabel": None
 		}
-		mods["objectClass"].remove("totpUser")	
+		mods["objectClass"].remove("totpUser")
 		open_database(env).modify_record(user, mods)
 		return True
 
 	else:
 		# Disable totp at the index specified
-		idx = index_from_id(user, id)	
+		idx = index_from_id(user, id)
 		if idx<0 or idx>=len(user['totpSecret']):
 			return False
 		mods = {

management/ssl_certificates.py

@@ -13,7 +13,8 @@
 import os, os.path, re, shutil, subprocess, tempfile
 
 from utils import shell, safe_domain_name, sort_domains
-import idna
+import functools
+import operator
 
 # SELECTING SSL CERTIFICATES FOR USE IN WEB
 
@@ -92,9 +93,8 @@ def get_ssl_certificates(env):
 		for domain in cert_domains:
 			# The primary hostname can only use a certificate mapped
 			# to the system private key.
-			if domain == env['PRIMARY_HOSTNAME']:
+			if domain == env['PRIMARY_HOSTNAME'] and cert["private_key"]["filename"] != os.path.join(env['STORAGE_ROOT'], 'ssl', 'ssl_private_key.pem'):
-				if cert["private_key"]["filename"] != os.path.join(env['STORAGE_ROOT'], 'ssl', 'ssl_private_key.pem'):
+				continue
-					continue
 
 			domains.setdefault(domain, []).append(cert)
 
@@ -162,13 +162,12 @@ def get_domain_ssl_files(domain, ssl_certificates, env, allow_missing_cert=False
 			"certificate_object": load_pem(load_cert_chain(ssl_certificate)[0]),
 		}
 
-	if use_main_cert:
+	if use_main_cert and domain == env['PRIMARY_HOSTNAME']:
-		if domain == env['PRIMARY_HOSTNAME']:
+		# The primary domain must use the server certificate because
-			# The primary domain must use the server certificate because
+		# it is hard-coded in some service configuration files.
-			# it is hard-coded in some service configuration files.
+		return system_certificate
-			return system_certificate
 
-	wildcard_domain = re.sub("^[^\\.]+", "*", domain)
+	wildcard_domain = re.sub(r"^[^\.]+", "*", domain)
 	if domain in ssl_certificates:
 		return ssl_certificates[domain]
 	elif wildcard_domain in ssl_certificates:
@@ -224,7 +223,7 @@ def get_certificates_to_provision(env, limit_domains=None, show_valid_certs=True
 				if not value: continue # IPv6 is not configured
 				response = query_dns(domain, rtype)
 				if response != normalize_ip(value):
-					bad_dns.append("%s (%s)" % (response, rtype))
+					bad_dns.append(f"{response} ({rtype})")
 
 			if bad_dns:
 				domains_cant_provision[domain] = "The domain name does not resolve to this machine: " \
@@ -277,11 +276,11 @@ def provision_certificates(env, limit_domains):
 	# primary domain listed in each certificate.
 	from dns_update import get_dns_zones
 	certs = { }
-	for zone, zonefile in get_dns_zones(env):
+	for zone, _zonefile in get_dns_zones(env):
 		certs[zone] = [[]]
 	for domain in sort_domains(domains, env):
 		# Does the domain end with any domain we've seen so far.
-		for parent in certs.keys():
+		for parent in certs:
 			if domain.endswith("." + parent):
 				# Add this to the parent's list of domains.
 				# Start a new group if the list already has
@@ -298,7 +297,7 @@ def provision_certificates(env, limit_domains):
 
 	# Flatten to a list of lists of domains (from a mapping). Remove empty
 	# lists (zones with no domains that need certs).
-	certs = sum(certs.values(), [])
+	certs = functools.reduce(operator.iadd, certs.values(), [])
 	certs = [_ for _ in certs if len(_) > 0]
 
 	# Prepare to provision.
@@ -426,7 +425,7 @@ def create_csr(domain, ssl_key, country_code, env):
 				"openssl", "req", "-new",
 				"-key", ssl_key,
 				"-sha256",
-				"-subj", "/C=%s/CN=%s" % (country_code, domain)])
+				"-subj", f"/C={country_code}/CN={domain}"])
 
 def install_cert(domain, ssl_cert, ssl_chain, env, raw=False):
 	# Write the combined cert+chain to a temporary path and validate that it is OK.
@@ -462,8 +461,8 @@ def install_cert_copy_file(fn, env):
 	from cryptography.hazmat.primitives import hashes
 	from binascii import hexlify
 	cert = load_pem(load_cert_chain(fn)[0])
-	all_domains, cn = get_certificate_domains(cert)
+	_all_domains, cn = get_certificate_domains(cert)
-	path = "%s-%s-%s.pem" % (
+	path = "{}-{}-{}.pem".format(
 		safe_domain_name(cn), # common name, which should be filename safe because it is IDNA-encoded, but in case of a malformed cert make sure it's ok to use as a filename
 		cert.not_valid_after.date().isoformat().replace("-", ""), # expiration date
 		hexlify(cert.fingerprint(hashes.SHA256())).decode("ascii")[0:8], # fingerprint prefix
@@ -535,12 +534,12 @@ def check_certificate(domain, ssl_certificate, ssl_private_key, warn_if_expiring
 	# First check that the domain name is one of the names allowed by
 	# the certificate.
 	if domain is not None:
-		certificate_names, cert_primary_name = get_certificate_domains(cert)
+		certificate_names, _cert_primary_name = get_certificate_domains(cert)
 
 		# Check that the domain appears among the acceptable names, or a wildcard
 		# form of the domain name (which is a stricter check than the specs but
 		# should work in normal cases).
-		wildcard_domain = re.sub("^[^\\.]+", "*", domain)
+		wildcard_domain = re.sub(r"^[^\.]+", "*", domain)
 		if domain not in certificate_names and wildcard_domain not in certificate_names:
 			return ("The certificate is for the wrong domain name. It is for %s."
 				% ", ".join(sorted(certificate_names)), None)
@@ -551,7 +550,7 @@ def check_certificate(domain, ssl_certificate, ssl_private_key, warn_if_expiring
 			with open(ssl_private_key, 'rb') as f:
 				priv_key = load_pem(f.read())
 		except ValueError as e:
-			return ("The private key file %s is not a private key file: %s" % (ssl_private_key, str(e)), None)
+			return (f"The private key file {ssl_private_key} is not a private key file: {e!s}", None)
 
 		if not isinstance(priv_key, RSAPrivateKey):
 			return ("The private key file %s is not a private key file." % ssl_private_key, None)
@@ -582,7 +581,7 @@ def check_certificate(domain, ssl_certificate, ssl_private_key, warn_if_expiring
 	import datetime
 	now = datetime.datetime.utcnow()
 	if not(cert.not_valid_before <= now <= cert.not_valid_after):
-		return ("The certificate has expired or is not yet valid. It is valid from %s to %s." % (cert.not_valid_before, cert.not_valid_after), None)
+		return (f"The certificate has expired or is not yet valid. It is valid from {cert.not_valid_before} to {cert.not_valid_after}.", None)
 
 	# Next validate that the certificate is valid. This checks whether the certificate
 	# is self-signed, that the chain of trust makes sense, that it is signed by a CA
@@ -641,7 +640,8 @@ def load_cert_chain(pemfile):
 		pem = f.read() + b"\n" # ensure trailing newline
 		pemblocks = re.findall(re_pem, pem)
 		if len(pemblocks) == 0:
-			raise ValueError("File does not contain valid PEM data.")
+			msg = "File does not contain valid PEM data."
+			raise ValueError(msg)
 		return pemblocks
 
 def load_pem(pem):
@@ -652,9 +652,10 @@ def load_pem(pem):
 	from cryptography.hazmat.backends import default_backend
 	pem_type = re.match(b"-+BEGIN (.*?)-+[\r\n]", pem)
 	if pem_type is None:
-		raise ValueError("File is not a valid PEM-formatted file.")
+		msg = "File is not a valid PEM-formatted file."
+		raise ValueError(msg)
 	pem_type = pem_type.group(1)
-	if pem_type in (b"RSA PRIVATE KEY", b"PRIVATE KEY"):
+	if pem_type in {b"RSA PRIVATE KEY", b"PRIVATE KEY"}:
 		return serialization.load_pem_private_key(pem, password=None, backend=default_backend())
 	if pem_type == b"CERTIFICATE":
 		return load_pem_x509_certificate(pem, default_backend())

management/status_checks.py

@@ -14,11 +14,10 @@
 # TLS certificates have been signed, etc., and if not tells the user
 # what to do next.
 
-import sys, os, os.path, re, subprocess, datetime, multiprocessing.pool
+import sys, os, os.path, re, datetime, multiprocessing.pool
 import asyncio
 
 import dns.reversename, dns.resolver
-import dateutil.parser, dateutil.tz
 import idna
 import psutil
 import postfix_mta_sts_resolver.resolver
@@ -101,7 +100,7 @@ def run_services_checks(env, output, pool):
 	all_running = True
 	fatal = False
 	ret = pool.starmap(check_service, ((i, service, env) for i, service in enumerate(get_services())), chunksize=1)
-	for i, running, fatal2, output2 in sorted(ret):
+	for _i, running, fatal2, output2 in sorted(ret):
 		if output2 is None: continue # skip check (e.g. no port was set, e.g. no sshd)
 		all_running = all_running and running
 		fatal = fatal or fatal2
@@ -137,7 +136,7 @@ def check_service(i, service, env):
 		try:
 			s.connect((ip, service["port"]))
 			return True
-		except OSError as e:
+		except OSError:
 			# timed out or some other odd error
 			return False
 		finally:
@@ -164,18 +163,17 @@ def check_service(i, service, env):
 			output.print_error("%s is not running (port %d)." % (service['name'], service['port']))
 
 		# Why is nginx not running?
-		if not running and service["port"] in (80, 443):
+		if not running and service["port"] in {80, 443}:
 			output.print_line(shell('check_output', ['nginx', '-t'], capture_stderr=True, trap=True)[1].strip())
 
+	# Service should be running locally.
+	elif try_connect("127.0.0.1"):
+		running = True
 	else:
-		# Service should be running locally.
+		output.print_error("%s is not running (port %d)." % (service['name'], service['port']))
-		if try_connect("127.0.0.1"):
-			running = True
-		else:
-			output.print_error("%s is not running (port %d)." % (service['name'], service['port']))
 
 	# Flag if local DNS is not running.
-	if not running and service["port"] == 53 and service["public"] == False:
+	if not running and service["port"] == 53 and service["public"] is False:
 		fatal = True
 
 	return (i, running, fatal, output)
@@ -207,7 +205,7 @@ def check_ufw(env, output):
 		for service in get_services():
 			if service["public"] and not is_port_allowed(ufw, service["port"]):
 				not_allowed_ports += 1
-				output.print_error("Port %s (%s) should be allowed in the firewall, please re-run the setup." % (service["port"], service["name"]))
+				output.print_error("Port {} ({}) should be allowed in the firewall, please re-run the setup.".format(service["port"], service["name"]))
 
 		if not_allowed_ports == 0:
 			output.print_ok("Firewall is active.")
@@ -225,10 +223,10 @@ def check_ssh_password(env, output):
 	# the configuration file.
 	if not os.path.exists("/etc/ssh/sshd_config"):
 		return
-	with open("/etc/ssh/sshd_config", "r") as f:
+	with open("/etc/ssh/sshd_config", encoding="utf-8") as f:
 		sshd = f.read()
-	if re.search("\nPasswordAuthentication\s+yes", sshd) \
+	if re.search("\nPasswordAuthentication\\s+yes", sshd) \
-		or not re.search("\nPasswordAuthentication\s+no", sshd):
+		or not re.search("\nPasswordAuthentication\\s+no", sshd):
 		output.print_error("""The SSH server on this machine permits password-based login. A more secure
 			way to log in is using a public key. Add your SSH public key to $HOME/.ssh/authorized_keys, check
 			that you can log in without a password, set the option 'PasswordAuthentication no' in
@@ -249,7 +247,7 @@ def check_software_updates(env, output):
 	else:
 		output.print_error("There are %d software packages that can be updated." % len(pkgs))
 		for p in pkgs:
-			output.print_line("%s (%s)" % (p["package"], p["version"]))
+			output.print_line("{} ({})".format(p["package"], p["version"]))
 
 def check_system_aliases(env, output):
 	# Check that the administrator alias exists since that's where all
@@ -281,8 +279,7 @@ def check_free_disk_space(rounded_values, env, output):
 	except:
 		backup_cache_count = 0
 	if backup_cache_count > 1:
-		output.print_warning("The backup cache directory {} has more than one backup target cache. Consider clearing this directory to save disk space."
+		output.print_warning(f"The backup cache directory {backup_cache_path} has more than one backup target cache. Consider clearing this directory to save disk space.")
-			.format(backup_cache_path))
 
 def check_free_memory(rounded_values, env, output):
 	# Check free memory.
@@ -308,7 +305,7 @@ def run_network_checks(env, output):
 	# Stop if we cannot make an outbound connection on port 25. Many residential
 	# networks block outbound port 25 to prevent their network from sending spam.
 	# See if we can reach one of Google's MTAs with a 5-second timeout.
-	code, ret = shell("check_call", ["/bin/nc", "-z", "-w5", "aspmx.l.google.com", "25"], trap=True)
+	_code, ret = shell("check_call", ["/bin/nc", "-z", "-w5", "aspmx.l.google.com", "25"], trap=True)
 	if ret == 0:
 		output.print_ok("Outbound mail (SMTP port 25) is not blocked.")
 	else:
@@ -321,18 +318,26 @@ def run_network_checks(env, output):
 	# The user might have ended up on an IP address that was previously in use
 	# by a spammer, or the user may be deploying on a residential network. We
 	# will not be able to reliably send mail in these cases.
+
+	# See https://www.spamhaus.org/news/article/807/using-our-public-mirrors-check-your-return-codes-now. for
+	# information on spamhaus return codes
 	rev_ip4 = ".".join(reversed(env['PUBLIC_IP'].split('.')))
 	zen = query_dns(rev_ip4+'.zen.spamhaus.org', 'A', nxdomain=None)
 	if zen is None:
 		output.print_ok("IP address is not blacklisted by zen.spamhaus.org.")
 	elif zen == "[timeout]":
-		output.print_warning("Connection to zen.spamhaus.org timed out. We could not determine whether your server's IP address is blacklisted. Please try again later.")
+		output.print_warning("Connection to zen.spamhaus.org timed out. Could not determine whether this box's IP address is blacklisted. Please try again later.")
 	elif zen == "[Not Set]":
-		output.print_warning("Could not connect to zen.spamhaus.org. We could not determine whether your server's IP address is blacklisted. Please try again later.")
+		output.print_warning("Could not connect to zen.spamhaus.org. Could not determine whether this box's IP address is blacklisted. Please try again later.")
+	elif zen == "127.255.255.252":
+		output.print_warning("Incorrect spamhaus query: %s. Could not determine whether this box's IP address is blacklisted." % (rev_ip4+'.zen.spamhaus.org'))
+	elif zen == "127.255.255.254":
+		output.print_warning("Mail-in-a-Box is configured to use a public DNS server. This is not supported by spamhaus. Could not determine whether this box's IP address is blacklisted.")
+	elif zen == "127.255.255.255":
+		output.print_warning("Too many queries have been performed on the spamhaus server. Could not determine whether this box's IP address is blacklisted.")
 	else:
-		output.print_error("""The IP address of this machine %s is listed in the Spamhaus Block List (code %s),
+		output.print_error("""The IP address of this machine {} is listed in the Spamhaus Block List (code {}),
-			which may prevent recipients from receiving your email. See http://www.spamhaus.org/query/ip/%s."""
+			which may prevent recipients from receiving your email. See http://www.spamhaus.org/query/ip/{}.""".format(env['PUBLIC_IP'], zen, env['PUBLIC_IP']))
-			% (env['PUBLIC_IP'], zen, env['PUBLIC_IP']))
 
 def run_domain_checks(rounded_time, env, output, pool, domains_to_check=None):
 	# Get the list of domains we handle mail for.
@@ -353,7 +358,7 @@ def run_domain_checks(rounded_time, env, output, pool, domains_to_check=None):
 	domains_to_check = [
 		d for d in domains_to_check
 		if not (
-		   d.split(".", 1)[0] in ("www", "autoconfig", "autodiscover", "mta-sts")
+		   d.split(".", 1)[0] in {"www", "autoconfig", "autodiscover", "mta-sts"}
 		   and len(d.split(".", 1)) == 2
 		   and d.split(".", 1)[1] in domains_to_check
 		)
@@ -435,10 +440,9 @@ def check_primary_hostname_dns(domain, env, output, dns_domains, dns_zonefiles):
 	# If a DS record is set on the zone containing this domain, check DNSSEC now.
 	has_dnssec = False
 	for zone in dns_domains:
-		if zone == domain or domain.endswith("." + zone):
+		if (zone == domain or domain.endswith("." + zone)) and query_dns(zone, "DS", nxdomain=None) is not None:
-			if query_dns(zone, "DS", nxdomain=None) is not None:
+			has_dnssec = True
-				has_dnssec = True
+			check_dnssec(zone, env, output, dns_zonefiles, is_checking_primary=True)
-				check_dnssec(zone, env, output, dns_zonefiles, is_checking_primary=True)
 
 	ip = query_dns(domain, "A")
 	ns_ips = query_dns("ns1." + domain, "A") + '/' + query_dns("ns2." + domain, "A")
@@ -450,44 +454,41 @@ def check_primary_hostname_dns(domain, env, output, dns_domains, dns_zonefiles):
 	# the nameserver, are reporting the right info --- but if the glue is incorrect this
 	# will probably fail.
 	if ns_ips == env['PUBLIC_IP'] + '/' + env['PUBLIC_IP']:
-		output.print_ok("Nameserver glue records are correct at registrar. [ns1/ns2.%s ↦ %s]" % (env['PRIMARY_HOSTNAME'], env['PUBLIC_IP']))
+		output.print_ok("Nameserver glue records are correct at registrar. [ns1/ns2.{} ↦ {}]".format(env['PRIMARY_HOSTNAME'], env['PUBLIC_IP']))
 
 	elif ip == env['PUBLIC_IP']:
 		# The NS records are not what we expect, but the domain resolves correctly, so
 		# the user may have set up external DNS. List this discrepancy as a warning.
-		output.print_warning("""Nameserver glue records (ns1.%s and ns2.%s) should be configured at your domain name
+		output.print_warning("""Nameserver glue records (ns1.{} and ns2.{}) should be configured at your domain name
-			registrar as having the IP address of this box (%s). They currently report addresses of %s. If you have set up External DNS, this may be OK."""
+			registrar as having the IP address of this box ({}). They currently report addresses of {}. If you have set up External DNS, this may be OK.""".format(env['PRIMARY_HOSTNAME'], env['PRIMARY_HOSTNAME'], env['PUBLIC_IP'], ns_ips))
-			% (env['PRIMARY_HOSTNAME'], env['PRIMARY_HOSTNAME'], env['PUBLIC_IP'], ns_ips))
 
 	else:
-		output.print_error("""Nameserver glue records are incorrect. The ns1.%s and ns2.%s nameservers must be configured at your domain name
+		output.print_error("""Nameserver glue records are incorrect. The ns1.{} and ns2.{} nameservers must be configured at your domain name
-			registrar as having the IP address %s. They currently report addresses of %s. It may take several hours for
+			registrar as having the IP address {}. They currently report addresses of {}. It may take several hours for
-			public DNS to update after a change."""
+			public DNS to update after a change.""".format(env['PRIMARY_HOSTNAME'], env['PRIMARY_HOSTNAME'], env['PUBLIC_IP'], ns_ips))
-			% (env['PRIMARY_HOSTNAME'], env['PRIMARY_HOSTNAME'], env['PUBLIC_IP'], ns_ips))
 
 	# Check that PRIMARY_HOSTNAME resolves to PUBLIC_IP[V6] in public DNS.
 	ipv6 = query_dns(domain, "AAAA") if env.get("PUBLIC_IPV6") else None
 	if ip == env['PUBLIC_IP'] and not (ipv6 and env['PUBLIC_IPV6'] and ipv6 != normalize_ip(env['PUBLIC_IPV6'])):
-		output.print_ok("Domain resolves to box's IP address. [%s ↦ %s]" % (env['PRIMARY_HOSTNAME'], my_ips))
+		output.print_ok("Domain resolves to box's IP address. [{} ↦ {}]".format(env['PRIMARY_HOSTNAME'], my_ips))
 	else:
-		output.print_error("""This domain must resolve to your box's IP address (%s) in public DNS but it currently resolves
+		output.print_error("""This domain must resolve to this box's IP address ({}) in public DNS but it currently resolves
-			to %s. It may take several hours for public DNS to update after a change. This problem may result from other
+			to {}. It may take several hours for public DNS to update after a change. This problem may result from other
-			issues listed above."""
+			issues listed above.""".format(my_ips, ip + ((" / " + ipv6) if ipv6 is not None else "")))
-			% (my_ips, ip + ((" / " + ipv6) if ipv6 is not None else "")))
 
 
 	# Check reverse DNS matches the PRIMARY_HOSTNAME. Note that it might not be
 	# a DNS zone if it is a subdomain of another domain we have a zone for.
 	existing_rdns_v4 = query_dns(dns.reversename.from_address(env['PUBLIC_IP']), "PTR")
 	existing_rdns_v6 = query_dns(dns.reversename.from_address(env['PUBLIC_IPV6']), "PTR") if env.get("PUBLIC_IPV6") else None
-	if existing_rdns_v4 == domain and existing_rdns_v6 in (None, domain):
+	if existing_rdns_v4 == domain and existing_rdns_v6 in {None, domain}:
-		output.print_ok("Reverse DNS is set correctly at ISP. [%s ↦ %s]" % (my_ips, env['PRIMARY_HOSTNAME']))
+		output.print_ok("Reverse DNS is set correctly at ISP. [{} ↦ {}]".format(my_ips, env['PRIMARY_HOSTNAME']))
 	elif existing_rdns_v4 == existing_rdns_v6 or existing_rdns_v6 is None:
-		output.print_error("""Your box's reverse DNS is currently %s, but it should be %s. Your ISP or cloud provider will have instructions
+		output.print_error(f"""This box's reverse DNS is currently {existing_rdns_v4}, but it should be {domain}. Your ISP or cloud provider will have instructions
-			on setting up reverse DNS for your box.""" % (existing_rdns_v4, domain) )
+			on setting up reverse DNS for this box.""" )
 	else:
-		output.print_error("""Your box's reverse DNS is currently %s (IPv4) and %s (IPv6), but it should be %s. Your ISP or cloud provider will have instructions
+		output.print_error(f"""This box's reverse DNS is currently {existing_rdns_v4} (IPv4) and {existing_rdns_v6} (IPv6), but it should be {domain}. Your ISP or cloud provider will have instructions
-			on setting up reverse DNS for your box.""" % (existing_rdns_v4, existing_rdns_v6, domain) )
+			on setting up reverse DNS for this box.""" )
 
 	# Check the TLSA record.
 	tlsa_qname = "_25._tcp." + domain
@@ -501,9 +502,8 @@ def check_primary_hostname_dns(domain, env, output, dns_domains, dns_zonefiles):
 			# since TLSA shouldn't be used without DNSSEC.
 			output.print_warning("""The DANE TLSA record for incoming mail is not set. This is optional.""")
 	else:
-		output.print_error("""The DANE TLSA record for incoming mail (%s) is not correct. It is '%s' but it should be '%s'.
+		output.print_error(f"""The DANE TLSA record for incoming mail ({tlsa_qname}) is not correct. It is '{tlsa25}' but it should be '{tlsa25_expected}'.
-			It may take several hours for public DNS to update after a change."""
+			It may take several hours for public DNS to update after a change.""")
-                        % (tlsa_qname, tlsa25, tlsa25_expected))
 
 	# Check that the hostmaster@ email address exists.
 	check_alias_exists("Hostmaster contact address", "hostmaster@" + domain, env, output)
@@ -538,7 +538,7 @@ def check_dns_zone(domain, env, output, dns_zonefiles):
 	secondary_ns = custom_secondary_ns or ["ns2." + env['PRIMARY_HOSTNAME']]
 
 	existing_ns = query_dns(domain, "NS")
-	correct_ns = "; ".join(sorted(["ns1." + env['PRIMARY_HOSTNAME']] + secondary_ns))
+	correct_ns = "; ".join(sorted(["ns1." + env["PRIMARY_HOSTNAME"], *secondary_ns]))
 	ip = query_dns(domain, "A")
 
 	probably_external_dns = False
@@ -547,14 +547,12 @@ def check_dns_zone(domain, env, output, dns_zonefiles):
 		output.print_ok("Nameservers are set correctly at registrar. [%s]" % correct_ns)
 	elif ip == correct_ip:
 		# The domain resolves correctly, so maybe the user is using External DNS.
-		output.print_warning("""The nameservers set on this domain at your domain name registrar should be %s. They are currently %s.
+		output.print_warning(f"""The nameservers set on this domain at your domain name registrar should be {correct_ns}. They are currently {existing_ns}.
-			If you are using External DNS, this may be OK."""
+			If you are using External DNS, this may be OK.""" )
-				% (correct_ns, existing_ns) )
 		probably_external_dns = True
 	else:
-		output.print_error("""The nameservers set on this domain are incorrect. They are currently %s. Use your domain name registrar's
+		output.print_error(f"""The nameservers set on this domain are incorrect. They are currently {existing_ns}. Use your domain name registrar's
-			control panel to set the nameservers to %s."""
+			control panel to set the nameservers to {correct_ns}.""" )
-				% (existing_ns, correct_ns) )
 
 	# Check that each custom secondary nameserver resolves the IP address.
 
@@ -575,7 +573,7 @@ def check_dns_zone(domain, env, output, dns_zonefiles):
 			elif ip is None:
 				output.print_error("Secondary nameserver %s is not configured to resolve this domain." % ns)
 			else:
-				output.print_error("Secondary nameserver %s is not configured correctly. (It resolved this domain as %s. It should be %s.)" % (ns, ip, correct_ip))
+				output.print_error(f"Secondary nameserver {ns} is not configured correctly. (It resolved this domain as {ip}. It should be {correct_ip}.)")
 
 def check_dns_zone_suggestions(domain, env, output, dns_zonefiles, domains_with_a_records):
 	# Warn if a custom DNS record is preventing this or the automatic www redirect from
@@ -604,7 +602,7 @@ def check_dnssec(domain, env, output, dns_zonefiles, is_checking_primary=False):
 	expected_ds_records = { }
 	ds_file = '/etc/nsd/zones/' + dns_zonefiles[domain] + '.ds'
 	if not os.path.exists(ds_file): return # Domain is in our database but DNS has not yet been updated.
-	with open(ds_file) as f:
+	with open(ds_file, encoding="utf-8") as f:
 		for rr_ds in f:
 			rr_ds = rr_ds.rstrip()
 			ds_keytag, ds_alg, ds_digalg, ds_digest = rr_ds.split("\t")[4].split(" ")
@@ -613,7 +611,7 @@ def check_dnssec(domain, env, output, dns_zonefiles, is_checking_primary=False):
 			# record that we suggest using is for the KSK (and that's how the DS records were generated).
 			# We'll also give the nice name for the key algorithm.
 			dnssec_keys = load_env_vars_from_file(os.path.join(env['STORAGE_ROOT'], 'dns/dnssec/%s.conf' % alg_name_map[ds_alg]))
-			with open(os.path.join(env['STORAGE_ROOT'], 'dns/dnssec/' + dnssec_keys['KSK'] + '.key'), 'r') as f:
+			with open(os.path.join(env['STORAGE_ROOT'], 'dns/dnssec/' + dnssec_keys['KSK'] + '.key'), encoding="utf-8") as f:
 				dnsssec_pubkey = f.read().split("\t")[3].split(" ")[3]
 
 			expected_ds_records[ (ds_keytag, ds_alg, ds_digalg, ds_digest) ] = {
@@ -646,10 +644,10 @@ def check_dnssec(domain, env, output, dns_zonefiles, is_checking_primary=False):
 			#
 			# But it may not be preferred. Only algorithm 13 is preferred. Warn if any of the
 			# matched zones uses a different algorithm.
-			if set(r[1] for r in matched_ds) == { '13' } and set(r[2] for r in matched_ds) <= { '2', '4' }: # all are alg 13 and digest type 2 or 4
+			if {r[1] for r in matched_ds} == { '13' } and {r[2] for r in matched_ds} <= { '2', '4' }: # all are alg 13 and digest type 2 or 4
 				output.print_ok("DNSSEC 'DS' record is set correctly at registrar.")
 				return
-			elif len([r for r in matched_ds if r[1] == '13' and r[2] in ( '2', '4' )]) > 0: # some but not all are alg 13
+			elif len([r for r in matched_ds if r[1] == '13' and r[2] in { '2', '4' }]) > 0: # some but not all are alg 13
 				output.print_ok("DNSSEC 'DS' record is set correctly at registrar. (Records using algorithm other than ECDSAP256SHA256 and digest types other than SHA-256/384 should be removed.)")
 				return
 			else: # no record uses alg 13
@@ -681,8 +679,8 @@ def check_dnssec(domain, env, output, dns_zonefiles, is_checking_primary=False):
 		output.print_line("----------")
 		output.print_line("Key Tag: " + ds_suggestion['keytag'])
 		output.print_line("Key Flags: KSK / 257")
-		output.print_line("Algorithm: %s / %s" % (ds_suggestion['alg'], ds_suggestion['alg_name']))
+		output.print_line("Algorithm: {} / {}".format(ds_suggestion['alg'], ds_suggestion['alg_name']))
-		output.print_line("Digest Type: %s / %s" % (ds_suggestion['digalg'], ds_suggestion['digalg_name']))
+		output.print_line("Digest Type: {} / {}".format(ds_suggestion['digalg'], ds_suggestion['digalg_name']))
 		output.print_line("Digest: " + ds_suggestion['digest'])
 		output.print_line("Public Key: ")
 		output.print_line(ds_suggestion['pubkey'], monospace=True)
@@ -693,7 +691,7 @@ def check_dnssec(domain, env, output, dns_zonefiles, is_checking_primary=False):
 		output.print_line("")
 		output.print_line("The DS record is currently set to:")
 		for rr in sorted(ds):
-			output.print_line("Key Tag: {0}, Algorithm: {1}, Digest Type: {2}, Digest: {3}".format(*rr))
+			output.print_line("Key Tag: {}, Algorithm: {}, Digest Type: {}, Digest: {}".format(*rr))
 
 def check_mail_domain(domain, env, output):
 	# Check the MX record.
@@ -701,21 +699,19 @@ def check_mail_domain(domain, env, output):
 	recommended_mx = "10 " + env['PRIMARY_HOSTNAME']
 	mx = query_dns(domain, "MX", nxdomain=None)
 
-	if mx is None:
+	if mx is None or mx == "[timeout]":
-		mxhost = None
-	elif mx == "[timeout]":
 		mxhost = None
 	else:
 		# query_dns returns a semicolon-delimited list
 		# of priority-host pairs.
 		mxhost = mx.split('; ')[0].split(' ')[1]
 
-	if mxhost == None:
+	if mxhost is None:
 		# A missing MX record is okay on the primary hostname because
 		# the primary hostname's A record (the MX fallback) is... itself,
 		# which is what we want the MX to be.
 		if domain == env['PRIMARY_HOSTNAME']:
-			output.print_ok("Domain's email is directed to this domain. [%s has no MX record, which is ok]" % (domain,))
+			output.print_ok(f"Domain's email is directed to this domain. [{domain} has no MX record, which is ok]")
 
 		# And a missing MX record is okay on other domains if the A record
 		# matches the A record of the PRIMARY_HOSTNAME. Actually this will
@@ -723,17 +719,17 @@ def check_mail_domain(domain, env, output):
 		else:
 			domain_a = query_dns(domain, "A", nxdomain=None)
 			primary_a = query_dns(env['PRIMARY_HOSTNAME'], "A", nxdomain=None)
-			if domain_a != None and domain_a == primary_a:
+			if domain_a is not None and domain_a == primary_a:
-				output.print_ok("Domain's email is directed to this domain. [%s has no MX record but its A record is OK]" % (domain,))
+				output.print_ok(f"Domain's email is directed to this domain. [{domain} has no MX record but its A record is OK]")
 			else:
-				output.print_error("""This domain's DNS MX record is not set. It should be '%s'. Mail will not
+				output.print_error(f"""This domain's DNS MX record is not set. It should be '{recommended_mx}'. Mail will not
 					be delivered to this box. It may take several hours for public DNS to update after a
-					change. This problem may result from other issues listed here.""" % (recommended_mx,))
+					change. This problem may result from other issues listed here.""")
 
 	elif mxhost == env['PRIMARY_HOSTNAME']:
-		good_news = "Domain's email is directed to this domain. [%s ↦ %s]" % (domain, mx)
+		good_news = f"Domain's email is directed to this domain. [{domain} ↦ {mx}]"
 		if mx != recommended_mx:
-			good_news += "  This configuration is non-standard.  The recommended configuration is '%s'." % (recommended_mx,)
+			good_news += f"  This configuration is non-standard.  The recommended configuration is '{recommended_mx}'."
 		output.print_ok(good_news)
 
 		# Check MTA-STS policy.
@@ -744,14 +740,14 @@ def check_mail_domain(domain, env, output):
 			if policy[1].get("mx") == [env['PRIMARY_HOSTNAME']] and policy[1].get("mode") == "enforce": # policy[0] is the policyid
 				output.print_ok("MTA-STS policy is present.")
 			else:
-				output.print_error("MTA-STS policy is present but has unexpected settings. [{}]".format(policy[1]))
+				output.print_error(f"MTA-STS policy is present but has unexpected settings. [{policy[1]}]")
 		else:
-			output.print_error("MTA-STS policy is missing: {}".format(valid))
+			output.print_error(f"MTA-STS policy is missing: {valid}")
 
 	else:
-		output.print_error("""This domain's DNS MX record is incorrect. It is currently set to '%s' but should be '%s'. Mail will not
+		output.print_error(f"""This domain's DNS MX record is incorrect. It is currently set to '{mx}' but should be '{recommended_mx}'. Mail will not
 			be delivered to this box. It may take several hours for public DNS to update after a change. This problem may result from
-			other issues listed here.""" % (mx, recommended_mx))
+			other issues listed here.""")
 
 	# Check that the postmaster@ email address exists. Not required if the domain has a
 	# catch-all address or domain alias.
@@ -761,17 +757,26 @@ def check_mail_domain(domain, env, output):
 	# Stop if the domain is listed in the Spamhaus Domain Block List.
 	# The user might have chosen a domain that was previously in use by a spammer
 	# and will not be able to reliably send mail.
+
+	# See https://www.spamhaus.org/news/article/807/using-our-public-mirrors-check-your-return-codes-now. for
+	# information on spamhaus return codes
 	dbl = query_dns(domain+'.dbl.spamhaus.org', "A", nxdomain=None)
 	if dbl is None:
 		output.print_ok("Domain is not blacklisted by dbl.spamhaus.org.")
 	elif dbl == "[timeout]":
-		output.print_warning("Connection to dbl.spamhaus.org timed out. We could not determine whether the domain {} is blacklisted. Please try again later.".format(domain))
+		output.print_warning(f"Connection to dbl.spamhaus.org timed out. Could not determine whether the domain {domain} is blacklisted. Please try again later.")
 	elif dbl == "[Not Set]":
-		output.print_warning("Could not connect to dbl.spamhaus.org. We could not determine whether the domain {} is blacklisted. Please try again later.".format(domain))
+		output.print_warning(f"Could not connect to dbl.spamhaus.org. Could not determine whether the domain {domain} is blacklisted. Please try again later.")
+	elif dbl == "127.255.255.252":
+		output.print_warning("Incorrect spamhaus query: %s. Could not determine whether the domain %s is blacklisted." % (domain+'.dbl.spamhaus.org', domain))
+	elif dbl == "127.255.255.254":
+		output.print_warning("Mail-in-a-Box is configured to use a public DNS server. This is not supported by spamhaus. Could not determine whether the domain {} is blacklisted.".format(domain))
+	elif dbl == "127.255.255.255":
+		output.print_warning("Too many queries have been performed on the spamhaus server. Could not determine whether the domain {} is blacklisted.".format(domain))
 	else:
-		output.print_error("""This domain is listed in the Spamhaus Domain Block List (code %s),
+		output.print_error(f"""This domain is listed in the Spamhaus Domain Block List (code {dbl}),
 			which may prevent recipients from receiving your mail.
-			See http://www.spamhaus.org/dbl/ and http://www.spamhaus.org/query/domain/%s.""" % (dbl, domain))
+			See http://www.spamhaus.org/dbl/ and http://www.spamhaus.org/query/domain/{domain}.""")
 
 def check_web_domain(domain, rounded_time, ssl_certificates, env, output):
 	# See if the domain's A record resolves to our PUBLIC_IP. This is already checked
@@ -785,13 +790,13 @@ def check_web_domain(domain, rounded_time, ssl_certificates, env, output):
 			if value == normalize_ip(expected):
 				ok_values.append(value)
 			else:
-				output.print_error("""This domain should resolve to your box's IP address (%s %s) if you would like the box to serve
+				output.print_error(f"""This domain should resolve to this box's IP address ({rtype} {expected}) if you would like the box to serve
-					webmail or a website on this domain. The domain currently resolves to %s in public DNS. It may take several hours for
+					webmail or a website on this domain. The domain currently resolves to {value} in public DNS. It may take several hours for
-					public DNS to update after a change. This problem may result from other issues listed here.""" % (rtype, expected, value))
+					public DNS to update after a change. This problem may result from other issues listed here.""")
 				return
 
 		# If both A and AAAA are correct...
-		output.print_ok("Domain resolves to this box's IP address. [%s ↦ %s]" % (domain, '; '.join(ok_values)))
+		output.print_ok("Domain resolves to this box's IP address. [{} ↦ {}]".format(domain, '; '.join(ok_values)))
 
 
 	# We need a TLS certificate for PRIMARY_HOSTNAME because that's where the
@@ -838,7 +843,7 @@ def query_dns(qname, rtype, nxdomain='[Not Set]', at=None, as_list=False):
 	# be expressed in equivalent string forms. Canonicalize the form before
 	# returning them. The caller should normalize any IP addresses the result
 	# of this method is compared with.
-	if rtype in ("A", "AAAA"):
+	if rtype in {"A", "AAAA"}:
 		response = [normalize_ip(str(r)) for r in response]
 
 	if as_list:
@@ -854,7 +859,7 @@ def check_ssl_cert(domain, rounded_time, ssl_certificates, env, output):
 	# Check that TLS certificate is signed.
 
 	# Skip the check if the A record is not pointed here.
-	if query_dns(domain, "A", None) not in (env['PUBLIC_IP'], None): return
+	if query_dns(domain, "A", None) not in {env['PUBLIC_IP'], None}: return
 
 	# Where is the certificate file stored?
 	tls_cert = get_domain_ssl_files(domain, ssl_certificates, env, allow_missing_cert=True)
@@ -928,18 +933,16 @@ def what_version_is_this(env):
 	# Git may not be installed and Mail-in-a-Box may not have been cloned from github,
 	# so this function may raise all sorts of exceptions.
 	miab_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-	tag = shell("check_output", ["/usr/bin/git", "describe", "--always", "--abbrev=0"], env={"GIT_DIR": os.path.join(miab_dir, '.git')}).strip()
+	return shell("check_output", ["/usr/bin/git", "describe", "--always", "--abbrev=0"], env={"GIT_DIR": os.path.join(miab_dir, '.git')}).strip()
-	return tag
 
 def get_latest_miab_version():
 	# This pings https://mailinabox.email/setup.sh and extracts the tag named in
 	# the script to determine the current product version.
     from urllib.request import urlopen, HTTPError, URLError
-    from socket import timeout
 
     try:
         return re.search(b'TAG=(.*)', urlopen("https://mailinabox.email/setup.sh?ping=1", timeout=5).read()).group(1).decode("utf8")
-    except (HTTPError, URLError, timeout):
+    except (TimeoutError, HTTPError, URLError):
         return None
 
 def check_miab_version(env, output):
@@ -960,7 +963,7 @@ def check_miab_version(env, output):
 		elif latest_ver is None:
 			output.print_error("Latest Mail-in-a-Box LDAP version could not be determined. You are running version %s." % this_ver)
 		else:
-			output.print_error("A new version of Mail-in-a-Box LDAP is available. You are running version %s. The latest version is %s. For upgrade instructions, see https://mailinabox.email. "
+			output.print_error(f"A new version of Mail-in-a-Box LDAP is available. You are running version {this_ver}. The latest version is {latest_ver}. For upgrade instructions, see https://mailinabox.email. "
 				% (this_ver, latest_ver))
 
 def run_and_output_changes(env, pool):
@@ -976,8 +979,11 @@ def run_and_output_changes(env, pool):
 	# Load previously saved status checks.
 	cache_fn = "/var/cache/mailinabox/status_checks.json"
 	if os.path.exists(cache_fn):
-		with open(cache_fn, 'r') as f:
+		with open(cache_fn, encoding="utf-8") as f:
-			prev = json.load(f)
+			try:
+				prev = json.load(f)
+			except json.JSONDecodeError:
+				prev = []
 
 		# execute hooks
 		hook_data = {
@@ -1022,14 +1028,14 @@ def run_and_output_changes(env, pool):
 						out.add_heading(category + " -- Previously:")
 					elif op == "delete":
 						out.add_heading(category + " -- Removed")
-					if op in ("replace", "delete"):
+					if op in {"replace", "delete"}:
 						BufferedOutput(with_lines=prev_lines[i1:i2]).playback(out)
 
 					if op == "replace":
 						out.add_heading(category + " -- Currently:")
 					elif op == "insert":
 						out.add_heading(category + " -- Added")
-					if op in ("replace", "insert"):
+					if op in {"replace", "insert"}:
 						BufferedOutput(with_lines=cur_lines[j1:j2]).playback(out)
 
 		for category, prev_lines in prev_status.items():
@@ -1045,10 +1051,10 @@ def run_and_output_changes(env, pool):
 		'output':out
 	}
 	hooks.exec_hooks('status_checks', hook_data)
-	
+
 	# Store the current status checks output for next time.
 	os.makedirs(os.path.dirname(cache_fn), exist_ok=True)
-	with open(cache_fn, "w") as f:
+	with open(cache_fn, "w", encoding="utf-8") as f:
 		json.dump(cur.buf, f, indent=True)
 
 
@@ -1086,8 +1092,8 @@ class FileOutput:
 
 	def print_block(self, message, first_line="   "):
 		print(first_line, end='', file=self.buf)
-		message = re.sub("\n\s*", " ", message)
+		message = re.sub("\n\\s*", " ", message)
-		words = re.split("(\s+)", message)
+		words = re.split(r"(\s+)", message)
 		linelen = 0
 		for w in words:
 			if self.width and (linelen + len(w) > self.width-1-len(first_line)):
@@ -1126,7 +1132,7 @@ class ConsoleOutput(FileOutput):
 class BufferedOutput:
 	# Record all of the instance method calls so we can play them back later.
 	def __init__(self, with_lines=None):
-		self.buf = [] if not with_lines else with_lines
+		self.buf = with_lines if with_lines else []
 	def __getattr__(self, attr):
 		if attr not in ("add_heading", "print_ok", "print_error", "print_warning", "print_info", "print_block", "print_line"):
 			raise AttributeError

management/utils.py

@@ -30,13 +30,13 @@ def load_environment():
     # It won't exist exist until migration 13 completes...
     if os.path.exists(os.path.join(env["STORAGE_ROOT"],"ldap/miab_ldap.conf")):
         load_env_vars_from_file(os.path.join(env["STORAGE_ROOT"],"ldap/miab_ldap.conf"), strip_quotes=True, merge_env=env)
-    
+
     return env
 
 def load_env_vars_from_file(fn, strip_quotes=False, merge_env=None):
     # Load settings from a KEY=VALUE file.
     env = Environment()
-    with open(fn, 'r') as f:
+    with open(fn, encoding="utf-8") as f:
         for line in f:
             env.setdefault(*line.strip().split("=", 1))
     if strip_quotes:
@@ -46,25 +46,25 @@ def load_env_vars_from_file(fn, strip_quotes=False, merge_env=None):
     return env
 
 def save_environment(env):
-    with open("/etc/mailinabox.conf", "w") as f:
+    with open("/etc/mailinabox.conf", "w", encoding="utf-8") as f:
         for k, v in env.items():
-            f.write("%s=%s\n" % (k, v))
+            f.write(f"{k}={v}\n")
 
 # THE SETTINGS FILE AT STORAGE_ROOT/settings.yaml.
 
 def write_settings(config, env):
     import rtyaml
     fn = os.path.join(env['STORAGE_ROOT'], 'settings.yaml')
-    with open(fn, "w") as f:
+    with open(fn, "w", encoding="utf-8") as f:
         f.write(rtyaml.dump(config))
 
 def load_settings(env):
     import rtyaml
     fn = os.path.join(env['STORAGE_ROOT'], 'settings.yaml')
     try:
-        with open(fn, "r") as f:
+        with open(fn, encoding="utf-8") as f:
             config = rtyaml.load(f)
-        if not isinstance(config, dict): raise ValueError() # caught below
+        if not isinstance(config, dict): raise ValueError # caught below
         return config
     except:
         return { }
@@ -85,7 +85,7 @@ def sort_domains(domain_names, env):
     # from shortest to longest since zones are always shorter than their
     # subdomains.
     zones = { }
-    for domain in sorted(domain_names, key=lambda d : len(d)):
+    for domain in sorted(domain_names, key=len):
         for z in zones.values():
             if domain.endswith("." + z):
                 # We found a parent domain already in the list.
@@ -107,7 +107,7 @@ def sort_domains(domain_names, env):
       ))
 
     # Now sort the domain names that fall within each zone.
-    domain_names = sorted(domain_names,
+    return sorted(domain_names,
       key = lambda d : (
         # First by zone.
         zone_domains.index(zones[d]),
@@ -121,25 +121,26 @@ def sort_domains(domain_names, env):
         # Then in right-to-left lexicographic order of the .-separated parts of the name.
         list(reversed(d.split("."))),
       ))
-    
+
-    return domain_names
 
 def sort_email_addresses(email_addresses, env):
     email_addresses = set(email_addresses)
-    domains = set(email.split("@", 1)[1] for email in email_addresses if "@" in email)
+    domains = {email.split("@", 1)[1] for email in email_addresses if "@" in email}
     ret = []
     for domain in sort_domains(domains, env):
-        domain_emails = set(email for email in email_addresses if email.endswith("@" + domain))
+        domain_emails = {email for email in email_addresses if email.endswith("@" + domain)}
         ret.extend(sorted(domain_emails))
         email_addresses -= domain_emails
     ret.extend(sorted(email_addresses)) # whatever is left
     return ret
 
-def shell(method, cmd_args, env={}, capture_stderr=False, return_bytes=False, trap=False, input=None):
+def shell(method, cmd_args, env=None, capture_stderr=False, return_bytes=False, trap=False, input=None):
     # A safe way to execute processes.
     # Some processes like apt-get require being given a sane PATH.
     import subprocess
 
+    if env is None:
+        env = {}
     env.update({ "PATH": "/sbin:/bin:/usr/sbin:/usr/bin" })
     kwargs = {
         'env': env,
@@ -175,7 +176,7 @@ def du(path):
     # soft and hard links.
     total_size = 0
     seen = set()
-    for dirpath, dirnames, filenames in os.walk(path):
+    for dirpath, _dirnames, filenames in os.walk(path):
         for f in filenames:
             fp = os.path.join(dirpath, f)
             try:

management/web_update.py

@@ -34,16 +34,17 @@ def get_web_domains(env, include_www_redirects=True, include_auto=True, exclude_
 		# Add 'www.' subdomains that we want to provide default redirects
 		# to the main domain for. We'll add 'www.' to any DNS zones, i.e.
 		# the topmost of each domain we serve.
-		domains |= set('www.' + zone for zone, zonefile in get_dns_zones(env))
+		domains |= {'www.' + zone for zone, zonefile in get_dns_zones(env)}
 
 	if 'mail' in categories and include_auto:
 		# Add Autoconfiguration domains for domains that there are user accounts at:
 		# 'autoconfig.' for Mozilla Thunderbird auto setup.
-		# 'autodiscover.' for Activesync autodiscovery.
+		# 'autodiscover.' for ActiveSync autodiscovery (Z-Push).
-		domains |= set('autoconfig.' + maildomain for maildomain in get_mail_domains(env, users_only=True))
+		domains |= {'autoconfig.' + maildomain for maildomain in get_mail_domains(env, users_only=True)}
-		domains |= set('autodiscover.' + maildomain for maildomain in get_mail_domains(env, users_only=True))
+		domains |= {'autodiscover.' + maildomain for maildomain in get_mail_domains(env, users_only=True)}
+
 		# 'mta-sts.' for MTA-STS support for all domains that have email addresses.
-		domains |= set('mta-sts.' + maildomain for maildomain in get_mail_domains(env))
+		domains |= {'mta-sts.' + maildomain for maildomain in get_mail_domains(env)}
 
 	if exclude_dns_elsewhere:
 		# ...Unless the domain has an A/AAAA record that maps it to a different
@@ -56,15 +57,14 @@ def get_web_domains(env, include_www_redirects=True, include_auto=True, exclude_
 	domains.add(env['PRIMARY_HOSTNAME'])
 
 	# Sort the list so the nginx conf gets written in a stable order.
-	domains = sort_domains(domains, env)
+	return sort_domains(domains, env)
 
-	return domains
 
 def get_domains_with_a_records(env):
 	domains = set()
 	dns = get_custom_dns_config(env)
 	for domain, rtype, value in dns:
-		if rtype == "CNAME" or (rtype in ("A", "AAAA") and value not in ("local", env['PUBLIC_IP'])):
+		if rtype == "CNAME" or (rtype in {"A", "AAAA"} and value not in {"local", env['PUBLIC_IP']}):
 			domains.add(domain)
 	return domains
 
@@ -74,7 +74,7 @@ def get_web_domains_with_root_overrides(env):
 	root_overrides = { }
 	nginx_conf_custom_fn = os.path.join(env["STORAGE_ROOT"], "www/custom.yaml")
 	if os.path.exists(nginx_conf_custom_fn):
-		with open(nginx_conf_custom_fn, 'r') as f:
+		with open(nginx_conf_custom_fn, encoding='utf-8') as f:
 			custom_settings = rtyaml.load(f)
 		for domain, settings in custom_settings.items():
 			for type, value in [('redirect', settings.get('redirects', {}).get('/')),
@@ -89,7 +89,7 @@ def do_web_update(env):
 
 	# Helper for reading config files and templates
 	def read_conf(conf_fn):
-		with open(os.path.join(os.path.dirname(__file__), "../conf", conf_fn), "r") as f:
+		with open(os.path.join(os.path.dirname(__file__), "../conf", conf_fn), encoding='utf-8') as f:
 			return f.read()
 
 	# Build an nginx configuration file.
@@ -132,12 +132,12 @@ def do_web_update(env):
 	# Did the file change? If not, don't bother writing & restarting nginx.
 	nginx_conf_fn = "/etc/nginx/conf.d/local.conf"
 	if os.path.exists(nginx_conf_fn):
-		with open(nginx_conf_fn) as f:
+		with open(nginx_conf_fn, encoding='utf-8') as f:
 			if f.read() == nginx_conf:
 				return ""
 
 	# Save the file.
-	with open(nginx_conf_fn, "w") as f:
+	with open(nginx_conf_fn, "w", encoding='utf-8') as f:
 		f.write(nginx_conf)
 
 	# Kick nginx. Since this might be called from the web admin
@@ -169,13 +169,13 @@ def make_domain_config(domain, templates, ssl_certificates, env):
 		with open(filepath, 'rb') as f:
 			sha1.update(f.read())
 		return sha1.hexdigest()
-	nginx_conf_extra += "\t# ssl files sha1: %s / %s\n" % (hashfile(tls_cert["private-key"]), hashfile(tls_cert["certificate"]))
+	nginx_conf_extra += "\t# ssl files sha1: {} / {}\n".format(hashfile(tls_cert["private-key"]), hashfile(tls_cert["certificate"]))
 
 	# Add in any user customizations in YAML format.
 	hsts = "yes"
 	nginx_conf_custom_fn = os.path.join(env["STORAGE_ROOT"], "www/custom.yaml")
 	if os.path.exists(nginx_conf_custom_fn):
-		with open(nginx_conf_custom_fn, 'r') as f:
+		with open(nginx_conf_custom_fn, encoding='utf-8') as f:
 			yaml = rtyaml.load(f)
 		if domain in yaml:
 			yaml = yaml[domain]
@@ -215,16 +215,16 @@ def make_domain_config(domain, templates, ssl_certificates, env):
 				nginx_conf_extra += "\n\t\talias %s;" % alias
 				nginx_conf_extra += "\n\t}\n"
 			for path, url in yaml.get("redirects", {}).items():
-				nginx_conf_extra += "\trewrite %s %s permanent;\n" % (path, url)
+				nginx_conf_extra += f"\trewrite {path} {url} permanent;\n"
 
 			# override the HSTS directive type
 			hsts = yaml.get("hsts", hsts)
 
 	# Add the HSTS header.
 	if hsts == "yes":
-		nginx_conf_extra += "\tadd_header Strict-Transport-Security \"max-age=15768000\" always;\n"
+		nginx_conf_extra += '\tadd_header Strict-Transport-Security "max-age=15768000" always;\n'
 	elif hsts == "preload":
-		nginx_conf_extra += "\tadd_header Strict-Transport-Security \"max-age=15768000; includeSubDomains; preload\" always;\n"
+		nginx_conf_extra += '\tadd_header Strict-Transport-Security "max-age=15768000; includeSubDomains; preload" always;\n'
 
 	# Add in any user customizations in the includes/ folder.
 	nginx_conf_custom_include = os.path.join(env["STORAGE_ROOT"], "www", safe_domain_name(domain) + ".conf")
@@ -235,7 +235,7 @@ def make_domain_config(domain, templates, ssl_certificates, env):
 	# Combine the pieces. Iteratively place each template into the "# ADDITIONAL DIRECTIVES HERE" placeholder
 	# of the previous template.
 	nginx_conf = "# ADDITIONAL DIRECTIVES HERE\n"
-	for t in templates + [nginx_conf_extra]:
+	for t in [*templates, nginx_conf_extra]:
 		nginx_conf = re.sub("[ \t]*# ADDITIONAL DIRECTIVES HERE *\n", t, nginx_conf)
 
 	# Replace substitution strings in the template & return.
@@ -244,9 +244,8 @@ def make_domain_config(domain, templates, ssl_certificates, env):
 	nginx_conf = nginx_conf.replace("$ROOT", root)
 	nginx_conf = nginx_conf.replace("$SSL_KEY", tls_cert["private-key"])
 	nginx_conf = nginx_conf.replace("$SSL_CERTIFICATE", tls_cert["certificate"])
-	nginx_conf = nginx_conf.replace("$REDIRECT_DOMAIN", re.sub(r"^www\.", "", domain)) # for default www redirects to parent domain
+	return nginx_conf.replace("$REDIRECT_DOMAIN", re.sub(r"^www\.", "", domain)) # for default www redirects to parent domain
 
-	return nginx_conf
 
 def get_web_root(domain, env, test_exists=True):
 	# Try STORAGE_ROOT/web/domain_name if it exists, but fall back to STORAGE_ROOT/web/default.
@@ -285,4 +284,3 @@ def get_web_domains_info(env):
 		}
 		for domain in get_web_domains(env)
 	]
-

management/wsgi.py

@@ -8,9 +8,9 @@
 #####
 
 from daemon import app
-import auth, utils
+import utils
 
 app.logger.addHandler(utils.create_syslog_handler())
 
 if __name__ == "__main__":
-    app.run(port=10222)
+    app.run(port=10222)

setup/mail-postfix.sh

@@ -70,7 +70,7 @@ tools/editconf.py /etc/postfix/main.cf \
 	myhostname=$PRIMARY_HOSTNAME\
 	smtpd_banner="\$myhostname ESMTP Hi, I'm a Mail-in-a-Box (Ubuntu/Postfix; see https://mailinabox.email/)" \
 	mydestination=localhost
-    
+
 # Tweak some queue settings:
 # * Inform users when their e-mail delivery is delayed more than 3 hours (default is not to warn).
 # * Stop trying to send an undeliverable e-mail after 2 days (instead of 5), and for bounce messages just try for 1 day.
@@ -245,14 +245,15 @@ tools/editconf.py /etc/postfix/main.cf  -e lmtp_destination_recipient_limit=
 # * `reject_unlisted_recipient`: Although Postfix will reject mail to unknown recipients, it's nicer to reject such mail ahead of greylisting rather than after.
 # * `check_policy_service`: Apply greylisting using postgrey.
 #
+# Note the spamhaus rbl return codes are taken into account as adviced here: https://docs.spamhaus.com/datasets/docs/source/40-real-world-usage/PublicMirrors/MTAs/020-Postfix.html
 # Notes: #NODOC
 # permit_dnswl_client can pass through mail from whitelisted IP addresses, which would be good to put before greylisting #NODOC
 # so these IPs get mail delivered quickly. But when an IP is not listed in the permit_dnswl_client list (i.e. it is not #NODOC
 # whitelisted) then postfix does a DEFER_IF_REJECT, which results in all "unknown user" sorts of messages turning into #NODOC
 # "450 4.7.1 Client host rejected: Service unavailable". This is a retry code, so the mail doesn't properly bounce. #NODOC
 tools/editconf.py /etc/postfix/main.cf \
-	smtpd_sender_restrictions="reject_non_fqdn_sender,reject_unknown_sender_domain,reject_authenticated_sender_login_mismatch,reject_rhsbl_sender dbl.spamhaus.org" \
+	smtpd_sender_restrictions="reject_non_fqdn_sender,reject_unknown_sender_domain,reject_authenticated_sender_login_mismatch,reject_rhsbl_sender dbl.spamhaus.org=127.0.1.[2..99]" \
-	smtpd_recipient_restrictions=permit_sasl_authenticated,permit_mynetworks,"reject_rbl_client zen.spamhaus.org",reject_unlisted_recipient,"check_policy_service unix:private/policy-spf","check_policy_service inet:127.0.0.1:10023"
+	smtpd_recipient_restrictions="permit_sasl_authenticated,permit_mynetworks,reject_rbl_client zen.spamhaus.org=127.0.0.[2..11],reject_unlisted_recipient,check_policy_service unix:private/policy-spf,check_policy_service inet:127.0.0.1:10023"
 
 # Postfix connects to Postgrey on the 127.0.0.1 interface specifically. Ensure that
 # Postgrey listens on the same interface (and not IPv6, for instance).

setup/management.sh

@@ -75,18 +75,18 @@ rm -rf $assets_dir
 mkdir -p $assets_dir
 
 # jQuery CDN URL
-jquery_version=2.1.4
+jquery_version=2.2.4
 jquery_url=https://code.jquery.com
 
 # Get jQuery
-wget_verify $jquery_url/jquery-$jquery_version.min.js 43dc554608df885a59ddeece1598c6ace434d747 $assets_dir/jquery.min.js
+wget_verify $jquery_url/jquery-$jquery_version.min.js 69bb69e25ca7d5ef0935317584e6153f3fd9a88c $assets_dir/jquery.min.js
 
 # Bootstrap CDN URL
-bootstrap_version=3.3.7
+bootstrap_version=3.4.1
 bootstrap_url=https://github.com/twbs/bootstrap/releases/download/v$bootstrap_version/bootstrap-$bootstrap_version-dist.zip
 
 # Get Bootstrap
-wget_verify $bootstrap_url e6b1000b94e835ffd37f4c6dcbdad43f4b48a02a /tmp/bootstrap.zip
+wget_verify $bootstrap_url 0bb64c67c2552014d48ab4db81c2e8c01781f580 /tmp/bootstrap.zip
 unzip -q /tmp/bootstrap.zip -d $assets_dir
 mv $assets_dir/bootstrap-$bootstrap_version-dist $assets_dir/bootstrap
 rm -f /tmp/bootstrap.zip

setup/migrate.py

@@ -19,6 +19,7 @@ import sys, os, os.path, glob, re, shutil
 
 sys.path.insert(0, 'management')
 from utils import load_environment, load_env_vars_from_file, save_environment, shell
+import contextlib
 
 def migration_1(env):
 	# Re-arrange where we store SSL certificates. There was a typo also.
@@ -41,10 +42,8 @@ def migration_1(env):
 			move_file(sslfn, domain_name, file_type)
 
 	# Move the old domains directory if it is now empty.
-	try:
+	with contextlib.suppress(Exception):
 		os.rmdir(os.path.join( env["STORAGE_ROOT"], 'ssl/domains'))
-	except:
-		pass
 
 def migration_2(env):
 	# Delete the .dovecot_sieve script everywhere. This was formerly a copy of our spam -> Spam
@@ -178,7 +177,7 @@ def migration_12(env):
                         dropcmd = "DROP TABLE %s" % table
                         c.execute(dropcmd)
                     except:
-                        print("Failed to drop table", table, e)
+                        print("Failed to drop table", table)
             # Save.
             conn.commit()
             conn.close()
@@ -215,7 +214,7 @@ def migration_miabldap_1(env):
 	#   maildrop: [email]
 	#   userPassword: [password]
 	#   mailaccess: [privilege]   # multi-valued
-	# 
+	#
 	# aliases table:
 	# for each row create an ldap entry of the form:
 	#   dn: cn=[uuid],ou=aliases,ou=Users,dc=mailinabox
@@ -229,15 +228,15 @@ def migration_miabldap_1(env):
 	#   objectClass: mailGroup
 	#   mail: [source]
 	#   member: [user-dn]         # multi-valued
-	
+
 	print("Migrating users and aliases from sqlite to ldap")
-	
+
 	# Get the ldap server up and running
 	shell("check_call", ["setup/ldap.sh", "-v"])
-	
+
 	import sqlite3, ldap3
 	import migration_13 as m13
-				
+
 	# 2. get ldap site details (miab_ldap.conf was created by ldap.sh)
 	ldapvars = load_env_vars_from_file(os.path.join(env["STORAGE_ROOT"], "ldap/miab_ldap.conf"), strip_quotes=True)
 	ldap_base = ldapvars.LDAP_BASE
@@ -253,7 +252,7 @@ def migration_miabldap_1(env):
 	conn = sqlite3.connect(os.path.join(env["STORAGE_ROOT"], "mail/users.sqlite"))
 	ldap = ldap3.Connection('127.0.0.1', ldap_admin_dn, ldap_admin_pass, raise_exceptions=True)
 	ldap.bind()
-	
+
 	# 4. perform the migration
 	users=m13.create_users(env, conn, ldap, ldap_base, ldap_users_base, ldap_domains_base)
 	aliases=m13.create_aliases(env, conn, ldap, ldap_aliases_base)
@@ -301,7 +300,7 @@ def migration_miabldap_2(env):
 		"-LLL",
 		"olcObjectClasses"
 	])
-	
+
 	if "rfc822MailMember" in ret:
 		def ldif_change_fn(ldif):
 			return ldif.replace("rfc822MailMember: ", "mailMember: ")
@@ -327,10 +326,10 @@ def migration_miabldap_2(env):
 
 	print("Ensure all required aliases are created")
 	m14.ensure_required_aliases(env, ldapvars, ldap)
-	
+
 	ldap.unbind()
 
-	
+
 def get_current_migration():
 	ver = 0
 	while True:
@@ -350,8 +349,8 @@ def run_migrations():
 	migration_id_file = os.path.join(env['STORAGE_ROOT'], 'mailinabox.version')
 	migration_id = None
 	if os.path.exists(migration_id_file):
-		with open(migration_id_file) as f:
+		with open(migration_id_file, encoding='utf-8') as f:
-			migration_id = f.read().strip();
+			migration_id = f.read().strip()
 
 	if migration_id is None:
 		# Load the legacy location of the migration ID. We'll drop support
@@ -360,7 +359,7 @@ def run_migrations():
 
 	if migration_id is None:
 		print()
-		print("%s file doesn't exists. Skipping migration..." % (migration_id_file,))
+		print(f"{migration_id_file} file doesn't exists. Skipping migration...")
 		return
 
 	ourver = int(migration_id)
@@ -391,7 +390,7 @@ def run_migrations():
 
 		# Write out our current version now. Do this sooner rather than later
 		# in case of any problems.
-		with open(migration_id_file, "w") as f:
+		with open(migration_id_file, "w", encoding='utf-8') as f:
 			f.write(str(ourver) + "\n")
 
 		# Delete the legacy location of this field.
@@ -422,7 +421,7 @@ def run_miabldap_migrations():
 			print()
 			print("%s file doesn't exists. Skipping migration..." % (migration_id_file,))
 			return
-	
+
 	ourver = int(migration_id)
 
 	while True:
@@ -464,13 +463,12 @@ if __name__ == "__main__":
 	elif sys.argv[-1] == "--migrate":
 		# Perform migrations.
 		env = load_environment()
-		
+
 		# if miab-ldap already installed, only run miab-ldap migrations
 		if 'LDAP_USERS_BASE' in env:
 			run_miabldap_migrations()
-			
+
 		# otherwise, run both
 		else:
 			run_migrations()
 			run_miabldap_migrations()
-

setup/nextcloud.sh

@@ -38,31 +38,48 @@ echo "Installing Nextcloud (contacts/calendar)..."
 #   we automatically install intermediate versions as needed.
 # * The hash is the SHA1 hash of the ZIP package, which you can find by just running this script and
 #   copying it from the error message when it doesn't match what is below.
-nextcloud_ver=25.0.7
+nextcloud_ver=26.0.12
-nextcloud_hash=a5a565c916355005c7b408dd41a1e53505e1a080
+nextcloud_hash=b55e9f51171c0a9b9ab3686cf5c8ad1a4292ca15
 
 # Nextcloud apps
 # --------------
-# * Find the most recent tag that is compatible with the Nextcloud version above by
+# * Find the most recent tag that is compatible with the Nextcloud version above by:
-#   consulting the <dependencies>...<nextcloud> node at:
+#   https://github.com/nextcloud-releases/contacts/tags
-#   https://github.com/nextcloud-releases/contacts/blob/main/appinfo/info.xml
+#   https://github.com/nextcloud-releases/calendar/tags
-#   https://github.com/nextcloud-releases/calendar/blob/main/appinfo/info.xml
+#   https://github.com/nextcloud/user_external/tags
-#   https://github.com/nextcloud/user_external/blob/master/appinfo/info.xml
+#
-# * The hash is the SHA1 hash of the ZIP package, which you can find by just running this script and
+# * For these three packages, contact, calendar and user_external, the hash is the SHA1 hash of
-#   copying it from the error message when it doesn't match what is below.
+# the ZIP package, which you can find by just running this script and copying it from
-contacts_ver=5.3.0
+# the error message when it doesn't match what is below:
-contacts_hash=4b0a6666374e3b55cfd2ae9b72e1d458b87d4c8c
+
+# Always ensure the versions are supported, see https://apps.nextcloud.com/apps/contacts
+contacts_ver=5.5.3
+contacts_hash=799550f38e46764d90fa32ca1a6535dccd8316e5
 
 # Always ensure the versions are supported, see https://apps.nextcloud.com/apps/calendar
-calendar_ver=4.4.2
+calendar_ver=4.6.6
-calendar_hash=21a42e15806adc9b2618760ef94f1797ef399e2f
+calendar_hash=e34a71669a52d997e319d64a984dcd041389eb22
 
-# And https://apps.nextcloud.com/apps/user_external
+# Always ensure the versions are supported, see https://apps.nextcloud.com/apps/user_external
 user_external_ver=3.2.0
 user_external_hash=a494073dcdecbbbc79a9c77f72524ac9994d2eec
 
-# Clear prior packages and install dependencies from apt.
+# Developer advice (test plan)
+# ----------------------------
+# When upgrading above versions, how to test?
+#
+# 1. Enter your server instance (or on the Vagrant image)
+# 1. Git clone <your fork>
+# 2. Git checkout <your fork>
+# 3. Run `sudo ./setup/nextcloud.sh`
+# 4. Ensure the installation completes. If any hashes mismatch, correct them.
+# 5. Enter nextcloud web, run following tests:
+# 5.1 You still can create, edit and delete contacts
+# 5.2 You still can create, edit and delete calendar events
+# 5.3 You still can create, edit and delete users
+# 5.4 Go to Administration > Logs and ensure no new errors are shown
 
+# Clear prior packages and install dependencies from apt.
 apt-get purge -qq -y owncloud* # we used to use the package manager
 
 apt_install curl php${PHP_VER} php${PHP_VER}-fpm \
@@ -167,7 +184,7 @@ InstallNextcloud() {
 
 # Current Nextcloud Version, #1623
 # Checking /usr/local/lib/owncloud/version.php shows version of the Nextcloud application, not the DB
-# $STORAGE_ROOT/owncloud is kept together even during a backup.  It is better to rely on config.php than
+# $STORAGE_ROOT/owncloud is kept together even during a backup. It is better to rely on config.php than
 # version.php since the restore procedure can leave the system in a state where you have a newer Nextcloud
 # application version than the database.
 
@@ -225,6 +242,11 @@ if [ ! -d /usr/local/lib/owncloud/ ] || [[ ! ${CURRENT_NEXTCLOUD_VER} =~ ^$nextc
 			return 0
 		fi
 
+		# Hint: whenever you bump, remember this:
+		# - Run a server with the previous version
+		# - On a new if-else block, copy the versions/hashes from the previous version
+		# - Run sudo ./setup/start.sh on the new machine. Upon completion, test its basic functionalities.
+
 		if [[ ${CURRENT_NEXTCLOUD_VER} =~ ^20 ]]; then
 			InstallNextcloud 21.0.7 f5c7079c5b56ce1e301c6a27c0d975d608bb01c9 4.0.7 45e7cf4bfe99cd8d03625cf9e5a1bb2e90549136 3.0.4 d0284b68135777ec9ca713c307216165b294d0fe
 			CURRENT_NEXTCLOUD_VER="21.0.7"
@@ -241,6 +263,10 @@ if [ ! -d /usr/local/lib/owncloud/ ] || [[ ! ${CURRENT_NEXTCLOUD_VER} =~ ^$nextc
 			InstallNextcloud 24.0.12 7aa5d61632c1ccf4ca3ff00fb6b295d318c05599 4.1.0 697f6b4a664e928d72414ea2731cb2c9d1dc3077 3.2.2 ce4030ab57f523f33d5396c6a81396d440756f5f 3.0.0 0df781b261f55bbde73d8c92da3f99397000972f
 			CURRENT_NEXTCLOUD_VER="24.0.12"
 		fi
+        if [[ ${CURRENT_NEXTCLOUD_VER} =~ ^24 ]]; then
+			InstallNextcloud 25.0.7 a5a565c916355005c7b408dd41a1e53505e1a080 5.3.0 4b0a6666374e3b55cfd2ae9b72e1d458b87d4c8c 4.4.2 21a42e15806adc9b2618760ef94f1797ef399e2f 3.2.0 a494073dcdecbbbc79a9c77f72524ac9994d2eec
+			CURRENT_NEXTCLOUD_VER="25.0.7"
+		fi
 	fi
 
 	InstallNextcloud $nextcloud_ver $nextcloud_hash $contacts_ver $contacts_hash $calendar_ver $calendar_hash $user_external_ver $user_external_hash

setup/preflight.sh

@@ -1,3 +1,4 @@
+#!/bin/bash
 #####
 ##### This file is part of Mail-in-a-Box-LDAP which is released under the
 ##### terms of the GNU Affero General Public License as published by the
@@ -35,16 +36,16 @@ fi
 #
 # Skip the check if we appear to be running inside of Vagrant, because that's really just for testing.
 TOTAL_PHYSICAL_MEM=$(head -n 1 /proc/meminfo | awk '{print $2}')
-if [ $TOTAL_PHYSICAL_MEM -lt 490000 ]; then
+if [ "$TOTAL_PHYSICAL_MEM" -lt 490000 ]; then
 if [ ! -d /vagrant ]; then
-	TOTAL_PHYSICAL_MEM=$(expr \( \( $TOTAL_PHYSICAL_MEM \* 1024 \) / 1000 \) / 1000)
+	TOTAL_PHYSICAL_MEM=$(( TOTAL_PHYSICAL_MEM * 1024 / 1000 / 1000 ))
 	echo "Your Mail-in-a-Box needs more memory (RAM) to function properly."
 	echo "Please provision a machine with at least 512 MB, 1 GB recommended."
 	echo "This machine has $TOTAL_PHYSICAL_MEM MB memory."
 	exit
 fi
 fi
-if [ $TOTAL_PHYSICAL_MEM -lt 750000 ]; then
+if [ "$TOTAL_PHYSICAL_MEM" -lt 750000 ]; then
 	echo "WARNING: Your Mail-in-a-Box has less than 768 MB of memory."
 	echo "         It might run unreliably when under heavy load."
 fi

setup/webmail.sh

@@ -49,8 +49,8 @@ apt_install php${PHP_VER}-ldap
 #   https://github.com/mstilkerich/rcmcarddav/releases
 # The easiest way to get the package hashes is to run this script and get the hash from
 # the error message.
-VERSION=1.6.5
+VERSION=1.6.6
-HASH=326fcc206cddc00355e98d1e40fd0bcd9baec69f
+HASH=7705d2736890c49e7ae3ac75e3ae00ba56187056
 PERSISTENT_LOGIN_VERSION=version-5.3.0
 HTML5_NOTIFIER_VERSION=68d9ca194212e15b3c7225eb6085dbcf02fd13d7 # version 0.6.4+
 CARDDAV_VERSION=4.4.3
@@ -302,4 +302,3 @@ cat > /etc/cron.daily/mailinabox-roundcubemail << EOF
 cd $RCM_DIR && bin/cleandb.sh >/dev/null
 EOF
 chmod +x /etc/cron.daily/mailinabox-roundcubemail
-

tests/fail2ban.py

@@ -15,12 +15,12 @@
 # try to log in to.
 ######################################################################
 
-import sys, os, time, functools
+import sys, os, time
 
 # parse command line
 
 if len(sys.argv) != 4:
-	print("Usage: tests/fail2ban.py \"ssh user@hostname\" hostname owncloud_user")
+	print('Usage: tests/fail2ban.py "ssh user@hostname" hostname owncloud_user')
 	sys.exit(1)
 
 ssh_command, hostname, owncloud_user = sys.argv[1:4]
@@ -33,7 +33,6 @@ socket.setdefaulttimeout(10)
 class IsBlocked(Exception):
 	"""Tests raise this exception when it appears that a fail2ban
 	jail is in effect, i.e. on a connection refused error."""
-	pass
 
 def smtp_test():
 	import smtplib
@@ -42,13 +41,14 @@ def smtp_test():
 		server = smtplib.SMTP(hostname, 587)
 	except ConnectionRefusedError:
 		# looks like fail2ban worked
-		raise IsBlocked()
+		raise IsBlocked
 	server.starttls()
 	server.ehlo_or_helo_if_needed()
 
 	try:
 		server.login("fakeuser", "fakepassword")
-		raise Exception("authentication didn't fail")
+		msg = "authentication didn't fail"
+		raise Exception(msg)
 	except smtplib.SMTPAuthenticationError:
 		# athentication should fail
 		pass
@@ -66,11 +66,12 @@ def imap_test():
 		M = imaplib.IMAP4_SSL(hostname)
 	except ConnectionRefusedError:
 		# looks like fail2ban worked
-		raise IsBlocked()
+		raise IsBlocked
 
 	try:
 		M.login("fakeuser", "fakepassword")
-		raise Exception("authentication didn't fail")
+		msg = "authentication didn't fail"
+		raise Exception(msg)
 	except imaplib.IMAP4.error:
 		# authentication should fail
 		pass
@@ -84,17 +85,18 @@ def pop_test():
 		M = poplib.POP3_SSL(hostname)
 	except ConnectionRefusedError:
 		# looks like fail2ban worked
-		raise IsBlocked()
+		raise IsBlocked
 	try:
 		M.user('fakeuser')
 		try:
 			M.pass_('fakepassword')
-		except poplib.error_proto as e:
+		except poplib.error_proto:
 			# Authentication should fail.
 			M = None # don't .quit()
 			return
 		M.list()
-		raise Exception("authentication didn't fail")
+		msg = "authentication didn't fail"
+		raise Exception(msg)
 	finally:
 		if M:
 			M.quit()
@@ -108,11 +110,12 @@ def managesieve_test():
 		M = imaplib.IMAP4(hostname, 4190)
 	except ConnectionRefusedError:
 		# looks like fail2ban worked
-		raise IsBlocked()
+		raise IsBlocked
 
 	try:
 		M.login("fakeuser", "fakepassword")
-		raise Exception("authentication didn't fail")
+		msg = "authentication didn't fail"
+		raise Exception(msg)
 	except imaplib.IMAP4.error:
 		# authentication should fail
 		pass
@@ -138,17 +141,17 @@ def http_test(url, expected_status, postdata=None, qsargs=None, auth=None):
 			headers={'User-Agent': 'Mail-in-a-Box fail2ban tester'},
 			timeout=8,
 			verify=False) # don't bother with HTTPS validation, it may not be configured yet
-	except requests.exceptions.ConnectTimeout as e:
+	except requests.exceptions.ConnectTimeout:
-		raise IsBlocked()
+		raise IsBlocked
 	except requests.exceptions.ConnectionError as e:
 		if "Connection refused" in str(e):
-			raise IsBlocked()
+			raise IsBlocked
 		raise # some other unexpected condition
 
 	# return response status code
 	if r.status_code != expected_status:
 		r.raise_for_status() # anything but 200
-		raise IOError("Got unexpected status code %s." % r.status_code)
+		raise OSError("Got unexpected status code %s." % r.status_code)
 
 # define how to run a test
 
@@ -158,7 +161,7 @@ def restart_fail2ban_service(final=False):
 	if not final:
 		# Stop recidive jails during testing.
 		command += " && sudo fail2ban-client stop recidive"
-	os.system("%s \"%s\"" % (ssh_command, command))
+	os.system(f'{ssh_command} "{command}"')
 
 def testfunc_runner(i, testfunc, *args):
 	print(i+1, end=" ", flush=True)
@@ -172,7 +175,6 @@ def run_test(testfunc, args, count, within_seconds, parallel):
 	# run testfunc sequentially and still get to count requests within
 	# the required time. So we split the requests across threads.
 
-	import requests.exceptions
 	from multiprocessing import Pool
 
 	restart_fail2ban_service()
@@ -188,7 +190,7 @@ def run_test(testfunc, args, count, within_seconds, parallel):
 		# Distribute the requests across the pool.
 		asyncresults = []
 		for i in range(count):
-			ar = p.apply_async(testfunc_runner, [i, testfunc] + list(args))
+			ar = p.apply_async(testfunc_runner, [i, testfunc, *list(args)])
 			asyncresults.append(ar)
 
 		# Wait for all runs to finish.

tests/test_dns.py

@@ -16,7 +16,7 @@
 # where ipaddr is the IP address of your Mail-in-a-Box
 # and hostname is the domain name to check the DNS for.
 
-import sys, re, difflib
+import sys, re
 import dns.reversename, dns.resolver
 
 if len(sys.argv) < 3:
@@ -36,10 +36,10 @@ def test(server, description):
 		("ns2." + primary_hostname, "A", ipaddr),
 		("www." + hostname, "A", ipaddr),
 		(hostname, "MX", "10 " + primary_hostname + "."),
-		(hostname, "TXT", "\"v=spf1 mx -all\""),
+		(hostname, "TXT", '"v=spf1 mx -all"'),
-		("mail._domainkey." + hostname, "TXT", "\"v=DKIM1; k=rsa; s=email; \" \"p=__KEY__\""),
+		("mail._domainkey." + hostname, "TXT", '"v=DKIM1; k=rsa; s=email; " "p=__KEY__"'),
 		#("_adsp._domainkey." + hostname, "TXT", "\"dkim=all\""),
-		("_dmarc." + hostname, "TXT", "\"v=DMARC1; p=quarantine;\""),
+		("_dmarc." + hostname, "TXT", '"v=DMARC1; p=quarantine;"'),
 	]
 	return test2(tests, server, description)
 
@@ -68,7 +68,7 @@ def test2(tests, server, description):
 			response = ["[no value]"]
 		response = ";".join(str(r) for r in response)
 		response = re.sub(r"(\"p=).*(\")", r"\1__KEY__\2", response) # normalize DKIM key
-		response = response.replace("\"\" ", "") # normalize TXT records (DNSSEC signing inserts empty text string components)
+		response = response.replace('"" ', "") # normalize TXT records (DNSSEC signing inserts empty text string components)
 
 		# is it right?
 		if response == expected_answer:
@@ -107,7 +107,7 @@ else:
 		# And if that's OK, also check reverse DNS (the PTR record).
 		if not test_ptr("8.8.8.8", "Google Public DNS (Reverse DNS)"):
 			print ()
-			print ("The reverse DNS for %s is not correct. Consult your ISP for how to set the reverse DNS (also called the PTR record) for %s to %s." % (hostname, hostname, ipaddr))
+			print (f"The reverse DNS for {hostname} is not correct. Consult your ISP for how to set the reverse DNS (also called the PTR record) for {hostname} to {ipaddr}.")
 			sys.exit(1)
 		else:
 			print ("And the reverse DNS for the domain is correct.")

tests/test_mail.py

@@ -83,7 +83,7 @@ while argi<len(sys.argv):
 		argi+=2
 	else:
 		usage()
-		
+
 if not smtpd:
 	if len(sys.argv) - argi != 3: usage()
 	host, login, pw = sys.argv[argi:argi+3]
@@ -222,13 +222,12 @@ if delete_msg:
 		if not found:
 			print("Test message not present in the inbox yet...")
 			time.sleep(wait_cycle_sleep)
-		
+
 	M.close()
 	M.logout()
-	
+
 	if not found:
 		raise TimeoutError("Timeout waiting for message")
 
 if send_msg and delete_msg:
 	print("Test message sent & received successfully.")
-

tests/test_smtp_server.py

@@ -15,11 +15,11 @@ if len(sys.argv) < 3:
         sys.exit(1)
 
 host, toaddr, fromaddr = sys.argv[1:4]
-msg = """From: %s
+msg = f"""From: {fromaddr}
-To: %s
+To: {toaddr}
 Subject: SMTP server test
 
-This is a test message.""" % (fromaddr, toaddr)
+This is a test message."""
 
 server = smtplib.SMTP(host, 25)
 server.set_debuglevel(1)

tests/tls.py

@@ -97,14 +97,14 @@ def sslyze(opts, port, ok_ciphers):
 
 	try:
 		# Execute SSLyze.
-		out = subprocess.check_output([SSLYZE] + common_opts + opts + [connection_string])
+		out = subprocess.check_output([SSLYZE, *common_opts, *opts, connection_string])
 		out = out.decode("utf8")
 
 		# Trim output to make better for storing in git.
 		if "SCAN RESULTS FOR" not in out:
 			# Failed. Just output the error.
-			out = re.sub("[\w\W]*CHECKING HOST\(S\) AVAILABILITY\n\s*-+\n", "", out) # chop off header that shows the host we queried
+			out = re.sub("[\\w\\W]*CHECKING HOST\\(S\\) AVAILABILITY\n\\s*-+\n", "", out) # chop off header that shows the host we queried
-		out = re.sub("[\w\W]*SCAN RESULTS FOR.*\n\s*-+\n", "", out) # chop off header that shows the host we queried
+		out = re.sub("[\\w\\W]*SCAN RESULTS FOR.*\n\\s*-+\n", "", out) # chop off header that shows the host we queried
 		out = re.sub("SCAN COMPLETED IN .*", "", out)
 		out = out.rstrip(" \n-") + "\n"
 
@@ -114,8 +114,8 @@ def sslyze(opts, port, ok_ciphers):
 		# Pull out the accepted ciphers list for each SSL/TLS protocol
 		# version outputted.
 		accepted_ciphers = set()
-		for ciphers in re.findall(" Accepted:([\w\W]*?)\n *\n", out):
+		for ciphers in re.findall(" Accepted:([\\w\\W]*?)\n *\n", out):
-			accepted_ciphers |= set(re.findall("\n\s*(\S*)", ciphers))
+			accepted_ciphers |= set(re.findall("\n\\s*(\\S*)", ciphers))
 
 		# Compare to what Mozilla recommends, for a given modernness-level.
 		print("  Should Not Offer: " + (", ".join(sorted(accepted_ciphers-set(ok_ciphers))) or "(none -- good)"))
@@ -151,7 +151,7 @@ for cipher in csv.DictReader(io.StringIO(urllib.request.urlopen("https://raw.git
 client_compatibility = json.loads(urllib.request.urlopen("https://raw.githubusercontent.com/mail-in-a-box/user-agent-tls-capabilities/master/clients.json").read().decode("utf8"))
 cipher_clients = { }
 for client in client_compatibility:
-	if len(set(client['protocols']) & set(["TLS 1.0", "TLS 1.1", "TLS 1.2"])) == 0: continue # does not support TLS
+	if len(set(client['protocols']) & {"TLS 1.0", "TLS 1.1", "TLS 1.2"}) == 0: continue # does not support TLS
 	for cipher in client['ciphers']:
 		cipher_clients.setdefault(cipher_names.get(cipher), set()).add("/".join(x for x in [client['client']['name'], client['client']['version'], client['client']['platform']] if x))
 

tests/vagrant/Vagrantfile

@@ -25,7 +25,7 @@ export FEATURE_MUNIN=false
 export EHDD_KEYFILE=$HOME/keyfile
 echo -n "boo" >$EHDD_KEYFILE
 tests/system-setup/remote-nextcloud-docker.sh || exit 1
-tests/runner.sh remote-nextcloud ehdd default || exit 2
+tests/runner.sh -no-smtp-remote remote-nextcloud ehdd default || exit 2
 SH
     end
   end
@@ -38,25 +38,26 @@ cd /mailinabox
 export PRIMARY_HOSTNAME=qa2.abc.com
 export FEATURE_MUNIN=false
 tests/system-setup/remote-nextcloud-docker.sh upgrade --populate=basic || exit 1
-tests/runner.sh remote-nextcloud upgrade-basic default || exit 2
+tests/runner.sh -no-smtp-remote remote-nextcloud upgrade-basic default || exit 2
 SH
   end
 
 
   # upgrade-from-upstream
-  
+
   config.vm.define "upgrade-from-upstream" do |m1|
     m1.vm.provision :shell, :inline => <<-SH
 cd /mailinabox
 export PRIMARY_HOSTNAME=qa3.abc.com
-export UPSTREAM_TAG=main
+# TODO: change UPSTREAM_TAG to 'main' once upstream is installable
+export UPSTREAM_TAG=v67
 tests/system-setup/upgrade-from-upstream.sh --populate=basic --populate=totpuser || exit 1
-tests/runner.sh upgrade-basic upgrade-totpuser default || exit 2
+tests/runner.sh -no-smtp-remote upgrade-basic upgrade-totpuser default || exit 2
 SH
   end
 
   # upgrade
-  
+
   # this test is only needed when testing migrations from miabldap
   # to a newer miabldap with a migration step
   #
@@ -65,9 +66,11 @@ SH
   config.vm.define "upgrade" do |m1|
     m1.vm.provision :shell, :inline => <<-SH
 cd /mailinabox
+# TODO: remove DEB_PYTHON_INSTALL_LAYOUT once MIABLDAP_RELEASE_TAG >= v66 (see https://github.com/downtownallday/mailinabox-ldap/commit/371f5bc1b236de40a1ed5d9118140ee13fddf5dc)
+export DEB_PYTHON_INSTALL_LAYOUT='deb'
 export PRIMARY_HOSTNAME=upgrade.abc.com
 tests/system-setup/upgrade.sh --populate=basic --populate=totpuser || exit 1
-tests/runner.sh upgrade-basic upgrade-totpuser default || exit 2
+tests/runner.sh -no-smtp-remote upgrade-basic upgrade-totpuser default || exit 2
 SH
   end
 
@@ -96,6 +99,6 @@ setup/start.sh
 SH
     end
   end
-  
+
 
 end

tools/editconf.py

@@ -110,7 +110,7 @@ except:
 
 found = set()
 buf = ""
-with open(filename, "r") as f:
+with open(filename, encoding="utf-8") as f:
 	input_lines = list(f)
 cur_section = None
 
@@ -119,7 +119,7 @@ while len(input_lines) > 0:
 
 	# If this configuration file uses folded lines, append any folded lines
 	# into our input buffer.
-	if folded_lines and line[0] not in (comment_char, " ", ""):
+	if folded_lines and line[0] not in {comment_char, " ", ""}:
 		while len(input_lines) > 0 and input_lines[0][0] in " \t":
 			line += input_lines.pop(0)
 
@@ -147,9 +147,9 @@ while len(input_lines) > 0:
 		name, val = (settings[i].name, settings[i].val)
 		flags = re.S | (re.I if case_insensitive_names else 0)
 		m = re.match(
-			   "(\\s*)"
+			   r"(\s*)"
-			 + "(" + re.escape(comment_char) + "\\s*)?"
+			 + "(" + re.escape(comment_char) + r"\s*)?"
-			 + re.escape(name) + delimiter_re + "(.*?)\\s*$",
+			 + re.escape(name) + delimiter_re + r"(.*?)\s*$",
 			 line, flags)
 		if not m: continue
 		indent, is_comment, existing_val = m.groups()
@@ -206,7 +206,7 @@ if not ini_section or cur_section == ini_section.lower():
 
 if not testing:
 	# Write out the new file.
-	with open(filename, "w") as f:
+	with open(filename, "w", encoding="utf-8") as f:
 		f.write(buf)
 else:
 	# Just print the new file to stdout.

tools/parse-nginx-log-bootstrap-accesses.py

@@ -47,7 +47,7 @@ for date, ip in accesses:
 # Since logs are rotated, store the statistics permanently in a JSON file.
 # Load in the stats from an existing file.
 if os.path.exists(outfn):
-	with open(outfn, "r") as f:
+	with open(outfn, encoding="utf-8") as f:
 		existing_data = json.load(f)
 	for date, count in existing_data:
 		if date not in by_date:
@@ -60,5 +60,5 @@ by_date = sorted(by_date.items())
 by_date.pop(-1)
 
 # Write out.
-with open(outfn, "w") as f:
+with open(outfn, "w", encoding="utf-8") as f:
 	json.dump(by_date, f, sort_keys=True, indent=True)

tools/readable_bash.py

@@ -133,7 +133,7 @@ def generate_documentation():
  """)
 
 	parser = Source.parser()
-	with open("setup/start.sh", "r") as start_file: 
+	with open("setup/start.sh", "r") as start_file:
                 for line in start_file:
                         try:
                                 fn = parser.parse_string(line).filename()