Merge remote-tracking branch 'origin/master' into configurablebackupfolder

management/auth.py

@@ -1,6 +1,7 @@
-import base64, os, os.path, hmac, json
+import base64, os, os.path, hmac, json, secrets
+from datetime import timedelta
 
-from flask import make_response
+from expiringdict import ExpiringDict
 
 import utils
 from mailconfig import get_mail_password, get_mail_user_privileges
@@ -9,25 +10,18 @@ from mfa import get_hash_mfa_state, validate_auth_mfa
 DEFAULT_KEY_PATH   = '/var/lib/mailinabox/api.key'
 DEFAULT_AUTH_REALM = 'Mail-in-a-Box Management Server'
 
-class KeyAuthService:
-	"""Generate an API key for authenticating clients
-
-	Clients must read the key from the key file and send the key with all HTTP
-	requests. The key is passed as the username field in the standard HTTP
-	Basic Auth header.
-	"""
+class AuthService:
 	def __init__(self):
 		self.auth_realm = DEFAULT_AUTH_REALM
-		self.key = self._generate_key()
 		self.key_path = DEFAULT_KEY_PATH
+		self.max_session_duration = timedelta(days=2)
 
-	def write_key(self):
-		"""Write key to file so authorized clients can get the key
+		self.init_system_api_key()
+		self.sessions = ExpiringDict(max_len=64, max_age_seconds=self.max_session_duration.total_seconds())
+
+	def init_system_api_key(self):
+		"""Write an API key to a local file so local processes can use the API"""
 
-		The key file is created with mode 0640 so that additional users can be
-		authorized to access the API by granting group/ACL read permissions on
-		the key file.
-		"""
 		def create_file_with_mode(path, mode):
 			# Based on answer by A-B-B: http://stackoverflow.com/a/15015748
 			old_umask = os.umask(0)
@@ -36,123 +30,137 @@ class KeyAuthService:
 			finally:
 				os.umask(old_umask)
 
+		self.key = secrets.token_hex(32)
+
 		os.makedirs(os.path.dirname(self.key_path), exist_ok=True)
 
 		with create_file_with_mode(self.key_path, 0o640) as key_file:
 			key_file.write(self.key + '\n')
 
-	def authenticate(self, request, env):
-		"""Test if the client key passed in HTTP Authorization header matches the service key
-		or if the or username/password passed in the header matches an administrator user.
+	def authenticate(self, request, env, login_only=False, logout=False):
+		"""Test if the HTTP Authorization header's username matches the system key, a session key,
+		or if the username/password passed in the header matches a local user.
 		Returns a tuple of the user's email address and list of user privileges (e.g.
 		('my@email', []) or ('my@email', ['admin']); raises a ValueError on login failure.
-		If the user used an API key, the user's email is returned as None."""
+		If the user used the system API key, the user's email is returned as None since
+		this key is not associated with a user."""
 
-		def decode(s):
-			return base64.b64decode(s.encode('ascii')).decode('ascii')
-
-		def parse_basic_auth(header):
+		def parse_http_authorization_basic(header):
+			def decode(s):
+				return base64.b64decode(s.encode('ascii')).decode('ascii')
 			if " " not in header:
 				return None, None
 			scheme, credentials = header.split(maxsplit=1)
 			if scheme != 'Basic':
 				return None, None
-
 			credentials = decode(credentials)
 			if ":" not in credentials:
 				return None, None
 			username, password = credentials.split(':', maxsplit=1)
 			return username, password
 
-		header = request.headers.get('Authorization')
-		if not header:
-			raise ValueError("No authorization header provided.")
-
-		username, password = parse_basic_auth(header)
-
+		username, password = parse_http_authorization_basic(request.headers.get('Authorization', ''))
 		if username in (None, ""):
 			raise ValueError("Authorization header invalid.")
-		elif username == self.key:
-			# The user passed the master API key which grants administrative privs.
+
+		if username.strip() == "" and password.strip() == "":
+			raise ValueError("No email address, password, session key, or API key provided.")
+
+		# If user passed the system API key, grant administrative privs. This key
+		# is not associated with a user.
+		if username == self.key and not login_only:
 			return (None, ["admin"])
+
+		# If the password corresponds with a session token for the user, grant access for that user.
+		if self.get_session(username, password, "login", env) and not login_only:
+			sessionid = password
+			session = self.sessions[sessionid]
+			if logout:
+				# Clear the session.
+				del self.sessions[sessionid]
+			else:
+				# Re-up the session so that it does not expire.
+				self.sessions[sessionid] = session
+
+		# If no password was given, but a username was given, we're missing some information.
+		elif password.strip() == "":
+			raise ValueError("Enter a password.")
+
 		else:
-			# The user is trying to log in with a username and either a password
-			# (and possibly a MFA token) or a user-specific API key.
-			return (username, self.check_user_auth(username, password, request, env))
+			# The user is trying to log in with a username and a password
+			# (and possibly a MFA token). On failure, an exception is raised.
+			self.check_user_auth(username, password, request, env)
+
+		# Get privileges for authorization. This call should never fail because by this
+		# point we know the email address is a valid user --- unless the user has been
+		# deleted after the session was granted. On error the call will return a tuple
+		# of an error message and an HTTP status code.
+		privs = get_mail_user_privileges(username, env)
+		if isinstance(privs, tuple): raise ValueError(privs[0])
+
+		# Return the authorization information.
+		return (username, privs)
 
 	def check_user_auth(self, email, pw, request, env):
 		# Validate a user's login email address and password. If MFA is enabled,
 		# check the MFA token in the X-Auth-Token header.
 		#
-		# On success returns a list of privileges (e.g. [] or ['admin']). On login
-		# failure, raises a ValueError with a login error message.
+		# On login failure, raises a ValueError with a login error message. On
+		# success, nothing is returned.
 
-		# Sanity check.
-		if email == "" or pw == "":
-			raise ValueError("Enter an email address and password.")
-
-		# The password might be a user-specific API key. create_user_key raises
-		# a ValueError if the user does not exist.
-		if hmac.compare_digest(self.create_user_key(email, env), pw):
-			# OK.
-			pass
-		else:
+		# Authenticate.
+		try:
 			# Get the hashed password of the user. Raise a ValueError if the
-			# email address does not correspond to a user.
+			# email address does not correspond to a user. But wrap it in the
+			# same exception as if a password fails so we don't easily reveal
+			# if an email address is valid.
 			pw_hash = get_mail_password(email, env)
 
-			# Authenticate.
-			try:
-				# Use 'doveadm pw' to check credentials. doveadm will return
-				# a non-zero exit status if the credentials are no good,
-				# and check_call will raise an exception in that case.
-				utils.shell('check_call', [
-					"/usr/bin/doveadm", "pw",
-					"-p", pw,
-					"-t", pw_hash,
-					])
-			except:
-				# Login failed.
-				raise ValueError("Invalid password.")
+			# Use 'doveadm pw' to check credentials. doveadm will return
+			# a non-zero exit status if the credentials are no good,
+			# and check_call will raise an exception in that case.
+			utils.shell('check_call', [
+				"/usr/bin/doveadm", "pw",
+				"-p", pw,
+				"-t", pw_hash,
+				])
+		except:
+			# Login failed.
+			raise ValueError("Incorrect email address or password.")
 
-			# If MFA is enabled, check that MFA passes.
-			status, hints = validate_auth_mfa(email, request, env)
-			if not status:
-				# Login valid. Hints may have more info.
-				raise ValueError(",".join(hints))
+		# If MFA is enabled, check that MFA passes.
+		status, hints = validate_auth_mfa(email, request, env)
+		if not status:
+			# Login valid. Hints may have more info.
+			raise ValueError(",".join(hints))
 
-		# Get privileges for authorization. This call should never fail because by this
-		# point we know the email address is a valid user. But on error the call will
-		# return a tuple of an error message and an HTTP status code.
-		privs = get_mail_user_privileges(email, env)
-		if isinstance(privs, tuple): raise ValueError(privs[0])
-
-		# Return a list of privileges.
-		return privs
-
-	def create_user_key(self, email, env):
-		# Create a user API key, which is a shared secret that we can re-generate from
-		# static information in our database. The shared secret contains the user's
-		# email address, current hashed password, and current MFA state, so that the
-		# key becomes invalid if any of that information changes.
-		#
-		# Use an HMAC to generate the API key using our master API key as a key,
-		# which also means that the API key becomes invalid when our master API key
-		# changes --- i.e. when this process is restarted.
-		#
-		# Raises ValueError via get_mail_password if the user doesn't exist.
-
-		# Construct the HMAC message from the user's email address and current password.
-		msg = b"AUTH:" + email.encode("utf8") + b" " + get_mail_password(email, env).encode("utf8")
+	def create_user_password_state_token(self, email, env):
+		# Create a token that changes if the user's password or MFA options change
+		# so that sessions become invalid if any of that information changes.
+		msg = get_mail_password(email, env).encode("utf8")
 
 		# Add to the message the current MFA state, which is a list of MFA information.
 		# Turn it into a string stably.
 		msg += b" " + json.dumps(get_hash_mfa_state(email, env), sort_keys=True).encode("utf8")
 
-		# Make the HMAC.
+		# Make a HMAC using the system API key as a hash key.
 		hash_key = self.key.encode('ascii')
 		return hmac.new(hash_key, msg, digestmod="sha256").hexdigest()
 
-	def _generate_key(self):
-		raw_key = os.urandom(32)
-		return base64.b64encode(raw_key).decode('ascii')
+	def create_session_key(self, username, env, type=None):
+		# Create a new session.
+		token = secrets.token_hex(32)
+		self.sessions[token] = {
+			"email": username,
+			"password_token": self.create_user_password_state_token(username, env),
+			"type": type,
+		}
+		return token
+
+	def get_session(self, user_email, session_key, session_type, env):
+		if session_key not in self.sessions: return None
+		session = self.sessions[session_key]
+		if session_type == "login" and session["email"] != user_email: return None
+		if session["type"] != session_type: return None
+		if session["password_token"] != self.create_user_password_state_token(session["email"], env): return None
+		return session

management/daemon.py

@@ -1,5 +1,8 @@
 #!/usr/local/lib/mailinabox/env/bin/python3
 #
+# The API can be accessed on the command line, e.g. use `curl` like so:
+#    curl --user $(</var/lib/mailinabox/api.key): http://localhost:10222/mail/users
+#
 # During development, you can start the Mail-in-a-Box control panel
 # by running this script, e.g.:
 #
@@ -9,6 +12,7 @@
 
 import os, os.path, re, json, time
 import multiprocessing.pool, subprocess
+import logging
 
 from functools import wraps
 
@@ -22,7 +26,7 @@ from mfa import get_public_mfa_state, provision_totp, validate_totp_secret, enab
 
 env = utils.load_environment()
 
-auth_service = auth.KeyAuthService()
+auth_service = auth.AuthService()
 
 # We may deploy via a symbolic link, which confuses flask's template finding.
 me = __file__
@@ -53,8 +57,10 @@ def authorized_personnel_only(viewfunc):
 		try:
 			email, privs = auth_service.authenticate(request, env)
 		except ValueError as e:
-			# Write a line in the log recording the failed login
-			log_failed_login(request)
+			# Write a line in the log recording the failed login, unless no authorization header
+			# was given which can happen on an initial request before a 403 response.
+			if "Authorization" in request.headers:
+				log_failed_login(request)
 
 			# Authentication failed.
 			error = str(e)
@@ -131,11 +137,12 @@ def index():
 		csr_country_codes=csr_country_codes,
 	)
 
-@app.route('/me')
-def me():
+# Create a session key by checking the username/password in the Authorization header.
+@app.route('/login', methods=["POST"])
+def login():
 	# Is the caller authorized?
 	try:
-		email, privs = auth_service.authenticate(request, env)
+		email, privs = auth_service.authenticate(request, env, login_only=True)
 	except ValueError as e:
 		if "missing-totp-token" in str(e):
 			return json_response({
@@ -150,19 +157,29 @@ def me():
 				"reason": str(e),
 			})
 
+	# Return a new session for the user.
 	resp = {
 		"status": "ok",
 		"email": email,
 		"privileges": privs,
+		"api_key": auth_service.create_session_key(email, env, type='login'),
 	}
 
-	# Is authorized as admin? Return an API key for future use.
-	if "admin" in privs:
-		resp["api_key"] = auth_service.create_user_key(email, env)
+	app.logger.info("New login session created for {}".format(email))
 
 	# Return.
 	return json_response(resp)
 
+@app.route('/logout', methods=["POST"])
+def logout():
+	try:
+		email, _ = auth_service.authenticate(request, env, logout=True)
+		app.logger.info("{} logged out".format(email))
+	except ValueError as e:
+		pass
+	finally:
+		return json_response({ "status": "ok" })
+
 # MAIL
 
 @app.route('/mail/users')
@@ -219,7 +236,7 @@ def mail_aliases():
 	if request.args.get("format", "") == "json":
 		return json_response(get_mail_aliases_ex(env))
 	else:
-		return "".join(address+"\t"+receivers+"\t"+(senders or "")+"\n" for address, receivers, senders in get_mail_aliases(env))
+		return "".join(address+"\t"+receivers+"\t"+(senders or "")+"\n" for address, receivers, senders, auto in get_mail_aliases(env))
 
 @app.route('/mail/aliases/add', methods=['POST'])
 @authorized_personnel_only
@@ -257,6 +274,7 @@ def dns_update():
 	try:
 		return do_dns_update(env, force=request.form.get('force', '') == '1')
 	except Exception as e:
+		logging.exception('dns update exc')
 		return (str(e), 500)
 
 @app.route('/dns/secondary-nameserver')
@@ -314,7 +332,7 @@ def dns_get_records(qname=None, rtype=None):
 		r["sort-order"]["created"] = i
 	domain_sort_order = utils.sort_domains([r["qname"] for r in records], env)
 	for i, r in enumerate(sorted(records, key = lambda r : (
-			zones.index(r["zone"]),
+			zones.index(r["zone"]) if r.get("zone") else 0, # record is not within a zone managed by the box
 			domain_sort_order.index(r["qname"]),
 			r["rtype"]))):
 		r["sort-order"]["qname"] = i
@@ -512,10 +530,7 @@ def web_get_domains():
 @authorized_personnel_only
 def web_update():
 	from web_update import do_web_update
-	try:
-		return do_web_update(env)
-	except Exception as e:
-		return (str(e), 500)
+	return do_web_update(env)
 
 # System
 
@@ -641,16 +656,42 @@ def privacy_status_set():
 # MUNIN
 
 @app.route('/munin/')
-@app.route('/munin/<path:filename>')
 @authorized_personnel_only
-def munin(filename=""):
-	# Checks administrative access (@authorized_personnel_only) and then just proxies
-	# the request to static files.
+def munin_start():
+	# Munin pages, static images, and dynamically generated images are served
+	# outside of the AJAX API. We'll start with a 'start' API that sets a cookie
+	# that subsequent requests will read for authorization. (We don't use cookies
+	# for the API to avoid CSRF vulnerabilities.)
+	response = make_response("OK")
+	response.set_cookie("session", auth_service.create_session_key(request.user_email, env, type='cookie'),
+	    max_age=60*30, secure=True, httponly=True, samesite="Strict") # 30 minute duration
+	return response
+
+def check_request_cookie_for_admin_access():
+	session = auth_service.get_session(None, request.cookies.get("session", ""), "cookie", env)
+	if not session: return False
+	privs = get_mail_user_privileges(session["email"], env)
+	if not isinstance(privs, list): return False
+	if "admin" not in privs: return False
+	return True
+
+def authorized_personnel_only_via_cookie(f):
+	@wraps(f)
+	def g(*args, **kwargs):
+		if not check_request_cookie_for_admin_access():
+			return Response("Unauthorized", status=403, mimetype='text/plain', headers={})
+		return f(*args, **kwargs)
+	return g
+
+@app.route('/munin/<path:filename>')
+@authorized_personnel_only_via_cookie
+def munin_static_file(filename=""):
+	# Proxy the request to static files.
 	if filename == "": filename = "index.html"
 	return send_from_directory("/var/cache/munin/www", filename)
 
 @app.route('/munin/cgi-graph/<path:filename>')
-@authorized_personnel_only
+@authorized_personnel_only_via_cookie
 def munin_cgi(filename):
 	""" Relay munin cgi dynazoom requests
 	/usr/lib/munin/cgi/munin-cgi-graph is a perl cgi script in the munin package
@@ -723,34 +764,21 @@ def log_failed_login(request):
 # APP
 
 if __name__ == '__main__':
+	logging_level = logging.DEBUG
+	
 	if "DEBUG" in os.environ:
 		# Turn on Flask debugging.
 		app.debug = True
-
-		# Use a stable-ish master API key so that login sessions don't restart on each run.
-		# Use /etc/machine-id to seed the key with a stable secret, but add something
-		# and hash it to prevent possibly exposing the machine id, using the time so that
-		# the key is not valid indefinitely.
-		import hashlib
-		with open("/etc/machine-id") as f:
-			api_key = f.read()
-		api_key += "|" + str(int(time.time() / (60*60*2)))
-		hasher = hashlib.sha1()
-		hasher.update(api_key.encode("ascii"))
-		auth_service.key = hasher.hexdigest()
-
-	if "APIKEY" in os.environ: auth_service.key = os.environ["APIKEY"]
+		logging_level = logging.DEBUG
 
 	if not app.debug:
 		app.logger.addHandler(utils.create_syslog_handler())
 
-	# For testing on the command line, you can use `curl` like so:
-	#    curl --user $(</var/lib/mailinabox/api.key): http://localhost:10222/mail/users
-	auth_service.write_key()
+	#app.logger.info('API key: ' + auth_service.key)
 
-	# For testing in the browser, you can copy the API key that's output to the
-	# debug console and enter that as the username
-	app.logger.info('API key: ' + auth_service.key)
+	logging.basicConfig(level=logging_level, format='%(levelname)s:%(module)s.%(funcName)s %(message)s')
+	logging.info('Logging level set to %s', logging.getLevelName(logging_level))
 
 	# Start the application server. Listens on 127.0.0.1 (IPv4 only).
 	app.run(port=10222)
+	

management/daily_tasks.sh

@@ -14,7 +14,7 @@ source /etc/mailinabox.conf
 # On Mondays, i.e. once a week, send the administrator a report of total emails
 # sent and received so the admin might notice server abuse.
 if [ `date "+%u"` -eq 1 ]; then
-    management/mail_log.py -t week | management/email_administrator.py "Mail-in-a-Box Usage Report"
+    management/mail_log.py -t week -r -s -l -g -b | management/email_administrator.py "Mail-in-a-Box Usage Report"
     
     /usr/sbin/pflogsumm -u 5 -h 5 --problems_first /var/log/mail.log.1 | management/email_administrator.py "Postfix log analysis summary"
 fi

management/dns_update.py

@@ -8,6 +8,7 @@ import sys, os, os.path, urllib.parse, datetime, re, hashlib, base64
 import ipaddress
 import rtyaml
 import dns.resolver
+import logging
 
 from utils import shell, load_env_vars_from_file, safe_domain_name, sort_domains
 from ssl_certificates import get_ssl_certificates, check_certificate
@@ -105,21 +106,22 @@ def do_dns_update(env, force=False):
 	if len(updated_domains) > 0:
 		shell('check_call', ["/usr/sbin/service", "nsd", "restart"])
 
-	# Write the OpenDKIM configuration tables for all of the mail domains.
+	# Write the DKIM configuration tables for all of the mail domains.
 	from mailconfig import get_mail_domains
-	if write_opendkim_tables(get_mail_domains(env), env):
-		# Settings changed. Kick opendkim.
-		shell('check_call', ["/usr/sbin/service", "opendkim", "restart"])
+	
+	if write_dkim_tables(get_mail_domains(env), env):
+		# Settings changed. Kick dkimpy.
+		shell('check_call', ["/usr/sbin/service", "dkimpy-milter", "restart"])
 		if len(updated_domains) == 0:
 			# If this is the only thing that changed?
-			updated_domains.append("OpenDKIM configuration")
+			updated_domains.append("DKIM configuration")
 
-	# Clear bind9's DNS cache so our own DNS resolver is up to date.
+	# Clear unbound's DNS cache so our own DNS resolver is up to date.
 	# (ignore errors with trap=True)
-	shell('check_call', ["/usr/sbin/rndc", "flush"], trap=True)
+	shell('check_call', ["/usr/sbin/unbound-control", "flush_zone", "."], trap=True, capture_stdout=False)
 
 	if len(updated_domains) == 0:
-		# if nothing was updated (except maybe OpenDKIM's files), don't show any output
+		# if nothing was updated (except maybe DKIM's files), don't show any output
 		return ""
 	else:
 		return "updated DNS: " + ",".join(updated_domains) + "\n"
@@ -303,10 +305,18 @@ def build_zone(domain, domain_properties, additional_records, env, is_zone=True)
 		if not has_rec(None, "TXT", prefix="v=spf1 "):
 			records.append((None,  "TXT", 'v=spf1 mx -all', "Recommended. Specifies that only the box is permitted to send @%s mail." % domain))
 
-		# Append the DKIM TXT record to the zone as generated by OpenDKIM.
+		# Append the DKIM TXT record to the zone as generated by DKIMpy.
 		# Skip if the user has set a DKIM record already.
-		opendkim_record_file = os.path.join(env['STORAGE_ROOT'], 'mail/dkim/mail.txt')
-		with open(opendkim_record_file) as orf:
+		dkim_record_file = os.path.join(env['STORAGE_ROOT'], 'mail/dkim/box-rsa.dns')
+		with open(dkim_record_file) as orf:
+			m = re.match(r'(\S+)\s+IN\s+TXT\s+\( ((?:"[^"]+"\s+)+)\)', orf.read(), re.S)
+			val = "".join(re.findall(r'"([^"]+)"', m.group(2)))
+			if not has_rec(m.group(1), "TXT", prefix="v=DKIM1; "):
+				records.append((m.group(1), "TXT", val, "Recommended. Provides a way for recipients to verify that this machine sent @%s mail." % domain))
+		
+		# Also add a ed25519 DKIM record
+		dkim_record_file = os.path.join(env['STORAGE_ROOT'], 'mail/dkim/box-ed25519.dns')
+		with open(dkim_record_file) as orf:
 			m = re.match(r'(\S+)\s+IN\s+TXT\s+\( ((?:"[^"]+"\s+)+)\)', orf.read(), re.S)
 			val = "".join(re.findall(r'"([^"]+)"', m.group(2)))
 			if not has_rec(m.group(1), "TXT", prefix="v=DKIM1; "):
@@ -501,7 +511,7 @@ def write_nsd_zone(domain, zonefile, records, env, force):
 	# @ the PRIMARY_HOSTNAME. Hopefully that's legit.
 	#
 	# For the refresh through TTL fields, a good reference is:
-	# http://www.peerwisdom.org/2013/05/15/dns-understanding-the-soa-record/
+	# https://www.ripe.net/publications/docs/ripe-203
 	
 	# Time To Refresh – How long in seconds a nameserver should wait prior to checking for a Serial Number 
 	#     increase within the primary zone file. An increased Serial Number means a transfer is needed to sync
@@ -670,7 +680,7 @@ def get_dns_zonefile(zone, env):
 
 def write_nsd_conf(zonefiles, additional_records, env):
 	# Write the list of zones to a configuration file.
-	nsd_conf_file = "/etc/nsd/zones.conf"
+	nsd_conf_file = "/etc/nsd/nsd.conf.d/zones.conf"
 	nsdconf = ""
 
 	# Append the zones.
@@ -817,14 +827,15 @@ def sign_zone(domain, zonefile, env):
 
 ########################################################################
 
-def write_opendkim_tables(domains, env):
-	# Append a record to OpenDKIM's KeyTable and SigningTable for each domain
+def write_dkim_tables(domains, env):
+	# Append a record to DKIMpy's KeyTable and SigningTable for each domain
 	# that we send mail from (zones and all subdomains).
 
-	opendkim_key_file = os.path.join(env['STORAGE_ROOT'], 'mail/dkim/mail.private')
+	dkim_rsa_key_file = os.path.join(env['STORAGE_ROOT'], 'mail/dkim/box-rsa.key')
+	dkim_ed_key_file = os.path.join(env['STORAGE_ROOT'], 'mail/dkim/box-ed25519.key')
 
-	if not os.path.exists(opendkim_key_file):
-		# Looks like OpenDKIM is not installed.
+	if not os.path.exists(dkim_rsa_key_file) or not os.path.exists(dkim_ed_key_file):
+		# Looks like DKIMpy is not installed.
 		return False
 
 	config = {
@@ -846,7 +857,12 @@ def write_opendkim_tables(domains, env):
 		# signing domain must match the sender's From: domain.
 		"KeyTable":
 			"".join(
-				"{domain} {domain}:mail:{key_file}\n".format(domain=domain, key_file=opendkim_key_file)
+				"{domain} {domain}:box-rsa:{key_file}\n".format(domain=domain, key_file=dkim_rsa_key_file)
+				for domain in domains
+			),
+		"KeyTableEd25519":
+			"".join(
+				"{domain} {domain}:box-ed25519:{key_file}\n".format(domain=domain, key_file=dkim_ed_key_file)
 				for domain in domains
 			),
 	}
@@ -854,18 +870,18 @@ def write_opendkim_tables(domains, env):
 	did_update = False
 	for filename, content in config.items():
 		# Don't write the file if it doesn't need an update.
-		if os.path.exists("/etc/opendkim/" + filename):
-			with open("/etc/opendkim/" + filename) as f:
+		if os.path.exists("/etc/dkim/" + filename):
+			with open("/etc/dkim/" + filename) as f:
 				if f.read() == content:
 					continue
 
 		# The contents needs to change.
-		with open("/etc/opendkim/" + filename, "w") as f:
+		with open("/etc/dkim/" + filename, "w") as f:
 			f.write(content)
 		did_update = True
 
 	# Return whether the files changed. If they didn't change, there's
-	# no need to kick the opendkim process.
+	# no need to kick the dkimpy process.
 	return did_update
 
 ########################################################################
@@ -1049,6 +1065,7 @@ def set_custom_dns_record(qname, rtype, value, action, env):
 def get_secondary_dns(custom_dns, mode=None):
 	resolver = dns.resolver.get_default_resolver()
 	resolver.timeout = 10
+	resolver.lifetime = 10
 
 	values = []
 	for qname, rtype, value in custom_dns:
@@ -1066,10 +1083,17 @@ def get_secondary_dns(custom_dns, mode=None):
 			# doesn't.
 			if not hostname.startswith("xfr:"):
 				if mode == "xfr":
-					response = dns.resolver.resolve(hostname+'.', "A", raise_on_no_answer=False)
-					values.extend(map(str, response))
-					response = dns.resolver.resolve(hostname+'.', "AAAA", raise_on_no_answer=False)
-					values.extend(map(str, response))
+					try:
+						response = resolver.resolve(hostname+'.', "A", raise_on_no_answer=False)
+						values.extend(map(str, response))
+					except dns.exception.DNSException:
+						logging.debug("Secondary dns A lookup exception %s", hostname)
+						
+					try:
+						response = resolver.resolve(hostname+'.', "AAAA", raise_on_no_answer=False)
+						values.extend(map(str, response))
+					except dns.exception.DNSException:
+						logging.debug("Secondary dns AAAA lookup exception %s", hostname)
 					continue
 				values.append(hostname)
 
@@ -1087,16 +1111,32 @@ def set_secondary_dns(hostnames, env):
 		# Validate that all hostnames are valid and that all zone-xfer IP addresses are valid.
 		resolver = dns.resolver.get_default_resolver()
 		resolver.timeout = 5
+		resolver.lifetime = 5
 		for item in hostnames:
 			if not item.startswith("xfr:"):
 				# Resolve hostname.
-				try:
-					response = resolver.resolve(item, "A")
-				except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
+				tries = 2
+				
+				while tries > 0:
+					tries = tries - 1
 					try:
-						response = resolver.query(item, "AAAA")
+						response = resolver.resolve(item, "A")
+						tries = 0
 					except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
-						raise ValueError("Could not resolve the IP address of %s." % item)
+						logging.debug('Error on resolving ipv4 address, trying ipv6')
+						try:
+							response = resolver.resolve(item, "AAAA")
+							tries = 0
+						except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
+							raise ValueError("Could not resolve the IP address of %s." % item)
+						except (dns.resolver.Timeout):
+							logging.debug('Timeout on resolving ipv6 address')
+							if tries < 1:
+								raise ValueError("Could not resolve the IP address of %s due to timeout." % item)
+					except (dns.resolver.Timeout):
+						logging.debug('Timeout on resolving ipv4 address')
+						if tries < 1:
+							raise ValueError("Could not resolve the IP address of %s due to timeout." % item)
 			else:
 				# Validate IP address.
 				try:

management/email_administrator.py

@@ -2,7 +2,7 @@
 
 # Reads in STDIN. If the stream is not empty, mail it to the system administrator.
 
-import sys
+import sys, traceback
 
 import html
 import smtplib
@@ -29,6 +29,7 @@ try:
         content = sys.stdin.read().strip()
 except:
         print("error occured while cleaning input text")
+        traceback.print_exc()
         sys.exit(1)
 
 # If there's nothing coming in, just exit.

management/mail_log.py

@@ -376,7 +376,7 @@ def scan_mail_log_line(line, collector):
         if SCAN_BLOCKED:
             scan_postfix_smtpd_line(date, log, collector)
     elif service in ("postfix/qmgr", "postfix/pickup", "postfix/cleanup", "postfix/scache",
-                     "spampd", "postfix/anvil", "postfix/master", "opendkim", "postfix/lmtp",
+                     "spampd", "postfix/anvil", "postfix/master", "dkimpy", "postfix/lmtp",
                      "postfix/tlsmgr", "anvil"):
         # nothing to look at
         return True
@@ -549,8 +549,9 @@ def scan_postfix_submission_line(date, log, collector):
     """
 
     # Match both the 'plain' and 'login' sasl methods, since both authentication methods are
-    # allowed by Dovecot
-    m = re.match("([A-Z0-9]+): client=(\S+), sasl_method=(PLAIN|LOGIN), sasl_username=(\S+)", log)
+    # allowed by Dovecot. Exclude trailing comma after the username when additional fields 
+	# follow after.
+    m = re.match("([A-Z0-9]+): client=(\S+), sasl_method=(PLAIN|LOGIN), sasl_username=(\S+)(?<!,)", log)
 
     if m:
         _, client, method, user = m.groups()
@@ -586,7 +587,7 @@ def scan_postfix_submission_line(date, log, collector):
 def readline(filename):
     """ A generator that returns the lines of a file
     """
-    with open(filename) as file:
+    with open(filename, errors='replace') as file:
         while True:
           line = file.readline()
           if not line:

management/mailconfig.py

@@ -16,8 +16,8 @@ import idna
 
 def validate_email(email, mode=None):
 	# Checks that an email address is syntactically valid. Returns True/False.
-	# Until Postfix supports SMTPUTF8, an email address may contain ASCII
-	# characters only; IDNs must be IDNA-encoded.
+	# An email address may contain ASCII characters only because Dovecot's
+	# authentication mechanism gets confused with other character encodings.
 	#
 	# When mode=="user", we're checking that this can be a user account name.
 	# Dovecot has tighter restrictions - letters, numbers, underscore, and
@@ -186,9 +186,9 @@ def get_admins(env):
 	return users
 
 def get_mail_aliases(env):
-	# Returns a sorted list of tuples of (address, forward-tos, permitted-senders).
+	# Returns a sorted list of tuples of (address, forward-tos, permitted-senders, auto).
 	c = open_database(env)
-	c.execute('SELECT source, destination, permitted_senders FROM aliases')
+	c.execute('SELECT source, destination, permitted_senders, 0 as auto FROM aliases UNION SELECT source, destination, permitted_senders, 1 as auto FROM auto_aliases')
 	aliases = { row[0]: row for row in c.fetchall() } # make dict
 
 	# put in a canonical order: sort by domain, then by email address lexicographically
@@ -208,7 +208,7 @@ def get_mail_aliases_ex(env):
 	#         address_display: "name@domain.tld", # full Unicode
 	#         forwards_to: ["user1@domain.com", "receiver-only1@domain.com", ...],
 	#         permitted_senders: ["user1@domain.com", "sender-only1@domain.com", ...] OR null,
-	#         required: True|False
+	#         auto: True|False
 	#       },
 	#       ...
 	#     ]
@@ -216,12 +216,13 @@ def get_mail_aliases_ex(env):
 	#   ...
 	# ]
 
-	required_aliases = get_required_aliases(env)
 	domains = {}
-	for address, forwards_to, permitted_senders in get_mail_aliases(env):
+	for address, forwards_to, permitted_senders, auto in get_mail_aliases(env):
+		# skip auto domain maps since these are not informative in the control panel's aliases list
+		if auto and address.startswith("@"): continue
+
 		# get alias info
 		domain = get_domain(address)
-		required = (address in required_aliases)
 
 		# add to list
 		if not domain in domains:
@@ -234,7 +235,7 @@ def get_mail_aliases_ex(env):
 			"address_display": prettify_idn_email_address(address),
 			"forwards_to": [prettify_idn_email_address(r.strip()) for r in forwards_to.split(",")],
 			"permitted_senders": [prettify_idn_email_address(s.strip()) for s in permitted_senders.split(",")] if permitted_senders is not None else None,
-			"required": required,
+			"auto": bool(auto),
 		})
 
 	# Sort domains.
@@ -242,7 +243,7 @@ def get_mail_aliases_ex(env):
 
 	# Sort aliases within each domain first by required-ness then lexicographically by address.
 	for domain in domains:
-		domain["aliases"].sort(key = lambda alias : (alias["required"], alias["address"]))
+		domain["aliases"].sort(key = lambda alias : (alias["auto"], alias["address"]))
 	return domains
 
 def get_domain(emailaddr, as_unicode=True):
@@ -261,11 +262,12 @@ def get_domain(emailaddr, as_unicode=True):
 def get_mail_domains(env, filter_aliases=lambda alias : True, users_only=False):
 	# Returns the domain names (IDNA-encoded) of all of the email addresses
 	# configured on the system. If users_only is True, only return domains
-	# with email addresses that correspond to user accounts.
+	# with email addresses that correspond to user accounts. Exclude Unicode
+	# forms of domain names listed in the automatic aliases table.
 	domains = []
 	domains.extend([get_domain(login, as_unicode=False) for login in get_mail_users(env)])
 	if not users_only:
-		domains.extend([get_domain(address, as_unicode=False) for address, *_ in get_mail_aliases(env) if filter_aliases(address) ])
+		domains.extend([get_domain(address, as_unicode=False) for address, _, _, auto in get_mail_aliases(env) if filter_aliases(address) and not auto ])
 	return set(domains)
 
 def add_mail_user(email, pw, privs, env):
@@ -512,6 +514,13 @@ def remove_mail_alias(address, env, do_kick=True):
 		# Update things in case any domains are removed.
 		return kick(env, "alias removed")
 
+def add_auto_aliases(aliases, env):
+	conn, c = open_database(env, with_connection=True)
+	c.execute("DELETE FROM auto_aliases");
+	for source, destination in aliases.items():
+		c.execute("INSERT INTO auto_aliases (source, destination) VALUES (?, ?)", (source, destination))
+	conn.commit()
+
 def get_system_administrator(env):
 	return "administrator@" + env['PRIMARY_HOSTNAME']
 
@@ -558,39 +567,34 @@ def kick(env, mail_result=None):
 	if mail_result is not None:
 		results.append(mail_result + "\n")
 
-	# Ensure every required alias exists.
+	auto_aliases = { }
 
-	existing_users = get_mail_users(env)
-	existing_alias_records = get_mail_aliases(env)
-	existing_aliases = set(a for a, *_ in existing_alias_records) # just first entry in tuple
+	# Map required aliases to the administrator alias (which should be created manually).
+	administrator = get_system_administrator(env)
 	required_aliases = get_required_aliases(env)
+	for alias in required_aliases:
+		if alias == administrator: continue # don't make an alias from the administrator to itself --- this alias must be created manually
+		auto_aliases[alias] = administrator
 
-	def ensure_admin_alias_exists(address):
-		# If a user account exists with that address, we're good.
-		if address in existing_users:
-			return
+	# Add domain maps from Unicode forms of IDNA domains to the ASCII forms stored in the alias table.
+	for domain in get_mail_domains(env):
+		try:
+			domain_unicode = idna.decode(domain.encode("ascii"))
+			if domain == domain_unicode: continue # not an IDNA/Unicode domain
+			auto_aliases["@" + domain_unicode] = "@" + domain
+		except (ValueError, UnicodeError, idna.IDNAError):
+			continue
 
-		# If the alias already exists, we're good.
-		if address in existing_aliases:
-			return
+	add_auto_aliases(auto_aliases, env)
 
-		# Doesn't exist.
-		administrator = get_system_administrator(env)
-		if address == administrator: return # don't make an alias from the administrator to itself --- this alias must be created manually
-		add_mail_alias(address, administrator, "", env, do_kick=False)
-		if administrator not in existing_aliases: return # don't report the alias in output if the administrator alias isn't in yet -- this is a hack to supress confusing output on initial setup
-		results.append("added alias %s (=> %s)\n" % (address, administrator))
-
-	for address in required_aliases:
-		ensure_admin_alias_exists(address)
-
-	# Remove auto-generated postmaster/admin on domains we no
-	# longer have any other email addresses for.
-	for address, forwards_to, *_ in existing_alias_records:
+	# Remove auto-generated postmaster/admin/abuse alises from the main aliases table.
+	# They are now stored in the auto_aliases table.
+	for address, forwards_to, permitted_senders, auto in get_mail_aliases(env):
 		user, domain = address.split("@")
 		if user in ("postmaster", "admin", "abuse") \
 			and address not in required_aliases \
-			and forwards_to == get_system_administrator(env):
+			and forwards_to == get_system_administrator(env) \
+			and not auto:
 			remove_mail_alias(address, env, do_kick=False)
 			results.append("removed alias %s (was to %s; domain no longer used for email)\n" % (address, forwards_to))
 

management/mfa.py

@@ -109,15 +109,7 @@ def validate_auth_mfa(email, request, env):
 	# If no MFA modes are added, return True.
 	if len(mfa_state) == 0:
 		return (True, [])
-		
-	# munin routes are proxied by our control panel. We do not have
-	# full control over their routes so credentials are supplied via
-	# a basic HTTP authentication prompt.
-	# There is neither a way to input a mfa credential there nor can we pass
-	# the user_api_key from localStorage so mfa should be disabled for these routes.
-	if request.full_path.startswith("/munin"):
-		return (True, [])
-		
+
 	# Try the enabled MFA modes.
 	hints = set()
 	for mfa_mode in mfa_state:

management/status_checks.py

@@ -12,6 +12,7 @@ import dateutil.parser, dateutil.tz
 import idna
 import psutil
 import postfix_mta_sts_resolver.resolver
+import logging
 
 from dns_update import get_dns_zones, build_tlsa_record, get_custom_dns_config, get_secondary_dns, get_custom_dns_records
 from web_update import get_web_domains, get_domains_with_a_records
@@ -22,13 +23,12 @@ from utils import shell, sort_domains, load_env_vars_from_file, load_settings
 
 def get_services():
 	return [
-		{ "name": "Local DNS (bind9)", "port": 53, "public": False, },
-		#{ "name": "NSD Control", "port": 8952, "public": False, },
-		{ "name": "Local DNS Control (bind9/rndc)", "port": 953, "public": False, },
+		{ "name": "Local DNS (unbound)", "port": 53, "public": False, },
+		{ "name": "Local DNS Control (unbound)", "port": 953, "public": False, },
 		{ "name": "Dovecot LMTP LDA", "port": 10026, "public": False, },
 		{ "name": "Postgrey", "port": 10023, "public": False, },
 		{ "name": "Spamassassin", "port": 10025, "public": False, },
-		{ "name": "OpenDKIM", "port": 8891, "public": False, },
+		{ "name": "DKIMpy", "port": 8892, "public": False, },
 		{ "name": "OpenDMARC", "port": 8893, "public": False, },
 		{ "name": "Mail-in-a-Box Management Daemon", "port": 10222, "public": False, },
 		{ "name": "SSH Login (ssh)", "port": get_ssh_port(), "public": True, },
@@ -49,15 +49,15 @@ def run_checks(rounded_values, env, output, pool, domains_to_check=None):
 
 	# check that services are running
 	if not run_services_checks(env, output, pool):
-		# If critical services are not running, stop. If bind9 isn't running,
+		# If critical services are not running, stop. If unbound isn't running,
 		# all later DNS checks will timeout and that will take forever to
 		# go through, and if running over the web will cause a fastcgi timeout.
 		return
 
-	# clear bind9's DNS cache so our DNS checks are up to date
-	# (ignore errors; if bind9/rndc isn't running we'd already report
+	# clear unbound's DNS cache so our DNS checks are up to date
+	# (ignore errors; if unbound isn't running we'd already report
 	# that in run_services checks.)
-	shell('check_call', ["/usr/sbin/rndc", "flush"], trap=True)
+	shell('check_call', ["/usr/sbin/unbound-control", "flush_zone", "."], trap=True, capture_stdout=False)
 
 	run_system_checks(rounded_values, env, output)
 
@@ -73,6 +73,9 @@ def get_ssh_port():
 	except FileNotFoundError:
 		# sshd is not installed. That's ok.
 		return None
+	except subprocess.CalledProcessError:
+		# error while calling shell command
+		return None
 
 	returnNext = False
 	for e in output.split():
@@ -293,7 +296,7 @@ def run_network_checks(env, output):
 	# by a spammer, or the user may be deploying on a residential network. We
 	# will not be able to reliably send mail in these cases.
 	rev_ip4 = ".".join(reversed(env['PUBLIC_IP'].split('.')))
-	zen = query_dns(rev_ip4+'.zen.spamhaus.org', 'A', nxdomain=None)
+	zen = query_dns(rev_ip4+'.zen.spamhaus.org', 'A', nxdomain=None, retry = False)
 	if zen is None:
 		output.print_ok("IP address is not blacklisted by zen.spamhaus.org.")
 	elif zen == "[timeout]":
@@ -547,14 +550,17 @@ def check_dns_zone(domain, env, output, dns_zonefiles):
 			# Choose the first IP if nameserver returns multiple
 			ns_ip = ns_ips.split('; ')[0]
 
-			# Now query it to see what it says about this domain.
-			ip = query_dns(domain, "A", at=ns_ip, nxdomain=None)
-			if ip == correct_ip:
-				output.print_ok("Secondary nameserver %s resolved the domain correctly." % ns)
-			elif ip is None:
-				output.print_error("Secondary nameserver %s is not configured to resolve this domain." % ns)
+			if ns_ip == '[Not Set]':
+				output.print_error("Secondary nameserver %s could not be resolved correctly. (dns result: %s used %s)" % (ns, ns_ips, ns_ip))
 			else:
-				output.print_error("Secondary nameserver %s is not configured correctly. (It resolved this domain as %s. It should be %s.)" % (ns, ip, correct_ip))
+				# Now query it to see what it says about this domain.
+				ip = query_dns(domain, "A", at=ns_ip, nxdomain=None)
+				if ip == correct_ip:
+					output.print_ok("Secondary nameserver %s resolved the domain correctly." % ns)
+				elif ip is None:
+					output.print_error("Secondary nameserver %s is not configured to resolve this domain." % ns)
+				else:
+					output.print_error("Secondary nameserver %s is not configured correctly. (It resolved this domain as %s. It should be %s.)" % (ns, ip, correct_ip))
 
 def check_dns_zone_suggestions(domain, env, output, dns_zonefiles, domains_with_a_records):
 	# Warn if a custom DNS record is preventing this or the automatic www redirect from
@@ -626,14 +632,16 @@ def check_dnssec(domain, env, output, dns_zonefiles, is_checking_primary=False):
 			#
 			# But it may not be preferred. Only algorithm 13 is preferred. Warn if any of the
 			# matched zones uses a different algorithm.
-			if set(r[1] for r in matched_ds) == { '13' }: # all are alg 13
+			if set(r[1] for r in matched_ds) == { '13' } and set(r[2] for r in matched_ds) <= { '2', '4' }: # all are alg 13 and digest type 2 or 4
 				output.print_ok("DNSSEC 'DS' record is set correctly at registrar.")
 				return
-			elif '13' in set(r[1] for r in matched_ds): # some but not all are alg 13
-				output.print_ok("DNSSEC 'DS' record is set correctly at registrar. (Records using algorithm other than ECDSAP256SHA256 should be removed.)")
+			elif len([r for r in matched_ds if r[1] == '13' and r[2] in ( '2', '4' )]) > 0: # some but not all are alg 13
+				output.print_ok("DNSSEC 'DS' record is set correctly at registrar. (Records using algorithm other than ECDSAP256SHA256 and digest types other than SHA-256/384 should be removed.)")
 				return
 			else: # no record uses alg 13
-				output.print_warning("DNSSEC 'DS' record set at registrar is valid but should be updated to ECDSAP256SHA256 (see below).")
+				output.print_warning("""DNSSEC 'DS' record set at registrar is valid but should be updated to ECDSAP256SHA256 and SHA-256 (see below).
+				IMPORTANT: Do not delete existing DNSSEC 'DS' records for this domain until confirmation that the new DNSSEC 'DS' record
+				for this domain is valid.""")
 		else:
 			if is_checking_primary:
 				output.print_error("""The DNSSEC 'DS' record for %s is incorrect. See further details below.""" % domain)
@@ -644,7 +652,8 @@ def check_dnssec(domain, env, output, dns_zonefiles, is_checking_primary=False):
 
 	output.print_line("""Follow the instructions provided by your domain name registrar to set a DS record.
 		Registrars support different sorts of DS records. Use the first option that works:""")
-	preferred_ds_order = [(7, 1), (7, 2), (8, 4), (13, 4), (8, 1), (8, 2), (13, 1), (13, 2)] # low to high
+	preferred_ds_order = [(7, 2), (8, 4), (13, 4), (8, 2), (13, 2)] # low to high, see https://github.com/mail-in-a-box/mailinabox/issues/1998
+
 	def preferred_ds_order_func(ds_suggestion):
 		k = (int(ds_suggestion['alg']), int(ds_suggestion['digalg']))
 		if k in preferred_ds_order:
@@ -652,11 +661,12 @@ def check_dnssec(domain, env, output, dns_zonefiles, is_checking_primary=False):
 		return -1 # index before first item
 	output.print_line("")
 	for i, ds_suggestion in enumerate(sorted(expected_ds_records.values(), key=preferred_ds_order_func, reverse=True)):
+		if preferred_ds_order_func(ds_suggestion) == -1: continue # don't offer record types that the RFC says we must not offer
 		output.print_line("")
 		output.print_line("Option " + str(i+1) + ":")
 		output.print_line("----------")
 		output.print_line("Key Tag: " + ds_suggestion['keytag'])
-		output.print_line("Key Flags: KSK")
+		output.print_line("Key Flags: KSK / 257")
 		output.print_line("Algorithm: %s / %s" % (ds_suggestion['alg'], ds_suggestion['alg_name']))
 		output.print_line("Digest Type: %s / %s" % (ds_suggestion['digalg'], ds_suggestion['digalg_name']))
 		output.print_line("Digest: " + ds_suggestion['digest'])
@@ -737,7 +747,7 @@ def check_mail_domain(domain, env, output):
 	# Stop if the domain is listed in the Spamhaus Domain Block List.
 	# The user might have chosen a domain that was previously in use by a spammer
 	# and will not be able to reliably send mail.
-	dbl = query_dns(domain+'.dbl.spamhaus.org', "A", nxdomain=None)
+	dbl = query_dns(domain+'.dbl.spamhaus.org', "A", nxdomain=None, retry=False)
 	if dbl is None:
 		output.print_ok("Domain is not blacklisted by dbl.spamhaus.org.")
 	elif dbl == "[timeout]":
@@ -773,7 +783,7 @@ def check_web_domain(domain, rounded_time, ssl_certificates, env, output):
 	# website for also needs a signed certificate.
 	check_ssl_cert(domain, rounded_time, ssl_certificates, env, output)
 
-def query_dns(qname, rtype, nxdomain='[Not Set]', at=None, as_list=False):
+def query_dns(qname, rtype, nxdomain='[Not Set]', at=None, as_list=False, retry=True):
 	# Make the qname absolute by appending a period. Without this, dns.resolver.query
 	# will fall back a failed lookup to a second query with this machine's hostname
 	# appended. This has been causing some false-positive Spamhaus reports. The
@@ -783,7 +793,7 @@ def query_dns(qname, rtype, nxdomain='[Not Set]', at=None, as_list=False):
 		qname += "."
 
 	# Use the default nameservers (as defined by the system, which is our locally
-	# running bind server), or if the 'at' argument is specified, use that host
+	# running unbound server), or if the 'at' argument is specified, use that host
 	# as the nameserver.
 	resolver = dns.resolver.get_default_resolver()
 	if at:
@@ -792,16 +802,29 @@ def query_dns(qname, rtype, nxdomain='[Not Set]', at=None, as_list=False):
 
 	# Set a timeout so that a non-responsive server doesn't hold us back.
 	resolver.timeout = 5
+	resolver.lifetime = 5
 
+	if retry:
+		tries = 2
+	else:
+		tries = 1
+	
 	# Do the query.
-	try:
-		response = resolver.resolve(qname, rtype)
-	except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
-		# Host did not have an answer for this query; not sure what the
-		# difference is between the two exceptions.
-		return nxdomain
-	except dns.exception.Timeout:
-		return "[timeout]"
+	while tries > 0:
+		tries = tries - 1
+		try:
+			response = resolver.resolve(qname, rtype, search=True)
+			tries = 0
+		except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
+			# Host did not have an answer for this query; not sure what the
+			# difference is between the two exceptions.
+			logging.debug("No result for dns lookup %s, %s (%d)", qname, rtype, tries)
+			if tries < 1:
+				return nxdomain
+		except dns.exception.Timeout:
+			logging.debug("Timeout on dns lookup %s, %s (%d)", qname, rtype, tries)
+			if tries < 1:
+				return "[timeout]"
 
 	# Normalize IP addresses. IP address --- especially IPv6 addresses --- can
 	# be expressed in equivalent string forms. Canonicalize the form before

management/templates/aliases.html

@@ -1,6 +1,6 @@
 <style>
 #alias_table .actions > * { padding-right: 3px; }
-#alias_table .alias-required .remove { display: none }
+#alias_table .alias-auto .actions > * { display: none }
 </style>
 
 <h2>Aliases</h2>
@@ -163,7 +163,7 @@ function show_aliases() {
           var n = $("#alias-template").clone();
           n.attr('id', '');
 
-          if (alias.required) n.addClass('alias-required');
+          if (alias.auto) n.addClass('alias-auto');
           n.attr('data-address', alias.address_display); // this is decoded from IDNA, but will get re-coded to IDNA on the backend
           n.find('td.address').text(alias.address_display)
           for (var j = 0; j < alias.forwards_to.length; j++)

management/templates/external-dns.html

@@ -38,7 +38,7 @@
 <p class="alert" role="alert">
 	<span class="glyphicon glyphicon-info-sign"></span>
 	You may encounter zone file errors when attempting to create a TXT record with a long string.
-	<a href="http://tools.ietf.org/html/rfc4408#section-3.1.3">RFC 4408</a> states a TXT record is allowed to contain multiple strings, and this technique can be used to construct records that would exceed the 255-byte maximum length.
+	<a href="https://tools.ietf.org/html/rfc4408#section-3.1.3">RFC 4408</a> states a TXT record is allowed to contain multiple strings, and this technique can be used to construct records that would exceed the 255-byte maximum length.
 	You may need to adopt this technique when adding DomainKeys. Use a tool like <code>named-checkzone</code> to validate your zone file.
 </p>
 

management/templates/index.html

@@ -62,6 +62,37 @@
 	    ol li {
 	       margin-bottom: 1em;
 	    }
+          
+            .if-logged-in { display: none; }
+            .if-logged-in-admin { display: none; }
+
+      /* The below only gets used if it is supported */
+      @media (prefers-color-scheme: dark) {
+        /* Invert invert lightness but not hue */
+        html {
+          filter: invert(100%) hue-rotate(180deg);
+        }
+
+        /* Set explicit background color (necessary for Firefox) */
+        html {
+          background-color: #111;
+        }
+          
+        /* Override Boostrap theme here to give more contrast. The black turns to white by the filter. */
+        .form-control {
+          color: black !important;
+        }          
+
+        /* Revert the invert for the navbar */
+        button, div.navbar {
+          filter: invert(100%) hue-rotate(180deg);
+        }
+          
+        /* Revert the revert for the dropdowns */
+        ul.dropdown-menu {
+          filter: invert(100%) hue-rotate(180deg);
+        }
+      }
         </style>
         <link rel="stylesheet" href="/admin/assets/bootstrap/css/bootstrap-theme.min.css">
     </head>
@@ -83,7 +114,7 @@
         </div>
         <div class="navbar-collapse collapse">
           <ul class="nav navbar-nav">
-            <li class="dropdown admin-links">
+            <li class="dropdown if-logged-in-admin">
               <a href="#" class="dropdown-toggle" data-toggle="dropdown">System <b class="caret"></b></a>
               <ul class="dropdown-menu">
                 <li><a href="#system_status" onclick="return show_panel(this);">Status Checks</a></li>
@@ -93,31 +124,36 @@
                 <li class="dropdown-header">Advanced Pages</li>
                 <li><a href="#custom_dns" onclick="return show_panel(this);">Custom DNS</a></li>
                 <li><a href="#external_dns" onclick="return show_panel(this);">External DNS</a></li>
-                <li><a href="/admin/munin" target="_blank">Munin Monitoring</a></li>
+                <li><a href="#munin" onclick="return show_panel(this);">Munin Monitoring</a></li>
               </ul>
             </li>
-            <li class="dropdown">
+            <li><a href="#mail-guide" onclick="return show_panel(this);" class="if-logged-in-not-admin">Mail</a></li>
+            <li class="dropdown if-logged-in-admin">
               <a href="#" class="dropdown-toggle" data-toggle="dropdown">Mail &amp; Users <b class="caret"></b></a>
               <ul class="dropdown-menu">
                 <li><a href="#mail-guide" onclick="return show_panel(this);">Instructions</a></li>
-                <li class="admin-links"><a href="#users" onclick="return show_panel(this);">Users</a></li>
-                <li class="admin-links"><a href="#aliases" onclick="return show_panel(this);">Aliases</a></li>
-                <li class="divider admin-links"></li>
-                <li class="dropdown-header admin-links">Your Account</li>
-                <li class="admin-links"><a href="#mfa" onclick="return show_panel(this);">Two-Factor Authentication</a></li>
+                <li><a href="#users" onclick="return show_panel(this);">Users</a></li>
+                <li><a href="#aliases" onclick="return show_panel(this);">Aliases</a></li>
+                <li class="divider"></li>
+                <li class="dropdown-header">Your Account</li>
+                <li><a href="#mfa" onclick="return show_panel(this);">Two-Factor Authentication</a></li>
               </ul>
             </li>
-            <li><a href="#sync_guide" onclick="return show_panel(this);">Contacts/Calendar</a></li>
-            <li class="admin-links"><a href="#web" onclick="return show_panel(this);">Web</a></li>
+            <li><a href="#sync_guide" onclick="return show_panel(this);" class="if-logged-in">Contacts/Calendar</a></li>
+            <li><a href="#web" onclick="return show_panel(this);" class="if-logged-in-admin">Web</a></li>
           </ul>
-          <ul class="admin-links nav navbar-nav navbar-right">
-            <li><a href="#" onclick="do_logout(); return false;" style="color: white">Log out</a></li>
+          <ul class="nav navbar-nav navbar-right">
+            <li class="if-logged-in"><a href="#" onclick="do_logout(); return false;" style="color: white">Log out</a></li>
           </ul>
         </div><!--/.navbar-collapse -->
       </div>
     </div>
 
     <div class="container">
+      <div id="panel_welcome" class="admin_panel">
+      {% include "welcome.html" %}
+      </div>
+
       <div id="panel_system_status" class="admin_panel">
       {% include "system-status.html" %}
       </div>
@@ -166,6 +202,10 @@
       {% include "ssl.html" %}
       </div>
 
+      <div id="panel_munin" class="admin_panel">
+      {% include "munin.html" %}
+      </div>
+
       <hr>
 
       <footer>
@@ -298,7 +338,7 @@ function ajax_with_indicator(options) {
   return false; // handy when called from onclick
 }
 
-var api_credentials = ["", ""];
+var api_credentials = null;
 function api(url, method, data, callback, callback_error, headers) {
   // from http://www.webtoolkit.info/javascript-base64.html
   function base64encode(input) {
@@ -346,9 +386,10 @@ function api(url, method, data, callback, callback_error, headers) {
       // We don't store user credentials in a cookie to avoid the hassle of CSRF
       // attacks. The Authorization header only gets set in our AJAX calls triggered
       // by user actions.
-      xhr.setRequestHeader(
-        'Authorization',
-        'Basic ' + base64encode(api_credentials[0] + ':' + api_credentials[1]));
+      if (api_credentials)
+        xhr.setRequestHeader(
+          'Authorization',
+          'Basic ' + base64encode(api_credentials.username + ':' + api_credentials.session_key));
     },
     success: callback,
     error: callback_error || default_error,
@@ -367,12 +408,21 @@ var current_panel = null;
 var switch_back_to_panel = null;
 
 function do_logout() {
-  api_credentials = ["", ""];
+  // Clear the session from the backend.
+  api("/logout", "POST");
+
+  // Forget the token.
+  api_credentials = null;
   if (typeof localStorage != 'undefined')
     localStorage.removeItem("miab-cp-credentials");
   if (typeof sessionStorage != 'undefined')
     sessionStorage.removeItem("miab-cp-credentials");
+
+  // Return to the start.
   show_panel('login');
+
+  // Reset menus.
+  show_hide_menus();
 }
 
 function show_panel(panelid) {
@@ -395,21 +445,22 @@ function show_panel(panelid) {
 
 $(function() {
   // Recall saved user credentials.
-  if (typeof sessionStorage != 'undefined' && sessionStorage.getItem("miab-cp-credentials"))
-    api_credentials = sessionStorage.getItem("miab-cp-credentials").split(":");
-  else if (typeof localStorage != 'undefined' && localStorage.getItem("miab-cp-credentials"))
-    api_credentials = localStorage.getItem("miab-cp-credentials").split(":");
+  try {
+    if (typeof sessionStorage != 'undefined' && sessionStorage.getItem("miab-cp-credentials"))
+      api_credentials = JSON.parse(sessionStorage.getItem("miab-cp-credentials"));
+    else if (typeof localStorage != 'undefined' && localStorage.getItem("miab-cp-credentials"))
+      api_credentials = JSON.parse(localStorage.getItem("miab-cp-credentials"));
+  } catch (_) {
+  }
+
+  // Toggle menu state.
+  show_hide_menus();
 
-  if (!api_credentials[0] && !api_credentials[1]) {
-    $('.admin-links').hide()
-  }
-  else {
-    $('.admin-links').show()
-  }
-  
   // Recall what the user was last looking at.
-  if (typeof localStorage != 'undefined' && localStorage.getItem("miab-cp-lastpanel")) {
+  if (api_credentials != null && typeof localStorage != 'undefined' && localStorage.getItem("miab-cp-lastpanel")) {
     show_panel(localStorage.getItem("miab-cp-lastpanel"));
+  } else if (api_credentials != null) {
+    show_panel('welcome');
   } else {
     show_panel('login');
   }

management/templates/login.html

@@ -64,7 +64,7 @@ sudo management/cli.py user make-admin me@{{hostname}}</pre>
     <div class="form-group" id="loginOtp">
       <label for="loginOtpInput" class="col-sm-3 control-label">Code</label>
       <div class="col-sm-9">
-          <input type="text" class="form-control" id="loginOtpInput" placeholder="6-digit code">
+          <input type="text" class="form-control" id="loginOtpInput" placeholder="6-digit code" autocomplete="off">
           <div class="help-block" style="margin-top: 5px; font-size: 90%">Enter the six-digit code generated by your two factor authentication app.</div>
       </div>
     </div>
@@ -102,11 +102,11 @@ function do_login() {
   }
 
   // Exchange the email address & password for an API key.
-  api_credentials = [$('#loginEmail').val(), $('#loginPassword').val()]
+  api_credentials = { username: $('#loginEmail').val(), session_key: $('#loginPassword').val() }
 
   api(
-  "/me",
-  "GET",
+  "/login",
+  "POST",
   {},
   function(response) {
     // This API call always succeeds. It returns a JSON object indicating
@@ -141,7 +141,9 @@ function do_login() {
       // Login succeeded.
 
       // Save the new credentials.
-      api_credentials = [response.email, response.api_key];
+      api_credentials = { username: response.email,
+                          session_key: response.api_key,
+                          privileges: response.privileges };
 
       // Try to wipe the username/password information.
       $('#loginEmail').val('');
@@ -152,18 +154,21 @@ function do_login() {
       // Remember the credentials.
       if (typeof localStorage != 'undefined' && typeof sessionStorage != 'undefined') {
         if ($('#loginRemember').val()) {
-          localStorage.setItem("miab-cp-credentials", api_credentials.join(":"));
+          localStorage.setItem("miab-cp-credentials", JSON.stringify(api_credentials));
           sessionStorage.removeItem("miab-cp-credentials");
         } else {
           localStorage.removeItem("miab-cp-credentials");
-          sessionStorage.setItem("miab-cp-credentials", api_credentials.join(":"));
+          sessionStorage.setItem("miab-cp-credentials", JSON.stringify(api_credentials));
         }
       }
 
+      // Toggle menus.
+      show_hide_menus();
+
       // Open the next panel the user wants to go to. Do this after the XHR response
       // is over so that we don't start a new XHR request while this one is finishing,
       // which confuses the loading indicator.
-      setTimeout(function() { show_panel(!switch_back_to_panel || switch_back_to_panel == "login" ? 'system_status' : switch_back_to_panel) }, 300);
+      setTimeout(function() { show_panel(!switch_back_to_panel || switch_back_to_panel == "login" ? 'welcome' : switch_back_to_panel) }, 300);
     }
   },
   undefined,
@@ -183,4 +188,19 @@ function show_login() {
     }
   });
 }
+
+function show_hide_menus() {
+  var is_logged_in = (api_credentials != null);
+  var privs = api_credentials ? api_credentials.privileges : [];
+  $('.if-logged-in').toggle(is_logged_in);
+  $('.if-logged-in-admin, .if-logged-in-not-admin').toggle(false);
+  if (is_logged_in) {
+    $('.if-logged-in-not-admin').toggle(true);
+    privs.forEach(function(priv) {
+      $('.if-logged-in-' + priv).toggle(true);
+      $('.if-logged-in-not-' + priv).toggle(false);
+    });
+  }
+  $('.if-not-logged-in').toggle(!is_logged_in);
+}
 </script>

management/templates/munin.html

@@ -0,0 +1,20 @@
+<h2>Munin Monitoring</h2>
+
+<style>
+</style>
+
+<p>Opening munin in a new tab... You may need to allow pop-ups for this site.</p>
+
+<script>
+function show_munin() {
+  // Set the cookie.
+  api(
+    "/munin",
+    "GET",
+    { },
+    function(r) {
+      // Redirect.
+      window.open("/admin/munin/index.html", "_blank");
+    });
+}
+</script>

management/templates/sync-guide.html

@@ -30,9 +30,9 @@
 
 			<table class="table">
 			<thead><tr><th>For...</th> <th>Use...</th></tr></thead>
-			<tr><td>Contacts and Calendar</td> <td><a href="https://play.google.com/store/apps/details?id=at.bitfire.davdroid">DAVdroid</a> ($3.69; free <a href="https://f-droid.org/packages/at.bitfire.davdroid/">here</a>)</td></tr>
-			<tr><td>Only Contacts</td> <td><a href="https://play.google.com/store/apps/details?id=org.dmfs.carddav.sync">CardDAV-Sync free beta</a> (free)</td></tr>
-			<tr><td>Only Calendar</td> <td><a href="https://play.google.com/store/apps/details?id=org.dmfs.caldav.lib">CalDAV-Sync</a> ($2.89)</td></tr>
+			<tr><td>Contacts and Calendar</td> <td><a href="https://play.google.com/store/apps/details?id=at.bitfire.davdroid">DAVx⁵</a> ($5.99; free <a href="https://f-droid.org/packages/at.bitfire.davdroid/">here</a>)</td></tr>
+			<tr><td>Only Contacts</td> <td><a href="https://play.google.com/store/apps/details?id=org.dmfs.carddav.sync">CardDAV-Sync free</a> (free)</td></tr>
+			<tr><td>Only Calendar</td> <td><a href="https://play.google.com/store/apps/details?id=org.dmfs.caldav.lib">CalDAV-Sync</a> ($2.99)</td></tr>
 			</table>
 
 			<p>Use the following settings:</p>

management/templates/system-backup.html

@@ -5,7 +5,7 @@
 
 <h2>Backup Status</h2>
 
-<p>The box makes an incremental backup each night. By default the backup is stored on the machine itself, but you can also store in on S3-compatible services like Amazon Web Services (AWS).</p>
+<p>The box makes an incremental backup each night. By default the backup is stored on the machine itself, but you can also store it on S3-compatible services like Amazon Web Services (AWS).</p>
 
 <h3>Configuration</h3>
 
@@ -138,7 +138,7 @@
       </div>
   </div>
   <!-- Common -->
-  <div class="form-group backup-target-local backup-target-rsync backup-target-s3">
+  <div class="form-group backup-target-local backup-target-rsync backup-target-s3 backup-target-b2">
     <label for="min-age" class="col-sm-2 control-label">Retention Days:</label>
     <div class="col-sm-8">
       <input type="number" class="form-control" rows="1" id="min-age">

management/templates/users.html

@@ -203,7 +203,7 @@ function users_set_password(elem) {
   var email = $(elem).parents('tr').attr('data-email');
 
   var yourpw = "";
-  if (api_credentials != null && email == api_credentials[0])
+  if (api_credentials != null && email == api_credentials.username)
     yourpw = "<p class='text-danger'>If you change your own password, you will be logged out of this control panel and will need to log in again.</p>";
 
   show_modal_confirm(
@@ -232,7 +232,7 @@ function users_remove(elem) {
   var email = $(elem).parents('tr').attr('data-email');
 
   // can't remove yourself
-  if (api_credentials != null && email == api_credentials[0]) {
+  if (api_credentials != null && email == api_credentials.username) {
     show_modal_error("Archive User", "You cannot archive your own account.");
     return;
   }
@@ -264,7 +264,7 @@ function mod_priv(elem, add_remove) {
   var priv = $(elem).parents('td').find('.name').text();
 
   // can't remove your own admin access
-  if (priv == "admin" && add_remove == "remove" && api_credentials != null && email == api_credentials[0]) {
+  if (priv == "admin" && add_remove == "remove" && api_credentials != null && email == api_credentials.username) {
     show_modal_error("Modify Privileges", "You cannot remove the admin privilege from yourself.");
     return;
   }

management/templates/welcome.html

@@ -0,0 +1,16 @@
+<style>
+  .title {
+    margin: 1em;
+    text-align: center;
+  }
+
+  .subtitle {
+    margin: 2em;
+    text-align: center;
+  }
+</style>
+
+<h1 class="title">{{hostname}}</h1>
+
+<p class="subtitle">Welcome to your Mail-in-a-Box control panel.</p>
+

management/utils.py

@@ -106,7 +106,7 @@ def sort_email_addresses(email_addresses, env):
     ret.extend(sorted(email_addresses)) # whatever is left
     return ret
 
-def shell(method, cmd_args, env={}, capture_stderr=False, return_bytes=False, trap=False, input=None):
+def shell(method, cmd_args, env={}, capture_stdout=True, capture_stderr=False, return_bytes=False, trap=False, input=None):
     # A safe way to execute processes.
     # Some processes like apt-get require being given a sane PATH.
     import subprocess
@@ -116,6 +116,8 @@ def shell(method, cmd_args, env={}, capture_stderr=False, return_bytes=False, tr
         'env': env,
         'stderr': None if not capture_stderr else subprocess.STDOUT,
     }
+    if not capture_stdout:
+        kwargs['stdout'] = subprocess.DEVNULL
     if method == "check_output" and input is not None:
         kwargs['input'] = input
 

management/web_update.py

@@ -211,9 +211,14 @@ def make_domain_config(domain, templates, ssl_certificates, env):
 
 	# Add the HSTS header.
 	if hsts == "yes":
-		nginx_conf_extra += "\tadd_header Strict-Transport-Security \"max-age=15768000\" always;\n"
+		nginx_conf_extra += "\tadd_header Strict-Transport-Security \"max-age=31536000; includeSubDomains\" always;\n"
 	elif hsts == "preload":
-		nginx_conf_extra += "\tadd_header Strict-Transport-Security \"max-age=15768000; includeSubDomains; preload\" always;\n"
+		nginx_conf_extra += "\tadd_header Strict-Transport-Security \"max-age=31536000; includeSubDomains; preload\" always;\n"
+		
+	nginx_conf_extra += "\tadd_header X-Frame-Options \"SAMEORIGIN\" always;\n"
+	nginx_conf_extra += "\tadd_header X-Content-Type-Options nosniff;\n"
+	nginx_conf_extra += "\tadd_header Content-Security-Policy-Report-Only \"default-src 'self'; font-src *;img-src * data:; script-src *; style-src *;frame-ancestors 'self'\";\n"
+	nginx_conf_extra += "\tadd_header Referrer-Policy \"strict-origin\";\n"
 
 	# Add in any user customizations in the includes/ folder.
 	nginx_conf_custom_include = os.path.join(env["STORAGE_ROOT"], "www", safe_domain_name(domain) + ".conf")