Remove ssl stapling from nginx configuration (#2520)

fix: MissingContentLength error in boto3 version 1.36.1 and up

conf/mozilla-autoconfig.xml

@@ -1,7 +1,7 @@
 <?xml version="1.0"?>
 <clientConfig version="1.1">
     <emailProvider id="PRIMARY_HOSTNAME">
-      <domain>PRIMARY_HOSTNAME</domain>
+      <domain purpose="mx">PRIMARY_HOSTNAME</domain>
 
       <displayName>PRIMARY_HOSTNAME (Mail-in-a-Box)</displayName>
       <displayShortName>PRIMARY_HOSTNAME</displayShortName>
@@ -14,6 +14,14 @@
          <authentication>password-cleartext</authentication>
       </incomingServer>
 
+      <incomingServer type="pop3">
+         <hostname>PRIMARY_HOSTNAME</hostname>
+         <port>995</port>
+         <socketType>SSL</socketType>
+         <username>%EMAILADDRESS%</username>
+         <authentication>password-cleartext</authentication>
+      </incomingServer>
+
       <outgoingServer type="smtp">
          <hostname>PRIMARY_HOSTNAME</hostname>
          <port>465</port>
@@ -29,6 +37,20 @@
       </documentation>
     </emailProvider>
 
+    <addressbook type="carddav">
+      <username>%EMAILADDRESS%</username>
+      <authentication system="http">basic</authentication>
+      <!-- Redirects to: https://PRIMARY_HOSTNAME/cloud/remote.php/carddav/ -->
+      <url>https://PRIMARY_HOSTNAME/.well-known/carddav</url>
+    </addressbook>
+
+    <calendar type="caldav">
+      <username>%EMAILADDRESS%</username>
+      <authentication system="http">basic</authentication>
+      <!-- Redirects to: https://PRIMARY_HOSTNAME/cloud/remote.php/caldav/ -->
+      <url>https://PRIMARY_HOSTNAME/.well-known/caldav</url>
+    </calendar>
+
     <webMail>
       <loginPage url="https://PRIMARY_HOSTNAME/mail/" />
       <loginPageInfo url="https://PRIMARY_HOSTNAME/mail/" >

conf/nginx-ssl.conf

@@ -12,8 +12,6 @@ ssl_session_timeout 1d;
 # nginx 1.5.9+ ONLY
 ssl_buffer_size 1400; 
 
-ssl_stapling on;
-ssl_stapling_verify on;
 resolver 127.0.0.1 valid=86400;
 resolver_timeout 10;
 

management/backup.py

@@ -14,6 +14,7 @@ import rtyaml
 from exclusiveprocess import Lock
 
 from utils import load_environment, shell, wait_for_service
+import operator
 
 def backup_status(env):
 	# If backups are disabled, return no status.
@@ -91,7 +92,7 @@ def backup_status(env):
 
 	# Ensure the rows are sorted reverse chronologically.
 	# This is relied on by should_force_full() and the next step.
-	backups = sorted(backups.values(), key = lambda b : b["date"], reverse=True)
+	backups = sorted(backups.values(), key = operator.itemgetter("date"), reverse=True)
 
 	# Get the average size of incremental backups, the size of the
 	# most recent full backup, and the date of the most recent
@@ -177,9 +178,7 @@ def should_force_full(config, env):
 				if dateutil.parser.parse(bak["date"]) + datetime.timedelta(days=config["min_age_in_days"]*10+1) < datetime.datetime.now(dateutil.tz.tzlocal()):
 					return True
 			return False
-	else:
 	# If we got here there are no (full) backups, so make one.
-		# (I love for/else blocks. Here it's just to show off.)
 	return True
 
 def get_passphrase(env):
@@ -236,7 +235,7 @@ def get_duplicity_additional_args(env):
 			f"--ssh-options='-i /root/.ssh/id_rsa_miab -p {port}'",
 			f"--rsync-options='-e \"/usr/bin/ssh -oStrictHostKeyChecking=no -oBatchMode=yes -p {port} -i /root/.ssh/id_rsa_miab\"'",
 		]
-	elif get_target_type(config) == 's3':
+	if get_target_type(config) == 's3':
 		# See note about hostname in get_duplicity_target_url.
 		# The region name, which is required by some non-AWS endpoints,
 		# is saved inside the username portion of the URL.
@@ -258,6 +257,8 @@ def get_duplicity_env_vars(env):
 	if get_target_type(config) == 's3':
 		env["AWS_ACCESS_KEY_ID"] = config["target_user"]
 		env["AWS_SECRET_ACCESS_KEY"] = config["target_pass"]
+		env["AWS_REQUEST_CHECKSUM_CALCULATION"] = "WHEN_REQUIRED"
+		env["AWS_RESPONSE_CHECKSUM_VALIDATION"] = "WHEN_REQUIRED"
 
 	return env
 
@@ -447,7 +448,7 @@ def list_target_files(config):
 	if target.scheme == "file":
 		return [(fn, os.path.getsize(os.path.join(target.path, fn))) for fn in os.listdir(target.path)]
 
-	elif target.scheme == "rsync":
+	if target.scheme == "rsync":
 		rsync_fn_size_re = re.compile(r'.*    ([^ ]*) [^ ]* [^ ]* (.*)')
 		rsync_target = '{host}:{path}'
 
@@ -463,9 +464,8 @@ def list_target_files(config):
 
 		target_path = target.path
 		if not target_path.endswith('/'):
-			target_path = target_path + '/'
-		if target_path.startswith('/'):
-			target_path = target_path[1:]
+			target_path += '/'
+		target_path = target_path.removeprefix('/')
 
 		rsync_command = [ 'rsync',
 					'-e',
@@ -485,7 +485,6 @@ def list_target_files(config):
 				if match:
 					ret.append( (match.groups()[1], int(match.groups()[0].replace(',',''))) )
 			return ret
-		else:
 		if 'Permission denied (publickey).' in listing:
 			reason = "Invalid user or check you correctly copied the SSH key."
 		elif 'No such file or directory' in listing:
@@ -501,7 +500,7 @@ def list_target_files(config):
 		msg = f"Connection to rsync host failed: {reason}"
 		raise ValueError(msg)
 
-	elif target.scheme == "s3":
+	if target.scheme == "s3":
 		import boto3.s3
 		from botocore.exceptions import ClientError
 
@@ -519,6 +518,9 @@ def list_target_files(config):
 
 		# connect to the region & bucket
 		try:
+			if config['target_user'] == "" and config['target_pass'] == "":
+				s3 = boto3.client('s3', endpoint_url=f'https://{target.hostname}')
+			else:
 				s3 = boto3.client('s3', \
 					endpoint_url=f'https://{target.hostname}', \
 					aws_access_key_id=config['target_user'], \
@@ -528,7 +530,7 @@ def list_target_files(config):
 		except ClientError as e:
 			raise ValueError(e)
 		return backup_list
-	elif target.scheme == 'b2':
+	if target.scheme == 'b2':
 		from b2sdk.v1 import InMemoryAccountInfo, B2Api
 		from b2sdk.v1.exception import NonExistentBucket
 		info = InMemoryAccountInfo()
@@ -547,7 +549,6 @@ def list_target_files(config):
 			raise ValueError(msg)
 		return [(key.file_name, key.size) for key, _ in bucket.ls()]
 
-	else:
 	raise ValueError(config["target"])
 
 
@@ -602,8 +603,7 @@ def get_backup_config(env, for_save=False, for_ui=False):
 	# authentication details. The user will have to re-enter it.
 	if for_ui:
 		for field in ("target_user", "target_pass"):
-			if field in config:
-				del config[field]
+			config.pop(field, None)
 
 	# helper fields for the admin
 	config["file_target_directory"] = os.path.join(backup_root, 'encrypted')

management/cli.py

@@ -65,6 +65,7 @@ if len(sys.argv) < 2:
   {cli} user password user@domain.com [password]
   {cli} user remove user@domain.com
   {cli} user make-admin user@domain.com
+  {cli} user quota user@domain [new-quota]       (get or set user quota)
   {cli} user remove-admin user@domain.com
   {cli} user admins                              (lists admins)
   {cli} user mfa show user@domain.com            (shows MFA devices for user, if any)
@@ -117,6 +118,14 @@ elif sys.argv[1] == "user" and sys.argv[2] == "admins":
 			if "admin" in user['privileges']:
 				print(user['email'])
 
+elif sys.argv[1] == "user" and sys.argv[2] == "quota" and len(sys.argv) == 4:
+	# Get a user's quota
+	print(mgmt(f"/mail/users/quota?text=1&email={sys.argv[3]}"))
+
+elif sys.argv[1] == "user" and sys.argv[2] == "quota" and len(sys.argv) == 5:
+	# Set a user's quota
+	users = mgmt("/mail/users/quota", { "email": sys.argv[3], "quota": sys.argv[4] })
+
 elif sys.argv[1] == "user" and len(sys.argv) == 5 and sys.argv[2:4] == ["mfa", "show"]:
 	# Show MFA status for a user.
 	status = mgmt("/mfa/status", { "user": sys.argv[4] }, is_json=True)
@@ -141,4 +150,3 @@ elif sys.argv[1] == "alias" and sys.argv[2] == "remove" and len(sys.argv) == 4:
 else:
 	print("Invalid command-line arguments.")
 	sys.exit(1)
-

management/daemon.py

@@ -21,6 +21,7 @@ import auth, utils
 from mailconfig import get_mail_users, get_mail_users_ex, get_admins, add_mail_user, set_mail_password, remove_mail_user
 from mailconfig import get_mail_user_privileges, add_remove_mail_user_privilege
 from mailconfig import get_mail_aliases, get_mail_aliases_ex, get_mail_domains, add_mail_alias, remove_mail_alias
+from mailconfig import get_mail_quota, set_mail_quota
 from mfa import get_public_mfa_state, provision_totp, validate_totp_secret, enable_mfa, disable_mfa
 import contextlib
 
@@ -92,7 +93,6 @@ def authorized_personnel_only(viewfunc):
 		if request.headers.get('Accept') in {None, "", "*/*"}:
 			# Return plain text output.
 			return Response(error+"\n", status=status, mimetype='text/plain', headers=headers)
-		else:
 		# Return JSON output.
 		return Response(json.dumps({
 			"status": "error",
@@ -147,7 +147,6 @@ def login():
 				"status": "missing-totp-token",
 				"reason": str(e),
 			})
-		else:
 		# Log the failed login
 		log_failed_login(request)
 		return json_response({
@@ -163,7 +162,7 @@ def login():
 		"api_key": auth_service.create_session_key(email, env, type='login'),
 	}
 
-	app.logger.info(f"New login session created for {email}")
+	app.logger.info("New login session created for %s", email)
 
 	# Return.
 	return json_response(resp)
@@ -172,7 +171,7 @@ def login():
 def logout():
 	try:
 		email, _ = auth_service.authenticate(request, env, logout=True)
-		app.logger.info(f"{email} logged out")
+		app.logger.info("%s logged out", email)
 	except ValueError:
 		pass
 	finally:
@@ -185,14 +184,36 @@ def logout():
 def mail_users():
 	if request.args.get("format", "") == "json":
 		return json_response(get_mail_users_ex(env, with_archived=True))
-	else:
 	return "".join(x+"\n" for x in get_mail_users(env))
 
 @app.route('/mail/users/add', methods=['POST'])
 @authorized_personnel_only
 def mail_users_add():
+	quota = request.form.get('quota', '0')
 	try:
-		return add_mail_user(request.form.get('email', ''), request.form.get('password', ''), request.form.get('privileges', ''), env)
+		return add_mail_user(request.form.get('email', ''), request.form.get('password', ''), request.form.get('privileges', ''), quota, env)
+	except ValueError as e:
+		return (str(e), 400)
+
+@app.route('/mail/users/quota', methods=['GET'])
+@authorized_personnel_only
+def get_mail_users_quota():
+	email = request.values.get('email', '')
+	quota = get_mail_quota(email, env)
+
+	if request.values.get('text'):
+		return quota
+
+	return json_response({
+		"email": email,
+		"quota": quota
+	})
+
+@app.route('/mail/users/quota', methods=['POST'])
+@authorized_personnel_only
+def mail_users_quota():
+	try:
+		return set_mail_quota(request.form.get('email', ''), request.form.get('quota'), env)
 	except ValueError as e:
 		return (str(e), 400)
 
@@ -233,7 +254,6 @@ def mail_user_privs_remove():
 def mail_aliases():
 	if request.args.get("format", "") == "json":
 		return json_response(get_mail_aliases_ex(env))
-	else:
 	return "".join(address+"\t"+receivers+"\t"+(senders or "")+"\n" for address, receivers, senders, auto in get_mail_aliases(env))
 
 @app.route('/mail/aliases/add', methods=['POST'])
@@ -354,7 +374,7 @@ def dns_set_record(qname, rtype="A"):
 			# Get the existing records matching the qname and rtype.
 			return dns_get_records(qname, rtype)
 
-		elif request.method in {"POST", "PUT"}:
+		if request.method in {"POST", "PUT"}:
 			# There is a default value for A/AAAA records.
 			if rtype in {"A", "AAAA"} and value == "":
 				value = request.environ.get("HTTP_X_FORWARDED_FOR") # normally REMOTE_ADDR but we're behind nginx as a reverse proxy
@@ -512,7 +532,7 @@ def totp_post_disable():
 		return (str(e), 400)
 	if result: # success
 		return "OK"
-	else: # error
+	# error
 	return ("Invalid user or MFA id.", 400)
 
 # WEB
@@ -597,7 +617,6 @@ def needs_reboot():
 	from status_checks import is_reboot_needed_due_to_package_installation
 	if is_reboot_needed_due_to_package_installation():
 		return json_response(True)
-	else:
 	return json_response(False)
 
 @app.route('/system/reboot', methods=["POST"])
@@ -607,7 +626,6 @@ def do_reboot():
 	from status_checks import is_reboot_needed_due_to_package_installation
 	if is_reboot_needed_due_to_package_installation():
 		return utils.shell("check_output", ["/sbin/shutdown", "-r", "now"], capture_stderr=True)
-	else:
 	return "No reboot is required, so it is not allowed."
 
 
@@ -670,8 +688,7 @@ def check_request_cookie_for_admin_access():
 	if not session: return False
 	privs = get_mail_user_privileges(session["email"], env)
 	if not isinstance(privs, list): return False
-	if "admin" not in privs: return False
-	return True
+	return "admin" in privs
 
 def authorized_personnel_only_via_cookie(f):
 	@wraps(f)
@@ -719,7 +736,7 @@ def munin_cgi(filename):
 
 	query_str = request.query_string.decode("utf-8", 'ignore')
 
-	env = {'PATH_INFO': '/%s/' % filename, 'REQUEST_METHOD': 'GET', 'QUERY_STRING': query_str}
+	env = {'PATH_INFO': f'/{filename}/', 'REQUEST_METHOD': 'GET', 'QUERY_STRING': query_str}
 	code, binout = utils.shell('check_output',
 							   COMMAND.split(" ", 5),
 							   # Using a maxsplit of 5 keeps the last arguments together
@@ -753,7 +770,7 @@ def log_failed_login(request):
 
 	# We need to add a timestamp to the log message, otherwise /dev/log will eat the "duplicate"
 	# message.
-	app.logger.warning( f"Mail-in-a-Box Management Daemon: Failed login attempt from ip {ip} - timestamp {time.time()}")
+	app.logger.warning("Mail-in-a-Box Management Daemon: Failed login attempt from ip %s - timestamp %s", ip, time.time())
 
 
 # APP

management/dns_update.py

@@ -11,7 +11,6 @@ import dns.resolver
 
 from utils import shell, load_env_vars_from_file, safe_domain_name, sort_domains, get_ssh_port
 from ssl_certificates import get_ssl_certificates, check_certificate
-import contextlib
 
 # From https://stackoverflow.com/questions/3026957/how-to-validate-a-domain-name-using-regex-php/16491074#16491074
 # This regular expression matches domain names according to RFCs, it also accepts fqdn with an leading dot,
@@ -124,7 +123,6 @@ def do_dns_update(env, force=False):
 	if len(updated_domains) == 0:
 		# if nothing was updated (except maybe OpenDKIM's files), don't show any output
 		return ""
-	else:
 	return "updated DNS: " + ",".join(updated_domains) + "\n"
 
 ########################################################################
@@ -187,7 +185,7 @@ def build_zone(domain, domain_properties, additional_records, env, is_zone=True)
 	# is managed outside of the box.
 	if is_zone:
 		# Obligatory NS record to ns1.PRIMARY_HOSTNAME.
-		records.append((None,  "NS",  "ns1.%s." % env["PRIMARY_HOSTNAME"], False))
+		records.append((None,  "NS",  "ns1.{}.".format(env["PRIMARY_HOSTNAME"]), False))
 
 		# NS record to ns2.PRIMARY_HOSTNAME or whatever the user overrides.
 		# User may provide one or more additional nameservers
@@ -254,16 +252,16 @@ def build_zone(domain, domain_properties, additional_records, env, is_zone=True)
 	# was set. So set has_rec_base to a clone of the current set of DNS settings, and don't update
 	# during this process.
 	has_rec_base = list(records)
-	a_expl = "Required. May have a different value. Sets the IP address that %s resolves to for web hosting and other services besides mail. The A record must be present but its value does not affect mail delivery." % domain
+	a_expl = f"Required. May have a different value. Sets the IP address that {domain} resolves to for web hosting and other services besides mail. The A record must be present but its value does not affect mail delivery."
 	if domain_properties[domain]["auto"]:
 		if domain.startswith(("ns1.", "ns2.")): a_expl = False # omit from 'External DNS' page since this only applies if box is its own DNS server
-		if domain.startswith("www."): a_expl = "Optional. Sets the IP address that %s resolves to so that the box can provide a redirect to the parent domain." % domain
+		if domain.startswith("www."): a_expl = f"Optional. Sets the IP address that {domain} resolves to so that the box can provide a redirect to the parent domain."
 		if domain.startswith("mta-sts."): a_expl = "Optional. MTA-STS Policy Host serving /.well-known/mta-sts.txt."
 		if domain.startswith("autoconfig."): a_expl = "Provides email configuration autodiscovery support for Thunderbird Autoconfig."
 		if domain.startswith("autodiscover."): a_expl = "Provides email configuration autodiscovery support for Z-Push ActiveSync Autodiscover."
 	defaults = [
 		(None,  "A",    env["PUBLIC_IP"], a_expl),
-		(None,  "AAAA", env.get('PUBLIC_IPV6'), "Optional. Sets the IPv6 address that %s resolves to, e.g. for web hosting. (It is not necessary for receiving mail on this domain.)" % domain),
+		(None,  "AAAA", env.get('PUBLIC_IPV6'), f"Optional. Sets the IPv6 address that {domain} resolves to, e.g. for web hosting. (It is not necessary for receiving mail on this domain.)"),
 	]
 	for qname, rtype, value, explanation in defaults:
 		if value is None or value.strip() == "": continue # skip IPV6 if not set
@@ -281,13 +279,13 @@ def build_zone(domain, domain_properties, additional_records, env, is_zone=True)
 	if domain_properties[domain]["mail"]:
 		# The MX record says where email for the domain should be delivered: Here!
 		if not has_rec(None, "MX", prefix="10 "):
-			records.append((None,  "MX",  "10 %s." % env["PRIMARY_HOSTNAME"], "Required. Specifies the hostname (and priority) of the machine that handles @%s mail." % domain))
+			records.append((None,  "MX",  "10 {}.".format(env["PRIMARY_HOSTNAME"]), f"Required. Specifies the hostname (and priority) of the machine that handles @{domain} mail."))
 
 		# SPF record: Permit the box ('mx', see above) to send mail on behalf of
 		# the domain, and no one else.
 		# Skip if the user has set a custom SPF record.
 		if not has_rec(None, "TXT", prefix="v=spf1 "):
-			records.append((None,  "TXT", 'v=spf1 mx -all', "Recommended. Specifies that only the box is permitted to send @%s mail." % domain))
+			records.append((None,  "TXT", 'v=spf1 mx -all', f"Recommended. Specifies that only the box is permitted to send @{domain} mail."))
 
 		# Append the DKIM TXT record to the zone as generated by OpenDKIM.
 		# Skip if the user has set a DKIM record already.
@@ -296,12 +294,12 @@ def build_zone(domain, domain_properties, additional_records, env, is_zone=True)
 			m = re.match(r'(\S+)\s+IN\s+TXT\s+\( ((?:"[^"]+"\s+)+)\)', orf.read(), re.S)
 			val = "".join(re.findall(r'"([^"]+)"', m.group(2)))
 			if not has_rec(m.group(1), "TXT", prefix="v=DKIM1; "):
-				records.append((m.group(1), "TXT", val, "Recommended. Provides a way for recipients to verify that this machine sent @%s mail." % domain))
+				records.append((m.group(1), "TXT", val, f"Recommended. Provides a way for recipients to verify that this machine sent @{domain} mail."))
 
 		# Append a DMARC record.
 		# Skip if the user has set a DMARC record already.
 		if not has_rec("_dmarc", "TXT", prefix="v=DMARC1; "):
-			records.append(("_dmarc", "TXT", 'v=DMARC1; p=quarantine;', "Recommended. Specifies that mail that does not originate from the box but claims to be from @%s or which does not have a valid DKIM signature is suspect and should be quarantined by the recipient's mail system." % domain))
+			records.append(("_dmarc", "TXT", 'v=DMARC1; p=quarantine;', f"Recommended. Specifies that mail that does not originate from the box but claims to be from @{domain} or which does not have a valid DKIM signature is suspect and should be quarantined by the recipient's mail system."))
 
 	if domain_properties[domain]["user"]:
 		# Add CardDAV/CalDAV SRV records on the non-primary hostname that points to the primary hostname
@@ -364,9 +362,9 @@ def build_zone(domain, domain_properties, additional_records, env, is_zone=True)
 			# Mark this domain as not sending mail with hard-fail SPF and DMARC records.
 			d = (qname+"." if qname else "") + domain
 			if not has_rec(qname, "TXT", prefix="v=spf1 "):
-				records.append((qname,  "TXT", 'v=spf1 -all', "Recommended. Prevents use of this domain name for outbound mail by specifying that no servers are valid sources for mail from @%s. If you do send email from this domain name you should either override this record such that the SPF rule does allow the originating server, or, take the recommended approach and have the box handle mail for this domain (simply add any receiving alias at this domain name to make this machine treat the domain name as one of its mail domains)." % d))
+				records.append((qname,  "TXT", 'v=spf1 -all', f"Recommended. Prevents use of this domain name for outbound mail by specifying that no servers are valid sources for mail from @{d}. If you do send email from this domain name you should either override this record such that the SPF rule does allow the originating server, or, take the recommended approach and have the box handle mail for this domain (simply add any receiving alias at this domain name to make this machine treat the domain name as one of its mail domains)."))
 			if not has_rec("_dmarc" + ("."+qname if qname else ""), "TXT", prefix="v=DMARC1; "):
-				records.append(("_dmarc" + ("."+qname if qname else ""), "TXT", 'v=DMARC1; p=reject;', "Recommended. Prevents use of this domain name for outbound mail by specifying that the SPF rule should be honoured for mail from @%s." % d))
+				records.append(("_dmarc" + ("."+qname if qname else ""), "TXT", 'v=DMARC1; p=reject;', f"Recommended. Prevents use of this domain name for outbound mail by specifying that the SPF rule should be honoured for mail from @{d}."))
 
 			# And with a null MX record (https://explained-from-first-principles.com/email/#null-mx-record)
 			if not has_rec(qname, "MX"):
@@ -592,7 +590,8 @@ def get_dns_zonefile(zone, env):
 		if zone == domain:
 			break
 	else:
-		raise ValueError("%s is not a domain name that corresponds to a zone." % zone)
+		msg = f"{zone} is not a domain name that corresponds to a zone."
+		raise ValueError(msg)
 
 	nsd_zonefile = "/etc/nsd/zones/" + fn
 	with open(nsd_zonefile, encoding="utf-8") as f:
@@ -617,8 +616,8 @@ zone:
 		# and, if not a subnet, notifies to them.
 		for ipaddr in get_secondary_dns(additional_records, mode="xfr"):
 			if "/" not in ipaddr:
-				nsdconf += "\n\tnotify: %s NOKEY" % (ipaddr)
-			nsdconf += "\n\tprovide-xfr: %s NOKEY\n" % (ipaddr)
+				nsdconf += f"\n\tnotify: {ipaddr} NOKEY"
+			nsdconf += f"\n\tprovide-xfr: {ipaddr} NOKEY\n"
 
 	# Check if the file is changing. If it isn't changing,
 	# return False to flag that no change was made.
@@ -717,9 +716,9 @@ def sign_zone(domain, zonefile, env):
 
 		# zonefile to sign
 		"/etc/nsd/zones/" + zonefile,
-	]
 		# keys to sign with (order doesn't matter -- it'll figure it out)
-		+ all_keys
+		*all_keys
+	]
 	)
 
 	# Create a DS record based on the patched-up key files. The DS record is specific to the
@@ -898,7 +897,8 @@ def set_custom_dns_record(qname, rtype, value, action, env):
 	else:
 		# No match.
 		if qname != "_secondary_nameserver":
-			raise ValueError("%s is not a domain name or a subdomain of a domain name managed by this box." % qname)
+			msg = f"{qname} is not a domain name or a subdomain of a domain name managed by this box."
+			raise ValueError(msg)
 
 	# validate rtype
 	rtype = rtype.upper()
@@ -919,7 +919,7 @@ def set_custom_dns_record(qname, rtype, value, action, env):
 
 			# ensure value has a trailing dot
 			if not value.endswith("."):
-				value = value + "."
+				value += "."
 
 			if not re.search(DOMAIN_RE, value):
 				msg = "Invalid value."
@@ -928,7 +928,8 @@ def set_custom_dns_record(qname, rtype, value, action, env):
 			# anything goes
 			pass
 		else:
-			raise ValueError("Unknown record type '%s'." % rtype)
+			msg = f"Unknown record type '{rtype}'."
+			raise ValueError(msg)
 
 	# load existing config
 	config = list(get_custom_dns_config(env))
@@ -1039,7 +1040,8 @@ def set_secondary_dns(hostnames, env):
 					try:
 						resolver.resolve(item, "AAAA")
 					except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer, dns.resolver.Timeout):
-						raise ValueError("Could not resolve the IP address of %s." % item)
+						msg = f"Could not resolve the IP address of {item}."
+						raise ValueError(msg)
 			else:
 				# Validate IP address.
 				try:
@@ -1048,7 +1050,8 @@ def set_secondary_dns(hostnames, env):
 					else:
 						ipaddress.ip_address(item[4:]) # raises a ValueError if there's a problem
 				except ValueError:
-					raise ValueError("'%s' is not an IPv4 or IPv6 address or subnet." % item[4:])
+					msg = f"'{item[4:]}' is not an IPv4 or IPv6 address or subnet."
+					raise ValueError(msg)
 
 		# Set.
 		set_custom_dns_record("_secondary_nameserver", "A", " ".join(hostnames), "set", env)

management/mail_log.py

@@ -71,7 +71,7 @@ def scan_files(collector):
 
         if not os.path.exists(fn):
             continue
-        elif fn[-3:] == '.gz':
+        if fn[-3:] == '.gz':
             tmp_file = tempfile.NamedTemporaryFile()
             with gzip.open(fn, 'rb') as f:
                 shutil.copyfileobj(f, tmp_file)
@@ -302,7 +302,7 @@ def scan_mail_log(env):
                 for date, sender, message in user_data["blocked"]:
                     if len(sender) > 64:
                         sender = sender[:32] + "…" + sender[-32:]
-                    user_rejects.extend((f'{date} - {sender} ', '  %s' % message))
+                    user_rejects.extend((f'{date} - {sender} ', f'  {message}'))
                 rejects.append(user_rejects)
 
         print_user_table(
@@ -355,7 +355,7 @@ def scan_mail_log_line(line, collector):
     if date > END_DATE:
         # Don't process, and halt
         return False
-    elif date < START_DATE:
+    if date < START_DATE:
         # Don't process, but continue
         return True
 
@@ -391,7 +391,7 @@ def scan_postgrey_line(date, log, collector):
     """ Scan a postgrey log line and extract interesting data """
 
     m = re.match(r"action=(greylist|pass), reason=(.*?), (?:delay=\d+, )?client_name=(.*), "
-                 "client_address=(.*), sender=(.*), recipient=(.*)",
+                 r"client_address=(.*), sender=(.*), recipient=(.*)",
                  log)
 
     if m:
@@ -423,7 +423,7 @@ def scan_postfix_smtpd_line(date, log, collector):
 
     # Check if the incoming mail was rejected
 
-    m = re.match("NOQUEUE: reject: RCPT from .*?: (.*?); from=<(.*?)> to=<(.*?)>", log)
+    m = re.match(r"NOQUEUE: reject: RCPT from .*?: (.*?); from=<(.*?)> to=<(.*?)>", log)
 
     if m:
         message, sender, user = m.groups()
@@ -467,7 +467,7 @@ def scan_postfix_smtpd_line(date, log, collector):
 def scan_dovecot_login_line(date, log, collector, protocol_name):
     """ Scan a dovecot login log line and extract interesting data """
 
-    m = re.match("Info: Login: user=<(.*?)>, method=PLAIN, rip=(.*?),", log)
+    m = re.match(r"Info: Login: user=<(.*?)>, method=PLAIN, rip=(.*?),", log)
 
     if m:
         # TODO: CHECK DIT
@@ -495,7 +495,7 @@ def add_login(user, date, protocol_name, host, collector):
             data["latest"] = date
 
             data["totals_by_protocol"][protocol_name] += 1
-            data["totals_by_protocol_and_host"][(protocol_name, host)] += 1
+            data["totals_by_protocol_and_host"][protocol_name, host] += 1
 
             if host not in {"127.0.0.1", "::1"} or True:
                 data["activity-by-hour"][protocol_name][date.hour] += 1
@@ -608,7 +608,8 @@ def valid_date(string):
     try:
         date = dateutil.parser.parse(string)
     except ValueError:
-        raise argparse.ArgumentTypeError("Unrecognized date and/or time '%s'" % string)
+        msg = f"Unrecognized date and/or time '{string}'"
+        raise argparse.ArgumentTypeError(msg)
     return date
 
 
@@ -634,7 +635,6 @@ def print_time_table(labels, data, do_print=True):
     if do_print:
         print("\n".join(lines))
         return None
-    else:
     return lines
 
 
@@ -670,7 +670,7 @@ def print_user_table(users, data=None, sub_data=None, activity=None, latest=None
                 col_str = f"{d[row]!s:<20}"
                 col_left[col] = True
             else:
-                temp = "{:>%s}" % max(5, len(l) + 1, len(str(d[row])) + 1)
+                temp = f"{{:>{max(5, len(l) + 1, len(str(d[row])) + 1)}}}"
                 col_str = temp.format(str(d[row]))
             col_widths[col] = max(col_widths[col], len(col_str))
             line += col_str
@@ -679,7 +679,7 @@ def print_user_table(users, data=None, sub_data=None, activity=None, latest=None
                 data_accum[col] += d[row]
 
         try:
-            if None not in [latest, earliest]: # noqa PLR6201
+            if None not in [latest, earliest]: # noqa: PLR6201
                 vert_pos = len(line)
                 e = earliest[row]
                 l = latest[row]
@@ -707,10 +707,10 @@ def print_user_table(users, data=None, sub_data=None, activity=None, latest=None
                 if sub_data is not None:
                     for l, d in sub_data:
                         if d[row]:
-                            lines.extend(('┬', '│ %s' % l, '├─%s─' % (len(l) * '─'), '│'))
+                            lines.extend(('┬', f'│ {l}', '├─%s─' % (len(l) * '─'), '│'))
                             max_len = 0
                             for v in list(d[row]):
-                                lines.append("│ %s" % v)
+                                lines.append(f"│ {v}")
                                 max_len = max(max_len, len(v))
                             lines.append("└" + (max_len + 1) * "─")
 
@@ -732,7 +732,7 @@ def print_user_table(users, data=None, sub_data=None, activity=None, latest=None
         else:
             header += l.rjust(max(5, len(l) + 1, col_widths[col]))
 
-    if None not in [latest, earliest]: # noqa PLR6201
+    if None not in [latest, earliest]: # noqa: PLR6201
         header += " │ timespan   "
 
     lines.insert(0, header.rstrip())
@@ -757,7 +757,7 @@ def print_user_table(users, data=None, sub_data=None, activity=None, latest=None
         footer += temp.format(data_accum[row])
 
     try:
-        if None not in [latest, earliest]: # noqa PLR6201
+        if None not in [latest, earliest]: # noqa: PLR6201
             max_l = max(latest)
             min_e = min(earliest)
             timespan = relativedelta(max_l, min_e)

management/mailconfig.py

@@ -10,9 +10,12 @@
 # address entered by the user.
 
 import os, sqlite3, re
+import subprocess
+
 import utils
 from email_validator import validate_email as validate_email_, EmailNotValidError
 import idna
+import operator
 
 def validate_email(email, mode=None):
 	# Checks that an email address is syntactically valid. Returns True/False.
@@ -92,7 +95,6 @@ def open_database(env, with_connection=False):
 	conn = sqlite3.connect(env["STORAGE_ROOT"] + "/mail/users.sqlite")
 	if not with_connection:
 		return conn.cursor()
-	else:
 	return conn, conn.cursor()
 
 def get_mail_users(env):
@@ -102,6 +104,17 @@ def get_mail_users(env):
 	users = [ row[0] for row in c.fetchall() ]
 	return utils.sort_email_addresses(users, env)
 
+def sizeof_fmt(num):
+	for unit in ['','K','M','G','T']:
+		if abs(num) < 1024.0:
+			if abs(num) > 99:
+				return f"{num:3.0f}{unit}"
+			return f"{num:2.1f}{unit}"
+
+		num /= 1024.0
+
+	return str(num)
+
 def get_mail_users_ex(env, with_archived=False):
 	# Returns a complex data structure of all user accounts, optionally
 	# including archived (status="inactive") accounts.
@@ -125,13 +138,42 @@ def get_mail_users_ex(env, with_archived=False):
 	users = []
 	active_accounts = set()
 	c = open_database(env)
-	c.execute('SELECT email, privileges FROM users')
-	for email, privileges in c.fetchall():
+	c.execute('SELECT email, privileges, quota FROM users')
+	for email, privileges, quota in c.fetchall():
 		active_accounts.add(email)
 
+		(user, domain) = email.split('@')
+		box_size = 0
+		box_quota = 0
+		percent = ''
+		try:
+			dirsize_file = os.path.join(env['STORAGE_ROOT'], f'mail/mailboxes/{domain}/{user}/maildirsize')
+			with open(dirsize_file, encoding="utf-8") as f:
+				box_quota = int(f.readline().split('S')[0])
+				for line in f:
+					(size, _count) = line.split(' ')
+					box_size += int(size)
+
+			try:
+				percent = (box_size / box_quota) * 100
+			except:
+				percent = 'Error'
+
+		except:
+			box_size = '?'
+			box_quota = '?'
+			percent = '?'
+
+		if quota == '0':
+			percent = ''
+
 		user = {
 			"email": email,
 			"privileges": parse_privs(privileges),
+            "quota": quota,
+			"box_quota": box_quota,
+			"box_size": sizeof_fmt(box_size) if box_size != '?' else box_size,
+			"percent": f'{percent:3.0f}%' if type(percent) != str else percent,
 			"status": "active",
 		}
 		users.append(user)
@@ -150,6 +192,9 @@ def get_mail_users_ex(env, with_archived=False):
 						"privileges": [],
 						"status": "inactive",
 						"mailbox": mbox,
+                        "box_size": '?',
+                        "box_quota": '?',
+                        "percent": '?',
 					}
 					users.append(user)
 
@@ -239,7 +284,7 @@ def get_mail_aliases_ex(env):
 
 	# Sort aliases within each domain first by required-ness then lexicographically by address.
 	for domain in domains:
-		domain["aliases"].sort(key = lambda alias : (alias["auto"], alias["address"]))
+		domain["aliases"].sort(key = operator.itemgetter("auto", "address"))
 	return domains
 
 def get_domain(emailaddr, as_unicode=True):
@@ -266,15 +311,15 @@ def get_mail_domains(env, filter_aliases=lambda alias : True, users_only=False):
 		domains.extend([get_domain(address, as_unicode=False) for address, _, _, auto in get_mail_aliases(env) if filter_aliases(address) and not auto ])
 	return set(domains)
 
-def add_mail_user(email, pw, privs, env):
+def add_mail_user(email, pw, privs, quota, env):
 	# validate email
 	if email.strip() == "":
 		return ("No email address provided.", 400)
-	elif not validate_email(email):
+	if not validate_email(email):
 		return ("Invalid email address.", 400)
-	elif not validate_email(email, mode='user'):
+	if not validate_email(email, mode='user'):
 		return ("User account email addresses may only use the lowercase ASCII letters a-z, the digits 0-9, underscore (_), hyphen (-), and period (.).", 400)
-	elif is_dcv_address(email) and len(get_mail_users(env)) > 0:
+	if is_dcv_address(email) and len(get_mail_users(env)) > 0:
 		# Make domain control validation hijacking a little harder to mess up by preventing the usual
 		# addresses used for DCV from being user accounts. Except let it be the first account because
 		# during box setup the user won't know the rules.
@@ -292,6 +337,14 @@ def add_mail_user(email, pw, privs, env):
 			validation = validate_privilege(p)
 			if validation: return validation
 
+	if quota is None:
+		quota = '0'
+
+	try:
+		quota = validate_quota(quota)
+	except ValueError as e:
+		return (str(e), 400)
+
 	# get the database
 	conn, c = open_database(env, with_connection=True)
 
@@ -300,14 +353,16 @@ def add_mail_user(email, pw, privs, env):
 
 	# add the user to the database
 	try:
-		c.execute("INSERT INTO users (email, password, privileges) VALUES (?, ?, ?)",
-			(email, pw, "\n".join(privs)))
+		c.execute("INSERT INTO users (email, password, privileges, quota) VALUES (?, ?, ?, ?)",
+			(email, pw, "\n".join(privs), quota))
 	except sqlite3.IntegrityError:
 		return ("User already exists.", 400)
 
 	# write databasebefore next step
 	conn.commit()
 
+	dovecot_quota_recalc(email)
+
 	# Update things in case any new domains are added.
 	return kick(env, "mail user added")
 
@@ -322,7 +377,7 @@ def set_mail_password(email, pw, env):
 	conn, c = open_database(env, with_connection=True)
 	c.execute("UPDATE users SET password=? WHERE email=?", (pw, email))
 	if c.rowcount != 1:
-		return ("That's not a user (%s)." % email, 400)
+		return (f"That's not a user ({email}).", 400)
 	conn.commit()
 	return "OK"
 
@@ -332,6 +387,58 @@ def hash_password(pw):
 	# http://wiki2.dovecot.org/Authentication/PasswordSchemes
 	return utils.shell('check_output', ["/usr/bin/doveadm", "pw", "-s", "SHA512-CRYPT", "-p", pw]).strip()
 
+
+def get_mail_quota(email, env):
+	_conn, c = open_database(env, with_connection=True)
+	c.execute("SELECT quota FROM users WHERE email=?", (email,))
+	rows = c.fetchall()
+	if len(rows) != 1:
+		return (f"That's not a user ({email}).", 400)
+
+	return rows[0][0]
+
+
+def set_mail_quota(email, quota, env):
+	# validate that password is acceptable
+	quota = validate_quota(quota)
+
+	# update the database
+	conn, c = open_database(env, with_connection=True)
+	c.execute("UPDATE users SET quota=? WHERE email=?", (quota, email))
+	if c.rowcount != 1:
+		return (f"That's not a user ({email}).", 400)
+	conn.commit()
+
+	dovecot_quota_recalc(email)
+
+	return "OK"
+
+def dovecot_quota_recalc(email):
+	# dovecot processes running for the user will not recognize the new quota setting
+	# a reload is necessary to reread the quota setting, but it will also shut down
+	# running dovecot processes.  Email clients generally log back in when they lose
+	# a connection.
+	# subprocess.call(['doveadm', 'reload'])
+
+	# force dovecot to recalculate the quota info for the user.
+	subprocess.call(["doveadm", "quota", "recalc", "-u", email])
+
+def validate_quota(quota):
+	# validate quota
+	quota = quota.strip().upper()
+
+	if quota == "":
+		msg = "No quota provided."
+		raise ValueError(msg)
+	if re.search(r"[\s,.]", quota):
+		msg = "Quotas cannot contain spaces, commas, or decimal points."
+		raise ValueError(msg)
+	if not re.match(r'^[\d]+[GM]?$', quota):
+		msg = "Invalid quota."
+		raise ValueError(msg)
+
+	return quota
+
 def get_mail_password(email, env):
 	# Gets the hashed password for a user. Passwords are stored in Dovecot's
 	# password format, with a prefixed scheme.
@@ -341,7 +448,8 @@ def get_mail_password(email, env):
 	c.execute('SELECT password FROM users WHERE email=?', (email,))
 	rows = c.fetchall()
 	if len(rows) != 1:
-		raise ValueError("That's not a user (%s)." % email)
+		msg = f"That's not a user ({email})."
+		raise ValueError(msg)
 	return rows[0][0]
 
 def remove_mail_user(email, env):
@@ -349,7 +457,7 @@ def remove_mail_user(email, env):
 	conn, c = open_database(env, with_connection=True)
 	c.execute("DELETE FROM users WHERE email=?", (email,))
 	if c.rowcount != 1:
-		return ("That's not a user (%s)." % email, 400)
+		return (f"That's not a user ({email}).", 400)
 	conn.commit()
 
 	# Update things in case any domains are removed.
@@ -365,12 +473,12 @@ def get_mail_user_privileges(email, env, empty_on_error=False):
 	rows = c.fetchall()
 	if len(rows) != 1:
 		if empty_on_error: return []
-		return ("That's not a user (%s)." % email, 400)
+		return (f"That's not a user ({email}).", 400)
 	return parse_privs(rows[0][0])
 
 def validate_privilege(priv):
 	if "\n" in priv or priv.strip() == "":
-		return ("That's not a valid privilege (%s)." % priv, 400)
+		return (f"That's not a valid privilege ({priv}).", 400)
 	return None
 
 def add_remove_mail_user_privilege(email, priv, action, env):
@@ -413,7 +521,7 @@ def add_mail_alias(address, forwards_to, permitted_senders, env, update_if_exist
 	if address == "":
 		return ("No email address provided.", 400)
 	if not validate_email(address, mode='alias'):
-		return ("Invalid email address (%s)." % address, 400)
+		return (f"Invalid email address ({address}).", 400)
 
 	# validate forwards_to
 	validated_forwards_to = []
@@ -442,7 +550,7 @@ def add_mail_alias(address, forwards_to, permitted_senders, env, update_if_exist
 				# Strip any +tag from email alias and check privileges
 				privileged_email = re.sub(r"(?=\+)[^@]*(?=@)",'',email)
 				if not validate_email(email):
-					return ("Invalid receiver email address (%s)." % email, 400)
+					return (f"Invalid receiver email address ({email}).", 400)
 				if is_dcv_source and not is_dcv_address(email) and "admin" not in get_mail_user_privileges(privileged_email, env, empty_on_error=True):
 					# Make domain control validation hijacking a little harder to mess up by
 					# requiring aliases for email addresses typically used in DCV to forward
@@ -462,7 +570,7 @@ def add_mail_alias(address, forwards_to, permitted_senders, env, update_if_exist
 			login = login.strip()
 			if login == "": continue
 			if login not in valid_logins:
-				return ("Invalid permitted sender: %s is not a user on this system." % login, 400)
+				return (f"Invalid permitted sender: {login} is not a user on this system.", 400)
 			validated_permitted_senders.append(login)
 
 	# Make sure the alias has either a forwards_to or a permitted_sender.
@@ -481,8 +589,7 @@ def add_mail_alias(address, forwards_to, permitted_senders, env, update_if_exist
 		return_status = "alias added"
 	except sqlite3.IntegrityError:
 		if not update_if_exists:
-			return ("Alias already exists (%s)." % address, 400)
-		else:
+			return (f"Alias already exists ({address}).", 400)
 		c.execute("UPDATE aliases SET destination = ?, permitted_senders = ? WHERE source = ?", (forwards_to, permitted_senders, address))
 		return_status = "alias updated"
 
@@ -501,7 +608,7 @@ def remove_mail_alias(address, env, do_kick=True):
 	conn, c = open_database(env, with_connection=True)
 	c.execute("DELETE FROM aliases WHERE source=?", (address,))
 	if c.rowcount != 1:
-		return ("That's not an alias (%s)." % address, 400)
+		return (f"That's not an alias ({address}).", 400)
 	conn.commit()
 
 	if do_kick:

management/ssl_certificates.py

@@ -63,9 +63,7 @@ def get_ssl_certificates(env):
 		if isinstance(pem, Certificate):
 			certificates.append({ "filename": fn, "cert": pem })
 		# It is a private key
-		elif (isinstance(pem, rsa.RSAPrivateKey)
-			or isinstance(pem, dsa.DSAPrivateKey)
-			or isinstance(pem, ec.EllipticCurvePrivateKey)):
+		elif (isinstance(pem, (rsa.RSAPrivateKey, dsa.DSAPrivateKey, ec.EllipticCurvePrivateKey))):
 			private_keys[pem.public_key().public_numbers()] = { "filename": fn, "key": pem }
 
 
@@ -160,12 +158,11 @@ def get_domain_ssl_files(domain, ssl_certificates, env, allow_missing_cert=False
 	wildcard_domain = re.sub(r"^[^\.]+", "*", domain)
 	if domain in ssl_certificates:
 		return ssl_certificates[domain]
-	elif wildcard_domain in ssl_certificates:
+	if wildcard_domain in ssl_certificates:
 		return ssl_certificates[wildcard_domain]
-	elif not allow_missing_cert:
+	if not allow_missing_cert:
 		# No valid certificate is available for this domain! Return default files.
 		return system_certificate
-	else:
 	# No valid certificate is available for this domain.
 	return None
 
@@ -518,7 +515,7 @@ def check_certificate(domain, ssl_certificate, ssl_private_key, warn_if_expiring
 		cert = load_pem(ssl_cert_chain[0])
 		if not isinstance(cert, Certificate): raise ValueError("This is not a certificate file.")
 	except ValueError as e:
-		return ("There is a problem with the certificate file: %s" % str(e), None)
+		return (f"There is a problem with the certificate file: {e!s}", None)
 
 	# First check that the domain name is one of the names allowed by
 	# the certificate.
@@ -530,8 +527,7 @@ def check_certificate(domain, ssl_certificate, ssl_private_key, warn_if_expiring
 		# should work in normal cases).
 		wildcard_domain = re.sub(r"^[^\.]+", "*", domain)
 		if domain not in certificate_names and wildcard_domain not in certificate_names:
-			return ("The certificate is for the wrong domain name. It is for %s."
-				% ", ".join(sorted(certificate_names)), None)
+			return ("The certificate is for the wrong domain name. It is for {}.".format(", ".join(sorted(certificate_names))), None)
 
 	# Second, check that the certificate matches the private key.
 	if ssl_private_key is not None:
@@ -544,10 +540,10 @@ def check_certificate(domain, ssl_certificate, ssl_private_key, warn_if_expiring
 		if (not isinstance(priv_key, rsa.RSAPrivateKey)
 			and not isinstance(priv_key, dsa.DSAPrivateKey)
 			and not isinstance(priv_key, ec.EllipticCurvePrivateKey)):
-			return ("The private key file %s is not a private key file." % ssl_private_key, None)
+			return (f"The private key file {ssl_private_key} is not a private key file.", None)
 
 		if priv_key.public_key().public_numbers() != cert.public_key().public_numbers():
-			return ("The certificate does not correspond to the private key at %s." % ssl_private_key, None)
+			return (f"The certificate does not correspond to the private key at {ssl_private_key}.", None)
 
 		# We could also use the openssl command line tool to get the modulus
 		# listed in each file. The output of each command below looks like "Modulus=XXXXX".
@@ -591,14 +587,13 @@ def check_certificate(domain, ssl_certificate, ssl_private_key, warn_if_expiring
 		# Certificate is self-signed. Probably we detected this above.
 		return ("SELF-SIGNED", None)
 
-	elif retcode != 0:
+	if retcode != 0:
 		if "unable to get local issuer certificate" in verifyoutput:
-			return ("The certificate is missing an intermediate chain or the intermediate chain is incorrect or incomplete. (%s)" % verifyoutput, None)
+			return (f"The certificate is missing an intermediate chain or the intermediate chain is incorrect or incomplete. ({verifyoutput})", None)
 
 		# There is some unknown problem. Return the `openssl verify` raw output.
 		return ("There is a problem with the certificate.", verifyoutput.strip())
 
-	else:
 	# `openssl verify` returned a zero exit status so the cert is currently
 	# good.
 
@@ -610,7 +605,7 @@ def check_certificate(domain, ssl_certificate, ssl_private_key, warn_if_expiring
 		expiry_info = "The certificate expires in %d days on %s." % (ndays, cert_expiration_date.date().isoformat())
 	else:
 		# We'll renew it with Lets Encrypt.
-			expiry_info = "The certificate expires on %s." % cert_expiration_date.date().isoformat()
+		expiry_info = f"The certificate expires on {cert_expiration_date.date().isoformat()}."
 
 	if warn_if_expiring_soon and ndays <= warn_if_expiring_soon:
 		# Warn on day 10 to give 4 days for us to automatically renew the
@@ -672,13 +667,11 @@ def get_certificate_domains(cert):
 	def idna_decode_dns_name(dns_name):
 		if dns_name.startswith("*."):
 			return "*." + idna.encode(dns_name[2:]).decode('ascii')
-		else:
 		return idna.encode(dns_name).decode('ascii')
 
 	try:
 		sans = cert.extensions.get_extension_for_oid(OID_SUBJECT_ALTERNATIVE_NAME).value.get_values_for_type(DNSName)
-		for san in sans:
-			names.add(idna_decode_dns_name(san))
+		names.update(idna_decode_dns_name(san) for san in sans)
 	except ExtensionNotFound:
 		pass
 

management/status_checks.py

@@ -6,6 +6,7 @@
 
 import sys, os, os.path, re, datetime, multiprocessing.pool
 import asyncio
+import dateutil.parser, dateutil.relativedelta, dateutil.tz
 
 import dns.reversename, dns.resolver
 import idna
@@ -18,6 +19,7 @@ from ssl_certificates import get_ssl_certificates, get_domain_ssl_files, check_c
 from mailconfig import get_mail_domains, get_mail_aliases
 
 from utils import shell, sort_domains, load_env_vars_from_file, load_settings, get_ssh_port, get_ssh_config_value
+from backup import get_backup_config, backup_status
 
 def get_services():
 	return [
@@ -155,6 +157,7 @@ def run_system_checks(rounded_values, env, output):
 	check_system_aliases(env, output)
 	check_free_disk_space(rounded_values, env, output)
 	check_free_memory(rounded_values, env, output)
+	check_backup(rounded_values, env, output)
 
 def check_ufw(env, output):
 	if not os.path.isfile('/usr/sbin/ufw'):
@@ -248,7 +251,7 @@ def check_free_disk_space(rounded_values, env, output):
 def check_free_memory(rounded_values, env, output):
 	# Check free memory.
 	percent_free = 100 - psutil.virtual_memory().percent
-	memory_msg = "System memory is %s%% free." % str(round(percent_free))
+	memory_msg = f"System memory is {round(percent_free)!s}% free."
 	if percent_free >= 20:
 		if rounded_values: memory_msg = "System free memory is at least 20%."
 		output.print_ok(memory_msg)
@@ -259,6 +262,37 @@ def check_free_memory(rounded_values, env, output):
 		if rounded_values: memory_msg = "System free memory is below 10%."
 		output.print_error(memory_msg)
 
+
+def check_backup(rounded_values, env, output):
+	# Check backups
+	backup_config = get_backup_config(env, for_ui=True)
+	
+	# Is the backup enabled?
+	if backup_config.get("target", "off") == "off":
+		output.print_warning("Backups are disabled. It is recommended to enable a backup for your box.")
+		return
+	else:
+		output.print_ok("Backups are enabled")
+	
+	# Get the age of the most recent backup
+	backup_stat = backup_status(env)
+	
+	backups = backup_stat.get("backups", {})
+	if backups and len(backups) > 0:
+		most_recent = backups[0]["date"]
+		
+		# Calculate time between most recent backup and current time
+		now = datetime.datetime.now(dateutil.tz.tzlocal())
+		bk_date = dateutil.parser.parse(most_recent).astimezone(dateutil.tz.tzlocal())
+		bk_age = dateutil.relativedelta.relativedelta(now, bk_date)
+		
+		if bk_age.days > 7:
+			output.print_error("Backup is more than a week old")
+	else:
+		output.print_error("Could not obtain backup status or no backup has been made (yet). "
+			"This could happen if you have just enabled backups. In that case, check back tomorrow.")
+
+
 def run_network_checks(env, output):
 	# Also see setup/network-checks.sh.
 
@@ -478,7 +512,7 @@ def check_primary_hostname_dns(domain, env, output, dns_domains, dns_zonefiles):
 	tlsa25 = query_dns(tlsa_qname, "TLSA", nxdomain=None)
 	tlsa25_expected = build_tlsa_record(env)
 	if tlsa25 == tlsa25_expected:
-		output.print_ok("""The DANE TLSA record for incoming mail is correct (%s).""" % tlsa_qname,)
+		output.print_ok(f"""The DANE TLSA record for incoming mail is correct ({tlsa_qname}).""",)
 	elif tlsa25 is None:
 		if has_dnssec:
 			# Omit a warning about it not being set if DNSSEC isn't enabled,
@@ -497,9 +531,9 @@ def check_alias_exists(alias_name, alias, env, output):
 		if mail_aliases[alias]:
 			output.print_ok(f"{alias_name} exists as a mail alias. [{alias} ↦ {mail_aliases[alias]}]")
 		else:
-			output.print_error("""You must set the destination of the mail alias for %s to direct email to you or another administrator.""" % alias)
+			output.print_error(f"""You must set the destination of the mail alias for {alias} to direct email to you or another administrator.""")
 	else:
-		output.print_error("""You must add a mail alias for %s which directs email to you or another administrator.""" % alias)
+		output.print_error(f"""You must add a mail alias for {alias} which directs email to you or another administrator.""")
 
 def check_dns_zone(domain, env, output, dns_zonefiles):
 	# If a DS record is set at the registrar, check DNSSEC first because it will affect the NS query.
@@ -527,7 +561,7 @@ def check_dns_zone(domain, env, output, dns_zonefiles):
 	probably_external_dns = False
 
 	if existing_ns.lower() == correct_ns.lower():
-		output.print_ok("Nameservers are set correctly at registrar. [%s]" % correct_ns)
+		output.print_ok(f"Nameservers are set correctly at registrar. [{correct_ns}]")
 	elif ip == correct_ip:
 		# The domain resolves correctly, so maybe the user is using External DNS.
 		output.print_warning(f"""The nameservers set on this domain at your domain name registrar should be {correct_ns}. They are currently {existing_ns}.
@@ -546,7 +580,7 @@ def check_dns_zone(domain, env, output, dns_zonefiles):
 			# We must first resolve the nameserver to an IP address so we can query it.
 			ns_ips = query_dns(ns, "A")
 			if not ns_ips or ns_ips in {'[Not Set]', '[timeout]'}:
-				output.print_error("Secondary nameserver %s is not valid (it doesn't resolve to an IP address)." % ns)
+				output.print_error(f"Secondary nameserver {ns} is not valid (it doesn't resolve to an IP address).")
 				continue
 			# Choose the first IP if nameserver returns multiple
 			ns_ip = ns_ips.split('; ')[0]
@@ -587,7 +621,7 @@ def check_dns_zone_suggestions(domain, env, output, dns_zonefiles, domains_with_
 	if domain in domains_with_a_records:
 		output.print_warning("""Web has been disabled for this domain because you have set a custom DNS record.""")
 	if "www." + domain in domains_with_a_records:
-		output.print_warning("""A redirect from 'www.%s' has been disabled for this domain because you have set a custom DNS record on the www subdomain.""" % domain)
+		output.print_warning(f"""A redirect from 'www.{domain}' has been disabled for this domain because you have set a custom DNS record on the www subdomain.""")
 
 	# Since DNSSEC is optional, if a DS record is NOT set at the registrar suggest it.
 	# (If it was set, we did the check earlier.)
@@ -616,11 +650,11 @@ def check_dnssec(domain, env, output, dns_zonefiles, is_checking_primary=False):
 			# Some registrars may want the public key so they can compute the digest. The DS
 			# record that we suggest using is for the KSK (and that's how the DS records were generated).
 			# We'll also give the nice name for the key algorithm.
-			dnssec_keys = load_env_vars_from_file(os.path.join(env['STORAGE_ROOT'], 'dns/dnssec/%s.conf' % alg_name_map[ds_alg]))
+			dnssec_keys = load_env_vars_from_file(os.path.join(env['STORAGE_ROOT'], f'dns/dnssec/{alg_name_map[ds_alg]}.conf'))
 			with open(os.path.join(env['STORAGE_ROOT'], 'dns/dnssec/' + dnssec_keys['KSK'] + '.key'), encoding="utf-8") as f:
 				dnsssec_pubkey = f.read().split("\t")[3].split(" ")[3]
 
-			expected_ds_records[ (ds_keytag, ds_alg, ds_digalg, ds_digest) ] = {
+			expected_ds_records[ ds_keytag, ds_alg, ds_digalg, ds_digest ] = {
 				"record": rr_ds,
 				"keytag": ds_keytag,
 				"alg": ds_alg,
@@ -653,16 +687,16 @@ def check_dnssec(domain, env, output, dns_zonefiles, is_checking_primary=False):
 			if {r[1] for r in matched_ds} == { '13' } and {r[2] for r in matched_ds} <= { '2', '4' }: # all are alg 13 and digest type 2 or 4
 				output.print_ok("DNSSEC 'DS' record is set correctly at registrar.")
 				return
-			elif len([r for r in matched_ds if r[1] == '13' and r[2] in { '2', '4' }]) > 0: # some but not all are alg 13
+			if len([r for r in matched_ds if r[1] == '13' and r[2] in { '2', '4' }]) > 0: # some but not all are alg 13
 				output.print_ok("DNSSEC 'DS' record is set correctly at registrar. (Records using algorithm other than ECDSAP256SHA256 and digest types other than SHA-256/384 should be removed.)")
 				return
-			else: # no record uses alg 13
+			# no record uses alg 13
 			output.print_warning("""DNSSEC 'DS' record set at registrar is valid but should be updated to ECDSAP256SHA256 and SHA-256 (see below).
 				IMPORTANT: Do not delete existing DNSSEC 'DS' records for this domain until confirmation that the new DNSSEC 'DS' record
 				for this domain is valid.""")
 		else:
 			if is_checking_primary:
-				output.print_error("""The DNSSEC 'DS' record for %s is incorrect. See further details below.""" % domain)
+				output.print_error(f"""The DNSSEC 'DS' record for {domain} is incorrect. See further details below.""")
 				return
 			output.print_error("""This domain's DNSSEC DS record is incorrect. The chain of trust is broken between the public DNS system
 				and this machine's DNS server. It may take several hours for public DNS to update after a change. If you did not recently
@@ -774,11 +808,11 @@ def check_mail_domain(domain, env, output):
 	elif dbl == "[Not Set]":
 		output.print_warning(f"Could not connect to dbl.spamhaus.org. Could not determine whether the domain {domain} is blacklisted. Please try again later.")
 	elif dbl == "127.255.255.252":
-		output.print_warning("Incorrect spamhaus query: %s. Could not determine whether the domain %s is blacklisted." % (domain+'.dbl.spamhaus.org', domain))
+		output.print_warning("Incorrect spamhaus query: {}. Could not determine whether the domain {} is blacklisted.".format(domain+'.dbl.spamhaus.org', domain))
 	elif dbl == "127.255.255.254":
-		output.print_warning("Mail-in-a-Box is configured to use a public DNS server. This is not supported by spamhaus. Could not determine whether the domain {} is blacklisted.".format(domain))
+		output.print_warning(f"Mail-in-a-Box is configured to use a public DNS server. This is not supported by spamhaus. Could not determine whether the domain {domain} is blacklisted.")
 	elif dbl == "127.255.255.255":
-		output.print_warning("Too many queries have been performed on the spamhaus server. Could not determine whether the domain {} is blacklisted.".format(domain))
+		output.print_warning(f"Too many queries have been performed on the spamhaus server. Could not determine whether the domain {domain} is blacklisted.")
 	else:
 		output.print_error(f"""This domain is listed in the Spamhaus Domain Block List (code {dbl}),
 			which may prevent recipients from receiving your mail.
@@ -960,14 +994,14 @@ def check_miab_version(env, output):
 		this_ver = "Unknown"
 
 	if config.get("privacy", True):
-		output.print_warning("You are running version Mail-in-a-Box %s. Mail-in-a-Box version check disabled by privacy setting." % this_ver)
+		output.print_warning(f"You are running version Mail-in-a-Box {this_ver}. Mail-in-a-Box version check disabled by privacy setting.")
 	else:
 		latest_ver = get_latest_miab_version()
 
 		if this_ver == latest_ver:
-			output.print_ok("Mail-in-a-Box is up to date. You are running version %s." % this_ver)
+			output.print_ok(f"Mail-in-a-Box is up to date. You are running version {this_ver}.")
 		elif latest_ver is None:
-			output.print_error("Latest Mail-in-a-Box version could not be determined. You are running version %s." % this_ver)
+			output.print_error(f"Latest Mail-in-a-Box version could not be determined. You are running version {this_ver}.")
 		else:
 			output.print_error(f"A new version of Mail-in-a-Box is available. You are running version {this_ver}. The latest version is {latest_ver}. For upgrade instructions, see https://mailinabox.email. ")
 
@@ -1114,7 +1148,7 @@ class ConsoleOutput(FileOutput):
 class BufferedOutput:
 	# Record all of the instance method calls so we can play them back later.
 	def __init__(self, with_lines=None):
-		self.buf = with_lines if with_lines else []
+		self.buf = with_lines or []
 	def __getattr__(self, attr):
 		if attr not in {"add_heading", "print_ok", "print_error", "print_warning", "print_block", "print_line"}:
 			raise AttributeError

management/templates/index.html

@@ -392,7 +392,9 @@ function api(url, method, data, callback, callback_error, headers) {
       403: function(xhr) {
         // Credentials are no longer valid. Try to login again.
         var p = current_panel;
+        clear_credentials();
         show_panel('login');
+        show_hide_menus();
         switch_back_to_panel = p;
       }
     }
@@ -402,16 +404,21 @@ function api(url, method, data, callback, callback_error, headers) {
 var current_panel = null;
 var switch_back_to_panel = null;
 
-function do_logout() {
-  // Clear the session from the backend.
-  api("/logout", "POST");
-
+function clear_credentials() {
   // Forget the token.
   api_credentials = null;
   if (typeof localStorage != 'undefined')
     localStorage.removeItem("miab-cp-credentials");
   if (typeof sessionStorage != 'undefined')
     sessionStorage.removeItem("miab-cp-credentials");
+}
+
+function do_logout() {
+  // Clear the session from the backend.
+  api("/logout", "POST");
+
+  // Remove locally stored credentials
+  clear_credentials();
 
   // Return to the start.
   show_panel('login');

management/templates/mail-guide.html

@@ -42,7 +42,7 @@
 
 				<h4>Exchange/ActiveSync settings</h4>
 
-					<p>On iOS devices, devices on this <a href="https://wiki.z-hub.io/display/ZP/Compatibility">compatibility list</a>, or using Outlook 2007 or later on Windows 7 and later, you may set up your mail as an Exchange or ActiveSync server. However, we&rsquo;ve found this to be more buggy than using IMAP as described above. If you encounter any problems, please use the manual settings above.</p>
+					<p>On iOS devices, devices on this <a href="https://github.com/Z-Hub/Z-Push/wiki/Compatibility">compatibility list</a>, or using Outlook 2007 or later on Windows 7 and later, you may set up your mail as an Exchange or ActiveSync server. However, we&rsquo;ve found this to be more buggy than using IMAP as described above. If you encounter any problems, please use the manual settings above.</p>
 
 					<table class="table">
 					<tr><th>Server</th> <td>{{hostname}}</td></tr>

management/templates/users.html

@@ -6,6 +6,7 @@
 #user_table .account_inactive .if_active { display: none; }
 #user_table .account_active .if_inactive { display: none; }
 #user_table .account_active.if_inactive { display: none; }
+.row-center { text-align: center; }
 </style>
 
 <h3>Add a mail user</h3>
@@ -27,6 +28,10 @@
       <option value="admin">Administrator</option>
     </select>
   </div>
+  <div class="form-group">
+      <label class="sr-only" for="adduserQuota">Quota</label>
+      <input type="text" class="form-control" id="adduserQuota" placeholder="Quota" style="width:5em;" value="0">
+  </div>
   <button type="submit" class="btn btn-primary">Add User</button>
 </form>
 <ul style="margin-top: 1em; padding-left: 1.5em; font-size: 90%;">
@@ -34,13 +39,17 @@
   <li>Use <a href="#aliases">aliases</a> to create email addresses that forward to existing accounts.</li>
   <li>Administrators get access to this control panel.</li>
   <li>User accounts cannot contain any international (non-ASCII) characters, but <a href="#aliases">aliases</a> can.</li>
+  <li>Quotas may not contain any spaces, commas or decimal points. Suffixes of G (gigabytes) and M (megabytes) are allowed.  For unlimited storage enter 0 (zero)</li>
 </ul>
 
 <h3>Existing mail users</h3>
 <table id="user_table" class="table" style="width: auto">
   <thead>
     <tr>
-      <th width="50%">Email Address</th>
+      <th width="35%">Email Address</th>
+      <th class="row-center">Size</th>
+      <th class="row-center">Used</th>
+      <th class="row-center">Quota</th>
       <th>Actions</th>
     </tr>
   </thead>
@@ -53,10 +62,21 @@
   <tr id="user-template">
     <td class='address'>
     </td>
+    <td class="box-size row-center"></td>
+    <td class="percent row-center"></td>
+	<td class="quota row-center">
+	</td>
     <td class='actions'>
         <span class='privs'>
         </span>
 
+        <span class="if_active">
+          <a href="#" onclick="users_set_quota(this); return false;" class='setquota' title="Set Quota">
+              set quota
+          </a>
+          |
+        </span>
+
         <span class="if_active">
           <a href="#" onclick="users_set_password(this); return false;" class='setpw' title="Set Password">
             set password
@@ -97,10 +117,28 @@
 <table class="table" style="margin-top: .5em">
 <thead><th>Verb</th> <th>Action</th><th></th></thead>
 <tr><td>GET</td><td><i>(none)</i></td> <td>Returns a list of existing mail users. Adding <code>?format=json</code> to the URL will give JSON-encoded results.</td></tr>
-<tr><td>POST</td><td>/add</td> <td>Adds a new mail user. Required POST-body parameters are <code>email</code> and <code>password</code>.</td></tr>
-<tr><td>POST</td><td>/remove</td> <td>Removes a mail user. Required POST-body parameter is <code>email</code>.</td></tr>
+<tr>
+    <td>POST</td>
+    <td>/add</td>
+    <td>Adds a new mail user. Required POST-body parameters are <code>email</code> and <code>password</code>. Optional parameters: <code>privilege=admin</code> and <code>quota</code></td>
+</tr>
+<tr>
+    <td>POST</td>
+    <td>/remove</td>
+    <td>Removes a mail user. Required POST-by parameter is <code>email</code>.</td>
+</tr>
 <tr><td>POST</td><td>/privileges/add</td> <td>Used to make a mail user an admin. Required POST-body parameters are <code>email</code> and <code>privilege=admin</code>.</td></tr>
 <tr><td>POST</td><td>/privileges/remove</td> <td>Used to remove the admin privilege from a mail user. Required POST-body parameter is <code>email</code>.</td></tr>
+<tr>
+    <td>GET</td>
+    <td>/quota</td>
+    <td>Get the quota for a mail user. Required POST-body parameters are <code>email</code> and will return JSON result</td>
+</tr>
+<tr>
+    <td>POST</td>
+    <td>/quota</td>
+    <td>Set the quota for a mail user. Required POST-body parameters are <code>email</code> and <code>quota</code>.</td>
+</tr>
 </table>
 
 <h4>Examples:</h4>
@@ -133,7 +171,7 @@ function show_users() {
     function(r) {
       $('#user_table tbody').html("");
       for (var i = 0; i < r.length; i++) {
-        var hdr = $("<tr><th colspan='2' style='background-color: #EEE'></th></tr>");
+        var hdr = $("<tr><th colspan='6' style='background-color: #EEE'></th></tr>");
         hdr.find('th').text(r[i].domain);
         $('#user_table tbody').append(hdr);
 
@@ -151,7 +189,14 @@ function show_users() {
           n2.addClass("account_" + user.status);
 
           n.attr('data-email', user.email);
-          n.find('.address').text(user.email)
+          n.attr('data-quota', user.quota);
+          n.find('.address').text(user.email);
+          n.find('.box-size').text(user.box_size);
+          if (user.box_size == '?') {
+            n.find('.box-size').attr('title', 'Mailbox size is unkown')
+          }
+          n.find('.percent').text(user.percent);
+          n.find('.quota').text((user.quota == '0') ? 'unlimited' : user.quota);
           n2.find('.restore_info tt').text(user.mailbox);
 
           if (user.status == 'inactive') continue;
@@ -180,13 +225,15 @@ function do_add_user() {
   var email = $("#adduserEmail").val();
   var pw = $("#adduserPassword").val();
   var privs = $("#adduserPrivs").val();
+  var quota = $("#adduserQuota").val();
   api(
     "/mail/users/add",
     "POST",
     {
       email: email,
       password: pw,
-      privileges: privs
+      privileges: privs,
+      quota: quota
     },
     function(r) {
       // Responses are multiple lines of pre-formatted text.
@@ -228,6 +275,36 @@ function users_set_password(elem) {
     });
 }
 
+function users_set_quota(elem) {
+    var email = $(elem).parents('tr').attr('data-email');
+    var quota = $(elem).parents('tr').attr('data-quota');
+
+    show_modal_confirm(
+        "Set Quota",
+        $("<p>Set quota for <b>" + email + "</b>?</p>" +
+            "<p>" +
+            "<label for='users_set_quota' style='display: block; font-weight: normal'>Quota:</label>" +
+            "<input type='text' id='users_set_quota' value='" + quota + "'></p>" +
+            "<p><small>Quotas may not contain any spaces or commas.  Suffixes of G (gigabytes) and M (megabytes) are allowed.</small></p>" +
+            "<p><small>For unlimited storage enter 0 (zero)</small></p>"),
+        "Set Quota",
+        function() {
+            api(
+                "/mail/users/quota",
+                "POST",
+                {
+                    email: email,
+                    quota: $('#users_set_quota').val()
+                },
+                function(r) {
+                    show_users();
+                },
+                function(r) {
+                    show_modal_error("Set Quota", r);
+                });
+        });
+}
+
 function users_remove(elem) {
   var email = $(elem).parents('tr').attr('data-email');
 
@@ -293,7 +370,7 @@ function generate_random_password() {
   var charset = "ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz23456789"; // confusable characters skipped
   for (var i = 0; i < 12; i++)
     pw += charset.charAt(Math.floor(Math.random() * charset.length));
-  show_modal_error("Random Password", "<p>Here, try this:</p> <p><code style='font-size: 110%'>" + pw + "</code></pr");
+  show_modal_error("Random Password", "<p>Here, try this:</p> <p><code style='font-size: 110%'>" + pw + "</code></p>");
   return false; // cancel click
 }
 </script>

management/utils.py

@@ -21,8 +21,7 @@ def load_env_vars_from_file(fn):
 
 def save_environment(env):
     with open("/etc/mailinabox.conf", "w", encoding="utf-8") as f:
-        for k, v in env.items():
-            f.write(f"{k}={v}\n")
+        f.writelines(f"{k}={v}\n" for k, v in env.items())
 
 # THE SETTINGS FILE AT STORAGE_ROOT/settings.yaml.
 
@@ -135,7 +134,6 @@ def shell(method, cmd_args, env=None, capture_stderr=False, return_bytes=False,
     if not return_bytes and isinstance(ret, bytes): ret = ret.decode("utf8")
     if not trap:
         return ret
-    else:
     return code, ret
 
 def create_syslog_handler():

management/web_update.py

@@ -52,7 +52,7 @@ def get_domains_with_a_records(env):
 	domains = set()
 	dns = get_custom_dns_config(env)
 	for domain, rtype, value in dns:
-		if rtype == "CNAME" or (rtype in {"A", "AAAA"} and value not in {"local", env['PUBLIC_IP']}):
+		if rtype == "CNAME" or (rtype in {"A", "AAAA"} and value not in {"local", env['PUBLIC_IP'], env['PUBLIC_IPV6']}):
 			domains.add(domain)
 	return domains
 
@@ -167,7 +167,7 @@ def make_domain_config(domain, templates, ssl_certificates, env):
 				proxy_redirect_off = False
 				frame_options_header_sameorigin = False
 				web_sockets = False
-				m = re.search("#(.*)$", url)
+				m = re.search(r"#(.*)$", url)
 				if m:
 					for flag in m.group(1).split(","):
 						if flag == "pass-http-host":
@@ -178,10 +178,10 @@ def make_domain_config(domain, templates, ssl_certificates, env):
 							frame_options_header_sameorigin = True
 						elif flag == "web-sockets":
 							web_sockets = True
-					url = re.sub("#(.*)$", "", url)
+					url = re.sub(r"#(.*)$", "", url)
 
-				nginx_conf_extra += "\tlocation %s {" % path
-				nginx_conf_extra += "\n\t\tproxy_pass %s;" % url
+				nginx_conf_extra += f"\tlocation {path} {{"
+				nginx_conf_extra += f"\n\t\tproxy_pass {url};"
 				if proxy_redirect_off:
 					nginx_conf_extra += "\n\t\tproxy_redirect off;"
 				if pass_http_host_header:
@@ -198,8 +198,8 @@ def make_domain_config(domain, templates, ssl_certificates, env):
 				nginx_conf_extra += "\n\t\tproxy_set_header X-Real-IP $remote_addr;"
 				nginx_conf_extra += "\n\t}\n"
 			for path, alias in yaml.get("aliases", {}).items():
-				nginx_conf_extra += "\tlocation %s {" % path
-				nginx_conf_extra += "\n\t\talias %s;" % alias
+				nginx_conf_extra += f"\tlocation {path} {{"
+				nginx_conf_extra += f"\n\t\talias {alias};"
 				nginx_conf_extra += "\n\t}\n"
 			for path, url in yaml.get("redirects", {}).items():
 				nginx_conf_extra += f"\trewrite {path} {url} permanent;\n"
@@ -216,7 +216,7 @@ def make_domain_config(domain, templates, ssl_certificates, env):
 	# Add in any user customizations in the includes/ folder.
 	nginx_conf_custom_include = os.path.join(env["STORAGE_ROOT"], "www", safe_domain_name(domain) + ".conf")
 	if os.path.exists(nginx_conf_custom_include):
-		nginx_conf_extra += "\tinclude %s;\n" % (nginx_conf_custom_include)
+		nginx_conf_extra += f"\tinclude {nginx_conf_custom_include};\n"
 	# PUT IT ALL TOGETHER
 
 	# Combine the pieces. Iteratively place each template into the "# ADDITIONAL DIRECTIVES HERE" placeholder
@@ -256,9 +256,8 @@ def get_web_domains_info(env):
 		cert_status, cert_status_details = check_certificate(domain, tls_cert["certificate"], tls_cert["private-key"])
 		if cert_status == "OK":
 			return ("success", "Signed & valid. " + cert_status_details)
-		elif cert_status == "SELF-SIGNED":
+		if cert_status == "SELF-SIGNED":
 			return ("warning", "Self-signed. Get a signed certificate to stop warnings.")
-		else:
 		return ("danger", "Certificate has a problem: " + cert_status)
 
 	return [

pyproject.toml

@@ -0,0 +1,69 @@
+[tool.ruff]
+line-length = 320 # https://github.com/astral-sh/ruff/issues/8106
+indent-width = 4
+
+target-version = "py310"
+
+preview = true
+
+output-format = "concise"
+
+extend-exclude = ["tools/mail.py"]
+
+[tool.ruff.lint]
+select = [
+	"F",
+	"E4",
+	"E7",
+	"E9",
+	"W",
+	"UP",
+	"YTT",
+	"S",
+	"BLE",
+	"B",
+	"A",
+	"C4",
+	"T10",
+	"DJ",
+	"EM",
+	"EXE",
+	"ISC",
+	"ICN",
+	"G",
+	"PIE",
+	"PYI",
+	"Q003",
+	"Q004",
+	"RSE",
+	"RET",
+	"SLF",
+	"SLOT",
+	"SIM",
+	"TID",
+	"TC",
+	"ARG",
+	"PGH",
+	"PL",
+	"TRY",
+	"FLY",
+	"PERF",
+	"FURB",
+	"LOG",
+	"RUF"
+]
+ignore = [
+	"W191",
+	"PLR09",
+	"PLR1702",
+	"PLR2004",
+	"RUF001",
+	"RUF002",
+	"RUF003",
+	"RUF023"
+]
+
+[tool.ruff.format]
+quote-style = "preserve"
+
+indent-style = "tab"

setup/bootstrap.sh

@@ -89,4 +89,3 @@ fi
 
 # Start setup script.
 setup/start.sh
-

setup/mail-dovecot.sh

@@ -67,6 +67,32 @@ tools/editconf.py /etc/dovecot/conf.d/10-mail.conf \
 
 # Create, subscribe, and mark as special folders: INBOX, Drafts, Sent, Trash, Spam and Archive.
 cp conf/dovecot-mailboxes.conf /etc/dovecot/conf.d/15-mailboxes.conf
+sed -i "s/#mail_plugins =\(.*\)/mail_plugins =\1 \$mail_plugins quota/" /etc/dovecot/conf.d/10-mail.conf
+if ! grep -q "mail_plugins.* imap_quota" /etc/dovecot/conf.d/20-imap.conf; then
+  sed -i "s/\(mail_plugins =.*\)/\1\n  mail_plugins = \$mail_plugins imap_quota/" /etc/dovecot/conf.d/20-imap.conf
+fi
+
+# configure stuff for quota support
+if ! grep -q "quota_status_success = DUNNO" /etc/dovecot/conf.d/90-quota.conf; then
+    cat > /etc/dovecot/conf.d/90-quota.conf << EOF;
+plugin {
+  quota = maildir
+
+  quota_grace = 10%%
+
+  quota_status_success = DUNNO
+  quota_status_nouser = DUNNO
+  quota_status_overquota = "522 5.2.2 Mailbox is full"
+}
+
+service quota-status {
+    executable = quota-status -p postfix
+    inet_listener {
+        port = 12340
+    }
+}
+EOF
+fi
 
 # ### IMAP/POP
 

setup/mail-postfix.sh

@@ -238,7 +238,7 @@ tools/editconf.py /etc/postfix/main.cf  -e lmtp_destination_recipient_limit=
 # "450 4.7.1 Client host rejected: Service unavailable". This is a retry code, so the mail doesn't properly bounce. #NODOC
 tools/editconf.py /etc/postfix/main.cf \
 	smtpd_sender_restrictions="reject_non_fqdn_sender,reject_unknown_sender_domain,reject_authenticated_sender_login_mismatch,reject_rhsbl_sender dbl.spamhaus.org=127.0.1.[2..99]" \
-	smtpd_recipient_restrictions="permit_sasl_authenticated,permit_mynetworks,reject_rbl_client zen.spamhaus.org=127.0.0.[2..11],reject_unlisted_recipient,check_policy_service inet:127.0.0.1:10023"
+	smtpd_recipient_restrictions="permit_sasl_authenticated,permit_mynetworks,reject_rbl_client zen.spamhaus.org=127.0.0.[2..11],reject_unlisted_recipient,check_policy_service inet:127.0.0.1:10023,check_policy_service inet:127.0.0.1:12340"
 
 # Postfix connects to Postgrey on the 127.0.0.1 interface specifically. Ensure that
 # Postgrey listens on the same interface (and not IPv6, for instance).

setup/mail-users.sh

@@ -20,7 +20,7 @@ db_path=$STORAGE_ROOT/mail/users.sqlite
 # Create an empty database if it doesn't yet exist.
 if [ ! -f "$db_path" ]; then
 	echo "Creating new user database: $db_path";
-	echo "CREATE TABLE users (id INTEGER PRIMARY KEY AUTOINCREMENT, email TEXT NOT NULL UNIQUE, password TEXT NOT NULL, extra, privileges TEXT NOT NULL DEFAULT '');" | sqlite3 "$db_path";
+	echo "CREATE TABLE users (id INTEGER PRIMARY KEY AUTOINCREMENT, email TEXT NOT NULL UNIQUE, password TEXT NOT NULL, extra, privileges TEXT NOT NULL DEFAULT '', quota TEXT NOT NULL DEFAULT '0');" | sqlite3 "$db_path";
 	echo "CREATE TABLE aliases (id INTEGER PRIMARY KEY AUTOINCREMENT, source TEXT NOT NULL UNIQUE, destination TEXT NOT NULL, permitted_senders TEXT);" | sqlite3 "$db_path";
 	echo "CREATE TABLE mfa (id INTEGER PRIMARY KEY AUTOINCREMENT, user_id INTEGER NOT NULL, type TEXT NOT NULL, secret TEXT NOT NULL, mru_token TEXT, label TEXT, FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE);" | sqlite3 "$db_path";
 	echo "CREATE TABLE auto_aliases (id INTEGER PRIMARY KEY AUTOINCREMENT, source TEXT NOT NULL UNIQUE, destination TEXT NOT NULL, permitted_senders TEXT);" | sqlite3 "$db_path";
@@ -51,7 +51,7 @@ driver = sqlite
 connect = $db_path
 default_pass_scheme = SHA512-CRYPT
 password_query = SELECT email as user, password FROM users WHERE email='%u';
-user_query = SELECT email AS user, "mail" as uid, "mail" as gid, "$STORAGE_ROOT/mail/mailboxes/%d/%n" as home FROM users WHERE email='%u';
+user_query = SELECT email AS user, "mail" as uid, "mail" as gid, "$STORAGE_ROOT/mail/mailboxes/%d/%n" as home, '*:bytes=' || quota AS quota_rule FROM users WHERE email='%u';
 iterate_query = SELECT email AS user FROM users;
 EOF
 chmod 0600 /etc/dovecot/dovecot-sql.conf.ext # per Dovecot instructions
@@ -159,4 +159,5 @@ EOF
 restart_service postfix
 restart_service dovecot
 
-
+# force a recalculation of all user quotas
+doveadm quota recalc -A

setup/migrate.py

@@ -23,7 +23,7 @@ def migration_1(env):
 	# Migrate the 'domains' directory.
 	for sslfn in glob.glob(os.path.join( env["STORAGE_ROOT"], 'ssl/domains/*' )):
 		fn = os.path.basename(sslfn)
-		m = re.match("(.*)_(certifiate.pem|cert_sign_req.csr|private_key.pem)$", fn)
+		m = re.match(r"(.*)_(certifiate.pem|cert_sign_req.csr|private_key.pem)$", fn)
 		if m:
 			# get the new name for the file
 			domain_name, file_type = m.groups()
@@ -164,7 +164,7 @@ def migration_12(env):
                     try:
                         table = table[0]
                         c = conn.cursor()
-                        dropcmd = "DROP TABLE %s" % table
+                        dropcmd = f"DROP TABLE {table}"
                         c.execute(dropcmd)
                     except:
                         print("Failed to drop table", table)
@@ -190,6 +190,12 @@ def migration_14(env):
 	db = os.path.join(env["STORAGE_ROOT"], 'mail/users.sqlite')
 	shell("check_call", ["sqlite3", db, "CREATE TABLE auto_aliases (id INTEGER PRIMARY KEY AUTOINCREMENT, source TEXT NOT NULL UNIQUE, destination TEXT NOT NULL, permitted_senders TEXT);"])
 
+def migration_15(env):
+	# Add a column to the users table to store their quota limit.  Default to '0' for unlimited.
+	db = os.path.join(env["STORAGE_ROOT"], 'mail/users.sqlite')
+	shell("check_call", ["sqlite3", db, "ALTER TABLE users ADD COLUMN quota TEXT NOT NULL DEFAULT '0';"])
+
+
 ###########################################################
 
 def get_current_migration():

setup/nextcloud.sh

@@ -413,8 +413,8 @@ sqlite3 "$STORAGE_ROOT/owncloud/owncloud.db" "UPDATE oc_users_external SET backe
 cat > /etc/cron.d/mailinabox-nextcloud << EOF;
 #!/bin/bash
 # Mail-in-a-Box
-*/5 * * * *	root	sudo -u www-data php$PHP_VER -f /usr/local/lib/owncloud/cron.php
-*/5 * * * *	root	sudo -u www-data php$PHP_VER -f /usr/local/lib/owncloud/occ dav:send-event-reminders
+*/5 * * * *	www-data	php$PHP_VER -f /usr/local/lib/owncloud/cron.php
+*/5 * * * *	www-data	php$PHP_VER -f /usr/local/lib/owncloud/occ dav:send-event-reminders
 EOF
 chmod +x /etc/cron.d/mailinabox-nextcloud
 

setup/preflight.sh

@@ -8,11 +8,14 @@ if [[ $EUID -ne 0 ]]; then
 	exit 1
 fi
 
-# Check that we are running on Ubuntu 20.04 LTS (or 20.04.xx).
-if [ "$( lsb_release --id --short )" != "Ubuntu" ] || [ "$( lsb_release --release --short )" != "22.04" ]; then
+# Check that we are running on Ubuntu 22.04 LTS (or 22.04.xx).
+# Pull in the variables defined in /etc/os-release but in a
+# namespace to avoid polluting our variables.
+source <(cat /etc/os-release | sed s/^/OS_RELEASE_/)
+if [ "${OS_RELEASE_ID:-}" != "ubuntu" ] || [ "${OS_RELEASE_VERSION_ID:-}" != "22.04" ]; then
 	echo "Mail-in-a-Box only supports being installed on Ubuntu 22.04, sorry. You are running:"
 	echo
-	lsb_release --description --short
+	echo "${OS_RELEASE_ID:-"Unknown linux distribution"} ${OS_RELEASE_VERSION_ID:-}"
 	echo
 	echo "We can't write scripts that run on every possible setup, sorry."
 	exit 1

setup/webmail.sh

@@ -145,6 +145,7 @@ cat > $RCM_CONFIG <<EOF;
 \$config['session_path'] = '/mail/';
 /* prevent CSRF, requires php 7.3+ */
 \$config['session_samesite'] = 'Strict';
+\$config['quota_zero_as_unlimited'] = true;
 ?>
 EOF
 

tests/fail2ban.py

@@ -142,7 +142,8 @@ def http_test(url, expected_status, postdata=None, qsargs=None, auth=None):
 	# return response status code
 	if r.status_code != expected_status:
 		r.raise_for_status() # anything but 200
-		raise OSError("Got unexpected status code %s." % r.status_code)
+		msg = f"Got unexpected status code {r.status_code}."
+		raise OSError(msg)
 
 # define how to run a test
 

tests/test_dns.py

@@ -51,7 +51,7 @@ def test2(tests, server, description):
 			response = dns.resolver.resolve(qname, rtype)
 		except dns.resolver.NoNameservers:
 			# host did not have an answer for this query
-			print("Could not connect to %s for DNS query." % server)
+			print(f"Could not connect to {server} for DNS query.")
 			sys.exit(1)
 		except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
 			# host did not have an answer for this query; not sure what the
@@ -79,7 +79,7 @@ def test2(tests, server, description):
 # Test the response from the machine itself.
 if not test(ipaddr, "Mail-in-a-Box"):
 	print ()
-	print ("Please run the Mail-in-a-Box setup script on %s again." % hostname)
+	print (f"Please run the Mail-in-a-Box setup script on {hostname} again.")
 	sys.exit(1)
 else:
 	print ("The Mail-in-a-Box provided correct DNS answers.")
@@ -89,7 +89,7 @@ else:
 	# to see if the machine is hooked up to recursive DNS properly.
 	if not test("8.8.8.8", "Google Public DNS"):
 		print ()
-		print ("Check that the nameserver settings for %s are correct at your domain registrar. It may take a few hours for Google Public DNS to update after changes on your Mail-in-a-Box." % hostname)
+		print (f"Check that the nameserver settings for {hostname} are correct at your domain registrar. It may take a few hours for Google Public DNS to update after changes on your Mail-in-a-Box.")
 		sys.exit(1)
 	else:
 		print ("Your domain registrar or DNS host appears to be configured correctly as well. Public DNS provides the same answers.")

tests/test_mail.py

@@ -46,7 +46,7 @@ reverse_ip = dns.reversename.from_address(ipaddr) # e.g. "1.0.0.127.in-addr.arpa
 try:
 	reverse_dns = dns.resolver.resolve(reverse_ip, 'PTR')[0].target.to_text(omit_final_dot=True) # => hostname
 except dns.resolver.NXDOMAIN:
-	print("Reverse DNS lookup failed for %s. SMTP EHLO name check skipped." % ipaddr)
+	print(f"Reverse DNS lookup failed for {ipaddr}. SMTP EHLO name check skipped.")
 	reverse_dns = None
 if reverse_dns is not None:
 	server.ehlo_or_helo_if_needed() # must send EHLO before getting the server's EHLO name
@@ -54,7 +54,7 @@ if reverse_dns is not None:
 	if helo_name != reverse_dns:
 		print("The server's EHLO name does not match its reverse hostname. Check DNS settings.")
 	else:
-		print("SMTP EHLO name (%s) is OK." % helo_name)
+		print(f"SMTP EHLO name ({helo_name}) is OK.")
 
 # Login and send a test email.
 server.login(emailaddress, pw)

tests/tls.py

@@ -96,7 +96,7 @@ def sslyze(opts, port, ok_ciphers):
 			# Failed. Just output the error.
 			out = re.sub("[\\w\\W]*CHECKING HOST\\(S\\) AVAILABILITY\n\\s*-+\n", "", out) # chop off header that shows the host we queried
 		out = re.sub("[\\w\\W]*SCAN RESULTS FOR.*\n\\s*-+\n", "", out) # chop off header that shows the host we queried
-		out = re.sub("SCAN COMPLETED IN .*", "", out)
+		out = re.sub(r"SCAN COMPLETED IN .*", "", out)
 		out = out.rstrip(" \n-") + "\n"
 
 		# Print.

tools/readable_bash.py

@@ -124,13 +124,13 @@ def generate_documentation():
  """)
 
 	parser = Source.parser()
-	with open("setup/start.sh", "r") as start_file:
+	with open("setup/start.sh", encoding="utf-8") as start_file:
 		for line in start_file:
 			try:
 				fn = parser.parse_string(line).filename()
 			except:
 				continue
-                        if fn in ("setup/start.sh", "setup/preflight.sh", "setup/questions.sh", "setup/firstuser.sh", "setup/management.sh"):
+			if fn in {"setup/start.sh", "setup/preflight.sh", "setup/questions.sh", "setup/firstuser.sh", "setup/management.sh"}:
 				continue
 
 		import sys
@@ -192,8 +192,7 @@ class CatEOF(Grammar):
 	def value(self):
 		content = self[9].string
 		content = re.sub(r"\\([$])", r"\1", content) # un-escape bash-escaped characters
-		return "<div class='write-to'><div class='filename'>%s <span>(%s)</span></div><pre>%s</pre></div>\n" \
-			% (self[4].string,
+		return "<div class='write-to'><div class='filename'>{} <span>({})</span></div><pre>{}</pre></div>\n".format(self[4].string,
 			   "overwrite" if ">>" not in self[2].string else "append to",
 			   cgi.escape(content))
 
@@ -223,14 +222,14 @@ class EditConf(Grammar):
 		EOL
 		)
 	def value(self):
-		conffile = self[1]
+		# conffile = self[1]
 		options = []
 		eq = "="
 		if self[3] and "-s" in self[3].string: eq = " "
-		for opt in re.split("\s+", self[4].string):
+		for opt in re.split(r"\s+", self[4].string):
 			k, v = opt.split("=", 1)
 			v = re.sub(r"\n+", "", fixup_tokens(v)) # not sure why newlines are getting doubled
-			options.append("%s%s%s" % (k, eq, v))
+			options.append(f"{k}{eq}{v}")
 		return "<div class='write-to'><div class='filename'>" + self[1].string + " <span>(change settings)</span></div><pre>" + "\n".join(cgi.escape(s) for s in options) + "</pre></div>\n"
 
 class CaptureOutput(Grammar):
@@ -248,8 +247,8 @@ class SedReplace(Grammar):
 class EchoPipe(Grammar):
 	grammar = OPTIONAL(SPACE), L("echo "), REST_OF_LINE, L(' | '), REST_OF_LINE, EOL
 	def value(self):
-		text = " ".join("\"%s\"" % s for s in self[2].string.split(" "))
-		return "<pre class='shell'><div>echo " + recode_bash(text) + " \<br> | " + recode_bash(self[4].string) + "</div></pre>\n"
+		text = " ".join(f'"{s}"' for s in self[2].string.split(" "))
+		return "<pre class='shell'><div>echo " + recode_bash(text) + r" \<br> | " + recode_bash(self[4].string) + "</div></pre>\n"
 
 def shell_line(bash):
 	return "<pre class='shell'><div>" + recode_bash(bash.strip()) + "</div></pre>\n"
@@ -324,7 +323,7 @@ def quasitokenize(bashscript):
 		elif c == "\\":
 			# Escaping next character.
 			escape_next = True
-		elif quote_mode is None and c in ('"', "'"):
+		elif quote_mode is None and c in {'"', "'"}:
 			# Starting a quoted word.
 			quote_mode = c
 		elif c == quote_mode:
@@ -364,7 +363,7 @@ def quasitokenize(bashscript):
 			newscript += c
 
 		# "<< EOF" escaping.
-		if quote_mode is None and re.search("<<\s*EOF\n$", newscript):
+		if quote_mode is None and re.search("<<\\s*EOF\n$", newscript):
 			quote_mode = "EOF"
 		elif quote_mode == "EOF" and re.search("\nEOF\n$", newscript):
 			quote_mode = None
@@ -378,7 +377,7 @@ def recode_bash(s):
 			tok = tok.replace(c, "\\" + c)
 		tok = fixup_tokens(tok)
 		if " " in tok or '"' in tok:
-			tok = tok.replace("\"", "\\\"")
+			tok = tok.replace('"', '\\"')
 			tok = '"' + tok +'"'
 		else:
 			tok = tok.replace("'", "\\'")
@@ -401,21 +400,20 @@ class BashScript(Grammar):
 
 	@staticmethod
 	def parse(fn):
-		if fn in ("setup/functions.sh", "/etc/mailinabox.conf"): return ""
-		with open(fn, "r") as f:
+		if fn in {"setup/functions.sh", "/etc/mailinabox.conf"}: return ""
+		with open(fn, encoding="utf-8") as f:
 			string = f.read()
 
 		# tokenize
 		string = re.sub(".* #NODOC\n", "", string)
-		string = re.sub("\n\s*if .*then.*|\n\s*fi|\n\s*else|\n\s*elif .*", "", string)
+		string = re.sub("\n\\s*if .*then.*|\n\\s*fi|\n\\s*else|\n\\s*elif .*", "", string)
 		string = quasitokenize(string)
-		string = re.sub("hide_output ", "", string)
+		string = string.replace(r"hide_output ", "")
 
 		parser = BashScript.parser()
 		result = parser.parse_string(string)
 
-		v = "<div class='row'><div class='col-xs-12 sourcefile'>view the bash source for the following section at <a href=\"%s\">%s</a></div></div>\n" \
-			 % ("https://github.com/mail-in-a-box/mailinabox/tree/master/" + fn, fn)
+		v = "<div class='row'><div class='col-xs-12 sourcefile'>view the bash source for the following section at <a href=\"{}\">{}</a></div></div>\n".format("https://github.com/mail-in-a-box/mailinabox/tree/master/" + fn, fn)
 
 		mode = 0
 		for item in result.value():
@@ -429,7 +427,7 @@ class BashScript(Grammar):
 					mode = 0
 					clz = "contd"
 				if mode == 0:
-					v += "<div class='row %s'>\n" % clz
+					v += f"<div class='row {clz}'>\n"
 					v += "<div class='col-md-6 prose'>\n"
 				v += item
 				mode = 1
@@ -460,17 +458,16 @@ class BashScript(Grammar):
 		v = fixup_tokens(v)
 
 		v = v.replace("</pre>\n<pre class='shell'>", "")
-		v = re.sub("<pre>([\w\W]*?)</pre>", lambda m : "<pre>" + strip_indent(m.group(1)) + "</pre>", v)
+		v = re.sub(r"<pre>([\w\W]*?)</pre>", lambda m : "<pre>" + strip_indent(m.group(1)) + "</pre>", v)
 
 		v = re.sub(r"(\$?)PRIMARY_HOSTNAME", r"<b>box.yourdomain.com</b>", v)
 		v = re.sub(r"\$STORAGE_ROOT", r"<b>$STORE</b>", v)
-		v = v.replace("`pwd`",  "<code><b>/path/to/mailinabox</b></code>")
+		return v.replace("`pwd`",  "<code><b>/path/to/mailinabox</b></code>")
 
-		return v
 
 def wrap_lines(text, cols=60):
 	ret = ""
-	words = re.split("(\s+)", text)
+	words = re.split(r"(\s+)", text)
 	linelen = 0
 	for w in words:
 		if linelen + len(w) > cols-1: