Merge pull request #40 from downtownallday/quota

half-a Quota

.github/workflows/commit-tests.yml

@@ -33,8 +33,7 @@ jobs:
     runs-on: ubuntu-22.04
     env:
       PRIMARY_HOSTNAME: box2.abc.com
-      # TODO: change UPSTREAM_TAG to 'main' once upstream is installable
+      UPSTREAM_TAG: main
-      UPSTREAM_TAG: v67
     steps:
       - uses: actions/checkout@v4
       - name: setup

conf/schema/postfix.schema

@@ -56,13 +56,74 @@ attributetype ( MiabLDAPmailAttributeType:1 NAME 'mailMember' DESC 'RFC6532 utf8
 # create a utf8 version of core 'domainComponent'
 attributetype ( MiabLDAPmailAttributeType:2 NAME 'dcIntl' DESC 'UTF8 domain component' EQUALITY caseIgnoreMatch SUBSTR caseIgnoreSubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE )
 
+# Create mda/lda user mailbox quota settings (for dovecot)
+# format: number | number 'B' | number 'K' | number 'M' | number 'G'
+#
+# Dovecot supports more than one quota rule (but no way to use a
+# multi-valued attribute). Also add additional attributes for
+# more quota rules even though we're not necessarily
+# using them because we might in the future which could help avoid a
+# schema update. Dovecot supports "as many quota rules as you want"
+
+attributetype ( MiabLDAPmailAttributeType:3
+	DESC 'MDA/LDA user mailbox quota'
+	NAME 'mailboxQuota'
+	SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE
+	EQUALITY caseExactMatch )
+
+attributetype ( MiabLDAPmailAttributeType:4
+	DESC 'MDA/LDA user mailbox quota 2'
+	NAME 'mailboxQuota2'
+	SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE
+	EQUALITY caseExactMatch )
+
+attributetype ( MiabLDAPmailAttributeType:5
+	DESC 'MDA/LDA user mailbox quota 3'
+	NAME 'mailboxQuota3'
+	SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE
+	EQUALITY caseExactMatch )
+
+attributetype ( MiabLDAPmailAttributeType:6
+	DESC 'MDA/LDA user mailbox quota 4'
+	NAME 'mailboxQuota4'
+	SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE
+	EQUALITY caseExactMatch )
+
+attributetype ( MiabLDAPmailAttributeType:7
+	DESC 'MDA/LDA user mailbox quota 5'
+	NAME 'mailboxQuota5'
+	SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE
+	EQUALITY caseExactMatch )
+
+attributetype ( MiabLDAPmailAttributeType:8
+	DESC 'MDA/LDA user mailbox quota 6'
+	NAME 'mailboxQuota6'
+	SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE
+	EQUALITY caseExactMatch )
+
+# Dovecot can maintain a flag indicating whether a user is over or
+# under quota. It's use is not required, but enables postfix to reject
+# messages without queuing them when a mailbox is full. The value
+# should be dovecot boolean value 'yes', or 'no'.
+
+attributetype ( MiabLDAPmailAttributeType:9
+	DESC 'MDA/LDA over quota flag'
+	NAME 'mailboxOverQuotaFlag'
+	SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE
+	EQUALITY caseIgnoreMatch )
+
+
+#
+# object classes
+#
+
 objectclass ( 1.3.6.1.4.1.15347.2.1
 	NAME 'mailUser'
 	DESC 'E-Mail User'
 	SUP top
 	AUXILIARY
 	MUST ( uid $ mail $ maildrop )
-	MAY ( cn $ mailbox $ maildest $ mailaccess )
+	MAY ( cn $ mailbox $ maildest $ mailaccess $ mailboxQuota $ mailboxQuota2 $ mailboxQuota3 $ mailboxQuota4 $ mailboxQuota5 $ mailboxQuota6 $ mailboxOverQuotaFlag )
 	)
 
 objectclass ( 1.3.6.1.4.1.15347.2.2

management/cli.py

@@ -74,6 +74,7 @@ if len(sys.argv) < 2:
   {cli} user password user@domain.com [password]
   {cli} user remove user@domain.com
   {cli} user make-admin user@domain.com
+  {cli} user quota user@domain [new-quota]       (get or set user quota)
   {cli} user remove-admin user@domain.com
   {cli} user admins                              (lists admins)
   {cli} user mfa show user@domain.com            (shows MFA devices for user, if any)
@@ -97,6 +98,10 @@ elif sys.argv[1] == "user" and len(sys.argv) == 2:
 			print(user['email'], end='')
 			if "admin" in user['privileges']:
 				print("*", end='')
+			if user['quota'] == '0':
+				print(" unlimited", end='')
+			else:
+				print(" " + user['quota'], end='')
 			print()
 
 elif sys.argv[1] == "user" and sys.argv[2] in {"add", "password"}:
@@ -126,6 +131,14 @@ elif sys.argv[1] == "user" and sys.argv[2] == "admins":
 			if "admin" in user['privileges']:
 				print(user['email'])
 
+elif sys.argv[1] == "user" and sys.argv[2] == "quota" and len(sys.argv) == 4:
+	# Get a user's quota
+	print(mgmt("/mail/users/quota?text=1&email=%s" % sys.argv[3]))
+
+elif sys.argv[1] == "user" and sys.argv[2] == "quota" and len(sys.argv) == 5:
+	# Set a user's quota
+	users = mgmt("/mail/users/quota", { "email": sys.argv[3], "quota": sys.argv[4] })
+
 elif sys.argv[1] == "user" and len(sys.argv) == 5 and sys.argv[2:4] == ["mfa", "show"]:
 	# Show MFA status for a user.
 	status = mgmt("/mfa/status", { "user": sys.argv[4] }, is_json=True)
@@ -150,4 +163,3 @@ elif sys.argv[1] == "alias" and sys.argv[2] == "remove" and len(sys.argv) == 4:
 else:
 	print("Invalid command-line arguments.")
 	sys.exit(1)
-

management/daemon.py

@@ -30,6 +30,7 @@ import auth, utils
 from mailconfig import get_mail_users, get_mail_users_ex, get_admins, add_mail_user, set_mail_password, set_mail_display_name, remove_mail_user
 from mailconfig import get_mail_user_privileges, add_remove_mail_user_privilege
 from mailconfig import get_mail_aliases, get_mail_aliases_ex, get_mail_domains, add_mail_alias, remove_mail_alias
+from mailconfig import get_mail_quota, set_mail_quota
 from mfa import get_public_mfa_state, enable_mfa, disable_mfa
 import mfa_totp
 import contextlib
@@ -201,8 +202,31 @@ def mail_users():
 @app.route('/mail/users/add', methods=['POST'])
 @authorized_personnel_only
 def mail_users_add():
+	quota = request.form.get('quota', '0')
 	try:
-		return add_mail_user(request.form.get('email', ''), request.form.get('password', ''), request.form.get('privileges', ''), request.form.get('display_name', ''), env)
+		return add_mail_user(request.form.get('email', ''), request.form.get('password', ''), request.form.get('privileges', ''), quota, request.form.get('display_name', ''), env)
+	except ValueError as e:
+		return (str(e), 400)
+
+@app.route('/mail/users/quota', methods=['GET'])
+@authorized_personnel_only
+def get_mail_users_quota():
+	email = request.values.get('email', '')
+	quota = get_mail_quota(email, env)
+
+	if request.values.get('text'):
+		return quota
+
+	return json_response({
+		"email": email,
+		"quota": quota
+	})
+
+@app.route('/mail/users/quota', methods=['POST'])
+@authorized_personnel_only
+def mail_users_quota():
+	try:
+		return set_mail_quota(request.form.get('email', ''), request.form.get('quota'), env)
 	except ValueError as e:
 		return (str(e), 400)
 

management/mailconfig.py

@@ -19,8 +19,11 @@
 # Python 3 in setup/questions.sh to validate the email
 # address entered by the user.
 
-import subprocess, shutil, os, sqlite3, re, ldap3, uuid, hashlib
+import os, sqlite3, re
-import utils, backend
+import subprocess
+import ldap3, uuid, hashlib, backend
+
+import utils
 from email_validator import validate_email as validate_email_, EmailNotValidError
 import idna
 import socket
@@ -283,6 +286,30 @@ def get_mail_users(env, as_map=False, map_by="maildrop"):
 		return utils.sort_email_addresses(users, env)
 
 
+def sizeof_fmt(num):
+	for unit in ['','K','M','G','T']:
+		if abs(num) < 1024.0:
+			if abs(num) > 99:
+				return "%3.0f%s" % (num, unit)
+			else:
+				return "%2.1f%s" % (num, unit)
+
+		num /= 1024.0
+
+	return str(num)
+
+def sizeof_fmt(num):
+	for unit in ['','K','M','G','T']:
+		if abs(num) < 1024.0:
+			if abs(num) > 99:
+				return "%3.0f%s" % (num, unit)
+			else:
+				return "%2.1f%s" % (num, unit)
+
+		num /= 1024.0
+
+	return str(num)
+
 def get_mail_users_ex(env, with_archived=False):
 	# Returns a complex data structure of all user accounts, optionally
 	# including archived (status="inactive") accounts.
@@ -307,20 +334,52 @@ def get_mail_users_ex(env, with_archived=False):
 	users = []
 	active_accounts = set()
 	c = open_database(env)
-	response = c.wait( c.search(env.LDAP_USERS_BASE, "(objectClass=mailUser)", attributes=['mail','maildrop','mailaccess','cn']) )
+	response = c.wait( c.search(env.LDAP_USERS_BASE, "(objectClass=mailUser)", attributes=['mail','maildrop','mailaccess','mailboxQuota','cn']) )
 
 	for rec in response:
 		#email = rec['maildrop'][0]
 		email = rec['mail'][0]
 		privileges = rec['mailaccess']
+		quota = rec['mailboxQuota'][0] if len(rec['mailboxQuota'])>0 else '0'
 		display_name = rec['cn'][0]
 		active_accounts.add(email)
+
+		(user, domain) = email.split('@')
+		box_size = 0
+		box_quota = 0
+		percent = ''
+		try:
+			dirsize_file = os.path.join(env['STORAGE_ROOT'], 'mail/mailboxes/%s/%s/maildirsize' % (domain, user))
+			with open(dirsize_file, 'r') as f:
+				box_quota = int(f.readline().split('S')[0])
+				for line in f.readlines():
+					(size, count) = line.split(' ')
+					box_size += int(size)
+
+			try:
+				percent = (box_size / box_quota) * 100
+			except:
+				percent = 'Error'
+
+		except:
+			box_size = '?'
+			box_quota = '?'
+			percent = '?'
+
+		if quota == '0':
+			percent = ''
+
 		user = {
 			"email": email,
 			"privileges": privileges,
+			"quota": quota,
 			"status": "active",
-			"display_name": display_name
+			"display_name": display_name,
+			"box_quota": box_quota,
+			"box_size": sizeof_fmt(box_size) if box_size != '?' else box_size,
+			"percent": '%3.0f%%' % percent if type(percent) != str else percent,
 		}
+
 		users.append(user)
 
 	# Add in archived accounts.
@@ -337,7 +396,10 @@ def get_mail_users_ex(env, with_archived=False):
 						"privileges": [],
 						"status": "inactive",
 						"mailbox": mbox,
-						"display_name": ""
+						"display_name": "",
+                        "box_size": '?',
+                        "box_quota": '?',
+                        "percent": '?',
 					}
 					users.append(user)
 
@@ -697,12 +759,13 @@ def remove_mail_domain(env, domain_idna, validate=True):
 	return True
 
 
-def add_mail_user(email, pw, privs, display_name, env):
+def add_mail_user(email, pw, privs, quota, display_name, env):
 	# Add a new mail user.
 	#
 	# email: the new user's email address (idna)
 	# pw: the new user's password
 	# privs: either an array of privilege strings, or a newline
+	# quota: a string (number | number 'M' | number 'G') or None
 	# separated string of privilege names
 	# display_name: a string with users givenname and surname (eg "Al Woods")
 	#
@@ -735,6 +798,22 @@ def add_mail_user(email, pw, privs, display_name, env):
 		validation = validate_privilege(p)
 		if validation: return validation
 
+	if quota is None:
+		quota = '0'
+
+	try:
+		quota = validate_quota(quota)
+	except ValueError as e:
+		return (str(e), 400)
+
+	if quota is None:
+		quota = '0'
+
+	try:
+		quota = validate_quota(quota)
+	except ValueError as e:
+		return (str(e), 400)
+
 	# get the database
 	conn = open_database(env)
 
@@ -773,6 +852,7 @@ def add_mail_user(email, pw, privs, display_name, env):
 		"maildrop" : email.lower(),
 		"uid" : uid,
 		"mailaccess": privs,
+		"mailboxQuota": quota,
 		"cn": cn,
 		"sn": sn,
 		"shadowLastChange": backend.get_shadowLastChanged()
@@ -805,6 +885,10 @@ def add_mail_user(email, pw, privs, display_name, env):
 	# convert alias's mailMember to member
 	convert_mailMember(env, conn, dn, email)
 
+	dovecot_quota_recalc(email)
+
+	dovecot_quota_recalc(email)
+
 	# Update things in case any new domains are added.
 	if domain_added:
 		return kick(env, return_status)
@@ -866,6 +950,51 @@ def validate_login(email, pw, env):
 	except ldap3.core.exceptions.LDAPInvalidCredentialsResult:
 		return False
 
+def validate_quota(quota):
+	# validate quota
+	quota = quota.strip().upper()
+
+	if quota == "":
+		raise ValueError("No quota provided.")
+	if re.search(r"[\s,.]", quota):
+		raise ValueError("Quotas cannot contain spaces, commas, or decimal points.")
+	if not re.match(r'^[\d]+[GMK]?$', quota):
+		raise ValueError("Invalid quota.")
+
+	return quota
+
+def get_mail_quota(email, env):
+	user = find_mail_user(env, email, ['mailboxQuota'])
+	if user is None:
+		return ("That's not a user (%s)." % email, 400)
+	if len(user['mailboxQuota'])==0:
+		return '0'
+	else:
+		return user['mailboxQuota'][0]
+
+def set_mail_quota(email, quota, env):
+	# validate that password is acceptable
+	quota = validate_quota(quota)
+
+	# update the database
+	conn = open_database(env)
+	user = find_mail_user(env, email, ['mailboxQuota'], conn)
+	if user is None:
+		return ("That's not a user (%s)." % email, 400)
+
+	conn.modify_record(user, { 'mailboxQuota': quota })
+	dovecot_quota_recalc(email)
+	return "OK"
+
+def dovecot_quota_recalc(email):
+	# dovecot processes running for the user will not recognize the new quota setting
+	# a reload is necessary to reread the quota setting, but it will also shut down
+	# running dovecot processes.  Email clients generally log back in when they lose
+	# a connection.
+	# subprocess.call(['doveadm', 'reload'])
+
+	# force dovecot to recalculate the quota info for the user.
+	subprocess.call(["doveadm", "quota", "recalc", "-u", email])
 
 def get_mail_password(email, env):
 	# Gets the hashed passwords for a user. In ldap, userPassword is
@@ -1329,12 +1458,12 @@ def remove_mail_alias(address_utf8, env, do_kick=True, auto=None, ignore_if_not_
 		return return_status
 
 
-def add_auto_aliases(aliases, env):
+# def add_auto_aliases(aliases, env):
-	conn, c = open_database(env, with_connection=True)
+# 	conn, c = open_database(env, with_connection=True)
-	c.execute("DELETE FROM auto_aliases")
+# 	c.execute("DELETE FROM auto_aliases")
-	for source, destination in aliases.items():
+# 	for source, destination in aliases.items():
-		c.execute("INSERT INTO auto_aliases (source, destination) VALUES (?, ?)", (source, destination))
+# 		c.execute("INSERT INTO auto_aliases (source, destination) VALUES (?, ?)", (source, destination))
-	conn.commit()
+# 	conn.commit()
 
 def get_system_administrator(env):
 	return "administrator@" + env['PRIMARY_HOSTNAME']

management/templates/users.html

@@ -6,6 +6,7 @@
 #user_table .account_inactive .if_active { display: none; }
 #user_table .account_active .if_inactive { display: none; }
 #user_table .account_active.if_inactive { display: none; }
+.row-center { text-align: center; }
 </style>
 
 <h3>Add a mail user</h3>
@@ -34,6 +35,10 @@
     <div>Display Name</div>
     <input id="adduserDisplayName" class="form-control" type="text" placeholder="eg: John Smith">
   </div>
+  <div class="form-group">
+    <div>Quota</div>
+    <input type="text" class="form-control" id="adduserQuota" placeholder="Quota" style="width:5em;" value="0">
+  </div>
   <div class="text-center">
     <div>&nbsp;</div>
     <button type="submit" class="btn btn-primary">Add User</button>
@@ -44,13 +49,17 @@
   <li>Use <a href="#aliases">aliases</a> to create email addresses that forward to existing accounts.</li>
   <li>Administrators get access to this control panel.</li>
   <li>User accounts cannot contain any international (non-ASCII) characters, but <a href="#aliases">aliases</a> can.</li>
+  <li>Quotas may not contain any spaces, commas or decimal points. Suffixes of G (gigabytes) and M (megabytes) are allowed.  For unlimited storage enter 0 (zero)</li>
 </ul>
 
 <h3>Existing mail users</h3>
 <table id="user_table" class="table" style="width: auto">
   <thead>
     <tr>
-      <th width="50%">Email Address</th>
+      <th width="35%">Email Address</th>
+      <th class="row-center">Size</th>
+      <th class="row-center">Used</th>
+      <th class="row-center">Quota</th>
       <th>Actions</th>
     </tr>
   </thead>
@@ -64,10 +73,21 @@
     <td>
       <span class="address"></span> <span class="display_name_wrapper">(<a class="display_name" href="#" onclick="users_set_displayname(this); return false;" title="Change display name"></a>)</span>
     </td>
+    <td class="box-size row-center"></td>
+    <td class="percent row-center"></td>
+	<td class="quota row-center">
+	</td>
     <td class='actions'>
         <span class='privs'>
         </span>
 
+        <span class="if_active">
+          <a href="#" onclick="users_set_quota(this); return false;" class='setquota' title="Set Quota">
+              set quota
+          </a>
+          |
+        </span>
+
         <span class="if_active">
           <a href="#" onclick="users_set_password(this); return false;" class='setpw' title="Set Password">
             set password
@@ -108,10 +128,28 @@
 <table class="table" style="margin-top: .5em">
 <thead><th>Verb</th> <th>Action</th><th></th></thead>
 <tr><td>GET</td><td><i>(none)</i></td> <td>Returns a list of existing mail users. Adding <code>?format=json</code> to the URL will give JSON-encoded results.</td></tr>
-<tr><td>POST</td><td>/add</td> <td>Adds a new mail user. Required POST-body parameters are <code>email</code> and <code>password</code>.</td></tr>
+<tr>
-<tr><td>POST</td><td>/remove</td> <td>Removes a mail user. Required POST-body parameter is <code>email</code>.</td></tr>
+    <td>POST</td>
+    <td>/add</td>
+    <td>Adds a new mail user. Required POST-body parameters are <code>email</code> and <code>password</code>. Optional parameters: <code>privilege=admin</code> and <code>quota</code></td>
+</tr>
+<tr>
+    <td>POST</td>
+    <td>/remove</td>
+    <td>Removes a mail user. Required POST-by parameter is <code>email</code>.</td>
+</tr>
 <tr><td>POST</td><td>/privileges/add</td> <td>Used to make a mail user an admin. Required POST-body parameters are <code>email</code> and <code>privilege=admin</code>.</td></tr>
 <tr><td>POST</td><td>/privileges/remove</td> <td>Used to remove the admin privilege from a mail user. Required POST-body parameter is <code>email</code>.</td></tr>
+<tr>
+    <td>GET</td>
+    <td>/quota</td>
+    <td>Get the quota for a mail user. Required POST-body parameters are <code>email</code> and will return JSON result</td>
+</tr>
+<tr>
+    <td>POST</td>
+    <td>/quota</td>
+    <td>Set the quota for a mail user. Required POST-body parameters are <code>email</code> and <code>quota</code>.</td>
+</tr>
 </table>
 
 <h4>Examples:</h4>
@@ -144,7 +182,7 @@ function show_users() {
     function(r) {
       $('#user_table tbody').html("");
       for (var i = 0; i < r.length; i++) {
-        var hdr = $("<tr><th colspan='2' style='background-color: #EEE'></th></tr>");
+        var hdr = $("<tr><th colspan='6' style='background-color: #EEE'></th></tr>");
         hdr.find('th').text(r[i].domain);
         $('#user_table tbody').append(hdr);
 
@@ -162,7 +200,14 @@ function show_users() {
           n2.addClass("account_" + user.status);
 
           n.attr('data-email', user.email);
+          n.attr('data-quota', user.quota);
           n.find('.address').text(user.email);
+          n.find('.box-size').text(user.box_size);
+          if (user.box_size == '?') {
+            n.find('.box-size').attr('title', 'Mailbox size is unkown')
+          }
+          n.find('.percent').text(user.percent);
+          n.find('.quota').text((user.quota == '0') ? 'unlimited' : user.quota);
           if (user.status == "inactive") {
             n.find('.display_name_wrapper').text('[archived]');
           }
@@ -197,6 +242,7 @@ function do_add_user() {
   var email = $("#adduserEmail").val();
   var pw = $("#adduserPassword").val();
   var privs = $("#adduserPrivs").val();
+  var quota = $("#adduserQuota").val();
   var display_name = $("#adduserDisplayName").val();
   api(
     "/mail/users/add",
@@ -205,6 +251,7 @@ function do_add_user() {
       email: email,
       password: pw,
       privileges: privs,
+      quota: quota,
       display_name: display_name
     },
     function(r) {
@@ -272,6 +319,36 @@ function users_set_displayname(elem) {
       });
 }
 
+function users_set_quota(elem) {
+    var email = $(elem).parents('tr').attr('data-email');
+    var quota = $(elem).parents('tr').attr('data-quota');
+
+    show_modal_confirm(
+        "Set Quota",
+        $("<p>Set quota for <b>" + email + "</b>?</p>" +
+            "<p>" +
+            "<label for='users_set_quota' style='display: block; font-weight: normal'>Quota:</label>" +
+            "<input type='text' id='users_set_quota' value='" + quota + "'></p>" +
+            "<p><small>Quotas may not contain any spaces or commas.  Suffixes of G (gigabytes) and M (megabytes) are allowed.</small></p>" +
+            "<p><small>For unlimited storage enter 0 (zero)</small></p>"),
+        "Set Quota",
+        function() {
+            api(
+                "/mail/users/quota",
+                "POST",
+                {
+                    email: email,
+                    quota: $('#users_set_quota').val()
+                },
+                function(r) {
+                    show_users();
+                },
+                function(r) {
+                    show_modal_error("Set Quota", r);
+                });
+        });
+}
+
 function users_remove(elem) {
   var email = $(elem).parents('tr').attr('data-email');
 
@@ -337,7 +414,7 @@ function generate_random_password() {
   var charset = "ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz23456789"; // confusable characters skipped
   for (var i = 0; i < 12; i++)
     pw += charset.charAt(Math.floor(Math.random() * charset.length));
-  show_modal_error("Random Password", "<p>Here, try this:</p> <p><code style='font-size: 110%'>" + pw + "</code></pr");
+  show_modal_error("Random Password", "<p>Here, try this:</p> <p><code style='font-size: 110%'>" + pw + "</code></p>");
   return false; // cancel click
 }
 </script>

setup/bootstrap.sh

@@ -147,4 +147,3 @@ if [ "${ENCRYPTION_AT_REST:-false}" = "true" ]; then
 else
     setup/start.sh </dev/tty
 fi
-

setup/ldap.sh

@@ -777,7 +777,7 @@ process_cmdline() {
 		if [ "$s" == "permitted-senders" -o "$s" == "ps" ]; then
 			echo ""
 			echo '--------------------------------'
-			local attrs=(mail member mailRoutingAddress rfc822MailMember)
+			local attrs=(mail member mailRoutingAddress mailMember)
 			[ ${verbose:-0} -gt 0 ] && attrs=()
 			debug_search "(objectClass=mailGroup)" "$LDAP_PERMITTED_SENDERS_BASE" ${attrs[@]}
 		fi

setup/mail-dovecot.sh

@@ -76,6 +76,32 @@ tools/editconf.py /etc/dovecot/conf.d/10-mail.conf \
 
 # Create, subscribe, and mark as special folders: INBOX, Drafts, Sent, Trash, Spam and Archive.
 cp conf/dovecot-mailboxes.conf /etc/dovecot/conf.d/15-mailboxes.conf
+sed -i "s/#mail_plugins =\(.*\)/mail_plugins =\1 \$mail_plugins quota/" /etc/dovecot/conf.d/10-mail.conf
+if ! grep -q "mail_plugins.* imap_quota" /etc/dovecot/conf.d/20-imap.conf; then
+  sed -i "s/\(mail_plugins =.*\)/\1\n  mail_plugins = \$mail_plugins imap_quota/" /etc/dovecot/conf.d/20-imap.conf
+fi
+
+# configure stuff for quota support
+if ! grep -q "quota_status_success = DUNNO" /etc/dovecot/conf.d/90-quota.conf; then
+    cat > /etc/dovecot/conf.d/90-quota.conf << EOF;
+plugin {
+  quota = maildir:User quota
+
+  quota_grace = 10%%
+
+  quota_status_success = DUNNO
+  quota_status_nouser = DUNNO
+  quota_status_overquota = "522 5.2.2 Mailbox is full"
+}
+
+service quota-status {
+    executable = quota-status -p postfix
+    inet_listener {
+        port = 12340
+    }
+}
+EOF
+fi
 
 # ### IMAP/POP
 

setup/mail-postfix.sh

@@ -259,7 +259,7 @@ tools/editconf.py /etc/postfix/main.cf  -e lmtp_destination_recipient_limit=
 # "450 4.7.1 Client host rejected: Service unavailable". This is a retry code, so the mail doesn't properly bounce. #NODOC
 tools/editconf.py /etc/postfix/main.cf \
 	smtpd_sender_restrictions="reject_non_fqdn_sender,reject_unknown_sender_domain,reject_authenticated_sender_login_mismatch,reject_rhsbl_sender dbl.spamhaus.org=127.0.1.[2..99]" \
-	smtpd_recipient_restrictions="permit_sasl_authenticated,permit_mynetworks,reject_rbl_client zen.spamhaus.org=127.0.0.[2..11],reject_unlisted_recipient,check_policy_service unix:private/policy-spf,check_policy_service inet:127.0.0.1:10023"
+	smtpd_recipient_restrictions="permit_sasl_authenticated,permit_mynetworks,reject_rbl_client zen.spamhaus.org=127.0.0.[2..11],reject_unlisted_recipient,check_policy_service unix:private/policy-spf,check_policy_service inet:127.0.0.1:10023,check_policy_service inet:127.0.0.1:12340"
 
 # Postfix connects to Postgrey on the 127.0.0.1 interface specifically. Ensure that
 # Postgrey listens on the same interface (and not IPv6, for instance).

setup/mail-users.sh

@@ -88,8 +88,13 @@ pass_attrs = maildrop=user
 #   Post-login information specific to the user (eg. quotas).  For
 #   lmtp delivery, pass_filter is not used, and postfix has already
 #   rewritten the envelope using the maildrop address.
+#   %$ is expanded to mailboxQuota's value.
 user_filter = (&(objectClass=mailUser)(|(mail=%u)(maildrop=%u)))
-user_attrs = maildrop=user
+user_attrs = maildrop=user, \
+             mailboxQuota=quota_rule=*:bytes=%\$, \
+             =quota_rule2=Trash:storage=+100M, \
+             =quota_rule3=Drafts:storage=+25M, \
+             =quota_rule4=Sent:storage=+50M
 
 # Account iteration for various dovecot tools (doveadm)
 iterate_filter = (objectClass=mailUser)
@@ -269,3 +274,6 @@ chmod 0640 /etc/postfix/virtual-alias-maps.cf
 
 restart_service postfix
 restart_service dovecot
+
+# force a recalculation of all user quotas
+doveadm quota recalc -A

setup/migrate.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/python3 -u
 # -*- indent-tabs-mode: t; tab-width: 8; python-indent-offset: 8; -*-
 #####
 ##### This file is part of Mail-in-a-Box-LDAP which is released under the
@@ -200,6 +200,12 @@ def migration_14(env):
 	db = os.path.join(env["STORAGE_ROOT"], 'mail/users.sqlite')
 	shell("check_call", ["sqlite3", db, "CREATE TABLE auto_aliases (id INTEGER PRIMARY KEY AUTOINCREMENT, source TEXT NOT NULL UNIQUE, destination TEXT NOT NULL, permitted_senders TEXT);"])
 
+def migration_15(env):
+	# Add a column to the users table to store their quota limit.  Default to '0' for unlimited.
+	db = os.path.join(env["STORAGE_ROOT"], 'mail/users.sqlite')
+	shell("check_call", ["sqlite3", db, "ALTER TABLE users ADD COLUMN quota TEXT NOT NULL DEFAULT '0';"])
+
+
 ###########################################################
 
 
@@ -306,7 +312,7 @@ def migration_miabldap_2(env):
 			return ldif.replace("rfc822MailMember: ", "mailMember: ")
 		# apply schema changes miabldap/1 -> miabldap/2
 		ldap.unbind()
-		print("Apply schema changes")
+		print("Apply schema changes to support utf8 email addresses")
 		m14.apply_schema_changes(env, ldapvars, ldif_change_fn)
 		# reconnect
 		ldap = connect(ldapvars)
@@ -329,6 +335,52 @@ def migration_miabldap_2(env):
 
 	ldap.unbind()
 
+def migration_miabldap_3(env):
+	# This migration step changes the ldap schema to support quotas
+	#
+	# possible states at this point:
+	#   miabldap was installed and is being upgraded
+	#      -> schema update needed
+	#   a miab install was present and step 1 upgaded it to miabldap
+	#      -> new schema already present
+	#
+	sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), "../management")))
+	import ldap3
+	from backend import connect
+	import migration_14 as m14
+
+	# 1. get ldap site details
+	ldapvars = load_env_vars_from_file(os.path.join(env["STORAGE_ROOT"], "ldap/miab_ldap.conf"), strip_quotes=True)
+
+	# connect before schema changes to ensure admin password works
+	ldap = connect(ldapvars)
+
+	# 2. if this is a miab -> maibldap install, the new schema is
+	# already in place and no schema changes are needed. however,
+	# if this is a miabldap/1 to miabldap/2 migration, we must
+	# upgrade the schema.
+	ret = shell("check_output", [
+		"ldapsearch",
+		"-Q",
+		"-Y", "EXTERNAL",
+		"-H", "ldapi:///",
+		"(&(objectClass=olcSchemaConfig)(cn={*}postfix))",
+		"-b", "cn=schema,cn=config",
+		"-o", "ldif_wrap=no",
+		"-LLL",
+		"olcObjectClasses"
+	])
+
+	ldap.unbind()
+
+	if "mailboxQuota" not in ret:
+		def ldif_change_fn(ldif):
+			# the schema change we're making does not require any data changes
+			return ldif
+		# apply schema changes miabldap/2 -> miabldap/3
+		print("Apply schema changes to support mailbox quotas")
+		m14.apply_schema_changes(env, ldapvars, ldif_change_fn)
+
 
 def get_current_migration():
 	ver = 0
@@ -419,7 +471,7 @@ def run_miabldap_migrations():
 			migration_id = 0
 		else:
 			print()
-			print("%s file doesn't exists. Skipping migration..." % (migration_id_file,))
+			print("%s file doesn't exist. Skipping migration..." % (migration_id_file,))
 			return
 
 	ourver = int(migration_id)

setup/migration_13.py

@@ -17,7 +17,7 @@
 import uuid, os, sqlite3, ldap3, hashlib
 
 
-def add_user(env, ldapconn, search_base, users_base, domains_base, email, password, privs, totp, cn=None):
+def add_user(env, ldapconn, search_base, users_base, domains_base, email, password, privs, quota, totp, cn=None):
 	# Add a sqlite user to ldap
 	#   env are the environment variables
 	#   ldapconn is the bound ldap connection
@@ -27,6 +27,7 @@ def add_user(env, ldapconn, search_base, users_base, domains_base, email, passwo
 	#   email is the user's email
 	#   password is the user's current sqlite password hash
 	#   privs is an array of privilege names for the user
+	#   quota is the users mailbox quota (string; defaults to '0')
 	#   totp contains the list of secrets, mru tokens, and labels
 	#   cn is the user's common name [optional]
 	#
@@ -52,6 +53,7 @@ def add_user(env, ldapconn, search_base, users_base, domains_base, email, passwo
 		"mail" : email,
 		"maildrop" : email,
 		"uid" : uid,
+		"mailboxQuota": quota,
 		# Openldap uses prefix {CRYPT} for all crypt(3) formats
 		"userPassword" : password.replace('{SHA512-CRYPT}','{CRYPT}')
 	}
@@ -116,7 +118,7 @@ def create_users(env, conn, ldapconn, ldap_base, ldap_users_base, ldap_domains_b
 
 	# select users
 	c = conn.cursor()
-	c.execute("SELECT id, email, password, privileges from users")
+	c.execute("SELECT id, email, password, privileges, quota from users")
 
 	users = {}
 	for row in c:
@@ -124,6 +126,7 @@ def create_users(env, conn, ldapconn, ldap_base, ldap_users_base, ldap_domains_b
 		email=row[1]
 		password=row[2]
 		privs=row[3]
+		quota=row[4]
 		totp = None
 
 		c2 = conn.cursor()
@@ -143,7 +146,7 @@ def create_users(env, conn, ldapconn, ldap_base, ldap_users_base, ldap_domains_b
 			totp["label"].append("{%s}%s" % (rowidx, row2[2] or ''))
 			rowidx += 1
 
-		dn = add_user(env, ldapconn, ldap_base, ldap_users_base, ldap_domains_base, email, password, privs.split("\n"), totp)
+		dn = add_user(env, ldapconn, ldap_base, ldap_users_base, ldap_domains_base, email, password, privs.split("\n"), quota, totp)
 		users[email] = dn
 	return users
 

setup/webmail.sh

@@ -214,6 +214,7 @@ cat > $RCM_CONFIG <<EOF;
 
 /* prevent CSRF, requires php 7.3+ */
 \$config['session_samesite'] = 'Strict';
+\$config['quota_zero_as_unlimited'] = true;
 ?>
 EOF
 

tests/lib/installed-state.sh

@@ -138,6 +138,7 @@ installed_state_compare() {
     RELEASE_A="${RELEASE:-0}"
     PROD_A="miab"
     grep "mailinabox-ldap" <<<"$GIT_ORIGIN" >/dev/null && PROD_A="miabldap"
+    MIGRATION_ML_VERSION_A="${MIGRATION_ML_VERSION:-0}"
 
     source "$s2/info.txt"
     MAJOR_B="$MAJOR"
@@ -145,6 +146,7 @@ installed_state_compare() {
     RELEASE_B="${RELEASE:-0}"
     PROD_B="miab"
     grep "mailinabox-ldap" <<<"$GIT_ORIGIN" >/dev/null && PROD_B="miabldap"
+    MIGRATION_ML_VERSION_B="${MIGRATION_ML_VERSION:-0}"
 
     cmptype="${PROD_A}2${PROD_B}"
 
@@ -176,6 +178,13 @@ installed_state_compare() {
 
         # s2: re-sort aliases
         jq -c ".[] | .aliases | sort_by(.address) | .[] | {address:.address, forwards_to:.forwards_to, permitted_senders:.permitted_senders, auto:.auto, description:.description}"  "$s2/aliases.json" > "$s2/aliases-cmp.json"
+
+        if [ $MIGRATION_ML_VERSION_A -le 2 -a $MIGRATION_ML_VERSION_B -ge 3 ]; then
+            # miabldap migration level <=2 does not have quota fields, so
+            # remove them from the comparison
+            grep -vE '"(quota|box_quota|box_size|percent)":' "$s2/users-cmp.json" > "$s2/users-cmp2.json" || changed="true"
+            cp "$s2/users-cmp2.json" "$s2/users-cmp.json" && rm -f "$s2/users-cmp2.json"
+        fi
     fi
 
 

tests/suites/_mgmt-functions.sh

@@ -400,3 +400,34 @@ mgmt_assert_admin_login() {
 	fi
 	return 0
 }
+
+
+mgmt_get_user_quota() {
+    local user="$1"
+    record "[get user $user quota]"
+	mgmt_rest GET "/admin/mail/users/quota?email=$user"
+	local rc=$?
+    # REST_OUTPUT contains json, eg:
+    #   { "email": "alice@somedomain.com", "quota": "5000" }
+    if [ $rc -eq 0 ]; then
+        # output to stdout the quota value
+        QUOTA="$(/usr/bin/jq -r ".quota" <<<"$REST_OUTPUT" 2>>$TEST_OF)"
+        if [ $? -ne 0 ]; then
+            record "could not obtain quota member from json using jq"
+            rc=1
+        fi
+    else
+        QUOTA="error"
+    fi
+	return $rc
+
+}
+
+mgmt_set_user_quota() {
+    local user="$1"
+    local quota="$2"
+    record "[set user $user quota to $quota]"
+	mgmt_rest POST "/admin/mail/users/quota" "email=$user" "quota=$quota"
+	local rc=$?
+	return $rc
+}

tests/suites/management-users.sh

@@ -361,12 +361,123 @@ test_totp() {
 
 
 
+test_mailbox_quotas() {
+    test_start "mailbox-quotas"
+
+    # create standard user alice
+    local alice="alice@somedomain.com"
+    create_user "$alice" "alice"
+
+    # quota should be unlimited for newly added users
+    if ! mgmt_get_user_quota "$alice"; then
+        test_failure "Unable to get $alice's quota: $REST_ERROR"
+    elif [ "$QUOTA" != "0" -a "$QUOTA" != "unlimited" ]; then
+        test_failure "A newly created user should have unlimited quota"
+    fi
+
+    # get alice's current total number of messages. should be 0 unless
+    # the account was "archived"
+    local count_messages="$(doveadm -f json quota get -u "$alice" | jq -r '.[] | select(.type=="MESSAGE") | .value')"
+    record "$alice currently has $count_messages messages"
+
+    # set alice's quota to a small number
+    local quota_value="5K"
+    if ! mgmt_set_user_quota "$alice" "$quota_value"
+    then
+        test_failure "Unable to set $alice's quota: $REST_ERROR"
+    else
+        # read back the quota - make sure it's what we set
+        if ! mgmt_get_user_quota "$alice" || [ "$QUOTA" != "$quota_value" ]
+        then
+            test_failure "Setting quota failed - expected quota does not match current quota: $REST_OUTPUT $REST_ERROR QUOTA=$QUOTA"
+
+        else
+            record_search "(mail=$alice)"
+        fi
+    fi
+
+    if ! have_test_failures; then
+        # send messages large enough to exceed the quota
+        local output
+        local subjects=()
+        local msgidx=0
+        local body="$(python3 -c 'for i in range(0,int(512/4)): print("abc\n", end="")')"
+        local quota_exceeded="no"
+
+        while ! have_test_failures && [ $msgidx -lt 10 ]; do
+            record ""
+            record "[send msg $msgidx]"
+            local subj="msg$msgidx - $(generate_password)"
+            output="$($PYMAIL -smtp-debug -body-from-stdin -no-delete -subj "$subj" $PRIVATE_IP $alice alice <<<"$body" 2>&1)"
+		    if ! assert_python_success $? "$output"; then
+                break
+            fi
+
+            # You'd expect that the send would fail when the quota is
+            # exceeded, but it doesn't. Postfix accepts it into it's
+            # queue, then bounces the message back to sender with
+            # delivery status notification (DSN) of 5.2.2 when it
+            # processes the queue.
+            #
+            # The debugging messages (turned on by the -smtp-debug
+            # argument) hold the internal postfix message id, so
+            # extract that, then grep the logs to see if the message
+            # was bounced due to 5.2.2.
+
+            local postid="$(awk '/^data: .* queued as/  { match($0," as "); print substr($0,RSTART+4,10); exit }' <<<"$output" 2>>$TEST_OF)"
+            record "Extracted POSTID=$postid"
+            if [ ! -z "$postid" ]; then
+                /usr/sbin/postqueue -f >>"$TEST_OF" 2>&1
+                flush_logs
+                record "[dovecot and postfix logs for msg $msgidx]"
+                record "logs: $(grep "$postid" /var/log/mail.log)"
+
+                if grep "$postid" /var/log/mail.log | grep "status=bounced" | grep -Fq "5.2.2"; then
+                    # success - message was rejected
+                    quota_exceeded="yes"
+                    break
+                fi
+            fi
+
+            subjects+=( "$subj" )
+            let msgidx+=1
+            # doveadm quota get -u "$alice" >>"$TEST_OF" 2>&1
+        done
+
+        if ! have_test_failures && [ "$quota_exceeded" = "no" ]; then
+            test_failure "Quota restriction was not enforced by dovecot after sending $msgidx messages"
+        fi
+
+        # cleanup: delete the messages
+        msgidx=0
+        for subj in "${subjects[@]}"; do
+            record "[delete msg $msgidx]"
+            record "subj=$subj"
+            $PYMAIL -no-send -timeout 2 -subj "$subj" $PRIVATE_IP $alice alice >>$TEST_OF 2>&1
+            let msgidx+=1
+        done
+
+        # verify cleanup worked
+        local cur_count_messages="$(doveadm -f json quota get -u "$alice" | jq -r '.[] | select(.type=="MESSAGE") | .value')"
+        if [ $count_messages -ne $cur_count_messages ]; then
+            test_failure "Cleanup failed: test account $alice started with $count_messages but ended up with $cur_count_messages"
+        fi
+    fi
+
+    # cleanup: delete the test user
+    delete_user "$alice"
+    test_end
+}
+
+
+
+
 suite_start "management-users" mgmt_start
 
 test_totp
 test_mixed_case_domains
 test_mixed_case_users
 test_intl_domains
+test_mailbox_quotas
 
 suite_end mgmt_end
-

tests/system-setup/setup-defaults.sh

@@ -64,7 +64,7 @@ export UPSTREAM_TAG="${UPSTREAM_TAG:-}"
 
 # For setup scripts that install miabldap releases (eg. upgrade tests)
 export MIABLDAP_GIT="${MIABLDAP_GIT:-https://github.com/downtownallday/mailinabox-ldap.git}"
-export MIABLDAP_RELEASE_TAG="${MIABLDAP_RELEASE_TAG:-v60}"
+export MIABLDAP_RELEASE_TAG="${MIABLDAP_RELEASE_TAG:-v70}"
 
 # When running tests that require php, use this version of php. This
 # should be the same as what's in setup/functions.sh.
@@ -73,3 +73,4 @@ export PHP_VER=8.0
 # Tag of last version supported on Ubuntu Bionic 18.04
 UPSTREAM_FINAL_RELEASE_TAG_BIONIC64=v57a
 MIABLDAP_FINAL_RELEASE_TAG_BIONIC64=f6cd8f56c3bcb20969c6cf66c040c8efa3773f3a
+MIABLDAP_INITIAL_RELEASE_TAG_JAMMY64=v60

tests/test_mail.py

@@ -28,6 +28,8 @@ def usage():
 	print("    -no-send: don't send, just delete")
 	print("    -no-delete: don't delete, just send")
 	print("    -timeout <seconds>: how long to wait for message")
+	print("    -body-from-stdin: read the message body from stdin")
+	print("    -smtp-debug: output debugging messages")
 	print("");
 	sys.exit(1)
 
@@ -48,6 +50,8 @@ delete_msg=True  # login to imap and delete message
 wait_timeout=30  # abandon timeout wiating for message delivery
 wait_cycle_sleep=5  # delay between delivery checks
 subject="Mail-in-a-Box Automated Test Message " + uuid.uuid4().hex  # message subject
+body_from_stdin=False
+smtp_debug=False
 
 # process command line
 argi=1
@@ -81,6 +85,12 @@ while argi<len(sys.argv):
 	elif arg=="-timeout" and arg_remaining>1:
 		wait_timeout=int(sys.argv[argi+1])
 		argi+=2
+	elif arg=="-body-from-stdin":
+		body_from_stdin = True
+		argi+=1
+	elif arg=="-smtp-debug":
+		smtp_debug = True
+		argi+=1
 	else:
 		usage()
 
@@ -100,14 +110,20 @@ headerfrom = if_unset(headerfrom, emailfrom)
 emailto = if_unset(emailto, login)
 emailto_pw = if_unset(emailto_pw, pw)
 
+if body_from_stdin:
+	body=sys.stdin.readlines()
+else:
+	body=['This is a test message. It should be automatically deleted by the test script.']
+
 msg = """From: {headerfrom}
 To: {emailto}
 Subject: {subject}
 
-This is a test message. It should be automatically deleted by the test script.""".format(
+{body}""".format(
 	headerfrom=headerfrom,
 	emailto=emailto,
 	subject=subject,
+	body=''.join(body)
 	)
 
 def imap_login(host, login, pw):
@@ -162,7 +178,8 @@ def smtp_login(host, login, pw, port):
 		server.starttls()
 	else:
 		server = smtplib.SMTP_SSL(host)
-	#server.set_debuglevel(1)
+	if smtp_debug:
+		server.set_debuglevel(1)
 
 	# Verify that the EHLO name matches the server's reverse DNS.
 	ipaddr = socket.gethostbyname(host) # IPv4 only!
@@ -191,7 +208,10 @@ def smtp_login(host, login, pw, port):
 if send_msg:
 	# Attempt to send a mail.
 	server = smtp_login(host, login, pw, port)
-	server.sendmail(emailfrom, [emailto], msg)
+	# sendmail: "If this method does not raise an exception, it returns a dictionary, with one entry for each recipient that was refused. Each entry contains a tuple of the SMTP error code and the accompanying error message sent by the server."
+	errors = server.sendmail(emailfrom, [emailto], msg)
+	#print(errors)
+	#if errors: raise ValueError(errors)
 	server.quit()
 	print("SMTP submission is OK.")