Merge pull request #40 from downtownallday/quota

half-a Quota

.github/workflows/commit-tests.yml

@@ -33,8 +33,7 @@ jobs:
     runs-on: ubuntu-22.04
     env:
       PRIMARY_HOSTNAME: box2.abc.com
-      # TODO: change UPSTREAM_TAG to 'main' once upstream is installable
+      UPSTREAM_TAG: main
-      UPSTREAM_TAG: v67
     steps:
       - uses: actions/checkout@v4
       - name: setup

conf/schema/postfix.schema

@@ -18,12 +18,12 @@ objectIdentifier MiabLDAPmail MiabLDAProot:2
 objectIdentifier MiabLDAPmailAttributeType MiabLDAPmail:1
 objectIdentifier MiabLDAPmailObjectClass MiabLDAPmail:2
 
-attributetype ( 1.3.6.1.4.1.15347.2.102 
+attributetype ( 1.3.6.1.4.1.15347.2.102
-	NAME 'transport' 
+	NAME 'transport'
 	SUP name)
 
-attributetype ( 1.3.6.1.4.1.15347.2.101 
+attributetype ( 1.3.6.1.4.1.15347.2.101
-	NAME 'mailRoutingAddress' 
+	NAME 'mailRoutingAddress'
 	SUP mail )
 
 attributetype ( 1.3.6.1.4.1.15347.2.110 NAME 'maildest'
@@ -56,13 +56,74 @@ attributetype ( MiabLDAPmailAttributeType:1 NAME 'mailMember' DESC 'RFC6532 utf8
 # create a utf8 version of core 'domainComponent'
 attributetype ( MiabLDAPmailAttributeType:2 NAME 'dcIntl' DESC 'UTF8 domain component' EQUALITY caseIgnoreMatch SUBSTR caseIgnoreSubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE )
 
+# Create mda/lda user mailbox quota settings (for dovecot)
+# format: number | number 'B' | number 'K' | number 'M' | number 'G'
+#
+# Dovecot supports more than one quota rule (but no way to use a
+# multi-valued attribute). Also add additional attributes for
+# more quota rules even though we're not necessarily
+# using them because we might in the future which could help avoid a
+# schema update. Dovecot supports "as many quota rules as you want"
+
+attributetype ( MiabLDAPmailAttributeType:3
+	DESC 'MDA/LDA user mailbox quota'
+	NAME 'mailboxQuota'
+	SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE
+	EQUALITY caseExactMatch )
+
+attributetype ( MiabLDAPmailAttributeType:4
+	DESC 'MDA/LDA user mailbox quota 2'
+	NAME 'mailboxQuota2'
+	SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE
+	EQUALITY caseExactMatch )
+
+attributetype ( MiabLDAPmailAttributeType:5
+	DESC 'MDA/LDA user mailbox quota 3'
+	NAME 'mailboxQuota3'
+	SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE
+	EQUALITY caseExactMatch )
+
+attributetype ( MiabLDAPmailAttributeType:6
+	DESC 'MDA/LDA user mailbox quota 4'
+	NAME 'mailboxQuota4'
+	SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE
+	EQUALITY caseExactMatch )
+
+attributetype ( MiabLDAPmailAttributeType:7
+	DESC 'MDA/LDA user mailbox quota 5'
+	NAME 'mailboxQuota5'
+	SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE
+	EQUALITY caseExactMatch )
+
+attributetype ( MiabLDAPmailAttributeType:8
+	DESC 'MDA/LDA user mailbox quota 6'
+	NAME 'mailboxQuota6'
+	SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE
+	EQUALITY caseExactMatch )
+
+# Dovecot can maintain a flag indicating whether a user is over or
+# under quota. It's use is not required, but enables postfix to reject
+# messages without queuing them when a mailbox is full. The value
+# should be dovecot boolean value 'yes', or 'no'.
+
+attributetype ( MiabLDAPmailAttributeType:9
+	DESC 'MDA/LDA over quota flag'
+	NAME 'mailboxOverQuotaFlag'
+	SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE
+	EQUALITY caseIgnoreMatch )
+
+
+#
+# object classes
+#
+
 objectclass ( 1.3.6.1.4.1.15347.2.1
 	NAME 'mailUser'
 	DESC 'E-Mail User'
 	SUP top
 	AUXILIARY
 	MUST ( uid $ mail $ maildrop )
-	MAY ( cn $ mailbox $ maildest $ mailaccess )
+	MAY ( cn $ mailbox $ maildest $ mailaccess $ mailboxQuota $ mailboxQuota2 $ mailboxQuota3 $ mailboxQuota4 $ mailboxQuota5 $ mailboxQuota6 $ mailboxOverQuotaFlag )
 	)
 
 objectclass ( 1.3.6.1.4.1.15347.2.2

management/cli.py

@@ -74,6 +74,7 @@ if len(sys.argv) < 2:
   {cli} user password user@domain.com [password]
   {cli} user remove user@domain.com
   {cli} user make-admin user@domain.com
+  {cli} user quota user@domain [new-quota]       (get or set user quota)
   {cli} user remove-admin user@domain.com
   {cli} user admins                              (lists admins)
   {cli} user mfa show user@domain.com            (shows MFA devices for user, if any)
@@ -97,6 +98,10 @@ elif sys.argv[1] == "user" and len(sys.argv) == 2:
 			print(user['email'], end='')
 			if "admin" in user['privileges']:
 				print("*", end='')
+			if user['quota'] == '0':
+				print(" unlimited", end='')
+			else:
+				print(" " + user['quota'], end='')
 			print()
 
 elif sys.argv[1] == "user" and sys.argv[2] in {"add", "password"}:
@@ -126,6 +131,14 @@ elif sys.argv[1] == "user" and sys.argv[2] == "admins":
 			if "admin" in user['privileges']:
 				print(user['email'])
 
+elif sys.argv[1] == "user" and sys.argv[2] == "quota" and len(sys.argv) == 4:
+	# Get a user's quota
+	print(mgmt("/mail/users/quota?text=1&email=%s" % sys.argv[3]))
+
+elif sys.argv[1] == "user" and sys.argv[2] == "quota" and len(sys.argv) == 5:
+	# Set a user's quota
+	users = mgmt("/mail/users/quota", { "email": sys.argv[3], "quota": sys.argv[4] })
+
 elif sys.argv[1] == "user" and len(sys.argv) == 5 and sys.argv[2:4] == ["mfa", "show"]:
 	# Show MFA status for a user.
 	status = mgmt("/mfa/status", { "user": sys.argv[4] }, is_json=True)
@@ -150,4 +163,3 @@ elif sys.argv[1] == "alias" and sys.argv[2] == "remove" and len(sys.argv) == 4:
 else:
 	print("Invalid command-line arguments.")
 	sys.exit(1)
-

management/daemon.py

@@ -30,6 +30,7 @@ import auth, utils
 from mailconfig import get_mail_users, get_mail_users_ex, get_admins, add_mail_user, set_mail_password, set_mail_display_name, remove_mail_user
 from mailconfig import get_mail_user_privileges, add_remove_mail_user_privilege
 from mailconfig import get_mail_aliases, get_mail_aliases_ex, get_mail_domains, add_mail_alias, remove_mail_alias
+from mailconfig import get_mail_quota, set_mail_quota
 from mfa import get_public_mfa_state, enable_mfa, disable_mfa
 import mfa_totp
 import contextlib
@@ -201,8 +202,31 @@ def mail_users():
 @app.route('/mail/users/add', methods=['POST'])
 @authorized_personnel_only
 def mail_users_add():
+	quota = request.form.get('quota', '0')
 	try:
-		return add_mail_user(request.form.get('email', ''), request.form.get('password', ''), request.form.get('privileges', ''), request.form.get('display_name', ''), env)
+		return add_mail_user(request.form.get('email', ''), request.form.get('password', ''), request.form.get('privileges', ''), quota, request.form.get('display_name', ''), env)
+	except ValueError as e:
+		return (str(e), 400)
+
+@app.route('/mail/users/quota', methods=['GET'])
+@authorized_personnel_only
+def get_mail_users_quota():
+	email = request.values.get('email', '')
+	quota = get_mail_quota(email, env)
+
+	if request.values.get('text'):
+		return quota
+
+	return json_response({
+		"email": email,
+		"quota": quota
+	})
+
+@app.route('/mail/users/quota', methods=['POST'])
+@authorized_personnel_only
+def mail_users_quota():
+	try:
+		return set_mail_quota(request.form.get('email', ''), request.form.get('quota'), env)
 	except ValueError as e:
 		return (str(e), 400)
 

management/mailconfig.py

@@ -19,8 +19,11 @@
 # Python 3 in setup/questions.sh to validate the email
 # address entered by the user.
 
-import subprocess, shutil, os, sqlite3, re, ldap3, uuid, hashlib
+import os, sqlite3, re
-import utils, backend
+import subprocess
+import ldap3, uuid, hashlib, backend
+
+import utils
 from email_validator import validate_email as validate_email_, EmailNotValidError
 import idna
 import socket
@@ -283,6 +286,30 @@ def get_mail_users(env, as_map=False, map_by="maildrop"):
 		return utils.sort_email_addresses(users, env)
 
 
+def sizeof_fmt(num):
+	for unit in ['','K','M','G','T']:
+		if abs(num) < 1024.0:
+			if abs(num) > 99:
+				return "%3.0f%s" % (num, unit)
+			else:
+				return "%2.1f%s" % (num, unit)
+
+		num /= 1024.0
+
+	return str(num)
+
+def sizeof_fmt(num):
+	for unit in ['','K','M','G','T']:
+		if abs(num) < 1024.0:
+			if abs(num) > 99:
+				return "%3.0f%s" % (num, unit)
+			else:
+				return "%2.1f%s" % (num, unit)
+
+		num /= 1024.0
+
+	return str(num)
+
 def get_mail_users_ex(env, with_archived=False):
 	# Returns a complex data structure of all user accounts, optionally
 	# including archived (status="inactive") accounts.
@@ -307,20 +334,52 @@ def get_mail_users_ex(env, with_archived=False):
 	users = []
 	active_accounts = set()
 	c = open_database(env)
-	response = c.wait( c.search(env.LDAP_USERS_BASE, "(objectClass=mailUser)", attributes=['mail','maildrop','mailaccess','cn']) )
+	response = c.wait( c.search(env.LDAP_USERS_BASE, "(objectClass=mailUser)", attributes=['mail','maildrop','mailaccess','mailboxQuota','cn']) )
 
 	for rec in response:
 		#email = rec['maildrop'][0]
 		email = rec['mail'][0]
 		privileges = rec['mailaccess']
+		quota = rec['mailboxQuota'][0] if len(rec['mailboxQuota'])>0 else '0'
 		display_name = rec['cn'][0]
 		active_accounts.add(email)
+
+		(user, domain) = email.split('@')
+		box_size = 0
+		box_quota = 0
+		percent = ''
+		try:
+			dirsize_file = os.path.join(env['STORAGE_ROOT'], 'mail/mailboxes/%s/%s/maildirsize' % (domain, user))
+			with open(dirsize_file, 'r') as f:
+				box_quota = int(f.readline().split('S')[0])
+				for line in f.readlines():
+					(size, count) = line.split(' ')
+					box_size += int(size)
+
+			try:
+				percent = (box_size / box_quota) * 100
+			except:
+				percent = 'Error'
+
+		except:
+			box_size = '?'
+			box_quota = '?'
+			percent = '?'
+
+		if quota == '0':
+			percent = ''
+
 		user = {
 			"email": email,
 			"privileges": privileges,
+			"quota": quota,
 			"status": "active",
-			"display_name": display_name
+			"display_name": display_name,
+			"box_quota": box_quota,
+			"box_size": sizeof_fmt(box_size) if box_size != '?' else box_size,
+			"percent": '%3.0f%%' % percent if type(percent) != str else percent,
 		}
+
 		users.append(user)
 
 	# Add in archived accounts.
@@ -337,7 +396,10 @@ def get_mail_users_ex(env, with_archived=False):
 						"privileges": [],
 						"status": "inactive",
 						"mailbox": mbox,
-						"display_name": ""
+						"display_name": "",
+                        "box_size": '?',
+                        "box_quota": '?',
+                        "percent": '?',
 					}
 					users.append(user)
 
@@ -697,12 +759,13 @@ def remove_mail_domain(env, domain_idna, validate=True):
 	return True
 
 
-def add_mail_user(email, pw, privs, display_name, env):
+def add_mail_user(email, pw, privs, quota, display_name, env):
 	# Add a new mail user.
 	#
 	# email: the new user's email address (idna)
 	# pw: the new user's password
 	# privs: either an array of privilege strings, or a newline
+	# quota: a string (number | number 'M' | number 'G') or None
 	# separated string of privilege names
 	# display_name: a string with users givenname and surname (eg "Al Woods")
 	#
@@ -735,6 +798,22 @@ def add_mail_user(email, pw, privs, display_name, env):
 		validation = validate_privilege(p)
 		if validation: return validation
 
+	if quota is None:
+		quota = '0'
+
+	try:
+		quota = validate_quota(quota)
+	except ValueError as e:
+		return (str(e), 400)
+
+	if quota is None:
+		quota = '0'
+
+	try:
+		quota = validate_quota(quota)
+	except ValueError as e:
+		return (str(e), 400)
+
 	# get the database
 	conn = open_database(env)
 
@@ -773,6 +852,7 @@ def add_mail_user(email, pw, privs, display_name, env):
 		"maildrop" : email.lower(),
 		"uid" : uid,
 		"mailaccess": privs,
+		"mailboxQuota": quota,
 		"cn": cn,
 		"sn": sn,
 		"shadowLastChange": backend.get_shadowLastChanged()
@@ -805,6 +885,10 @@ def add_mail_user(email, pw, privs, display_name, env):
 	# convert alias's mailMember to member
 	convert_mailMember(env, conn, dn, email)
 
+	dovecot_quota_recalc(email)
+
+	dovecot_quota_recalc(email)
+
 	# Update things in case any new domains are added.
 	if domain_added:
 		return kick(env, return_status)
@@ -866,6 +950,51 @@ def validate_login(email, pw, env):
 	except ldap3.core.exceptions.LDAPInvalidCredentialsResult:
 		return False
 
+def validate_quota(quota):
+	# validate quota
+	quota = quota.strip().upper()
+
+	if quota == "":
+		raise ValueError("No quota provided.")
+	if re.search(r"[\s,.]", quota):
+		raise ValueError("Quotas cannot contain spaces, commas, or decimal points.")
+	if not re.match(r'^[\d]+[GMK]?$', quota):
+		raise ValueError("Invalid quota.")
+
+	return quota
+
+def get_mail_quota(email, env):
+	user = find_mail_user(env, email, ['mailboxQuota'])
+	if user is None:
+		return ("That's not a user (%s)." % email, 400)
+	if len(user['mailboxQuota'])==0:
+		return '0'
+	else:
+		return user['mailboxQuota'][0]
+
+def set_mail_quota(email, quota, env):
+	# validate that password is acceptable
+	quota = validate_quota(quota)
+
+	# update the database
+	conn = open_database(env)
+	user = find_mail_user(env, email, ['mailboxQuota'], conn)
+	if user is None:
+		return ("That's not a user (%s)." % email, 400)
+
+	conn.modify_record(user, { 'mailboxQuota': quota })
+	dovecot_quota_recalc(email)
+	return "OK"
+
+def dovecot_quota_recalc(email):
+	# dovecot processes running for the user will not recognize the new quota setting
+	# a reload is necessary to reread the quota setting, but it will also shut down
+	# running dovecot processes.  Email clients generally log back in when they lose
+	# a connection.
+	# subprocess.call(['doveadm', 'reload'])
+
+	# force dovecot to recalculate the quota info for the user.
+	subprocess.call(["doveadm", "quota", "recalc", "-u", email])
 
 def get_mail_password(email, env):
 	# Gets the hashed passwords for a user. In ldap, userPassword is
@@ -1329,12 +1458,12 @@ def remove_mail_alias(address_utf8, env, do_kick=True, auto=None, ignore_if_not_
 		return return_status
 
 
-def add_auto_aliases(aliases, env):
+# def add_auto_aliases(aliases, env):
-	conn, c = open_database(env, with_connection=True)
+# 	conn, c = open_database(env, with_connection=True)
-	c.execute("DELETE FROM auto_aliases")
+# 	c.execute("DELETE FROM auto_aliases")
-	for source, destination in aliases.items():
+# 	for source, destination in aliases.items():
-		c.execute("INSERT INTO auto_aliases (source, destination) VALUES (?, ?)", (source, destination))
+# 		c.execute("INSERT INTO auto_aliases (source, destination) VALUES (?, ?)", (source, destination))
-	conn.commit()
+# 	conn.commit()
 
 def get_system_administrator(env):
 	return "administrator@" + env['PRIMARY_HOSTNAME']

management/templates/users.html

@@ -6,6 +6,7 @@
 #user_table .account_inactive .if_active { display: none; }
 #user_table .account_active .if_inactive { display: none; }
 #user_table .account_active.if_inactive { display: none; }
+.row-center { text-align: center; }
 </style>
 
 <h3>Add a mail user</h3>
@@ -34,8 +35,12 @@
     <div>Display Name</div>
     <input id="adduserDisplayName" class="form-control" type="text" placeholder="eg: John Smith">
   </div>
+  <div class="form-group">
+    <div>Quota</div>
+    <input type="text" class="form-control" id="adduserQuota" placeholder="Quota" style="width:5em;" value="0">
+  </div>
   <div class="text-center">
-    <div>&nbsp;</div>    
+    <div>&nbsp;</div>
     <button type="submit" class="btn btn-primary">Add User</button>
   </div>
 </form>
@@ -44,13 +49,17 @@
   <li>Use <a href="#aliases">aliases</a> to create email addresses that forward to existing accounts.</li>
   <li>Administrators get access to this control panel.</li>
   <li>User accounts cannot contain any international (non-ASCII) characters, but <a href="#aliases">aliases</a> can.</li>
+  <li>Quotas may not contain any spaces, commas or decimal points. Suffixes of G (gigabytes) and M (megabytes) are allowed.  For unlimited storage enter 0 (zero)</li>
 </ul>
 
 <h3>Existing mail users</h3>
 <table id="user_table" class="table" style="width: auto">
   <thead>
     <tr>
-      <th width="50%">Email Address</th>
+      <th width="35%">Email Address</th>
+      <th class="row-center">Size</th>
+      <th class="row-center">Used</th>
+      <th class="row-center">Quota</th>
       <th>Actions</th>
     </tr>
   </thead>
@@ -64,10 +73,21 @@
     <td>
       <span class="address"></span> <span class="display_name_wrapper">(<a class="display_name" href="#" onclick="users_set_displayname(this); return false;" title="Change display name"></a>)</span>
     </td>
+    <td class="box-size row-center"></td>
+    <td class="percent row-center"></td>
+	<td class="quota row-center">
+	</td>
     <td class='actions'>
         <span class='privs'>
         </span>
 
+        <span class="if_active">
+          <a href="#" onclick="users_set_quota(this); return false;" class='setquota' title="Set Quota">
+              set quota
+          </a>
+          |
+        </span>
+
         <span class="if_active">
           <a href="#" onclick="users_set_password(this); return false;" class='setpw' title="Set Password">
             set password
@@ -108,10 +128,28 @@
 <table class="table" style="margin-top: .5em">
 <thead><th>Verb</th> <th>Action</th><th></th></thead>
 <tr><td>GET</td><td><i>(none)</i></td> <td>Returns a list of existing mail users. Adding <code>?format=json</code> to the URL will give JSON-encoded results.</td></tr>
-<tr><td>POST</td><td>/add</td> <td>Adds a new mail user. Required POST-body parameters are <code>email</code> and <code>password</code>.</td></tr>
+<tr>
-<tr><td>POST</td><td>/remove</td> <td>Removes a mail user. Required POST-body parameter is <code>email</code>.</td></tr>
+    <td>POST</td>
+    <td>/add</td>
+    <td>Adds a new mail user. Required POST-body parameters are <code>email</code> and <code>password</code>. Optional parameters: <code>privilege=admin</code> and <code>quota</code></td>
+</tr>
+<tr>
+    <td>POST</td>
+    <td>/remove</td>
+    <td>Removes a mail user. Required POST-by parameter is <code>email</code>.</td>
+</tr>
 <tr><td>POST</td><td>/privileges/add</td> <td>Used to make a mail user an admin. Required POST-body parameters are <code>email</code> and <code>privilege=admin</code>.</td></tr>
 <tr><td>POST</td><td>/privileges/remove</td> <td>Used to remove the admin privilege from a mail user. Required POST-body parameter is <code>email</code>.</td></tr>
+<tr>
+    <td>GET</td>
+    <td>/quota</td>
+    <td>Get the quota for a mail user. Required POST-body parameters are <code>email</code> and will return JSON result</td>
+</tr>
+<tr>
+    <td>POST</td>
+    <td>/quota</td>
+    <td>Set the quota for a mail user. Required POST-body parameters are <code>email</code> and <code>quota</code>.</td>
+</tr>
 </table>
 
 <h4>Examples:</h4>
@@ -144,7 +182,7 @@ function show_users() {
     function(r) {
       $('#user_table tbody').html("");
       for (var i = 0; i < r.length; i++) {
-        var hdr = $("<tr><th colspan='2' style='background-color: #EEE'></th></tr>");
+        var hdr = $("<tr><th colspan='6' style='background-color: #EEE'></th></tr>");
         hdr.find('th').text(r[i].domain);
         $('#user_table tbody').append(hdr);
 
@@ -162,7 +200,14 @@ function show_users() {
           n2.addClass("account_" + user.status);
 
           n.attr('data-email', user.email);
+          n.attr('data-quota', user.quota);
           n.find('.address').text(user.email);
+          n.find('.box-size').text(user.box_size);
+          if (user.box_size == '?') {
+            n.find('.box-size').attr('title', 'Mailbox size is unkown')
+          }
+          n.find('.percent').text(user.percent);
+          n.find('.quota').text((user.quota == '0') ? 'unlimited' : user.quota);
           if (user.status == "inactive") {
             n.find('.display_name_wrapper').text('[archived]');
           }
@@ -197,6 +242,7 @@ function do_add_user() {
   var email = $("#adduserEmail").val();
   var pw = $("#adduserPassword").val();
   var privs = $("#adduserPrivs").val();
+  var quota = $("#adduserQuota").val();
   var display_name = $("#adduserDisplayName").val();
   api(
     "/mail/users/add",
@@ -205,6 +251,7 @@ function do_add_user() {
       email: email,
       password: pw,
       privileges: privs,
+      quota: quota,
       display_name: display_name
     },
     function(r) {
@@ -272,6 +319,36 @@ function users_set_displayname(elem) {
       });
 }
 
+function users_set_quota(elem) {
+    var email = $(elem).parents('tr').attr('data-email');
+    var quota = $(elem).parents('tr').attr('data-quota');
+
+    show_modal_confirm(
+        "Set Quota",
+        $("<p>Set quota for <b>" + email + "</b>?</p>" +
+            "<p>" +
+            "<label for='users_set_quota' style='display: block; font-weight: normal'>Quota:</label>" +
+            "<input type='text' id='users_set_quota' value='" + quota + "'></p>" +
+            "<p><small>Quotas may not contain any spaces or commas.  Suffixes of G (gigabytes) and M (megabytes) are allowed.</small></p>" +
+            "<p><small>For unlimited storage enter 0 (zero)</small></p>"),
+        "Set Quota",
+        function() {
+            api(
+                "/mail/users/quota",
+                "POST",
+                {
+                    email: email,
+                    quota: $('#users_set_quota').val()
+                },
+                function(r) {
+                    show_users();
+                },
+                function(r) {
+                    show_modal_error("Set Quota", r);
+                });
+        });
+}
+
 function users_remove(elem) {
   var email = $(elem).parents('tr').attr('data-email');
 
@@ -337,7 +414,7 @@ function generate_random_password() {
   var charset = "ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz23456789"; // confusable characters skipped
   for (var i = 0; i < 12; i++)
     pw += charset.charAt(Math.floor(Math.random() * charset.length));
-  show_modal_error("Random Password", "<p>Here, try this:</p> <p><code style='font-size: 110%'>" + pw + "</code></pr");
+  show_modal_error("Random Password", "<p>Here, try this:</p> <p><code style='font-size: 110%'>" + pw + "</code></p>");
   return false; // cancel click
 }
 </script>

setup/bootstrap.sh

@@ -133,7 +133,7 @@ fi
 if [ -z "${ENCRYPTION_AT_REST:-}" ]; then
     source ehdd/ehdd_funcs.sh || exit 1
     hdd_exists && ENCRYPTION_AT_REST=true
-elif [ "${ENCRYPTION_AT_REST:-}" = "false" ]; then 
+elif [ "${ENCRYPTION_AT_REST:-}" = "false" ]; then
     source ehdd/ehdd_funcs.sh || exit 1
     if hdd_exists; then
         echo "Encryption-at-rest must be disabled manually"
@@ -147,4 +147,3 @@ if [ "${ENCRYPTION_AT_REST:-false}" = "true" ]; then
 else
     setup/start.sh </dev/tty
 fi
-

setup/ldap.sh

@@ -99,7 +99,7 @@ create_miab_conf() {
 		_add_if_missing "${prefix}_DN" "cn=$cn,$LDAP_SERVICES_BASE"
 		_add_if_missing "${prefix}_PASSWORD" "$(generate_password 64)"
 	done
-	
+
 	chmod 0640 "$MIAB_INTERNAL_CONF_FILE"
 	. "$MIAB_INTERNAL_CONF_FILE"
 }
@@ -126,7 +126,7 @@ create_service_accounts() {
 	# create service accounts. service accounts have special access
 	# rights, generally read-only to users, aliases, and configuration
 	# subtrees (see apply_access_control)
-	
+
 	local prefix dn pass
 	for prefix in ${SERVICE_ACCOUNTS[*]}
 	do
@@ -147,7 +147,7 @@ userPassword: $(slappasswd_hash "$pass")
 EOF
 		fi
 	done
-	
+
 }
 
 
@@ -155,7 +155,7 @@ install_system_packages() {
 	# install required deb packages, generate admin credentials
 	# and apply them to the installation
 	create_miab_conf
-	
+
 	# Set installation defaults to avoid interactive dialogs. See
 	# /var/lib/dpkg/info/slapd.templates for a list of what can be set
 	debconf-set-selections <<EOF
@@ -164,10 +164,10 @@ slapd slapd/domain string ${LDAP_DOMAIN}
 slapd slapd/password1 password ${LDAP_ADMIN_PASSWORD}
 slapd slapd/password2 password ${LDAP_ADMIN_PASSWORD}
 EOF
-	
+
 	# Install packages
 	say "Installing OpenLDAP server..."
-	
+
 	# we must install slapd without DEBIAN_FRONTEND=noninteractive or
 	# debconf selections are ignored
 	hide_output apt-get -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confnew" install slapd
@@ -227,7 +227,7 @@ EOF
 		say "  is set to: $ATTR_VALUE"
 		say "  expected : $LDAP_ADMIN_DN"
 		die
-	fi		
+	fi
 }
 
 relocate_slapd_data() {
@@ -277,11 +277,11 @@ relocate_slapd_data() {
 	say_verbose "		DB='${DB_DIR}'"
 	say_verbose "	to:"
 	say_verbose "	   CONF=${MIAB_SLAPD_CONF}"
-	say_verbose "		 DB=${MIAB_SLAPD_DB_DIR}"	
+	say_verbose "		 DB=${MIAB_SLAPD_DB_DIR}"
 	say_verbose ""
 	say_verbose "Stopping slapd"
 	systemctl stop slapd || die "Could not stop slapd"
-	
+
 	# Modify the path to dc=mailinabox's database directory
 	say_verbose "Dump config database"
 	local TMP="/tmp/miab_relocate_ldap.ldif"
@@ -320,7 +320,7 @@ schema_to_ldif() {
 			cat="curl -s"
 		fi
 	fi
-	
+
 	cat >"$ldif" <<EOF
 dn: cn=$cn,cn=schema,cn=config
 objectClass: olcSchemaConfig
@@ -358,7 +358,7 @@ EOF
 
 
 add_schemas() {
-	# Add necessary schema's for MiaB operaion		 
+	# Add necessary schema's for MiaB operaion
 	#
 	# Note: the postfix schema originally came from the ldapadmin
 	# project (GPL)(*), but has been modified to support the needs of
@@ -385,7 +385,7 @@ add_schemas() {
 			ldapadd -Q -Y EXTERNAL -H ldapi:/// -f "$ldif" >/dev/null
 			rm -f "$ldif"
 		fi
-	done	
+	done
 }
 
 
@@ -481,7 +481,7 @@ EOF
 add_overlays() {
 	# Apply slapd overlays - apply the commonly used member-of overlay
 	# now because adding it later is harder.
-	
+
 	# Get the config dn for the database
 	get_attribute "cn=config" "olcSuffix=${LDAP_BASE}" "dn"
 	[ -z "$ATTR_DN" ] &&
@@ -498,7 +498,7 @@ add: olcModuleLoad
 olcModuleLoad: memberof.la
 EOF
 	fi
-	
+
 	get_attribute "$cdn" "(olcOverlay=memberof)" "olcOverlay"
 	if [ -z "$ATTR_DN" ]; then
 		say_verbose "Adding memberof overlay to $LDAP_BASE"
@@ -516,7 +516,7 @@ EOF
 
 add_indexes() {
 	# Index mail-related attributes
-	
+
 	# Get the config dn for the database
 	get_attribute "cn=config" "olcSuffix=${LDAP_BASE}" "dn"
 	[ -z "$ATTR_DN" ] &&
@@ -678,7 +678,7 @@ EOF
 #
 process_cmdline() {
 	[ -e "$MIAB_INTERNAL_CONF_FILE" ] && . "$MIAB_INTERNAL_CONF_FILE"
-	
+
 	if [ "$1" == "-d" ]; then
 		# Start slapd in interactive/debug mode
 		echo "!! SERVER DEBUG MODE !!"
@@ -688,7 +688,7 @@ process_cmdline() {
 		echo "Listening on $SLAPD_SERVICES..."
 		/usr/sbin/slapd -h "$SLAPD_SERVICES" -g openldap -u openldap -F $MIAB_SLAPD_CONF -d ${2:-1}
 		exit 0
-		
+
 	elif [ "$1" == "-config" ]; then
 		# Apply a certain configuration
 		if [ "$2" == "server" ]; then
@@ -734,7 +734,7 @@ process_cmdline() {
 		local hide_attrs="(structuralObjectClass|entryUUID|creatorsName|createTimestamp|entryCSN|modifiersName|modifyTimestamp)"
 		local slapcat_args=(-F "$MIAB_SLAPD_CONF" -o ldif-wrap=no)
 		[ ${verbose:-0} -gt 0 ] && hide_attrs="(_____NEVERMATCHES)"
-		
+
 		if [ "$s" == "all" ]; then
 			echo ""
 			echo '--------------------------------'
@@ -777,7 +777,7 @@ process_cmdline() {
 		if [ "$s" == "permitted-senders" -o "$s" == "ps" ]; then
 			echo ""
 			echo '--------------------------------'
-			local attrs=(mail member mailRoutingAddress rfc822MailMember)
+			local attrs=(mail member mailRoutingAddress mailMember)
 			[ ${verbose:-0} -gt 0 ] && attrs=()
 			debug_search "(objectClass=mailGroup)" "$LDAP_PERMITTED_SENDERS_BASE" ${attrs[@]}
 		fi
@@ -814,7 +814,7 @@ process_cmdline() {
 		rm -f "/etc/default/slapd"
 		echo "Done"
 		exit 0
-		
+
 	elif [ ! -z "$1" ]; then
 		echo "Invalid command line argument '$1'"
 		exit 1

setup/mail-dovecot.sh

@@ -76,6 +76,32 @@ tools/editconf.py /etc/dovecot/conf.d/10-mail.conf \
 
 # Create, subscribe, and mark as special folders: INBOX, Drafts, Sent, Trash, Spam and Archive.
 cp conf/dovecot-mailboxes.conf /etc/dovecot/conf.d/15-mailboxes.conf
+sed -i "s/#mail_plugins =\(.*\)/mail_plugins =\1 \$mail_plugins quota/" /etc/dovecot/conf.d/10-mail.conf
+if ! grep -q "mail_plugins.* imap_quota" /etc/dovecot/conf.d/20-imap.conf; then
+  sed -i "s/\(mail_plugins =.*\)/\1\n  mail_plugins = \$mail_plugins imap_quota/" /etc/dovecot/conf.d/20-imap.conf
+fi
+
+# configure stuff for quota support
+if ! grep -q "quota_status_success = DUNNO" /etc/dovecot/conf.d/90-quota.conf; then
+    cat > /etc/dovecot/conf.d/90-quota.conf << EOF;
+plugin {
+  quota = maildir:User quota
+
+  quota_grace = 10%%
+
+  quota_status_success = DUNNO
+  quota_status_nouser = DUNNO
+  quota_status_overquota = "522 5.2.2 Mailbox is full"
+}
+
+service quota-status {
+    executable = quota-status -p postfix
+    inet_listener {
+        port = 12340
+    }
+}
+EOF
+fi
 
 # ### IMAP/POP
 

setup/mail-postfix.sh

@@ -259,7 +259,7 @@ tools/editconf.py /etc/postfix/main.cf  -e lmtp_destination_recipient_limit=
 # "450 4.7.1 Client host rejected: Service unavailable". This is a retry code, so the mail doesn't properly bounce. #NODOC
 tools/editconf.py /etc/postfix/main.cf \
 	smtpd_sender_restrictions="reject_non_fqdn_sender,reject_unknown_sender_domain,reject_authenticated_sender_login_mismatch,reject_rhsbl_sender dbl.spamhaus.org=127.0.1.[2..99]" \
-	smtpd_recipient_restrictions="permit_sasl_authenticated,permit_mynetworks,reject_rbl_client zen.spamhaus.org=127.0.0.[2..11],reject_unlisted_recipient,check_policy_service unix:private/policy-spf,check_policy_service inet:127.0.0.1:10023"
+	smtpd_recipient_restrictions="permit_sasl_authenticated,permit_mynetworks,reject_rbl_client zen.spamhaus.org=127.0.0.[2..11],reject_unlisted_recipient,check_policy_service unix:private/policy-spf,check_policy_service inet:127.0.0.1:10023,check_policy_service inet:127.0.0.1:12340"
 
 # Postfix connects to Postgrey on the 127.0.0.1 interface specifically. Ensure that
 # Postgrey listens on the same interface (and not IPv6, for instance).

setup/mail-users.sh

@@ -88,8 +88,13 @@ pass_attrs = maildrop=user
 #   Post-login information specific to the user (eg. quotas).  For
 #   lmtp delivery, pass_filter is not used, and postfix has already
 #   rewritten the envelope using the maildrop address.
+#   %$ is expanded to mailboxQuota's value.
 user_filter = (&(objectClass=mailUser)(|(mail=%u)(maildrop=%u)))
-user_attrs = maildrop=user
+user_attrs = maildrop=user, \
+             mailboxQuota=quota_rule=*:bytes=%\$, \
+             =quota_rule2=Trash:storage=+100M, \
+             =quota_rule3=Drafts:storage=+25M, \
+             =quota_rule4=Sent:storage=+50M
 
 # Account iteration for various dovecot tools (doveadm)
 iterate_filter = (objectClass=mailUser)
@@ -269,3 +274,6 @@ chmod 0640 /etc/postfix/virtual-alias-maps.cf
 
 restart_service postfix
 restart_service dovecot
+
+# force a recalculation of all user quotas
+doveadm quota recalc -A

setup/migrate.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/python3 -u
 # -*- indent-tabs-mode: t; tab-width: 8; python-indent-offset: 8; -*-
 #####
 ##### This file is part of Mail-in-a-Box-LDAP which is released under the
@@ -200,6 +200,12 @@ def migration_14(env):
 	db = os.path.join(env["STORAGE_ROOT"], 'mail/users.sqlite')
 	shell("check_call", ["sqlite3", db, "CREATE TABLE auto_aliases (id INTEGER PRIMARY KEY AUTOINCREMENT, source TEXT NOT NULL UNIQUE, destination TEXT NOT NULL, permitted_senders TEXT);"])
 
+def migration_15(env):
+	# Add a column to the users table to store their quota limit.  Default to '0' for unlimited.
+	db = os.path.join(env["STORAGE_ROOT"], 'mail/users.sqlite')
+	shell("check_call", ["sqlite3", db, "ALTER TABLE users ADD COLUMN quota TEXT NOT NULL DEFAULT '0';"])
+
+
 ###########################################################
 
 
@@ -306,7 +312,7 @@ def migration_miabldap_2(env):
 			return ldif.replace("rfc822MailMember: ", "mailMember: ")
 		# apply schema changes miabldap/1 -> miabldap/2
 		ldap.unbind()
-		print("Apply schema changes")
+		print("Apply schema changes to support utf8 email addresses")
 		m14.apply_schema_changes(env, ldapvars, ldif_change_fn)
 		# reconnect
 		ldap = connect(ldapvars)
@@ -329,6 +335,52 @@ def migration_miabldap_2(env):
 
 	ldap.unbind()
 
+def migration_miabldap_3(env):
+	# This migration step changes the ldap schema to support quotas
+	#
+	# possible states at this point:
+	#   miabldap was installed and is being upgraded
+	#      -> schema update needed
+	#   a miab install was present and step 1 upgaded it to miabldap
+	#      -> new schema already present
+	#
+	sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), "../management")))
+	import ldap3
+	from backend import connect
+	import migration_14 as m14
+
+	# 1. get ldap site details
+	ldapvars = load_env_vars_from_file(os.path.join(env["STORAGE_ROOT"], "ldap/miab_ldap.conf"), strip_quotes=True)
+
+	# connect before schema changes to ensure admin password works
+	ldap = connect(ldapvars)
+
+	# 2. if this is a miab -> maibldap install, the new schema is
+	# already in place and no schema changes are needed. however,
+	# if this is a miabldap/1 to miabldap/2 migration, we must
+	# upgrade the schema.
+	ret = shell("check_output", [
+		"ldapsearch",
+		"-Q",
+		"-Y", "EXTERNAL",
+		"-H", "ldapi:///",
+		"(&(objectClass=olcSchemaConfig)(cn={*}postfix))",
+		"-b", "cn=schema,cn=config",
+		"-o", "ldif_wrap=no",
+		"-LLL",
+		"olcObjectClasses"
+	])
+
+	ldap.unbind()
+
+	if "mailboxQuota" not in ret:
+		def ldif_change_fn(ldif):
+			# the schema change we're making does not require any data changes
+			return ldif
+		# apply schema changes miabldap/2 -> miabldap/3
+		print("Apply schema changes to support mailbox quotas")
+		m14.apply_schema_changes(env, ldapvars, ldif_change_fn)
+
 
 def get_current_migration():
 	ver = 0
@@ -419,7 +471,7 @@ def run_miabldap_migrations():
 			migration_id = 0
 		else:
 			print()
-			print("%s file doesn't exists. Skipping migration..." % (migration_id_file,))
+			print("%s file doesn't exist. Skipping migration..." % (migration_id_file,))
 			return
 
 	ourver = int(migration_id)

setup/migration_13.py

@@ -17,7 +17,7 @@
 import uuid, os, sqlite3, ldap3, hashlib
 
 
-def add_user(env, ldapconn, search_base, users_base, domains_base, email, password, privs, totp, cn=None):
+def add_user(env, ldapconn, search_base, users_base, domains_base, email, password, privs, quota, totp, cn=None):
 	# Add a sqlite user to ldap
 	#   env are the environment variables
 	#   ldapconn is the bound ldap connection
@@ -27,6 +27,7 @@ def add_user(env, ldapconn, search_base, users_base, domains_base, email, passwo
 	#   email is the user's email
 	#   password is the user's current sqlite password hash
 	#   privs is an array of privilege names for the user
+	#   quota is the users mailbox quota (string; defaults to '0')
 	#   totp contains the list of secrets, mru tokens, and labels
 	#   cn is the user's common name [optional]
 	#
@@ -45,13 +46,14 @@ def add_user(env, ldapconn, search_base, users_base, domains_base, email, passwo
 	m = hashlib.sha1()
 	m.update(bytearray(email.lower(),'utf-8'))
 	uid = m.hexdigest()
-	
+
 	# Attributes to apply to the new ldap entry
 	objectClasses = [ 'inetOrgPerson','mailUser','shadowAccount' ]
 	attrs = {
 		"mail" : email,
 		"maildrop" : email,
 		"uid" : uid,
+		"mailboxQuota": quota,
 		# Openldap uses prefix {CRYPT} for all crypt(3) formats
 		"userPassword" : password.replace('{SHA512-CRYPT}','{CRYPT}')
 	}
@@ -91,10 +93,10 @@ def add_user(env, ldapconn, search_base, users_base, domains_base, email, passwo
 		attrs['totpMruToken'] = totp["mru_token"]
 		attrs['totpMruTokenTime'] = totp["mru_token_time"]
 		attrs['totpLabel'] = totp["label"]
-	
+
 	# Add user
 	dn = "uid=%s,%s" % (uid, users_base)
-	
+
 	print("adding user %s" % email)
 	ldapconn.add(dn, objectClasses, attrs)
 
@@ -116,14 +118,15 @@ def create_users(env, conn, ldapconn, ldap_base, ldap_users_base, ldap_domains_b
 
 	# select users
 	c = conn.cursor()
-	c.execute("SELECT id, email, password, privileges from users")
+	c.execute("SELECT id, email, password, privileges, quota from users")
 
 	users = {}
 	for row in c:
 		user_id=row[0]
 		email=row[1]
 		password=row[2]
-		privs=row[3]	
+		privs=row[3]
+		quota=row[4]
 		totp = None
 
 		c2 = conn.cursor()
@@ -143,7 +146,7 @@ def create_users(env, conn, ldapconn, ldap_base, ldap_users_base, ldap_domains_b
 			totp["label"].append("{%s}%s" % (rowidx, row2[2] or ''))
 			rowidx += 1
 
-		dn = add_user(env, ldapconn, ldap_base, ldap_users_base, ldap_domains_base, email, password, privs.split("\n"), totp)
+		dn = add_user(env, ldapconn, ldap_base, ldap_users_base, ldap_domains_base, email, password, privs.split("\n"), quota, totp)
 		users[email] = dn
 	return users
 
@@ -164,7 +167,7 @@ def create_aliases(env, conn, ldapconn, aliases_base):
 			cn="%s" % uuid.uuid4()
 			dn="cn=%s,%s" % (cn, aliases_base)
 			description="Mail group %s" % alias
-			
+
 			if alias.startswith("postmaster@") or \
 			   alias.startswith("hostmaster@") or \
 			   alias.startswith("abuse@") or \
@@ -172,7 +175,7 @@ def create_aliases(env, conn, ldapconn, aliases_base):
 			   alias == "administrator@" + env['PRIMARY_HOSTNAME']:
 				description = "Required alias"
 
-			print("adding alias %s" % alias)			
+			print("adding alias %s" % alias)
 			ldapconn.add(dn, ['mailGroup'], {
 				"mail": alias,
 				"description": description
@@ -196,7 +199,7 @@ def populate_aliases(conn, ldapconn, users_map, aliases_map):
 		alias_dn=aliases_map[alias]
 		members = []
 		mailMembers = []
-		
+
 		for email in row[1].split(','):
 			email=email.strip()
 			if email=="":
@@ -207,13 +210,13 @@ def populate_aliases(conn, ldapconn, users_map, aliases_map):
 				members.append(aliases_map[email])
 			else:
 				mailMembers.append(email)
-		
+
 		print("populate alias group %s" % alias)
 		changes = {}
 		if len(members)>0:
 			changes["member"]=[(ldap3.MODIFY_REPLACE, members)]
 		if len(mailMembers)>0:
-			changes["rfc822MailMember"]=[(ldap3.MODIFY_REPLACE, mailMembers)]			
+			changes["rfc822MailMember"]=[(ldap3.MODIFY_REPLACE, mailMembers)]
 		ldapconn.modify(alias_dn, changes)
 
 

setup/webmail.sh

@@ -214,6 +214,7 @@ cat > $RCM_CONFIG <<EOF;
 
 /* prevent CSRF, requires php 7.3+ */
 \$config['session_samesite'] = 'Strict';
+\$config['quota_zero_as_unlimited'] = true;
 ?>
 EOF
 

tests/lib/installed-state.sh

@@ -19,16 +19,16 @@ parse_miab_version_string() {
     local tmpfile
     tmpfile=$(mktemp)
     awk -F- '
-/^v[0-9]+\./ { split($1,a,"."); print "MAJOR="substr(a[1],2); print "MINOR="a[2]; print "RELEASE="$2; next }  
+/^v[0-9]+\./ { split($1,a,"."); print "MAJOR="substr(a[1],2); print "MINOR="a[2]; print "RELEASE="$2; next }
 
 $1 ~ /^v[0-9]+[a-z]$/ { print "MAJOR="substr($1,2,length($1)-2); print "MINOR="substr($1,length($1))-"a"+1; print "RELEASE="; next }
 
 $1 ~ /^v[0-9]+[A-Z]$/ { print "MAJOR="substr($1,2,length($1)-2); print "MINOR="substr($1,length($1))-"A"+1; print "RELEASE="; next }
 
-$1 ~ /^v[0-9]+$/ { print "MAJOR="substr($1,2); print "MINOR="; print "RELEASE="; next } 
+$1 ~ /^v[0-9]+$/ { print "MAJOR="substr($1,2); print "MINOR="; print "RELEASE="; next }
 
 { exit 1 }' >> "$tmpfile" <<< "$1"
-    
+
     if [ $? -ne 0 ]; then
         rm -f "$tmpfile"
         return 1
@@ -79,7 +79,7 @@ installed_state_capture() {
     echo "MAJOR=$MAJOR" >>"$info"
     echo "MINOR=$MINOR" >>"$info"
     echo "RELEASE=$RELEASE" >>"$info"
-    
+
     echo "GIT_ORIGIN='$(git remote -v | grep ^origin | grep 'fetch)$' | awk '{print $2}')'" >>"$info"
     echo "MIGRATION_VERSION=$([ -e "$STORAGE_ROOT/mailinabox.version" ] && cat "$STORAGE_ROOT/mailinabox.version")" >>"$info"
     echo "MIGRATION_ML_VERSION=$([ -e "$STORAGE_ROOT/mailinabox-ldap.version" ] && cat "$STORAGE_ROOT/mailinabox-ldap.version")" >>"$info"
@@ -114,7 +114,7 @@ installed_state_capture() {
             return 3
         fi
     done
-    
+
     return 0
 }
 
@@ -123,7 +123,7 @@ installed_state_capture() {
 installed_state_compare() {
     local s1="$1"
     local s2="$2"
-    
+
     local output
     local changed="false"
 
@@ -138,13 +138,15 @@ installed_state_compare() {
     RELEASE_A="${RELEASE:-0}"
     PROD_A="miab"
     grep "mailinabox-ldap" <<<"$GIT_ORIGIN" >/dev/null && PROD_A="miabldap"
-    
+    MIGRATION_ML_VERSION_A="${MIGRATION_ML_VERSION:-0}"
+
     source "$s2/info.txt"
     MAJOR_B="$MAJOR"
     MINOR_B="${MINOR:-0}"
     RELEASE_B="${RELEASE:-0}"
     PROD_B="miab"
     grep "mailinabox-ldap" <<<"$GIT_ORIGIN" >/dev/null && PROD_B="miabldap"
+    MIGRATION_ML_VERSION_B="${MIGRATION_ML_VERSION:-0}"
 
     cmptype="${PROD_A}2${PROD_B}"
 
@@ -155,7 +157,7 @@ installed_state_compare() {
     cp "$s1/aliases.json" "$s1/aliases-cmp.json" || changed="true"
     cp "$s2/users.json" "$s2/users-cmp.json" || changed="true"
     cp "$s2/aliases.json" "$s2/aliases-cmp.json" || changed="true"
-    
+
     if [ "$cmptype" = "miab2miabldap" ]
     then
         # user display names is a feature added to MiaB-LDAP that is
@@ -164,7 +166,7 @@ installed_state_compare() {
 
         # alias descriptions is a feature added to MiaB-LDAP that is
         # not in MiaB
-        grep -v '"description":' "$s2/aliases.json" > "$s2/aliases-cmp.json" || changed="true"        
+        grep -v '"description":' "$s2/aliases.json" > "$s2/aliases-cmp.json" || changed="true"
     fi
 
     # cmp: v0.54 to current
@@ -176,9 +178,16 @@ installed_state_compare() {
 
         # s2: re-sort aliases
         jq -c ".[] | .aliases | sort_by(.address) | .[] | {address:.address, forwards_to:.forwards_to, permitted_senders:.permitted_senders, auto:.auto, description:.description}"  "$s2/aliases.json" > "$s2/aliases-cmp.json"
+
+        if [ $MIGRATION_ML_VERSION_A -le 2 -a $MIGRATION_ML_VERSION_B -ge 3 ]; then
+            # miabldap migration level <=2 does not have quota fields, so
+            # remove them from the comparison
+            grep -vE '"(quota|box_quota|box_size|percent)":' "$s2/users-cmp.json" > "$s2/users-cmp2.json" || changed="true"
+            cp "$s2/users-cmp2.json" "$s2/users-cmp.json" && rm -f "$s2/users-cmp2.json"
+        fi
     fi
-    
+
-    
+
     #
     # users
     #
@@ -195,7 +204,7 @@ installed_state_compare() {
     #
     # aliases
     #
-    
+
     H2 "Aliases"
     output="$(diff "$s1/aliases-cmp.json" "$s2/aliases-cmp.json" 2>&1)"
     if [ $? -ne 0 ]; then
@@ -241,13 +250,13 @@ installed_state_compare() {
 # $4 == "SOA" { print $1" "$3" "$4" "$5" "$6" "$8" "$10" "$12; next } \
 # { for(i=1;i<=NF;i++) if (i!=2) printf("%s ",$i); print ""; }' \
 #                 "$s1/zones/$zone" > "$t1"
-            
+
 #             awk '\
 # $4 == "RRSIG" || $4 == "NSEC3" { next; } \
 # $4 == "SOA" { print $1" "$3" "$4" "$5" "$6" "$8" "$10" "$12; next } \
 # { for(i=1;i<=NF;i++) if (i!=2) printf("%s ",$i); print ""; }' \
 #                 "$s2/zones/$zone" > "$t2"
-            
+
 #             output="$(diff "$t1" "$t2" 2>&1)"
 #             if [ $? -ne 0 ]; then
 #                 echo "CHANGED zone: $zone"

tests/suites/_mgmt-functions.sh

@@ -34,11 +34,11 @@ mgmt_start() {
 	MGMT_ADMIN_PW="$(generate_password)"
 
 	delete_user "$MGMT_ADMIN_EMAIL"
-	
+
 	record "[Creating a new account with admin rights for management tests]"
 	create_user "$MGMT_ADMIN_EMAIL" "$MGMT_ADMIN_PW" "admin"
 	MGMT_ADMIN_DN="$ATTR_DN"
-	record "Created: $MGMT_ADMIN_EMAIL at $MGMT_ADMIN_DN"	
+	record "Created: $MGMT_ADMIN_EMAIL at $MGMT_ADMIN_DN"
 }
 
 mgmt_end() {
@@ -191,7 +191,7 @@ mgmt_assert_privileges_add() {
 mgmt_get_totp_token() {
 	local secret="$1"
 	local mru_token="$2"
-	
+
 	TOTP_TOKEN="" # this is set to the acquired token on success
 
 	# the user would normally give the secret to an authenticator app
@@ -202,7 +202,7 @@ mgmt_get_totp_token() {
 	record "[Get the current token for the secret '$secret']"
 
 	local count=0
-	
+
 	while [ -z "$TOTP_TOKEN" -a $count -lt 10 ]; do
 		TOTP_TOKEN="$(totp_current_token "$secret" 2>>"$TEST_OF")"
 		if [ $? -ne 0 ]; then
@@ -218,13 +218,13 @@ mgmt_get_totp_token() {
 			record "Success: token is '$TOTP_TOKEN'"
 			return 0
 		fi
-		
+
 		let count+=1
 	done
 
 	record "Failed: timeout !"
 	TOTP_TOKEN=""
-	return 1	
+	return 1
 }
 
 mgmt_mfa_status() {
@@ -246,7 +246,7 @@ mgmt_totp_enable() {
 	#   returns 1 if a REST error occured. $REST_ERROR has the message
 	#   returns 2 if some other error occured
 	#
-	
+
 	local user="$1"
 	local pw="$2"
 	local label="$3"  # optional
@@ -258,7 +258,7 @@ mgmt_totp_enable() {
 	if ! mgmt_mfa_status "$user" "$pw"; then
 		return 1
 	fi
-	
+
 	TOTP_SECRET="$(/usr/bin/jq -r ".new_mfa.totp.secret" <<<"$REST_OUTPUT")"
 	if [ $? -ne 0 ]; then
 		record "Unable to obtain setup totp secret - is 'jq' installed?"
@@ -271,11 +271,11 @@ mgmt_totp_enable() {
 	else
 		record "Found TOTP secret '$TOTP_SECRET'"
 	fi
-	
+
 	if ! mgmt_get_totp_token "$TOTP_SECRET"; then
 		return 2
 	fi
-	
+
 	# 2. enable TOTP
 	record "Enabling TOTP using the secret and token"
 	if ! mgmt_rest_as_user "POST" "/admin/mfa/totp/enable" "$user" "$pw" "secret=$TOTP_SECRET" "token=$TOTP_TOKEN" "label=$label"; then
@@ -284,7 +284,7 @@ mgmt_totp_enable() {
 	else
 		record "Success: POST /mfa/totp/enable: '$REST_OUTPUT'"
 	fi
-		
+
 	return 0
 }
 
@@ -318,7 +318,7 @@ mgmt_mfa_disable() {
 	local user="$1"
 	local pw="$2"
 	local mfa_id="$3"
-	
+
 	record "[Disable MFA for $user]"
 	if [ "$mfa_id" == "all" ]; then
 		mfa_id=""
@@ -327,7 +327,7 @@ mgmt_mfa_disable() {
 		if ! mgmt_mfa_status "$user" "$pw"; then
 			return 1
 		fi
-		
+
 		mfa_id="$(/usr/bin/jq -r ".enabled_mfa[0].id" <<<"$REST_OUTPUT")"
 		if [ $? -ne 0 ]; then
 			record "Unable to use /usr/bin/jq - is it installed?"
@@ -338,9 +338,9 @@ mgmt_mfa_disable() {
 			return 3
 		fi
 	fi
-	
 
-	
+
+
 	if ! mgmt_rest_as_user "POST" "/admin/mfa/disable" "$user" "$pw" "mfa-id=$mfa_id"
 	then
 		REST_ERROR="Failed: POST /admin/mfa/disable: $REST_ERROR"
@@ -387,7 +387,7 @@ mgmt_assert_admin_login() {
 		if [ $code -ne 0 ]; then
 			test_failure "Unable to run jq ($code) on /admin/login json"
 			return 1
-				
+
 		elif [ "$status" == "null" ]; then
 			test_failure "No 'status' in /admin/login json"
 			return 1
@@ -395,8 +395,39 @@ mgmt_assert_admin_login() {
 		elif [ "$status" != "$expected_status" ]; then
 			test_failure "Expected a login status of '$expected_status', but got '$status'"
 			return 1
-			
+
 		fi
 	fi
 	return 0
 }
+
+
+mgmt_get_user_quota() {
+    local user="$1"
+    record "[get user $user quota]"
+	mgmt_rest GET "/admin/mail/users/quota?email=$user"
+	local rc=$?
+    # REST_OUTPUT contains json, eg:
+    #   { "email": "alice@somedomain.com", "quota": "5000" }
+    if [ $rc -eq 0 ]; then
+        # output to stdout the quota value
+        QUOTA="$(/usr/bin/jq -r ".quota" <<<"$REST_OUTPUT" 2>>$TEST_OF)"
+        if [ $? -ne 0 ]; then
+            record "could not obtain quota member from json using jq"
+            rc=1
+        fi
+    else
+        QUOTA="error"
+    fi
+	return $rc
+
+}
+
+mgmt_set_user_quota() {
+    local user="$1"
+    local quota="$2"
+    record "[set user $user quota to $quota]"
+	mgmt_rest POST "/admin/mail/users/quota" "email=$user" "quota=$quota"
+	local rc=$?
+	return $rc
+}

tests/suites/management-users.sh

@@ -8,7 +8,7 @@
 ##### details.
 #####
 
-#	
+#
 # User management tests
 
 _test_mixed_case() {
@@ -29,16 +29,16 @@ _test_mixed_case() {
 			test_failure "Creation of a user with the same email address, but different case, succeeded."
 			test_failure "${REST_ERROR}"
 		fi
-		
+
 		# create an alias group with alice in it
 		mgmt_assert_create_alias_group "${aliases[0]}" "${alices[1]}"
 	fi
 
 	# create local user bob
 	mgmt_assert_create_user "${bobs[0]}" "$bob_pw"
-	
+
 	assert_check_logs
-	
+
 
 	# send mail from bob to alice
 	#
@@ -60,7 +60,7 @@ _test_mixed_case() {
 		output="$($PYMAIL -subj "$subject" -no-send $PRIVATE_IP ${alices[3]} "$alice_pw" 2>&1)"
 		assert_python_success $? "$output"
 		assert_check_logs
-		
+
 		# send mail from alice as the alias to bob, ensure bob got it
 		#
 		record "[Mailing to bob as alias from alice]"
@@ -87,7 +87,7 @@ test_mixed_case_users() {
 	# send mail from that user as the alias to the other user
 
 	test_start "mixed-case-users"
-	
+
 	local alices=(alice@mgmt.somedomain.com
 				  aLICE@mgmt.somedomain.com
 				  aLiCe@mgmt.somedomain.com
@@ -102,7 +102,7 @@ test_mixed_case_users() {
 				   ALICE@mgmt.anotherdomain.com)
 
 	_test_mixed_case "${alices[*]}" "${bobs[*]}" "${aliases[*]}"
-	
+
 	test_end
 }
 
@@ -115,7 +115,7 @@ test_mixed_case_domains() {
 	# send mail from that user as the alias to the other user
 
 	test_start "mixed-case-domains"
-	
+
 	local alices=(alice@mgmt.somedomain.com
 				  alice@MGMT.somedomain.com
 				  alice@mgmt.SOMEDOMAIN.com
@@ -128,9 +128,9 @@ test_mixed_case_domains() {
 	local aliases=(alice@MGMT.anotherdomain.com
 				   alice@mgmt.ANOTHERDOMAIN.com
 				   alice@Mgmt.AnotherDomain.Com)
-	
+
 	_test_mixed_case "${alices[*]}" "${bobs[*]}" "${aliases[*]}"
-	
+
 	test_end
 }
 
@@ -178,7 +178,7 @@ test_intl_domains() {
 			[ ! -z "$ATTR_DN" ] && record_search "$ATTR_DN"
 		else
 			record_search "$ATTR_DN"
-			
+
 			# required aliases are automatically created and should
 			# have both mail addresses (idna and utf8)
 			get_attribute "$LDAP_ALIASES_BASE" "(mail=abuse@$intl_person_idna_domain)" "mail"
@@ -191,7 +191,7 @@ test_intl_domains() {
 				test_failure "Require alias abuse@$intl_person_idna_domain expected to contain both idna and utf8 mail addresses"
 				record_search "$ATTR_DN"
 			fi
-			
+
 			# ensure user is removed as is expected by the remaining tests
 			mgmt_delete_user "$intl_person_idna"
 		fi
@@ -204,7 +204,7 @@ test_intl_domains() {
 		test_failure "No required alias should not exist for the $intl_person_domain domain"
 		record_search "$ATTR_DN"
 	fi
-	
+
 	# create local users bob and mary
 	mgmt_assert_create_user "$bob" "$bob_pw"
 	mgmt_assert_create_user "$mary" "$mary_pw"
@@ -212,7 +212,7 @@ test_intl_domains() {
 	# create intl alias with local user bob and intl_person in it
 	if mgmt_assert_create_alias_group "$alias" "$bob" "$intl_person"; then
 		# examine LDAP server to verify IDNA-encodings
-		
+
 		# 1. the mail attribute for the alias should have both the
 		# idna and utf8 addresses
 		get_attribute "$LDAP_ALIASES_BASE" "(mail=$alias)" "mail"
@@ -238,7 +238,7 @@ test_intl_domains() {
 
 	# re-create intl alias with local user bob only
 	mgmt_assert_create_alias_group "$alias" "$bob"
-	
+
 	assert_check_logs
 
 	if ! have_test_failures; then
@@ -300,18 +300,18 @@ test_totp() {
 		record "Expect a login failure..."
 		mgmt_assert_admin_login "$alice" "$alice_pw" "missing-totp-token"
 	fi
-	
+
 
 	# logging into /admin/me with a password and a token should
 	# succeed, and an api_key generated
 	local api_key
-	if ! have_test_failures; then		
+	if ! have_test_failures; then
 		record "Try using a password and a token to get the user api key, we may have to wait 30 seconds to get a new token..."
 
 		local old_totp_token="$TOTP_TOKEN"
 		if ! mgmt_get_totp_token "$TOTP_SECRET" "$TOTP_TOKEN"; then
 			test_failure "Could not obtain a new TOTP token"
-			
+
 		else
 			# we have a new token, try logging in ...
 			# the token must be placed in the header "x-auth-token"
@@ -331,7 +331,7 @@ test_totp() {
 	fi
 
 	# we should be able to login using the user's api key
-	if ! have_test_failures; then		
+	if ! have_test_failures; then
 		record "[Use the session key to enum users]"
 		if ! mgmt_rest_as_user "GET" "/admin/mail/users?format=json" "$alice" "$api_key"; then
 			test_failure "Unable to use the session key to issue a rest call: $REST_ERROR"
@@ -342,7 +342,7 @@ test_totp() {
 
 	# disable totp on the account - login should work with just the password
 	# and the ldap entry should not have the 'totpUser' objectClass
-	if ! have_test_failures; then		
+	if ! have_test_failures; then
 		if mgmt_assert_mfa_disable "$alice" "$api_key"; then
 			mgmt_assert_admin_login "$alice" "$alice_pw" "ok"
 		fi
@@ -354,19 +354,130 @@ test_totp() {
 	else
 		check_logs
 	fi
-	
+
 	mgmt_assert_delete_user "$alice"
 	test_end
 }
 
 
 
+test_mailbox_quotas() {
+    test_start "mailbox-quotas"
+
+    # create standard user alice
+    local alice="alice@somedomain.com"
+    create_user "$alice" "alice"
+
+    # quota should be unlimited for newly added users
+    if ! mgmt_get_user_quota "$alice"; then
+        test_failure "Unable to get $alice's quota: $REST_ERROR"
+    elif [ "$QUOTA" != "0" -a "$QUOTA" != "unlimited" ]; then
+        test_failure "A newly created user should have unlimited quota"
+    fi
+
+    # get alice's current total number of messages. should be 0 unless
+    # the account was "archived"
+    local count_messages="$(doveadm -f json quota get -u "$alice" | jq -r '.[] | select(.type=="MESSAGE") | .value')"
+    record "$alice currently has $count_messages messages"
+
+    # set alice's quota to a small number
+    local quota_value="5K"
+    if ! mgmt_set_user_quota "$alice" "$quota_value"
+    then
+        test_failure "Unable to set $alice's quota: $REST_ERROR"
+    else
+        # read back the quota - make sure it's what we set
+        if ! mgmt_get_user_quota "$alice" || [ "$QUOTA" != "$quota_value" ]
+        then
+            test_failure "Setting quota failed - expected quota does not match current quota: $REST_OUTPUT $REST_ERROR QUOTA=$QUOTA"
+
+        else
+            record_search "(mail=$alice)"
+        fi
+    fi
+
+    if ! have_test_failures; then
+        # send messages large enough to exceed the quota
+        local output
+        local subjects=()
+        local msgidx=0
+        local body="$(python3 -c 'for i in range(0,int(512/4)): print("abc\n", end="")')"
+        local quota_exceeded="no"
+
+        while ! have_test_failures && [ $msgidx -lt 10 ]; do
+            record ""
+            record "[send msg $msgidx]"
+            local subj="msg$msgidx - $(generate_password)"
+            output="$($PYMAIL -smtp-debug -body-from-stdin -no-delete -subj "$subj" $PRIVATE_IP $alice alice <<<"$body" 2>&1)"
+		    if ! assert_python_success $? "$output"; then
+                break
+            fi
+
+            # You'd expect that the send would fail when the quota is
+            # exceeded, but it doesn't. Postfix accepts it into it's
+            # queue, then bounces the message back to sender with
+            # delivery status notification (DSN) of 5.2.2 when it
+            # processes the queue.
+            #
+            # The debugging messages (turned on by the -smtp-debug
+            # argument) hold the internal postfix message id, so
+            # extract that, then grep the logs to see if the message
+            # was bounced due to 5.2.2.
+
+            local postid="$(awk '/^data: .* queued as/  { match($0," as "); print substr($0,RSTART+4,10); exit }' <<<"$output" 2>>$TEST_OF)"
+            record "Extracted POSTID=$postid"
+            if [ ! -z "$postid" ]; then
+                /usr/sbin/postqueue -f >>"$TEST_OF" 2>&1
+                flush_logs
+                record "[dovecot and postfix logs for msg $msgidx]"
+                record "logs: $(grep "$postid" /var/log/mail.log)"
+
+                if grep "$postid" /var/log/mail.log | grep "status=bounced" | grep -Fq "5.2.2"; then
+                    # success - message was rejected
+                    quota_exceeded="yes"
+                    break
+                fi
+            fi
+
+            subjects+=( "$subj" )
+            let msgidx+=1
+            # doveadm quota get -u "$alice" >>"$TEST_OF" 2>&1
+        done
+
+        if ! have_test_failures && [ "$quota_exceeded" = "no" ]; then
+            test_failure "Quota restriction was not enforced by dovecot after sending $msgidx messages"
+        fi
+
+        # cleanup: delete the messages
+        msgidx=0
+        for subj in "${subjects[@]}"; do
+            record "[delete msg $msgidx]"
+            record "subj=$subj"
+            $PYMAIL -no-send -timeout 2 -subj "$subj" $PRIVATE_IP $alice alice >>$TEST_OF 2>&1
+            let msgidx+=1
+        done
+
+        # verify cleanup worked
+        local cur_count_messages="$(doveadm -f json quota get -u "$alice" | jq -r '.[] | select(.type=="MESSAGE") | .value')"
+        if [ $count_messages -ne $cur_count_messages ]; then
+            test_failure "Cleanup failed: test account $alice started with $count_messages but ended up with $cur_count_messages"
+        fi
+    fi
+
+    # cleanup: delete the test user
+    delete_user "$alice"
+    test_end
+}
+
+
+
+
 suite_start "management-users" mgmt_start
 
 test_totp
 test_mixed_case_domains
 test_mixed_case_users
 test_intl_domains
+test_mailbox_quotas
 
 suite_end mgmt_end
-

tests/system-setup/setup-defaults.sh

@@ -64,7 +64,7 @@ export UPSTREAM_TAG="${UPSTREAM_TAG:-}"
 
 # For setup scripts that install miabldap releases (eg. upgrade tests)
 export MIABLDAP_GIT="${MIABLDAP_GIT:-https://github.com/downtownallday/mailinabox-ldap.git}"
-export MIABLDAP_RELEASE_TAG="${MIABLDAP_RELEASE_TAG:-v60}"
+export MIABLDAP_RELEASE_TAG="${MIABLDAP_RELEASE_TAG:-v70}"
 
 # When running tests that require php, use this version of php. This
 # should be the same as what's in setup/functions.sh.
@@ -73,3 +73,4 @@ export PHP_VER=8.0
 # Tag of last version supported on Ubuntu Bionic 18.04
 UPSTREAM_FINAL_RELEASE_TAG_BIONIC64=v57a
 MIABLDAP_FINAL_RELEASE_TAG_BIONIC64=f6cd8f56c3bcb20969c6cf66c040c8efa3773f3a
+MIABLDAP_INITIAL_RELEASE_TAG_JAMMY64=v60

tests/test_mail.py

@@ -28,6 +28,8 @@ def usage():
 	print("    -no-send: don't send, just delete")
 	print("    -no-delete: don't delete, just send")
 	print("    -timeout <seconds>: how long to wait for message")
+	print("    -body-from-stdin: read the message body from stdin")
+	print("    -smtp-debug: output debugging messages")
 	print("");
 	sys.exit(1)
 
@@ -48,6 +50,8 @@ delete_msg=True  # login to imap and delete message
 wait_timeout=30  # abandon timeout wiating for message delivery
 wait_cycle_sleep=5  # delay between delivery checks
 subject="Mail-in-a-Box Automated Test Message " + uuid.uuid4().hex  # message subject
+body_from_stdin=False
+smtp_debug=False
 
 # process command line
 argi=1
@@ -81,6 +85,12 @@ while argi<len(sys.argv):
 	elif arg=="-timeout" and arg_remaining>1:
 		wait_timeout=int(sys.argv[argi+1])
 		argi+=2
+	elif arg=="-body-from-stdin":
+		body_from_stdin = True
+		argi+=1
+	elif arg=="-smtp-debug":
+		smtp_debug = True
+		argi+=1
 	else:
 		usage()
 
@@ -100,14 +110,20 @@ headerfrom = if_unset(headerfrom, emailfrom)
 emailto = if_unset(emailto, login)
 emailto_pw = if_unset(emailto_pw, pw)
 
+if body_from_stdin:
+	body=sys.stdin.readlines()
+else:
+	body=['This is a test message. It should be automatically deleted by the test script.']
+
 msg = """From: {headerfrom}
 To: {emailto}
 Subject: {subject}
 
-This is a test message. It should be automatically deleted by the test script.""".format(
+{body}""".format(
 	headerfrom=headerfrom,
 	emailto=emailto,
 	subject=subject,
+	body=''.join(body)
 	)
 
 def imap_login(host, login, pw):
@@ -162,7 +178,8 @@ def smtp_login(host, login, pw, port):
 		server.starttls()
 	else:
 		server = smtplib.SMTP_SSL(host)
-	#server.set_debuglevel(1)
+	if smtp_debug:
+		server.set_debuglevel(1)
 
 	# Verify that the EHLO name matches the server's reverse DNS.
 	ipaddr = socket.gethostbyname(host) # IPv4 only!
@@ -191,7 +208,10 @@ def smtp_login(host, login, pw, port):
 if send_msg:
 	# Attempt to send a mail.
 	server = smtp_login(host, login, pw, port)
-	server.sendmail(emailfrom, [emailto], msg)
+	# sendmail: "If this method does not raise an exception, it returns a dictionary, with one entry for each recipient that was refused. Each entry contains a tuple of the SMTP error code and the accompanying error message sent by the server."
+	errors = server.sendmail(emailfrom, [emailto], msg)
+	#print(errors)
+	#if errors: raise ValueError(errors)
 	server.quit()
 	print("SMTP submission is OK.")