Merge remote-tracking branch 'chadfurman/master' into chads-quota

# Conflicts:
#	management/daemon.py
#	management/mailconfig.py
#	management/templates/users.html
#	setup/bootstrap.sh
#	setup/mail-postfix.sh
#	setup/mail-users.sh
#	setup/migrate.py

setup/bootstrap.sh

@@ -133,7 +133,7 @@ fi
 if [ -z "${ENCRYPTION_AT_REST:-}" ]; then
     source ehdd/ehdd_funcs.sh || exit 1
     hdd_exists && ENCRYPTION_AT_REST=true
-elif [ "${ENCRYPTION_AT_REST:-}" = "false" ]; then 
+elif [ "${ENCRYPTION_AT_REST:-}" = "false" ]; then
     source ehdd/ehdd_funcs.sh || exit 1
     if hdd_exists; then
         echo "Encryption-at-rest must be disabled manually"
@@ -147,4 +147,3 @@ if [ "${ENCRYPTION_AT_REST:-false}" = "true" ]; then
 else
     setup/start.sh </dev/tty
 fi
-

setup/ldap.sh

@@ -99,7 +99,7 @@ create_miab_conf() {
 		_add_if_missing "${prefix}_DN" "cn=$cn,$LDAP_SERVICES_BASE"
 		_add_if_missing "${prefix}_PASSWORD" "$(generate_password 64)"
 	done
-	
+
 	chmod 0640 "$MIAB_INTERNAL_CONF_FILE"
 	. "$MIAB_INTERNAL_CONF_FILE"
 }
@@ -126,7 +126,7 @@ create_service_accounts() {
 	# create service accounts. service accounts have special access
 	# rights, generally read-only to users, aliases, and configuration
 	# subtrees (see apply_access_control)
-	
+
 	local prefix dn pass
 	for prefix in ${SERVICE_ACCOUNTS[*]}
 	do
@@ -147,7 +147,7 @@ userPassword: $(slappasswd_hash "$pass")
 EOF
 		fi
 	done
-	
+
 }
 
 
@@ -155,7 +155,7 @@ install_system_packages() {
 	# install required deb packages, generate admin credentials
 	# and apply them to the installation
 	create_miab_conf
-	
+
 	# Set installation defaults to avoid interactive dialogs. See
 	# /var/lib/dpkg/info/slapd.templates for a list of what can be set
 	debconf-set-selections <<EOF
@@ -164,10 +164,10 @@ slapd slapd/domain string ${LDAP_DOMAIN}
 slapd slapd/password1 password ${LDAP_ADMIN_PASSWORD}
 slapd slapd/password2 password ${LDAP_ADMIN_PASSWORD}
 EOF
-	
+
 	# Install packages
 	say "Installing OpenLDAP server..."
-	
+
 	# we must install slapd without DEBIAN_FRONTEND=noninteractive or
 	# debconf selections are ignored
 	hide_output apt-get -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confnew" install slapd
@@ -227,7 +227,7 @@ EOF
 		say "  is set to: $ATTR_VALUE"
 		say "  expected : $LDAP_ADMIN_DN"
 		die
-	fi		
+	fi
 }
 
 relocate_slapd_data() {
@@ -277,11 +277,11 @@ relocate_slapd_data() {
 	say_verbose "		DB='${DB_DIR}'"
 	say_verbose "	to:"
 	say_verbose "	   CONF=${MIAB_SLAPD_CONF}"
-	say_verbose "		 DB=${MIAB_SLAPD_DB_DIR}"	
+	say_verbose "		 DB=${MIAB_SLAPD_DB_DIR}"
 	say_verbose ""
 	say_verbose "Stopping slapd"
 	systemctl stop slapd || die "Could not stop slapd"
-	
+
 	# Modify the path to dc=mailinabox's database directory
 	say_verbose "Dump config database"
 	local TMP="/tmp/miab_relocate_ldap.ldif"
@@ -320,7 +320,7 @@ schema_to_ldif() {
 			cat="curl -s"
 		fi
 	fi
-	
+
 	cat >"$ldif" <<EOF
 dn: cn=$cn,cn=schema,cn=config
 objectClass: olcSchemaConfig
@@ -358,7 +358,7 @@ EOF
 
 
 add_schemas() {
-	# Add necessary schema's for MiaB operaion		 
+	# Add necessary schema's for MiaB operaion
 	#
 	# Note: the postfix schema originally came from the ldapadmin
 	# project (GPL)(*), but has been modified to support the needs of
@@ -385,7 +385,7 @@ add_schemas() {
 			ldapadd -Q -Y EXTERNAL -H ldapi:/// -f "$ldif" >/dev/null
 			rm -f "$ldif"
 		fi
-	done	
+	done
 }
 
 
@@ -481,7 +481,7 @@ EOF
 add_overlays() {
 	# Apply slapd overlays - apply the commonly used member-of overlay
 	# now because adding it later is harder.
-	
+
 	# Get the config dn for the database
 	get_attribute "cn=config" "olcSuffix=${LDAP_BASE}" "dn"
 	[ -z "$ATTR_DN" ] &&
@@ -498,7 +498,7 @@ add: olcModuleLoad
 olcModuleLoad: memberof.la
 EOF
 	fi
-	
+
 	get_attribute "$cdn" "(olcOverlay=memberof)" "olcOverlay"
 	if [ -z "$ATTR_DN" ]; then
 		say_verbose "Adding memberof overlay to $LDAP_BASE"
@@ -516,7 +516,7 @@ EOF
 
 add_indexes() {
 	# Index mail-related attributes
-	
+
 	# Get the config dn for the database
 	get_attribute "cn=config" "olcSuffix=${LDAP_BASE}" "dn"
 	[ -z "$ATTR_DN" ] &&
@@ -678,7 +678,7 @@ EOF
 #
 process_cmdline() {
 	[ -e "$MIAB_INTERNAL_CONF_FILE" ] && . "$MIAB_INTERNAL_CONF_FILE"
-	
+
 	if [ "$1" == "-d" ]; then
 		# Start slapd in interactive/debug mode
 		echo "!! SERVER DEBUG MODE !!"
@@ -688,7 +688,7 @@ process_cmdline() {
 		echo "Listening on $SLAPD_SERVICES..."
 		/usr/sbin/slapd -h "$SLAPD_SERVICES" -g openldap -u openldap -F $MIAB_SLAPD_CONF -d ${2:-1}
 		exit 0
-		
+
 	elif [ "$1" == "-config" ]; then
 		# Apply a certain configuration
 		if [ "$2" == "server" ]; then
@@ -734,7 +734,7 @@ process_cmdline() {
 		local hide_attrs="(structuralObjectClass|entryUUID|creatorsName|createTimestamp|entryCSN|modifiersName|modifyTimestamp)"
 		local slapcat_args=(-F "$MIAB_SLAPD_CONF" -o ldif-wrap=no)
 		[ ${verbose:-0} -gt 0 ] && hide_attrs="(_____NEVERMATCHES)"
-		
+
 		if [ "$s" == "all" ]; then
 			echo ""
 			echo '--------------------------------'
@@ -777,7 +777,7 @@ process_cmdline() {
 		if [ "$s" == "permitted-senders" -o "$s" == "ps" ]; then
 			echo ""
 			echo '--------------------------------'
-			local attrs=(mail member mailRoutingAddress rfc822MailMember)
+			local attrs=(mail member mailRoutingAddress mailMember)
 			[ ${verbose:-0} -gt 0 ] && attrs=()
 			debug_search "(objectClass=mailGroup)" "$LDAP_PERMITTED_SENDERS_BASE" ${attrs[@]}
 		fi
@@ -814,7 +814,7 @@ process_cmdline() {
 		rm -f "/etc/default/slapd"
 		echo "Done"
 		exit 0
-		
+
 	elif [ ! -z "$1" ]; then
 		echo "Invalid command line argument '$1'"
 		exit 1

setup/mail-dovecot.sh

@@ -76,6 +76,32 @@ tools/editconf.py /etc/dovecot/conf.d/10-mail.conf \
 
 # Create, subscribe, and mark as special folders: INBOX, Drafts, Sent, Trash, Spam and Archive.
 cp conf/dovecot-mailboxes.conf /etc/dovecot/conf.d/15-mailboxes.conf
+sed -i "s/#mail_plugins =\(.*\)/mail_plugins =\1 \$mail_plugins quota/" /etc/dovecot/conf.d/10-mail.conf
+if ! grep -q "mail_plugins.* imap_quota" /etc/dovecot/conf.d/20-imap.conf; then
+  sed -i "s/\(mail_plugins =.*\)/\1\n  mail_plugins = \$mail_plugins imap_quota/" /etc/dovecot/conf.d/20-imap.conf
+fi
+
+# configure stuff for quota support
+if ! grep -q "quota_status_success = DUNNO" /etc/dovecot/conf.d/90-quota.conf; then
+    cat > /etc/dovecot/conf.d/90-quota.conf << EOF;
+plugin {
+  quota = maildir
+
+  quota_grace = 10%
+
+  quota_status_success = DUNNO
+  quota_status_nouser = DUNNO
+  quota_status_overquota = "522 5.2.2 Mailbox is full"
+}
+
+service quota-status {
+    executable = quota-status -p postfix
+    inet_listener {
+        port = 12340
+    }
+}
+EOF
+fi
 
 # ### IMAP/POP
 

setup/mail-postfix.sh

@@ -259,7 +259,7 @@ tools/editconf.py /etc/postfix/main.cf  -e lmtp_destination_recipient_limit=
 # "450 4.7.1 Client host rejected: Service unavailable". This is a retry code, so the mail doesn't properly bounce. #NODOC
 tools/editconf.py /etc/postfix/main.cf \
 	smtpd_sender_restrictions="reject_non_fqdn_sender,reject_unknown_sender_domain,reject_authenticated_sender_login_mismatch,reject_rhsbl_sender dbl.spamhaus.org=127.0.1.[2..99]" \
-	smtpd_recipient_restrictions="permit_sasl_authenticated,permit_mynetworks,reject_rbl_client zen.spamhaus.org=127.0.0.[2..11],reject_unlisted_recipient,check_policy_service unix:private/policy-spf,check_policy_service inet:127.0.0.1:10023"
+	smtpd_recipient_restrictions="permit_sasl_authenticated,permit_mynetworks,reject_rbl_client zen.spamhaus.org=127.0.0.[2..11],reject_unlisted_recipient,check_policy_service unix:private/policy-spf,check_policy_service inet:127.0.0.1:10023,check_policy_service inet:127.0.0.1:12340"
 
 # Postfix connects to Postgrey on the 127.0.0.1 interface specifically. Ensure that
 # Postgrey listens on the same interface (and not IPv6, for instance).

setup/mail-users.sh

@@ -89,7 +89,7 @@ pass_attrs = maildrop=user
 #   lmtp delivery, pass_filter is not used, and postfix has already
 #   rewritten the envelope using the maildrop address.
 user_filter = (&(objectClass=mailUser)(|(mail=%u)(maildrop=%u)))
-user_attrs = maildrop=user
+user_attrs = maildrop=user mailboxQuota=quota_rule=*:bytes=%\$
 
 # Account iteration for various dovecot tools (doveadm)
 iterate_filter = (objectClass=mailUser)
@@ -269,3 +269,6 @@ chmod 0640 /etc/postfix/virtual-alias-maps.cf
 
 restart_service postfix
 restart_service dovecot
+
+# force a recalculation of all user quotas
+doveadm quota recalc -A

setup/migrate.py

@@ -200,6 +200,12 @@ def migration_14(env):
 	db = os.path.join(env["STORAGE_ROOT"], 'mail/users.sqlite')
 	shell("check_call", ["sqlite3", db, "CREATE TABLE auto_aliases (id INTEGER PRIMARY KEY AUTOINCREMENT, source TEXT NOT NULL UNIQUE, destination TEXT NOT NULL, permitted_senders TEXT);"])
 
+def migration_15(env):
+	# Add a column to the users table to store their quota limit.  Default to '0' for unlimited.
+	db = os.path.join(env["STORAGE_ROOT"], 'mail/users.sqlite')
+	shell("check_call", ["sqlite3", db, "ALTER TABLE users ADD COLUMN quota TEXT NOT NULL DEFAULT '0';"])
+
+
 ###########################################################
 
 
@@ -306,7 +312,7 @@ def migration_miabldap_2(env):
 			return ldif.replace("rfc822MailMember: ", "mailMember: ")
 		# apply schema changes miabldap/1 -> miabldap/2
 		ldap.unbind()
-		print("Apply schema changes")
+		print("Apply schema changes to support utf8 email addresses")
 		m14.apply_schema_changes(env, ldapvars, ldif_change_fn)
 		# reconnect
 		ldap = connect(ldapvars)
@@ -329,6 +335,52 @@ def migration_miabldap_2(env):
 
 	ldap.unbind()
 
+def migration_miabldap_3(env):
+	# This migration step changes the ldap schema to support quotas
+	#
+	# possible states at this point:
+	#   miabldap was installed and is being upgraded
+	#      -> schema update needed
+	#   a miab install was present and step 1 upgaded it to miabldap
+	#      -> new schema already present
+	#
+	sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), "../management")))
+	import ldap3
+	from backend import connect
+	import migration_14 as m14
+
+	# 1. get ldap site details
+	ldapvars = load_env_vars_from_file(os.path.join(env["STORAGE_ROOT"], "ldap/miab_ldap.conf"), strip_quotes=True)
+
+	# connect before schema changes to ensure admin password works
+	ldap = connect(ldapvars)
+
+	# 2. if this is a miab -> maibldap install, the new schema is
+	# already in place and no schema changes are needed. however,
+	# if this is a miabldap/1 to miabldap/2 migration, we must
+	# upgrade the schema.
+	ret = shell("check_output", [
+		"ldapsearch",
+		"-Q",
+		"-Y", "EXTERNAL",
+		"-H", "ldapi:///",
+		"(&(objectClass=olcSchemaConfig)(cn={*}postfix))",
+		"-b", "cn=schema,cn=config",
+		"-o", "ldif_wrap=no",
+		"-LLL",
+		"olcObjectClasses"
+	])
+
+	ldap.unbind()
+
+	if "mailboxQuota" not in ret:
+		def ldif_change_fn(ldif):
+			# the schema change we're making does not require any data changes
+			return ldif
+		# apply schema changes miabldap/2 -> miabldap/3
+		print("Apply schema changes to support mailbox quotas")
+		m14.apply_schema_changes(env, ldapvars, ldif_change_fn)
+
 
 def get_current_migration():
 	ver = 0

setup/migration_13.py

@@ -17,7 +17,7 @@
 import uuid, os, sqlite3, ldap3, hashlib
 
 
-def add_user(env, ldapconn, search_base, users_base, domains_base, email, password, privs, totp, cn=None):
+def add_user(env, ldapconn, search_base, users_base, domains_base, email, password, privs, quota, totp, cn=None):
 	# Add a sqlite user to ldap
 	#   env are the environment variables
 	#   ldapconn is the bound ldap connection
@@ -27,6 +27,7 @@ def add_user(env, ldapconn, search_base, users_base, domains_base, email, passwo
 	#   email is the user's email
 	#   password is the user's current sqlite password hash
 	#   privs is an array of privilege names for the user
+	#   quota is the users mailbox quota (string; defaults to '0')
 	#   totp contains the list of secrets, mru tokens, and labels
 	#   cn is the user's common name [optional]
 	#
@@ -45,13 +46,14 @@ def add_user(env, ldapconn, search_base, users_base, domains_base, email, passwo
 	m = hashlib.sha1()
 	m.update(bytearray(email.lower(),'utf-8'))
 	uid = m.hexdigest()
-	
+
 	# Attributes to apply to the new ldap entry
 	objectClasses = [ 'inetOrgPerson','mailUser','shadowAccount' ]
 	attrs = {
 		"mail" : email,
 		"maildrop" : email,
 		"uid" : uid,
+		"mailboxQuota": quota,
 		# Openldap uses prefix {CRYPT} for all crypt(3) formats
 		"userPassword" : password.replace('{SHA512-CRYPT}','{CRYPT}')
 	}
@@ -91,10 +93,10 @@ def add_user(env, ldapconn, search_base, users_base, domains_base, email, passwo
 		attrs['totpMruToken'] = totp["mru_token"]
 		attrs['totpMruTokenTime'] = totp["mru_token_time"]
 		attrs['totpLabel'] = totp["label"]
-	
+
 	# Add user
 	dn = "uid=%s,%s" % (uid, users_base)
-	
+
 	print("adding user %s" % email)
 	ldapconn.add(dn, objectClasses, attrs)
 
@@ -116,14 +118,15 @@ def create_users(env, conn, ldapconn, ldap_base, ldap_users_base, ldap_domains_b
 
 	# select users
 	c = conn.cursor()
-	c.execute("SELECT id, email, password, privileges from users")
+	c.execute("SELECT id, email, password, privileges, quota from users")
 
 	users = {}
 	for row in c:
 		user_id=row[0]
 		email=row[1]
 		password=row[2]
-		privs=row[3]	
+		privs=row[3]
+		quota=row[4]
 		totp = None
 
 		c2 = conn.cursor()
@@ -143,7 +146,7 @@ def create_users(env, conn, ldapconn, ldap_base, ldap_users_base, ldap_domains_b
 			totp["label"].append("{%s}%s" % (rowidx, row2[2] or ''))
 			rowidx += 1
 
-		dn = add_user(env, ldapconn, ldap_base, ldap_users_base, ldap_domains_base, email, password, privs.split("\n"), totp)
+		dn = add_user(env, ldapconn, ldap_base, ldap_users_base, ldap_domains_base, email, password, privs.split("\n"), quota, totp)
 		users[email] = dn
 	return users
 
@@ -164,7 +167,7 @@ def create_aliases(env, conn, ldapconn, aliases_base):
 			cn="%s" % uuid.uuid4()
 			dn="cn=%s,%s" % (cn, aliases_base)
 			description="Mail group %s" % alias
-			
+
 			if alias.startswith("postmaster@") or \
 			   alias.startswith("hostmaster@") or \
 			   alias.startswith("abuse@") or \
@@ -172,7 +175,7 @@ def create_aliases(env, conn, ldapconn, aliases_base):
 			   alias == "administrator@" + env['PRIMARY_HOSTNAME']:
 				description = "Required alias"
 
-			print("adding alias %s" % alias)			
+			print("adding alias %s" % alias)
 			ldapconn.add(dn, ['mailGroup'], {
 				"mail": alias,
 				"description": description
@@ -196,7 +199,7 @@ def populate_aliases(conn, ldapconn, users_map, aliases_map):
 		alias_dn=aliases_map[alias]
 		members = []
 		mailMembers = []
-		
+
 		for email in row[1].split(','):
 			email=email.strip()
 			if email=="":
@@ -207,13 +210,13 @@ def populate_aliases(conn, ldapconn, users_map, aliases_map):
 				members.append(aliases_map[email])
 			else:
 				mailMembers.append(email)
-		
+
 		print("populate alias group %s" % alias)
 		changes = {}
 		if len(members)>0:
 			changes["member"]=[(ldap3.MODIFY_REPLACE, members)]
 		if len(mailMembers)>0:
-			changes["rfc822MailMember"]=[(ldap3.MODIFY_REPLACE, mailMembers)]			
+			changes["rfc822MailMember"]=[(ldap3.MODIFY_REPLACE, mailMembers)]
 		ldapconn.modify(alias_dn, changes)
 
 

setup/webmail.sh

@@ -214,6 +214,7 @@ cat > $RCM_CONFIG <<EOF;
 
 /* prevent CSRF, requires php 7.3+ */
 \$config['session_samesite'] = 'Strict';
+\$config['quota_zero_as_unlimited'] = true;
 ?>
 EOF