Merge branch 'main' of https://github.com/mail-in-a-box/mailinabox

Upstream is adding handling for utf8 domains by creating a domain alias @utf8 -> @idna. I'm deviating from this approach by setting multiple email address (idna and utf8) per user and alias where a domain contains non-ascii characters. The maildrop (mailbox) remains the same - all mail goes to the user's mailbox regardless of which email address was used. This is more in line with how other systems (eg. active directory), handle multiple email addresses for a single user.

# Conflicts:
#	README.md
#	management/mailconfig.py
#	management/templates/index.html
#	setup/dns.sh
#	setup/mail-users.sh

CHANGELOG.md

@@ -1,6 +1,35 @@
 CHANGELOG
 =========
 
+In Development
+--------------
+
+Mail:
+
+* "SMTPUTF8" is now disabled in Postfix. Because Dovecot still does not support SMTPUTF8, incoming mail to internationalized addresses was bouncing. This fixes incoming mail to internationalized domains (which was probably working prior to v0.40), but it will prevent sending outbound mail to addresses with internationalized local-parts.
+* Upgraded to Roundcube 1.5 Release Candidate.
+
+Firewall:
+
+* Fail2ban's IPv6 support is enabled.
+
+Control panel:
+
+* The control panel menus are now hidden before login, but now non-admins can log in to access the mail and contacts/calendar instruction pages.
+* The login form now disables browser autocomplete in the two-factor authentication code field.
+* After logging in, the default page is now a fast-loading welcome page rather than the slow-loading system status checks page.
+* The backup retention period option now displays for B2 backup targets.
+* The DNSSEC DS record recommendations are cleaned up and now recommend changing records that use SHA1.
+* The Munin monitoring pages no longer require a separate HTTP basic authentication login and can be used if two-factor authentication is turned on.
+* Control panel logins are now tied to a session backend that allows true logouts (rather than an encrypted cookie).
+* Failed logins no longer directly reveal whether the email address corresponds to a user account.
+* Browser dark mode now inverts the color scheme.
+
+Other:
+
+* The mail log tool now doesn't crash if there are email addresess in log messages with invalid UTF-8 characters.
+* Additional nsd.conf files can be placed in /etc/nsd.conf.d.
+
 v0.54 (June 20, 2021)
 ---------------------
 

conf/postfix.schema

@@ -1,60 +0,0 @@
-# LDAP Admin Extensions for Postfix MTA support
-
-attributetype ( 1.3.6.1.4.1.15347.2.102 
-	NAME 'transport' 
-	SUP name)
-
-attributetype ( 1.3.6.1.4.1.15347.2.101 
-	NAME 'mailRoutingAddress' 
-	SUP mail )
-
-attributetype ( 1.3.6.1.4.1.15347.2.110 NAME 'maildest'
-	DESC 'Restricted to send only to local network'
-	EQUALITY caseIgnoreMatch
-	SUBSTR caseIgnoreSubstringsMatch
-	SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{32768} )
-
-attributetype ( 1.3.6.1.4.1.15347.2.111 NAME 'mailaccess'
-	DESC 'Can be mailed to restricted groups'
-	EQUALITY caseIgnoreMatch
-	SUBSTR caseIgnoreSubstringsMatch
-	SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{32768} )
-
-attributetype ( 1.3.6.1.4.1.15347.2.100
-	NAME ( 'maildrop' )
-	DESC 'RFC1274: RFC822 Mailbox'
-	EQUALITY caseIgnoreIA5Match
-	SUBSTR caseIgnoreIA5SubstringsMatch
-	SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} )
-
-attributetype ( 1.3.6.1.4.1.10018.1.1.1 NAME 'mailbox'
-	DESC 'The absolute path to the mailbox for a mail account in a non-default location'
-	EQUALITY caseExactIA5Match
-	SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 SINGLE-VALUE )
-
-objectclass ( 1.3.6.1.4.1.15347.2.1
-	NAME 'mailUser'
-    	DESC 'E-Mail User'
-	SUP top
-	AUXILIARY
-	MUST ( uid $ mail $ maildrop )
-	MAY ( cn $ mailbox $ maildest $ mailaccess )
-	)
-
-objectclass ( 1.3.6.1.4.1.15347.2.2
-	NAME 'mailGroup'
-    	DESC 'E-Mail Group'
-	SUP top
-	STRUCTURAL
-	MUST ( cn $ mail )
-	MAY ( mailRoutingAddress $ member $ description )
-	)
-
-objectclass ( 1.3.6.1.4.1.15347.2.3
-	NAME 'transportTable'
-    	DESC 'MTA Transport Table'
-	SUP top
-	STRUCTURAL
-	MUST ( cn $ transport )
-	)
-

conf/schema/mfa-totp.schema

@@ -1,11 +1,6 @@
 #
 # MiaB-LDAP's directory schema for time-based one time passwords (TOTP)
 #
-# MiaB LDAP UUID(v4): 7392cdda-5ec8-431f-9936-0000273c0167
-#                 or: 1939000794.24264.17183.39222.658243943
-#
-
-objectIdentifier MiabLDAProot 2.25.1939000794.24264.17183.39222.658243943
 
 objectIdentifier MiabLDAPmfa MiabLDAProot:1
 objectIdentifier MiabLDAPmfaAttributeType MiabLDAPmfa:2

conf/schema/namedProperties.schema

@@ -0,0 +1,23 @@
+#
+# Auxiliary objectclass to add named properties to an entry
+#
+
+objectIdentifier MiabLDAPadmin MiabLDAProot:3
+objectIdentifier MiabLDAPadminAttributeType MiabLDAPadmin:1
+objectIdentifier MiabLDAPadminObjectClass MiabLDAPadmin:2
+
+attributetype ( MiabLDAPadminAttributeType:1
+	DESC 'Named property'
+	NAME 'namedProperty'
+	EQUALITY caseIgnoreMatch
+	SUBSTR caseIgnoreSubstringsMatch
+	SYNTAX 1.3.6.1.4.1.1466.115.121.1.15
+	)
+
+objectClass ( MiabLDAPadminObjectClass:1
+	NAME 'namedProperties'
+	DESC 'Entry contains named properties'
+	SUP top
+	AUXILIARY
+	MAY ( namedProperty )
+	)

conf/schema/postfix.schema

@@ -0,0 +1,77 @@
+# LDAP Admin Extensions for Postfix MTA support
+#
+# MiaB LDAP UUID(v4): 7392cdda-5ec8-431f-9936-0000273c0167
+#                 or: 1939000794.24264.17183.39222.658243943
+#
+
+objectIdentifier MiabLDAProot 2.25.1939000794.24264.17183.39222.658243943
+objectIdentifier MiabLDAPmail MiabLDAProot:2
+objectIdentifier MiabLDAPmailAttributeType MiabLDAPmail:1
+objectIdentifier MiabLDAPmailObjectClass MiabLDAPmail:2
+
+attributetype ( 1.3.6.1.4.1.15347.2.102 
+	NAME 'transport' 
+	SUP name)
+
+attributetype ( 1.3.6.1.4.1.15347.2.101 
+	NAME 'mailRoutingAddress' 
+	SUP mail )
+
+attributetype ( 1.3.6.1.4.1.15347.2.110 NAME 'maildest'
+	DESC 'Restricted to send only to local network'
+	EQUALITY caseIgnoreMatch
+	SUBSTR caseIgnoreSubstringsMatch
+	SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{32768} )
+
+attributetype ( 1.3.6.1.4.1.15347.2.111 NAME 'mailaccess'
+	DESC 'Can be mailed to restricted groups'
+	EQUALITY caseIgnoreMatch
+	SUBSTR caseIgnoreSubstringsMatch
+	SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{32768} )
+
+attributetype ( 1.3.6.1.4.1.15347.2.100
+	NAME ( 'maildrop' )
+	DESC 'RFC1274: RFC822 Mailbox'
+	EQUALITY caseIgnoreMatch
+	SUBSTR caseIgnoreSubstringsMatch
+	SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 )
+
+attributetype ( 1.3.6.1.4.1.10018.1.1.1 NAME 'mailbox'
+	DESC 'The absolute path to the mailbox for a mail account in a non-default location'
+	EQUALITY caseExactMatch
+	SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE )
+
+# create a mailMember for utf8 email addresses in mailGroups
+attributetype ( MiabLDAPmailAttributeType:1 NAME 'mailMember' DESC 'RFC6532 utf8 email address of group member(s)' EQUALITY caseIgnoreMatch SUBSTR caseIgnoreSubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 )
+
+# create a utf8 version of core 'domainComponent'
+attributetype ( MiabLDAPmailAttributeType:2 NAME 'dcIntl' DESC 'UTF8 domain component' EQUALITY caseIgnoreMatch SUBSTR caseIgnoreSubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE )
+
+objectclass ( 1.3.6.1.4.1.15347.2.1
+	NAME 'mailUser'
+	DESC 'E-Mail User'
+	SUP top
+	AUXILIARY
+	MUST ( uid $ mail $ maildrop )
+	MAY ( cn $ mailbox $ maildest $ mailaccess )
+	)
+
+objectclass ( 1.3.6.1.4.1.15347.2.2
+	NAME 'mailGroup'
+	DESC 'E-Mail Group'
+	SUP top
+	STRUCTURAL
+	MUST ( cn $ mail )
+	MAY ( mailRoutingAddress $ member $ mailMember $ description )
+	)
+
+objectclass ( 1.3.6.1.4.1.15347.2.3
+	NAME 'transportTable'
+	DESC 'MTA Transport Table'
+	SUP top
+	STRUCTURAL
+	MUST ( cn $ transport )
+	)
+
+# create an auxiliary class to attach to 'domain' objects
+objectClass ( MiabLDAPmailObjectClass:1 NAME 'mailDomain' DESC 'Domain we handle mail for' SUP top AUXILIARY MUST ( dcIntl ) )

management/auth.py

@@ -73,14 +73,9 @@ class AuthService:
 			return (None, ["admin"])
 
 		# If the password corresponds with a session token for the user, grant access for that user.
-		if password in self.sessions and self.sessions[password]["email"] == username and not login_only:
+		if self.get_session(username, password, "login", env) and not login_only:
 			sessionid = password
 			session = self.sessions[sessionid]
-			if session["password_token"] != self.create_user_password_state_token(username, env):
-				# This session is invalid because the user's password/MFA state changed
-				# after the session was created.
-				del self.sessions[sessionid]
-				raise ValueError("Session expired.")
 			if logout:
 				# Clear the session.
 				del self.sessions[sessionid]
@@ -144,5 +139,14 @@ class AuthService:
 		self.sessions[token] = {
 			"email": username,
 			"password_token": self.create_user_password_state_token(username, env),
+			"type": type,
 		}
 		return token
+
+	def get_session(self, user_email, session_key, session_type, env):
+		if session_key not in self.sessions: return None
+		session = self.sessions[session_key]
+		if session_type == "login" and session["email"] != user_email: return None
+		if session["type"] != session_type: return None
+		if session["password_token"] != self.create_user_password_state_token(session["email"], env): return None
+		return session

management/daemon.py

@@ -244,7 +244,7 @@ def mail_aliases():
 	if request.args.get("format", "") == "json":
 		return json_response(get_mail_aliases_ex(env))
 	else:
-		return "".join(address+"\t"+receivers+"\t"+(senders or "")+"\n" for address, receivers, senders in get_mail_aliases(env))
+		return "".join(address+"\t"+receivers+"\t"+(senders or "")+"\n" for address, receivers, senders, auto in get_mail_aliases(env))
 
 @app.route('/mail/aliases/add', methods=['POST'])
 @authorized_personnel_only
@@ -691,16 +691,42 @@ def postgrey_whitelist_handler():
 # MUNIN
 
 @app.route('/munin/')
-@app.route('/munin/<path:filename>')
 @authorized_personnel_only
-def munin(filename=""):
+def munin_start():
-	# Checks administrative access (@authorized_personnel_only) and then just proxies
+	# Munin pages, static images, and dynamically generated images are served
-	# the request to static files.
+	# outside of the AJAX API. We'll start with a 'start' API that sets a cookie
+	# that subsequent requests will read for authorization. (We don't use cookies
+	# for the API to avoid CSRF vulnerabilities.)
+	response = make_response("OK")
+	response.set_cookie("session", auth_service.create_session_key(request.user_email, env, type='cookie'),
+	    max_age=60*30, secure=True, httponly=True, samesite="Strict") # 30 minute duration
+	return response
+
+def check_request_cookie_for_admin_access():
+	session = auth_service.get_session(None, request.cookies.get("session", ""), "cookie", env)
+	if not session: return False
+	privs = get_mail_user_privileges(session["email"], env)
+	if not isinstance(privs, list): return False
+	if "admin" not in privs: return False
+	return True
+
+def authorized_personnel_only_via_cookie(f):
+	@wraps(f)
+	def g(*args, **kwargs):
+		if not check_request_cookie_for_admin_access():
+			return Response("Unauthorized", status=403, mimetype='text/plain', headers={})
+		return f(*args, **kwargs)
+	return g
+
+@app.route('/munin/<path:filename>')
+@authorized_personnel_only_via_cookie
+def munin_static_file(filename=""):
+	# Proxy the request to static files.
 	if filename == "": filename = "index.html"
 	return send_from_directory("/var/cache/munin/www", filename)
 
 @app.route('/munin/cgi-graph/<path:filename>')
-@authorized_personnel_only
+@authorized_personnel_only_via_cookie
 def munin_cgi(filename):
 	""" Relay munin cgi dynazoom requests
 	/usr/lib/munin/cgi/munin-cgi-graph is a perl cgi script in the munin package

management/dns_update.py

@@ -604,7 +604,7 @@ def get_dns_zonefile(zone, env):
 
 def write_nsd_conf(zonefiles, additional_records, env):
 	# Write the list of zones to a configuration file.
-	nsd_conf_file = "/etc/nsd/zones.conf"
+	nsd_conf_file = "/etc/nsd/nsd.conf.d/zones.conf"
 	nsdconf = ""
 
 	# Append the zones.

management/mail_log.py

@@ -114,8 +114,8 @@ def scan_mail_log(env):
 
     try:
         import mailconfig
-        users = mailconfig.get_mail_users(env, as_map=True)
+        users = mailconfig.get_mail_users(env, as_map=True, map_by="mail")
-        aliases = mailconfig.get_mail_aliases(env, as_map=True)
+        aliases = mailconfig.get_mail_aliases(env, as_map=True, map_by="mail")
         collector["known_addresses"] = (set(users.keys()) |
                                         set(aliases.keys()))
     except ImportError:

management/status_checks.py

@@ -476,7 +476,7 @@ def check_primary_hostname_dns(domain, env, output, dns_domains, dns_zonefiles):
 	check_alias_exists("Hostmaster contact address", "hostmaster@" + domain, env, output)
 
 def check_alias_exists(alias_name, alias, env, output):
-	mail_aliases = get_mail_aliases(env, as_map=True)
+	mail_aliases = get_mail_aliases(env, as_map=True, map_by="mail")
 	if alias in mail_aliases:
 		if mail_aliases[alias]["forward_tos"]:
 			output.print_ok("%s exists as a mail alias. [%s ↦ %s]" % (alias_name, alias, ",".join(mail_aliases[alias]["forward_tos"])))

management/templates/aliases.html

@@ -1,6 +1,6 @@
 <style>
 #alias_table .actions > * { padding-right: 3px; }
-#alias_table .alias-required .remove { display: none }
+#alias_table .alias-auto .actions > * { display: none }
 </style>
 
 <h2>Aliases</h2>
@@ -174,7 +174,7 @@ function show_aliases() {
           var n = $("#alias-template").clone();
           n.attr('id', '');
 
-          if (alias.required) n.addClass('alias-required');
+          if (alias.auto) n.addClass('alias-auto');
           n.attr('data-address', alias.address_display); // this is decoded from IDNA, but will get re-coded to IDNA on the backend
           n.find('td.address').text(alias.address_display)
           for (var j = 0; j < alias.forwards_to.length; j++)

management/templates/index.html

@@ -124,7 +124,7 @@
                 <li class="dropdown-header">Advanced Pages</li>
                 <li><a href="#custom_dns" onclick="return show_panel(this);">Custom DNS</a></li>
                 <li><a href="#external_dns" onclick="return show_panel(this);">External DNS</a></li>
-                <li><a href="/admin/munin" target="_blank">Munin Monitoring</a></li>
+                <li><a href="#munin" onclick="return show_panel(this);">Munin Monitoring</a></li>
                 <li><a href="#postgrey_whitelist" onclick="return show_panel(this);">Postgrey Whitelist</a></li>
               </ul>
             </li>
@@ -208,6 +208,10 @@
       {% include "ssl.html" %}
       </div>
 
+      <div id="panel_munin" class="admin_panel">
+      {% include "munin.html" %}
+      </div>
+
       <hr>
 
       <footer>

management/templates/munin.html

@@ -0,0 +1,20 @@
+<h2>Munin Monitoring</h2>
+
+<style>
+</style>
+
+<p>Opening munin in a new tab... You may need to allow pop-ups for this site.</p>
+
+<script>
+function show_munin() {
+  // Set the cookie.
+  api(
+    "/munin",
+    "GET",
+    { },
+    function(r) {
+      // Redirect.
+      window.open("/admin/munin/index.html", "_blank");
+    });
+}
+</script>

setup/dns.sh

@@ -80,7 +80,13 @@ remote-control:
 EOF
 fi
 
-echo "include: /etc/nsd/zones.conf" >> /etc/nsd/nsd.conf;
+# Create a directory for additional configuration directives, including
+# the zones.conf file written out by our management daemon.
+echo "include: /etc/nsd/nsd.conf.d/*.conf" >> /etc/nsd/nsd.conf;
+
+# Remove the old location of zones.conf that we generate. It will
+# now be stored in /etc/nsd/nsd.conf.d.
+rm -f /etc/nsd/zones.conf
 
 # Create DNSSEC signing keys.
 

setup/functions-ldap.sh

@@ -13,8 +13,13 @@ get_attribute_from_ldif() {
 	local line
 	while read line; do
 		[ -z "$line" ] && break
-		local v=$(awk "/^$attr:/ { print substr(\$0, length(\"$attr\")+3) }" <<<$line)
+		local v=$(awk "/^$attr: / { print substr(\$0, length(\"$attr\")+3) }" <<<"$line")
-		[ ! -z "$v" ] && ATTR_VALUE+=( "$v" )
+		if [ ! -z "$v" ]; then
+			ATTR_VALUE+=( "$v" )
+		else
+			v=$(awk "/^$attr:: / { print substr(\$0, length(\"$attr\")+4) }" <<<"$line")
+			[ ! -z "$v" ] && ATTR_VALUE+=( $(base64 --decode --wrap=0 <<<"$v") )
+		fi
 	done <<< "$ldif"
 	return 0
 }

setup/ldap.sh

@@ -295,8 +295,8 @@ schema_to_ldif() {
 	local cn="$3"	   # schema common name, eg "postfix"
 	local cat='cat'
 	if [ ! -e "$schema" ]; then
-		if [ -e "conf/$(basename $schema)" ]; then
+		if [ -e "conf/schema/$(basename $schema)" ]; then
-			schema="conf/$(basename $schema)"
+			schema="conf/schema/$(basename $schema)"
 		else
 			cat="curl -s"
 		fi
@@ -316,56 +316,57 @@ EOF
 		| sed 's/^\s*$/#/g' >> "$ldif"
 }
 
+change_core_mail_syntax() {
+	# output the ldif to change mail to utf8 from ia5 in the core schema
+	get_attribute "cn=schema,cn=config" "(cn={0}core)" "olcAttributeTypes"
+	case "${ATTR_VALUE[48]}" in
+		*\'mail\'*caseIgnoreIA5Match* )
+			newval=$(sed 's/SYNTAX[ \t][^ ]*/SYNTAX 1.3.6.1.4.1.1466.115.121.1.15/g' <<<"${ATTR_VALUE[48]}" |
+					 sed 's/caseIgnoreIA5Match/caseIgnoreMatch/g' |
+					 sed 's/caseIgnoreIA5SubstringsMatch/caseIgnoreSubstringsMatch/g')
+			cat <<EOF
+dn: cn={0}core,cn=schema,cn=config
+changetype: modify
+delete: olcAttributeTypes
+olcAttributeTypes: ${ATTR_VALUE[48]}
+-
+add: olcAttributeTypes
+olcAttributeTypes: $newval
+EOF
+			;;
+	esac
+}
+
 
 add_schemas() {
-	# Add necessary schema's for MiaB operaion
+	# Add necessary schema's for MiaB operaion		 
 	#
-	# First, apply rfc822MailMember from OpenLDAP's "misc"
+	# Note: the postfix schema originally came from the ldapadmin
-	# schema. Don't apply the whole schema file because much is from
+	# project (GPL)(*), but has been modified to support the needs of
-	# expired RFC's, and we just need rfc822MailMember
+	# this project.
-	local cn="misc"
-	get_attribute "cn=schema,cn=config" "(&(cn={*}$cn)(objectClass=olcSchemaConfig))" "cn"
-	if [ -z "$ATTR_DN" ]; then
-		say_verbose "Adding '$cn' schema"
-		cat "/etc/ldap/schema/misc.ldif" | awk 'BEGIN {C=0}
-/^(dn|objectClass|cn):/ { print $0; next }
-/^olcAttributeTypes:/ && /27\.2\.1\.15/ { print $0; C=1; next }
-/^(olcAttributeTypes|olcObjectClasses):/ { C=0; next }
-/^ / && C==1 { print $0 }' | ldapadd -Q -Y EXTERNAL -H ldapi:/// >/dev/null
-	fi
-	
-	# Next, apply the postfix schema from the ldapadmin project
-	# (GPL)(*).
 	#	see: http://ldapadmin.org
 	#		 http://ldapadmin.org/docs/postfix.schema
 	#		 http://www.postfix.org/LDAP_README.html
-	# (*) mailGroup modified to include rfc822MailMember
+	for cn in "postfix" "mfa-totp" "namedProperties"; do
-	local schema="http://ldapadmin.org/docs/postfix.schema"
+		schema="$cn.schema"
-	local cn="postfix"
+		get_attribute "cn=schema,cn=config" "(&(cn={*}$cn)(objectClass=olcSchemaConfig))" "cn"
-	get_attribute "cn=schema,cn=config" "(&(cn={*}$cn)(objectClass=olcSchemaConfig))" "cn"
+		if [ -z "$ATTR_DN" ]; then
-	if [ -z "$ATTR_DN" ]; then
+			local ldif="/tmp/$cn.$$.ldif"
-		local ldif="/tmp/$cn.$$.ldif"
+			schema_to_ldif "$schema" "$ldif" "$cn"
-		schema_to_ldif "$schema" "$ldif" "$cn"
+			if [ "$cn" = "postfix" ]; then
-		sed -i 's/\$ member \$/$ member $ rfc822MailMember $/' "$ldif"
+				local ldif2="/tmp/$cn.$$-2.ldif"
-		say_verbose "Adding '$cn' schema"
+				change_core_mail_syntax >"$ldif2"
-		[ ${verbose:-0} -gt 1 ] && cat "$ldif"
+				echo "" >>"$ldif2"
-		ldapadd -Q -Y EXTERNAL -H ldapi:/// -f "$ldif" >/dev/null
+				sed 's/^dn: cn=postfix,\(.*\)$/dn: cn=postfix,\1\nchangetype: add/g' "$ldif" >> "$ldif2"
-		rm -f "$ldif"
+				rm "$ldif"
-	fi
+				ldif="$ldif2"
-
+			fi
-	# apply the mfa-totp schema
+			say_verbose "Adding '$cn' schema"
-	# this adds the totpUser class to store the totp secret
+			[ ${verbose:-0} -gt 1 ] && cat "$ldif"
-	local schema="mfa-totp.schema"
+			ldapadd -Q -Y EXTERNAL -H ldapi:/// -f "$ldif" >/dev/null
-	local cn="mfa-totp"
+			rm -f "$ldif"
-	get_attribute "cn=schema,cn=config" "(&(cn={*}$cn)(objectClass=olcSchemaConfig))" "cn"
+		fi
-	if [ -z "$ATTR_DN" ]; then
+	done	
-		local ldif="/tmp/$cn.$$.ldif"
-		schema_to_ldif "$schema" "$ldif" "$cn"
-		say_verbose "Adding '$cn' schema"
-		[ ${verbose:-0} -gt 1 ] && cat "$ldif"
-		ldapadd -Q -Y EXTERNAL -H ldapi:/// -f "$ldif" >/dev/null
-		rm -f "$ldif"
-	fi
 }
 
 
@@ -504,7 +505,7 @@ add_indexes() {
 	# Add the indexes
 	get_attribute "$cdn" "(objectClass=*)" "olcDbIndex" base
 	local attr
-	for attr in mail maildrop mailaccess dc rfc822MailMember; do
+	for attr in mail maildrop mailaccess dc dcIntl mailMember; do
 		local type="eq" atype="" aindex=""
 		[ "$attr" == "mail" ] && type="eq,sub"
 
@@ -661,6 +662,7 @@ process_cmdline() {
 	elif [ "$1" == "-config" ]; then
 		# Apply a certain configuration
 		if [ "$2" == "server" ]; then
+			add_schemas
 			modify_global_config
 			add_overlays
 			add_indexes
@@ -675,8 +677,23 @@ process_cmdline() {
 
 	elif [ "$1" == "-search" ]; then
 		# search for email addresses, distinguished names and general
-		# ldap filters
+		# ldap filters. Args:
-		debug_search "$2"
+		#    search filter [optional]
+		#    base dn [optional]
+		#    remaining args are attributes to output [optional]
+		shift
+		if [ $# -eq 0 ]; then
+			debug_search "(objectclass=*)"
+		else
+			debug_search "$@"
+		fi
+		exit 0
+
+	elif [ "$1" == "-schema-to-ldif" ]; then
+		cn="$2"
+		output="${3:-/dev/stdout}"
+		schema="$cn.schema"
+		schema_to_ldif "$schema" "$output" "$cn"
 		exit 0
 
 	elif [ "$1" == "-dumpdb" ]; then
@@ -699,6 +716,11 @@ process_cmdline() {
 			echo ""
 			slapcat ${slapcat_args[@]} -s "$ATTR_DN" | grep -Ev "^$hide_attrs:"
 		fi
+		if [ "$s" == "all" -o "$s" == "schema" ]; then
+			echo ""
+			echo '--------------------------------'
+			slapcat ${slapcat_args[@]} -s "cn=schema,cn=config" | grep -Ev "^$hide_attrs:"
+		fi
 		if [ "$s" == "all" -o "$s" == "frontend" ]; then
 			echo ""
 			echo '--------------------------------'
@@ -716,7 +738,7 @@ process_cmdline() {
 		if [ "$s" == "aliases" ]; then
 			echo ""
 			echo '--------------------------------'
-			local attrs=(mail member mailRoutingAddress rfc822MailMember)
+			local attrs=(mail member mailRoutingAddress mailMember namedProperty)
 			[ ${verbose:-0} -gt 0 ] && attrs=()
 			debug_search "(objectClass=mailGroup)" "$LDAP_ALIASES_BASE" ${attrs[@]}
 		fi

setup/mail-users.sh

@@ -63,7 +63,7 @@ base = ${LDAP_USERS_BASE}
 #   filter below. If found, the user is authenticated against this dn
 #   (a bind is attempted as that user). The attribute 'mail' is
 #   multi-valued and contains all the user's email addresses. We use
-#   maildrop as the dovecot mailbox address and forbid then from using
+#   maildrop as the dovecot mailbox address and forbid them from using
 #   it for authentication by excluding maildrop from the filter.
 pass_filter = (&(objectClass=mailUser)(mail=%u))
 pass_attrs = maildrop=user
@@ -166,6 +166,7 @@ chmod 0640 /etc/postfix/sender-login-maps-aliases.cf
 # Check whether a destination email address exists, and to perform any
 # email alias rewrites in Postfix.
 tools/editconf.py /etc/postfix/main.cf \
+	smtputf8_enable=no \
 	virtual_mailbox_domains=ldap:/etc/postfix/virtual-mailbox-domains.cf \
 	virtual_mailbox_maps=ldap:/etc/postfix/virtual-mailbox-maps.cf \
 	virtual_alias_maps=ldap:/etc/postfix/virtual-alias-maps.cf \
@@ -180,7 +181,7 @@ bind_dn = ${LDAP_POSTFIX_DN}
 bind_pw = ${LDAP_POSTFIX_PASSWORD}
 version = 3
 search_base = ${LDAP_DOMAINS_BASE}
-query_filter = (&(dc=%s)(businessCategory=mail))
+query_filter = (&(|(dc=%s)(dcIntl=%s))(businessCategory=mail))
 result_attribute = dc
 EOF
 chgrp postfix /etc/postfix/virtual-mailbox-domains.cf
@@ -228,7 +229,6 @@ chmod 0640 /etc/postfix/virtual-mailbox-maps.cf
 # it might have just permitted_senders, skip any records with an
 # empty destination here so that other lower priority rules might match.
 
-
 #
 # This is the ldap version of aliases(5) but for virtual
 # addresses. Postfix queries this recursively to determine delivery
@@ -242,7 +242,7 @@ bind_pw = ${LDAP_POSTFIX_PASSWORD}
 version = 3
 search_base = ${LDAP_USERS_BASE}
 query_filter = (mail=%s)
-result_attribute = maildrop, rfc822MailMember
+result_attribute = maildrop, mailMember
 special_result_attribute = member
 EOF
 chgrp postfix /etc/postfix/virtual-alias-maps.cf

setup/migrate.py

@@ -187,6 +187,11 @@ def migration_13(env):
 	db = os.path.join(env["STORAGE_ROOT"], 'mail/users.sqlite')
 	shell("check_call", ["sqlite3", db, "CREATE TABLE mfa (id INTEGER PRIMARY KEY AUTOINCREMENT, user_id INTEGER NOT NULL, type TEXT NOT NULL, secret TEXT NOT NULL, mru_token TEXT, label TEXT, FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE);"])
 
+def migration_14(env):
+	# Add the "auto_aliases" table.
+	db = os.path.join(env["STORAGE_ROOT"], 'mail/users.sqlite')
+	shell("check_call", ["sqlite3", db, "CREATE TABLE auto_aliases (id INTEGER PRIMARY KEY AUTOINCREMENT, source TEXT NOT NULL UNIQUE, destination TEXT NOT NULL, permitted_senders TEXT);"])
+
 ###########################################################
 
 
@@ -245,9 +250,77 @@ def migration_miabldap_1(env):
 	aliases=m13.create_aliases(env, conn, ldap, ldap_aliases_base)
 	permitted=m13.create_permitted_senders(conn, ldap, ldap_users_base, ldap_permitted_senders_base)
 	m13.populate_aliases(conn, ldap, users, aliases)
-		
+
 	ldap.unbind()
 	conn.close()
+
+
+def migration_miabldap_2(env):
+	# This migration step changes the ldap schema to support utf8 email
+	#
+	# possible states at this point:
+	#   miabldap was installed and is being upgraded
+	#      -> old pre-utf8 schema present
+	#   a miab install was present and step 1 upgaded it to miabldap
+	#      -> new utf8 schema present
+	#
+	sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), "../management")))
+	import ldap3
+	from backend import connect
+	import migration_14 as m14
+
+	# 1. get ldap site details
+	ldapvars = load_env_vars_from_file(os.path.join(env["STORAGE_ROOT"], "ldap/miab_ldap.conf"), strip_quotes=True)
+	ldap_domains_base = ldapvars.LDAP_DOMAINS_BASE
+	ldap_aliases_base = ldapvars.LDAP_ALIASES_BASE
+	ldap_users_base = ldapvars.LDAP_USERS_BASE
+
+	# connect before schema changes to ensure admin password works
+	ldap = connect(ldapvars)
+
+	# 2. if this is a miab -> maibldap install, the new schema is
+	# already in place and no schema changes are needed. however,
+	# if this is a miabldap/1 to miabldap/2 migration, we must
+	# upgrade the schema.
+	ret = shell("check_output", [
+		"ldapsearch",
+		"-Q",
+		"-Y", "EXTERNAL",
+		"-H", "ldapi:///",
+		"(&(objectClass=olcSchemaConfig)(cn={*}postfix))",
+		"-b", "cn=schema,cn=config",
+		"-LLL",
+		"olcObjectClasses"
+	])
+	
+	if "rfc822MailMember" in ret:
+		def ldif_change_fn(ldif):
+			return ldif.replace("rfc822MailMember: ", "mailMember: ")
+		# apply schema changes miabldap/1 -> miabldap/2
+		ldap.unbind()
+		print("Apply schema changes")
+		m14.apply_schema_changes(env, ldapvars, ldif_change_fn)
+		# reconnect
+		ldap = connect(ldapvars)
+
+	# 3. migrate to utf8: users, aliases and domains
+	print("Create utf8 entries for users and aliases having IDNA domains")
+	m14.add_utf8_mail_addresses(env, ldap, ldap_users_base)
+
+	print("Add namedProperties objectclass to aliases")
+	m14.add_namedProperties_objectclass(env, ldap, ldap_aliases_base)
+
+	print("Add mailDomain objectclass to domains")
+	m14.add_mailDomain_objectclass(env, ldap, ldap_domains_base)
+
+	print("Mark required aliases with 'auto' property")
+	m14.add_auto_tag(env, ldap, ldap_aliases_base)
+
+	print("Ensure all required aliases are created")
+	m14.ensure_required_aliases(env, ldapvars, ldap)
+	
+	ldap.unbind()
+
 	
 def get_current_migration():
 	ver = 0
@@ -327,11 +400,20 @@ def run_miabldap_migrations():
 	env = load_environment()
 
 	migration_id_file = os.path.join(env['STORAGE_ROOT'], 'mailinabox-ldap.version')
-	migration_id = 0
+	migration_id = None
 	if os.path.exists(migration_id_file):
 		with open(migration_id_file) as f:
 			migration_id = f.read().strip();
+		if migration_id.strip()=='': migration_id = None
 
+	if migration_id is None:
+		if os.path.exists(os.path.join(env['STORAGE_ROOT'], 'mailinabox.version')):
+			migration_id = 0
+		else:
+			print()
+			print("%s file doesn't exists. Skipping migration..." % (migration_id_file,))
+			return
+	
 	ourver = int(migration_id)
 
 	while True:

setup/migration_14.py

@@ -0,0 +1,232 @@
+#!/usr/bin/python3
+# -*- indent-tabs-mode: t; tab-width: 4; python-indent-offset: 4; -*-
+
+#
+# helper functions for migration #14 / miabldap-migration #2
+#
+
+import sys, os, ldap3, idna
+from utils import shell
+from mailconfig import (
+	add_required_aliases,
+	required_alias_names,
+	get_mail_domains
+)
+
+
+def utf8_from_idna(domain_idna):
+	try:
+		return idna.decode(domain_idna.encode("ascii"))
+	except (UnicodeError, idna.IDNAError):
+		# Failed to decode IDNA, should never happen
+		return domain_idna
+
+
+def apply_schema_changes(env, ldapvars, ldif_change_fn):
+	# 1. save LDAP_BASE data to ldif
+	slapd_conf = os.path.join(env["STORAGE_ROOT"], "ldap/slapd.d")
+	fail_fn = os.path.join(env["STORAGE_ROOT"], "ldap/failed_migration.txt")
+	ldif = shell("check_output", [
+		"/usr/sbin/slapcat",
+		"-F", slapd_conf,
+		"-b", ldapvars.LDAP_BASE
+	])
+
+	# 2. wipe out existing database configuration and database
+	#    2a. set the creation parameters
+	ORGANIZATION="Mail-In-A-Box"
+	LDAP_DOMAIN="mailinabox"
+	shell("check_output", [
+		"/usr/bin/debconf-set-selections"
+	], input=f'''slapd shared/organization string {ORGANIZATION}
+slapd slapd/domain string {LDAP_DOMAIN}
+slapd slapd/password1 password {ldapvars.LDAP_ADMIN_PASSWORD}
+slapd slapd/password2 password {ldapvars.LDAP_ADMIN_PASSWORD}
+'''.encode('utf-8')
+	)
+
+	#    2b. recreate ldap config and database
+	shell("check_call", [
+		"/usr/sbin/dpkg-reconfigure",
+		"--frontend=noninteractive",
+		"slapd"
+	])
+
+	#    2c. clear passwords from debconf
+	shell("check_output", [
+		"/usr/bin/debconf-set-selections"
+	], input='''slapd slapd/password1 password
+slapd slapd/password2 password
+'''.encode('utf-8')
+	)
+
+	# 3. make desired ldif changes
+	#   3a. first, remove dc=mailinabox and
+	#       cn=admin,dc=mailinabox. they were both created during
+	#       dpkg-reconfigure and can't be readded
+	entries = ldif.split("\n\n")
+	keep = []
+	removed = []
+	remove = [
+		"dn: " + ldapvars.LDAP_BASE,
+		"dn: " + ldapvars.LDAP_ADMIN_DN
+	]
+	for entry in entries:
+		dn = entry.split("\n")[0]
+		if dn not in remove:
+			keep.append(entry)
+		else:
+			removed.append(entry)
+			
+	#   3b. call the given ldif change function
+	ldif = ldif_change_fn("\n\n".join(keep))
+	#ldif = ldif_change_fn(ldif)
+	
+	# 4. re-create schemas and other config
+	shell("check_call", [
+		"setup/ldap.sh",
+		"-v",
+		"-config", "server"
+	])
+
+	# 5. restore LDAP_BASE data
+	code, ret = shell("check_output", [
+		"/usr/sbin/slapadd",
+		"-F", slapd_conf,
+		"-b", ldapvars.LDAP_BASE,
+		"-v",
+		"-c"
+	], input=ldif.encode('utf-8'), trap=True, capture_stderr=True)
+	
+	if code != 0:
+		try:
+			with open(fail_fn, "w") as of:
+				of.write("# slapadd -F %s -b %s -v -c\n" %
+						 (slapd_conf, ldapvars.LDAP_BASE))
+				of.write(ldif)
+			print("See saved data in %s" % fail_fn)
+		except Exception:
+			pass
+		
+		raise ValueError("Could not restore data: exit code=%s: output=%s" % (code, ret))
+
+
+
+def add_utf8_mail_addresses(env, ldap, ldap_users_base):
+	# if the mail attribute of users or aliases is idna encoded, also
+	# add a utf8 version of the address to the mail attribute so the
+	# user or alias will be known by multiple addresses (idna and
+	# utf8)
+	pager = ldap.paged_search(ldap_users_base, "(|(objectClass=mailGroup)(objectClass=mailUser))", attributes=['mail'])
+	changes = []
+	for rec in pager:
+		mail_idna_lc = []
+		for addr in rec['mail']:
+			mail_idna_lc = addr.lower()
+		
+		changed = False
+		new_mail = []
+		for addr in rec['mail']:
+			new_mail.append(addr)
+			name = addr.split('@')[0]
+			domain = addr.split('@', 1)[1]
+			addr_utf8 = name + '@' + utf8_from_idna(domain)
+			addr_utf8_lc = addr_utf8.lower()
+			if addr_utf8 != addr and addr_utf8_lc not in mail_lc:
+				new_mail.append(addr_utf8)
+				print("Add '%s' for %s" % (addr_utf8, addr))
+				changed = True
+		if changed:
+			changes.append({"rec":rec, "mail":new_mail})
+
+	for change in changes:
+		ldap.modify_record(
+			change["rec"],
+			{ "mail": change["mail"] }
+		)
+
+
+
+def add_namedProperties_objectclass(env, ldap, ldap_aliases_base):
+	# ensure every alias has a namedProperties objectClass attached
+	pager = ldap.paged_search(ldap_aliases_base, "(&(objectClass=mailGroup)(!(objectClass=namedProperties)))", attributes=['objectClass'])
+	changes = []
+	for rec in pager:
+		newoc = rec['objectClass'].copy()
+		newoc.append('namedProperties')
+		changelist = {
+			'objectClass': newoc,
+		}
+		changes.append({'rec': rec, 'changelist': changelist})
+
+	for change in changes:
+		ldap.modify_record(change['rec'], change['changelist'])
+
+		
+def add_auto_tag(env, ldap, ldap_aliases_base):
+	# add namedProperty=auto to existing required aliases
+	# this step is needed to upgrade miabldap systems
+	name_q = [
+		"(mail=hostmaster@"+env['PRIMARY_HOSTNAME']+")"
+	]
+	for name in required_alias_names:
+		name_q.append("(mail=%s@*)" % name)
+		
+	q = [
+		"(objectClass=mailGroup)",
+		"(!(namedProperty=auto))",
+		"(|%s)" % "".join(name_q)
+	]
+	pager = ldap.paged_search(
+		ldap_aliases_base,
+		"(&%s)" % "".join(q),
+		attributes=['namedProperty']
+	)
+	changes = []
+	for rec in pager:
+		newval = rec["namedProperty"].copy()
+		newval.append("auto")
+		changes.append({"rec": rec, "namedProperty": newval})
+		
+	for change in changes:
+		ldap.modify_record(
+			change["rec"],
+			{"namedProperty": change["namedProperty"]}
+		)
+
+
+
+def add_mailDomain_objectclass(env, ldap, ldap_domains_base):
+	# ensure every domain has a mailDomain objectClass attached
+	pager = ldap.paged_search(ldap_domains_base, "(&(objectClass=domain)(!(objectClass=mailDomain)))", attributes=['objectClass', 'dc', 'dcIntl'])
+	changes = []
+	for rec in pager:
+		newoc = rec['objectClass'].copy()
+		newoc.append('mailDomain')
+		changelist = {
+			'objectClass': newoc,
+			'dcIntl': [ utf8_from_idna(rec['dc'][0]) ]
+		}
+		changes.append({'rec': rec, 'changelist': changelist})
+
+	for change in changes:
+		ldap.modify_record(change['rec'], change['changelist'])
+
+
+def ensure_required_aliases(env, ldapvars, ldap):
+	# ensure every domain has its required aliases
+	env_combined = env.copy()
+	env_combined.update(ldapvars)
+	errors = []
+	for domain_idna in get_mail_domains(ldapvars):
+		results = add_required_aliases(env_combined, ldap, domain_idna)
+		for result in results:
+			if isinstance(result, str):
+				print(result)
+			else:
+				print("Error: %s" % result[0])
+				errors.append(result[0])
+	if len(errors)>0:
+		raise ValueError("Some required aliases could not be added")
+	
+

tests/lib/installed-state.sh

@@ -13,6 +13,7 @@ installed_state_capture() {
     # TOOD: tls certificates: expected CN's
 
     local state_dir="$1"
+    local install_dir="${2:-.}"
     local info="$state_dir/info.txt"
 
     H1 "Capture installed state to $state_dir"
@@ -22,11 +23,18 @@ installed_state_capture() {
     mkdir -p "$state_dir"
 
     # create info.json
+    if ! pushd "$install_dir" >/dev/null; then
+        echo "Directory '$install_dir' no accessible"
+        return 1
+    fi
     H2 "create info.txt"
     echo "STATE_VERSION=1" > "$info"
-    echo "GIT_VERSION='$(git describe --abbrev=0)'" >>"$info"
+    echo "GIT_VERSION='$(git describe)'" >>"$info"
+    git describe | awk -F- '{ split($1,a,"."); print "MAJOR="substr(a[1],2); print "MINOR="a[2]; print "RELEASE="$2 }' >>"$info"
     echo "GIT_ORIGIN='$(git remote -v | grep ^origin | grep 'fetch)$' | awk '{print $2}')'" >>"$info"
-    echo "MIGRATION_VERSION=$(cat "$STORAGE_ROOT/mailinabox.version")" >>"$info"
+    echo "MIGRATION_VERSION=$([ -e "$STORAGE_ROOT/mailinabox.version" ] && cat "$STORAGE_ROOT/mailinabox.version")" >>"$info"
+    echo "MIGRATION_ML_VERSION=$([ -e "$STORAGE_ROOT/mailinabox-ldap.version" ] && cat "$STORAGE_ROOT/mailinabox-ldap.version")" >>"$info"
+    popd >/dev/null
 
     # record users
     H2 "record users"
@@ -75,17 +83,21 @@ installed_state_compare() {
     #
     # determine compare type id (incorporating repo, branch, version, etc)
     #
-    local compare_type="all"
+    source "$s1/info.txt"
+    MAJOR_A="$MAJOR"
+    MINOR_A="$MINOR"
+    RELEASE_A="${RELEASE:-0}"
+    PROD_A="miab"
+    grep "mailinabox-ldap" <<<"$GIT_ORIGIN" >/dev/null && PROD_A="miabldap"
     
     source "$s2/info.txt"
-    if grep "mailinabox-ldap" <<<"$GIT_ORIGIN" >/dev/null; then
+    MAJOR_B="$MAJOR"
-        GIT_ORIGIN=""
+    MINOR_B="$MINOR"
-        source "$s1/info.txt"
+    RELEASE_B="${RELEASE:-0}"
-        if ! grep "mailinabox-ldap.git" <<<"$GIT_ORIGIN" >/dev/null; then
+    PROD_B="miab"
-            compare_type="miab2miab-ldap"
+    grep "mailinabox-ldap" <<<"$GIT_ORIGIN" >/dev/null && PROD_B="miabldap"
-        fi
+
-    fi
+    cmptype="${PROD_A}2${PROD_B}"
-    echo "Compare type: $compare_type"
 
     #
     # filter data for compare type
@@ -95,7 +107,7 @@ installed_state_compare() {
     cp "$s2/users.json" "$s2/users-cmp.json" || changed="true"
     cp "$s2/aliases.json" "$s2/aliases-cmp.json" || changed="true"
     
-    if [ "$compare_type" == "miab2miab-ldap" ]
+    if [ "$cmptype" = "miab2miabldap" ]
     then
         # user display names is a feature added to MiaB-LDAP that is
         # not in MiaB
@@ -104,7 +116,19 @@ installed_state_compare() {
         # alias descriptions is a feature added to MiaB-LDAP that is
         # not in MiaB
         grep -v '"description":' "$s2/aliases.json" > "$s2/aliases-cmp.json" || changed="true"        
-    fi    
+    fi
+
+    # cmp: v0.54 to current
+    if [ "$cmptype" = "miabldap2miabldap" -a $MAJOR_A -eq 0 -a $MINOR_A -le 54 -a $RELEASE_A -eq 0 ]
+    then
+        # s1: convert aliases 'required' to 'auto' and resort
+        jq -c ".[] | .aliases | sort_by(.address) | .[] | {address:.address, forwards_to:.forwards_to, permitted_senders:.permitted_senders, auto:.required, description:.description}"  "$s1/aliases.json" > "$s1/aliases-cmp.json"
+        sed -i 's/\("address":"administrator@.*"auto":\)true/\1false/' "$s1/aliases-cmp.json"
+
+        # s2: re-sort aliases
+        jq -c ".[] | .aliases | sort_by(.address) | .[] | {address:.address, forwards_to:.forwards_to, permitted_senders:.permitted_senders, auto:.auto, description:.description}"  "$s2/aliases.json" > "$s2/aliases-cmp.json"
+    fi
+    
     
     #
     # users
@@ -122,6 +146,7 @@ installed_state_compare() {
     #
     # aliases
     #
+    
     H2 "Aliases"
     output="$(diff "$s1/aliases-cmp.json" "$s2/aliases-cmp.json" 2>&1)"
     if [ $? -ne 0 ]; then
@@ -154,36 +179,36 @@ installed_state_compare() {
     done
     echo "$count added"
 
-    H2 "DNS - zones changed"
+#     H2 "DNS - zones changed"
-    count=0
+#     count=0
-    for zone in $(cd "$s1/zones"; ls *.signed); do
+#     for zone in $(cd "$s1/zones"; ls *.signed); do
-        if [ -e "$s2/zones/$zone" ]; then
+#         if [ -e "$s2/zones/$zone" ]; then
-            # all the signatures change if we're using self-signed certs
+#             # all the signatures change if we're using self-signed certs
-            # ignore ttl changes
+#             # ignore ttl changes
-            local t1="/tmp/s1.$$.txt"
+#             local t1="/tmp/s1.$$.txt"
-            local t2="/tmp/s2.$$.txt"
+#             local t2="/tmp/s2.$$.txt"
-            awk '\
+#             awk '\
-$4 == "RRSIG" || $4 == "NSEC3" { next; } \
+# $4 == "RRSIG" || $4 == "NSEC3" { next; } \
-$4 == "SOA" { print $1" "$3" "$4" "$5" "$6" "$8" "$10" "$12; next } \
+# $4 == "SOA" { print $1" "$3" "$4" "$5" "$6" "$8" "$10" "$12; next } \
-{ for(i=1;i<=NF;i++) if (i!=2) printf("%s ",$i); print ""; }' \
+# { for(i=1;i<=NF;i++) if (i!=2) printf("%s ",$i); print ""; }' \
-                "$s1/zones/$zone" > "$t1"
+#                 "$s1/zones/$zone" > "$t1"
             
-            awk '\
+#             awk '\
-$4 == "RRSIG" || $4 == "NSEC3" { next; } \
+# $4 == "RRSIG" || $4 == "NSEC3" { next; } \
-$4 == "SOA" { print $1" "$3" "$4" "$5" "$6" "$8" "$10" "$12; next } \
+# $4 == "SOA" { print $1" "$3" "$4" "$5" "$6" "$8" "$10" "$12; next } \
-{ for(i=1;i<=NF;i++) if (i!=2) printf("%s ",$i); print ""; }' \
+# { for(i=1;i<=NF;i++) if (i!=2) printf("%s ",$i); print ""; }' \
-                "$s2/zones/$zone" > "$t2"
+#                 "$s2/zones/$zone" > "$t2"
             
-            output="$(diff "$t1" "$t2" 2>&1)"
+#             output="$(diff "$t1" "$t2" 2>&1)"
-            if [ $? -ne 0 ]; then
+#             if [ $? -ne 0 ]; then
-                echo "CHANGED zone: $zone"
+#                 echo "CHANGED zone: $zone"
-                echo "$output"
+#                 echo "$output"
-                changed="true"
+#                 changed="true"
-                let count+=1
+#                 let count+=1
-            fi
+#             fi
-        fi
+#         fi
-    done
+#     done
-    echo "$count zone files had differences"
+#    echo "$count zone files had differences"
 
     if $changed; then
         return 1

tests/suites/_init.sh

@@ -133,6 +133,7 @@ test_success() {
 test_failure() {
 	local why="$1"
 	[ -z "$TEST_OF" ] && return
+	record "** TEST_FAILURE: $why **"
 	TEST_STATE="FAILURE"
 	TEST_STATE_MSG+=( "$why" )
 }

tests/suites/_ldap-functions.sh

@@ -147,8 +147,8 @@ EOF
 	for member; do
 		case $member in
 			*@* )
-				echo "rfc822MailMember: $member" >>$TEST_OF
+				echo "mailMember: $member" >>$TEST_OF
-				echo "rfc822MailMember: $member" >>$of 2>>$TEST_OF
+				echo "mailMember: $member" >>$of 2>>$TEST_OF
 				;;
 			* )
 				echo "member: $member" >>$TEST_OF

tests/suites/mail-aliases.sh

@@ -134,7 +134,7 @@ test_shared_alias_delivery() {
 
 test_trial_nonlocal_alias_delivery() {
 	# verify that mail sent to an alias with a non-local address
-	# (rfc822MailMember) can be delivered
+	# (mailMember) can be delivered
 	test_start "trial-nonlocal-alias-delivery"
 	if skip_test remote-smtp; then
 		test_end

tests/suites/management-users.sh

@@ -136,6 +136,8 @@ test_intl_domains() {
 	# remote intl user / forward-to
 	local intl_person="hans@bücher.example"
 	local intl_person_idna="hans@xn--bcher-kva.example"
+	local intl_person_domain=$(email_domainpart "$intl_person")
+	local intl_person_idna_domain=$(email_domainpart "$intl_person_idna")
 
 	# local users
 	local bob="bob@somedomain.com"
@@ -149,10 +151,50 @@ test_intl_domains() {
 	if mgmt_create_user "$intl_person" "$bob_pw"; then
 		test_failure "A user account is not permitted to have an international domain"
 		# ensure user is removed as is expected by the remaining tests
-		mgmt_delele_user "$intl_person"
+		mgmt_delete_user "$intl_person"
 		delete_user "$intl_person"
 		delete_user "$intl_person_idna"
 	fi
+
+	# given an idna encoded user - the user should have 2 mail addresses
+	if ! mgmt_create_user "$intl_person_idna" "$bob_pw"; then
+		test_failure "Could not create idna-encoded user account $intl_person_idna"
+	else
+		get_attribute "$LDAP_USERS_BASE" "(mail=$intl_person_idna)" "mail"
+		if [ -z "$ATTR_DN" ] || \
+			   ! array_contains "$intl_person" "${ATTR_VALUE[@]}" || \
+			   ! array_contains "$intl_person_idna" "${ATTR_VALUE[@]}"
+		then
+			test_failure "Alias's ($intl_person) mail attribute expected to have both the idna and utf8 names, got ${#ATTR_VALUE[@]}: ${ATTR_VALUE[*]}, expected: $intl_person,$intl_person_idna"
+			[ ! -z "$ATTR_DN" ] && record_search "$ATTR_DN"
+		else
+			record_search "$ATTR_DN"
+			
+			# required aliases are automatically created and should
+			# have both mail addresses (idna and utf8)
+			get_attribute "$LDAP_ALIASES_BASE" "(mail=abuse@$intl_person_idna_domain)" "mail"
+			if [ -z "$ATTR_DN" ]; then
+				test_failure "Required alias not created!"
+				debug_search "(objectClass=mailGroup)" >>$TEST_OF
+			elif  ! array_contains "abuse@$intl_person_domain" "${ATTR_VALUE[@]}" || \
+					! array_contains "abuse@$intl_person_idna_domain" "${ATTR_VALUE[@]}"
+			then
+				test_failure "Require alias abuse@$intl_person_idna_domain expected to contain both idna and utf8 mail addresses"
+				record_search "$ATTR_DN"
+			fi
+			
+			# ensure user is removed as is expected by the remaining tests
+			mgmt_delete_user "$intl_person_idna"
+		fi
+	fi
+
+	# at this point intl_person does not exist, so all required aliases
+	# should also not be present
+	get_attribute "$LDAP_ALIASES_BASE" "(mail=*@$intl_person_idna_domain)"
+	if [ ! -z "$ATTR_DN" ]; then
+		test_failure "No required alias should not exist for the $intl_person_domain domain"
+		record_search "$ATTR_DN"
+	fi
 	
 	# create local users bob and mary
 	mgmt_assert_create_user "$bob" "$bob_pw"
@@ -161,11 +203,27 @@ test_intl_domains() {
 	# create intl alias with local user bob and intl_person in it
 	if mgmt_assert_create_alias_group "$alias" "$bob" "$intl_person"; then
 		# examine LDAP server to verify IDNA-encodings
-		get_attribute "$LDAP_ALIASES_BASE" "(mail=$alias_idna)" "rfc822MailMember"
+		
+		# 1. the mail attribute for the alias should have both the
+		# idna and utf8 addresses
+		get_attribute "$LDAP_ALIASES_BASE" "(mail=$alias)" "mail"
+		if [ -z "$ATTR_DN" ] || \
+			   ! array_contains "$alias" "${ATTR_VALUE[@]}" || \
+			   ! array_contains "$alias_idna" "${ATTR_VALUE[@]}"
+		then
+			test_failure "Alias's ($alias) mail attribute expected to have both the idna and utf8 names, got: ${ATTR_VALUE[*]}, expected: $alias,$alias_idna"
+			[ ! -z "$ATTR_DN" ] && record_search "$ATTR_DN"
+		fi
+
+		record_search "$ATTR_DN"
+
+		# 2. the mailMember attribute for the alias should contain the
+		# idna encoded intl_person (who is external - not a system user)
+		get_attribute "$LDAP_ALIASES_BASE" "(mail=$alias_idna)" "mailMember"
 		if [ -z "$ATTR_DN" ]; then
 			test_failure "IDNA-encoded alias group not found! created as:$alias expected:$alias_idna"
 		elif [ "$ATTR_VALUE" != "$intl_person_idna" ]; then
-			test_failure "Alias group with user having an international domain was not ecoded properly. added as:$intl_person expected:$intl_person_idna"
+			test_failure "Alias group with user having an international domain was not encoded properly. added as:$intl_person expected:$intl_person_idna"
 		fi
 	fi
 

tests/system-setup/setup-defaults.sh

@@ -39,3 +39,7 @@ export NC_ADMIN_PASSWORD="${NC_ADMIN_PASSWORD:-Test_1234}"
 # For setup scripts that install upstream versions
 export MIAB_UPSTREAM_GIT="${MIAB_UPSTREAM_GIT:-https://github.com/mail-in-a-box/mailinabox.git}"
 export UPSTREAM_TAG="${UPSTREAM_TAG:-}"
+
+# For setup scripts that install miabldap releases
+export MIABLDAP_GIT="${MIABLDAP_GIT:-https://github.com/downtownallday/mailinabox-ldap.git}"
+export MIABLDAP_RELEASE_TAG="${MIABLDAP_RELEASE_TAG:-v0.54}"

tests/system-setup/setup-funcs.sh

@@ -224,6 +224,14 @@ miab_ldap_install() {
         die "Cannot install: the working directory is not MiaB-LDAP!"
     fi
 
+    # setup/questions.sh installs the email_validator python3 module
+    # but only when in interactive mode. make sure it's also installed
+    # in non-interactive mode
+    if [ ! -z "${NONINTERACTIVE:-}" ]; then
+        H2 "Install email_validator python3 module"
+        pip3 install -q "email_validator>=1.0.0" || die "Unable to install email_validator python3 module!"
+    fi
+
     # if EHDD_KEYFILE is set, use encryption-at-rest support
     if [ ! -z "$EHDD_KEYFILE" ]; then
         ehdd/start-encrypted.sh

tests/system-setup/upgrade-from-upstream.sh

@@ -96,13 +96,13 @@ upstream_install() {
         echo "$F_RESET"
         die "Upstream setup failed!"
     fi
-    popd >/dev/null
     
     workaround_dovecot_sieve_bug
 
     H2 "Upstream info"
     echo "Code version: $(git describe)"
     echo "Migration version: $(cat "$STORAGE_ROOT/mailinabox.version")"
+    popd >/dev/null
 }
 
 
@@ -154,9 +154,7 @@ else
     fi
 
     # capture upstream state
-    pushd "$upstream_dir" >/dev/null
+    installed_state_capture "/tmp/state/upstream" "$upstream_dir"
-    installed_state_capture "/tmp/state/upstream"
-    popd >/dev/null
 fi
 
 # install miab-ldap and capture state

tests/system-setup/upgrade.sh

@@ -0,0 +1,111 @@
+#!/bin/bash
+
+# setup MiaB-LDAP by:
+#    1. installing upstream MiaB
+#    2. adding some data (users/aliases/etc)
+#    3. upgrading to MiaB-LDAP
+#
+# See setup-defaults.sh for usernames and passwords.
+#
+
+
+usage() {
+    echo "Usage: $(basename "$0")"
+    echo "Install MiaB-LDAP after installing upstream MiaB"
+    exit 1
+}
+
+# ensure working directory
+if [ ! -d "tests/system-setup" ]; then
+    echo "This script must be run from the MiaB root directory"
+    exit 1
+fi
+
+# load helper scripts
+. "tests/lib/all.sh" "tests/lib" || die "Could not load lib scripts"
+. "tests/system-setup/setup-defaults.sh" || die "Could not load setup-defaults"
+. "tests/system-setup/setup-funcs.sh" || die "Could not load setup-funcs"
+
+# ensure running as root
+if [ "$EUID" != "0" ]; then
+    die "This script must be run as root (sudo)"
+fi
+
+
+init() {
+    H1 "INIT"
+    init_test_system
+    init_miab_testing || die "Initialization failed"
+}
+
+install_release() {
+    install_dir="$1"
+    H1 "INSTALL RELEASE $MIABLDAP_RELEASE_TAG"
+    [ ! -x /usr/bin/git ] && apt-get install -y -qq git
+    
+    if [ ! -d "$install_dir" ] || [ -z "$(ls -A "$install_dir")" ] ; then
+        H2 "Cloning $MIABLDAP_GIT"
+        rm -rf "$install_dir"
+        git clone "$MIABLDAP_GIT" "$install_dir"
+        if [ $? -ne 0 ]; then
+            rm -rf "$install_dir"
+            die "git clone failed!"
+        fi
+    fi
+
+    pushd "$install_dir" >/dev/null
+    H2 "Checkout $MIABLDAP_RELEASE_TAG"
+    git checkout "$MIABLDAP_RELEASE_TAG" || die "git checkout $MIABLDAP_RELEASE_TAG failed"
+    
+    H2 "Run setup"
+    if ! setup/start.sh; then
+        echo "$F_WARN"
+        dump_file /var/log/syslog 100
+        echo "$F_RESET"
+        die "Release $RELEASE_TAG setup failed!"
+    fi
+    
+    workaround_dovecot_sieve_bug
+
+    H2 "Release info"
+    echo "Code version: $(git describe)"
+    echo "Migration version (miabldap): $(cat "$STORAGE_ROOT/mailinabox-ldap.version")"
+    popd >/dev/null
+}
+
+
+# install basic stuff, set the hostname, time, etc
+init
+
+# install release
+release_dir="$HOME/miabldap_$MIABLDAP_RELEASE_TAG"
+install_release "$release_dir"
+. /etc/mailinabox.conf
+    
+# populate some data
+if [ $# -gt 0 ]; then
+    populate_by_name "$@"
+else
+    populate_by_name "basic" "totpuser"
+fi
+
+# capture release state
+installed_state_capture "/tmp/state/release" "$release_dir"
+
+# install master miab-ldap and capture state
+H2 "New miabldap"
+echo "git branch: $(git branch | grep '*')"
+miab_ldap_install
+installed_state_capture "/tmp/state/master"
+
+# compare states
+if ! installed_state_compare "/tmp/state/release" "/tmp/state/master"; then
+    dump_file "/tmp/state/release/info.txt"
+    dump_file "/tmp/state/master/info.txt"
+    die "Release $RELEASE_TAG and master states are different !"
+fi
+
+#
+# actual verification that mail sends/receives properly is done via
+# the test runner ...
+#

tests/vagrant/Vagrantfile

@@ -49,6 +49,23 @@ tests/runner.sh upgrade-basic upgrade-totpuser default || exit 2
 SH
   end
 
+  # upgrade
+  
+  # this test is only needed when testing migrations from miabldap
+  # to a newer miabldap with a migration step
+  #
+  # upgrade will handle testing upgrades of
+  # miabldap with or without a new migration step
+  config.vm.define "upgrade" do |m1|
+    m1.vm.provision :shell, :inline => <<-SH
+cd /mailinabox
+source tests/vagrant/globals.sh || exit 1
+export PRIMARY_HOSTNAME=upgrade.abc.com
+tests/system-setup/upgrade.sh basic totpuser || exit 1
+tests/runner.sh upgrade-basic upgrade-totpuser default || exit 2
+SH
+  end
+
   # unsetvars: because miab sets bash '-e' to fail any setup script
   # when a script command returns a non-zero exit code, and more
   # importantly '-u' which fails scripts when any unset variable is