Issue #1340 - LDAP backend for accounts

This commit will:

1. Change the user account database from sqlite to OpenLDAP
2. Add policyd-spf to postfix for SPF validation
3. Add a test runner with some automated test suites

Notes:

User account password hashes are preserved.

There is a new Roundcube contact list called "Directory" that lists the users in LDAP (MiaB users), similar to what Google Suite does.

Users can still change their password in Roundcube.

OpenLDAP is configured with TLS, but all remote access is blocked by firewall rules. Manual changes are required to open it for remote access (eg. "ufw allow proto tcp from <HOST> to any port ldaps").

The test runner is started by executing tests/runner.sh. Be aware that it will make changes to your system, including adding new users, domains, mailboxes, start/stop services, etc. It is highly unadvised to run it on a production system!

The LDAP schema that supports mail delivery with postfix and dovecot is located in conf/postfix.schema. This file is copied verbatim from the LdapAdmin project (GPL, ldapadmin.org). Instead of including the file in git, it could be referenced by URL and downloaded by the setup script if GPL is an issue or apply for a PEN from IANA.

Mangement console and other services should not appear or behave any differently than before.

management/auth.py

@@ -3,7 +3,7 @@ import base64, os, os.path, hmac
 from flask import make_response
 
 import utils
-from mailconfig import get_mail_password, get_mail_user_privileges
+from mailconfig import validate_login, get_mail_password, get_mail_user_privileges
 
 DEFAULT_KEY_PATH   = '/var/lib/mailinabox/api.key'
 DEFAULT_AUTH_REALM = 'Mail-in-a-Box Management Server'
@@ -96,19 +96,7 @@ class KeyAuthService:
 		else:
 			# Get the hashed password of the user. Raise a ValueError if the
 			# email address does not correspond to a user.
-			pw_hash = get_mail_password(email, env)
-
-			# Authenticate.
-			try:
-				# Use 'doveadm pw' to check credentials. doveadm will return
-				# a non-zero exit status if the credentials are no good,
-				# and check_call will raise an exception in that case.
-				utils.shell('check_call', [
-					"/usr/bin/doveadm", "pw",
-					"-p", pw,
-					"-t", pw_hash,
-					])
-			except:
+			if not validate_login(email, pw, env):
 				# Login failed.
 				raise ValueError("Invalid password.")
 
@@ -130,7 +118,7 @@ class KeyAuthService:
 		# a user's password is reset, the HMAC changes and they will correctly need to log
 		# in to the control panel again. This method raises a ValueError if the user does
 		# not exist, due to get_mail_password.
-		msg = b"AUTH:" + email.encode("utf8") + b" " + get_mail_password(email, env).encode("utf8")
+		msg = b"AUTH:" + email.encode("utf8") + b" " + ";".join(get_mail_password(email, env)).encode("utf8")
 		return hmac.new(self.key.encode('ascii'), msg, digestmod="sha256").hexdigest()
 
 	def _generate_key(self):

management/backend.py

@@ -0,0 +1,294 @@
+#!/usr/local/lib/mailinabox/env/bin/python
+# -*- indent-tabs-mode: t; tab-width: 4; python-indent-offset: 4; -*-
+
+import ldap3, time
+
+
+class Response:
+	#
+	# helper class for iterating over ldap search results
+	# example:
+	#    conn = connect()
+	#    response = conn.wait( conn.search(...) )
+	#    for record in response:
+	#        print(record['dn'])
+	#
+	rowidx=0
+	def __init__(self, result, response):
+		self.result = result
+		self.response = response
+		if result is None:
+			raise ValueError("INVALID RESULT: None")
+		
+	def __iter__(self):
+		self.rowidx=0
+		return self
+	
+	def __next__(self):
+		# return the next record in the result set
+		rtn = self.next()
+		if rtn is None:	raise StopIteration
+		return rtn
+	
+	def next(self):
+		# return the next record in the result set or None
+		if self.rowidx >= len(self.response):
+			return None
+		rtn=self.response[self.rowidx]['attributes']
+		rtn['dn'] = self.response[self.rowidx]['dn']
+		self.rowidx += 1
+		
+		return rtn
+	
+	def count(self):
+		# return the number of records in the result set
+		return len(self.response)
+		
+
+
+class PagedSearch:
+	#
+	# Helper class for iterating over and handling paged searches.
+	# Use PagedSearch when expecting a large number of matching
+	# entries. slapd by default limits each result set to 500 entries.
+	#
+	# example:
+	#  conn=connection()
+	#  response = conn.paged_search(...)
+	#  for record in response:
+	#     print(record['dn'])
+	#
+	# PagedSearch is limited to one iteration pass. In the above
+	# example the 'for' statement could not be repeated.
+	#
+
+	def __init__(self, conn, search_base, search_filter, search_scope, attributes, page_size):
+		self.conn = conn
+		self.search_base = search_base
+		self.search_filter = search_filter
+		self.search_scope = search_scope
+		self.attributes = attributes
+		self.page_size = page_size
+
+		# issue the search
+		self.response = None
+		self.id = conn.search(search_base, search_filter, search_scope, attributes=attributes, paged_size=page_size, paged_criticality=True)
+		self.page_count = 0
+		
+	def __iter__(self):
+		# wait for the search result on first iteration
+		self.response = self.conn.wait(self.id)
+		self.page_count += 1
+		return self
+
+	def __next__(self):
+		# return the next record in the result set
+		r = self.response.next()
+		if r is None:
+			cookie=self.response.result['controls']['1.2.840.113556.1.4.319']['value']['cookie']
+			if not cookie:
+				raise StopIteration
+			self.id = self.conn.search(self.search_base, self.search_filter, self.search_scope, attributes=self.attributes, paged_size=self.page_size, paged_cookie=cookie)
+			self.response = self.conn.wait(self.id)
+			self.page_count += 1
+			r = self.response.next()
+			if r is None:
+				raise StopIteration
+		return r
+
+	def abandon(self):
+		# "If you send 0 as paged_size and a valid cookie the search
+		# operation referred by that cookie is abandoned."
+		cookie=self.response.result['controls']['1.2.840.113556.1.4.319']['value']['cookie']
+		if not cookie: return
+		self.id = self.conn.search(self.search_base, self.search_filter, self.search_scope, attributes=self.attributes, paged_size=0, paged_cookie=cookie)
+		
+
+
+
+class LdapConnection(ldap3.Connection):
+	# This is a subclass ldap3.Connection with our own methods for
+	# simplifying searches and modifications
+	
+	def wait(self, id):
+		# Wait for results from an async search and return the result
+		# set in a Response object.  If a syncronous strategy is in
+		# use, it returns immediately with the Response object.
+		if type(id)==int:
+			# async
+			tup = self.get_response(id)
+			return Response(tup[1], tup[0])
+		else:
+			# sync - conn has returned results
+			return Response(self.result, self.response)
+
+	def paged_search(self, search_base, search_filter, search_scope=ldap3.SUBTREE, attributes=None, page_size=200):
+		# Perform a paged search - see PagedSearch above
+		return PagedSearch(self,
+				   search_base,
+				   search_filter,
+				   search_scope,
+				   attributes,
+				   page_size)
+
+	def add(self, dn, object_class=None, attrs=None, controls=None):
+		# This overrides ldap3's add method to automatically remove
+		# empty attributes from the attribute list for a ldap ADD
+		# operation, which cause exceptions.
+		if attrs is not None:
+			keys = [ k for k in attrs ]
+			for k in keys:
+				if attrs[k] is None or \
+				   type(attrs[k]) is list and len(attrs[k])==0 or \
+				   type(attrs[k]) is str and attrs[k]=='':
+					del attrs[k]
+		return ldap3.Connection.add(self, dn, object_class, attrs, controls)
+
+	def chase_members(self, members, attrib, env):
+		# Given a list of distinguished names (members), lookup each
+		# one and return the list of values for `attrib` in an
+		# array. It is *not* recursive. `attrib` is a string holding
+		# the name of an attribute.
+		resolved=[]
+		for dn in members:
+			try:
+				response = self.wait(self.search(dn, "(objectClass=*)", ldap3.BASE, attributes=attrib))
+				rec = response.next()
+				if rec is not None: resolved.append(rec[attrib])
+			except ldap3.core.exceptions.LDAPNoSuchObjectResult:
+				# ignore orphans
+				pass
+		return resolved
+
+	# static
+	def ldap_modify_op(attr, record, new_values):
+		# Return an ldap operation to change record[attr] so that it
+		# has values `new_values`. `new_values` is a list.
+		
+		if not type(new_values) is list:
+			new_values = [ new_values ]
+		# remove None values
+		new_values = [ v for v in new_values if v is not None ]
+
+		if len(record[attr]) == 0:
+			if len(new_values) == 0:
+				# NOP: current=empty, desired=empty
+				return None
+			# ADD: current=empty, desired=non-empty
+			return [(ldap3.MODIFY_ADD, new_values)]
+		if len(new_values) == 0:
+			# DELETE: current=non-empty, desired=empty
+			return [(ldap3.MODIFY_DELETE, [])]
+		# MODIFY: current=non-empty, desired=non-empty
+		return [(ldap3.MODIFY_REPLACE, new_values)]
+
+
+
+	def add_or_modify(self, dn, existing_record, attrs_to_update, objectClasses, values):
+		# Add or modify an existing database entry.
+		#   dn: dn for a new entry
+		#   existing_record: the existing data, if any
+		#         (a dict from Response.next()). The data must
+		#         have values for each attribute in `attrs_to_update`
+		#   attrs_to_update: an array of attribute names to update
+		#   objectClasses: a list of object classes for a new entry
+		#   values: a dict of attributes and values for a new entry
+		if existing_record:
+			# modify existing
+			changes = {}
+			dn = existing_record['dn']
+			for attr in attrs_to_update:
+				modify_op = LdapConnection.ldap_modify_op(
+					attr,
+					existing_record,
+					values[attr])
+				if modify_op: changes[attr] = modify_op
+			self.wait ( self.modify(dn, changes) )
+			return 'modify'
+		else:
+			# add new alias
+			self.wait ( self.add(dn, objectClasses, values) )
+			return 'add'
+
+		
+	def modify_record(self, rec, modifications):
+		# Modify an existing record by changing the attributes of
+		# `modifications` to their associated values. `modifications`
+		# is a dict with key of attribute name and value of list of
+		# string.
+		dn = rec['dn']
+		attrs_to_update=modifications.keys()
+		self.add_or_modify(dn, rec, attrs_to_update, None, modifications)
+		return True
+	
+
+def get_shadowLastChanged():
+	# get the number of days from the epoch
+	days = int(time.time() / (24 * 60 * 60))
+	return days
+
+
+def get_ldap_server(env):
+	# return a ldap3.Server object for a ldap connection
+	tls=None
+	if env.LDAP_SERVER_TLS=="yes":
+		import ssl
+		tls=ldap3.Tls(
+			validate=ssl.CERT_REQUIRED,
+			ca_certs_file="/etc/ssl/certs/ca-certificates.crt")
+
+	server=ldap3.Server(
+		host=env.LDAP_SERVER,
+		port=int(env.LDAP_SERVER_PORT),
+		use_ssl=False if tls is None else True,
+		get_info=ldap3.NONE,
+		tls=tls)
+	
+	return server
+
+
+def connect(env):
+	# connect to the ldap server
+	#
+	# problems and observations:
+	#
+	# using thread-local storage does not work with Flask.  nor does
+	# REUSABLE strategy - both fail with LDAPResponseTimeoutError
+	#
+	# there are a lot of connections left open with ASYNC strategy
+	#
+	# there are more unclosed connections with manual bind vs auto
+	# bind
+	#
+	# pooled connections only work for REUSABLE strategy
+	#
+	# paging does not work with REUSABLE strategy at all:
+	# get_response() always returns a protocol error (invalid cookie)
+	# when retrieving the second page
+	#
+	server = get_ldap_server(env)
+
+	auto_bind=ldap3.AUTO_BIND_NO_TLS
+	if env.LDAP_SERVER_STARTTLS=="yes":
+		auto_bind=ldap3.AUTO_BIND_TLS_BEFORE_BIND
+		
+	conn = LdapConnection(
+		server,
+		env.LDAP_MANAGEMENT_DN,
+		env.LDAP_MANAGEMENT_PASSWORD,
+		auto_bind=auto_bind,
+		lazy=False,
+		#client_strategy=ldap3.ASYNC, # ldap3.REUSABLE,
+		client_strategy=ldap3.SYNC,
+		#pool_name="default",
+		#pool_size=5,
+		#pool_lifetime=20*60,  # 20 minutes
+		#pool_keepalive=60,
+		raise_exceptions=True)
+	
+	#conn.bind()
+	return conn
+
+
+
+

management/backup.py

@@ -1,4 +1,5 @@
 #!/usr/local/lib/mailinabox/env/bin/python
+# -*- indent-tabs-mode: t; tab-width: 4; python-indent-offset: 4; -*-
 
 # This script performs a backup of all user data:
 # 1) System services are stopped.
@@ -250,6 +251,7 @@ def perform_backup(full_backup):
 	service_command("php7.2-fpm", "stop", quit=True)
 	service_command("postfix", "stop", quit=True)
 	service_command("dovecot", "stop", quit=True)
+	service_command("slapd", "stop", quit=True)
 
 	# Execute a pre-backup script that copies files outside the homedir.
 	# Run as the STORAGE_USER user, not as root. Pass our settings in
@@ -279,6 +281,7 @@ def perform_backup(full_backup):
 			get_env(env))
 	finally:
 		# Start services again.
+		service_command("slapd", "start", quit=False)
 		service_command("dovecot", "start", quit=False)
 		service_command("postfix", "start", quit=False)
 		service_command("php7.2-fpm", "start", quit=False)

management/dns_update.py

@@ -22,7 +22,8 @@ def get_dns_domains(env):
 	# Add all domain names in use by email users and mail aliases and ensure
 	# PRIMARY_HOSTNAME is in the list.
 	domains = set()
-	domains |= get_mail_domains(env)
+	domains |= get_mail_domains(env, category="mail")
+	domains |= get_mail_domains(env, category="ssl")
 	domains.add(env['PRIMARY_HOSTNAME'])
 	return domains
 

management/mail_log.py

@@ -115,8 +115,10 @@ def scan_mail_log(env):
 
     try:
         import mailconfig
-        collector["known_addresses"] = (set(mailconfig.get_mail_users(env)) |
-                                        set(alias[0] for alias in mailconfig.get_mail_aliases(env)))
+        users = mailconfig.get_mail_users(env, as_map=True)
+        aliases = mailconfig.get_mail_aliases(env, as_map=True)
+        collector["known_addresses"] = (set(users.keys()) |
+                                        set(aliases.keys()))
     except ImportError:
         pass
 

management/mailconfig.py

@@ -1,4 +1,5 @@
 #!/usr/local/lib/mailinabox/env/bin/python
+# -*- indent-tabs-mode: t; tab-width: 4; python-indent-offset: 4; -*-
 
 # NOTE:
 # This script is run both using the system-wide Python 3
@@ -9,11 +10,51 @@
 # Python 3 in setup/questions.sh to validate the email
 # address entered by the user.
 
-import subprocess, shutil, os, sqlite3, re
-import utils
+import subprocess, shutil, os, sqlite3, re, ldap3, uuid
+import utils, backend
 from email_validator import validate_email as validate_email_, EmailNotValidError
 import idna
 
+
+#
+# LDAP notes:
+#
+#    Users have an objectClass of mailUser with a mail and maildrop
+#    attribute for the email address. For historical reasons, the
+#    management interface only permits lowercase email addresses.
+#
+#    In the current implementation, both attributes will have the same
+#    lowercase value, but it's not a requirement of the underlying
+#    postfix and dovecot configurations that it be this way. Postfix
+#    and dovecot use the mail attribute to find the user and maildrop
+#    is where the mail is delivered. We use maildrop in the management
+#    interface because that's the address that will always map
+#    directly to the filesystem for archived users.
+#
+#    Email addresses and domain comparisons performed by the LDAP
+#    server are not case sensitive because their respective schemas
+#    define a case-insensitive comparison for those attributes.
+#
+#    User privileges are maintained in the mailaccess attribute of
+#    users.
+#
+#    Aliases and permitted-senders are separate entities in the LDAP
+#    database, but both are of objectClass mailGroup with a
+#    single-valued mail attribute. Alias addresses are forced to
+#    lowercase, again for historical reasons.
+#
+#    All alias and permitted-sender email addresses in the database
+#    are IDNA encoded. International domains for users is not
+#    supported or permitted.
+#
+#    Domains that are handled by this mail server are maintained
+#    on-the-fly as users are added and deleted. They have an
+#    objectClass of domain with attribute dc.
+#
+#    LDAP "records" in this code are dictionaries containing the
+#    attributes and distinguished name of the entry.
+#
+
 def validate_email(email, mode=None):
 	# Checks that an email address is syntactically valid. Returns True/False.
 	# Until Postfix supports SMTPUTF8, an email address may contain ASCII
@@ -91,20 +132,90 @@ def is_dcv_address(email):
 			return True
 	return False
 
-def open_database(env, with_connection=False):
-	conn = sqlite3.connect(env["STORAGE_ROOT"] + "/mail/users.sqlite")
-	if not with_connection:
-		return conn.cursor()
+def open_database(env):
+		return backend.connect(env)
+
+def find_mail_user(env, email, attributes=None, conn=None):
+	# Find the user with the given email address and return the ldap
+	# record for it.
+	#
+	# email is the users email address
+	# attributes are a list of attributes to return eg ["mail","maildrop"]
+	# conn is a ldap database connection, if not specified a new one
+	# is established	
+	#
+	# The ldap record for the user is returned or None if not found.
+	if not conn: conn = open_database(env)
+	id=conn.search(env.LDAP_USERS_BASE,
+		       "(&(objectClass=mailUser)(maildrop=%s))" % email,
+		       attributes=attributes)
+	response = conn.wait(id)
+	if response.count() > 1:
+		dns = [ rec['dn'] for rec in response ]
+		raise LookupError("Detected more than one user with the same email address (%s): %s" % (email, ";".join(dns)))
 	else:
-		return conn, conn.cursor()
+		return response.next()
+	
+def find_mail_alias(env, email, attributes=None, conn=None):
+	# Find the alias with the given address and return the ldap
+	# records for it and the associated permitted senders (if one).
+	#
+	# email is the alias address. It must be IDNA encoded.
+	#
+	# attributes are a list of attributes to return, eg
+	# ["member","rfc822MailMember"]
+	#
+	# conn is a ldap database connection, if not specified a new one
+	# is established.
+	#
+	# A tuple having the two ldap records for the alias and it's
+	# permitted senders (alias, permitted_senders) is returned. If
+	# either is not found, the corresponding tuple value will be None.
+	# 
+	if not conn: conn = open_database(env)
+	# get alias
+	id=conn.search(env.LDAP_ALIASES_BASE,
+		       "(&(objectClass=mailGroup)(mail=%s))" % email,
+		       attributes=attributes)
+	response = conn.wait(id)
+	if response.count() > 1:
+		dns = [ rec['dn'] for rec in response ]
+		raise LookupError("Detected more than one alias with the same email address (%s): %s" % (email, ";".join(dns)))
+	alias = response.next()
 
-def get_mail_users(env):
-	# Returns a flat, sorted list of all user accounts.
+	# get permitted senders for alias
+	id=conn.search(env.LDAP_PERMITTED_SENDERS_BASE,
+		       "(&(objectClass=mailGroup)(mail=%s))" % email,
+		       attributes=attributes)
+	response = conn.wait(id)
+	if response.count() > 1:
+		dns = [ rec['dn'] for rec in response ]
+		raise LookupError("Detected more than one permitted senders group with the same email address (%s): %s" % (email, ";".join(dns)))
+	permitted_senders = response.next()
+	return (alias, permitted_senders)
+	
+
+def get_mail_users(env, as_map=False):
+	# When `as_map` is False, this function returns a flat, sorted
+	# array of all user accounts. If True, it returns a dict where key
+	# is the user and value is a dict having, dn, maildrop and
+	# mail addresses
 	c = open_database(env)
-	c.execute('SELECT email FROM users')
-	users = [ row[0] for row in c.fetchall() ]
-	return utils.sort_email_addresses(users, env)
+	pager = c.paged_search(env.LDAP_USERS_BASE, "(objectClass=mailUser)", attributes=['maildrop','mail'])
+	if as_map:
+		users = {}
+		for rec in pager:
+			users[rec['maildrop'][0]] = {
+				"dn":   rec['dn'],
+				"mail": rec['mail'],
+				"maildrop": rec['maildrop'][0]
+			}
+		return users
+	else:
+		users = [ rec['maildrop'][0] for rec in pager ]
+		return utils.sort_email_addresses(users, env)
 
+	
 def get_mail_users_ex(env, with_archived=False):
 	# Returns a complex data structure of all user accounts, optionally
 	# including archived (status="inactive") accounts.
@@ -128,13 +239,15 @@ def get_mail_users_ex(env, with_archived=False):
 	users = []
 	active_accounts = set()
 	c = open_database(env)
-	c.execute('SELECT email, privileges FROM users')
-	for email, privileges in c.fetchall():
-		active_accounts.add(email)
+	response = c.wait( c.search(env.LDAP_USERS_BASE, "(objectClass=mailUser)", attributes=['maildrop','mailaccess']) )
 
+	for rec in response:
+		email = rec['maildrop'][0]
+		privileges = rec['mailaccess']
+		active_accounts.add(email)
 		user = {
 			"email": email,
-			"privileges": parse_privs(privileges),
+			"privileges": privileges,
 			"status": "active",
 		}
 		users.append(user)
@@ -179,21 +292,82 @@ def get_mail_users_ex(env, with_archived=False):
 def get_admins(env):
 	# Returns a set of users with admin privileges.
 	users = set()
-	for domain in get_mail_users_ex(env):
-		for user in domain["users"]:
-			if "admin" in user["privileges"]:
-				users.add(user["email"])
+	c = open_database(env)
+	response = c.wait( c.search(env.LDAP_USERS_BASE, "(&(objectClass=mailUser)(mailaccess=admin))", attributes=['maildrop']) )
+	for rec in response:
+		users.add(rec['maildrop'][0])
 	return users
 
-def get_mail_aliases(env):
-	# Returns a sorted list of tuples of (address, forward-tos, permitted-senders).
-	c = open_database(env)
-	c.execute('SELECT source, destination, permitted_senders FROM aliases')
-	aliases = { row[0]: row for row in c.fetchall() } # make dict
 
-	# put in a canonical order: sort by domain, then by email address lexicographically
-	aliases = [ aliases[address] for address in utils.sort_email_addresses(aliases.keys(), env) ]
-	return aliases
+def get_mail_aliases(env, as_map=False):
+	# Retrieve all mail aliases.
+	#
+	# If as_map is False, the function returns a sorted array of tuples:
+	#
+	#   (address(lowercase), forward-tos{string,csv}, permitted-senders{string,csv})
+	#
+	# If as-map is True, it returns a dict whose keys are
+	# address(lowercase) and whose values are:
+	#
+	#    { dn: {string},
+	#      mail: {string}
+	#      forward_tos: {array of string},
+	#      permited_senders: {array of string}
+	#    }
+	#
+	c = open_database(env)
+	# get all permitted senders
+	pager = c.paged_search(env.LDAP_PERMITTED_SENDERS_BASE, "(objectClass=mailGroup)", attributes=["mail", "member"])
+	
+	# make a dict of permitted senders, key=mail(lowercase) value=members
+	permitted_senders = { rec["mail"][0].lower(): rec["member"] for rec in pager }
+	
+	# get all alias groups
+	pager = c.paged_search(env.LDAP_ALIASES_BASE, "(objectClass=mailGroup)", attributes=['mail','member','rfc822MailMember'])
+	
+	# make a dict of aliases
+	# key=email(lowercase), value=(email, forward-tos, permitted-senders).
+	aliases = {}
+	for alias in pager:
+		alias_email = alias['mail'][0]
+		alias_email_lc = alias_email.lower()
+		
+		# chase down each member's email address, because a member is a dn
+		forward_tos = []
+		for fwd_to in c.chase_members(alias['member'], 'mail', env):
+			forward_tos.append(fwd_to[0])
+			
+		for fwd_to in alias['rfc822MailMember']:
+			forward_tos.append(fwd_to)
+		
+		# chase down permitted senders' email addresses
+		allowed_senders = []
+		if alias_email_lc in permitted_senders:
+			members = permitted_senders[alias_email_lc]
+			for mail_list in c.chase_members(members, 'mail', env):
+				for mail in mail_list:
+					allowed_senders.append(mail)
+
+		aliases[alias_email_lc] = {
+			"dn": alias["dn"],
+			"mail": alias_email,
+			"forward_tos": forward_tos,
+			"permitted_senders": allowed_senders
+		}
+
+	if not as_map:
+		# put in a canonical order: sort by domain, then by email address lexicographically
+		list = []
+		for address in utils.sort_email_addresses(aliases.keys(), env):
+			alias = aliases[address]
+			xft = ",".join(alias["forward_tos"])
+			xas = ",".join(alias["permitted_senders"])
+			list.append( (address, xft, xas) )
+		return list
+	
+	else:
+		return aliases
+			
 
 def get_mail_aliases_ex(env):
 	# Returns a complex data structure of all mail aliases, similar
@@ -258,15 +432,174 @@ def get_domain(emailaddr, as_unicode=True):
 			pass
 	return ret
 
-def get_mail_domains(env, filter_aliases=lambda alias : True):
-	# Returns the domain names (IDNA-encoded) of all of the email addresses
-	# configured on the system.
-	return set(
-		   [get_domain(login, as_unicode=False) for login in get_mail_users(env)]
-		 + [get_domain(address, as_unicode=False) for address, *_ in get_mail_aliases(env) if filter_aliases(address) ]
-		 )
+def get_mail_domains(env, as_map=False, filter_aliases=None, category=None):
+	# Retrieves all domains, IDNA-encoded, we accept mail for.
+	#
+	# If as_map is False, the function returns the lowercase domain
+	# names (IDNA-encoded) as an array.
+	#
+	# If as_map is True, it returns a dict whose keys are
+	# domain(idna,lowercase) and whose values are:
+	#
+	#   { dn:{string}, domain:{string(idna)} }
+	#
+	# filter_aliases is an inclusion filter - a True return value from
+	# this lambda function indicates an alias to include.
+	#
+	# With filter_aliases, there has to be at least one user or
+	# filtered (included) alias in the domain for the domain to be
+	# part of the returned set. If None, all domains are returned.
+	#
+	# category is another type of filter. Set to a string value to
+	# return only those domains of that category. ie. the
+	# "businessCategory" attribute of the domain must include this
+	# category.
+	#
+	conn = open_database(env)
+	filter = "(&(objectClass=domain)(businessCategory=mail))"
+	if category:
+		filter = "(&(objectClass=domain)(businessCategory=%s))" % category
+	id = conn.search(env.LDAP_DOMAINS_BASE, filter, attributes="dc")
+	response = conn.wait(id)
+	filter_candidates=[]
+	domains=None
+	if as_map:
+		domains = {}
+		for rec in response:
+			domains[ rec["dc"][0].lower() ] = {
+				"dn": rec["dn"],
+				"domain": rec["dc"][0]
+			}
+			if filter_aliases: filter_candidates.append(rec['dc'][0].lower())
+	else:
+		domains = set([ rec["dc"][0].lower() for rec in response ])
+		if filter_aliases: filter_candidates += domains
+	
+	for candidate in filter_candidates:
+		# with the filter, there has to be at least one user or
+		# filtered (included) alias in the domain for the domain to be
+		# part of the returned set
+
+		# any users ?
+		response = conn.wait( conn.search(env.LDAP_USERS_BASE, "(&(objectClass=mailUser)(mail=*@%s))" % candidate, size_limit=1) )
+		if response.next():
+			# yes, that domain needs to be in the returned set
+			continue
+
+		# any filtered aliases ?
+		pager = conn.paged_search(
+			env.LDAP_ALIASES_BASE,
+			"(&(objectClass=mailGroup)(mail=*@%s))" % candidate,
+			attributes=['mail'])
+
+		remove = True
+		for rec in pager:
+			if filter_aliases(rec['mail'][0]):
+				remove = False
+				pager.abandon()
+				break
+
+		if remove:
+			if as_map: del domains[candidate]
+			else: domains.remove(candidate)
+			
+	return domains
+
+
+def add_mail_domain(env, domain, validate=True):
+	# Create domain entry indicating that we are handling
+	# mail for that domain.
+	#
+	# We only care about domains for users, not for aliases.
+	#
+	# domain: IDNA encoded domain name.
+	# validate: If True, ensure there is at least one user on the
+	# system using that domain.
+	#
+	# returns True if added, False if it already exists or fails
+	# validation.
+	
+	conn = open_database(env)
+	if validate:
+		# check to ensure there is at least one user with that domain
+		id = conn.search(env.LDAP_USERS_BASE,
+						 "(&(objectClass=mailUser)(mail=*@%s))" % domain,
+						 size_limit=1)
+		if conn.wait(id).count() == 0:
+			# no mail users are using that domain!
+			return False
+		
+	dn = 'dc=%s,%s' % (domain, env.LDAP_DOMAINS_BASE)
+	try:
+		response = conn.wait( conn.add(dn, [ 'domain' ], {
+			"businessCategory": "mail"
+		}) )
+		return True
+	except ldap3.core.exceptions.LDAPEntryAlreadyExistsResult:
+		try:
+			# add 'mail' as a businessCategory
+			changes = {	"businessCategory": [(ldap3.MODIFY_ADD, ['mail'])] }
+			response = conn.wait ( conn.modify(dn, changes) )
+		except ldap3.core.exceptions.LDAPAttributeOrValueExistsResult:
+			pass
+		return False
+
+	
+def remove_mail_domain(env, domain, validate=True):
+	# Remove the specified domain from the list of domains that we
+	# handle mail for. The domain must be IDNA encoded.
+	#
+	# If validate is True, ensure there are no valid users on the system
+	# currently using the domain.
+	#
+	# Returns True if removed or does not exist, False if the domain
+	# fails validation.
+	conn = open_database(env)
+	if validate:
+		# check to ensure no users are using the domain
+		id = conn.search(env.LDAP_USERS_BASE,
+						 "(&(objectClass=mailUser)(mail=*@%s))" % domain,
+						 size_limit=1)
+		if conn.wait(id).count() > 0:
+			# there is one or more mailUser's with that domain
+			return False
+		
+	id = conn.search(env.LDAP_DOMAINS_BASE,
+					 "(&(objectClass=domain)(dc=%s))" % domain,
+					 attributes=['businessCategory'])
+	
+	existing = conn.wait(id).next()
+	if existing is None:
+		# the domain doesn't exist!
+		return True
+
+	newvals=existing['businessCategory'].copy()
+	if 'mail' in newvals:
+		newvals.remove('mail')
+	else:
+		# we only remove mail-related entries
+		return False
+
+	if len(newvals)==0:
+		conn.wait ( conn.delete(existing['dn']) )
+	else:
+		conn.wait ( conn.modify_record(existing, {'businessCategory', newvals}))
+	return True
+
 
 def add_mail_user(email, pw, privs, env):
+	# Add a new mail user.
+	#
+	# email: the new user's email address
+	# pw: the new user's password
+	# privs: either an array of privilege strings, or a newline
+	# separated string of privilege names
+	#
+	# If an error occurs, the function returns a tuple of (message,
+	# http-status).
+	#
+	# If successful, the string "OK" is returned.
+	
 	# validate email
 	if email.strip() == "":
 		return ("No email address provided.", 400)
@@ -284,73 +617,131 @@ def add_mail_user(email, pw, privs, env):
 	validate_password(pw)
 
 	# validate privileges
-	if privs is None or privs.strip() == "":
-		privs = []
-	else:
-		privs = privs.split("\n")
-		for p in privs:
-			validation = validate_privilege(p)
-			if validation: return validation
+	privs = []
+	if privs is not None and type(privs) is str and privs.strip() != "":
+		privs = parse_privs(privs)
+	for p in privs:
+		validation = validate_privilege(p)
+		if validation: return validation
 
 	# get the database
-	conn, c = open_database(env, with_connection=True)
+	conn = open_database(env)
 
-	# hash the password
-	pw = hash_password(pw)
+	# ensure another user doesn't have that address
+	id=conn.search(env.LDAP_USERS_BASE, "(&(objectClass=mailUser)(|(mail=%s)(maildrop=%s)))" % (email, email))
+	if conn.wait(id).count() > 0:
+		return ("User alreay exists.", 400)
+
+	# ensure an alias doesn't have that address
+	id=conn.search(env.LDAP_ALIASES_BASE, "(&(objectClass=mailGroup)(mail=%s))" % email)
+	if conn.wait(id).count() > 0:
+		return ("An alias exists with that address.", 400)
+	
+	# Generate a unique id for uid
+	uid = '%s' % uuid.uuid4()
+
+	# choose a common name and surname (required attributes)
+	cn = email.split("@")[0].replace('.',' ').replace('_',' ')
+	sn = cn[cn.find(' ')+1:]
+	
+	# compile user's attributes
+	# for historical reasons, make the email address lowercase
+	attrs = {
+		"mail" : email,
+		"maildrop" : email.lower(),
+		"uid" : uid,
+		"mailaccess": privs,
+		"cn": cn,
+		"sn": sn,
+		"shadowLastChange": backend.get_shadowLastChanged()
+	}
 
 	# add the user to the database
-	try:
-		c.execute("INSERT INTO users (email, password, privileges) VALUES (?, ?, ?)",
-			(email, pw, "\n".join(privs)))
-	except sqlite3.IntegrityError:
-		return ("User already exists.", 400)
+	dn = 'uid=%s,%s' % (uid, env.LDAP_USERS_BASE)
+	id=conn.add(dn, [ 'inetOrgPerson','mailUser','shadowAccount' ], attrs);
+	conn.wait(id)
 
-	# write databasebefore next step
-	conn.commit()
+	# set the password - the ldap server will hash it
+	conn.extend.standard.modify_password(user=dn, new_password=pw)
 
+	# tell postfix the domain is local, if needed
+	add_mail_domain(env, get_domain(email, as_unicode=False), validate=False)
+
+	# convert alias's rfc822MailMember to member
+	convert_rfc822MailMember(env, conn, dn, email)
+	
 	# Update things in case any new domains are added.
 	return kick(env, "mail user added")
 
 def set_mail_password(email, pw, env):
-	# validate that password is acceptable
+	# validate that the password is acceptable
 	validate_password(pw)
 
-	# hash the password
-	pw = hash_password(pw)
-
-	# update the database
-	conn, c = open_database(env, with_connection=True)
-	c.execute("UPDATE users SET password=? WHERE email=?", (pw, email))
-	if c.rowcount != 1:
+	# find the user				  
+	conn = open_database(env)
+	user = find_mail_user(env, email, ['shadowLastChange'], conn)
+	if user is None:
 		return ("That's not a user (%s)." % email, 400)
-	conn.commit()
+
+	# update the database - the ldap server will hash the password
+	conn.extend.standard.modify_password(user=user['dn'], new_password=pw)
+
+	# update shadowLastChange
+	conn.modify_record(user, {'shadowLastChange': backend.get_shadowLastChanged()})
+
 	return "OK"
 
-def hash_password(pw):
-	# Turn the plain password into a Dovecot-format hashed password, meaning
-	# something like "{SCHEME}hashedpassworddata".
-	# http://wiki2.dovecot.org/Authentication/PasswordSchemes
-	return utils.shell('check_output', ["/usr/bin/doveadm", "pw", "-s", "SHA512-CRYPT", "-p", pw]).strip()
+def validate_login(email, pw, env):
+	# Validate that `email` exists and has password `pw`.
+	# Returns True if valid, or False if invalid.
+	user = find_mail_user(env, email)
+	if user is None:
+		raise ValueError("That's not a user (%s)" % email)
+	try:
+		# connect as that user to validate the login
+		server = backend.get_ldap_server(env)
+		conn = ldap3.Connection(
+			server,
+			user=user['dn'],
+			password=pw,
+			raise_exceptions=True)
+		conn.bind()
+		conn.unbind()
+		return True
+	except ldap3.core.exceptions.LDAPInvalidCredentialsResult:
+		return False
+
 
 def get_mail_password(email, env):
-	# Gets the hashed password for a user. Passwords are stored in Dovecot's
-	# password format, with a prefixed scheme.
-	# http://wiki2.dovecot.org/Authentication/PasswordSchemes
-	# update the database
-	c = open_database(env)
-	c.execute('SELECT password FROM users WHERE email=?', (email,))
-	rows = c.fetchall()
-	if len(rows) != 1:
+	# Gets the hashed passwords for a user. In ldap, userPassword is
+	# multi-valued and each value can have different hash. This
+	# function returns all hashes as an array.
+	user = find_mail_user(env, email, attributes=["userPassword"])
+	if user is None:
 		raise ValueError("That's not a user (%s)." % email)
-	return rows[0][0]
+	if len(user['userPassword'])==0:
+		raise ValueError("The user has no password (%s)" % email)
+	return user['userPassword']
+
 
 def remove_mail_user(email, env):
-	# remove
-	conn, c = open_database(env, with_connection=True)
-	c.execute("DELETE FROM users WHERE email=?", (email,))
-	if c.rowcount != 1:
+	# Remove the user as a valid user of the system.
+	# If an error occurs, the function returns a tuple of (message,
+	# http-status).
+	#
+	# If successful, the string "OK" is returned.
+	conn = open_database(env)
+	
+	# find the user
+	user = find_mail_user(env, email, conn=conn)
+	if user is None:
 		return ("That's not a user (%s)." % email, 400)
-	conn.commit()
+	
+	# delete the user
+	conn.wait( conn.delete(user['dn']) )
+
+	# remove as a handled domain, if needed
+	remove_mail_domain(env, get_domain(email, as_unicode=False))
 
 	# Update things in case any domains are removed.
 	return kick(env, "mail user removed")
@@ -359,14 +750,19 @@ def parse_privs(value):
 	return [p for p in value.split("\n") if p.strip() != ""]
 
 def get_mail_user_privileges(email, env, empty_on_error=False):
-	# get privs
+	# Get an array of privileges held by the specified user.
 	c = open_database(env)
-	c.execute('SELECT privileges FROM users WHERE email=?', (email,))
-	rows = c.fetchall()
-	if len(rows) != 1:
+	try:
+		user = find_mail_user(env, email, ['mailaccess'], c)
+	except LookupError as e:
+		if empty_on_error: return []
+		raise e
+	
+	if user is None:
 		if empty_on_error: return []
 		return ("That's not a user (%s)." % email, 400)
-	return parse_privs(rows[0][0])
+	
+	return user['mailaccess']
 
 def validate_privilege(priv):
 	if "\n" in priv or priv.strip() == "":
@@ -374,38 +770,93 @@ def validate_privilege(priv):
 	return None
 
 def add_remove_mail_user_privilege(email, priv, action, env):
+	# Add or remove a privilege from a user.
+	# priv: the name of the privilege to add or remove
+	# action: "add" to add the privilege, or "remove" to remove it
+	# email: the user
+	#
+	# If an error occurs, the function returns a tuple of (message,
+	# http-status).
+	#
+	# If successful, the string "OK" is returned.
+	
 	# validate
 	validation = validate_privilege(priv)
 	if validation: return validation
 
 	# get existing privs, but may fail
-	privs = get_mail_user_privileges(email, env)
-	if isinstance(privs, tuple): return privs # error
+	user = find_mail_user(env, email, attributes=['mailaccess'])
+	if user is None:
+		return ("That's not a user (%s)." % email, 400)
+		
+	privs = user['mailaccess'].copy()
 
 	# update privs set
+	changed = False
 	if action == "add":
 		if priv not in privs:
 			privs.append(priv)
+			changed = True
+			
 	elif action == "remove":
-		privs = [p for p in privs if p != priv]
+		if priv in privs:
+			privs.remove(priv)
+			changed = True
 	else:
 		return ("Invalid action.", 400)
 
 	# commit to database
-	conn, c = open_database(env, with_connection=True)
-	c.execute("UPDATE users SET privileges=? WHERE email=?", ("\n".join(privs), email))
-	if c.rowcount != 1:
-		return ("Something went wrong.", 400)
-	conn.commit()
-
+	if changed:
+		conn = open_database(env)
+		conn.modify_record( user, {'mailaccess': privs} )
+		
 	return "OK"
 
+
+
+def convert_rfc822MailMember(env, conn, dn, mail):
+	# if a newly added alias or user exists as an rfc822MailMember,
+	# convert it to a member dn
+	# the new alias or user is specified by arguments mail and dn.
+	# mail is the new alias or user's email address
+	# dn is the new alias or user's distinguished name
+	# conn is an existing ldap database connection
+	id=conn.search(env.LDAP_ALIASES_BASE,
+				   "(&(objectClass=mailGroup)(rfc822MailMember=%s))" % mail,
+				   attributes=[ 'member','rfc822MailMember' ])
+	response = conn.wait(id)
+	for rec in response:
+		# remove mail from rfc822MailMember
+		changes={ "rfc822MailMember": [(ldap3.MODIFY_DELETE, [mail])] }
+		conn.wait( conn.modify(rec['dn'], changes) )
+
+		# add dn to member
+		rec['member'].append(dn)
+		changes={ "member": [(ldap3.MODIFY_ADD, rec['member'])] }
+		try:
+			conn.wait( conn.modify(rec['dn'], changes) )
+		except ldap3.core.exceptions.LDAPAttributeOrValueExistsResult:
+			pass
+
+		
 def add_mail_alias(address, forwards_to, permitted_senders, env, update_if_exists=False, do_kick=True):
+	# Add a new alias group with permitted senders.
+	#
+	# address: the email address of the alias
+	# forwards_to: a string of newline and comma separated email address
+	# where mail is delivered
+	# permitted_senders: a string of newline and comma separated email addresses of local users that are permitted to MAIL FROM the alias.
+	# update_if_exists: if False and the alias exists fail, otherwise update the existing alias with the new values
+	#
+	# If an error occurs, the function returns a tuple of (message,
+	# http-status).
+	#
+	# If successful, the string "OK" is returned.
+
 	# convert Unicode domain to IDNA
 	address = sanitize_idn_email_address(address)
 
-	# Our database is case sensitive (oops), which affects mail delivery
-	# (Postfix always queries in lowercase?), so force lowercase.
+	# for historical reasons, force the address to lowercase
 	address = address.lower()
 
 	# validate address
@@ -415,8 +866,11 @@ def add_mail_alias(address, forwards_to, permitted_senders, env, update_if_exist
 	if not validate_email(address, mode='alias'):
 		return ("Invalid email address (%s)." % address, 400)
 
+	# retrieve all logins as a map ( email.lower() => {mail,dn} )
+	valid_logins = get_mail_users( env, as_map=True )
+	
 	# validate forwards_to
-	validated_forwards_to = []
+	validated_forwards_to = [ ]
 	forwards_to = forwards_to.strip()
 
 	# extra checks for email addresses used in domain control validation
@@ -451,9 +905,8 @@ def add_mail_alias(address, forwards_to, permitted_senders, env, update_if_exist
 				validated_forwards_to.append(email)
 
 	# validate permitted_senders
-	valid_logins = get_mail_users(env)
-	validated_permitted_senders = []
-	permitted_senders = permitted_senders.strip()
+	validated_permitted_senders = [ ]  # list of dns
+	permitted_senders = permitted_senders.strip( )
 
 	# Parse comma and \n-separated sender logins & validate. The permitted_senders must be
 	# valid usernames.
@@ -461,51 +914,111 @@ def add_mail_alias(address, forwards_to, permitted_senders, env, update_if_exist
 		for login in line.split(","):
 			login = login.strip()
 			if login == "": continue
-			if login not in valid_logins:
+			if login.lower() not in valid_logins:
 				return ("Invalid permitted sender: %s is not a user on this system." % login, 400)
-			validated_permitted_senders.append(login)
+			validated_permitted_senders.append(valid_logins[login.lower()]['dn'])
 
 	# Make sure the alias has either a forwards_to or a permitted_sender.
 	if len(validated_forwards_to) + len(validated_permitted_senders) == 0:
 		return ("The alias must either forward to an address or have a permitted sender.", 400)
 
+
+	# break validated_forwards_to into 'local' where an email
+	# address is a local user, or 'remote' where the email doesn't
+	# exist on this system
+
+	vfwd_tos_local = []   # list of dn's
+	vfwd_tos_remote = []  # list of email
+	for fwd_to in validated_forwards_to:
+		fwd_to_lc = fwd_to.lower()
+		if fwd_to_lc in valid_logins:
+			dn = valid_logins[fwd_to_lc]['dn']
+			vfwd_tos_local.append(dn)
+		else:
+			vfwd_tos_remote.append(fwd_to)
+			
 	# save to db
 
-	forwards_to = ",".join(validated_forwards_to)
-
-	if len(validated_permitted_senders) == 0:
-		permitted_senders = None
+	conn = open_database(env)
+	existing_alias, existing_permitted_senders = find_mail_alias(env, address, ['member','rfc822MailMember'], conn)
+	if existing_alias and not update_if_exists:
+		return ("Alias already exists (%s)." % address, 400)
+	
+	cn="%s" % uuid.uuid4()
+	dn="cn=%s,%s" % (cn, env.LDAP_ALIASES_BASE)
+	if address.startswith('@') and \
+	   len(validated_forwards_to)==1 and \
+	   validated_forwards_to[0].startswith('@'):
+		description = "Domain alias %s->%s" % (address, validated_forwards_to[0])
+	elif address.startswith('@'):
+		description = "Catch-all for %s" % address
+	else:
+		description ="Mail group %s" % address
+	attrs = {
+		"mail": address,
+		"description": description,
+		"member": vfwd_tos_local,
+		"rfc822MailMember": vfwd_tos_remote
+	}
+	
+	op = conn.add_or_modify(dn, existing_alias,
+			['member', 'rfc822MailMember' ],
+			['mailGroup'],
+			attrs)
+	if op == 'modify':
+		return_status = "alias updated"
 	else:
-		permitted_senders = ",".join(validated_permitted_senders)
-
-	conn, c = open_database(env, with_connection=True)
-	try:
-		c.execute("INSERT INTO aliases (source, destination, permitted_senders) VALUES (?, ?, ?)", (address, forwards_to, permitted_senders))
 		return_status = "alias added"
-	except sqlite3.IntegrityError:
-		if not update_if_exists:
-			return ("Alias already exists (%s)." % address, 400)
-		else:
-			c.execute("UPDATE aliases SET destination = ?, permitted_senders = ? WHERE source = ?", (forwards_to, permitted_senders, address))
-			return_status = "alias updated"
-
-	conn.commit()
+		convert_rfc822MailMember(env, conn, dn, address)
+		
+	# add or modify permitted-senders group
+	
+	cn = '%s' % uuid.uuid4()
+	dn = "cn=%s,%s" % (cn, env.LDAP_PERMITTED_SENDERS_BASE)
+	attrs = {
+		"mail" : address,
+		"description": "Permitted to MAIL FROM this address",
+		"member" : validated_permitted_senders
+	}
+	if len(validated_permitted_senders)==0:
+		if existing_permitted_senders:
+			dn = existing_permitted_senders['dn']
+			conn.wait( conn.delete(dn) )
+	else:
+		op=conn.add_or_modify(dn, existing_permitted_senders,
+							  [ 'member' ], [ 'mailGroup' ],
+							  attrs)
 
 	if do_kick:
 		# Update things in case any new domains are added.
 		return kick(env, return_status)
+	
 
 def remove_mail_alias(address, env, do_kick=True):
+	# Remove an alias group and it's associated permitted senders
+	# group.
+	#
+	# address is the email address of the alias
+	#
+	# If an error occurs, the function returns a tuple of (message,
+	# http-status).
+	#
+	# If successful, the string "OK" is returned.
+	
 	# convert Unicode domain to IDNA
 	address = sanitize_idn_email_address(address)
 
 	# remove
-	conn, c = open_database(env, with_connection=True)
-	c.execute("DELETE FROM aliases WHERE source=?", (address,))
-	if c.rowcount != 1:
+	conn = open_database(env)
+	existing_alias, existing_permitted_senders = find_mail_alias(env, address, conn=conn)
+	if existing_alias:
+		conn.delete(existing_alias['dn'])
+	else:
 		return ("That's not an alias (%s)." % address, 400)
-	conn.commit()
 
+	if existing_permitted_senders:
+		conn.delete(existing_permitted_senders['dn'])
+		
 	if do_kick:
 		# Update things in case any domains are removed.
 		return kick(env, "alias removed")
@@ -515,6 +1028,8 @@ def get_system_administrator(env):
 
 def get_required_aliases(env):
 	# These are the aliases that must exist.
+	# Returns a set of email addresses.
+	
 	aliases = set()
 
 	# The system administrator alias is required.
@@ -555,9 +1070,8 @@ def kick(env, mail_result=None):
 
 	# Ensure every required alias exists.
 
-	existing_users = get_mail_users(env)
-	existing_alias_records = get_mail_aliases(env)
-	existing_aliases = set(a for a, *_ in existing_alias_records) # just first entry in tuple
+	existing_users = get_mail_users(env, as_map=True)
+	existing_aliases = get_mail_aliases(env, as_map=True)
 	required_aliases = get_required_aliases(env)
 
 	def ensure_admin_alias_exists(address):
@@ -581,8 +1095,9 @@ def kick(env, mail_result=None):
 
 	# Remove auto-generated postmaster/admin on domains we no
 	# longer have any other email addresses for.
-	for address, forwards_to, *_ in existing_alias_records:
+	for address in existing_aliases:
 		user, domain = address.split("@")
+		forwards_to = ",".join(existing_aliases[address]["forward_tos"])
 		if user in ("postmaster", "admin", "abuse") \
 			and address not in required_aliases \
 			and forwards_to == get_system_administrator(env):

management/ssl_certificates.py

@@ -105,6 +105,9 @@ def get_ssl_certificates(env):
 			# prefer one that is not self-signed
 			cert.issuer != cert.subject,
 
+                        # prefer one that is not our temporary ca
+			"Temporary-Mail-In-A-Box-CA" not in "%s" % cert.issuer.rdns,
+
 			###########################################################
 			# The above lines ensure that valid certificates are chosen
 			# over invalid certificates. The lines below choose between
@@ -453,7 +456,7 @@ def post_install_func(env):
 
 	# Symlink the best cert for PRIMARY_HOSTNAME to the system
 	# certificate path, which is hard-coded for various purposes, and then
-	# restart postfix and dovecot.
+	# restart postfix, dovecot and openldap.
 	system_ssl_certificate = os.path.join(os.path.join(env["STORAGE_ROOT"], 'ssl', 'ssl_certificate.pem'))
 	if cert and os.readlink(system_ssl_certificate) != cert['certificate']:
 		# Update symlink.
@@ -463,6 +466,7 @@ def post_install_func(env):
 		os.symlink(ssl_certificate, system_ssl_certificate)
 
 		# Restart postfix and dovecot so they pick up the new file.
+		shell('check_call', ["/usr/sbin/service", "slapd", "restart"])
 		shell('check_call', ["/usr/sbin/service", "postfix", "restart"])
 		shell('check_call', ["/usr/sbin/service", "dovecot", "restart"])
 		ret.append("mail services restarted")
@@ -531,6 +535,9 @@ def check_certificate(domain, ssl_certificate, ssl_private_key, warn_if_expiring
 	if cert.issuer == cert.subject:
 		return ("SELF-SIGNED", None)
 
+	elif "Temporary-Mail-In-A-Box-CA" in "%s" % cert.issuer.rdns:
+		return ("SELF-SIGNED", None)
+
 	# When selecting which certificate to use for non-primary domains, we check if the primary
 	# certificate or a www-parent-domain certificate is good for the domain. There's no need
 	# to run extra checks beyond this point.

management/status_checks.py

@@ -434,10 +434,10 @@ def check_primary_hostname_dns(domain, env, output, dns_domains, dns_zonefiles):
 	check_alias_exists("Hostmaster contact address", "hostmaster@" + domain, env, output)
 
 def check_alias_exists(alias_name, alias, env, output):
-	mail_aliases = dict([(address, receivers) for address, receivers, *_ in get_mail_aliases(env)])
+	mail_aliases = get_mail_aliases(env, as_map=True)
 	if alias in mail_aliases:
-		if mail_aliases[alias]:
-			output.print_ok("%s exists as a mail alias. [%s ↦ %s]" % (alias_name, alias, mail_aliases[alias]))
+		if mail_aliases[alias]["forward_tos"]:
+			output.print_ok("%s exists as a mail alias. [%s ↦ %s]" % (alias_name, alias, ",".join(mail_aliases[alias]["forward_tos"])))
 		else:
 			output.print_error("""You must set the destination of the mail alias for %s to direct email to you or another administrator.""" % alias)
 	else:
@@ -618,7 +618,7 @@ def check_mail_domain(domain, env, output):
 
 	# Check that the postmaster@ email address exists. Not required if the domain has a
 	# catch-all address or domain alias.
-	if "@" + domain not in [address for address, *_ in get_mail_aliases(env)]:
+	if "@" + domain not in get_mail_aliases(env, as_map=True):
 		check_alias_exists("Postmaster contact address", "postmaster@" + domain, env, output)
 
 	# Stop if the domain is listed in the Spamhaus Domain Block List.

management/utils.py

@@ -1,4 +1,5 @@
-import os.path
+# -*- indent-tabs-mode: nil; -*-
+import os.path, collections
 
 # DO NOT import non-standard modules. This module is imported by
 # migrate.py which runs on fresh machines before anything is installed
@@ -6,15 +7,32 @@ import os.path
 
 # THE ENVIRONMENT FILE AT /etc/mailinabox.conf
 
+class Environment(collections.OrderedDict):
+    # subclass OrderedDict and provide attribute lookups to the
+    # underlying dictionary
+    def __getattr__(self, attr):
+        return self[attr]
+
 def load_environment():
     # Load settings from /etc/mailinabox.conf.
-    return load_env_vars_from_file("/etc/mailinabox.conf")
+    env = load_env_vars_from_file("/etc/mailinabox.conf")
 
-def load_env_vars_from_file(fn):
+    # Load settings from STORAGE_ROOT/ldap/miab_ldap.conf
+    # It won't exist exist until migration 13 completes...
+    if os.path.exists(os.path.join(env["STORAGE_ROOT"],"ldap/miab_ldap.conf")):
+        load_env_vars_from_file(os.path.join(env["STORAGE_ROOT"],"ldap/miab_ldap.conf"), strip_quotes=True, merge_env=env)
+    
+    return env
+
+def load_env_vars_from_file(fn, strip_quotes=False, merge_env=None):
     # Load settings from a KEY=VALUE file.
-    import collections
-    env = collections.OrderedDict()
-    for line in open(fn): env.setdefault(*line.strip().split("=", 1))
+    env = Environment()
+    for line in open(fn):
+        env.setdefault(*line.strip().split("=", 1))
+    if strip_quotes:
+        for k in env: env[k]=env[k].strip('"')
+    if merge_env is not None:
+        for k in env: merge_env[k]=env[k]
     return env
 
 def save_environment(env):

management/web_update.py

@@ -1,3 +1,4 @@
+# -*- indent-tabs-mode: t; tab-width: 4; python-indent-offset: 4; -*-
 # Creates an nginx configuration file so we serve HTTP/HTTPS on all
 # domains for which a mail account has been set up.
 ########################################################################
@@ -9,14 +10,15 @@ from dns_update import get_custom_dns_config, get_dns_zones
 from ssl_certificates import get_ssl_certificates, get_domain_ssl_files, check_certificate
 from utils import shell, safe_domain_name, sort_domains
 
-def get_web_domains(env, include_www_redirects=True, exclude_dns_elsewhere=True):
+def get_web_domains(env, include_www_redirects=True, exclude_dns_elsewhere=True, categories=['mail', 'ssl']):
 	# What domains should we serve HTTP(S) for?
 	domains = set()
 
 	# Serve web for all mail domains so that we might at least
 	# provide auto-discover of email settings, and also a static website
 	# if the user wants to make one.
-	domains |= get_mail_domains(env)
+	for category in categories:
+		domains |= get_mail_domains(env, category=category)
 
 	if include_www_redirects:
 		# Add 'www.' subdomains that we want to provide default redirects
@@ -27,8 +29,9 @@ def get_web_domains(env, include_www_redirects=True, exclude_dns_elsewhere=True)
 	# Add Autoconfiguration domains, allowing us to serve correct SSL certs.
 	# 'autoconfig.' for Mozilla Thunderbird auto setup.
 	# 'autodiscover.' for Activesync autodiscovery.
-	domains |= set('autoconfig.' + maildomain for maildomain in get_mail_domains(env))
-	domains |= set('autodiscover.' + maildomain for maildomain in get_mail_domains(env))
+	if 'mail' in categories:
+		domains |= set('autoconfig.' + maildomain for maildomain in get_mail_domains(env, category='mail'))
+		domains |= set('autodiscover.' + maildomain for maildomain in get_mail_domains(env, category='mail'))
 
 	if exclude_dns_elsewhere:
 		# ...Unless the domain has an A/AAAA record that maps it to a different
@@ -85,8 +88,8 @@ def do_web_update(env):
 
 	# Add configuration all other web domains.
 	has_root_proxy_or_redirect = get_web_domains_with_root_overrides(env)
-	web_domains_not_redirect = get_web_domains(env, include_www_redirects=False)
-	for domain in get_web_domains(env):
+	web_domains_not_redirect = get_web_domains(env, include_www_redirects=False, categories=['mail'])
+	for domain in get_web_domains(env, categories=['mail']):
 		if domain == env['PRIMARY_HOSTNAME']:
 			# PRIMARY_HOSTNAME is handled above.
 			continue