Issue #1340 - LDAP backend for accounts

This commit will:

1. Change the user account database from sqlite to OpenLDAP
2. Add policyd-spf to postfix for SPF validation
3. Add a test runner with some automated test suites

Notes:

User account password hashes are preserved.

There is a new Roundcube contact list called "Directory" that lists the users in LDAP (MiaB users), similar to what Google Suite does.

Users can still change their password in Roundcube.

OpenLDAP is configured with TLS, but all remote access is blocked by firewall rules. Manual changes are required to open it for remote access (eg. "ufw allow proto tcp from <HOST> to any port ldaps").

The test runner is started by executing tests/runner.sh. Be aware that it will make changes to your system, including adding new users, domains, mailboxes, start/stop services, etc. It is highly unadvised to run it on a production system!

The LDAP schema that supports mail delivery with postfix and dovecot is located in conf/postfix.schema. This file is copied verbatim from the LdapAdmin project (GPL, ldapadmin.org). Instead of including the file in git, it could be referenced by URL and downloaded by the setup script if GPL is an issue or apply for a PEN from IANA.

Mangement console and other services should not appear or behave any differently than before.

conf/fail2ban/jails.conf

@@ -76,3 +76,7 @@ enabled  = true
 enabled = true
 maxretry = 7
 bantime = 3600
+
+[slapd]
+enabled = true
+logpath = /var/log/ldap/slapd.log

conf/postfix.schema

@@ -0,0 +1,60 @@
+# LDAP Admin Extensions for Postfix MTA support
+
+attributetype ( 1.3.6.1.4.1.15347.2.102 
+	NAME 'transport' 
+	SUP name)
+
+attributetype ( 1.3.6.1.4.1.15347.2.101 
+	NAME 'mailRoutingAddress' 
+	SUP mail )
+
+attributetype ( 1.3.6.1.4.1.15347.2.110 NAME 'maildest'
+	DESC 'Restricted to send only to local network'
+	EQUALITY caseIgnoreMatch
+	SUBSTR caseIgnoreSubstringsMatch
+	SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{32768} )
+
+attributetype ( 1.3.6.1.4.1.15347.2.111 NAME 'mailaccess'
+	DESC 'Can be mailed to restricted groups'
+	EQUALITY caseIgnoreMatch
+	SUBSTR caseIgnoreSubstringsMatch
+	SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{32768} )
+
+attributetype ( 1.3.6.1.4.1.15347.2.100
+	NAME ( 'maildrop' )
+	DESC 'RFC1274: RFC822 Mailbox'
+	EQUALITY caseIgnoreIA5Match
+	SUBSTR caseIgnoreIA5SubstringsMatch
+	SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} )
+
+attributetype ( 1.3.6.1.4.1.10018.1.1.1 NAME 'mailbox'
+	DESC 'The absolute path to the mailbox for a mail account in a non-default location'
+	EQUALITY caseExactIA5Match
+	SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 SINGLE-VALUE )
+
+objectclass ( 1.3.6.1.4.1.15347.2.1
+	NAME 'mailUser'
+    	DESC 'E-Mail User'
+	SUP top
+	AUXILIARY
+	MUST ( uid $ mail $ maildrop )
+	MAY ( cn $ mailbox $ maildest $ mailaccess )
+	)
+
+objectclass ( 1.3.6.1.4.1.15347.2.2
+	NAME 'mailGroup'
+    	DESC 'E-Mail Group'
+	SUP top
+	STRUCTURAL
+	MUST ( cn $ mail )
+	MAY ( mailRoutingAddress $ member $ description )
+	)
+
+objectclass ( 1.3.6.1.4.1.15347.2.3
+	NAME 'transportTable'
+    	DESC 'MTA Transport Table'
+	SUP top
+	STRUCTURAL
+	MUST ( cn $ transport )
+	)
+

conf/slapd-logging.conf

@@ -0,0 +1,2 @@
+local4.*			-/var/log/ldap/slapd.log
+& stop

management/auth.py

@@ -3,7 +3,7 @@ import base64, os, os.path, hmac
 from flask import make_response
 
 import utils
-from mailconfig import get_mail_password, get_mail_user_privileges
+from mailconfig import validate_login, get_mail_password, get_mail_user_privileges
 
 DEFAULT_KEY_PATH   = '/var/lib/mailinabox/api.key'
 DEFAULT_AUTH_REALM = 'Mail-in-a-Box Management Server'
@@ -96,19 +96,7 @@ class KeyAuthService:
 		else:
 			# Get the hashed password of the user. Raise a ValueError if the
 			# email address does not correspond to a user.
-			pw_hash = get_mail_password(email, env)
-
-			# Authenticate.
-			try:
-				# Use 'doveadm pw' to check credentials. doveadm will return
-				# a non-zero exit status if the credentials are no good,
-				# and check_call will raise an exception in that case.
-				utils.shell('check_call', [
-					"/usr/bin/doveadm", "pw",
-					"-p", pw,
-					"-t", pw_hash,
-					])
-			except:
+			if not validate_login(email, pw, env):
 				# Login failed.
 				raise ValueError("Invalid password.")
 
@@ -130,7 +118,7 @@ class KeyAuthService:
 		# a user's password is reset, the HMAC changes and they will correctly need to log
 		# in to the control panel again. This method raises a ValueError if the user does
 		# not exist, due to get_mail_password.
-		msg = b"AUTH:" + email.encode("utf8") + b" " + get_mail_password(email, env).encode("utf8")
+		msg = b"AUTH:" + email.encode("utf8") + b" " + ";".join(get_mail_password(email, env)).encode("utf8")
 		return hmac.new(self.key.encode('ascii'), msg, digestmod="sha256").hexdigest()
 
 	def _generate_key(self):

management/backend.py

@@ -0,0 +1,294 @@
+#!/usr/local/lib/mailinabox/env/bin/python
+# -*- indent-tabs-mode: t; tab-width: 4; python-indent-offset: 4; -*-
+
+import ldap3, time
+
+
+class Response:
+	#
+	# helper class for iterating over ldap search results
+	# example:
+	#    conn = connect()
+	#    response = conn.wait( conn.search(...) )
+	#    for record in response:
+	#        print(record['dn'])
+	#
+	rowidx=0
+	def __init__(self, result, response):
+		self.result = result
+		self.response = response
+		if result is None:
+			raise ValueError("INVALID RESULT: None")
+		
+	def __iter__(self):
+		self.rowidx=0
+		return self
+	
+	def __next__(self):
+		# return the next record in the result set
+		rtn = self.next()
+		if rtn is None:	raise StopIteration
+		return rtn
+	
+	def next(self):
+		# return the next record in the result set or None
+		if self.rowidx >= len(self.response):
+			return None
+		rtn=self.response[self.rowidx]['attributes']
+		rtn['dn'] = self.response[self.rowidx]['dn']
+		self.rowidx += 1
+		
+		return rtn
+	
+	def count(self):
+		# return the number of records in the result set
+		return len(self.response)
+		
+
+
+class PagedSearch:
+	#
+	# Helper class for iterating over and handling paged searches.
+	# Use PagedSearch when expecting a large number of matching
+	# entries. slapd by default limits each result set to 500 entries.
+	#
+	# example:
+	#  conn=connection()
+	#  response = conn.paged_search(...)
+	#  for record in response:
+	#     print(record['dn'])
+	#
+	# PagedSearch is limited to one iteration pass. In the above
+	# example the 'for' statement could not be repeated.
+	#
+
+	def __init__(self, conn, search_base, search_filter, search_scope, attributes, page_size):
+		self.conn = conn
+		self.search_base = search_base
+		self.search_filter = search_filter
+		self.search_scope = search_scope
+		self.attributes = attributes
+		self.page_size = page_size
+
+		# issue the search
+		self.response = None
+		self.id = conn.search(search_base, search_filter, search_scope, attributes=attributes, paged_size=page_size, paged_criticality=True)
+		self.page_count = 0
+		
+	def __iter__(self):
+		# wait for the search result on first iteration
+		self.response = self.conn.wait(self.id)
+		self.page_count += 1
+		return self
+
+	def __next__(self):
+		# return the next record in the result set
+		r = self.response.next()
+		if r is None:
+			cookie=self.response.result['controls']['1.2.840.113556.1.4.319']['value']['cookie']
+			if not cookie:
+				raise StopIteration
+			self.id = self.conn.search(self.search_base, self.search_filter, self.search_scope, attributes=self.attributes, paged_size=self.page_size, paged_cookie=cookie)
+			self.response = self.conn.wait(self.id)
+			self.page_count += 1
+			r = self.response.next()
+			if r is None:
+				raise StopIteration
+		return r
+
+	def abandon(self):
+		# "If you send 0 as paged_size and a valid cookie the search
+		# operation referred by that cookie is abandoned."
+		cookie=self.response.result['controls']['1.2.840.113556.1.4.319']['value']['cookie']
+		if not cookie: return
+		self.id = self.conn.search(self.search_base, self.search_filter, self.search_scope, attributes=self.attributes, paged_size=0, paged_cookie=cookie)
+		
+
+
+
+class LdapConnection(ldap3.Connection):
+	# This is a subclass ldap3.Connection with our own methods for
+	# simplifying searches and modifications
+	
+	def wait(self, id):
+		# Wait for results from an async search and return the result
+		# set in a Response object.  If a syncronous strategy is in
+		# use, it returns immediately with the Response object.
+		if type(id)==int:
+			# async
+			tup = self.get_response(id)
+			return Response(tup[1], tup[0])
+		else:
+			# sync - conn has returned results
+			return Response(self.result, self.response)
+
+	def paged_search(self, search_base, search_filter, search_scope=ldap3.SUBTREE, attributes=None, page_size=200):
+		# Perform a paged search - see PagedSearch above
+		return PagedSearch(self,
+				   search_base,
+				   search_filter,
+				   search_scope,
+				   attributes,
+				   page_size)
+
+	def add(self, dn, object_class=None, attrs=None, controls=None):
+		# This overrides ldap3's add method to automatically remove
+		# empty attributes from the attribute list for a ldap ADD
+		# operation, which cause exceptions.
+		if attrs is not None:
+			keys = [ k for k in attrs ]
+			for k in keys:
+				if attrs[k] is None or \
+				   type(attrs[k]) is list and len(attrs[k])==0 or \
+				   type(attrs[k]) is str and attrs[k]=='':
+					del attrs[k]
+		return ldap3.Connection.add(self, dn, object_class, attrs, controls)
+
+	def chase_members(self, members, attrib, env):
+		# Given a list of distinguished names (members), lookup each
+		# one and return the list of values for `attrib` in an
+		# array. It is *not* recursive. `attrib` is a string holding
+		# the name of an attribute.
+		resolved=[]
+		for dn in members:
+			try:
+				response = self.wait(self.search(dn, "(objectClass=*)", ldap3.BASE, attributes=attrib))
+				rec = response.next()
+				if rec is not None: resolved.append(rec[attrib])
+			except ldap3.core.exceptions.LDAPNoSuchObjectResult:
+				# ignore orphans
+				pass
+		return resolved
+
+	# static
+	def ldap_modify_op(attr, record, new_values):
+		# Return an ldap operation to change record[attr] so that it
+		# has values `new_values`. `new_values` is a list.
+		
+		if not type(new_values) is list:
+			new_values = [ new_values ]
+		# remove None values
+		new_values = [ v for v in new_values if v is not None ]
+
+		if len(record[attr]) == 0:
+			if len(new_values) == 0:
+				# NOP: current=empty, desired=empty
+				return None
+			# ADD: current=empty, desired=non-empty
+			return [(ldap3.MODIFY_ADD, new_values)]
+		if len(new_values) == 0:
+			# DELETE: current=non-empty, desired=empty
+			return [(ldap3.MODIFY_DELETE, [])]
+		# MODIFY: current=non-empty, desired=non-empty
+		return [(ldap3.MODIFY_REPLACE, new_values)]
+
+
+
+	def add_or_modify(self, dn, existing_record, attrs_to_update, objectClasses, values):
+		# Add or modify an existing database entry.
+		#   dn: dn for a new entry
+		#   existing_record: the existing data, if any
+		#         (a dict from Response.next()). The data must
+		#         have values for each attribute in `attrs_to_update`
+		#   attrs_to_update: an array of attribute names to update
+		#   objectClasses: a list of object classes for a new entry
+		#   values: a dict of attributes and values for a new entry
+		if existing_record:
+			# modify existing
+			changes = {}
+			dn = existing_record['dn']
+			for attr in attrs_to_update:
+				modify_op = LdapConnection.ldap_modify_op(
+					attr,
+					existing_record,
+					values[attr])
+				if modify_op: changes[attr] = modify_op
+			self.wait ( self.modify(dn, changes) )
+			return 'modify'
+		else:
+			# add new alias
+			self.wait ( self.add(dn, objectClasses, values) )
+			return 'add'
+
+		
+	def modify_record(self, rec, modifications):
+		# Modify an existing record by changing the attributes of
+		# `modifications` to their associated values. `modifications`
+		# is a dict with key of attribute name and value of list of
+		# string.
+		dn = rec['dn']
+		attrs_to_update=modifications.keys()
+		self.add_or_modify(dn, rec, attrs_to_update, None, modifications)
+		return True
+	
+
+def get_shadowLastChanged():
+	# get the number of days from the epoch
+	days = int(time.time() / (24 * 60 * 60))
+	return days
+
+
+def get_ldap_server(env):
+	# return a ldap3.Server object for a ldap connection
+	tls=None
+	if env.LDAP_SERVER_TLS=="yes":
+		import ssl
+		tls=ldap3.Tls(
+			validate=ssl.CERT_REQUIRED,
+			ca_certs_file="/etc/ssl/certs/ca-certificates.crt")
+
+	server=ldap3.Server(
+		host=env.LDAP_SERVER,
+		port=int(env.LDAP_SERVER_PORT),
+		use_ssl=False if tls is None else True,
+		get_info=ldap3.NONE,
+		tls=tls)
+	
+	return server
+
+
+def connect(env):
+	# connect to the ldap server
+	#
+	# problems and observations:
+	#
+	# using thread-local storage does not work with Flask.  nor does
+	# REUSABLE strategy - both fail with LDAPResponseTimeoutError
+	#
+	# there are a lot of connections left open with ASYNC strategy
+	#
+	# there are more unclosed connections with manual bind vs auto
+	# bind
+	#
+	# pooled connections only work for REUSABLE strategy
+	#
+	# paging does not work with REUSABLE strategy at all:
+	# get_response() always returns a protocol error (invalid cookie)
+	# when retrieving the second page
+	#
+	server = get_ldap_server(env)
+
+	auto_bind=ldap3.AUTO_BIND_NO_TLS
+	if env.LDAP_SERVER_STARTTLS=="yes":
+		auto_bind=ldap3.AUTO_BIND_TLS_BEFORE_BIND
+		
+	conn = LdapConnection(
+		server,
+		env.LDAP_MANAGEMENT_DN,
+		env.LDAP_MANAGEMENT_PASSWORD,
+		auto_bind=auto_bind,
+		lazy=False,
+		#client_strategy=ldap3.ASYNC, # ldap3.REUSABLE,
+		client_strategy=ldap3.SYNC,
+		#pool_name="default",
+		#pool_size=5,
+		#pool_lifetime=20*60,  # 20 minutes
+		#pool_keepalive=60,
+		raise_exceptions=True)
+	
+	#conn.bind()
+	return conn
+
+
+
+

management/backup.py

@@ -1,4 +1,5 @@
 #!/usr/local/lib/mailinabox/env/bin/python
+# -*- indent-tabs-mode: t; tab-width: 4; python-indent-offset: 4; -*-
 
 # This script performs a backup of all user data:
 # 1) System services are stopped.
@@ -250,6 +251,7 @@ def perform_backup(full_backup):
 	service_command("php7.2-fpm", "stop", quit=True)
 	service_command("postfix", "stop", quit=True)
 	service_command("dovecot", "stop", quit=True)
+	service_command("slapd", "stop", quit=True)
 
 	# Execute a pre-backup script that copies files outside the homedir.
 	# Run as the STORAGE_USER user, not as root. Pass our settings in
@@ -279,6 +281,7 @@ def perform_backup(full_backup):
 			get_env(env))
 	finally:
 		# Start services again.
+		service_command("slapd", "start", quit=False)
 		service_command("dovecot", "start", quit=False)
 		service_command("postfix", "start", quit=False)
 		service_command("php7.2-fpm", "start", quit=False)

management/dns_update.py

@@ -22,7 +22,8 @@ def get_dns_domains(env):
 	# Add all domain names in use by email users and mail aliases and ensure
 	# PRIMARY_HOSTNAME is in the list.
 	domains = set()
-	domains |= get_mail_domains(env)
+	domains |= get_mail_domains(env, category="mail")
+	domains |= get_mail_domains(env, category="ssl")
 	domains.add(env['PRIMARY_HOSTNAME'])
 	return domains
 

management/mail_log.py

@@ -115,8 +115,10 @@ def scan_mail_log(env):
 
     try:
         import mailconfig
-        collector["known_addresses"] = (set(mailconfig.get_mail_users(env)) |
-                                        set(alias[0] for alias in mailconfig.get_mail_aliases(env)))
+        users = mailconfig.get_mail_users(env, as_map=True)
+        aliases = mailconfig.get_mail_aliases(env, as_map=True)
+        collector["known_addresses"] = (set(users.keys()) |
+                                        set(aliases.keys()))
     except ImportError:
         pass
 

management/mailconfig.py

@@ -1,4 +1,5 @@
 #!/usr/local/lib/mailinabox/env/bin/python
+# -*- indent-tabs-mode: t; tab-width: 4; python-indent-offset: 4; -*-
 
 # NOTE:
 # This script is run both using the system-wide Python 3
@@ -9,11 +10,51 @@
 # Python 3 in setup/questions.sh to validate the email
 # address entered by the user.
 
-import subprocess, shutil, os, sqlite3, re
-import utils
+import subprocess, shutil, os, sqlite3, re, ldap3, uuid
+import utils, backend
 from email_validator import validate_email as validate_email_, EmailNotValidError
 import idna
 
+
+#
+# LDAP notes:
+#
+#    Users have an objectClass of mailUser with a mail and maildrop
+#    attribute for the email address. For historical reasons, the
+#    management interface only permits lowercase email addresses.
+#
+#    In the current implementation, both attributes will have the same
+#    lowercase value, but it's not a requirement of the underlying
+#    postfix and dovecot configurations that it be this way. Postfix
+#    and dovecot use the mail attribute to find the user and maildrop
+#    is where the mail is delivered. We use maildrop in the management
+#    interface because that's the address that will always map
+#    directly to the filesystem for archived users.
+#
+#    Email addresses and domain comparisons performed by the LDAP
+#    server are not case sensitive because their respective schemas
+#    define a case-insensitive comparison for those attributes.
+#
+#    User privileges are maintained in the mailaccess attribute of
+#    users.
+#
+#    Aliases and permitted-senders are separate entities in the LDAP
+#    database, but both are of objectClass mailGroup with a
+#    single-valued mail attribute. Alias addresses are forced to
+#    lowercase, again for historical reasons.
+#
+#    All alias and permitted-sender email addresses in the database
+#    are IDNA encoded. International domains for users is not
+#    supported or permitted.
+#
+#    Domains that are handled by this mail server are maintained
+#    on-the-fly as users are added and deleted. They have an
+#    objectClass of domain with attribute dc.
+#
+#    LDAP "records" in this code are dictionaries containing the
+#    attributes and distinguished name of the entry.
+#
+
 def validate_email(email, mode=None):
 	# Checks that an email address is syntactically valid. Returns True/False.
 	# Until Postfix supports SMTPUTF8, an email address may contain ASCII
@@ -91,20 +132,90 @@ def is_dcv_address(email):
 			return True
 	return False
 
-def open_database(env, with_connection=False):
-	conn = sqlite3.connect(env["STORAGE_ROOT"] + "/mail/users.sqlite")
-	if not with_connection:
-		return conn.cursor()
+def open_database(env):
+		return backend.connect(env)
+
+def find_mail_user(env, email, attributes=None, conn=None):
+	# Find the user with the given email address and return the ldap
+	# record for it.
+	#
+	# email is the users email address
+	# attributes are a list of attributes to return eg ["mail","maildrop"]
+	# conn is a ldap database connection, if not specified a new one
+	# is established	
+	#
+	# The ldap record for the user is returned or None if not found.
+	if not conn: conn = open_database(env)
+	id=conn.search(env.LDAP_USERS_BASE,
+		       "(&(objectClass=mailUser)(maildrop=%s))" % email,
+		       attributes=attributes)
+	response = conn.wait(id)
+	if response.count() > 1:
+		dns = [ rec['dn'] for rec in response ]
+		raise LookupError("Detected more than one user with the same email address (%s): %s" % (email, ";".join(dns)))
 	else:
-		return conn, conn.cursor()
+		return response.next()
+	
+def find_mail_alias(env, email, attributes=None, conn=None):
+	# Find the alias with the given address and return the ldap
+	# records for it and the associated permitted senders (if one).
+	#
+	# email is the alias address. It must be IDNA encoded.
+	#
+	# attributes are a list of attributes to return, eg
+	# ["member","rfc822MailMember"]
+	#
+	# conn is a ldap database connection, if not specified a new one
+	# is established.
+	#
+	# A tuple having the two ldap records for the alias and it's
+	# permitted senders (alias, permitted_senders) is returned. If
+	# either is not found, the corresponding tuple value will be None.
+	# 
+	if not conn: conn = open_database(env)
+	# get alias
+	id=conn.search(env.LDAP_ALIASES_BASE,
+		       "(&(objectClass=mailGroup)(mail=%s))" % email,
+		       attributes=attributes)
+	response = conn.wait(id)
+	if response.count() > 1:
+		dns = [ rec['dn'] for rec in response ]
+		raise LookupError("Detected more than one alias with the same email address (%s): %s" % (email, ";".join(dns)))
+	alias = response.next()
 
-def get_mail_users(env):
-	# Returns a flat, sorted list of all user accounts.
+	# get permitted senders for alias
+	id=conn.search(env.LDAP_PERMITTED_SENDERS_BASE,
+		       "(&(objectClass=mailGroup)(mail=%s))" % email,
+		       attributes=attributes)
+	response = conn.wait(id)
+	if response.count() > 1:
+		dns = [ rec['dn'] for rec in response ]
+		raise LookupError("Detected more than one permitted senders group with the same email address (%s): %s" % (email, ";".join(dns)))
+	permitted_senders = response.next()
+	return (alias, permitted_senders)
+	
+
+def get_mail_users(env, as_map=False):
+	# When `as_map` is False, this function returns a flat, sorted
+	# array of all user accounts. If True, it returns a dict where key
+	# is the user and value is a dict having, dn, maildrop and
+	# mail addresses
 	c = open_database(env)
-	c.execute('SELECT email FROM users')
-	users = [ row[0] for row in c.fetchall() ]
-	return utils.sort_email_addresses(users, env)
+	pager = c.paged_search(env.LDAP_USERS_BASE, "(objectClass=mailUser)", attributes=['maildrop','mail'])
+	if as_map:
+		users = {}
+		for rec in pager:
+			users[rec['maildrop'][0]] = {
+				"dn":   rec['dn'],
+				"mail": rec['mail'],
+				"maildrop": rec['maildrop'][0]
+			}
+		return users
+	else:
+		users = [ rec['maildrop'][0] for rec in pager ]
+		return utils.sort_email_addresses(users, env)
 
+	
 def get_mail_users_ex(env, with_archived=False):
 	# Returns a complex data structure of all user accounts, optionally
 	# including archived (status="inactive") accounts.
@@ -128,13 +239,15 @@ def get_mail_users_ex(env, with_archived=False):
 	users = []
 	active_accounts = set()
 	c = open_database(env)
-	c.execute('SELECT email, privileges FROM users')
-	for email, privileges in c.fetchall():
-		active_accounts.add(email)
+	response = c.wait( c.search(env.LDAP_USERS_BASE, "(objectClass=mailUser)", attributes=['maildrop','mailaccess']) )
 
+	for rec in response:
+		email = rec['maildrop'][0]
+		privileges = rec['mailaccess']
+		active_accounts.add(email)
 		user = {
 			"email": email,
-			"privileges": parse_privs(privileges),
+			"privileges": privileges,
 			"status": "active",
 		}
 		users.append(user)
@@ -179,21 +292,82 @@ def get_mail_users_ex(env, with_archived=False):
 def get_admins(env):
 	# Returns a set of users with admin privileges.
 	users = set()
-	for domain in get_mail_users_ex(env):
-		for user in domain["users"]:
-			if "admin" in user["privileges"]:
-				users.add(user["email"])
+	c = open_database(env)
+	response = c.wait( c.search(env.LDAP_USERS_BASE, "(&(objectClass=mailUser)(mailaccess=admin))", attributes=['maildrop']) )
+	for rec in response:
+		users.add(rec['maildrop'][0])
 	return users
 
-def get_mail_aliases(env):
-	# Returns a sorted list of tuples of (address, forward-tos, permitted-senders).
-	c = open_database(env)
-	c.execute('SELECT source, destination, permitted_senders FROM aliases')
-	aliases = { row[0]: row for row in c.fetchall() } # make dict
 
-	# put in a canonical order: sort by domain, then by email address lexicographically
-	aliases = [ aliases[address] for address in utils.sort_email_addresses(aliases.keys(), env) ]
-	return aliases
+def get_mail_aliases(env, as_map=False):
+	# Retrieve all mail aliases.
+	#
+	# If as_map is False, the function returns a sorted array of tuples:
+	#
+	#   (address(lowercase), forward-tos{string,csv}, permitted-senders{string,csv})
+	#
+	# If as-map is True, it returns a dict whose keys are
+	# address(lowercase) and whose values are:
+	#
+	#    { dn: {string},
+	#      mail: {string}
+	#      forward_tos: {array of string},
+	#      permited_senders: {array of string}
+	#    }
+	#
+	c = open_database(env)
+	# get all permitted senders
+	pager = c.paged_search(env.LDAP_PERMITTED_SENDERS_BASE, "(objectClass=mailGroup)", attributes=["mail", "member"])
+	
+	# make a dict of permitted senders, key=mail(lowercase) value=members
+	permitted_senders = { rec["mail"][0].lower(): rec["member"] for rec in pager }
+	
+	# get all alias groups
+	pager = c.paged_search(env.LDAP_ALIASES_BASE, "(objectClass=mailGroup)", attributes=['mail','member','rfc822MailMember'])
+	
+	# make a dict of aliases
+	# key=email(lowercase), value=(email, forward-tos, permitted-senders).
+	aliases = {}
+	for alias in pager:
+		alias_email = alias['mail'][0]
+		alias_email_lc = alias_email.lower()
+		
+		# chase down each member's email address, because a member is a dn
+		forward_tos = []
+		for fwd_to in c.chase_members(alias['member'], 'mail', env):
+			forward_tos.append(fwd_to[0])
+			
+		for fwd_to in alias['rfc822MailMember']:
+			forward_tos.append(fwd_to)
+		
+		# chase down permitted senders' email addresses
+		allowed_senders = []
+		if alias_email_lc in permitted_senders:
+			members = permitted_senders[alias_email_lc]
+			for mail_list in c.chase_members(members, 'mail', env):
+				for mail in mail_list:
+					allowed_senders.append(mail)
+
+		aliases[alias_email_lc] = {
+			"dn": alias["dn"],
+			"mail": alias_email,
+			"forward_tos": forward_tos,
+			"permitted_senders": allowed_senders
+		}
+
+	if not as_map:
+		# put in a canonical order: sort by domain, then by email address lexicographically
+		list = []
+		for address in utils.sort_email_addresses(aliases.keys(), env):
+			alias = aliases[address]
+			xft = ",".join(alias["forward_tos"])
+			xas = ",".join(alias["permitted_senders"])
+			list.append( (address, xft, xas) )
+		return list
+	
+	else:
+		return aliases
+			
 
 def get_mail_aliases_ex(env):
 	# Returns a complex data structure of all mail aliases, similar
@@ -258,15 +432,174 @@ def get_domain(emailaddr, as_unicode=True):
 			pass
 	return ret
 
-def get_mail_domains(env, filter_aliases=lambda alias : True):
-	# Returns the domain names (IDNA-encoded) of all of the email addresses
-	# configured on the system.
-	return set(
-		   [get_domain(login, as_unicode=False) for login in get_mail_users(env)]
-		 + [get_domain(address, as_unicode=False) for address, *_ in get_mail_aliases(env) if filter_aliases(address) ]
-		 )
+def get_mail_domains(env, as_map=False, filter_aliases=None, category=None):
+	# Retrieves all domains, IDNA-encoded, we accept mail for.
+	#
+	# If as_map is False, the function returns the lowercase domain
+	# names (IDNA-encoded) as an array.
+	#
+	# If as_map is True, it returns a dict whose keys are
+	# domain(idna,lowercase) and whose values are:
+	#
+	#   { dn:{string}, domain:{string(idna)} }
+	#
+	# filter_aliases is an inclusion filter - a True return value from
+	# this lambda function indicates an alias to include.
+	#
+	# With filter_aliases, there has to be at least one user or
+	# filtered (included) alias in the domain for the domain to be
+	# part of the returned set. If None, all domains are returned.
+	#
+	# category is another type of filter. Set to a string value to
+	# return only those domains of that category. ie. the
+	# "businessCategory" attribute of the domain must include this
+	# category.
+	#
+	conn = open_database(env)
+	filter = "(&(objectClass=domain)(businessCategory=mail))"
+	if category:
+		filter = "(&(objectClass=domain)(businessCategory=%s))" % category
+	id = conn.search(env.LDAP_DOMAINS_BASE, filter, attributes="dc")
+	response = conn.wait(id)
+	filter_candidates=[]
+	domains=None
+	if as_map:
+		domains = {}
+		for rec in response:
+			domains[ rec["dc"][0].lower() ] = {
+				"dn": rec["dn"],
+				"domain": rec["dc"][0]
+			}
+			if filter_aliases: filter_candidates.append(rec['dc'][0].lower())
+	else:
+		domains = set([ rec["dc"][0].lower() for rec in response ])
+		if filter_aliases: filter_candidates += domains
+	
+	for candidate in filter_candidates:
+		# with the filter, there has to be at least one user or
+		# filtered (included) alias in the domain for the domain to be
+		# part of the returned set
+
+		# any users ?
+		response = conn.wait( conn.search(env.LDAP_USERS_BASE, "(&(objectClass=mailUser)(mail=*@%s))" % candidate, size_limit=1) )
+		if response.next():
+			# yes, that domain needs to be in the returned set
+			continue
+
+		# any filtered aliases ?
+		pager = conn.paged_search(
+			env.LDAP_ALIASES_BASE,
+			"(&(objectClass=mailGroup)(mail=*@%s))" % candidate,
+			attributes=['mail'])
+
+		remove = True
+		for rec in pager:
+			if filter_aliases(rec['mail'][0]):
+				remove = False
+				pager.abandon()
+				break
+
+		if remove:
+			if as_map: del domains[candidate]
+			else: domains.remove(candidate)
+			
+	return domains
+
+
+def add_mail_domain(env, domain, validate=True):
+	# Create domain entry indicating that we are handling
+	# mail for that domain.
+	#
+	# We only care about domains for users, not for aliases.
+	#
+	# domain: IDNA encoded domain name.
+	# validate: If True, ensure there is at least one user on the
+	# system using that domain.
+	#
+	# returns True if added, False if it already exists or fails
+	# validation.
+	
+	conn = open_database(env)
+	if validate:
+		# check to ensure there is at least one user with that domain
+		id = conn.search(env.LDAP_USERS_BASE,
+						 "(&(objectClass=mailUser)(mail=*@%s))" % domain,
+						 size_limit=1)
+		if conn.wait(id).count() == 0:
+			# no mail users are using that domain!
+			return False
+		
+	dn = 'dc=%s,%s' % (domain, env.LDAP_DOMAINS_BASE)
+	try:
+		response = conn.wait( conn.add(dn, [ 'domain' ], {
+			"businessCategory": "mail"
+		}) )
+		return True
+	except ldap3.core.exceptions.LDAPEntryAlreadyExistsResult:
+		try:
+			# add 'mail' as a businessCategory
+			changes = {	"businessCategory": [(ldap3.MODIFY_ADD, ['mail'])] }
+			response = conn.wait ( conn.modify(dn, changes) )
+		except ldap3.core.exceptions.LDAPAttributeOrValueExistsResult:
+			pass
+		return False
+
+	
+def remove_mail_domain(env, domain, validate=True):
+	# Remove the specified domain from the list of domains that we
+	# handle mail for. The domain must be IDNA encoded.
+	#
+	# If validate is True, ensure there are no valid users on the system
+	# currently using the domain.
+	#
+	# Returns True if removed or does not exist, False if the domain
+	# fails validation.
+	conn = open_database(env)
+	if validate:
+		# check to ensure no users are using the domain
+		id = conn.search(env.LDAP_USERS_BASE,
+						 "(&(objectClass=mailUser)(mail=*@%s))" % domain,
+						 size_limit=1)
+		if conn.wait(id).count() > 0:
+			# there is one or more mailUser's with that domain
+			return False
+		
+	id = conn.search(env.LDAP_DOMAINS_BASE,
+					 "(&(objectClass=domain)(dc=%s))" % domain,
+					 attributes=['businessCategory'])
+	
+	existing = conn.wait(id).next()
+	if existing is None:
+		# the domain doesn't exist!
+		return True
+
+	newvals=existing['businessCategory'].copy()
+	if 'mail' in newvals:
+		newvals.remove('mail')
+	else:
+		# we only remove mail-related entries
+		return False
+
+	if len(newvals)==0:
+		conn.wait ( conn.delete(existing['dn']) )
+	else:
+		conn.wait ( conn.modify_record(existing, {'businessCategory', newvals}))
+	return True
+
 
 def add_mail_user(email, pw, privs, env):
+	# Add a new mail user.
+	#
+	# email: the new user's email address
+	# pw: the new user's password
+	# privs: either an array of privilege strings, or a newline
+	# separated string of privilege names
+	#
+	# If an error occurs, the function returns a tuple of (message,
+	# http-status).
+	#
+	# If successful, the string "OK" is returned.
+	
 	# validate email
 	if email.strip() == "":
 		return ("No email address provided.", 400)
@@ -284,73 +617,131 @@ def add_mail_user(email, pw, privs, env):
 	validate_password(pw)
 
 	# validate privileges
-	if privs is None or privs.strip() == "":
-		privs = []
-	else:
-		privs = privs.split("\n")
-		for p in privs:
-			validation = validate_privilege(p)
-			if validation: return validation
+	privs = []
+	if privs is not None and type(privs) is str and privs.strip() != "":
+		privs = parse_privs(privs)
+	for p in privs:
+		validation = validate_privilege(p)
+		if validation: return validation
 
 	# get the database
-	conn, c = open_database(env, with_connection=True)
+	conn = open_database(env)
 
-	# hash the password
-	pw = hash_password(pw)
+	# ensure another user doesn't have that address
+	id=conn.search(env.LDAP_USERS_BASE, "(&(objectClass=mailUser)(|(mail=%s)(maildrop=%s)))" % (email, email))
+	if conn.wait(id).count() > 0:
+		return ("User alreay exists.", 400)
+
+	# ensure an alias doesn't have that address
+	id=conn.search(env.LDAP_ALIASES_BASE, "(&(objectClass=mailGroup)(mail=%s))" % email)
+	if conn.wait(id).count() > 0:
+		return ("An alias exists with that address.", 400)
+	
+	# Generate a unique id for uid
+	uid = '%s' % uuid.uuid4()
+
+	# choose a common name and surname (required attributes)
+	cn = email.split("@")[0].replace('.',' ').replace('_',' ')
+	sn = cn[cn.find(' ')+1:]
+	
+	# compile user's attributes
+	# for historical reasons, make the email address lowercase
+	attrs = {
+		"mail" : email,
+		"maildrop" : email.lower(),
+		"uid" : uid,
+		"mailaccess": privs,
+		"cn": cn,
+		"sn": sn,
+		"shadowLastChange": backend.get_shadowLastChanged()
+	}
 
 	# add the user to the database
-	try:
-		c.execute("INSERT INTO users (email, password, privileges) VALUES (?, ?, ?)",
-			(email, pw, "\n".join(privs)))
-	except sqlite3.IntegrityError:
-		return ("User already exists.", 400)
+	dn = 'uid=%s,%s' % (uid, env.LDAP_USERS_BASE)
+	id=conn.add(dn, [ 'inetOrgPerson','mailUser','shadowAccount' ], attrs);
+	conn.wait(id)
 
-	# write databasebefore next step
-	conn.commit()
+	# set the password - the ldap server will hash it
+	conn.extend.standard.modify_password(user=dn, new_password=pw)
 
+	# tell postfix the domain is local, if needed
+	add_mail_domain(env, get_domain(email, as_unicode=False), validate=False)
+
+	# convert alias's rfc822MailMember to member
+	convert_rfc822MailMember(env, conn, dn, email)
+	
 	# Update things in case any new domains are added.
 	return kick(env, "mail user added")
 
 def set_mail_password(email, pw, env):
-	# validate that password is acceptable
+	# validate that the password is acceptable
 	validate_password(pw)
 
-	# hash the password
-	pw = hash_password(pw)
-
-	# update the database
-	conn, c = open_database(env, with_connection=True)
-	c.execute("UPDATE users SET password=? WHERE email=?", (pw, email))
-	if c.rowcount != 1:
+	# find the user				  
+	conn = open_database(env)
+	user = find_mail_user(env, email, ['shadowLastChange'], conn)
+	if user is None:
 		return ("That's not a user (%s)." % email, 400)
-	conn.commit()
+
+	# update the database - the ldap server will hash the password
+	conn.extend.standard.modify_password(user=user['dn'], new_password=pw)
+
+	# update shadowLastChange
+	conn.modify_record(user, {'shadowLastChange': backend.get_shadowLastChanged()})
+
 	return "OK"
 
-def hash_password(pw):
-	# Turn the plain password into a Dovecot-format hashed password, meaning
-	# something like "{SCHEME}hashedpassworddata".
-	# http://wiki2.dovecot.org/Authentication/PasswordSchemes
-	return utils.shell('check_output', ["/usr/bin/doveadm", "pw", "-s", "SHA512-CRYPT", "-p", pw]).strip()
+def validate_login(email, pw, env):
+	# Validate that `email` exists and has password `pw`.
+	# Returns True if valid, or False if invalid.
+	user = find_mail_user(env, email)
+	if user is None:
+		raise ValueError("That's not a user (%s)" % email)
+	try:
+		# connect as that user to validate the login
+		server = backend.get_ldap_server(env)
+		conn = ldap3.Connection(
+			server,
+			user=user['dn'],
+			password=pw,
+			raise_exceptions=True)
+		conn.bind()
+		conn.unbind()
+		return True
+	except ldap3.core.exceptions.LDAPInvalidCredentialsResult:
+		return False
+
 
 def get_mail_password(email, env):
-	# Gets the hashed password for a user. Passwords are stored in Dovecot's
-	# password format, with a prefixed scheme.
-	# http://wiki2.dovecot.org/Authentication/PasswordSchemes
-	# update the database
-	c = open_database(env)
-	c.execute('SELECT password FROM users WHERE email=?', (email,))
-	rows = c.fetchall()
-	if len(rows) != 1:
+	# Gets the hashed passwords for a user. In ldap, userPassword is
+	# multi-valued and each value can have different hash. This
+	# function returns all hashes as an array.
+	user = find_mail_user(env, email, attributes=["userPassword"])
+	if user is None:
 		raise ValueError("That's not a user (%s)." % email)
-	return rows[0][0]
+	if len(user['userPassword'])==0:
+		raise ValueError("The user has no password (%s)" % email)
+	return user['userPassword']
+
 
 def remove_mail_user(email, env):
-	# remove
-	conn, c = open_database(env, with_connection=True)
-	c.execute("DELETE FROM users WHERE email=?", (email,))
-	if c.rowcount != 1:
+	# Remove the user as a valid user of the system.
+	# If an error occurs, the function returns a tuple of (message,
+	# http-status).
+	#
+	# If successful, the string "OK" is returned.
+	conn = open_database(env)
+	
+	# find the user
+	user = find_mail_user(env, email, conn=conn)
+	if user is None:
 		return ("That's not a user (%s)." % email, 400)
-	conn.commit()
+	
+	# delete the user
+	conn.wait( conn.delete(user['dn']) )
+
+	# remove as a handled domain, if needed
+	remove_mail_domain(env, get_domain(email, as_unicode=False))
 
 	# Update things in case any domains are removed.
 	return kick(env, "mail user removed")
@@ -359,14 +750,19 @@ def parse_privs(value):
 	return [p for p in value.split("\n") if p.strip() != ""]
 
 def get_mail_user_privileges(email, env, empty_on_error=False):
-	# get privs
+	# Get an array of privileges held by the specified user.
 	c = open_database(env)
-	c.execute('SELECT privileges FROM users WHERE email=?', (email,))
-	rows = c.fetchall()
-	if len(rows) != 1:
+	try:
+		user = find_mail_user(env, email, ['mailaccess'], c)
+	except LookupError as e:
+		if empty_on_error: return []
+		raise e
+	
+	if user is None:
 		if empty_on_error: return []
 		return ("That's not a user (%s)." % email, 400)
-	return parse_privs(rows[0][0])
+	
+	return user['mailaccess']
 
 def validate_privilege(priv):
 	if "\n" in priv or priv.strip() == "":
@@ -374,38 +770,93 @@ def validate_privilege(priv):
 	return None
 
 def add_remove_mail_user_privilege(email, priv, action, env):
+	# Add or remove a privilege from a user.
+	# priv: the name of the privilege to add or remove
+	# action: "add" to add the privilege, or "remove" to remove it
+	# email: the user
+	#
+	# If an error occurs, the function returns a tuple of (message,
+	# http-status).
+	#
+	# If successful, the string "OK" is returned.
+	
 	# validate
 	validation = validate_privilege(priv)
 	if validation: return validation
 
 	# get existing privs, but may fail
-	privs = get_mail_user_privileges(email, env)
-	if isinstance(privs, tuple): return privs # error
+	user = find_mail_user(env, email, attributes=['mailaccess'])
+	if user is None:
+		return ("That's not a user (%s)." % email, 400)
+		
+	privs = user['mailaccess'].copy()
 
 	# update privs set
+	changed = False
 	if action == "add":
 		if priv not in privs:
 			privs.append(priv)
+			changed = True
+			
 	elif action == "remove":
-		privs = [p for p in privs if p != priv]
+		if priv in privs:
+			privs.remove(priv)
+			changed = True
 	else:
 		return ("Invalid action.", 400)
 
 	# commit to database
-	conn, c = open_database(env, with_connection=True)
-	c.execute("UPDATE users SET privileges=? WHERE email=?", ("\n".join(privs), email))
-	if c.rowcount != 1:
-		return ("Something went wrong.", 400)
-	conn.commit()
-
+	if changed:
+		conn = open_database(env)
+		conn.modify_record( user, {'mailaccess': privs} )
+		
 	return "OK"
 
+
+
+def convert_rfc822MailMember(env, conn, dn, mail):
+	# if a newly added alias or user exists as an rfc822MailMember,
+	# convert it to a member dn
+	# the new alias or user is specified by arguments mail and dn.
+	# mail is the new alias or user's email address
+	# dn is the new alias or user's distinguished name
+	# conn is an existing ldap database connection
+	id=conn.search(env.LDAP_ALIASES_BASE,
+				   "(&(objectClass=mailGroup)(rfc822MailMember=%s))" % mail,
+				   attributes=[ 'member','rfc822MailMember' ])
+	response = conn.wait(id)
+	for rec in response:
+		# remove mail from rfc822MailMember
+		changes={ "rfc822MailMember": [(ldap3.MODIFY_DELETE, [mail])] }
+		conn.wait( conn.modify(rec['dn'], changes) )
+
+		# add dn to member
+		rec['member'].append(dn)
+		changes={ "member": [(ldap3.MODIFY_ADD, rec['member'])] }
+		try:
+			conn.wait( conn.modify(rec['dn'], changes) )
+		except ldap3.core.exceptions.LDAPAttributeOrValueExistsResult:
+			pass
+
+		
 def add_mail_alias(address, forwards_to, permitted_senders, env, update_if_exists=False, do_kick=True):
+	# Add a new alias group with permitted senders.
+	#
+	# address: the email address of the alias
+	# forwards_to: a string of newline and comma separated email address
+	# where mail is delivered
+	# permitted_senders: a string of newline and comma separated email addresses of local users that are permitted to MAIL FROM the alias.
+	# update_if_exists: if False and the alias exists fail, otherwise update the existing alias with the new values
+	#
+	# If an error occurs, the function returns a tuple of (message,
+	# http-status).
+	#
+	# If successful, the string "OK" is returned.
+
 	# convert Unicode domain to IDNA
 	address = sanitize_idn_email_address(address)
 
-	# Our database is case sensitive (oops), which affects mail delivery
-	# (Postfix always queries in lowercase?), so force lowercase.
+	# for historical reasons, force the address to lowercase
 	address = address.lower()
 
 	# validate address
@@ -415,8 +866,11 @@ def add_mail_alias(address, forwards_to, permitted_senders, env, update_if_exist
 	if not validate_email(address, mode='alias'):
 		return ("Invalid email address (%s)." % address, 400)
 
+	# retrieve all logins as a map ( email.lower() => {mail,dn} )
+	valid_logins = get_mail_users( env, as_map=True )
+	
 	# validate forwards_to
-	validated_forwards_to = []
+	validated_forwards_to = [ ]
 	forwards_to = forwards_to.strip()
 
 	# extra checks for email addresses used in domain control validation
@@ -451,9 +905,8 @@ def add_mail_alias(address, forwards_to, permitted_senders, env, update_if_exist
 				validated_forwards_to.append(email)
 
 	# validate permitted_senders
-	valid_logins = get_mail_users(env)
-	validated_permitted_senders = []
-	permitted_senders = permitted_senders.strip()
+	validated_permitted_senders = [ ]  # list of dns
+	permitted_senders = permitted_senders.strip( )
 
 	# Parse comma and \n-separated sender logins & validate. The permitted_senders must be
 	# valid usernames.
@@ -461,51 +914,111 @@ def add_mail_alias(address, forwards_to, permitted_senders, env, update_if_exist
 		for login in line.split(","):
 			login = login.strip()
 			if login == "": continue
-			if login not in valid_logins:
+			if login.lower() not in valid_logins:
 				return ("Invalid permitted sender: %s is not a user on this system." % login, 400)
-			validated_permitted_senders.append(login)
+			validated_permitted_senders.append(valid_logins[login.lower()]['dn'])
 
 	# Make sure the alias has either a forwards_to or a permitted_sender.
 	if len(validated_forwards_to) + len(validated_permitted_senders) == 0:
 		return ("The alias must either forward to an address or have a permitted sender.", 400)
 
+
+	# break validated_forwards_to into 'local' where an email
+	# address is a local user, or 'remote' where the email doesn't
+	# exist on this system
+
+	vfwd_tos_local = []   # list of dn's
+	vfwd_tos_remote = []  # list of email
+	for fwd_to in validated_forwards_to:
+		fwd_to_lc = fwd_to.lower()
+		if fwd_to_lc in valid_logins:
+			dn = valid_logins[fwd_to_lc]['dn']
+			vfwd_tos_local.append(dn)
+		else:
+			vfwd_tos_remote.append(fwd_to)
+			
 	# save to db
 
-	forwards_to = ",".join(validated_forwards_to)
-
-	if len(validated_permitted_senders) == 0:
-		permitted_senders = None
+	conn = open_database(env)
+	existing_alias, existing_permitted_senders = find_mail_alias(env, address, ['member','rfc822MailMember'], conn)
+	if existing_alias and not update_if_exists:
+		return ("Alias already exists (%s)." % address, 400)
+	
+	cn="%s" % uuid.uuid4()
+	dn="cn=%s,%s" % (cn, env.LDAP_ALIASES_BASE)
+	if address.startswith('@') and \
+	   len(validated_forwards_to)==1 and \
+	   validated_forwards_to[0].startswith('@'):
+		description = "Domain alias %s->%s" % (address, validated_forwards_to[0])
+	elif address.startswith('@'):
+		description = "Catch-all for %s" % address
+	else:
+		description ="Mail group %s" % address
+	attrs = {
+		"mail": address,
+		"description": description,
+		"member": vfwd_tos_local,
+		"rfc822MailMember": vfwd_tos_remote
+	}
+	
+	op = conn.add_or_modify(dn, existing_alias,
+			['member', 'rfc822MailMember' ],
+			['mailGroup'],
+			attrs)
+	if op == 'modify':
+		return_status = "alias updated"
 	else:
-		permitted_senders = ",".join(validated_permitted_senders)
-
-	conn, c = open_database(env, with_connection=True)
-	try:
-		c.execute("INSERT INTO aliases (source, destination, permitted_senders) VALUES (?, ?, ?)", (address, forwards_to, permitted_senders))
 		return_status = "alias added"
-	except sqlite3.IntegrityError:
-		if not update_if_exists:
-			return ("Alias already exists (%s)." % address, 400)
-		else:
-			c.execute("UPDATE aliases SET destination = ?, permitted_senders = ? WHERE source = ?", (forwards_to, permitted_senders, address))
-			return_status = "alias updated"
-
-	conn.commit()
+		convert_rfc822MailMember(env, conn, dn, address)
+		
+	# add or modify permitted-senders group
+	
+	cn = '%s' % uuid.uuid4()
+	dn = "cn=%s,%s" % (cn, env.LDAP_PERMITTED_SENDERS_BASE)
+	attrs = {
+		"mail" : address,
+		"description": "Permitted to MAIL FROM this address",
+		"member" : validated_permitted_senders
+	}
+	if len(validated_permitted_senders)==0:
+		if existing_permitted_senders:
+			dn = existing_permitted_senders['dn']
+			conn.wait( conn.delete(dn) )
+	else:
+		op=conn.add_or_modify(dn, existing_permitted_senders,
+							  [ 'member' ], [ 'mailGroup' ],
+							  attrs)
 
 	if do_kick:
 		# Update things in case any new domains are added.
 		return kick(env, return_status)
+	
 
 def remove_mail_alias(address, env, do_kick=True):
+	# Remove an alias group and it's associated permitted senders
+	# group.
+	#
+	# address is the email address of the alias
+	#
+	# If an error occurs, the function returns a tuple of (message,
+	# http-status).
+	#
+	# If successful, the string "OK" is returned.
+	
 	# convert Unicode domain to IDNA
 	address = sanitize_idn_email_address(address)
 
 	# remove
-	conn, c = open_database(env, with_connection=True)
-	c.execute("DELETE FROM aliases WHERE source=?", (address,))
-	if c.rowcount != 1:
+	conn = open_database(env)
+	existing_alias, existing_permitted_senders = find_mail_alias(env, address, conn=conn)
+	if existing_alias:
+		conn.delete(existing_alias['dn'])
+	else:
 		return ("That's not an alias (%s)." % address, 400)
-	conn.commit()
 
+	if existing_permitted_senders:
+		conn.delete(existing_permitted_senders['dn'])
+		
 	if do_kick:
 		# Update things in case any domains are removed.
 		return kick(env, "alias removed")
@@ -515,6 +1028,8 @@ def get_system_administrator(env):
 
 def get_required_aliases(env):
 	# These are the aliases that must exist.
+	# Returns a set of email addresses.
+	
 	aliases = set()
 
 	# The system administrator alias is required.
@@ -555,9 +1070,8 @@ def kick(env, mail_result=None):
 
 	# Ensure every required alias exists.
 
-	existing_users = get_mail_users(env)
-	existing_alias_records = get_mail_aliases(env)
-	existing_aliases = set(a for a, *_ in existing_alias_records) # just first entry in tuple
+	existing_users = get_mail_users(env, as_map=True)
+	existing_aliases = get_mail_aliases(env, as_map=True)
 	required_aliases = get_required_aliases(env)
 
 	def ensure_admin_alias_exists(address):
@@ -581,8 +1095,9 @@ def kick(env, mail_result=None):
 
 	# Remove auto-generated postmaster/admin on domains we no
 	# longer have any other email addresses for.
-	for address, forwards_to, *_ in existing_alias_records:
+	for address in existing_aliases:
 		user, domain = address.split("@")
+		forwards_to = ",".join(existing_aliases[address]["forward_tos"])
 		if user in ("postmaster", "admin", "abuse") \
 			and address not in required_aliases \
 			and forwards_to == get_system_administrator(env):

management/ssl_certificates.py

@@ -105,6 +105,9 @@ def get_ssl_certificates(env):
 			# prefer one that is not self-signed
 			cert.issuer != cert.subject,
 
+                        # prefer one that is not our temporary ca
+			"Temporary-Mail-In-A-Box-CA" not in "%s" % cert.issuer.rdns,
+
 			###########################################################
 			# The above lines ensure that valid certificates are chosen
 			# over invalid certificates. The lines below choose between
@@ -453,7 +456,7 @@ def post_install_func(env):
 
 	# Symlink the best cert for PRIMARY_HOSTNAME to the system
 	# certificate path, which is hard-coded for various purposes, and then
-	# restart postfix and dovecot.
+	# restart postfix, dovecot and openldap.
 	system_ssl_certificate = os.path.join(os.path.join(env["STORAGE_ROOT"], 'ssl', 'ssl_certificate.pem'))
 	if cert and os.readlink(system_ssl_certificate) != cert['certificate']:
 		# Update symlink.
@@ -463,6 +466,7 @@ def post_install_func(env):
 		os.symlink(ssl_certificate, system_ssl_certificate)
 
 		# Restart postfix and dovecot so they pick up the new file.
+		shell('check_call', ["/usr/sbin/service", "slapd", "restart"])
 		shell('check_call', ["/usr/sbin/service", "postfix", "restart"])
 		shell('check_call', ["/usr/sbin/service", "dovecot", "restart"])
 		ret.append("mail services restarted")
@@ -531,6 +535,9 @@ def check_certificate(domain, ssl_certificate, ssl_private_key, warn_if_expiring
 	if cert.issuer == cert.subject:
 		return ("SELF-SIGNED", None)
 
+	elif "Temporary-Mail-In-A-Box-CA" in "%s" % cert.issuer.rdns:
+		return ("SELF-SIGNED", None)
+
 	# When selecting which certificate to use for non-primary domains, we check if the primary
 	# certificate or a www-parent-domain certificate is good for the domain. There's no need
 	# to run extra checks beyond this point.

management/status_checks.py

@@ -434,10 +434,10 @@ def check_primary_hostname_dns(domain, env, output, dns_domains, dns_zonefiles):
 	check_alias_exists("Hostmaster contact address", "hostmaster@" + domain, env, output)
 
 def check_alias_exists(alias_name, alias, env, output):
-	mail_aliases = dict([(address, receivers) for address, receivers, *_ in get_mail_aliases(env)])
+	mail_aliases = get_mail_aliases(env, as_map=True)
 	if alias in mail_aliases:
-		if mail_aliases[alias]:
-			output.print_ok("%s exists as a mail alias. [%s ↦ %s]" % (alias_name, alias, mail_aliases[alias]))
+		if mail_aliases[alias]["forward_tos"]:
+			output.print_ok("%s exists as a mail alias. [%s ↦ %s]" % (alias_name, alias, ",".join(mail_aliases[alias]["forward_tos"])))
 		else:
 			output.print_error("""You must set the destination of the mail alias for %s to direct email to you or another administrator.""" % alias)
 	else:
@@ -618,7 +618,7 @@ def check_mail_domain(domain, env, output):
 
 	# Check that the postmaster@ email address exists. Not required if the domain has a
 	# catch-all address or domain alias.
-	if "@" + domain not in [address for address, *_ in get_mail_aliases(env)]:
+	if "@" + domain not in get_mail_aliases(env, as_map=True):
 		check_alias_exists("Postmaster contact address", "postmaster@" + domain, env, output)
 
 	# Stop if the domain is listed in the Spamhaus Domain Block List.

management/utils.py

@@ -1,4 +1,5 @@
-import os.path
+# -*- indent-tabs-mode: nil; -*-
+import os.path, collections
 
 # DO NOT import non-standard modules. This module is imported by
 # migrate.py which runs on fresh machines before anything is installed
@@ -6,15 +7,32 @@ import os.path
 
 # THE ENVIRONMENT FILE AT /etc/mailinabox.conf
 
+class Environment(collections.OrderedDict):
+    # subclass OrderedDict and provide attribute lookups to the
+    # underlying dictionary
+    def __getattr__(self, attr):
+        return self[attr]
+
 def load_environment():
     # Load settings from /etc/mailinabox.conf.
-    return load_env_vars_from_file("/etc/mailinabox.conf")
+    env = load_env_vars_from_file("/etc/mailinabox.conf")
 
-def load_env_vars_from_file(fn):
+    # Load settings from STORAGE_ROOT/ldap/miab_ldap.conf
+    # It won't exist exist until migration 13 completes...
+    if os.path.exists(os.path.join(env["STORAGE_ROOT"],"ldap/miab_ldap.conf")):
+        load_env_vars_from_file(os.path.join(env["STORAGE_ROOT"],"ldap/miab_ldap.conf"), strip_quotes=True, merge_env=env)
+    
+    return env
+
+def load_env_vars_from_file(fn, strip_quotes=False, merge_env=None):
     # Load settings from a KEY=VALUE file.
-    import collections
-    env = collections.OrderedDict()
-    for line in open(fn): env.setdefault(*line.strip().split("=", 1))
+    env = Environment()
+    for line in open(fn):
+        env.setdefault(*line.strip().split("=", 1))
+    if strip_quotes:
+        for k in env: env[k]=env[k].strip('"')
+    if merge_env is not None:
+        for k in env: merge_env[k]=env[k]
     return env
 
 def save_environment(env):

management/web_update.py

@@ -1,3 +1,4 @@
+# -*- indent-tabs-mode: t; tab-width: 4; python-indent-offset: 4; -*-
 # Creates an nginx configuration file so we serve HTTP/HTTPS on all
 # domains for which a mail account has been set up.
 ########################################################################
@@ -9,14 +10,15 @@ from dns_update import get_custom_dns_config, get_dns_zones
 from ssl_certificates import get_ssl_certificates, get_domain_ssl_files, check_certificate
 from utils import shell, safe_domain_name, sort_domains
 
-def get_web_domains(env, include_www_redirects=True, exclude_dns_elsewhere=True):
+def get_web_domains(env, include_www_redirects=True, exclude_dns_elsewhere=True, categories=['mail', 'ssl']):
 	# What domains should we serve HTTP(S) for?
 	domains = set()
 
 	# Serve web for all mail domains so that we might at least
 	# provide auto-discover of email settings, and also a static website
 	# if the user wants to make one.
-	domains |= get_mail_domains(env)
+	for category in categories:
+		domains |= get_mail_domains(env, category=category)
 
 	if include_www_redirects:
 		# Add 'www.' subdomains that we want to provide default redirects
@@ -27,8 +29,9 @@ def get_web_domains(env, include_www_redirects=True, exclude_dns_elsewhere=True)
 	# Add Autoconfiguration domains, allowing us to serve correct SSL certs.
 	# 'autoconfig.' for Mozilla Thunderbird auto setup.
 	# 'autodiscover.' for Activesync autodiscovery.
-	domains |= set('autoconfig.' + maildomain for maildomain in get_mail_domains(env))
-	domains |= set('autodiscover.' + maildomain for maildomain in get_mail_domains(env))
+	if 'mail' in categories:
+		domains |= set('autoconfig.' + maildomain for maildomain in get_mail_domains(env, category='mail'))
+		domains |= set('autodiscover.' + maildomain for maildomain in get_mail_domains(env, category='mail'))
 
 	if exclude_dns_elsewhere:
 		# ...Unless the domain has an A/AAAA record that maps it to a different
@@ -85,8 +88,8 @@ def do_web_update(env):
 
 	# Add configuration all other web domains.
 	has_root_proxy_or_redirect = get_web_domains_with_root_overrides(env)
-	web_domains_not_redirect = get_web_domains(env, include_www_redirects=False)
-	for domain in get_web_domains(env):
+	web_domains_not_redirect = get_web_domains(env, include_www_redirects=False, categories=['mail'])
+	for domain in get_web_domains(env, categories=['mail']):
 		if domain == env['PRIMARY_HOSTNAME']:
 			# PRIMARY_HOSTNAME is handled above.
 			continue

setup/functions-ldap.sh

@@ -0,0 +1,121 @@
+# -*- indent-tabs-mode: t; tab-width: 4; -*-
+#
+# some helpful ldap function that are shared between setup/ldap.sh and
+# test suites in tests/suites/*
+#
+get_attribute_from_ldif() {
+	local attr="$1"
+	local ldif="$2"
+	# Gather values - handle multivalued attributes and values that
+	# contain whitespace
+	ATTR_DN="$(awk "/^dn:/ { print substr(\$0, 4); exit }" <<< $ldif)"
+	ATTR_VALUE=()
+	local line
+	while read line; do
+		[ -z "$line" ] && break
+		local v=$(awk "/^$attr:/ { print substr(\$0, length(\"$attr\")+3) }" <<<$line)
+		[ ! -z "$v" ] && ATTR_VALUE+=( "$v" )
+	done <<< "$ldif"
+	return 0
+}
+
+get_attribute() {
+	# Returns first matching dn in $ATTR_DN (empty if not found),
+	# along with associated values of the specified attribute in
+	# $ATTR_VALUE as an array
+	local base="$1"
+	local filter="$2"
+	local attr="$3"
+	local scope="${4:-sub}"
+	local bind_dn="${5:-}"
+	local bind_pw="${6:-}"
+	local stderr_file="/tmp/ldap_search.$$.err"
+	local code_file="$stderr_file.code"
+
+	# Issue the search
+	local args=( "-Q" "-Y" "EXTERNAL" "-H" "ldapi:///" )
+	if [ ! -z "$bind_dn" ]; then
+		args=("-H" "$LDAP_URL" "-x" "-D" "$bind_dn" "-w" "$bind_pw" )
+	fi
+	args+=( "-LLL" "-s" "$scope" "-o" "ldif-wrap=no" "-b" "$base" )
+
+	local result
+	result=$(ldapsearch ${args[@]} "$filter" "$attr" 2>$stderr_file; echo $? >$code_file)
+	local exitcode=$(cat $code_file)
+	local stderr=$(cat $stderr_file)
+	rm -f "$stderr_file"
+	rm -f "$code_file"
+	if [ $exitcode -ne 0 -a $exitcode -ne 32 ]; then
+		# 255 == unable to contact server
+		# 32 == No such object
+		die "$stderr"
+	fi
+
+	get_attribute_from_ldif "$attr" "$result"
+}
+
+
+slappasswd_hash() {
+	# hash the given password with our preferred algorithm and in a
+	# format suitable for ldap. see crypt(3) for format
+	slappasswd -h {CRYPT} -c \$6\$%.16s -s "$1"
+}
+
+debug_search() {
+	# perform a search and output the results
+	# arg 1: the search criteria
+	# arg 2: [optional] the base rdn
+	# arg 3-: [optional] attributes to output, if not specified
+	#        all are output
+	local base="$LDAP_BASE"
+	local query="(objectClass=*)"
+	local scope="sub"
+	local attrs=( )
+	case "$1" in
+		\(* )
+			# filters start with an open paren...
+			query="$1"
+			;;
+		*@* )
+			# looks like an email address
+			query="(|(mail=$1)(maildrop=$1))"
+			;;
+		* )
+			# default: it's a dn
+			base="$1"
+			;;
+	esac
+	shift
+	
+	if [ $# -gt 0 ]; then
+		base="$1"
+		shift
+	fi
+
+	if [ $# -gt 0 ]; then
+		attrs=( $@ )
+	fi
+
+	local ldif=$(ldapsearch -H $LDAP_URL -o ldif-wrap=no -b "$base" -s $scope -LLL -x -D "$LDAP_ADMIN_DN" -w "$LDAP_ADMIN_PASSWORD" "$query" ${attrs[@]}; exit 0)
+
+	# expand 'member'
+	local line
+	while read line; do
+		case "$line" in
+			member:* )
+				local member_dn=$(cut -c9- <<<"$line")
+				get_attribute "$member_dn" "objectClass=*" mail base "$LDAP_ADMIN_DN" "$LDAP_ADMIN_PASSWORD"
+				if [ -z "$ATTR_DN" ]; then
+					echo "$line"
+					echo "#^ member DOES NOT EXIST"
+				else
+					echo "member: ${ATTR_VALUE[@]}"
+					echo "#^ $member_dn"
+				fi
+				;;
+			* )
+				echo "$line"
+				;;
+		esac
+	done <<<"$ldif"
+}

setup/functions.sh

@@ -1,3 +1,4 @@
+# -*- indent-tabs-mode: t; tab-width: 4; -*-
 # Turn on "strict mode." See http://redsymbol.net/articles/unofficial-bash-strict-mode/.
 # -e: exit if any command unexpectedly fails.
 # -u: exit if we have a variable typo.
@@ -223,3 +224,23 @@ function git_clone {
 	mv $TMPPATH/$SUBDIR $TARGETPATH
 	rm -rf $TMPPATH
 }
+
+function generate_password() {
+	# output a randomly generated password of the length specified as
+	# the first argument. If no length is given, a password of 64
+	# characters is generated.
+	#
+	# The actual returned password may be longer than requested to
+	# avoid base64 padding characters
+	#
+	local input_len extra pw_length="${1:-64}"
+	# choose a length (longer) that will avoid padding chars
+	let extra="4 - $pw_length % 4"
+	[ $extra -eq 4 ] && extra=0
+	let input_len="($pw_length + $extra) / 4 * 3"
+	# change forward slash to comma because forward slash causes problems
+	# when used in regular expressions (for instance sed) or curl using
+	# basic auth supplied in the url (https://user:pass@host)
+	dd if=/dev/urandom bs=1 count=$input_len 2>/dev/null | base64 --wrap=0 | awk '{ gsub("/", ",", $0); print $0}'
+}
+

setup/ldap.sh

@@ -0,0 +1,882 @@
+#!/bin/bash
+# -*- indent-tabs-mode: t; tab-width: 4; -*-
+
+#
+# LDAP server (slapd) for user authentication and directory services
+#
+source setup/functions.sh # load our functions
+source setup/functions-ldap.sh # load our ldap-specific functions
+source /etc/mailinabox.conf # load global vars
+
+ORGANIZATION="Mail-In-A-Box"
+LDAP_DOMAIN="mailinabox"
+LDAP_BASE="dc=mailinabox"
+LDAP_SERVICES_BASE="ou=Services,$LDAP_BASE"
+LDAP_CONFIG_BASE="ou=Config,$LDAP_BASE"
+LDAP_DOMAINS_BASE="ou=domains,$LDAP_CONFIG_BASE"
+LDAP_PERMITTED_SENDERS_BASE="ou=permitted-senders,$LDAP_CONFIG_BASE"
+LDAP_USERS_BASE="ou=Users,${LDAP_BASE}"
+LDAP_ALIASES_BASE="ou=aliases,${LDAP_USERS_BASE}"
+LDAP_ADMIN_DN="cn=admin,dc=mailinabox"
+
+STORAGE_LDAP_ROOT="$STORAGE_ROOT/ldap"
+MIAB_SLAPD_DB_DIR="$STORAGE_LDAP_ROOT/db"
+MIAB_SLAPD_CONF="$STORAGE_LDAP_ROOT/slapd.d"
+MIAB_INTERNAL_CONF_FILE="$STORAGE_LDAP_ROOT/miab_ldap.conf"
+
+SERVICE_ACCOUNTS=(LDAP_DOVECOT LDAP_POSTFIX LDAP_WEBMAIL LDAP_MANAGEMENT LDAP_NEXTCLOUD)
+
+declare -i verbose=0
+
+
+#
+# Helper functions
+#
+die() {
+	local msg="$1"
+	local rtn="${2:-1}"
+	[ ! -z "$msg" ] && echo "FATAL: $msg" || echo "An unrecoverable error occurred, exiting"
+	exit ${rtn}
+}
+
+say_debug() {
+	[ $verbose -gt 1 ] && echo $@
+	return 0
+}
+
+say_verbose() {
+	[ $verbose -gt 0 ] && echo $@
+	return 0
+}
+
+say() {
+	echo $@
+}
+
+ldap_debug_flag() {
+	[ $verbose -gt 1 ] && echo "-d 1"
+}
+
+wait_slapd_start() {
+	# Wait for slapd to start...
+	say_verbose -n "Waiting for slapd to start"
+	local let elapsed=0
+	until nc -z -w 4 127.0.0.1 389
+	do
+		[ $elapsed -gt 30 ] && die "Giving up waiting for slapd to start!"
+		[ $elapsed -gt 0 ] && say_verbose -n "...${elapsed}"
+		sleep 2
+		let elapsed+=2
+	done
+	say_verbose "...ok"
+}
+
+create_miab_conf() {
+	# create (if non-existing) or load (existing) ldap/miab_ldap.conf
+	if [ ! -e "$MIAB_INTERNAL_CONF_FILE" ]; then
+		say_verbose "Generating a new $MIAB_INTERNAL_CONF_FILE"
+		mkdir -p "$(dirname $MIAB_INTERNAL_CONF_FILE)"
+		
+		# Use 64-character secret keys of safe characters
+		cat > "$MIAB_INTERNAL_CONF_FILE" <<EOF
+LDAP_SERVER=127.0.0.1
+LDAP_SERVER_PORT=389
+LDAP_SERVER_STARTTLS=no
+LDAP_SERVER_TLS=no
+LDAP_URL=ldap://127.0.0.1/
+LDAP_BASE="${LDAP_BASE}"
+LDAP_SERVICES_BASE="${LDAP_SERVICES_BASE}"
+LDAP_CONFIG_BASE="${LDAP_CONFIG_BASE}"
+LDAP_DOMAINS_BASE="${LDAP_DOMAINS_BASE}"
+LDAP_PERMITTED_SENDERS_BASE="${LDAP_PERMITTED_SENDERS_BASE}"
+LDAP_USERS_BASE="${LDAP_USERS_BASE}"
+LDAP_ALIASES_BASE="${LDAP_ALIASES_BASE}"
+LDAP_ADMIN_DN="${LDAP_ADMIN_DN}"
+LDAP_ADMIN_PASSWORD="$(generate_password 64)"
+EOF
+	fi
+
+	# add service account credentials
+	local prefix
+	for prefix in ${SERVICE_ACCOUNTS[*]}
+	do
+		if [ $(grep -c "^$prefix" "$MIAB_INTERNAL_CONF_FILE") -eq 0 ]; then
+			local cn=$(awk -F_ '{print tolower($2)}' <<< $prefix)
+			cat >>"$MIAB_INTERNAL_CONF_FILE" <<EOF
+${prefix}_DN="cn=$cn,$LDAP_SERVICES_BASE"
+${prefix}_PASSWORD="$(generate_password 64)"
+EOF
+		fi
+	done
+	
+	chmod 0640 "$MIAB_INTERNAL_CONF_FILE"
+	. "$MIAB_INTERNAL_CONF_FILE"
+}
+
+
+create_directory_containers() {
+	# create organizationUnit containers
+	local basedn
+	for basedn in "$LDAP_SERVICES_BASE" "$LDAP_CONFIG_BASE" "$LDAP_DOMAINS_BASE" "$LDAP_PERMITTED_SENDERS_BASE" "$LDAP_USERS_BASE" "$LDAP_ALIASES_BASE"; do
+		# add ou container
+		get_attribute "$basedn" "objectClass=*" "ou" base
+		if [ -z "$ATTR_DN" ]; then
+			say_verbose "Adding $basedn"
+			ldapadd -H ldap://127.0.0.1/ -x -D "$LDAP_ADMIN_DN" -w "$LDAP_ADMIN_PASSWORD" >/dev/null <<EOF
+dn: $basedn
+objectClass: organizationalUnit
+ou: $(awk -F'[=,]' '{print $2}' <<< $basedn)
+EOF
+		fi
+	done
+}
+
+create_service_accounts() {
+	# create service accounts. service accounts have special access
+	# rights, generally read-only to users, aliases, and configuration
+	# subtrees (see apply_access_control)
+	
+	local prefix dn pass
+	for prefix in ${SERVICE_ACCOUNTS[*]}
+	do
+		eval "dn=\$${prefix}_DN"
+		eval "pass=\$${prefix}_PASSWORD"
+		get_attribute "$dn" "objectClass=*" "cn" base
+		say_debug "SERVICE_ACCOUNT $dn"
+		if [ -z "$ATTR_DN" ]; then
+			local cn=$(awk -F'[=,]' '{print $2}' <<< $dn)
+			say_verbose "Adding service account: $dn"
+			ldapadd -H ldap://127.0.0.1/ -x -D "$LDAP_ADMIN_DN" -w "$LDAP_ADMIN_PASSWORD" >/dev/null <<EOF
+dn: $dn
+objectClass: simpleSecurityObject
+objectClass: organizationalRole
+cn: $cn
+description: ${cn} service account
+userPassword: $(slappasswd_hash "$pass")
+EOF
+		fi
+	done
+	
+}
+
+
+install_system_packages() {
+	# install required deb packages, generate admin credentials
+	# and apply them to the installation
+	create_miab_conf
+	
+	# Set installation defaults to avoid interactive dialogs. See
+	# /var/lib/dpkg/info/slapd.templates for a list of what can be set
+	debconf-set-selections <<EOF
+slapd shared/organization string ${ORGANIZATION}
+slapd slapd/domain string ${LDAP_DOMAIN}
+slapd slapd/password1 password ${LDAP_ADMIN_PASSWORD}
+slapd slapd/password2 password ${LDAP_ADMIN_PASSWORD}
+EOF
+	
+	# Install packages
+	say "Installing OpenLDAP server..."
+	apt_install slapd ldap-utils python3-ldap3 python3-ldif3 ca-certificates xz-utils
+
+	# If slapd was not installed by us, the selections above did
+	# nothing.  To check this we see if SLAPD_CONF in
+	# /etc/default/slapd is empty and that the olc does not have our
+	# database. We could do 2 things in this situation:
+	#    1. ask the user for the current admin password and add our domain
+	#    2. reconfigure and wipe out the current database
+	# we do #2 ....
+	local SLAPD_CONF=""
+	eval "$(grep ^SLAPD_CONF= /etc/default/slapd)"
+	local cursuffix="$(slapcat -s "cn=config" | grep "^olcSuffix: ")"
+	if [ -z "$SLAPD_CONF" ] &&
+		   ! grep "$LDAP_DOMAIN" <<<"$cursuffix" >/dev/null
+	then
+		mkdir -p /var/backup
+		local tgz="/var/backup/slapd-$(date +%Y%m%d-%H%M%S).tgz"
+		(cd /var/lib/ldap; tar czf "$tgz" .)
+		chmod 600 "$tgz"
+		rm /var/lib/ldap/*
+		say "Reininstalling slapd! - existing database saved in $tgz"
+		dpkg-reconfigure --frontend=noninteractive slapd
+	fi
+
+	# Clear passwords out of debconf
+	debconf-set-selections <<EOF
+slapd slapd/password1 password
+slapd slapd/password2 password
+EOF
+
+	# Ensure slapd is running
+	systemctl start slapd && wait_slapd_start
+
+	# Change the admin password hash format in the server from slapd's
+	# default {SSHA} to SHA-512 {CRYPT} with 16 characters of salt
+	get_attribute "cn=config" "olcSuffix=${LDAP_BASE}" "olcRootPW"
+	if [ ${#ATTR_VALUE[*]} -eq 1 -a $(grep -c "{SSHA}" <<< "$ATTR_VALUE") -eq 1 ]; then
+		say_verbose "Updating root hash to SHA512-CRYPT"
+		local hash=$(slappasswd_hash "$LDAP_ADMIN_PASSWORD")
+		ldapmodify -Q -Y EXTERNAL -H ldapi:/// >/dev/null <<EOF
+dn: $ATTR_DN
+replace: olcRootPW
+olcRootPW: $hash
+EOF
+		say_verbose "Updating admin hash to SHA512-CRYPT"
+		ldapmodify -H ldap://127.0.0.1/ -x -D "$LDAP_ADMIN_DN" -w "$LDAP_ADMIN_PASSWORD"  >/dev/null <<EOF
+dn: $LDAP_ADMIN_DN
+replace: userPassword
+userPassword: $hash
+EOF
+	fi
+}
+
+relocate_slapd_data() {
+	#
+	# Move current ldap databases to user-data (eg. new install). A
+	# new slapd installation places the ldap configuration database in
+	# /etc/ldap/slapd.d and schema database in /var/lib/ldap. So that
+	# backups include the ldap database, move everything to user-data.
+	#
+	# On entry:
+	#	SLAPD_CONF must point to the current slapd.d directory
+	#	   (see /etc/default/slapd)
+	#	Global variables as defined above must be set
+	#	The slapd service must be running
+	#
+	# On success:
+	#	Config and data will be relocated to the new locations
+	#
+	say_verbose "Relocate ldap databases from current locations to user-data"
+
+	# Get the current database location from olc
+	get_attribute "cn=config" "olcSuffix=${LDAP_BASE}" "olcDbDirectory"
+	local DN="$ATTR_DN"
+	local DB_DIR="$ATTR_VALUE"
+	if [ -z "$DN" ]; then
+		say_verbose ""
+		say_verbose "ACK! ${LDAP_BASE} does not exist in the LDAP server!!!"
+		say_verbose "Something is amiss!!!!!"
+		say_verbose "... to ensure no data is lost, please manually fix the problem"
+		say_verbose "	 by running 'sudo dpkg-reconfigure slapd'"
+		say_verbose ""
+		say_verbose "CAUTION: running dbpg-reconfigure will remove ALL data"
+		say_verbose "for the existing domain!"
+		say_verbose ""
+		die "Unable to continue!"
+	fi
+
+	# Exit if destination directories are non-empty
+	[ ! -z "$(ls -A $MIAB_SLAPD_CONF)" ] && die "Cannot relocate system LDAP because $MIAB_SLAPD_CONF is not empty!"
+	[ ! -z "$(ls -A $MIAB_SLAPD_DB_DIR)" ] && die "Cannot relocate system LDAP because $MIAB_SLAPD_DB_DIR is not empty!"
+
+	# Stop slapd
+	say_verbose ""
+	say_verbose "Relocating ldap databases:"
+	say_verbose "	from: "
+	say_verbose "	   CONF='${SLAPD_CONF}'"
+	say_verbose "		DB='${DB_DIR}'"
+	say_verbose "	to:"
+	say_verbose "	   CONF=${MIAB_SLAPD_CONF}"
+	say_verbose "		 DB=${MIAB_SLAPD_DB_DIR}"	
+	say_verbose ""
+	say_verbose "Stopping slapd"
+	systemctl stop slapd || die "Could not stop slapd"
+	
+	# Modify the path to dc=mailinabox's database directory
+	say_verbose "Dump config database"
+	local TMP="/tmp/miab_relocate_ldap.ldif"
+	slapcat -F "${SLAPD_CONF}" -l "$TMP" -n 0 || die "slapcat failed"
+	awk -e "/olcDbDirectory:/ {print \$1 \"$MIAB_SLAPD_DB_DIR\"} !/^olcDbDirectory:/ { print \$0}" $TMP > $TMP.2
+	rm -f "$TMP"
+
+	# Copy the existing database files
+	say_verbose "Copy database files ($DB_DIR => $MIAB_SLAPD_DB_DIR)"
+	cp -p "${DB_DIR}"/* "${MIAB_SLAPD_DB_DIR}" || die "Could not copy files '${DB_DIR}/*' to '${MIAB_SLAPD_DB_DIR}'"
+
+	# Re-create the config
+	say_verbose "Create new slapd config"
+	local xargs=()
+	[ $verbose -gt 0 ] && xargs+=(-d 10 -v)
+	slapadd -F "${MIAB_SLAPD_CONF}" ${xargs[@]} -n 0 -l "$TMP.2" 2>/dev/null || die "slapadd failed!"
+	chown -R openldap:openldap "${MIAB_SLAPD_CONF}"
+	rm -f "$TMP.2"
+
+	# Remove the old database files
+	rm -f "${DB_DIR}/*"
+}
+
+
+schema_to_ldif() {
+	# Convert a .schema file to ldif. This function follows the
+	# conversion instructions found in /etc/ldap/schema/openldap.ldif
+	local schema="$1"  # path or url to schema
+	local ldif="$2"	   # destination file - will be overwritten
+	local cn="$3"	   # schema common name, eg "postfix"
+	local cat='cat'
+	if [ ! -e "$schema" ]; then
+		if [ -e "conf/$(basename $schema)" ]; then
+			schema="conf/$(basename $schema)"
+		else
+			cat="curl -s"
+		fi
+	fi
+	
+	cat >"$ldif" <<EOF
+dn: cn=$cn,cn=schema,cn=config
+objectClass: olcSchemaConfig
+cn: $cn
+EOF
+
+	$cat "$schema" \
+		| sed s/attributeType/olcAttributeTypes:/ig \
+		| sed s/objectClass/olcObjectClasses:/ig \
+		| sed s/objectIdentifier/olcObjectIdentifier:/ig \
+		| sed 's/\t/  /g' \
+		| sed 's/^\s*$/#/g' >> "$ldif"
+}
+
+
+add_schemas() {
+	# Add necessary schema's for MiaB operaion
+	#
+	# First, apply rfc822MailMember from OpenLDAP's "misc"
+	# schema. Don't apply the whole schema file because much is from
+	# expired RFC's, and we just need rfc822MailMember
+	local cn="misc"
+	get_attribute "cn=schema,cn=config" "(&(cn={*}$cn)(objectClass=olcSchemaConfig))" "cn"
+	if [ -z "$ATTR_DN" ]; then
+		say_verbose "Adding '$cn' schema"
+		cat "/etc/ldap/schema/misc.ldif" | awk 'BEGIN {C=0}
+/^(dn|objectClass|cn):/ { print $0; next }
+/^olcAttributeTypes:/ && /27\.2\.1\.15/ { print $0; C=1; next }
+/^(olcAttributeTypes|olcObjectClasses):/ { C=0; next }
+/^ / && C==1 { print $0 }' | ldapadd -Q -Y EXTERNAL -H ldapi:/// >/dev/null
+	fi
+	
+	# Next, apply the postfix schema from the ldapadmin project
+	# (GPL)(*).
+	#	see: http://ldapadmin.org
+	#		 http://ldapadmin.org/docs/postfix.schema
+	#		 http://www.postfix.org/LDAP_README.html
+	# (*) mailGroup modified to include rfc822MailMember
+	local schema="http://ldapadmin.org/docs/postfix.schema"
+	local cn="postfix"
+	get_attribute "cn=schema,cn=config" "(&(cn={*}$cn)(objectClass=olcSchemaConfig))" "cn"
+	if [ -z "$ATTR_DN" ]; then
+		local ldif="/tmp/$cn.$$.ldif"
+		schema_to_ldif "$schema" "$ldif" "$cn"
+		sed -i 's/\$ member \$/$ member $ rfc822MailMember $/' "$ldif"
+		say_verbose "Adding '$cn' schema"
+		[ $verbose -gt 1 ] && cat "$ldif"
+		ldapadd -Q -Y EXTERNAL -H ldapi:/// -f "$ldif" >/dev/null
+		rm -f "$ldif"
+	fi
+}
+
+
+modify_global_config() {
+	#
+	# Set ldap configuration attributes:
+	#  IdleTimeout: seconds to wait before forcibly closing idle connections
+	#  LogLevel: logging levels - see OpenLDAP docs
+	#  TLS configuration
+	#  Disable anonymous binds
+	#
+	say_verbose "Setting global ldap configuration"
+
+	# TLS requirements:
+	#
+	# The 'openldap' user must have read access to the TLS private key
+	# and certificate (file system permissions and apparmor). If
+	# access is not configured properly, slapd retuns error code 80
+	# and won't apply the TLS configuration, or won't start.
+	#
+	# Openldap TLS will not operate with a self-signed server
+	# certificate! The server will always log "unable to get TLS
+	# client DN, error=49." Ensure the certificate is signed by
+	# a certification authority.
+	#
+	# The list of trusted CA certificates must include the CA that
+	# signed the server's certificate!
+	#
+	# For the olcCiperSuite setting, see:
+	# https://www.gnutls.org/manual/gnutls.html#Priority-Strings
+	#
+
+	ldapmodify $(ldap_debug_flag) -Q -Y EXTERNAL -H ldapi:/// >/dev/null <<EOF
+dn: cn=config
+##
+## timeouts (1800=30 minutes) and logging
+##
+replace: olcIdleTimeout
+olcIdleTimeout: 1800
+-
+replace: olcLogLevel
+olcLogLevel: config stats shell
+#olcLogLevel: config stats shell filter ACL
+-
+##
+## TLS
+##
+replace: olcTLSCACertificateFile
+olcTLSCACertificateFile: /etc/ssl/certs/ca-certificates.crt
+-
+replace: olcTLSCertificateFile
+olcTLSCertificateFile: $STORAGE_ROOT/ssl/ssl_certificate.pem
+-
+replace: olcTLSCertificateKeyFile
+olcTLSCertificateKeyFile: $STORAGE_ROOT/ssl/ssl_private_key.pem
+-
+replace: olcTLSDHParamFile
+olcTLSDHParamFile: $STORAGE_ROOT/ssl/dh2048.pem
+-
+replace: olcTLSCipherSuite
+olcTLSCipherSuite: PFS
+-
+replace: olcTLSVerifyClient
+olcTLSVerifyClient: never
+-
+##
+## Password policies - use SHA512 with 16 characters of salt
+##
+replace: olcPasswordHash
+olcPasswordHash: {CRYPT}
+-
+replace: olcPasswordCryptSaltFormat
+olcPasswordCryptSaltFormat: \$6\$%.16s
+-
+##
+## Disable anonymous binds
+##
+replace: olcDisallows
+olcDisallows: bind_anon
+-
+replace: olcRequires
+olcRequires: authc
+
+dn: olcDatabase={-1}frontend,cn=config
+replace: olcRequires
+olcRequires: authc
+EOF
+}
+
+
+add_overlays() {
+	# Apply slapd overlays - apply the commonly used member-of overlay
+	# now because adding it later is harder.
+	
+	# Get the config dn for the database
+	get_attribute "cn=config" "olcSuffix=${LDAP_BASE}" "dn"
+	[ -z "$ATTR_DN" ] &&
+		die "No config found for olcSuffix=$LDAP_BASE in cn=config!"
+	local cdn="$ATTR_DN"
+
+	# Add member-of overlay (man 5 slapo-memberof)
+	get_attribute "cn=module{0},cn=config" "(olcModuleLoad=memberof.la)" "dn" base
+	if [ -z "$ATTR_DN" ]; then
+		say_verbose "Adding memberof overlay module"
+		ldapmodify -Q -Y EXTERNAL -H ldapi:/// >/dev/null <<EOF
+dn: cn=module{0},cn=config
+add: olcModuleLoad
+olcModuleLoad: memberof.la
+EOF
+	fi
+	
+	get_attribute "$cdn" "(olcOverlay=memberof)" "olcOverlay"
+	if [ -z "$ATTR_DN" ]; then
+		say_verbose "Adding memberof overlay to $LDAP_BASE"
+		ldapadd -Q -Y EXTERNAL -H ldapi:/// >/dev/null <<EOF
+dn: olcOverlay=memberof,$cdn
+objectClass: olcOverlayConfig
+objectClass: olcMemberOf
+olcOverlay: memberof
+#olcMemberOfGroupOC: mailGroup
+olcMemberOfRefint: TRUE
+EOF
+	fi
+}
+
+
+add_indexes() {
+	# Index mail-related attributes
+	
+	# Get the config dn for the database
+	get_attribute "cn=config" "olcSuffix=${LDAP_BASE}" "dn"
+	[ -z "$ATTR_DN" ] &&
+		die "No config found for olcSuffix=$LDAP_BASE in cn=config!"
+	local cdn="$ATTR_DN"
+
+	# Add the indexes
+	get_attribute "$cdn" "(objectClass=*)" "olcDbIndex" base
+	local attr
+	for attr in mail maildrop mailaccess dc rfc822MailMember; do
+		local type="eq" atype="" aindex=""
+		[ "$attr" == "mail" ] && type="eq,sub"
+
+		# find the existing index for the attribute
+		local item
+		for item in "${ATTR_VALUE[@]}"; do
+			local split=($item)  # eg "mail eq"
+			if [ "${split[0]}" == "$attr" ]; then
+				aindex="$item"
+				atype="${split[1]}"
+				break
+			fi
+		done
+
+		# if desired index type (eg "eq") is equal to actual type,
+		# continue, no change
+		[ "$type" == "$atype" ] && continue
+
+		# replace it or add a new index if not present
+		if [ ! -z "$atype" ]; then
+			say_verbose "Replace index $attr ($atype -> $type)"
+			ldapmodify -Q -Y EXTERNAL -H ldapi:/// >/dev/null <<EOF
+dn: $cdn
+delete: olcDbIndex
+olcDbIndex: $aindex
+-
+add: olcDbIndex
+olcDbIndex: $attr $type
+EOF
+		else
+			say_verbose "Add index for attribute $attr ($type)"
+			ldapmodify -Q -Y EXTERNAL -H ldapi:/// >/dev/null <<EOF
+dn: $cdn
+add: olcDbIndex
+olcDbIndex: $attr $type
+EOF
+		fi
+	done
+}
+
+
+apply_access_control() {
+	# Apply access control to the mail-in-a-box databse.
+	#
+	# Permission restrictions:
+	#	service accounts (except management):
+	#	   can bind but not change passwords, including their own
+	#	   can read all attributes of all users but not userPassword
+	#	   can read config subtree (permitted-senders, domains)
+	#	   no access to services subtree, except their own dn
+	#	management service account:
+	#	   can read and change password and shadowLastChange
+	#	   all other service account permissions are the same
+	#	users:
+	#	   can bind and change their own password
+	#	   can read and change their own shadowLastChange
+	#	   can read attributess of all users except mailaccess
+	#	   no access to config subtree
+	#	   no access to services subtree
+	#
+
+	# Get the config dn for the database
+	get_attribute "cn=config" "olcSuffix=${LDAP_BASE}" "dn"
+	[ -z "$ATTR_DN" ] &&
+		die "No config found for olcSuffix=$LDAP_BASE in cn=config!"
+	local cdn="$ATTR_DN"
+
+	say_verbose "Setting database permissions"
+	ldapmodify -Q -Y EXTERNAL -H ldapi:/// >/dev/null <<EOF
+dn: $cdn
+replace: olcAccess
+olcAccess: to attrs=userPassword
+  by dn.exact="cn=management,${LDAP_SERVICES_BASE}" write
+  by dn.exact="gidNumber=0+uidNumber=0,cn=peercred,cn=external,cn=auth" read
+  by dn.subtree="${LDAP_SERVICES_BASE}" none
+  by self =wx
+  by anonymous auth
+  by * none
+olcAccess: to attrs=shadowLastChange
+  by self write
+  by dn.exact="cn=management,${LDAP_SERVICES_BASE}" write
+  by dn.subtree="${LDAP_SERVICES_BASE}" read
+  by dn.exact="gidNumber=0+uidNumber=0,cn=peercred,cn=external,cn=auth" read
+  by * none
+olcAccess: to attrs=mailaccess
+  by dn.exact="cn=management,${LDAP_SERVICES_BASE}" write
+  by dn.subtree="${LDAP_SERVICES_BASE}" read
+  by dn.exact="gidNumber=0+uidNumber=0,cn=peercred,cn=external,cn=auth" read
+  by * none
+olcAccess: to dn.subtree="${LDAP_CONFIG_BASE}"
+  by dn.exact="cn=management,${LDAP_SERVICES_BASE}" write
+  by dn.subtree="${LDAP_SERVICES_BASE}" read
+  by dn.exact="gidNumber=0+uidNumber=0,cn=peercred,cn=external,cn=auth" read
+  by * none
+olcAccess: to dn.subtree="${LDAP_SERVICES_BASE}"
+  by self read
+  by dn.exact="gidNumber=0+uidNumber=0,cn=peercred,cn=external,cn=auth" read
+  by * none
+olcAccess: to dn.subtree="${LDAP_USERS_BASE}"
+  by dn.exact="cn=management,${LDAP_SERVICES_BASE}" write
+  by * read
+olcAccess: to *
+  by * read
+EOF
+}
+
+
+
+update_apparmor() {
+	# Update slapd's access rights under AppArmor so that it has
+	# access to database files in the user-data location
+	cat > /etc/apparmor.d/local/usr.sbin.slapd <<EOF
+	# database directories
+	$MIAB_SLAPD_CONF/** rw,
+	$MIAB_SLAPD_DB_DIR/ r,
+	$MIAB_SLAPD_DB_DIR/** rwk,
+	$MIAB_SLAPD_DB_DIR/alock kw,
+
+	# certificates and keys
+	$STORAGE_ROOT/ssl/* r,
+EOF
+	chmod 0644 /etc/apparmor.d/local/usr.sbin.slapd
+
+	# Load settings into the kernel
+	apparmor_parser -r /etc/apparmor.d/usr.sbin.slapd
+}
+
+
+#
+# Process command line arguments -- these are here for debugging and
+# testing purposes
+#
+process_cmdline() {
+	[ -e "$MIAB_INTERNAL_CONF_FILE" ] && . "$MIAB_INTERNAL_CONF_FILE"
+	
+	if [ "$1" == "-d" ]; then
+		# Start slapd in interactive/debug mode
+		echo "!! SERVER DEBUG MODE !!"
+		echo "Stopping slapd"
+		systemctl stop slapd
+		. /etc/default/slapd
+		echo "Listening on $SLAPD_SERVICES..."
+		/usr/sbin/slapd -h "$SLAPD_SERVICES" -g openldap -u openldap -F $MIAB_SLAPD_CONF -d ${2:-1}
+		exit 0
+		
+	elif [ "$1" == "-config" ]; then
+		# Apply a certain configuration
+		if [ "$2" == "server" ]; then
+			modify_global_config
+			add_overlays
+			add_indexes
+			apply_access_control
+		elif [ "$2" == "apparmor" ]; then
+			update_apparmor
+		else
+			echo "Invalid: '$2'. Only 'server' and 'apparmor' supported"
+			exit 1
+		fi
+		exit 0
+
+	elif [ "$1" == "-search" ]; then
+		# search for email addresses, distinguished names and general
+		# ldap filters
+		debug_search "$2"
+		exit 0
+
+	elif [ "$1" == "-dumpdb" ]; then
+		# Dump (to stdout) select ldap data and configuration
+		local s=${2:-all}
+		local hide_attrs="(structuralObjectClass|entryUUID|creatorsName|createTimestamp|entryCSN|modifiersName|modifyTimestamp)"
+		local slapcat_args=(-F "$MIAB_SLAPD_CONF" -o ldif-wrap=no)
+		[ $verbose -gt 0 ] && hide_attrs="(_____NEVERMATCHES)"
+		
+		if [ "$s" == "all" ]; then
+			echo ""
+			echo '--------------------------------'
+			slapcat ${slapcat_args[@]} -s "$LDAP_BASE" | grep -Ev "^$hide_attrs:"
+		fi
+		if [ "$s" == "all" -o "$s" == "config" ]; then
+			echo ""
+			echo '--------------------------------'
+			cat "$MIAB_SLAPD_CONF/cn=config.ldif" | grep -Ev "^$hide_attrs:"
+			get_attribute "cn=config" "olcSuffix=${LDAP_BASE}" "dn"
+			echo ""
+			slapcat ${slapcat_args[@]} -s "$ATTR_DN" | grep -Ev "^$hide_attrs:"
+		fi
+		if [ "$s" == "all" -o "$s" == "frontend" ]; then
+			echo ""
+			echo '--------------------------------'
+			cat "$MIAB_SLAPD_CONF/cn=config/olcDatabase={-1}frontend.ldif" | grep -Ev "^$hide_attrs:"
+		fi
+		if [ "$s" == "all" -o "$s" == "module" ]; then
+			echo ""
+			cat "$MIAB_SLAPD_CONF/cn=config/cn=module{0}.ldif" | grep -Ev "^$hide_attrs:"
+		fi
+		if [ "$s" == "users" ]; then
+			echo ""
+			echo '--------------------------------'
+			debug_search "(objectClass=mailUser)" "$LDAP_USERS_BASE"
+		fi
+		if [ "$s" == "aliases" ]; then
+			echo ""
+			echo '--------------------------------'
+			local attrs=(mail member mailRoutingAddress rfc822MailMember)
+			[ $verbose -gt 0 ] && attrs=()
+			debug_search "(objectClass=mailGroup)" "$LDAP_ALIASES_BASE" ${attrs[@]}
+		fi
+		if [ "$s" == "permitted-senders" -o "$s" == "ps" ]; then
+			echo ""
+			echo '--------------------------------'
+			local attrs=(mail member mailRoutingAddress rfc822MailMember)
+			[ $verbose -gt 0 ] && attrs=()
+			debug_search "(objectClass=mailGroup)" "$LDAP_PERMITTED_SENDERS_BASE" ${attrs[@]}
+		fi
+		if [ "$s" == "domains" ]; then
+			echo ""
+			echo '--------------------------------'
+			debug_search "(objectClass=domain)" "$LDAP_DOMAINS_BASE"
+		fi
+		exit 0
+
+	elif [ "$1" == "-reset" ]; then
+		#
+		# Delete and remove OpenLDAP
+		#
+		echo ""
+		echo "!!!!!			   WARNING!				  !!!!!"
+		echo "!!!!!		 OPENLDAP WILL BE REMOVED	  !!!!!"
+		echo "!!!!!	 ALL LDAP DATA WILL BE DESTROYED  !!!!!"
+		echo ""
+		echo -n "Type 'YES' to continue: "
+		read ans
+		if [ "$ans" != "YES" ]; then
+			echo "Aborted"
+			exit 1
+		fi
+		if [ -x /usr/sbin/slapd ]; then
+			apt-get remove --purge -y slapd
+			apt-get -y autoremove
+			apt-get autoclean
+		fi
+		rm -rf "$STORAGE_LDAP_ROOT"
+		rm -rf "/etc/ldap/slapd.d"
+		rm -rf "/var/lib/ldap"
+		rm -f "/etc/default/slapd"
+		echo "Done"
+		exit 0
+		
+	elif [ ! -z "$1" ]; then
+		echo "Invalid command line argument '$1'"
+		exit 1
+	fi
+}
+
+while [ $# -gt 0 ]; do
+	if [ "$1" == "-verbose" -o "$1" == "-v" ]; then
+		let verbose+=1
+		shift
+	else
+		break
+	fi
+done
+
+[ $# -gt 0 ] && process_cmdline $@
+
+
+
+####
+#### MAIN SCRIPT CODE STARTS HERE...
+####
+
+# Run apt installs
+install_system_packages
+
+# Update the ldap schema
+add_schemas
+
+#
+# Create user-data/ldap directory structure:
+#	db/			  - holds slapd database for "dc=mailinabox"
+#	slapd.d/	  - holds slapd configuration
+#	miab_ldap.conf	- holds values for other subsystems like postfix, management, etc
+#
+for d in "$STORAGE_LDAP_ROOT" "$MIAB_SLAPD_DB_DIR" "$MIAB_SLAPD_CONF"; do
+	mkdir -p "$d"
+	chown openldap:openldap "$d"
+	chmod 755 "$d"
+done
+
+# Ensure openldap can access the tls/ssl private key file
+usermod -a -G ssl-cert openldap
+
+# Ensure slapd can interact with the mailinabox database and config
+update_apparmor
+
+# Load slapd's init script startup options
+. /etc/default/slapd
+if [ -z "$SLAPD_CONF" ]; then
+	# when not defined, slapd uses its compiled-in default directory
+	SLAPD_CONF="/etc/ldap/slapd.d"
+fi
+
+
+# Relocate slapd databases to user-data, which is needed after a new
+# installation, we're restoring from backup, or STORAGE_ROOT changes
+if [ "$SLAPD_CONF" != "$MIAB_SLAPD_CONF" ]; then
+	if [ -z "$(ls -A $MIAB_SLAPD_CONF)" ]; then
+		# Empty destination - relocate databases
+		relocate_slapd_data
+	else
+		# Non-empty destination - use the backup data as-is
+		systemctl stop slapd
+	fi
+	# Tell the system startup script to use our config database
+	tools/editconf.py /etc/default/slapd \
+					  "SLAPD_CONF=$MIAB_SLAPD_CONF"
+	systemctl start slapd || die "slapd woudn't start! try running $0 -d"
+	wait_slapd_start
+fi
+
+
+# Configure syslog
+mkdir -p /var/log/ldap
+chmod 750 /var/log/ldap
+chown syslog:adm /var/log/ldap
+cp conf/slapd-logging.conf /etc/rsyslog.d/20-slapd.conf
+chmod 644 /etc/rsyslog.d/20-slapd.conf
+restart_service syslog
+
+# Add log rotation
+cat > /etc/logrotate.d/slapd <<EOF;
+/var/log/ldap/slapd.log {
+	weekly
+	missingok
+	rotate 52
+	compress
+	delaycompress
+	notifempty
+}
+EOF
+
+# Modify olc server config like TLS
+modify_global_config
+
+# Add overlays and ensure mail-related attributes are indexed
+add_overlays
+add_indexes
+
+# Lock down access
+apply_access_control
+
+# Create general db structure
+create_directory_containers
+
+# Create service accounts for dovecot, postfix, roundcube, etc
+create_service_accounts
+
+# Update where slapd listens for incoming requests
+tools/editconf.py /etc/default/slapd \
+				  "SLAPD_SERVICES=\"ldap://127.0.0.1:389/ ldaps:/// ldapi:///\""
+
+# Restart slapd
+restart_service slapd
+
+# Dump the database daily, before backups run at 3
+# This is not required, but nice to have just in case.
+cat > /etc/cron.d/mailinabox-ldap << EOF
+# Mail-in-a-Box
+# Dump database to ldif
+30 2 * * *	root	/usr/sbin/slapcat -F "$MIAB_SLAPD_CONF" -o ldif-wrap=no -s "$LDAP_BASE" | /usr/bin/xz > "$STORAGE_LDAP_ROOT/db.ldif.xz"; chmod 600 "$STORAGE_LDAP_ROOT/db.ldif.xz"
+EOF

setup/mail-dovecot.sh

@@ -26,7 +26,7 @@ source /etc/mailinabox.conf # load global vars
 echo "Installing Dovecot (IMAP server)..."
 apt_install \
 	dovecot-core dovecot-imapd dovecot-pop3d dovecot-lmtpd dovecot-sqlite sqlite3 \
-	dovecot-sieve dovecot-managesieved
+	dovecot-sieve dovecot-managesieved dovecot-ldap
 
 # The `dovecot-imapd`, `dovecot-pop3d`, and `dovecot-lmtpd` packages automatically
 # enable IMAP, POP and LMTP protocols.
@@ -84,6 +84,8 @@ tools/editconf.py /etc/dovecot/conf.d/10-ssl.conf \
 	ssl=required \
 	"ssl_cert=<$STORAGE_ROOT/ssl/ssl_certificate.pem" \
 	"ssl_key=<$STORAGE_ROOT/ssl/ssl_private_key.pem" \
+	"ssl_protocols=!SSLv3" \
+	"ssl_prefer_server_ciphers = yes" \
 	"ssl_protocols=TLSv1.2" \
 	"ssl_cipher_list=ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384" \
 	"ssl_prefer_server_ciphers=no" \

setup/mail-postfix.sh

@@ -42,7 +42,7 @@ source /etc/mailinabox.conf # load global vars
 # * `ca-certificates`: A trust store used to squelch postfix warnings about
 #   untrusted opportunistically-encrypted connections.
 echo "Installing Postfix (SMTP server)..."
-apt_install postfix postfix-sqlite postfix-pcre postgrey ca-certificates
+apt_install postfix postfix-sqlite postfix-pcre postgrey ca-certificates postfix-ldap postfix-policyd-spf-python
 
 # ### Basic Settings
 
@@ -53,6 +53,7 @@ apt_install postfix postfix-sqlite postfix-pcre postgrey ca-certificates
 # * Set our name (the Debian default seems to be "localhost" but make it our hostname).
 # * Set the name of the local machine to localhost, which means xxx@localhost is delivered locally, although we don't use it.
 # * Set the SMTP banner (which must have the hostname first, then anything).
+# * Extend the SPF time limit to avoid timeouts chasing SPF records
 tools/editconf.py /etc/postfix/main.cf \
 	inet_interfaces=all \
 	smtp_bind_address=$PRIVATE_IP \
@@ -67,7 +68,8 @@ tools/editconf.py /etc/postfix/main.cf \
 tools/editconf.py /etc/postfix/main.cf \
 	delay_warning_time=3h \
 	maximal_queue_lifetime=2d \
-	bounce_queue_lifetime=1d
+	bounce_queue_lifetime=1d \
+	policy-spf_time_limit=3600
 
 # ### Outgoing Mail
 
@@ -97,6 +99,16 @@ tools/editconf.py /etc/postfix/master.cf -s -w \
 	  -o header_checks=pcre:/etc/postfix/outgoing_mail_header_filters
 	  -o nested_header_checks="
 
+# enable the SPF service
+tools/editconf.py /etc/postfix/master.cf -s -w \
+		"policy-spf=unix  y       n       n       -       0       spawn user=policyd-spf argv=/usr/bin/policyd-spf"
+
+# configure policyd-spf configuration
+# * reject SPF softfail (eg ~all) for some domains that are configured
+#   not to reject
+tools/editconf.py /etc/postfix-policyd-spf-python/policyd-spf.conf \
+	"Reject_Not_Pass_Domains=gmail.com,google.com"
+
 # Install the `outgoing_mail_header_filters` file required by the new 'authclean' service.
 cp conf/postfix_outgoing_mail_header_filters /etc/postfix/outgoing_mail_header_filters
 
@@ -208,7 +220,7 @@ tools/editconf.py /etc/postfix/main.cf lmtp_destination_recipient_limit=1
 # "450 4.7.1 Client host rejected: Service unavailable". This is a retry code, so the mail doesn't properly bounce. #NODOC
 tools/editconf.py /etc/postfix/main.cf \
 	smtpd_sender_restrictions="reject_non_fqdn_sender,reject_unknown_sender_domain,reject_authenticated_sender_login_mismatch,reject_rhsbl_sender dbl.spamhaus.org" \
-	smtpd_recipient_restrictions=permit_sasl_authenticated,permit_mynetworks,"reject_rbl_client zen.spamhaus.org",reject_unlisted_recipient,"check_policy_service inet:127.0.0.1:10023"
+	smtpd_recipient_restrictions=permit_sasl_authenticated,permit_mynetworks,"reject_rbl_client zen.spamhaus.org",reject_unlisted_recipient,"check_policy_service unix:private/policy-spf","check_policy_service inet:127.0.0.1:10023"
 
 # Postfix connects to Postgrey on the 127.0.0.1 interface specifically. Ensure that
 # Postgrey listens on the same interface (and not IPv6, for instance).

setup/mail-users.sh

@@ -5,54 +5,86 @@
 #
 # This script configures user authentication for Dovecot
 # and Postfix (which relies on Dovecot) and destination
-# validation by quering an Sqlite3 database of mail users.
+# validation by quering a ldap database of mail users.
+
+# LDAP helpful links:
+#   http://www.postfix.org/LDAP_README.html
+#   http://www.postfix.org/postconf.5.html
+#   http://www.postfix.org/ldap_table.5.html
+#
 
 source setup/functions.sh # load our functions
 source /etc/mailinabox.conf # load global vars
+source ${STORAGE_ROOT}/ldap/miab_ldap.conf # user-data specific vars
 
-# ### User and Alias Database
-
-# The database of mail users (i.e. authenticated users, who have mailboxes)
-# and aliases (forwarders).
-
-db_path=$STORAGE_ROOT/mail/users.sqlite
-
-# Create an empty database if it doesn't yet exist.
-if [ ! -f $db_path ]; then
-	echo Creating new user database: $db_path;
-	echo "CREATE TABLE users (id INTEGER PRIMARY KEY AUTOINCREMENT, email TEXT NOT NULL UNIQUE, password TEXT NOT NULL, extra, privileges TEXT NOT NULL DEFAULT '');" | sqlite3 $db_path;
-	echo "CREATE TABLE aliases (id INTEGER PRIMARY KEY AUTOINCREMENT, source TEXT NOT NULL UNIQUE, destination TEXT NOT NULL, permitted_senders TEXT);" | sqlite3 $db_path;
-fi
 
 # ### User Authentication
 
 # Have Dovecot query our database, and not system users, for authentication.
 sed -i "s/#*\(\!include auth-system.conf.ext\)/#\1/"  /etc/dovecot/conf.d/10-auth.conf
-sed -i "s/#\(\!include auth-sql.conf.ext\)/\1/"  /etc/dovecot/conf.d/10-auth.conf
+sed -i "s/#*\(\!include auth-sql.conf.ext\)/#\1/"  /etc/dovecot/conf.d/10-auth.conf
+sed -i "s/#\(\!include auth-ldap.conf.ext\)/\1/"  /etc/dovecot/conf.d/10-auth.conf
+
 
 # Specify how the database is to be queried for user authentication (passdb)
 # and where user mailboxes are stored (userdb).
-cat > /etc/dovecot/conf.d/auth-sql.conf.ext << EOF;
+cat > /etc/dovecot/conf.d/auth-ldap.conf.ext << EOF;
 passdb {
-  driver = sql
-  args = /etc/dovecot/dovecot-sql.conf.ext
+  driver = ldap
+  args = /etc/dovecot/dovecot-ldap.conf.ext
 }
 userdb {
-  driver = sql
-  args = /etc/dovecot/dovecot-sql.conf.ext
+  driver = ldap
+  args = /etc/dovecot/dovecot-userdb-ldap.conf.ext
+  default_fields = uid=mail gid=mail home=$STORAGE_ROOT/mail/mailboxes/%d/%n
 }
 EOF
 
-# Configure the SQL to query for a user's metadata and password.
-cat > /etc/dovecot/dovecot-sql.conf.ext << EOF;
-driver = sqlite
-connect = $db_path
-default_pass_scheme = SHA512-CRYPT
-password_query = SELECT email as user, password FROM users WHERE email='%u';
-user_query = SELECT email AS user, "mail" as uid, "mail" as gid, "$STORAGE_ROOT/mail/mailboxes/%d/%n" as home FROM users WHERE email='%u';
-iterate_query = SELECT email AS user FROM users;
+# Dovecot ldap configuration
+cat > /etc/dovecot/dovecot-ldap.conf.ext << EOF;
+# LDAP server(s) to connect to
+uris = ${LDAP_URL}
+tls = ${LDAP_SERVER_TLS}
+
+# Credentials dovecot uses to perform searches
+dn = ${LDAP_DOVECOT_DN}
+dnpass = ${LDAP_DOVECOT_PASSWORD}
+
+# Use ldap authentication binding for verifying users' passwords
+# otherwise we have to give dovecot admin access to the database
+# so it can read userPassword, which is less secure
+auth_bind = yes
+# default_pass_scheme = SHA512-CRYPT
+
+# Search base (subtree)
+base = ${LDAP_USERS_BASE}
+
+# Find the user:
+#   Dovecot uses its service account to search for the user using the
+#   filter below. If found, the user is authenticated against this dn
+#   (a bind is attempted as that user). The attribute 'mail' is
+#   multi-valued and contains all the user's email addresses. We use
+#   maildrop as the dovecot mailbox address and forbid then from using
+#   it for authentication by excluding maildrop from the filter.
+pass_filter = (&(objectClass=mailUser)(mail=%u))
+pass_attrs = maildrop=user
+
+# Apply per-user settings:
+#   Post-login information specific to the user (eg. quotas).  For
+#   lmtp delivery, pass_filter is not used, and postfix has already
+#   rewritten the envelope using the maildrop address.
+user_filter = (&(objectClass=mailUser)(|(mail=%u)(maildrop=%u)))
+user_attrs = maildrop=user
+
+# Account iteration for various dovecot tools (doveadm)
+iterate_filter = (objectClass=mailUser)
+iterate_attrs = maildrop=user
+
 EOF
-chmod 0600 /etc/dovecot/dovecot-sql.conf.ext # per Dovecot instructions
+chmod 0600 /etc/dovecot/dovecot-ldap.conf.ext # per Dovecot instructions
+
+# symlink userdb ext file per dovecot instructions
+ln -sf /etc/dovecot/dovecot-ldap.conf.ext /etc/dovecot/dovecot-userdb-ldap.conf.ext
 
 # Have Dovecot provide an authorization service that Postfix can access & use.
 cat > /etc/dovecot/conf.d/99-local-auth.conf << EOF;
@@ -65,6 +97,7 @@ service auth {
 }
 EOF
 
+#
 # And have Postfix use that service. We *disable* it here
 # so that authentication is not permitted on port 25 (which
 # does not run DKIM on relayed mail, so outbound mail isn't
@@ -81,44 +114,97 @@ tools/editconf.py /etc/postfix/main.cf \
 # prevent intra-domain spoofing by logged in but untrusted users in outbound
 # email. In all outbound mail (the sender has authenticated), the MAIL FROM
 # address (aka envelope or return path address) must be "owned" by the user
-# who authenticated. An SQL query will find who are the owners of any given
-# address.
+# who authenticated.
+#
+# sender-login-maps is given a FROM address (%s), which it uses to
+# obtain all the users that are permitted to MAIL FROM that address
+# (from the docs: "Optional lookup table with the SASL login names
+# that own the sender (MAIL FROM) addresses")
+# see: http://www.postfix.org/postconf.5.html
+#
+# With multiple lookup tables specified, the first matching lookup
+# ends the search. So, if there is a permitted-senders ldap group,
+# alias group memberships are not considered for inclusion that may
+# MAIL FROM the FROM address being searched for.
 tools/editconf.py /etc/postfix/main.cf \
-	smtpd_sender_login_maps=sqlite:/etc/postfix/sender-login-maps.cf
+	smtpd_sender_login_maps="ldap:/etc/postfix/sender-login-maps-explicit.cf, ldap:/etc/postfix/sender-login-maps-aliases.cf"
 
-# Postfix will query the exact address first, where the priority will be alias
-# records first, then user records. If there are no matches for the exact
-# address, then Postfix will query just the domain part, which we call
-# catch-alls and domain aliases. A NULL permitted_senders column means to
-# take the value from the destination column.
-cat > /etc/postfix/sender-login-maps.cf << EOF;
-dbpath=$db_path
-query = SELECT permitted_senders FROM (SELECT permitted_senders, 0 AS priority FROM aliases WHERE source='%s' AND permitted_senders IS NOT NULL UNION SELECT destination AS permitted_senders, 1 AS priority FROM aliases WHERE source='%s' AND permitted_senders IS NULL UNION SELECT email as permitted_senders, 2 AS priority FROM users WHERE email='%s') ORDER BY priority LIMIT 1;
+
+# FROM addresses with an explicit list of "permitted senders"
+cat > /etc/postfix/sender-login-maps-explicit.cf <<EOF
+server_host = ${LDAP_URL}
+bind = yes
+bind_dn = ${LDAP_POSTFIX_DN}
+bind_pw = ${LDAP_POSTFIX_PASSWORD}
+version = 3
+search_base = ${LDAP_PERMITTED_SENDERS_BASE}
+query_filter = (mail=%s)
+result_attribute = maildrop
+special_result_attribute = member
 EOF
+# protect the password
+chgrp postfix /etc/postfix/sender-login-maps-explicit.cf
+chmod 0640 /etc/postfix/sender-login-maps-explicit.cf
+
+# Users may MAIL FROM any of their own aliases
+cat > /etc/postfix/sender-login-maps-aliases.cf <<EOF
+server_host = ${LDAP_URL}
+bind = yes
+bind_dn = ${LDAP_POSTFIX_DN}
+bind_pw = ${LDAP_POSTFIX_PASSWORD}
+version = 3
+search_base = ${LDAP_USERS_BASE}
+query_filter = (mail=%s)
+result_attribute = maildrop
+special_result_attribute = member
+EOF
+chgrp postfix /etc/postfix/sender-login-maps-aliases.cf
+chmod 0640 /etc/postfix/sender-login-maps-aliases.cf
+
 
 # ### Destination Validation
 
-# Use a Sqlite3 database to check whether a destination email address exists,
-# and to perform any email alias rewrites in Postfix.
+# Check whether a destination email address exists, and to perform any
+# email alias rewrites in Postfix.
 tools/editconf.py /etc/postfix/main.cf \
-	virtual_mailbox_domains=sqlite:/etc/postfix/virtual-mailbox-domains.cf \
-	virtual_mailbox_maps=sqlite:/etc/postfix/virtual-mailbox-maps.cf \
-	virtual_alias_maps=sqlite:/etc/postfix/virtual-alias-maps.cf \
+	virtual_mailbox_domains=ldap:/etc/postfix/virtual-mailbox-domains.cf \
+	virtual_mailbox_maps=ldap:/etc/postfix/virtual-mailbox-maps.cf \
+	virtual_alias_maps=ldap:/etc/postfix/virtual-alias-maps.cf \
 	local_recipient_maps=\$virtual_mailbox_maps
 
-# SQL statement to check if we handle incoming mail for a domain, either for users or aliases.
-cat > /etc/postfix/virtual-mailbox-domains.cf << EOF;
-dbpath=$db_path
-query = SELECT 1 FROM users WHERE email LIKE '%%@%s' UNION SELECT 1 FROM aliases WHERE source LIKE '%%@%s'
-EOF
 
-# SQL statement to check if we handle incoming mail for a user.
-cat > /etc/postfix/virtual-mailbox-maps.cf << EOF;
-dbpath=$db_path
-query = SELECT 1 FROM users WHERE email='%s'
+# the domains we handle mail for
+cat > /etc/postfix/virtual-mailbox-domains.cf << EOF
+server_host = ${LDAP_URL}
+bind = yes
+bind_dn = ${LDAP_POSTFIX_DN}
+bind_pw = ${LDAP_POSTFIX_PASSWORD}
+version = 3
+search_base = ${LDAP_DOMAINS_BASE}
+query_filter = (&(dc=%s)(businessCategory=mail))
+result_attribute = dc
 EOF
+chgrp postfix /etc/postfix/virtual-mailbox-domains.cf
+chmod 0640 /etc/postfix/virtual-mailbox-domains.cf
 
-# SQL statement to rewrite an email address if an alias is present.
+# check if we handle incoming mail for a user.
+# (this doesn't seem to ever be used by postfix)
+cat > /etc/postfix/virtual-mailbox-maps.cf << EOF
+server_host = ${LDAP_URL}
+bind = yes
+bind_dn = ${LDAP_POSTFIX_DN}
+bind_pw = ${LDAP_POSTFIX_PASSWORD}
+version = 3
+search_base = ${LDAP_USERS_BASE}
+query_filter = (&(objectClass=mailUser)(mail=%s)(!(|(maildrop="*|*")(maildrop="*:*")(maildrop="*/*"))))
+result_attribute = maildrop
+EOF
+chgrp postfix /etc/postfix/virtual-mailbox-maps.cf
+chmod 0640 /etc/postfix/virtual-mailbox-maps.cf
+
+
+
+# Rewrite an email address if an alias is present.
 #
 # Postfix makes multiple queries for each incoming mail. It first
 # queries the whole email address, then just the user part in certain
@@ -142,10 +228,26 @@ EOF
 # Since we might have alias records with an empty destination because
 # it might have just permitted_senders, skip any records with an
 # empty destination here so that other lower priority rules might match.
-cat > /etc/postfix/virtual-alias-maps.cf << EOF;
-dbpath=$db_path
-query = SELECT destination from (SELECT destination, 0 as priority FROM aliases WHERE source='%s' AND destination<>'' UNION SELECT email as destination, 1 as priority FROM users WHERE email='%s') ORDER BY priority LIMIT 1;
+
+
+#
+# This is the ldap version of aliases(5) but for virtual
+# addresses. Postfix queries this recursively to determine delivery
+# addresses. Aliases may be addresses, domains, and catch-alls.
+# 
+cat > /etc/postfix/virtual-alias-maps.cf <<EOF
+server_host = ${LDAP_URL}
+bind = yes
+bind_dn = ${LDAP_POSTFIX_DN}
+bind_pw = ${LDAP_POSTFIX_PASSWORD}
+version = 3
+search_base = ${LDAP_USERS_BASE}
+query_filter = (mail=%s)
+result_attribute = maildrop, rfc822MailMember
+special_result_attribute = member
 EOF
+chgrp postfix /etc/postfix/virtual-alias-maps.cf
+chmod 0640 /etc/postfix/virtual-alias-maps.cf
 
 # Restart Services
 ##################
@@ -153,4 +255,3 @@ EOF
 restart_service postfix
 restart_service dovecot
 
-

setup/management.sh

@@ -50,7 +50,7 @@ hide_output $venv/bin/pip install --upgrade pip
 hide_output $venv/bin/pip install --upgrade \
 	rtyaml "email_validator>=1.0.0" "exclusiveprocess" \
 	flask dnspython python-dateutil \
-	"idna>=2.0.0" "cryptography==2.2.2" boto psutil
+	"idna>=2.0.0" "cryptography==2.2.2" boto psutil ldap3
 
 # CONFIGURATION
 

setup/migrate.py

@@ -1,4 +1,5 @@
 #!/usr/bin/python3
+# -*- indent-tabs-mode: t; tab-width: 8; python-indent-offset: 8; -*-
 
 # Migrates any file structures, database schemas, etc. between versions of Mail-in-a-Box.
 
@@ -8,7 +9,7 @@
 import sys, os, os.path, glob, re, shutil
 
 sys.path.insert(0, 'management')
-from utils import load_environment, save_environment, shell
+from utils import load_environment, load_env_vars_from_file, save_environment, shell
 
 def migration_1(env):
 	# Re-arrange where we store SSL certificates. There was a typo also.
@@ -181,6 +182,65 @@ def migration_12(env):
             conn.commit()
             conn.close()
 
+def migration_13(env):
+	# This migration step moves users from sqlite3 to openldap
+
+	# users table:
+	# for each row create an ldap entry of the form:
+	#   dn: uid=[uuid],ou=Users,dc=mailinabox
+	#   objectClass: inetOrgPerson, mailUser, shadowAccount
+	#   mail: [email]
+	#   maildrop: [email]
+	#   userPassword: [password]
+	#   mailaccess: [privilege]   # multi-valued
+	# 
+	# aliases table:
+	# for each row create an ldap entry of the form:
+	#   dn: cn=[uuid],ou=aliases,ou=Users,dc=mailinabox
+	#   objectClass: mailGroup
+	#   mail: [source]
+	#   member: [destination-dn]   # multi-valued
+	#   rfc822MailMember: [email]  # multi-values
+	#
+	# if the alias has permitted_senders, create:
+	#   dn: cn=[uuid],ou=permitted-senders,ou=Config,dc=mailinabox
+	#   objectClass: mailGroup
+	#   mail: [source]
+	#   member: [user-dn]         # multi-valued
+
+	print("Migrating users and aliases from sqlite to ldap")
+	
+	# Get the ldap server up and running
+	shell("check_call", ["setup/ldap.sh", "-v"])
+	
+	import sqlite3, ldap3
+	import migration_13 as m13
+				
+	# 2. get ldap site details (miab_ldap.conf was created by ldap.sh)
+	ldapvars = load_env_vars_from_file(os.path.join(env["STORAGE_ROOT"], "ldap/miab_ldap.conf"), strip_quotes=True)
+	ldap_base = ldapvars.LDAP_BASE
+	ldap_domains_base = ldapvars.LDAP_DOMAINS_BASE
+	ldap_permitted_senders_base = ldapvars.LDAP_PERMITTED_SENDERS_BASE
+	ldap_users_base = ldapvars.LDAP_USERS_BASE
+	ldap_aliases_base = ldapvars.LDAP_ALIASES_BASE
+	ldap_services_base = ldapvars.LDAP_SERVICES_BASE
+	ldap_admin_dn = ldapvars.LDAP_ADMIN_DN
+	ldap_admin_pass = ldapvars.LDAP_ADMIN_PASSWORD
+
+	# 3. connect
+	conn = sqlite3.connect(os.path.join(env["STORAGE_ROOT"], "mail/users.sqlite"))
+	ldap = ldap3.Connection('127.0.0.1', ldap_admin_dn, ldap_admin_pass, raise_exceptions=True)
+	ldap.bind()
+	
+	# 4. perform the migration
+	users=m13.create_users(env, conn, ldap, ldap_base, ldap_users_base, ldap_domains_base)
+	aliases=m13.create_aliases(conn, ldap, ldap_aliases_base)
+	permitted=m13.create_permitted_senders(conn, ldap, ldap_users_base, ldap_permitted_senders_base)
+	m13.populate_aliases(conn, ldap, users, aliases)
+		
+	ldap.unbind()
+	conn.close()
+	
 
 def get_current_migration():
 	ver = 0

setup/migration_13.py

@@ -0,0 +1,220 @@
+#!/usr/bin/python3
+# -*- indent-tabs-mode: t; tab-width: 4; python-indent-offset: 4; -*-
+
+#
+# helper functions for migration #13
+#
+
+import uuid, os, sqlite3, ldap3
+
+
+def add_user(env, ldapconn, search_base, users_base, domains_base, email, password, privs, cn=None):
+	# Add a sqlite user to ldap
+	#   env are the environment variables
+	#   ldapconn is the bound ldap connection
+	#   search_base is for finding a user with the same email
+	#   users_base is the rdn where the user will be added
+	#   domains_base is the rdn for 'domain' entries
+	#   email is the user's email
+	#   password is the user's current sqlite password hash
+	#   privs is an array of privilege names for the user
+	#   cn is the user's common name [optional]
+	#
+	# the email address should be as-is from sqlite (encoded as
+	# ascii using IDNA rules)
+
+	# If the email address exists, return and do nothing
+	ldapconn.search(search_base, "(mail=%s)" % email)
+	if len(ldapconn.entries) > 0:
+		print("user already exists: %s" % email)
+		return ldapconn.response[0]['dn']
+
+	# Generate a unique id for uid
+	uid = '%s' % uuid.uuid4()
+
+	# Attributes to apply to the new ldap entry
+	attrs = {
+		"mail" : email,
+		"maildrop" : email,
+		"uid" : uid,
+		# Openldap uses prefix {CRYPT} for all crypt(3) formats
+		"userPassword" : password.replace('{SHA512-CRYPT}','{CRYPT}')
+	}
+
+	# Add privileges ('mailaccess' attribute)
+	privs_uniq = {}
+	for priv in privs:
+		if priv.strip() != '': privs_uniq[priv] = True
+	if len(privs_uniq) > 0:
+		attrs['mailaccess'] = privs_uniq.keys()
+
+	# Get a common name
+	localpart, domainpart = email.split("@")
+
+	if cn is None:
+		# Get the name for the email address from Roundcube and
+		# use that or `localpart` if no name
+		rconn = sqlite3.connect(os.path.join(env["STORAGE_ROOT"], "mail/roundcube/roundcube.sqlite"))
+		rc = rconn.cursor()
+		rc.execute("SELECT name FROM identities WHERE email = ? AND standard = 1 AND del = 0 AND name <> ''", (email,))
+		rc_all = rc.fetchall()
+		if len(rc_all)>0:
+			cn = rc_all[0][0]
+			attrs["displayName"] = cn
+		else:
+			cn = localpart.replace('.',' ').replace('_',' ')
+		rconn.close()
+	attrs["cn"] = cn
+
+	# Choose a surname for the user (required attribute)
+	attrs["sn"] = cn[cn.find(' ')+1:]
+
+	# Add user
+	dn = "uid=%s,%s" % (uid, users_base)
+	print("adding user %s" % email)
+	ldapconn.add(dn,
+		     [ 'inetOrgPerson','mailUser','shadowAccount' ],
+		     attrs);			 
+
+	# Create domain entry indicating that we are handling
+	# mail for that domain
+	domain_dn = 'dc=%s,%s' % (domainpart, domains_base)
+	try:
+		ldapconn.add(domain_dn, [ 'domain' ], {
+			"businessCategory": "mail"
+		})
+	except ldap3.core.exceptions.LDAPEntryAlreadyExistsResult:
+		pass
+	return dn
+
+
+def create_users(env, conn, ldapconn, ldap_base, ldap_users_base, ldap_domains_base):
+	# iterate through sqlite 'users' table and create each user in
+	# ldap. returns a map of email->dn
+	c = conn.cursor()
+	c.execute("SELECT email,password,privileges from users")
+	users = {}
+	for row in c:
+		email=row[0]
+		password=row[1]
+		privs=row[2]
+		dn = add_user(env, ldapconn, ldap_base, ldap_users_base, ldap_domains_base, email, password, privs.split("\n"))
+		users[email] = dn
+	return users
+
+
+def create_aliases(conn, ldapconn, aliases_base):
+	# iterate through sqlite 'aliases' table and create ldap
+	# aliases but without members.  returns a map of alias->dn
+	aliases={}
+	c = conn.cursor()
+	for row in c.execute("SELECT source FROM aliases WHERE destination<>''"):
+		alias=row[0]
+		ldapconn.search(aliases_base, "(mail=%s)" % alias)
+		if len(ldapconn.entries) > 0:
+			# Already present
+			print("alias already exists %s" % alias)
+			aliases[alias] = ldapconn.response[0]['dn']
+		else:
+			cn="%s" % uuid.uuid4()
+			dn="cn=%s,%s" % (cn, aliases_base)
+			print("adding alias %s" % alias)
+			ldapconn.add(dn, ['mailGroup'], {
+				"mail": alias,
+				"description": "Mail group %s" % alias
+			})
+			aliases[alias] = dn
+	return aliases
+
+
+def populate_aliases(conn, ldapconn, users_map, aliases_map):
+	# populate alias with members.
+	# conn is a connection to the users sqlite database
+	# ldapconn is a connecton to the ldap database
+	# users_map is a map of email -> dn for every user on the system
+	# aliases_map is a map of email -> dn for every pre-created alias
+	#
+	# email addresses should be encoded as-is from sqlite (IDNA
+	# domains)
+	c = conn.cursor()
+	for row in c.execute("SELECT source,destination FROM aliases where destination<>''"):
+		alias=row[0]
+		alias_dn=aliases_map[alias]
+		members = []
+		mailMembers = []
+		
+		for email in row[1].split(','):
+			email=email.strip()
+			if email=="":
+				continue
+			elif email in users_map:
+				members.append(users_map[email])
+			elif email in aliases_map:
+				members.append(aliases_map[email])
+			else:
+				mailMembers.append(email)
+		
+		print("populate alias group %s" % alias)
+		changes = {}
+		if len(members)>0:
+			changes["member"]=[(ldap3.MODIFY_REPLACE, members)]
+		if len(mailMembers)>0:
+			changes["rfc822MailMember"]=[(ldap3.MODIFY_REPLACE, mailMembers)]			
+		ldapconn.modify(alias_dn, changes)
+
+
+def add_permitted_senders_group(ldapconn, users_base, group_base, source, permitted_senders):
+	# creates a single permitted_senders ldap group
+	#
+	# email addresses should be encoded as-is from sqlite (IDNA
+	# domains)
+
+	# If the group already exists, return and do nothing
+	ldapconn.search(group_base, "(&(objectClass=mailGroup)(mail=%s))" % source)
+	if len(ldapconn.entries) > 0:
+		return ldapconn.response[0]['dn']
+
+	# get a dn for every permitted sender
+	permitted_dn = {}
+	for email in permitted_senders:
+		email = email.strip()
+		if email == "": continue
+		ldapconn.search(users_base, "(mail=%s)" % email)
+		for result in ldapconn.response:
+			permitted_dn[result["dn"]] = True
+	if len(permitted_dn) == 0:
+		return None
+
+	# add permitted senders group for the 'source' email
+	gid = '%s' % uuid.uuid4()
+	group_dn = "cn=%s,%s" % (gid, group_base)
+	print("adding permitted senders group for %s" % source)
+	try:
+		ldapconn.add(group_dn, [ "mailGroup" ], {
+			"cn" : gid,
+			"mail" : source,
+			"member" : permitted_dn.keys(),
+			"description": "Permitted to MAIL FROM this address"
+		})
+	except ldap3.core.exceptions.LDAPEntryAlreadyExistsResult:
+		pass
+	return group_dn
+
+
+def create_permitted_senders(conn, ldapconn, users_base, group_base):
+	# iterate through the 'aliases' table and create all
+	# permitted-senders groups
+	c = conn.cursor()
+	c.execute("SELECT source, permitted_senders from aliases WHERE permitted_senders is not null")
+	groups={}
+	for row in c:
+		source=row[0]
+		senders=[]
+		for line in row[1].split("\n"):
+			for sender in line.split(","):
+				if sender.strip() != "":
+					senders.append(sender.strip())
+		dn=add_permitted_senders_group(ldapconn, users_base, group_base, source, senders)
+		if dn is not None:
+			groups[source] = dn
+	return groups

setup/questions.sh

@@ -9,7 +9,7 @@ if [ -z "${NONINTERACTIVE:-}" ]; then
 	if [ ! -f /usr/bin/dialog ] || [ ! -f /usr/bin/python3 ] || [ ! -f /usr/bin/pip3 ]; then
 		echo Installing packages needed for setup...
 		apt-get -q -q update
-		apt_get_quiet install dialog python3 python3-pip  || exit 1
+		apt_get_quiet install dialog python3 python3-pip python3-ldap3 || exit 1
 	fi
 
 	# Installing email_validator is repeated in setup/management.sh, but in setup/management.sh

setup/ssl.sh

@@ -3,8 +3,9 @@
 # RSA private key, SSL certificate, Diffie-Hellman bits files
 # -------------------------------------------
 
-# Create an RSA private key, a self-signed SSL certificate, and some
-# Diffie-Hellman cipher bits, if they have not yet been created.
+# Create an RSA private key, a SSL certificate signed by a generated
+# CA, and some Diffie-Hellman cipher bits, if they have not yet been
+# created.
 #
 # The RSA private key and certificate are used for:
 #
@@ -12,6 +13,7 @@
 #  * IMAP
 #  * SMTP (opportunistic TLS for port 25 and submission on port 587)
 #  * HTTPS
+#  * SLAPD (OpenLDAP server)
 #
 # The certificate is created with its CN set to the PRIMARY_HOSTNAME. It is
 # also used for other domains served over HTTPS until the user installs a
@@ -25,8 +27,10 @@ source setup/functions.sh # load our functions
 source /etc/mailinabox.conf # load global vars
 
 # Show a status line if we are going to take any action in this file.
-if  [ ! -f /usr/bin/openssl ] \
- || [ ! -f $STORAGE_ROOT/ssl/ssl_private_key.pem ] \
+if	[ ! -f /usr/bin/openssl ] \
+ || [ ! -s $STORAGE_ROOT/ssl/ca_private_key.pem ] \
+ || [ ! -f $STORAGE_ROOT/ssl/ca_certificate.pem ] \
+ || [ ! -s $STORAGE_ROOT/ssl/ssl_private_key.pem ] \
  || [ ! -f $STORAGE_ROOT/ssl/ssl_certificate.pem ] \
  || [ ! -f $STORAGE_ROOT/ssl/dh2048.pem ]; then
 	echo "Creating initial SSL certificate and perfect forward secrecy Diffie-Hellman parameters..."
@@ -40,9 +44,9 @@ apt_install openssl
 
 mkdir -p $STORAGE_ROOT/ssl
 
-# Generate a new private key.
+# Generate new private keys.
 #
-# The key is only as good as the entropy available to openssl so that it
+# Keys are only as good as the entropy available to openssl so that it
 # can generate a random key. "OpenSSL’s built-in RSA key generator ....
 # is seeded on first use with (on Linux) 32 bytes read from /dev/urandom,
 # the process ID, user ID, and the current time in seconds. [During key
@@ -52,40 +56,144 @@ mkdir -p $STORAGE_ROOT/ssl
 #
 # A perfect storm of issues can cause the generated key to be not very random:
 #
-#   * improperly seeded /dev/urandom, but see system.sh for how we mitigate this
-#   * the user ID of this process is always the same (we're root), so that seed is useless
-#   * zero'd memory (plausible on embedded systems, cloud VMs?)
-#   * a predictable process ID (likely on an embedded/virtualized system)
-#   * a system clock reset to a fixed time on boot
+#	* improperly seeded /dev/urandom, but see system.sh for how we mitigate this
+#	* the user ID of this process is always the same (we're root), so that seed is useless
+#	* zero'd memory (plausible on embedded systems, cloud VMs?)
+#	* a predictable process ID (likely on an embedded/virtualized system)
+#	* a system clock reset to a fixed time on boot
 #
 # Since we properly seed /dev/urandom in system.sh we should be fine, but I leave
 # in the rest of the notes in case that ever changes.
-if [ ! -f $STORAGE_ROOT/ssl/ssl_private_key.pem ]; then
+if [ ! -s $STORAGE_ROOT/ssl/ca_private_key.pem ]; then
 	# Set the umask so the key file is never world-readable.
 	(umask 077; hide_output \
-		openssl genrsa -out $STORAGE_ROOT/ssl/ssl_private_key.pem 2048)
+		openssl genrsa -aes256 -passout 'pass:SECRET-PASSWORD' \
+		-out $STORAGE_ROOT/ssl/ca_private_key.pem 4096)
+
+	# remove the existing ca-certificate, it must be regenerated
+	rm -f $STORAGE_ROOT/ssl/ca_certificate.pem
+	
+	# Remove the ssl_certificate.pem symbolic link to force a
+	# regeneration of a self-signed server certificate. Old certs need
+	# to be signed by the new ca.
+	if [ -L $STORAGE_ROOT/ssl/ssl_certificate.pem ]; then
+        # Get the name of the certificate issuer
+        issuer="$(openssl x509 -issuer -nocert -in $STORAGE_ROOT/ssl/ssl_certificate.pem)"
+        
+        # Determine if the ssl cert if self-signed. If unique hashes is 1,
+        # the cert is self-signed (pior versions of MiaB used self-signed
+        # certs).
+        uniq_hashes="$(openssl x509 -subject_hash -issuer_hash -nocert -in $STORAGE_ROOT/ssl/ssl_certificate.pem | uniq | wc -l)"
+        
+        if [ "$uniq_hashes" == "1" ] || grep "Temporary-Mail-In-A-Box-CA" <<<"$issuer" >/dev/null
+        then
+		    rm -f $STORAGE_ROOT/ssl/ssl_certificate.pem
+	    fi
+    fi
 fi
 
-# Generate a self-signed SSL certificate because things like nginx, dovecot,
+if [ ! -s $STORAGE_ROOT/ssl/ssl_private_key.pem ]; then
+	# Set the umask so the key file is never world-readable.
+	(umask 037; hide_output \
+		openssl genrsa -out $STORAGE_ROOT/ssl/ssl_private_key.pem 2048)
+
+	# Give the group 'ssl-cert' read access so slapd can read it
+	groupadd -fr ssl-cert
+	chgrp ssl-cert $STORAGE_ROOT/ssl/ssl_private_key.pem
+	chmod g+r $STORAGE_ROOT/ssl/ssl_private_key.pem
+	
+	# Remove the ssl_certificate.pem symbolic link to force a
+	# regeneration of the server certificate. It needs to be
+	# signed by the new ca.
+	if [ -L $STORAGE_ROOT/ssl/ssl_certificate.pem ]; then
+		rm -f $STORAGE_ROOT/ssl/ssl_certificate.pem
+	fi
+fi
+
+#
+# Generate a root CA certificate
+#
+if [ ! -f $STORAGE_ROOT/ssl/ca_certificate.pem ]; then
+	# Generate the self-signed certificate.
+	CERT=$STORAGE_ROOT/ssl/ca_certificate.pem
+	hide_output \
+	openssl req -new -x509 \
+	  -days 3650 -sha256 \
+	  -key $STORAGE_ROOT/ssl/ca_private_key.pem \
+	  -passin 'pass:SECRET-PASSWORD' \
+	  -out $CERT \
+	  -subj '/CN=Temporary-Mail-In-A-Box-CA'
+
+	# add the certificate to the system's trusted root ca list
+	# this is required for openldap's TLS implementation
+	hide_output \
+	  cp $CERT /usr/local/share/ca-certificates/mailinabox.crt
+	hide_output \
+	  update-ca-certificates
+fi
+
+# Generate a signed SSL certificate because things like nginx, dovecot,
 # etc. won't even start without some certificate in place, and we need nginx
 # so we can offer the user a control panel to install a better certificate.
 if [ ! -f $STORAGE_ROOT/ssl/ssl_certificate.pem ]; then
-	# Generate a certificate signing request.
+	# # Generate a certificate signing request.
 	CSR=/tmp/ssl_cert_sign_req-$$.csr
 	hide_output \
 	openssl req -new -key $STORAGE_ROOT/ssl/ssl_private_key.pem -out $CSR \
 	  -sha256 -subj "/CN=$PRIMARY_HOSTNAME"
 
-	# Generate the self-signed certificate.
-	CERT=$STORAGE_ROOT/ssl/$PRIMARY_HOSTNAME-selfsigned-$(date --rfc-3339=date | sed s/-//g).pem
+	# create a ca database (directory) for openssl
+	CADIR=$STORAGE_ROOT/ssl/ca
+	mkdir -p $CADIR/newcerts
+	touch $CADIR/index.txt $CADIR/index.txt.attr
+	[ ! -e $CADIR/serial ] && date +%s > $CADIR/serial
+
+	# Generate the signed certificate.
+	CERT=$STORAGE_ROOT/ssl/$PRIMARY_HOSTNAME-cert-$(date --rfc-3339=date | sed s/-//g).pem
 	hide_output \
-	openssl x509 -req -days 365 \
-	  -in $CSR -signkey $STORAGE_ROOT/ssl/ssl_private_key.pem -out $CERT
+	openssl ca -batch \
+		-keyfile $STORAGE_ROOT/ssl/ca_private_key.pem \
+		-cert $STORAGE_ROOT/ssl/ca_certificate.pem \
+		-passin 'pass:SECRET-PASSWORD' \
+		-in $CSR \
+		-out $CERT \
+		-days 365 \
+		-name miab_ca \
+		-config - <<< "
+[ miab_ca ]
+dir		= $CADIR
+certs		= \$dir
+database	= \$dir/index.txt
+unique_subject	= no
+new_certs_dir	= \$dir/newcerts	# default place for new certs.
+serial		= \$dir/serial		# The current serial number
+x509_extensions	= server_cert		# The extensions to add to the cert
+name_opt	= ca_default		# Subject Name options
+cert_opt	= ca_default		# Certificate field options
+policy		= policy_anything
+default_md	= default		# use public key default MD
+
+[ policy_anything ]
+countryName		= optional
+stateOrProvinceName	= optional
+localityName		= optional
+organizationName	= optional
+organizationalUnitName	= optional
+commonName		= supplied
+emailAddress		= optional
+
+[ server_cert ]
+basicConstraints	= CA:FALSE
+nsCertType		= server
+nsComment		= \"Mail-In-A-Box Generated Certificate\"
+subjectKeyIdentifier	= hash
+authorityKeyIdentifier	= keyid,issuer
+"
 
 	# Delete the certificate signing request because it has no other purpose.
 	rm -f $CSR
 
-	# Symlink the certificate into the system certificate path, so system services
+	# Symlink the certificates into the system certificate path, so system services
 	# can find it.
 	ln -s $CERT $STORAGE_ROOT/ssl/ssl_certificate.pem
 fi

setup/start.sh

@@ -99,6 +99,7 @@ EOF
 source setup/system.sh
 source setup/ssl.sh
 source setup/dns.sh
+source setup/ldap.sh
 source setup/mail-postfix.sh
 source setup/mail-dovecot.sh
 source setup/mail-users.sh

setup/webmail.sh

@@ -4,6 +4,7 @@
 
 source setup/functions.sh # load our functions
 source /etc/mailinabox.conf # load global vars
+source ${STORAGE_ROOT}/ldap/miab_ldap.conf
 
 # ### Installing Roundcube
 
@@ -131,6 +132,33 @@ cat > $RCM_CONFIG <<EOF;
 \$config['login_autocomplete'] = 2;
 \$config['password_charset'] = 'UTF-8';
 \$config['junk_mbox'] = 'Spam';
+\$config['ldap_public']['public'] = array(
+    'name'              => 'Directory',
+    'hosts'             => array('${LDAP_SERVER}'),
+    'port'              => ${LDAP_SERVER_PORT},
+    'user_specific'     => false,
+    'scope'             => 'sub',
+    'base_dn'           => '${LDAP_USERS_BASE}',
+    'bind_dn'           => '${LDAP_WEBMAIL_DN}',
+    'bind_pass'         => '${LDAP_WEBMAIL_PASSWORD}',
+    'writable'          => false,
+    'ldap_version'      => 3,
+    'search_fields'     => array( 'mail' ),
+    'name_field'        => 'mail',
+    'email_field'       => 'mail',
+    'sort'              => 'mail',
+    'filter'            => '(objectClass=mailUser)',
+    'fuzzy_search'      => false,
+    'global_search'     => true,
+    # 'groups'            => array(
+    #     'base_dn'         => '${LDAP_ALIASES_BASE}',
+    #     'filter'          => '(objectClass=mailGroup)',
+    # 	'member_attr'     => 'member',
+    # 	'scope'           => 'sub',
+    # 	'name_attr'       => 'mail',
+    # 	'member_filter'   => '(|(objectClass=mailGroup)(objectClass=mailUser))',
+    # )
+);
 ?>
 EOF
 
@@ -169,22 +197,21 @@ cp ${RCM_PLUGIN_DIR}/password/config.inc.php.dist \
 	${RCM_PLUGIN_DIR}/password/config.inc.php
 
 tools/editconf.py ${RCM_PLUGIN_DIR}/password/config.inc.php \
-	"\$config['password_minimum_length']=8;" \
-	"\$config['password_db_dsn']='sqlite:///$STORAGE_ROOT/mail/users.sqlite';" \
-	"\$config['password_query']='UPDATE users SET password=%D WHERE email=%u';" \
-	"\$config['password_dovecotpw']='/usr/bin/doveadm pw';" \
-	"\$config['password_dovecotpw_method']='SHA512-CRYPT';" \
-	"\$config['password_dovecotpw_with_method']=true;"
-
-# so PHP can use doveadm, for the password changing plugin
-usermod -a -G dovecot www-data
-
-# set permissions so that PHP can use users.sqlite
-# could use dovecot instead of www-data, but not sure it matters
-chown root.www-data $STORAGE_ROOT/mail
-chmod 775 $STORAGE_ROOT/mail
-chown root.www-data $STORAGE_ROOT/mail/users.sqlite
-chmod 664 $STORAGE_ROOT/mail/users.sqlite
+	"\$config['password_driver']='ldap';" \
+	"\$config['password_ldap_host']='${LDAP_SERVER}';" \
+	"\$config['password_ldap_port']=${LDAP_SERVER_PORT};" \
+	"\$config['password_ldap_starttls']=$([ ${LDAP_SERVER_STARTTLS} == yes ] && echo true || echo false);" \
+	"\$config['password_ldap_basedn']='${LDAP_BASE}';" \
+	"\$config['password_ldap_userDN_mask']=null;" \
+	"\$config['password_ldap_searchDN']='${LDAP_WEBMAIL_DN}';" \
+	"\$config['password_ldap_searchPW']='${LDAP_WEBMAIL_PASSWORD}';" \
+	"\$config['password_ldap_search_base']='${LDAP_USERS_BASE}';" \
+	"\$config['password_ldap_search_filter']='(&(objectClass=mailUser)(mail=%login))';" \
+	"\$config['password_ldap_encodage']='default';" \
+	"\$config['password_ldap_lchattr']='shadowLastChange';" \
+	"\$config['password_algorithm']='sha512-crypt';" \
+	"\$config['password_algorithm_prefix']='{CRYPT}';" \
+	"\$config['password_minimum_length']=8;"
 
 # Fix Carddav permissions:
 chown -f -R root.www-data ${RCM_PLUGIN_DIR}/carddav
@@ -197,5 +224,5 @@ chown www-data:www-data $STORAGE_ROOT/mail/roundcube/roundcube.sqlite
 chmod 664 $STORAGE_ROOT/mail/roundcube/roundcube.sqlite
 
 # Enable PHP modules.
-phpenmod -v php mcrypt imap
+phpenmod -v php mcrypt imap ldap
 restart_service php7.2-fpm

tests/.gitignore

@@ -0,0 +1 @@
+out

tests/prep_vm.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+# Run this on a VM to pre-install all the packages, then
+# take a snapshot - it will greatly speed up subsequent
+# test installs
+
+
+remove_line_continuation() {
+    local file="$1"
+    awk '
+BEGIN            { C=0 } 
+C==1 && /[^\\]$/ { C=0; print $0; next } 
+C==1             { printf("%s",substr($0,0,length($0)-1)); next } 
+/\\$/            { C=1; printf("%s",substr($0,0,length($0)-1)); next } 
+                 { print $0 }' \
+                     "$file"
+}
+
+install_packages() {
+    while read line; do
+        pkgs=""
+        case "$line" in
+             apt_install* )
+                 pkgs="$(cut -c12- <<<"$line")"
+                 ;;
+             "apt-get install"* )
+                 pkgs="$(cut -c16- <<<"$line")"
+                 ;;
+             "apt install"* )
+                 pkgs="$(cut -c12- <<<"$line")"
+                 ;;
+        esac
+        
+        # don't install postfix - causes problems with setup scripts
+        pkgs="$(sed s/postfix//g <<<"$pkgs")"
+        
+        if [ ! -z "$pkgs" ]; then
+            echo "install: $pkgs"
+            apt-get install $pkgs -y
+        fi
+    done
+}
+
+apt-get update -y
+apt-get upgrade -y
+apt-get autoremove -y
+
+for file in $(ls setup/*.sh); do
+    remove_line_continuation "$file" | install_packages
+done
+
+apt-get install openssh-server -y
+apt-get install emacs-nox -y
+
+echo ""
+echo ""
+echo "Done. Take a snapshot...."
+echo ""

tests/runner.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+# -*- indent-tabs-mode: t; tab-width: 4; -*-
+
+#
+# Runner for test suites
+#
+
+# operate from the runner's directory
+cd "$(dirname $0)"
+
+# load global functions and variables
+. suites/_init.sh
+
+runner_suites=(
+	ldap-connection
+	ldap-access
+	mail-basic
+	mail-from
+	mail-aliases
+	mail-access
+	management-users
+)
+
+usage() {
+	echo ""
+	echo "Usage: $(basename $0) [-failfatal] [suite-name ...]"
+	echo "Valid suite names:"
+	for runner_suite in ${runner_suites[@]}; do
+		echo "	 $runner_suite"
+	done
+	echo "If no suite-name(s) given, all suites are run"
+	echo ""
+	echo "Options:"
+	echo "	 -failfatal	   The runner will stop if any test fails"
+	echo ""
+	echo "Output directory: $(dirname $0)/${base_outputdir}"
+	echo ""
+	exit 1
+}
+
+# process command line
+while [ $# -gt 0 ]; do
+	case "$1" in
+		-failfatal )
+			# failure is fatal (via global option, see _init.sh)
+			FAILURE_IS_FATAL=yes
+			;;
+		-* )
+			echo "Invalid argument $1" 1>&2
+			usage
+			;;
+		* )
+			# run named suite
+			if array_contains "$1" ${runner_suites[@]}; then
+				. "suites/$1.sh"
+			else
+				echo "Unknown suite '$1'" 1>&2
+				usage
+			fi
+			;;
+	esac
+	shift
+done
+
+# if no suites specified on command line, run all suites
+if [ $OVERALL_COUNT_SUITES -eq 0 ]; then
+	rm -rf "${base_outputdir}"
+	for runner_suite in ${runner_suites[@]}; do
+		. suites/$runner_suite.sh
+	done
+fi
+
+echo ""
+echo "Done"
+echo "$OVERALL_COUNT tests ($OVERALL_SUCCESSES success/$OVERALL_FAILURES failures) in $OVERALL_COUNT_SUITES test suites"
+
+if [ $OVERALL_FAILURES -gt 0 ]; then
+	exit 1
+else
+	exit 0
+fi

tests/suites/_init.sh

@@ -0,0 +1,176 @@
+# -*- indent-tabs-mode: t; tab-width: 4; -*-
+
+# load useful functions from setup
+. ../setup/functions.sh || exit 1
+. ../setup/functions-ldap.sh || exit 1
+set +eu
+
+# load test suite helper functions
+. suites/_ldap-functions.sh || exit 1
+. suites/_mail-functions.sh || exit 1
+. suites/_mgmt-functions.sh || exit 1
+
+# globals - all global variables are UPPERCASE
+BASE_OUTPUTDIR="out"
+PYMAIL="./test_mail.py"
+declare -i OVERALL_SUCCESSES=0
+declare -i OVERALL_FAILURES=0
+declare -i OVERALL_COUNT=0
+declare -i OVERALL_COUNT_SUITES=0
+
+# ansi escapes for hilighting text
+F_DANGER=$(echo -e "\033[31m")
+F_WARN=$(echo -e "\033[93m")
+F_RESET=$(echo -e "\033[39m")
+
+# options
+FAILURE_IS_FATAL=no
+
+
+suite_start() {
+	let TEST_NUM=1
+	let SUITE_COUNT_SUCCESS=0
+	let SUITE_COUNT_FAILURE=0
+	let SUITE_COUNT_TOTAL=0
+	SUITE_NAME="$1"
+	OUTDIR="$BASE_OUTPUTDIR/$SUITE_NAME"
+	mkdir -p "$OUTDIR"
+	echo ""
+	echo "Starting suite: $SUITE_NAME"
+	suite_setup "$2"
+}
+
+suite_end() {
+	suite_cleanup "$1"
+	echo "Suite $SUITE_NAME finished"
+	let OVERALL_SUCCESSES+=$SUITE_COUNT_SUCCESS
+	let OVERALL_FAILURES+=$SUITE_COUNT_FAILURE
+	let OVERALL_COUNT+=$SUITE_COUNT_TOTAL
+	let OVERALL_COUNT_SUITES+=1
+}
+
+suite_setup() {
+	[ -z "$1" ] && return 0
+	TEST_OF="$OUTDIR/setup"
+	eval "$1"
+	TEST_OF=""
+}
+
+suite_cleanup() {
+	[ -z "$1" ] && return 0
+	TEST_OF="$OUTDIR/cleanup"
+	eval "$1"
+	TEST_OF=""
+}
+
+test_start() {
+	TEST_DESC="${1:-}"
+	TEST_NAME="$(printf "%03d" $TEST_NUM)"
+	TEST_OF="$OUTDIR/$TEST_NAME"
+	TEST_STATE=""
+	TEST_STATE_MSG=()
+	echo "TEST-START \"${TEST_DESC:-unnamed}\"" >$TEST_OF
+	echo -n "  $TEST_NAME: $TEST_DESC: "
+	let TEST_NUM+=1
+	let SUITE_COUNT_TOTAL+=1
+}
+
+test_end() {
+	[ -z "$TEST_OF" ] && return
+	if [ $# -gt 0 ]; then
+		[ -z "$1" ] && test_success || test_failure "$1"
+	fi
+	case $TEST_STATE in
+		SUCCESS | "" )
+			record "[SUCCESS]"
+			echo "SUCCESS"
+			let SUITE_COUNT_SUCCESS+=1
+			;;
+		FAILURE )
+			record "[FAILURE]"
+			echo "${F_DANGER}FAILURE${F_RESET}:"
+			local idx=0
+			while [ $idx -lt ${#TEST_STATE_MSG[*]} ]; do
+				record "${TEST_STATE_MSG[$idx]}"
+				echo "	   why: ${TEST_STATE_MSG[$idx]}"
+				let idx+=1
+			done
+			echo "	   see: $(dirname $0)/$TEST_OF"
+			let SUITE_COUNT_FAILURE+=1
+			if [ "$FAILURE_IS_FATAL" == "yes" ]; then
+				record "FATAL: failures are fatal option enabled"
+				echo "FATAL: failures are fatal option enabled"
+				exit 1
+			fi
+			;;
+		* )
+			record "[INVALID TEST STATE '$TEST_STATE']"
+			echo "Invalid TEST_STATE=$TEST_STATE"
+			let SUITE_COUNT_FAILURE+=1
+			;;
+	esac
+	TEST_OF=""
+}
+
+test_success() {
+	[ -z "$TEST_OF" ] && return
+	[ -z "$TEST_STATE" ] && TEST_STATE="SUCCESS"
+}
+
+test_failure() {
+	local why="$1"
+	[ -z "$TEST_OF" ] && return
+	TEST_STATE="FAILURE"
+	TEST_STATE_MSG+=( "$why" )
+}
+
+have_test_failures() {
+	[ "$TEST_STATE" == "FAILURE" ] && return 0
+	return 1
+}
+
+record() {
+	if [ ! -z "$TEST_OF" ]; then
+		echo "$@" >>$TEST_OF
+	else
+		echo "$@"
+	fi
+}
+
+die() {
+	record "FATAL: $@"
+	test_failure "a fatal error occurred"
+	test_end
+	echo "FATAL: $@"
+	exit 1
+}
+
+array_contains() {
+	local searchfor="$1"
+	shift
+	local item
+	for item; do
+		[ "$item" == "$searchfor" ] && return 0
+	done
+	return 1
+}
+
+python_error() {
+	# finds tracebacks and outputs just the final error message of
+	# each
+	local output="$1"
+	awk 'BEGIN { TB=0; FOUND=0 } TB==0 && /^Traceback/ { TB=1; FOUND=1; next } TB==1 && /^[^ ]/ { print $0; TB=0 } END { if (FOUND==0) exit 1 }' <<< "$output"
+	[ $? -eq 1 ] && echo "$output"
+}
+
+
+
+##
+## Initialize
+##
+
+mkdir -p "$BASE_OUTPUTDIR"
+
+# load global vars
+. /etc/mailinabox.conf || die "Could not load '/etc/mailinabox.conf'"
+. "${STORAGE_ROOT}/ldap/miab_ldap.conf" || die "Could not load miab_ldap.conf"

tests/suites/_ldap-functions.sh

@@ -0,0 +1,427 @@
+# -*- indent-tabs-mode: t; tab-width: 4; -*-
+
+generate_uuid() {
+	local uuid
+	uuid=$(python3 -c "import uuid; print(uuid.uuid4())")
+	[ $? -ne 0 ] && die "Unable to generate a uuid"
+	echo "$uuid"
+}
+
+delete_user() {
+	local email="$1"
+	local domainpart="$(awk -F@ '{print $2}' <<< "$email")"
+	get_attribute "$LDAP_USERS_BASE" "mail=$email" "dn"
+	[ -z "$ATTR_DN" ] && return 0
+	record "[delete user $email]"
+	ldapdelete -H $LDAP_URL -x -D "$LDAP_ADMIN_DN" -w "$LDAP_ADMIN_PASSWORD" "$ATTR_DN" >>$TEST_OF 2>&1 || die "Unable to delete user $ATTR_DN (as admin)"
+	record "deleted"
+	# delete the domain if there are no more users in the domain
+	get_attribute "$LDAP_USERS_BASE" "mail=*@${domainpart}" "dn"
+	[ ! -z "$ATTR_DN" ] && return 0
+	get_attribute "$LDAP_DOMAINS_BASE" "dc=${domainpart}" "dn"
+	if [ ! -z "$ATTR_DN" ]; then
+		record "[delete domain $domainpart]"
+		ldapdelete -H $LDAP_URL -x -D "$LDAP_ADMIN_DN" -w "$LDAP_ADMIN_PASSWORD" "$ATTR_DN" >>$TEST_OF 2>&1 || die "Unable to delete domain $ATTR_DN (as admin)"
+		record "deleted"
+	fi
+}
+
+create_user() { 
+	local email="$1"
+	local pass="${2:-$email}"
+	local priv="${3:-test}"
+	local localpart="$(awk -F@ '{print $1}' <<< "$email")"
+	local domainpart="$(awk -F@ '{print $2}' <<< "$email")"
+	local uid="$localpart"
+	local dn="uid=${uid},${LDAP_USERS_BASE}"
+	
+	delete_user "$email"
+
+	record "[create user $email]"
+	delete_dn "$dn"
+	
+	ldapadd -H "$LDAP_URL" -x -D "$LDAP_ADMIN_DN" -w "$LDAP_ADMIN_PASSWORD" >>$TEST_OF 2>&1 <<EOF
+dn: $dn
+objectClass: inetOrgPerson
+objectClass: mailUser
+objectClass: shadowAccount
+uid: $uid
+cn: $localpart
+sn: $localpart
+displayName: $localpart
+mail: $email
+maildrop: $email
+mailaccess: $priv
+userPassword: $(slappasswd_hash "$pass")
+EOF
+	[ $? -ne 0 ] && die "Unable to add user $dn (as admin)"
+
+	# create domain entry, if needed
+	get_attribute "$LDAP_DOMAINS_BASE" "dc=${domainpart}" dn
+	if [ -z "$ATTR_DN" ]; then
+		record "[create domain entry $domainpart]"
+		ldapadd -H $LDAP_URL -x -D "$LDAP_ADMIN_DN" -w "$LDAP_ADMIN_PASSWORD" >>$TEST_OF 2>&1 <<EOF
+dn: dc=${domainpart},$LDAP_DOMAINS_BASE
+objectClass: domain
+dc: ${domainpart}
+businessCategory: mail
+EOF
+		[ $? -ne 0 ] && die "Unable to add domain ${domainpart} (as admin)"
+	fi
+	
+	ATTR_DN="$dn"
+}
+
+
+delete_dn() {
+	local dn="$1"
+	get_attribute "$dn" "objectClass=*" "dn" base
+	[ -z "$ATTR_DN" ] && return 0
+	record "delete dn: $dn"
+	ldapdelete -H $LDAP_URL -x -D "$LDAP_ADMIN_DN" -w "$LDAP_ADMIN_PASSWORD" "$dn" >>$TEST_OF 2>&1 || die "Unable to delete $dn (as admin)"
+}	 
+
+create_service_account() {
+	local cn="$1"
+	local pass="${2:-$cn}"
+	local dn="cn=${cn},${LDAP_SERVICES_BASE}"
+
+	record "[create service account $cn]"
+	delete_dn "$dn"
+	
+	ldapadd -H "$LDAP_URL" -x -D "$LDAP_ADMIN_DN" -w "$LDAP_ADMIN_PASSWORD" >>$TEST_OF 2>&1 <<EOF
+dn: $dn
+objectClass: simpleSecurityObject
+objectClass: organizationalRole
+cn: $cn
+description: TEST ${cn} service account
+userPassword: $(slappasswd_hash "$pass")
+EOF
+	[ $? -ne 0 ] && die "Unable to add service account $dn (as admin)"
+	ATTR_DN="$dn"
+}
+
+delete_service_account() {
+	local cn="$1"
+	local dn="cn=${cn},${LDAP_SERVICES_BASE}"
+	record "[delete service account $cn]"
+	delete_dn "$dn"
+}
+
+create_alias_group() {
+	local alias="$1"
+	shift
+	record "[Create new alias group $alias]"
+	# add alias group with dn's as members
+	get_attribute "$LDAP_ALIASES_BASE" "mail=$alias" "dn"
+	if [ ! -z "$ATTR_DN" ]; then
+		delete_dn "$ATTR_DN"
+	fi
+	
+	ATTR_DN="cn=$(generate_uuid),$LDAP_ALIASES_BASE"
+	of="/tmp/create_alias.$$.ldif"
+	cat >$of 2>>$TEST_OF <<EOF
+dn: $ATTR_DN
+objectClass: mailGroup
+mail: $alias
+EOF
+	local member
+	for member; do
+		case $member in
+			*@* )
+				echo "rfc822MailMember: $member" >>$TEST_OF
+				echo "rfc822MailMember: $member" >>$of 2>>$TEST_OF
+				;;
+			* )
+				echo "member: $member" >>$TEST_OF
+				echo "member: $member" >>$of 2>>$TEST_OF
+				;;
+		esac
+	done
+	ldapadd -H "$LDAP_URL" -x -D "$LDAP_ADMIN_DN" -w "$LDAP_ADMIN_PASSWORD" -f $of >>$TEST_OF 2>&1 || die "Unable to add alias group $alias"
+	rm -f $of
+}
+
+delete_alias_group() {
+	record "[delete alias group $1]"
+	get_attribute "$LDAP_ALIASES_BASE" "(mail=$1)" dn
+	[ ! -z "$ATTR_DN" ] && delete_dn "$ATTR_DN"
+}
+
+
+add_alias() {
+	local user_dn="$1"
+	local alias="$2"
+	local type="${3:-group}"
+	if [ $type == user ]; then
+		# add alias as additional 'mail' attribute to user's dn
+		record "[Add alias $alias to $user_dn]"
+		ldapmodify -H "$LDAP_URL" -x -D "$LDAP_ADMIN_DN" -w "$LDAP_ADMIN_PASSWORD" >>$TEST_OF 2>&1 <<EOF
+dn: $user_dn
+add: mail
+mail: $alias
+EOF
+		local r=$?
+		[ $r -ne 0 ] && die "Unable to modify $user_dn"
+		
+	elif [ $type == group ]; then
+		# add alias as additional 'member" to a mailGroup alias list
+		record "[Add member $user_dn to alias $alias]"
+		get_attribute "$LDAP_ALIASES_BASE" "mail=$alias" "dn"
+		if [ -z "$ATTR_DN" ]; then
+			# don't automatically add because it should be cleaned
+			# up by the caller
+			die "Alias grour $alias does not exist"
+		else
+			ldapmodify -H "$LDAP_URL" -x -D "$LDAP_ADMIN_DN" -w "$LDAP_ADMIN_PASSWORD" >>$TEST_OF 2>&1 <<EOF
+dn: $ATTR_DN
+add: member
+member: $user_dn
+EOF
+			local code=$?
+			if [ $code -ne 20 -a $code -ne 0 ]; then
+				# 20=Type or value exists
+				die "Unable to add user $user_dn to alias $alias"
+			fi
+		fi
+	else
+		die "Invalid type '$type' to add_alias"
+	fi
+
+}
+
+
+create_permitted_senders_group() {
+	# add a permitted senders group. specify the email address that
+	# the members may MAIL FROM as the first argument, followed by all
+	# member dns. If the group already exists, it is deleted first.
+	#
+	# on return, the global variable ATTR_DN is set to the dn of the
+	# created mailGroup
+	local mail_from="$1"
+	shift
+	record "[create permitted sender list $mail_from]"
+	get_attribute "$LDAP_PERMITTED_SENDERS_BASE" "(&(objectClass=mailGroup)(mail=$mail_from))" dn
+	if [ ! -z "$ATTR_DN" ]; then
+		delete_dn "$ATTR_DN"
+	fi
+
+	local tmp="/tmp/tests.$$.ldif"
+	ATTR_DN="cn=$(generate_uuid),$LDAP_PERMITTED_SENDERS_BASE"
+	cat >$tmp <<EOF
+dn: $ATTR_DN
+objectClass: mailGroup
+mail: $mail_from
+EOF
+	local member
+	for member; do
+		echo "member: $member" >>$tmp
+		echo "member: $member" >>$TEST_OF
+	done
+	
+	ldapadd -H "$LDAP_URL" -x -D "$LDAP_ADMIN_DN" -w "$LDAP_ADMIN_PASSWORD" -f $tmp >>$TEST_OF 2>&1
+	local r=$?
+	rm -f $tmp
+	[ $r -ne 0 ] && die "Unable to add permitted senders group $mail_from"
+}
+
+delete_permitted_senders_group() {
+	local mail_from="$1"
+	record "[delete permitted sender list $mail_from]"
+	get_attribute "$LDAP_PERMITTED_SENDERS_BASE" "(&(objectClass=mailGroup)(mail=$mail_from))" dn
+	if [ ! -z "$ATTR_DN" ]; then
+		delete_dn "$ATTR_DN"
+	fi
+}
+
+
+test_r_access() {
+	# tests read or unreadable access
+	# sets global variable FAILURE on return
+	local user_dn="$1"
+	local login_dn="$2"
+	local login_pass="$3"
+	local access="${4:-no-read}"  # should be "no-read" or "read"
+	shift; shift; shift; shift
+
+	if ! array_contains $access read no-read; then
+		die "Invalid parameter '$access' to function test_r_access"
+	fi
+
+	# get all attributes using login_dn's account
+	local attr
+	local search_output result=()
+	record "[Get attributes of $user_dn by $login_dn]"
+	search_output=$(ldapsearch -LLL -o ldif-wrap=no -H "$LDAP_URL" -b "$user_dn" -s base -x -D "$login_dn" -w "$login_pass" 2>>$TEST_OF)
+	local code=$?
+	# code 32: No such object (doesn't exist or login can't see it)
+	[ $code -ne 0 -a $code -ne 32 ] && die "Unable to find entry $user_dn by $login_dn"
+	while read attr; do
+		record "line: $attr"
+		attr=$(awk -F: '{print $1}' <<< "$attr")
+		[ "$attr" != "dn" -a "$attr" != "objectClass" ] && result+=($attr)
+	done <<< "$search_output"
+	record "check for $access access to ${@:-ALL}"
+	record "comparing to actual: ${result[@]}"
+
+	
+	local failure=""
+	if [ $access == "no-read" -a $# -eq 0 ]; then
+		# check that no attributes are readable
+		if [ ${#result[*]} -gt 0 ]; then
+			failure="Attributes '${result[*]}' of $user_dn should not be readable by $login_dn"
+		fi
+	else
+		# check that specified attributes are/aren't readable
+		for attr; do
+			if [ $access == "no-read" ]; then
+				if array_contains $attr ${result[@]}; then
+					failure="Attribute $attr of $user_dn should not be readable by $login_dn"
+					break
+				fi
+			else
+				if ! array_contains $attr ${result[@]}; then
+					failure="Attribute $attr of $user_dn should be readable by $login_dn got (${result[*]})"
+					break
+				fi
+			fi
+		done
+	fi
+
+	FAILURE="$failure"
+}
+
+
+assert_r_access() {
+	# asserts read or unreadable access
+	FAILURE=""
+	test_r_access "$@"
+	[ ! -z "$FAILURE" ] && test_failure "$FAILURE"
+}
+
+
+test_w_access() {
+	# tests write or unwritable access
+	# sets global variable FAILURE on return
+	# if no attributes given, test user attributes
+	#	uuid, cn, sn, mail, maildrop, mailaccess
+	local user_dn="$1"
+	local login_dn="$2"
+	local login_pass="$3"
+	local access="${4:-no-write}"  # should be "no-write" or "write"
+	shift; shift; shift; shift
+	local moddn=""
+	local attrs=( $@ )
+	
+	if ! array_contains $access write no-write; then
+		die "Invalid parameter '$access' to function test_w_access"
+	fi
+
+	if [ ${#attrs[*]} -eq 0 ]; then
+		moddn=uid
+		attrs=("cn=alice fiction" "sn=fiction" "mail" "maildrop" "mailaccess=admin")
+	fi
+
+	local failure=""
+	
+	# check that select attributes are not writable
+	if [ ! -z "$moddn" ]; then
+		record "[Change attribute ${moddn}]"
+		delete_dn "${moddn}=some-uuid,$LDAP_USERS_BASE"
+		ldapmodify -H "$LDAP_URL" -x -D "$login_dn" -w "$login_pass" >>$TEST_OF 2>&1 <<EOF
+dn: $user_dn
+changetype: moddn
+newrdn: ${moddn}=some-uuid
+deleteoldrdn: 1
+EOF
+		local r=$?
+		if [ $r -eq 0 ]; then
+			if [ "$access" == "no-write" ]; then
+				failure="Attribute $moddn of $user_dn should not be changeable by $login_dn"
+			fi
+		elif [ $r -eq 50 ]; then
+			if [ "$access" == "write" ]; then
+				failure="Attribute $moddn of $user_dn should be changeable by $login_dn"
+			fi
+		else
+			die "Error attempting moddn change of $moddn (code $?)"
+		fi
+	fi
+	
+
+	if [ -z "$failure" ]; then
+		local attrvalue attr value
+		for attrvalue in "${attrs[@]}"; do
+			attr="$(awk -F= '{print $1}' <<< "$attrvalue")"
+			value="$(awk -F= '{print substr($0,length($1)+2)}' <<< "$attrvalue")"
+			[ -z "$value" ] && value="alice2@abc.com"
+			record "[Change attribute $attr]"
+			ldapmodify -H "$LDAP_URL" -x -D "$login_dn" -w "$login_pass" >>$TEST_OF 2>&1 <<EOF
+dn: $user_dn
+replace: $attr
+$attr: $value
+EOF
+			r=$?
+			if [ $r -eq 0 ]; then
+				if [ $access == "no-write" ]; then
+					failure="Attribute $attr of $user_dn should not be changeable by $login_dn"
+					break
+				fi
+			elif [ $r -eq 50 ]; then
+				if [ $access == "write" ]; then
+					failure="Attribute $attr of $user_dn should be changeable by $login_dn"
+					break
+				fi
+			else
+				die "Error attempting change of $attr to '$value'"
+			fi
+		done
+	fi
+
+	FAILURE="$failure"
+}
+
+assert_w_access() {
+	# asserts write or unwritable access
+	FAILURE=""
+	test_w_access "$@"
+	[ ! -z "$FAILURE" ] && test_failure "$FAILURE"
+}
+
+
+test_search() {
+	# test if access to search something is allowed
+	# sets global variable SEARCH_DN_COUNT on return
+	local base_dn="$1"
+	local login_dn="$2"
+	local login_pass="$3"
+	local scope="${4:-sub}"
+	local filter="$5"
+
+	let SEARCH_DN_COUNT=0
+
+	local line search_output
+	record "[Perform $scope search of $base_dn by $login_dn]"
+	search_output=$(ldapsearch -H $LDAP_URL -o ldif-wrap=no -b "$base_dn" -s "$scope" -LLL -x -D "$login_dn" -w "$login_pass" $filter 2>>$TEST_OF)
+	local code=$?
+	# code 32: No such object (doesn't exist or login can't see it)
+	[ $code -ne 0 -a $code -ne 32 ] && die "Unable to search $base_dn by $login_dn"
+	
+	while read line; do
+		record "line: $line"
+		case $line in
+			dn:*)
+				let SEARCH_DN_COUNT+=1
+				;;
+		esac
+	done <<< "$search_output"
+	record "$SEARCH_DN_COUNT entries found"
+}
+
+
+record_search() {
+	local dn="$1"
+	record "[Contents of $dn]"
+	debug_search "$dn" >>$TEST_OF 2>&1
+	return 0
+}

tests/suites/_mail-functions.sh

@@ -0,0 +1,369 @@
+# -*- indent-tabs-mode: t; tab-width: 4; -*-
+
+clear_postfix_queue() {
+	record "[Clear postfix queue]"
+	postsuper -d ALL >>$TEST_OF 2>&1 || die "Unable to clear postfix undeliverable mail queue"
+}
+
+
+ensure_root_user() {
+	# ensure there is a local email account for root.
+	#
+	# on exit, ROOT, ROOT_MAILDROP, and ROOT_DN are set, and if no
+	# account exists, a new root@$(hostname) is created having a
+	# random password
+	#
+	if [ ! -z "$ROOT_MAILDROP" ]; then
+		# already have it
+		return
+	fi
+	ROOT="${USER}@$(hostname)"
+	record "[Find user $ROOT]"
+	get_attribute "$LDAP_USERS_BASE" "mail=$ROOT" "maildrop"
+	ROOT_MAILDROP="$ATTR_VALUE"
+	ROOT_DN="$ATTR_DN"
+	if [ -z "$ROOT_DN" ]; then
+		local pw="$(generate_password 128)"
+		create_user "$ROOT" "$pw"
+		record "new password is: $pw"
+		ROOT_DN="$ATTR_DN"
+		ROOT_MAILDROP="$ROOT"
+	else
+		record "$ROOT => $ROOT_DN ($ROOT_MAILDROP)"
+	fi
+}
+
+
+dovecot_mailbox_home() {
+	local email="$1"
+	echo -n "${STORAGE_ROOT}/mail/mailboxes/"
+	awk -F@ '{print $2"/"$1}' <<< "$email"
+}
+
+
+start_log_capture() {
+	SYS_LOG_LINECOUNT=$(wc -l /var/log/syslog 2>>$TEST_OF | awk '{print $1}') || die "could not access /var/log/syslog"
+	SLAPD_LOG_LINECOUNT=0
+	if [ -e /var/log/ldap/slapd.log ]; then
+		SLAPD_LOG_LINECOUNT=$(wc -l /var/log/ldap/slapd.log 2>>$TEST_OF | awk '{print $1}') || die "could not access /var/log/ldap/slapd.log"
+	fi
+	MAIL_ERRLOG_LINECOUNT=0
+	if [ -e /var/log/mail.err ]; then
+		MAIL_ERRLOG_LINECOUNT=$(wc -l /var/log/mail.err 2>>$TEST_OF | awk '{print $1}') || die "could not access /var/log/mail.err"
+	fi
+	MAIL_LOG_LINECOUNT=0
+	if [ -e /var/log/mail.log ]; then
+		MAIL_LOG_LINECOUNT=$(wc -l /var/log/mail.log 2>>$TEST_OF | awk '{print $1}') || die "could not access /var/log/mail.log"
+	fi
+	DOVECOT_LOG_LINECOUNT=$(doveadm log errors 2>>$TEST_OF | wc -l | awk '{print $1}') || die "could not access doveadm error logs"
+}
+
+start_mail_capture() {
+	local email="$1"
+	local newdir="$(dovecot_mailbox_home "$email")/new"
+	record "[Start mail capture $email]"
+	DOVECOT_CAPTURE_USER="$email"
+	DOVECOT_CAPTURE_FILECOUNT=0
+	if [ -e "$newdir" ]; then
+		DOVECOT_CAPTURE_FILECOUNT=$(ls "$newdir" 2>>$TEST_OF | wc -l)
+		[ $? -ne 0 ] && die "Error accessing mailbox of $email"
+	fi
+	record "mailbox: $(dirname $newdir)"
+	record "mailbox has $DOVECOT_CAPTURE_FILECOUNT files"
+}
+
+dump_capture_logs() {
+	# dump log files
+	record "[capture log dump]"
+	echo ""
+	echo "============= SYSLOG ================"
+	tail --lines=+$SYS_LOG_LINECOUNT /var/log/syslog 2>>$TEST_OF
+	echo ""
+	echo "============= SLAPD ================="
+	tail --lines=+$SLAPD_LOG_LINECOUNT /var/log/ldap/slapd.log 2>>$TEST_OF
+	echo ""
+	echo "============= MAIL.ERR =============="
+	tail --lines=+$MAIL_ERRLOG_LINECOUNT /var/log/mail.err 2>>$TEST_OF
+	echo ""
+	echo "============= MAIL.LOG =============="
+	tail --lines=+$MAIL_LOG_LINECOUNT /var/log/mail.log 2>>$TEST_OF
+	echo ""
+	echo "============= DOVECOT ERRORS =============="
+	doveadm log errors | tail --lines=+$DOVECOT_LOG_LINECOUNT 2>>$TEST_OF
+}
+
+detect_syslog_error() {
+	record
+	record "[Detect syslog errors]"
+	local count
+	let count="$SYS_LOG_LINECOUNT + 1"
+	tail --lines=+$count /var/log/syslog 2>>$TEST_OF | (
+		let ec=0 # error count
+		while read line; do
+			awk '
+/status=(bounced|deferred|undeliverable)/  { exit 1 }
+!/postfix\/qmgr/ && /warning:/	{ exit 1 }
+/(fatal|reject|error):/	 { exit 1 }
+/Error in /			{ exit 1 }
+/named\[\d+\]:.* verify failed/ { exit 1 }
+' \
+				>>$TEST_OF 2>&1 <<< "$line"
+			if [ $? -eq 1 ]; then
+				let ec+=1
+				record "$F_DANGER[ERROR] $line$F_RESET"
+			else
+				record "[	OK] $line"
+			fi
+		done
+		[ $ec -gt 0 ] && exit 0
+		exit 1 # no errors
+	)
+	local x=( ${PIPESTATUS[*]} )
+	[ ${x[0]} -ne 0 ] && die "Could not read /var/log/syslog"
+	return ${x[1]}
+}
+
+detect_slapd_log_error() {
+	record
+	record "[Detect slapd log errors]"
+	local count
+	let count="SLAPD_LOG_LINECOUNT + 1"
+	tail --lines=+$count /var/log/ldap/slapd.log 2>>$TEST_OF | (
+		let ec=0 # error count
+		let wc=0 # warning count
+		let ignored=0
+		while read line; do
+			# slapd error 68 = "entry already exists". Mark it as a
+			# warning because code often attempts to add entries
+			# silently ignoring the error, which is expected behavior
+			#
+			# slapd error 32 = "no such object". Mark it as a warning
+			# because code often attempts to resolve a dn (eg member)
+			# that is orphaned, so no entry exists. Code may or may
+			# not care about this.
+			#
+			# slapd error 4 - "size limit exceeded". Mark it as a warning
+			# because code often attempts to find just 1 entry so sets
+			# the limit to 1 purposefully.
+			#
+			# slapd error 20 - "attribute or value exists". Mark it as a
+			# warning becuase code often attempts to add a new value
+			# to an existing attribute and doesn't care if the new
+			# value fails to add because it already exists.
+			#
+			awk '
+/SEARCH RESULT.*err=(32|4) / { exit 2}
+/RESULT.*err=(68|20) / { exit 2 }
+/ not indexed/ { exit 2 }
+/RESULT.*err=[^0]/ { exit 1 }
+/(get|test)_filter/		  { exit 3 }
+/mdb_(filter|list)_candidates/	{ exit 3 }
+/:(		| #011| )(AND|OR|EQUALITY)/ { exit 3 }
+' \
+				>>$TEST_OF 2>&1 <<< "$line"
+			r=$?
+			if [ $r -eq 1 ]; then
+				let ec+=1
+				record "$F_DANGER[ERROR] $line$F_RESET"
+			elif [ $r -eq 2 ]; then
+				let wc+=1
+				record "$F_WARN[WARN ] $line$F_RESET"
+			elif [ $r -eq 3 ]; then
+				let ignored+=1
+			else
+				record "[OK   ] $line"
+			fi
+		done
+		record "$ignored unreported/ignored log lines"
+		[ $ec -gt 0 ] && exit 0
+		exit 1 # no errors
+	)
+	local x=( ${PIPESTATUS[*]} )
+	[ ${x[0]} -ne 0 ] && die "Could not read /var/log/ldap/slapd.log"
+	return ${x[1]}
+}
+
+
+detect_dovecot_log_error() {
+	record
+	record "[Detect dovecot log errors]"
+	local count
+	let count="$MAIL_LOG_LINECOUNT + 1"
+	if [ ! -e /var/log/mail.log ]; then
+		return 0
+	fi
+	# prefer mail.log over `dovadm log errors` because the latter does
+	# not have as much output - it's helpful to have success logs when
+	# diagnosing logs...
+	cat /var/log/mail.log 2>>$TEST_OF | tail --lines=+$count | (
+		let ec=0 # error count
+		let ignored=0
+		while read line; do
+			awk '
+/LDAP server, reconnecting/ { exit 2 }
+/postfix/ { exit 2 }
+/auth failed/  { exit 1 }
+/ Error: /			  { exit 1 }
+' \
+				>>$TEST_OF 2>&1 <<< "$line"
+			r=$?
+			if [ $r -eq 1 ]; then
+				let ec+=1
+				record "$F_DANGER[ERROR] $line$F_RESET"
+			elif [ $r -eq 2 ]; then
+				let ignored+=1
+			else
+				record "[	OK] $line"
+			fi
+		done
+		record "$ignored unreported/ignored log lines"
+		[ $ec -gt 0 ] && exit 0
+		exit 1 # no errors
+	)
+	local x=( ${PIPESTATUS[*]} )
+	[ ${x[0]} -ne 0 -o ${x[1]} -ne 0 ] && die "Could not read mail log"
+	return ${x[2]}
+}
+
+
+check_logs() {
+	local assert="${1:-false}"
+	[ "$1" == "true" -o "$1" == "false" ] && shift
+	local types=($@)
+	[ ${#types[@]} -eq 0 ] && types=(syslog slapd mail)
+	
+	# flush records
+	kill -HUP $(cat /var/run/rsyslogd.pid)
+	sleep 2
+
+	if array_contains syslog ${types[@]}; then
+		detect_syslog_error && $assert &&
+			test_failure "detected errors in syslog"
+	fi
+	
+	if array_contains slapd ${types[@]}; then
+		detect_slapd_log_error && $assert &&
+			test_failure "detected errors in slapd log"
+	fi
+	
+	if array_contains mail ${types[@]}; then
+		detect_dovecot_log_error && $assert &&
+			test_failure "detected errors in dovecot log"
+	fi
+}
+
+assert_check_logs() {
+	check_logs true $@
+}
+
+grep_postfix_log() {
+	local msg="$1"
+	local count
+	let count="$SYS_LOG_LINECOUNT + 1"
+	tail --lines=+$count /var/log/syslog 2>>$TEST_OF | grep -iF "$msg" >/dev/null 2>>$TEST_OF
+	return $?
+}
+
+wait_mail() {
+	local x mail_files elapsed max_s="${1:-60}"
+	let elapsed=0
+	record "[Waiting for mail to $DOVECOT_CAPTURE_USER]"
+	while [ $elapsed -lt $max_s ]; do
+		mail_files=( $(get_captured_mail_files) )
+		[ ${#mail_files[*]} -gt 0 ] && break
+		sleep 1
+		let elapsed+=1
+		let x="$elapsed % 10"
+		[ $x -eq 0 ] && record "...~${elapsed} seconds has passed"
+	done
+	if [ $elapsed -ge $max_s ]; then
+		record "Timeout waiting for mail"
+		return 1
+	fi
+	record "new mail files:"
+	for x in ${mail_files[@]}; do
+		record "$x"
+	done
+}
+
+get_captured_mail_files() {
+	local newdir="$(dovecot_mailbox_home "$DOVECOT_CAPTURE_USER")/new"
+	local count
+	let count="$DOVECOT_CAPTURE_FILECOUNT + 1"
+	[ ! -e "$newdir" ] && return 0
+	# output absolute path names
+	local file
+	for file in $(ls "$newdir" 2>>$TEST_OF | tail --lines=+${count}); do
+		echo "$newdir/$file"
+	done
+}
+
+record_captured_mail() {
+	local files=( $(get_captured_mail_files) )
+	local file
+	for file in ${files[@]}; do
+		record
+		record "[Captured mail file: $file]"
+		cat "$file" >>$TEST_OF 2>&1
+	done
+}
+
+
+sendmail_bv_send() {
+	# test sending mail, but don't actually send it...
+	local recpt="$1"
+	local timeout="$2"
+	local bvfrom from="$3"
+	# delivery status is emailed back to us, or 'from' if supplied
+	clear_postfix_queue
+	if [ -z "$from" ]; then
+		ensure_root_user
+		start_mail_capture "$ROOT"
+	else
+		bvfrom="-f $from"
+		start_mail_capture "$from"
+	fi
+	record "[Running sendmail -bv $bvfrom]"
+	sendmail $bvfrom -bv "$recpt" >>$TEST_OF 2>&1
+	if [ $? -ne 0 ]; then
+		test_failure "Error executing sendmail"
+	else
+		wait_mail $timeout || test_failure "Timeout waiting for delivery report"
+	fi	  
+}
+
+
+assert_python_success() {
+	local code="$1"
+	local output="$2"
+	record "$output"
+	record
+	record "python exit code: $code"
+	if [ $code -ne 0 ]; then
+		test_failure "unable to process mail: $(python_error "$output")"
+		return 1
+	fi
+	return 0
+}
+
+assert_python_failure() {
+	local code="$1"
+	local output="$2"
+	shift; shift
+	record "$output"
+	record
+	record "python exit code: $code"
+	if [ $code -eq 0 ]; then
+		test_failure "python succeeded but expected failure"
+		return 1
+	fi
+	local look_for
+	for look_for; do
+		if [ ! -z "$look_for" ] && ! grep "$look_for" <<< "$output" 1>/dev/null
+		then
+			test_failure "unexpected python failure: $(python_error "$output")"
+			return 1
+		fi
+	done
+	return 0
+}

tests/suites/_mgmt-functions.sh

@@ -0,0 +1,175 @@
+# -*- indent-tabs-mode: t; tab-width: 4; -*-
+
+# Available REST calls:
+#
+# general curl format:
+# curl -X <b>VERB</b> [-d "<b>parameters</b>"] --user {email}:{password} https://{{hostname}}/admin/mail/users[<b>action</b>]
+
+# ALIASES:
+# curl -X GET https://{{hostname}}/admin/mail/aliases?format=json
+# curl -X POST -d "address=new_alias@mydomail.com" -d "forwards_to=my_email@mydomain.com" https://{{hostname}}/admin/mail/aliases/add
+# curl -X POST -d "address=new_alias@mydomail.com" https://{{hostname}}/admin/mail/aliases/remove
+
+# USERS:
+# curl -X GET https://{{hostname}}/admin/mail/users?format=json
+# curl -X POST -d "email=new_user@mydomail.com" -d "password=s3curE_pa5Sw0rD" https://{{hostname}}/admin/mail/users/add
+# curl -X POST -d "email=new_user@mydomail.com" https://{{hostname}}/admin/mail/users/remove
+# curl -X POST -d "email=new_user@mydomail.com" -d "privilege=admin" https://{{hostname}}/admin/mail/users/privileges/add
+# curl -X POST -d "email=new_user@mydomail.com" https://{{hostname}}/admin/mail/users/privileges/remove
+
+
+mgmt_start() {
+	# Must be called before performing any REST calls
+	local domain="${1:-somedomain.com}"
+	MGMT_ADMIN_EMAIL="test_admin@$domain"
+	MGMT_ADMIN_PW="$(generate_password)"
+
+	delete_user "$MGMT_ADMIN_EMAIL"
+	
+	record "[Creating a new account with admin rights for management tests]"
+	create_user "$MGMT_ADMIN_EMAIL" "$MGMT_ADMIN_PW" "admin"
+	MGMT_ADMIN_DN="$ATTR_DN"
+	record "Created: $MGMT_ADMIN_EMAIL at $MGMT_ADMIN_DN"	
+}
+
+mgmt_end() {
+	# Clean up after mgmt_start
+	delete_user "$MGMT_ADMIN_EMAIL"
+}
+
+
+mgmt_rest() {
+	# Issue a REST call to the management subsystem
+	local verb="$1" # eg "POST"
+	local uri="$2"  # eg "/mail/users/add"
+	shift; shift;   # remaining arguments are data
+
+	local auth_user="${MGMT_ADMIN_EMAIL}"
+	local auth_pass="${MGMT_ADMIN_PW}"
+	local url="https://$PRIMARY_HOSTNAME${uri}"
+	local data=()
+	local item output
+	
+	for item; do data+=("--data-urlencode" "$item"); done
+
+	record "spawn: curl -w \"%{http_code}\" -X $verb --user \"${auth_user}:xxx\" ${data[@]} $url"
+	output=$(curl -s -S -w "%{http_code}" -X $verb --user "${auth_user}:${auth_pass}" "${data[@]}" $url 2>>$TEST_OF)
+	local code=$?
+
+	# http status is last 3 characters of output, extract it
+	REST_HTTP_CODE=$(awk '{S=substr($0,length($0)-2)} END {print S}' <<<"$output")
+	REST_OUTPUT=$(awk 'BEGIN{L=""}{ if(L!="") print L; L=$0 } END { print substr(L,1,length(L)-3) }' <<<"$output")
+	REST_ERROR=""
+	[ -z "$REST_HTTP_CODE" ] && REST_HTTP_CODE="000"
+
+	if [ $code -ne 0 ]; then
+		if [ $code -ne 16 -o $REST_HTTP_CODE -ne 200 ]; then
+			REST_ERROR="CURL failed with code $code"
+			record "${F_DANGER}$REST_ERROR${F_RESET}"
+			record "$output"
+			return 1
+		fi
+	fi
+	if [ $REST_HTTP_CODE -lt 200 -o $REST_HTTP_CODE -ge 300 ]; then
+		REST_ERROR="REST status $REST_HTTP_CODE: $REST_OUTPUT"
+		record "${F_DANGER}$REST_ERROR${F_RESET}"
+		return 2
+	fi
+	record "CURL succeded, HTTP status $REST_HTTP_CODE"
+	record "$output"
+	return 0	
+}
+
+mgmt_create_user() {
+	local email="$1"
+	local pass="${2:-$email}"
+	local delete_first="${3:-yes}"
+
+	# ensure the user is deleted (clean test run)
+	if [ "$delete_first" == "yes" ]; then
+		delete_user "$email"
+	fi
+	record "[create user $email]"
+	mgmt_rest POST /admin/mail/users/add "email=$email" "password=$pass"
+	return $?
+}
+
+mgmt_assert_create_user() {
+	local email="$1"
+	local pass="$2"
+	local delete_first="${3}"
+	if ! mgmt_create_user "$email" "$pass" "$delete_first"; then
+		test_failure "Unable to create user $email"
+		test_failure "${REST_ERROR}"
+		return 1
+	fi
+	return 0
+}
+
+mgmt_delete_user() {
+	local email="$1"
+	record "[delete user $email]"
+	mgmt_rest POST /admin/mail/users/remove "email=$email"
+	return $?
+}
+
+mgmt_assert_delete_user() {
+	local email="$1"
+	if ! mgmt_delete_user "$email"; then
+		test_failure "Unable to cleanup/delete user $email"
+		test_failure "$REST_ERROR"
+		return 1
+	fi
+	return 0
+}
+
+mgmt_create_alias_group() {
+	local alias="$1"
+	shift
+	record "[Create new alias group $alias]"
+	record "members: $@"
+	# ensure the group is deleted (clean test run)
+	record "Try deleting any existing entry"
+	if ! mgmt_rest POST /admin/mail/aliases/remove "address=$alias"; then
+		get_attribute "$LDAP_ALIASES_BASE" "mail=$alias" "dn"
+		if [ ! -z "$ATTR_DN" ]; then
+			delete_dn "$ATTR_DN"
+		fi
+	fi
+
+	record "Create the alias group"
+	local members="$1" member
+	shift
+	for member;	do members="${members},${member}"; done
+
+	mgmt_rest POST /admin/mail/aliases/add "address=$alias" "forwards_to=$members"
+	return $?
+}
+
+mgmt_assert_create_alias_group() {
+	local alias="$1"
+	shift
+	if ! mgmt_create_alias_group "$alias" "$@"; then
+		test_failure "Unable to create alias group $alias"
+		test_failure "${REST_ERROR}"
+		return 1
+	fi
+	return 0
+}
+
+mgmt_delete_alias_group() {
+	local alias="$1"
+	record "[Delete alias group $alias]"
+	mgmt_rest POST /admin/mail/aliases/remove "address=$alias"
+	return $?
+}
+
+mgmt_assert_delete_alias_group() {
+	local alias="$1"
+	if ! mgmt_delete_alias_group "$alias"; then
+		test_failure "Unable to cleanup/delete alias group $alias"
+		test_failure "$REST_ERROR"
+		return 1
+	fi
+	return 0
+}

tests/suites/ldap-access.sh

@@ -0,0 +1,233 @@
+# -*- indent-tabs-mode: t; tab-width: 4; -*-
+#
+# Access assertions:
+#	service accounts, except management:
+#	   can bind but not change passwords, including their own
+#	   can read all attributes of all users but not userPassword
+#	   can not write any user attributes, include shadowLastChange
+#	   can read config subtree (permitted-senders, domains)
+#	   no access to services subtree, except their own dn
+#	users:
+#	   can bind and change their own password
+#	   can read and change their own shadowLastChange
+#	   can read attributess of all users except mailaccess
+#	   no access to config subtree
+#	   no access to services subtree
+#	other:
+#	   no anonymous binds to root DSE
+#	   no anonymous binds to database
+#
+
+
+test_user_change_password() {
+	# users should be able to change their own passwords
+	test_start "user-change-password"
+
+	# create regular user with password "alice"
+	local alice="alice@somedomain.com"	  
+	create_user "$alice" "alice"
+	local alice_dn="$ATTR_DN"
+
+	# bind as alice and update userPassword
+	assert_w_access "$alice_dn" "$alice_dn" "alice" write "userPassword=$(slappasswd_hash "alice-new")"
+	delete_user "$alice"
+	test_end
+}
+
+
+test_user_access() {
+	# 1. can read attributess of all users except mailaccess
+	# 2. can read and change their own shadowLastChange
+	# 3. no access to config subtree
+	# 4. no access to services subtree
+	test_start "user-access"
+
+	# create regular user's alice and bob
+	local alice="alice@somedomain.com"
+	create_user "alice@somedomain.com" "alice"
+	local alice_dn="$ATTR_DN"
+
+	local bob="bob@somedomain.com"
+	create_user "bob@somedomain.com" "bob"
+	local bob_dn="$ATTR_DN"
+
+	# alice should be able to set her own shadowLastChange
+	assert_w_access "$alice_dn" "$alice_dn" "alice" write "shadowLastChange=0"
+
+	# test that alice can read her own attributes
+	assert_r_access "$alice_dn" "$alice_dn" "alice" read mail maildrop cn sn shadowLastChange
+	# alice should not have access to her own mailaccess, though
+	assert_r_access "$alice_dn" "$alice_dn" "alice" no-read mailaccess
+	# test that alice cannot change her own select attributes
+	assert_w_access "$alice_dn" "$alice_dn" "alice"
+
+	
+	# test that alice can read bob's attributes
+	assert_r_access "$bob_dn" "$alice_dn" "alice" read mail maildrop cn sn
+	# alice does not have access to bob's mailaccess though
+	assert_r_access "$bob_dn" "$alice_dn" "alice" no-read mailaccess
+	# test that alice cannot change bob's attributes
+	assert_w_access "$bob_dn" "$alice_dn" "alice"
+
+
+	# test that alice cannot read a service account's attributes
+	assert_r_access "$LDAP_POSTFIX_DN" "$alice_dn" "alice"
+
+	# test that alice cannot read config entries
+	assert_r_access "dc=somedomain.com,$LDAP_DOMAINS_BASE" "$alice_dn" "alice"
+	assert_r_access "$LDAP_PERMITTED_SENDERS_BASE" "$alice_dn" "alice"
+
+	# test that alice cannot find anything searching config
+	test_search "$LDAP_CONFIG_BASE" "$alice_dn" "alice"
+	[ $SEARCH_DN_COUNT -gt 0 ] && test_failure "Users should not be able to search config"
+
+	# test that alice cannot find anything searching config domains
+	test_search "$LDAP_DOMAINS_BASE" "$alice_dn" "alice"
+	[ $SEARCH_DN_COUNT -gt 0 ] && test_failure "Users should not be able to search config domains"
+
+	# test that alice cannot find anything searching services
+	test_search "$LDAP_SERVICES_BASE" "$alice_dn" "alice"
+	[ $SEARCH_DN_COUNT -gt 0 ] && test_failure "Users should not be able to search services"
+
+	delete_user "$alice"
+	delete_user "$bob"
+	test_end	
+}
+
+
+
+test_service_change_password() {
+	# service accounts should not be able to change other user's
+	# passwords
+	# service accounts should not be able to change their own password
+	test_start "service-change-password"
+
+	# create regular user with password "alice"
+	local alice="alice@somedomain.com"	  
+	create_user "alice@somedomain.com" "alice"
+	local alice_dn="$ATTR_DN"
+
+	# create a test service account
+	create_service_account "test" "test"
+	local service_dn="$ATTR_DN"
+
+	# update userPassword of user using service account
+	assert_w_access "$alice_dn" "$LDAP_POSTFIX_DN" "$LDAP_POSTFIX_PASSWORD" no-write "userPassword=$(slappasswd_hash "alice-new")"
+
+	# update userPassword of service account using service account
+	assert_w_access "$service_dn" "$service_dn" "test" no-write "userPassword=$(slappasswd_hash "test-new")"
+	
+	delete_user "$alice"
+	delete_service_account "test"
+	test_end
+}
+
+
+test_service_access() {
+	# service accounts should have read-only access to all attributes
+	# of all users except userPassword
+	# can not write any user attributes, include shadowLastChange
+	# can read config subtree (permitted-senders, domains)
+	# no access to services subtree, except their own dn
+	
+	test_start "service-access"
+
+	# create regular user with password "alice"
+	local alice="alice@somedomain.com"
+	create_user "alice@somedomain.com" "alice"
+
+	# create a test service account
+	create_service_account "test" "test"
+	local service_dn="$ATTR_DN"
+
+	# Use service account to find alice
+	record "[Use service account to find alice]"
+	get_attribute "$LDAP_USERS_BASE" "mail=${alice}" dn sub "$LDAP_POSTFIX_DN" "$LDAP_POSTFIX_PASSWORD"
+	if [ -z "$ATTR_DN" ]; then
+		test_failure "Unable to search for user account using service account"
+	else
+		local alice_dn="$ATTR_DN"
+		
+		# set shadowLastChange on alice's entry (to test reading it back)
+		assert_w_access "$alice_dn" "$alice_dn" "alice" write "shadowLastChange=0"
+		
+		# check that service account can read user attributes
+		assert_r_access "$alice_dn" "$LDAP_POSTFIX_DN" "$LDAP_POSTFIX_PASSWORD" read mail maildrop uid cn sn shadowLastChange
+		
+		# service account should not be able to read user's userPassword
+		assert_r_access "$alice_dn" "$LDAP_POSTFIX_DN" "$LDAP_POSTFIX_PASSWORD" no-read userPassword
+
+		# service accounts cannot change user attributes
+		assert_w_access "$alice_dn" "$LDAP_POSTFIX_DN" "$LDAP_POSTFIX_PASSWORD"
+		assert_w_access "$alice_dn" "$LDAP_POSTFIX_DN" "$LDAP_POSTFIX_PASSWORD" no-write "shadowLastChange=1"
+	fi
+
+	# service accounts can read config subtree (permitted-senders, domains)
+	assert_r_access "dc=somedomain.com,$LDAP_DOMAINS_BASE" "$LDAP_POSTFIX_DN" "$LDAP_POSTFIX_PASSWORD" read dc
+
+	# service accounts can search and find things in the config subtree
+	test_search "$LDAP_CONFIG_BASE" "$LDAP_POSTFIX_DN" "$LDAP_POSTFIX_PASSWORD" sub
+	[ $SEARCH_DN_COUNT -lt 4 ] && test_failure "Service accounts should be able to search config"
+
+	# service accounts can read attributes in their own dn
+	assert_r_access "$LDAP_POSTFIX_DN" "$LDAP_POSTFIX_DN" "$LDAP_POSTFIX_PASSWORD" read cn description
+	# ... but not userPassword
+	assert_r_access "$LDAP_POSTFIX_DN" "$LDAP_POSTFIX_DN" "$LDAP_POSTFIX_PASSWORD" no-read userPassword
+
+	# services cannot read other service's attributes
+	assert_r_access "$service_dn" "$LDAP_POSTFIX_DN" "$LDAP_POSTFIX_PASSWORD" no-read cn description userPassword
+
+	delete_user "$alice"
+	delete_service_account "test"
+	test_end
+}
+
+
+test_root_dse() {
+	# no anonymous binds to root dse
+	test_start "root-dse"
+
+	record "[bind anonymously to root dse]"
+	ldapsearch -H $LDAP_URL -x -b "" -s base >>$TEST_OF 2>&1
+	local r=$?
+	if [ $r -eq 0 ]; then
+		test_failure "Anonymous access to root dse should not be permitted"
+	elif [ $r -eq 48 ]; then
+		# 48=inappropriate authentication (anon binds not allowed)
+		test_success
+	else
+		die "Error accessing root dse"
+	fi
+	test_end
+}
+
+test_anon_bind() {
+	test_start "anon-bind"
+
+	record "[bind anonymously to $LDAP_BASE]"
+	ldapsearch -H $LDAP_URL -x -b "$LDAP_BASE" -s base >>$TEST_OF 2>&1
+	local r=$?
+	if [ $r -eq 0 ]; then
+		test_failure "Anonymous access should not be permitted"
+	elif [ $r -eq 48 ]; then
+		# 48=inappropriate authentication (anon binds not allowed)
+		test_success
+	else
+		die "Error accessing $LDAP_BASE"
+	fi
+
+	test_end
+}
+
+
+
+suite_start "ldap-access"
+
+test_user_change_password
+test_user_access
+test_service_change_password
+test_service_access
+test_root_dse
+test_anon_bind
+
+suite_end

tests/suites/ldap-connection.sh

@@ -0,0 +1,151 @@
+# -*- indent-tabs-mode: t; tab-width: 4; -*-
+
+exe_test() {
+	# run an executable and assert success or failure
+	# argument 1 must be:
+	#    "ZERO_RC" to assert the return code was 0
+	#    "NONZERO_RC" to assert the return code was not 0
+	# argument 2 is a description of the test for logging
+	# argument 3 and higher are the executable and arguments
+	local result_type=$1
+	shift
+	local desc="$1"
+	shift
+	test_start "$desc"
+	record "[CMD: $@]"
+	"$@" >>"$TEST_OF" 2>&1
+	local code=$?
+	case $result_type in
+		ZERO_RC)
+			if [ $code -ne 0 ]; then
+				test_failure "expected zero return code, got $code"
+			else
+				test_success
+			fi
+			;;
+
+		NONZERO_RC)
+			if [ $code -eq 0 ]; then
+				test_failure "expected non-zero return code"
+			else
+				test_success
+			fi
+			;;
+
+		*)
+			test_failure "unknown TEST type '$result_type'"
+			;;
+	esac
+	test_end
+}
+
+
+tests() {
+	# TLS: auth search to (local)host - expect success
+	exe_test ZERO_RC "TLS-auth-host" \
+			 ldapsearch -d 1 -b "dc=mailinabox" -H ldaps://$PRIMARY_HOSTNAME/ -x -D "$LDAP_ADMIN_DN" -w "$LDAP_ADMIN_PASSWORD"
+
+	# TLS: auth search to localhost - expect failure ("hostname does not match CN in peer certificate")
+	exe_test NONZERO_RC "TLS-auth-local" \
+			 ldapsearch -d 1 -b "dc=mailinabox" -H ldaps://127.0.0.1/ -x -D "$LDAP_ADMIN_DN" -w "$LDAP_ADMIN_PASSWORD"
+
+	# TLS: anon search - expect failure (anon bind disallowed)
+	exe_test NONZERO_RC "TLS-anon-host" \
+			 ldapsearch -d 1 -b "dc=mailinabox" -H ldaps://$PRIMARY_HOSTNAME/ -x
+	
+	# CLEAR: auth search to host - expected failure (not listening there)
+	exe_test NONZERO_RC "CLEAR-auth-host" \
+			 ldapsearch -d 1 -b "dc=mailinabox" -H ldap://$PRIVATE_IP/ -x -D "$LDAP_ADMIN_DN" -w "$LDAP_ADMIN_PASSWORD"
+
+	# CLEAR: auth search to localhost - expect success
+	exe_test ZERO_RC "CLEAR-auth-local" \
+			 ldapsearch -d 1 -b "dc=mailinabox" -H ldap://127.0.0.1/ -x -D "$LDAP_ADMIN_DN" -w "$LDAP_ADMIN_PASSWORD"
+
+	# CLEAR: anon search - expect failure (anon bind disallowed)
+	exe_test NONZERO_RC "CLEAR-anon-local" \
+			 ldapsearch -d 1 -b "dc=mailinabox" -H ldap://127.0.0.1/ -x
+	
+	# STARTTLS: auth search to localhost - expected failure ("hostname does not match CN in peer certificate")
+	exe_test NONZERO_RC "STARTTLS-auth-local" \
+			 ldapsearch -d 1 -b "dc=mailinabox" -H ldap://127.0.0.1/ -x -D "$LDAP_ADMIN_DN" -w "$LDAP_ADMIN_PASSWORD" -ZZ
+
+	# STARTTLS: auth search to host - expected failure (not listening there)
+	exe_test NONZERO_RC "STARTTLS-auth-host" \
+			 ldapsearch -d 1 -b "dc=mailinabox" -H ldap://$PRIVATE_IP/ -x -D "$LDAP_ADMIN_DN" -w "$LDAP_ADMIN_PASSWORD" -ZZ
+
+}
+
+
+test_fail2ban() {
+	test_start "fail2ban"
+	
+	# reset fail2ban
+	record "[reset fail2ban]"
+	fail2ban-client unban --all >>$TEST_OF 2>&1 ||
+		test_failure "Unable to execute unban --all"
+
+	# create regular user with password "alice"
+	local alice="alice@somedomain.com"	  
+	create_user "$alice" "alice"
+	local alice_dn="$ATTR_DN"
+
+	# log in a bunch of times with wrong password
+	local n=0
+	local total=25
+	local banned=no
+	record '[log in 25 times with wrong password]'
+	while ! have_test_failures && [ $n -lt $total ]; do
+		ldapsearch -H $LDAP_URL -D "$alice_dn" -w "bad-alice" -b "$LDAP_USERS_BASE" -s base "(objectClass=*)" 1>>$TEST_OF 2>&1
+		local code=$?
+		record "TRY $n: result code $code"
+		
+		if [ $code -eq 255 -a $n -gt 5 ]; then
+			# banned - could not connect
+			banned=yes
+			break
+			
+		elif [ $code -ne 49 ]; then
+			test_failure "Expected error code 49 (invalidCredentials), but got $code"
+			continue
+		fi
+		
+		let n+=1
+		if [ $n -lt $total ]; then
+			record "sleep 1"
+			sleep 1
+		fi
+	done
+
+	if ! have_test_failures && [ "$banned" == "no" ]; then
+		# wait for fail2ban to ban
+		record "[waiting for fail2ban]"
+		record "sleep 5"
+		sleep 5
+		ldapsearch -H ldap://$PRIVATE_IP -D "$alice_dn" -w "bad-alice" -b "$LDAP_USERS_BASE" -s base "(objectClass=*)" 1>>$TEST_OF 2>&1
+		local code=$?
+		record "$n result: $code"
+		if [ $code -ne 255 ]; then
+			test_failure "Expected to be banned after repeated login failures, but wasn't"
+		fi
+	fi
+
+	# delete alice
+	delete_user "$alice"
+	
+	# reset fail2ban
+	record "[reset fail2ban]"
+	fail2ban-client unban --all >>$TEST_OF 2>&1 ||
+		test_failure "Unable to execute unban --all"
+
+	# done
+	test_end
+}
+
+
+suite_start "ldap-connection"
+
+tests
+test_fail2ban
+
+suite_end
+

tests/suites/mail-access.sh

@@ -0,0 +1,199 @@
+# -*- indent-tabs-mode: t; tab-width: 4; -*-
+#
+
+_test_greylisting_x() {
+	# helper function sends mail and checks that it was greylisted
+	local email_to="$1"
+	local email_from="$2"
+	
+	start_log_capture
+	start_mail_capture "$email_to"
+	record "[Send mail anonymously TO $email_to FROM $email_from]"
+	local output
+	output="$($PYMAIL -no-delete -f $email_from -to $email_to '' $PRIVATE_IP '' '' 2>&1)"
+	local code=$?
+	if [ $code -eq 0 ]; then	
+		wait_mail
+		local file=( $(get_captured_mail_files) )
+		record "[Check captured mail for X-Greylist header]"
+		if ! grep "X-Greylist: delayed" <"$file" >/dev/null; then
+			record "not found"
+			test_failure "message not greylisted - X-Greylist header missing"
+			record_captured_mail
+		else
+			record "found"
+		fi
+	else
+		assert_python_failure $code "$output" "SMTPRecipientsRefused" "Greylisted"
+	fi
+
+	check_logs
+}
+
+
+postgrey_reset() {
+	# when postgrey receives a message for processing that is suspect,
+	# it will:
+	#	1. initally reject it
+	#	2. after a delay, permit delivery (end entity must resend),
+	#	   but with a X-Greyist header	  
+	#	3. subsequent deliveries will succeed with no header
+	#	   modifications
+	#
+	# because of #3, reset postgrey to establish a "clean" greylisting
+	# testing scenario
+	#
+	record "[Reset postgrey]"
+	if [ ! -d "/var/lib/postgrey" ]; then
+		die "Postgrey database directory /var/lib/postgrey does not exist!"
+	fi
+	systemctl stop postgrey >>$TEST_OF 2>&1 || die "unble to stop postgrey"
+	if ! rm -f /var/lib/postgrey/* >>$TEST_OF 2>&1; then
+		systemctl start postgrey >>$TEST_OF 2>&1
+		die "unable to remove the postgrey database files"
+	fi
+	systemctl start postgrey >>$TEST_OF 2>&1 || die "unble to start postgrey"
+}
+
+
+test_greylisting() {
+	# test that mail is delayed by greylisting
+	test_start "greylisting"
+
+	# reset postgrey's database to start the cycle over
+	postgrey_reset
+
+	# create standard user alice
+	local alice="alice@somedomain.com"
+	create_user "$alice" "alice"
+
+	# IMPORTANT: bob's domain must be from one that has no SPF record
+	# in DNS. At the time of creation of this script, yahoo.com did
+	# not...
+	local bob="bob@yahoo.com"
+	
+	# send to alice anonymously from bob
+	_test_greylisting_x "$alice" "$bob"
+	
+	delete_user "$alice"
+	test_end
+}
+
+
+test_relay_prohibited() {
+	# test that the server does not relay
+	test_start "relay-prohibited"
+
+	start_log_capture
+	record "[Attempt relaying mail anonymously]"
+	local output
+	output="$($PYMAIL -no-delete -f joe@badguy.com -to naive@gmail.com '' $PRIVATE_IP '' '' 2>&1)"
+	assert_python_failure $? "$output" "SMTPRecipientsRefused" "Relay access denied"
+	check_logs
+
+	test_end
+}
+
+
+test_spf() {
+	# test mail rejection due to SPF policy of FROM address
+	test_start "spf"
+	
+	# create standard user alice
+	local alice="alice@somedomain.com"
+	create_user "$alice" "alice"
+
+	# who we will impersonate
+	local from="test@google.com"
+	local domain=$(awk -F@ '{print $2}' <<<"$from")
+
+	# send to alice anonymously from imposter
+	start_log_capture
+	start_mail_capture "$alice"
+	record "[Test SPF for $domain FROM $from TO $alice]"
+	local output
+	output="$($PYMAIL -no-delete -f $from -to $alice '' $PRIVATE_IP '' '' 2>&1)"
+	local code=$?
+	if ! assert_python_failure $code "$output" "SMTPRecipientsRefused" "SPF" && [ $code -eq 0 ]
+	then
+		wait_mail
+		record_captured_mail
+	fi
+	check_logs
+	
+	delete_user "$alice"
+	test_end
+}
+
+
+test_mailbox_pipe() {
+	# postfix allows piped commands in aliases for local processing,
+	# which is a serious security issue. test that pipes are not
+	# permitted or don't work
+	test_start "mailbox-pipe"
+	
+	# create standard user alice
+	local alice="alice@somedomain.com"
+	create_user "$alice" "alice"
+	local alice_dn="$ATTR_DN"
+
+	# create the program to handle piped mail
+	local cmd="/tmp/pipedrop.$$.sh"
+	local outfile="/tmp/pipedrop.$$.out"
+	cat 2>>$TEST_OF >$cmd <<EOF
+#!/bin/bash
+cat > $outfile
+EOF
+	chmod 755 $cmd
+	rm -f $outfile
+	
+	# add a piped maildrop
+	record "[Add pipe command as alice's maildrop]"
+	ldapmodify -H $LDAP_URL -x -D "$LDAP_ADMIN_DN" -w "$LDAP_ADMIN_PASSWORD" >>$TEST_OF 2>&1 <<EOF
+dn: $alice_dn
+replace: maildrop
+maildrop: |$cmd
+EOF
+	[ $? -ne 0 ] && die "Could not modify ${alice}'s maildrop"
+
+	# send an email message to alice
+	start_log_capture
+	record "[Send an email to $alice - test pipe]"
+	local output
+	output="$($PYMAIL -no-delete $PRIVATE_IP $alice alice 2>&1)"
+	local code=$?
+	
+	if [ $code -ne 0 ]; then
+		assert_python_failure $code "$output" SMTPAuthenticationError
+		check_logs
+	else
+		sleep 5
+		if grep_postfix_log "User doesn't exist: |$cmd@"; then
+			# ok
+			check_logs
+		else
+			assert_check_logs
+		fi
+		
+		if [ -e $outfile ]; then
+			test_failure "a maildrop containing a pipe was executed by postfix"
+		fi
+	fi
+	
+	delete_user "$alice"
+	rm -f $cmd
+	rm -f $outfile
+	
+	test_end
+
+}
+
+
+suite_start "mail-access"
+
+test_greylisting
+test_relay_prohibited
+test_spf
+test_mailbox_pipe
+
+suite_end

tests/suites/mail-aliases.sh

@@ -0,0 +1,329 @@
+# -*- indent-tabs-mode: t; tab-width: 4; -*-
+# mail alias tests
+#
+
+test_shared_user_alias_login() {
+	# a login attempt should fail when using 'mail' aliases that map
+	# to two or more users
+	
+	test_start "shared-user-alias-login"
+	# create standard users alice and bob
+	local alice="alice@somedomain.com"
+	local bob="bob@anotherdomain.com"
+	create_user "$alice" "alice"
+	local alice_dn="$ATTR_DN"
+	create_user "$bob" "bob"
+	local bob_dn="$ATTR_DN"
+
+	# add common alias to alice and bob
+	local alias="us@somedomain.com"
+	add_alias $alice_dn $alias user
+	add_alias $bob_dn $alias user
+
+	start_log_capture
+	record "[Log in as alias to postfix]"
+	local output
+	local subject="Mail-In-A-Box test $(generate_uuid)"
+	
+	# login as the alias to postfix - should fail
+	output="$($PYMAIL -subj "$subject" -no-delete $PRIVATE_IP $alias alice 2>&1)"
+	assert_python_failure $? "$output" "SMTPAuthenticationError"
+
+	# login as the alias to dovecot - should fail
+	record "[Log in as alias to dovecot]"
+	local timeout=""
+	if have_test_failures; then
+		timeout="-timeout 0"
+	fi
+	output="$($PYMAIL -subj "$subject" $timeout -no-send $PRIVATE_IP $alias alice 2>&1)"
+	assert_python_failure $? "$output" "authentication failure"
+
+	check_logs
+
+	delete_user "$alice"
+	delete_user "$bob"
+	test_end
+}
+
+
+test_alias_group_member_login() {
+	# a login attempt should fail when using an alias defined in a
+	# mailGroup type alias
+	
+	test_start "alias-group-member-login"
+	# create standard user alice
+	local alice="alice@somedomain.com"
+	create_user "$alice" "alice"
+	local alice_dn="$ATTR_DN"
+
+	# create alias group with alice in it
+	local alias="us@somedomain.com"
+	create_alias_group "$alias" "$alice_dn"
+
+	start_log_capture
+	record "[Log in as alias to postfix]"
+	local output
+	local subject="Mail-In-A-Box test $(generate_uuid)"
+	
+	# login as the alias to postfix - should fail
+	output="$($PYMAIL -subj "$subject" -no-delete $PRIVATE_IP $alias alice 2>&1)"
+	assert_python_failure $? "$output" "SMTPAuthenticationError"
+
+	# login as the alias to dovecot - should fail
+	record "[Log in as alias to dovecot]"
+	local timeout=""
+	if have_test_failures; then
+		timeout="-timeout 0"
+	fi
+	output="$($PYMAIL -subj "$subject" $timeout -no-send $PRIVATE_IP $alias alice 2>&1)"
+	assert_python_failure $? "$output" "AUTHENTICATIONFAILED"
+
+	check_logs
+
+	delete_user "$alice"
+	delete_alias_group "$alias"
+	test_end
+}
+
+
+test_shared_alias_delivery() {
+	# mail sent to the shared alias of two users (eg. postmaster),
+	# should be sent to both users
+	test_start "shared-alias-delivery"
+	# create standard users alice, bob, and mary
+	local alice="alice@somedomain.com"
+	local bob="bob@anotherdomain.com"
+	local mary="mary@anotherdomain.com"
+	create_user "$alice" "alice"
+	local alice_dn="$ATTR_DN"
+	create_user "$bob" "bob"
+	local bob_dn="$ATTR_DN"
+	create_user "$mary" "mary"
+
+	# add common alias to alice and bob
+	local alias="us@somedomain.com"
+	create_alias_group $alias $alice_dn $bob_dn
+
+	# login as mary and send to alias
+	start_log_capture
+	record "[Sending mail to alias]"
+	local output
+	local subject="Mail-In-A-Box test $(generate_uuid)"
+	output="$($PYMAIL -subj "$subject" -no-delete -to $alias na $PRIVATE_IP $mary mary 2>&1)"
+	if assert_python_success $? "$output"; then
+		# check that alice and bob received it by deleting the mail in
+		# both mailboxes
+		record "[Delete mail alice's mailbox]"
+		output="$($PYMAIL -subj "$subject" -no-send $PRIVATE_IP $alice alice 2>&1)"
+		assert_python_success $? "$output"
+		record "[Delete mail bob's mailbox]"
+		output="$($PYMAIL -subj "$subject" -no-send $PRIVATE_IP $bob bob 2>&1)"
+		assert_python_success $? "$output"
+	fi
+
+	assert_check_logs
+	
+	delete_user "$alice"
+	delete_user "$bob"
+	delete_user "$mary"
+	delete_alias_group $alias
+	test_end
+
+}
+
+
+test_trial_nonlocal_alias_delivery() {
+	# verify that mail sent to an alias with a non-local address
+	# (rfc822MailMember) can be delivered
+	test_start "trial-nonlocal-alias-delivery"
+
+	# add alias
+	local alias="external@somedomain.com"
+	create_alias_group $alias "test@google.com"
+
+	# trail send...doesn't actually get delivered
+	start_log_capture
+	sendmail_bv_send "$alias" 120
+	assert_check_logs
+	have_test_failures && record_captured_mail
+	delete_alias_group $alias
+	test_end
+}
+
+
+
+
+test_catch_all() {
+	# 1. ensure users in the catch-all alias receive messages to
+	# invalid users for handled domains
+	#
+	# 2. ensure sending mail to valid user does not go to catch-all
+	#
+	test_start "catch-all"
+	# create standard users alice, bob, and mary
+	local alice="alice@somedomain.com"
+	local bob="bob@anotherdomain.com"
+	local mary="mary@anotherdomain.com"
+	create_user "$alice" "alice"
+	local alice_dn="$ATTR_DN"
+	create_user "$bob" "bob"
+	local bob_dn="$ATTR_DN"
+	create_user "$mary" "mary"
+
+	# add catch-all alias to alice and bob
+	local alias="@somedomain.com"
+	create_alias_group $alias $alice_dn $bob_dn
+
+	# login as mary, then send to an invalid address. alice and bob
+	# should receive that mail because they're aliases to the
+	# catch-all for the domain
+	record "[Sending mail to invalid user at catch-all domain]"
+	start_log_capture
+	local output
+	local subject="Mail-In-A-Box test $(generate_uuid)"
+	output="$($PYMAIL -subj "$subject" -no-delete -to INVALID${alias} na $PRIVATE_IP $mary mary 2>&1)"
+	if assert_python_success $? "$output"; then
+		# check that alice and bob received it by deleting the mail in
+		# both mailboxes
+		record "[Delete mail in alice's and bob's mailboxes]"
+		output="$($PYMAIL -subj "$subject" -no-send $PRIVATE_IP $alice alice 2>&1)"
+		assert_python_success $? "$output"
+		output="$($PYMAIL -subj "$subject" -no-send $PRIVATE_IP $bob bob 2>&1)"
+		assert_python_success $? "$output"
+	fi
+	assert_check_logs
+
+	# login as mary and send to a valid address at the catch-all
+	# domain. that user should receive it and the catch-all should not
+	record "[Sending mail to valid user at catch-all domain]"
+	start_log_capture
+	subject="Mail-In-A-Box test $(generate_uuid)"
+	output="$($PYMAIL -subj "$subject" -to $alice alice $PRIVATE_IP $mary mary 2>&1)"
+	if assert_python_success $? "$output"; then
+		# alice got the mail and it was deleted
+		# make sure bob didn't also receive the message
+		record "[Delete mail in bob's mailbox]"
+		output="$($PYMAIL -timeout 10 -subj "$subject" -no-send $PRIVATE_IP $bob bob 2>&1)"
+		assert_python_failure $? "$output" "TimeoutError"
+	fi
+	assert_check_logs
+
+	delete_user "$alice"
+	delete_user "$bob"
+	delete_user "$mary"
+	delete_alias_group $alias
+	test_end
+}
+
+
+test_nested_alias_groups() {
+	# sending to an alias with embedded aliases should reach final
+	# recipients
+	test_start "nested-alias-groups"
+	# create standard users alice and bob
+	local alice="alice@zdomain.z"
+	create_user "$alice" "alice"
+	local alice_dn="$ATTR_DN"
+	local bob="bob@zdomain.z"
+	create_user "$bob" "bob"
+	local bob_dn="$ATTR_DN"
+
+	# add nested alias groups [ alias1 -> alias2 -> alice ]
+	local alias1="z1@xyzdomain.z"
+	local alias2="z2@xyzdomain.z"
+	create_alias_group $alias2 $alice_dn
+	create_alias_group $alias1 $ATTR_DN
+
+	# send to alias1 from bob, then ensure alice received it
+	record "[Sending mail to alias $alias1]"
+	start_log_capture
+	local output
+	local subject="Mail-In-A-Box test $(generate_uuid)"
+	output="$($PYMAIL -subj "$subject" -no-delete -to $alias1 na $PRIVATE_IP $bob bob 2>&1)"
+	if assert_python_success $? "$output"; then
+		record "[Test delivery - delete mail in alice's mailbox]"
+		output="$($PYMAIL -subj "$subject" -no-send $PRIVATE_IP $alice alice 2>&1)"
+		assert_python_success $? "$output"
+	fi
+
+	assert_check_logs
+
+	delete_user "$alice"
+	delete_user "$bob"
+	delete_alias_group "$alias1"
+	delete_alias_group "$alias2"
+	
+	test_end
+}
+
+test_user_rename() {
+	# test the use case where someone's name changed
+	# in this test we rename the user's 'mail' address, but
+	# leave maildrop as-is
+	test_start "user-rename"
+
+	# create standard user alice
+	local alice1="alice.smith@somedomain.com"
+	local alice2="alice.jones@somedomain.com"
+	create_user "$alice1" "alice"
+	local alice_dn="$ATTR_DN"
+	local output
+
+	# send email to alice with subject1
+	record "[Testing mail to alice1]"
+	local subject1="Mail-In-A-Box test $(generate_uuid)"
+	local success1=false
+	start_mail_capture "$alice1"
+	record "[Sending mail to $alice1]"
+	output="$($PYMAIL -subj "$subject1" -no-delete $PRIVATE_IP $alice1 alice 2>&1)"
+	assert_python_success $? "$output" && success1=true
+
+	# alice1 got married, add a new mail address alice2
+	wait_mail	# rename too soon, and the first message is bounced
+	record "[Changing alice's mail address]"
+	ldapmodify -H $LDAP_URL -x -D "$LDAP_ADMIN_DN" -w "$LDAP_ADMIN_PASSWORD" >>$TEST_OF 2>&1 <<EOF
+dn: $alice_dn
+replace: mail
+mail: $alice2
+EOF
+	[ $? -ne 0 ] && die "Unable to modify ${alice1}'s mail address!"
+
+	# send email to alice with subject2
+	start_log_capture
+	local subject2="Mail-In-A-Box test $(generate_uuid)"
+	local success2=false
+	record "[Sending mail to $alice2]"
+	output="$($PYMAIL -subj "$subject2" -no-delete $PRIVATE_IP $alice2 alice 2>&1)"
+	assert_python_success $? "$output" && success2=true
+	assert_check_logs
+
+	# delete both messages
+	if $success1; then
+		record "[Deleting mail 1]"
+		output="$($PYMAIL -subj "$subject1" -no-send $PRIVATE_IP $alice2 alice 2>&1)"
+		assert_python_success $? "$output"
+	fi
+
+	if $success2; then
+		record "[Deleting mail 2]"
+		output="$($PYMAIL -subj "$subject2" -no-send $PRIVATE_IP $alice2 alice 2>&1)"
+		assert_python_success $? "$output"
+	fi
+
+	delete_user "$alice2"
+	test_end
+}
+
+
+
+suite_start "mail-aliases"
+
+test_shared_user_alias_login
+test_alias_group_member_login
+test_shared_alias_delivery	 # local alias delivery
+test_trial_nonlocal_alias_delivery
+test_catch_all
+test_nested_alias_groups
+test_user_rename
+
+suite_end

tests/suites/mail-basic.sh

@@ -0,0 +1,73 @@
+# -*- indent-tabs-mode: t; tab-width: 4; -*-
+#	
+# Test basic mail functionality
+
+
+
+test_trial_send_local() {
+	# use sendmail -bv to test mail delivery without actually mailing
+	# anything
+	test_start "trial_send_local"
+
+	# create a standard users alice and bobo
+	local alice="alice@somedomain.com" bob="bob@somedomain.com"
+	create_user "$alice" "alice"
+	create_user "$bob" "bob"
+
+	# test delivery, but don't actually mail it
+	start_log_capture
+	sendmail_bv_send "$alice" 30 "$bob"
+	assert_check_logs
+	have_test_failures && record_captured_mail
+
+	# clean up / end
+	delete_user "$alice"
+	delete_user "$bob"
+	test_end
+}
+
+test_trial_send_remote() {
+	# use sendmail -bv to test mail delivery without actually mailing
+	# anything
+	test_start "trial_send_remote"
+	start_log_capture
+	sendmail_bv_send "test@google.com" 120
+	assert_check_logs
+	have_test_failures && record_captured_mail
+	test_end
+}
+
+
+test_self_send_receive() {
+	# test sending mail to yourself
+	test_start "self-send-receive"
+	# create standard user alice
+	local alice="alice@somedomain.com"
+	create_user "$alice" "alice"
+
+	# test actual delivery
+	start_log_capture
+	record "[Sending mail to alice as alice]"
+	local output
+	output="$($PYMAIL $PRIVATE_IP $alice alice 2>&1)"
+	local code=$?
+	record "$output"
+	if [ $code -ne 0 ]; then
+		test_failure "$PYMAIL exit code $code: $output"
+	fi
+	assert_check_logs
+
+	delete_user "$alice"
+	test_end
+}
+
+
+
+suite_start "mail-basic"
+
+test_trial_send_local
+test_trial_send_remote
+test_self_send_receive
+
+suite_end
+

tests/suites/mail-from.sh

@@ -0,0 +1,141 @@
+# -*- indent-tabs-mode: t; tab-width: 4; -*-
+
+
+test_permitted_sender_fail() {
+	# a user may not send MAIL FROM someone else, when not permitted
+	test_start "permitted-sender-fail"
+	# create standard users alice, bob, and mary
+	local alice="alice@somedomain.com"
+	local bob="bob@anotherdomain.com"
+	local mary="mary@anotherdomain.com"
+	create_user "$alice" "alice"
+	create_user "$bob" "bob"
+	create_user "$mary" "mary"
+
+	# login as mary, send from bob, to alice
+	start_log_capture
+	record "[Mailing to alice from bob as mary]"
+	local output
+	output="$($PYMAIL -f $bob -to $alice alice $PRIVATE_IP $mary mary 2>&1)"
+	if ! assert_python_failure $? "$output" SMTPRecipientsRefused
+	then
+		# additional "color"
+		test_failure "user should not be permitted to send as another user"
+	fi
+
+	# expect errors, so don't assert
+	check_logs
+
+	delete_user "$alice"
+	delete_user "$bob"
+	delete_user "$mary"
+	test_end
+}
+
+
+test_permitted_sender_alias() {
+	# a user may send MAIL FROM one of their own aliases
+	test_start "permitted-sender-alias"
+	# create standard users alice and bob
+	local alice="alice@somedomain.com"
+	local bob="bob@anotherdomain.com"
+	local mary="mary@anotherdomain.com"
+	local jane="jane@google.com"
+	create_user "$alice" "alice"
+	create_user "$bob" "bob"
+	local bob_dn="$ATTR_DN"
+
+	# add mary as one of bob's aliases - to bob's 'mail' attribute
+	add_alias $bob_dn $mary user
+
+	# add jane as one of bob's aliases - to jane's alias group
+	create_alias_group $jane $bob_dn
+
+	# login as bob, send from mary, to alice
+	start_log_capture
+	record "[Mailing to alice from mary as bob]"
+	local output
+	output="$($PYMAIL -f $mary -to $alice alice $PRIVATE_IP $bob bob 2>&1)"
+	if ! assert_python_success $? "$output"; then
+		# additional "color"
+		test_failure "bob should be permitted to MAIL FROM $mary, his own alias: $(python_error "$output")"
+	fi
+
+	assert_check_logs
+
+	# login as bob, send from jane, to alice
+	start_log_capture
+	record "[Mailing to alice from jane as bob]"
+	local output
+	output="$($PYMAIL -f $jane -to $alice alice $PRIVATE_IP $bob bob 2>&1)"
+	if ! assert_python_success $? "$output"; then
+		# additional "color"
+		test_failure "bob should be permitted to MAIL FROM $jane, his own alias: $(python_error "$output")"
+	fi
+
+	assert_check_logs
+
+	delete_user "$alice"
+	delete_user "$bob"
+	delete_alias_group "$jane"
+	test_end
+}
+
+
+test_permitted_sender_explicit() {
+	# a user may send MAIL FROM an address that is explicitly allowed
+	# by a permitted-senders group
+	# a user may not send MAIL FROM an address that has a permitted
+	# senders list which they are not a member, even if they are an
+	# alias group member
+	test_start "permitted-sender-explicit"
+
+	# create standard users alice and bob
+	local alice="alice@somedomain.com"
+	local bob="bob@anotherdomain.com"
+	create_user "$alice" "alice"
+	local alice_dn="$ATTR_DN"
+	create_user "$bob" "bob"
+	local bob_dn="$ATTR_DN"
+
+	# create an alias that forwards to bob and alice
+	local alias="mary@anotherdomain.com"
+	create_alias_group $alias $bob_dn $alice_dn
+	
+	# create a permitted-senders group with only alice in it
+	create_permitted_senders_group $alias $alice_dn
+
+	# login as alice, send from alias to bob
+	start_log_capture
+	record "[Mailing to bob from alice as alias/mary]"
+	local output
+	output="$($PYMAIL -f $alias -to $bob bob $PRIVATE_IP $alice alice 2>&1)"
+	if ! assert_python_success $? "$output"; then
+		test_failure "user should be allowed to MAIL FROM a user for which they are a permitted sender: $(python_error "$output")"
+	fi
+	assert_check_logs
+
+	# login as bob, send from alias to alice
+	# expect failure because bob is not a permitted-sender
+	start_log_capture
+	record "[Mailing to alice from bob as alias/mary]"
+	output="$($PYMAIL -f $alias -to $alice alice $PRIVATE_IP $bob bob 2>&1)"
+	assert_python_failure $? "$output" "SMTPRecipientsRefused" "not owned by user"
+	check_logs
+
+	delete_user $alice
+	delete_user $bob
+	delete_permitted_senders_group $alias
+	create_alias_group $alias
+	test_end
+}
+
+
+
+suite_start "mail-from"
+
+test_permitted_sender_fail
+test_permitted_sender_alias
+test_permitted_sender_explicit
+
+suite_end

tests/suites/management-users.sh

@@ -0,0 +1,210 @@
+# -*- indent-tabs-mode: t; tab-width: 4; -*-
+#	
+# User management tests
+
+_test_mixed_case() {
+	# helper function sends multiple email messages to test mixed case
+	# input scenarios
+	local alices=($1)  # list of mixed-case email addresses for alice
+	local bobs=($2)    # list of mixed-case email addresses for bob
+	local aliases=($3) # list of mixed-case email addresses for an alias
+
+	start_log_capture
+
+	local alice_pw="$(generate_password 16)"
+	local bob_pw="$(generate_password 16)"
+	# create local user alice and alias group
+	if mgmt_assert_create_user "${alices[0]}" "$alice_pw"; then
+		# test that alice cannot also exist at the same time
+		if mgmt_create_user "${alices[1]}" "$alice_pw" no; then
+			test_failure "Creation of a user with the same email address, but different case, succeeded."
+			test_failure "${REST_ERROR}"
+		fi
+		
+		# create an alias group with alice in it
+		mgmt_assert_create_alias_group "${aliases[0]}" "${alices[1]}"
+	fi
+
+	# create local user bob
+	mgmt_assert_create_user "${bobs[0]}" "$bob_pw"
+	
+	assert_check_logs
+	
+
+	# send mail from bob to alice
+	#
+	if ! have_test_failures; then
+		record "[Mailing to alice from bob]"
+		start_log_capture
+		local output
+		output="$($PYMAIL -to ${alices[2]} "$alice_pw" $PRIVATE_IP ${bobs[1]} "$bob_pw" 2>&1)"
+		assert_python_success $? "$output"
+		assert_check_logs
+
+		# send mail from bob to the alias, ensure alice got it
+		#
+		record "[Mailing to alias from bob]"
+		start_log_capture
+		local subject="Mail-In-A-Box test $(generate_uuid)"
+		output="$($PYMAIL -subj "$subject" -no-delete -to ${aliases[1]} na $PRIVATE_IP ${bobs[2]} "$bob_pw" 2>&1)"
+		assert_python_success $? "$output"
+		output="$($PYMAIL -subj "$subject" -no-send $PRIVATE_IP ${alices[3]} "$alice_pw" 2>&1)"
+		assert_python_success $? "$output"
+		assert_check_logs
+		
+		# send mail from alice as the alias to bob, ensure bob got it
+		#
+		record "[Mailing to bob as alias from alice]"
+		start_log_capture
+		local subject="Mail-In-A-Box test $(generate_uuid)"
+		output="$($PYMAIL -subj "$subject" -no-delete -f ${aliases[2]} -to ${bobs[2]} "$bob_pw" $PRIVATE_IP ${alices[4]} "$alice_pw" 2>&1)"
+		assert_python_success $? "$output"
+		output="$($PYMAIL -subj "$subject" -no-send $PRIVATE_IP ${bobs[3]} "$bob_pw" 2>&1)"
+		assert_python_success $? "$output"
+		assert_check_logs
+	fi
+
+	mgmt_assert_delete_user "${alices[1]}"
+	mgmt_assert_delete_user "${bobs[1]}"
+	mgmt_assert_delete_alias_group "${aliases[1]}"
+}
+
+
+test_mixed_case_users() {
+	# create mixed-case user name
+	# add user to alias using different cases
+	# send mail from another user to that user - validates smtp, imap, delivery
+	# send mail from another user to the alias
+	# send mail from that user as the alias to the other user
+
+	test_start "mixed-case-users"
+	
+	local alices=(alice@mgmt.somedomain.com
+				  aLICE@mgmt.somedomain.com
+				  aLiCe@mgmt.somedomain.com
+				  ALICE@mgmt.somedomain.com
+				  alIce@mgmt.somedomain.com)
+	local bobs=(bob@mgmt.somedomain.com
+				Bob@mgmt.somedomain.com
+				boB@mgmt.somedomain.com
+				BOB@mgmt.somedomain.com)
+	local aliases=(aLICE@mgmt.anotherdomain.com
+				   aLiCe@mgmt.anotherdomain.com
+				   ALICE@mgmt.anotherdomain.com)
+
+	_test_mixed_case "${alices[*]}" "${bobs[*]}" "${aliases[*]}"
+	
+	test_end
+}
+
+
+test_mixed_case_domains() {
+	# create mixed-case domain names
+	# add user to alias using different cases
+	# send mail from another user to that user - validates smtp, imap, delivery
+	# send mail from another user to the alias
+	# send mail from that user as the alias to the other user
+
+	test_start "mixed-case-domains"
+	
+	local alices=(alice@mgmt.somedomain.com
+				  alice@MGMT.somedomain.com
+				  alice@mgmt.SOMEDOMAIN.com
+				  alice@mgmt.somedomain.COM
+				  alice@Mgmt.SomeDomain.Com)
+	local bobs=(bob@mgmt.somedomain.com
+				bob@MGMT.somedomain.com
+				bob@mgmt.SOMEDOMAIN.com
+				bob@Mgmt.SomeDomain.com)
+	local aliases=(alice@MGMT.anotherdomain.com
+				   alice@mgmt.ANOTHERDOMAIN.com
+				   alice@Mgmt.AnotherDomain.Com)
+	
+	_test_mixed_case "${alices[*]}" "${bobs[*]}" "${aliases[*]}"
+	
+	test_end
+}
+
+
+test_intl_domains() {
+	test_start "intl-domains"
+
+	# local intl alias
+	local alias="alice@bücher.example"
+	local alias_idna="alice@xn--bcher-kva.example"
+
+	# remote intl user / forward-to
+	local intl_person="hans@bücher.example"
+	local intl_person_idna="hans@xn--bcher-kva.example"
+
+	# local users
+	local bob="bob@somedomain.com"
+	local bob_pw="$(generate_password 16)"
+	local mary="mary@somedomain.com"
+	local mary_pw="$(generate_password 16)"
+
+	start_log_capture
+
+	# international domains are not permitted for user accounts
+	if mgmt_create_user "$intl_person" "$bob_pw"; then
+		test_failure "A user account is not permitted to have an international domain"
+		# ensure user is removed as is expected by the remaining tests
+		mgmt_delele_user "$intl_person"
+		delete_user "$intl_person"
+		delete_user "$intl_person_idna"
+	fi
+	
+	# create local users bob and mary
+	mgmt_assert_create_user "$bob" "$bob_pw"
+	mgmt_assert_create_user "$mary" "$mary_pw"
+
+	# create intl alias with local user bob and intl_person in it
+	if mgmt_assert_create_alias_group "$alias" "$bob" "$intl_person"; then
+		# examine LDAP server to verify IDNA-encodings
+		get_attribute "$LDAP_ALIASES_BASE" "(mail=$alias_idna)" "rfc822MailMember"
+		if [ -z "$ATTR_DN" ]; then
+			test_failure "IDNA-encoded alias group not found! created as:$alias expected:$alias_idna"
+		elif [ "$ATTR_VALUE" != "$intl_person_idna" ]; then
+			test_failure "Alias group with user having an international domain was not ecoded properly. added as:$intl_person expected:$intl_person_idna"
+		fi
+	fi
+
+	# re-create intl alias with local user bob only
+	mgmt_assert_create_alias_group "$alias" "$bob"
+	
+	assert_check_logs
+
+	if ! have_test_failures; then
+		# send mail to alias from mary, ensure bob got it
+		record "[Sending to intl alias from mary]"
+		# note PYMAIL does not do idna conversion - it'll throw
+		# "UnicodeEncodeError: 'ascii' codec can't encode character
+		# '\xfc' in position 38".
+		#
+		# we'll have to send to the idna address directly
+		start_log_capture
+		local subject="Mail-In-A-Box test $(generate_uuid)"
+		local output
+		output="$($PYMAIL -subj "$subject" -no-delete -to "$alias_idna" na $PRIVATE_IP $mary "$mary_pw" 2>&1)"
+		assert_python_success $? "$output"
+		output="$($PYMAIL -subj "$subject" -no-send $PRIVATE_IP $bob "$bob_pw" 2>&1)"
+		assert_python_success $? "$output"
+		assert_check_logs
+	fi
+
+	mgmt_assert_delete_alias_group "$alias"
+	mgmt_assert_delete_user "$bob"
+	mgmt_assert_delete_user "$mary"
+
+	test_end
+}
+
+
+suite_start "management-users" mgmt_start
+
+test_mixed_case_users
+test_mixed_case_domains
+test_intl_domains
+
+suite_end mgmt_end
+

tests/test_mail.py

@@ -1,109 +1,204 @@
 #!/usr/bin/env python3
+# -*- indent-tabs-mode: t; tab-width: 4; -*-
+#
 # Tests sending and receiving mail by sending a test message to yourself.
 
 import sys, imaplib, smtplib, uuid, time
 import socket, dns.reversename, dns.resolver
 
-if len(sys.argv) < 3:
-	print("Usage: tests/mail.py hostname emailaddress password")
+
+def usage():
+	print("Usage: test_mail.py [options] hostname login password")
+	print("Send, then delete message")
+	print("  options")
+	print("    -f <email>: use <email> as the MAIL FROM address")
+	print("    -to <email> <pass>: recipient of email and password")
+	print("    -subj <subject>: subject of the message (required with --no-send)")
+	print("    -no-send: don't send, just delete")
+	print("    -no-delete: don't delete, just send")
+	print("    -timeout <seconds>: how long to wait for message")
+	print("");
 	sys.exit(1)
 
-host, emailaddress, pw = sys.argv[1:4]
+def if_unset(a,b):
+	return b if a is None else a
 
-# Attempt to login with IMAP. Our setup uses email addresses
-# as IMAP/SMTP usernames.
-try:
-	M = imaplib.IMAP4_SSL(host)
-	M.login(emailaddress, pw)
-except OSError as e:
-	print("Connection error:", e)
-	sys.exit(1)
-except imaplib.IMAP4.error as e:
-	# any sort of login error
-	e = ", ".join(a.decode("utf8") for a in e.args)
-	print("IMAP error:", e)
-	sys.exit(1)
+# option defaults
+host=None        # smtp server address
+login=None       # smtp server login
+pw=None          # smtp server password
+emailfrom=None   # MAIL FROM address
+emailto=None     # RCPT TO address
+emailto_pw=None  # recipient password for imap login
+send_msg=True    # deliver message
+delete_msg=True  # login to imap and delete message
+wait_timeout=30  # abandon timeout wiating for message delivery
+wait_cycle_sleep=5  # delay between delivery checks
+subject="Mail-in-a-Box Automated Test Message " + uuid.uuid4().hex  # message subject
 
-M.select()
-print("IMAP login is OK.")
+# process command line
+argi=1
+while argi<len(sys.argv):
+	arg=sys.argv[argi]
+	arg_remaining = len(sys.argv) - argi - 1
+	if not arg.startswith('-'):
+		break
+	if (arg=="-f" or arg=="-from") and arg_remaining>0:
+		emailfrom=sys.argv[argi+1]
+		argi+=2
+	elif arg=="-to" and arg_remaining>1:
+		emailto=sys.argv[argi+1]
+		emailto_pw=sys.argv[argi+2]
+		argi+=3
+	elif arg=="-subj" and arg_remaining>1:
+		subject=sys.argv[argi+1]
+		argi+=2
+	elif arg=="-no-send":
+		send_msg=False
+		argi+=1
+	elif arg=="-no-delete":
+		delete_msg=False
+		argi+=1
+	elif arg=="-timeout" and arg_remaining>1:
+		wait_timeout=int(sys.argv[argi+1])
+		argi+=2
+	else:
+		usage()
+		
+        
+if len(sys.argv) - argi != 3: usage()
+host, login, pw = sys.argv[argi:argi+3]
+argi+=3
 
-# Attempt to send a mail to ourself.
-mailsubject = "Mail-in-a-Box Automated Test Message " + uuid.uuid4().hex
-emailto = emailaddress
-msg = """From: {emailaddress}
+emailfrom = if_unset(emailfrom, login)
+emailto = if_unset(emailto, login)
+emailto_pw = if_unset(emailto_pw, pw)
+
+msg = """From: {emailfrom}
 To: {emailto}
 Subject: {subject}
 
 This is a test message. It should be automatically deleted by the test script.""".format(
-	emailaddress=emailaddress,
+	emailfrom=emailfrom,
 	emailto=emailto,
-	subject=mailsubject,
+	subject=subject,
 	)
 
-# Connect to the server on the SMTP submission TLS port.
-server = smtplib.SMTP(host, 587)
-#server.set_debuglevel(1)
-server.starttls()
 
-# Verify that the EHLO name matches the server's reverse DNS.
-ipaddr = socket.gethostbyname(host) # IPv4 only!
-reverse_ip = dns.reversename.from_address(ipaddr) # e.g. "1.0.0.127.in-addr.arpa."
-try:
-	reverse_dns = dns.resolver.query(reverse_ip, 'PTR')[0].target.to_text(omit_final_dot=True) # => hostname
-except dns.resolver.NXDOMAIN:
-	print("Reverse DNS lookup failed for %s. SMTP EHLO name check skipped." % ipaddr)
-	reverse_dns = None
-if reverse_dns is not None:
-	server.ehlo_or_helo_if_needed() # must send EHLO before getting the server's EHLO name
-	helo_name = server.ehlo_resp.decode("utf8").split("\n")[0] # first line is the EHLO name
-	if helo_name != reverse_dns:
-		print("The server's EHLO name does not match its reverse hostname. Check DNS settings.")
-	else:
-		print("SMTP EHLO name (%s) is OK." % helo_name)
 
-# Login and send a test email.
-server.login(emailaddress, pw)
-server.sendmail(emailaddress, [emailto], msg)
-server.quit()
-print("SMTP submission is OK.")
+def imap_login(host, login, pw):
+	# Attempt to login with IMAP. Our setup uses email addresses
+	# as IMAP/SMTP usernames.
+	try:
+		M = imaplib.IMAP4_SSL(host)
+		M.login(login, pw)
+	except OSError as e:
+		print("Connection error:", e)
+		sys.exit(1)
+	except imaplib.IMAP4.error as e:
+		# any sort of login error
+		e = ", ".join(a.decode("utf8") for a in e.args)
+		print("IMAP error:", e)
+		sys.exit(1)
 
-while True:
-	# Wait so the message can propagate to the inbox.
-	time.sleep(10)
+	M.select()
+	print("IMAP login is OK.")
+	return M
 
+
+def imap_search_for(M, subject):
 	# Read the subject lines of all of the emails in the inbox
-	# to find our test message, and then delete it.
-	found = False
+	# to find our test message, then return the number
 	typ, data = M.search(None, 'ALL')
 	for num in data[0].split():
 		typ, data = M.fetch(num, '(BODY[HEADER.FIELDS (SUBJECT)])')
 		imapsubjectline = data[0][1].strip().decode("utf8")
-		if imapsubjectline == "Subject: " + mailsubject:
-			# We found our test message.
-			found = True
+		if imapsubjectline == "Subject: " + subject:
+			return num
+	return None
 
-			# To test DKIM, download the whole mssage body. Unfortunately,
-			# pydkim doesn't actually work.
-			# You must 'sudo apt-get install python3-dkim python3-dnspython' first.
-			#typ, msgdata = M.fetch(num, '(RFC822)')
-			#msg = msgdata[0][1]
-			#if dkim.verify(msg):
-			#	print("DKIM signature on the test message is OK (verified).")
-			#else:
-			#	print("DKIM signature on the test message failed verification.")
 
+def imap_test_dkim(M, num):
+	# To test DKIM, download the whole mssage body. Unfortunately,
+	# pydkim doesn't actually work.
+	# You must 'sudo apt-get install python3-dkim python3-dnspython' first.
+	#typ, msgdata = M.fetch(num, '(RFC822)')
+	#msg = msgdata[0][1]
+	#if dkim.verify(msg):
+	#	print("DKIM signature on the test message is OK (verified).")
+	#else:
+	#	print("DKIM signature on the test message failed verification.")
+	pass
+
+
+def smtp_login(host, login, pw):
+	# Connect to the server on the SMTP submission TLS port.
+	server = smtplib.SMTP(host, 587)
+	#server.set_debuglevel(1)
+	server.starttls()
+
+	# Verify that the EHLO name matches the server's reverse DNS.
+	ipaddr = socket.gethostbyname(host) # IPv4 only!
+	reverse_ip = dns.reversename.from_address(ipaddr) # e.g. "1.0.0.127.in-addr.arpa."
+	try:
+		reverse_dns = dns.resolver.query(reverse_ip, 'PTR')[0].target.to_text(omit_final_dot=True) # => hostname
+	except dns.resolver.NXDOMAIN:
+		print("Reverse DNS lookup failed for %s. SMTP EHLO name check skipped." % ipaddr)
+		reverse_dns = None
+	if reverse_dns is not None:
+		server.ehlo_or_helo_if_needed() # must send EHLO before getting the server's EHLO name
+		helo_name = server.ehlo_resp.decode("utf8").split("\n")[0] # first line is the EHLO name
+		if helo_name != reverse_dns:
+			print("The server's EHLO name does not match its reverse hostname. Check DNS settings.")
+		else:
+			print("SMTP EHLO name (%s) is OK." % helo_name)
+
+	# Login and send a test email.
+	if login is not None and login != "":
+		server.login(login, pw)
+	return server
+
+
+
+
+if send_msg:
+	# Attempt to send a mail.
+	server = smtp_login(host, login, pw)
+	server.sendmail(emailfrom, [emailto], msg)
+	server.quit()
+	print("SMTP submission is OK.")
+
+
+if delete_msg:
+	# Wait for mail and delete it.
+	M = imap_login(host, emailto, emailto_pw)
+
+	start_time = time.time()
+	found = False
+	if send_msg:
+		# Wait so the message can propagate to the inbox.
+		time.sleep(wait_cycle_sleep / 2)
+
+	while time.time() - start_time < wait_timeout:
+		num = imap_search_for(M, subject)
+		if num is not None:
 			# Delete the test message.
+			found = True
+			imap_test_dkim(M, num)
 			M.store(num, '+FLAGS', '\\Deleted')
 			M.expunge()
-
+			print("Message %s deleted successfully." % num)
 			break
+		
+		print("Test message not present in the inbox yet...")
+		time.sleep(wait_cycle_sleep)
+		
+	M.close()
+	M.logout()
+	
+	if not found:
+		raise TimeoutError("Timeout waiting for message")
 
-	if found:
-		break
+if send_msg and delete_msg:
+	print("Test message sent & received successfully.")
 
-	print("Test message not present in the inbox yet...")
-
-M.close()
-M.logout()
-
-print("Test message sent & received successfully.")