Issue #1340 - LDAP backend for accounts

This commit will:

1. Change the user account database from sqlite to OpenLDAP
2. Add policyd-spf to postfix for SPF validation
3. Add a test runner with some automated test suites

Notes:

User account password hashes are preserved.

There is a new Roundcube contact list called "Directory" that lists the users in LDAP (MiaB users), similar to what Google Suite does.

Users can still change their password in Roundcube.

OpenLDAP is configured with TLS, but all remote access is blocked by firewall rules. Manual changes are required to open it for remote access (eg. "ufw allow proto tcp from <HOST> to any port ldaps").

The test runner is started by executing tests/runner.sh. Be aware that it will make changes to your system, including adding new users, domains, mailboxes, start/stop services, etc. It is highly unadvised to run it on a production system!

The LDAP schema that supports mail delivery with postfix and dovecot is located in conf/postfix.schema. This file is copied verbatim from the LdapAdmin project (GPL, ldapadmin.org). Instead of including the file in git, it could be referenced by URL and downloaded by the setup script if GPL is an issue or apply for a PEN from IANA.

Mangement console and other services should not appear or behave any differently than before.

management/mailconfig.py

@@ -1,4 +1,5 @@
 #!/usr/local/lib/mailinabox/env/bin/python
+# -*- indent-tabs-mode: t; tab-width: 4; python-indent-offset: 4; -*-
 
 # NOTE:
 # This script is run both using the system-wide Python 3
@@ -9,11 +10,51 @@
 # Python 3 in setup/questions.sh to validate the email
 # address entered by the user.
 
-import subprocess, shutil, os, sqlite3, re
-import utils
+import subprocess, shutil, os, sqlite3, re, ldap3, uuid
+import utils, backend
 from email_validator import validate_email as validate_email_, EmailNotValidError
 import idna
 
+
+#
+# LDAP notes:
+#
+#    Users have an objectClass of mailUser with a mail and maildrop
+#    attribute for the email address. For historical reasons, the
+#    management interface only permits lowercase email addresses.
+#
+#    In the current implementation, both attributes will have the same
+#    lowercase value, but it's not a requirement of the underlying
+#    postfix and dovecot configurations that it be this way. Postfix
+#    and dovecot use the mail attribute to find the user and maildrop
+#    is where the mail is delivered. We use maildrop in the management
+#    interface because that's the address that will always map
+#    directly to the filesystem for archived users.
+#
+#    Email addresses and domain comparisons performed by the LDAP
+#    server are not case sensitive because their respective schemas
+#    define a case-insensitive comparison for those attributes.
+#
+#    User privileges are maintained in the mailaccess attribute of
+#    users.
+#
+#    Aliases and permitted-senders are separate entities in the LDAP
+#    database, but both are of objectClass mailGroup with a
+#    single-valued mail attribute. Alias addresses are forced to
+#    lowercase, again for historical reasons.
+#
+#    All alias and permitted-sender email addresses in the database
+#    are IDNA encoded. International domains for users is not
+#    supported or permitted.
+#
+#    Domains that are handled by this mail server are maintained
+#    on-the-fly as users are added and deleted. They have an
+#    objectClass of domain with attribute dc.
+#
+#    LDAP "records" in this code are dictionaries containing the
+#    attributes and distinguished name of the entry.
+#
+
 def validate_email(email, mode=None):
 	# Checks that an email address is syntactically valid. Returns True/False.
 	# Until Postfix supports SMTPUTF8, an email address may contain ASCII
@@ -91,20 +132,90 @@ def is_dcv_address(email):
 			return True
 	return False
 
-def open_database(env, with_connection=False):
-	conn = sqlite3.connect(env["STORAGE_ROOT"] + "/mail/users.sqlite")
-	if not with_connection:
-		return conn.cursor()
+def open_database(env):
+		return backend.connect(env)
+
+def find_mail_user(env, email, attributes=None, conn=None):
+	# Find the user with the given email address and return the ldap
+	# record for it.
+	#
+	# email is the users email address
+	# attributes are a list of attributes to return eg ["mail","maildrop"]
+	# conn is a ldap database connection, if not specified a new one
+	# is established	
+	#
+	# The ldap record for the user is returned or None if not found.
+	if not conn: conn = open_database(env)
+	id=conn.search(env.LDAP_USERS_BASE,
+		       "(&(objectClass=mailUser)(maildrop=%s))" % email,
+		       attributes=attributes)
+	response = conn.wait(id)
+	if response.count() > 1:
+		dns = [ rec['dn'] for rec in response ]
+		raise LookupError("Detected more than one user with the same email address (%s): %s" % (email, ";".join(dns)))
 	else:
-		return conn, conn.cursor()
+		return response.next()
+	
+def find_mail_alias(env, email, attributes=None, conn=None):
+	# Find the alias with the given address and return the ldap
+	# records for it and the associated permitted senders (if one).
+	#
+	# email is the alias address. It must be IDNA encoded.
+	#
+	# attributes are a list of attributes to return, eg
+	# ["member","rfc822MailMember"]
+	#
+	# conn is a ldap database connection, if not specified a new one
+	# is established.
+	#
+	# A tuple having the two ldap records for the alias and it's
+	# permitted senders (alias, permitted_senders) is returned. If
+	# either is not found, the corresponding tuple value will be None.
+	# 
+	if not conn: conn = open_database(env)
+	# get alias
+	id=conn.search(env.LDAP_ALIASES_BASE,
+		       "(&(objectClass=mailGroup)(mail=%s))" % email,
+		       attributes=attributes)
+	response = conn.wait(id)
+	if response.count() > 1:
+		dns = [ rec['dn'] for rec in response ]
+		raise LookupError("Detected more than one alias with the same email address (%s): %s" % (email, ";".join(dns)))
+	alias = response.next()
 
-def get_mail_users(env):
-	# Returns a flat, sorted list of all user accounts.
+	# get permitted senders for alias
+	id=conn.search(env.LDAP_PERMITTED_SENDERS_BASE,
+		       "(&(objectClass=mailGroup)(mail=%s))" % email,
+		       attributes=attributes)
+	response = conn.wait(id)
+	if response.count() > 1:
+		dns = [ rec['dn'] for rec in response ]
+		raise LookupError("Detected more than one permitted senders group with the same email address (%s): %s" % (email, ";".join(dns)))
+	permitted_senders = response.next()
+	return (alias, permitted_senders)
+	
+
+def get_mail_users(env, as_map=False):
+	# When `as_map` is False, this function returns a flat, sorted
+	# array of all user accounts. If True, it returns a dict where key
+	# is the user and value is a dict having, dn, maildrop and
+	# mail addresses
 	c = open_database(env)
-	c.execute('SELECT email FROM users')
-	users = [ row[0] for row in c.fetchall() ]
-	return utils.sort_email_addresses(users, env)
+	pager = c.paged_search(env.LDAP_USERS_BASE, "(objectClass=mailUser)", attributes=['maildrop','mail'])
+	if as_map:
+		users = {}
+		for rec in pager:
+			users[rec['maildrop'][0]] = {
+				"dn":   rec['dn'],
+				"mail": rec['mail'],
+				"maildrop": rec['maildrop'][0]
+			}
+		return users
+	else:
+		users = [ rec['maildrop'][0] for rec in pager ]
+		return utils.sort_email_addresses(users, env)
 
+	
 def get_mail_users_ex(env, with_archived=False):
 	# Returns a complex data structure of all user accounts, optionally
 	# including archived (status="inactive") accounts.
@@ -128,13 +239,15 @@ def get_mail_users_ex(env, with_archived=False):
 	users = []
 	active_accounts = set()
 	c = open_database(env)
-	c.execute('SELECT email, privileges FROM users')
-	for email, privileges in c.fetchall():
-		active_accounts.add(email)
+	response = c.wait( c.search(env.LDAP_USERS_BASE, "(objectClass=mailUser)", attributes=['maildrop','mailaccess']) )
 
+	for rec in response:
+		email = rec['maildrop'][0]
+		privileges = rec['mailaccess']
+		active_accounts.add(email)
 		user = {
 			"email": email,
-			"privileges": parse_privs(privileges),
+			"privileges": privileges,
 			"status": "active",
 		}
 		users.append(user)
@@ -179,21 +292,82 @@ def get_mail_users_ex(env, with_archived=False):
 def get_admins(env):
 	# Returns a set of users with admin privileges.
 	users = set()
-	for domain in get_mail_users_ex(env):
-		for user in domain["users"]:
-			if "admin" in user["privileges"]:
-				users.add(user["email"])
+	c = open_database(env)
+	response = c.wait( c.search(env.LDAP_USERS_BASE, "(&(objectClass=mailUser)(mailaccess=admin))", attributes=['maildrop']) )
+	for rec in response:
+		users.add(rec['maildrop'][0])
 	return users
 
-def get_mail_aliases(env):
-	# Returns a sorted list of tuples of (address, forward-tos, permitted-senders).
-	c = open_database(env)
-	c.execute('SELECT source, destination, permitted_senders FROM aliases')
-	aliases = { row[0]: row for row in c.fetchall() } # make dict
 
-	# put in a canonical order: sort by domain, then by email address lexicographically
-	aliases = [ aliases[address] for address in utils.sort_email_addresses(aliases.keys(), env) ]
-	return aliases
+def get_mail_aliases(env, as_map=False):
+	# Retrieve all mail aliases.
+	#
+	# If as_map is False, the function returns a sorted array of tuples:
+	#
+	#   (address(lowercase), forward-tos{string,csv}, permitted-senders{string,csv})
+	#
+	# If as-map is True, it returns a dict whose keys are
+	# address(lowercase) and whose values are:
+	#
+	#    { dn: {string},
+	#      mail: {string}
+	#      forward_tos: {array of string},
+	#      permited_senders: {array of string}
+	#    }
+	#
+	c = open_database(env)
+	# get all permitted senders
+	pager = c.paged_search(env.LDAP_PERMITTED_SENDERS_BASE, "(objectClass=mailGroup)", attributes=["mail", "member"])
+	
+	# make a dict of permitted senders, key=mail(lowercase) value=members
+	permitted_senders = { rec["mail"][0].lower(): rec["member"] for rec in pager }
+	
+	# get all alias groups
+	pager = c.paged_search(env.LDAP_ALIASES_BASE, "(objectClass=mailGroup)", attributes=['mail','member','rfc822MailMember'])
+	
+	# make a dict of aliases
+	# key=email(lowercase), value=(email, forward-tos, permitted-senders).
+	aliases = {}
+	for alias in pager:
+		alias_email = alias['mail'][0]
+		alias_email_lc = alias_email.lower()
+		
+		# chase down each member's email address, because a member is a dn
+		forward_tos = []
+		for fwd_to in c.chase_members(alias['member'], 'mail', env):
+			forward_tos.append(fwd_to[0])
+			
+		for fwd_to in alias['rfc822MailMember']:
+			forward_tos.append(fwd_to)
+		
+		# chase down permitted senders' email addresses
+		allowed_senders = []
+		if alias_email_lc in permitted_senders:
+			members = permitted_senders[alias_email_lc]
+			for mail_list in c.chase_members(members, 'mail', env):
+				for mail in mail_list:
+					allowed_senders.append(mail)
+
+		aliases[alias_email_lc] = {
+			"dn": alias["dn"],
+			"mail": alias_email,
+			"forward_tos": forward_tos,
+			"permitted_senders": allowed_senders
+		}
+
+	if not as_map:
+		# put in a canonical order: sort by domain, then by email address lexicographically
+		list = []
+		for address in utils.sort_email_addresses(aliases.keys(), env):
+			alias = aliases[address]
+			xft = ",".join(alias["forward_tos"])
+			xas = ",".join(alias["permitted_senders"])
+			list.append( (address, xft, xas) )
+		return list
+	
+	else:
+		return aliases
+			
 
 def get_mail_aliases_ex(env):
 	# Returns a complex data structure of all mail aliases, similar
@@ -258,15 +432,174 @@ def get_domain(emailaddr, as_unicode=True):
 			pass
 	return ret
 
-def get_mail_domains(env, filter_aliases=lambda alias : True):
-	# Returns the domain names (IDNA-encoded) of all of the email addresses
-	# configured on the system.
-	return set(
-		   [get_domain(login, as_unicode=False) for login in get_mail_users(env)]
-		 + [get_domain(address, as_unicode=False) for address, *_ in get_mail_aliases(env) if filter_aliases(address) ]
-		 )
+def get_mail_domains(env, as_map=False, filter_aliases=None, category=None):
+	# Retrieves all domains, IDNA-encoded, we accept mail for.
+	#
+	# If as_map is False, the function returns the lowercase domain
+	# names (IDNA-encoded) as an array.
+	#
+	# If as_map is True, it returns a dict whose keys are
+	# domain(idna,lowercase) and whose values are:
+	#
+	#   { dn:{string}, domain:{string(idna)} }
+	#
+	# filter_aliases is an inclusion filter - a True return value from
+	# this lambda function indicates an alias to include.
+	#
+	# With filter_aliases, there has to be at least one user or
+	# filtered (included) alias in the domain for the domain to be
+	# part of the returned set. If None, all domains are returned.
+	#
+	# category is another type of filter. Set to a string value to
+	# return only those domains of that category. ie. the
+	# "businessCategory" attribute of the domain must include this
+	# category.
+	#
+	conn = open_database(env)
+	filter = "(&(objectClass=domain)(businessCategory=mail))"
+	if category:
+		filter = "(&(objectClass=domain)(businessCategory=%s))" % category
+	id = conn.search(env.LDAP_DOMAINS_BASE, filter, attributes="dc")
+	response = conn.wait(id)
+	filter_candidates=[]
+	domains=None
+	if as_map:
+		domains = {}
+		for rec in response:
+			domains[ rec["dc"][0].lower() ] = {
+				"dn": rec["dn"],
+				"domain": rec["dc"][0]
+			}
+			if filter_aliases: filter_candidates.append(rec['dc'][0].lower())
+	else:
+		domains = set([ rec["dc"][0].lower() for rec in response ])
+		if filter_aliases: filter_candidates += domains
+	
+	for candidate in filter_candidates:
+		# with the filter, there has to be at least one user or
+		# filtered (included) alias in the domain for the domain to be
+		# part of the returned set
+
+		# any users ?
+		response = conn.wait( conn.search(env.LDAP_USERS_BASE, "(&(objectClass=mailUser)(mail=*@%s))" % candidate, size_limit=1) )
+		if response.next():
+			# yes, that domain needs to be in the returned set
+			continue
+
+		# any filtered aliases ?
+		pager = conn.paged_search(
+			env.LDAP_ALIASES_BASE,
+			"(&(objectClass=mailGroup)(mail=*@%s))" % candidate,
+			attributes=['mail'])
+
+		remove = True
+		for rec in pager:
+			if filter_aliases(rec['mail'][0]):
+				remove = False
+				pager.abandon()
+				break
+
+		if remove:
+			if as_map: del domains[candidate]
+			else: domains.remove(candidate)
+			
+	return domains
+
+
+def add_mail_domain(env, domain, validate=True):
+	# Create domain entry indicating that we are handling
+	# mail for that domain.
+	#
+	# We only care about domains for users, not for aliases.
+	#
+	# domain: IDNA encoded domain name.
+	# validate: If True, ensure there is at least one user on the
+	# system using that domain.
+	#
+	# returns True if added, False if it already exists or fails
+	# validation.
+	
+	conn = open_database(env)
+	if validate:
+		# check to ensure there is at least one user with that domain
+		id = conn.search(env.LDAP_USERS_BASE,
+						 "(&(objectClass=mailUser)(mail=*@%s))" % domain,
+						 size_limit=1)
+		if conn.wait(id).count() == 0:
+			# no mail users are using that domain!
+			return False
+		
+	dn = 'dc=%s,%s' % (domain, env.LDAP_DOMAINS_BASE)
+	try:
+		response = conn.wait( conn.add(dn, [ 'domain' ], {
+			"businessCategory": "mail"
+		}) )
+		return True
+	except ldap3.core.exceptions.LDAPEntryAlreadyExistsResult:
+		try:
+			# add 'mail' as a businessCategory
+			changes = {	"businessCategory": [(ldap3.MODIFY_ADD, ['mail'])] }
+			response = conn.wait ( conn.modify(dn, changes) )
+		except ldap3.core.exceptions.LDAPAttributeOrValueExistsResult:
+			pass
+		return False
+
+	
+def remove_mail_domain(env, domain, validate=True):
+	# Remove the specified domain from the list of domains that we
+	# handle mail for. The domain must be IDNA encoded.
+	#
+	# If validate is True, ensure there are no valid users on the system
+	# currently using the domain.
+	#
+	# Returns True if removed or does not exist, False if the domain
+	# fails validation.
+	conn = open_database(env)
+	if validate:
+		# check to ensure no users are using the domain
+		id = conn.search(env.LDAP_USERS_BASE,
+						 "(&(objectClass=mailUser)(mail=*@%s))" % domain,
+						 size_limit=1)
+		if conn.wait(id).count() > 0:
+			# there is one or more mailUser's with that domain
+			return False
+		
+	id = conn.search(env.LDAP_DOMAINS_BASE,
+					 "(&(objectClass=domain)(dc=%s))" % domain,
+					 attributes=['businessCategory'])
+	
+	existing = conn.wait(id).next()
+	if existing is None:
+		# the domain doesn't exist!
+		return True
+
+	newvals=existing['businessCategory'].copy()
+	if 'mail' in newvals:
+		newvals.remove('mail')
+	else:
+		# we only remove mail-related entries
+		return False
+
+	if len(newvals)==0:
+		conn.wait ( conn.delete(existing['dn']) )
+	else:
+		conn.wait ( conn.modify_record(existing, {'businessCategory', newvals}))
+	return True
+
 
 def add_mail_user(email, pw, privs, env):
+	# Add a new mail user.
+	#
+	# email: the new user's email address
+	# pw: the new user's password
+	# privs: either an array of privilege strings, or a newline
+	# separated string of privilege names
+	#
+	# If an error occurs, the function returns a tuple of (message,
+	# http-status).
+	#
+	# If successful, the string "OK" is returned.
+	
 	# validate email
 	if email.strip() == "":
 		return ("No email address provided.", 400)
@@ -284,73 +617,131 @@ def add_mail_user(email, pw, privs, env):
 	validate_password(pw)
 
 	# validate privileges
-	if privs is None or privs.strip() == "":
-		privs = []
-	else:
-		privs = privs.split("\n")
-		for p in privs:
-			validation = validate_privilege(p)
-			if validation: return validation
+	privs = []
+	if privs is not None and type(privs) is str and privs.strip() != "":
+		privs = parse_privs(privs)
+	for p in privs:
+		validation = validate_privilege(p)
+		if validation: return validation
 
 	# get the database
-	conn, c = open_database(env, with_connection=True)
+	conn = open_database(env)
 
-	# hash the password
-	pw = hash_password(pw)
+	# ensure another user doesn't have that address
+	id=conn.search(env.LDAP_USERS_BASE, "(&(objectClass=mailUser)(|(mail=%s)(maildrop=%s)))" % (email, email))
+	if conn.wait(id).count() > 0:
+		return ("User alreay exists.", 400)
+
+	# ensure an alias doesn't have that address
+	id=conn.search(env.LDAP_ALIASES_BASE, "(&(objectClass=mailGroup)(mail=%s))" % email)
+	if conn.wait(id).count() > 0:
+		return ("An alias exists with that address.", 400)
+	
+	# Generate a unique id for uid
+	uid = '%s' % uuid.uuid4()
+
+	# choose a common name and surname (required attributes)
+	cn = email.split("@")[0].replace('.',' ').replace('_',' ')
+	sn = cn[cn.find(' ')+1:]
+	
+	# compile user's attributes
+	# for historical reasons, make the email address lowercase
+	attrs = {
+		"mail" : email,
+		"maildrop" : email.lower(),
+		"uid" : uid,
+		"mailaccess": privs,
+		"cn": cn,
+		"sn": sn,
+		"shadowLastChange": backend.get_shadowLastChanged()
+	}
 
 	# add the user to the database
-	try:
-		c.execute("INSERT INTO users (email, password, privileges) VALUES (?, ?, ?)",
-			(email, pw, "\n".join(privs)))
-	except sqlite3.IntegrityError:
-		return ("User already exists.", 400)
+	dn = 'uid=%s,%s' % (uid, env.LDAP_USERS_BASE)
+	id=conn.add(dn, [ 'inetOrgPerson','mailUser','shadowAccount' ], attrs);
+	conn.wait(id)
 
-	# write databasebefore next step
-	conn.commit()
+	# set the password - the ldap server will hash it
+	conn.extend.standard.modify_password(user=dn, new_password=pw)
 
+	# tell postfix the domain is local, if needed
+	add_mail_domain(env, get_domain(email, as_unicode=False), validate=False)
+
+	# convert alias's rfc822MailMember to member
+	convert_rfc822MailMember(env, conn, dn, email)
+	
 	# Update things in case any new domains are added.
 	return kick(env, "mail user added")
 
 def set_mail_password(email, pw, env):
-	# validate that password is acceptable
+	# validate that the password is acceptable
 	validate_password(pw)
 
-	# hash the password
-	pw = hash_password(pw)
-
-	# update the database
-	conn, c = open_database(env, with_connection=True)
-	c.execute("UPDATE users SET password=? WHERE email=?", (pw, email))
-	if c.rowcount != 1:
+	# find the user				  
+	conn = open_database(env)
+	user = find_mail_user(env, email, ['shadowLastChange'], conn)
+	if user is None:
 		return ("That's not a user (%s)." % email, 400)
-	conn.commit()
+
+	# update the database - the ldap server will hash the password
+	conn.extend.standard.modify_password(user=user['dn'], new_password=pw)
+
+	# update shadowLastChange
+	conn.modify_record(user, {'shadowLastChange': backend.get_shadowLastChanged()})
+
 	return "OK"
 
-def hash_password(pw):
-	# Turn the plain password into a Dovecot-format hashed password, meaning
-	# something like "{SCHEME}hashedpassworddata".
-	# http://wiki2.dovecot.org/Authentication/PasswordSchemes
-	return utils.shell('check_output', ["/usr/bin/doveadm", "pw", "-s", "SHA512-CRYPT", "-p", pw]).strip()
+def validate_login(email, pw, env):
+	# Validate that `email` exists and has password `pw`.
+	# Returns True if valid, or False if invalid.
+	user = find_mail_user(env, email)
+	if user is None:
+		raise ValueError("That's not a user (%s)" % email)
+	try:
+		# connect as that user to validate the login
+		server = backend.get_ldap_server(env)
+		conn = ldap3.Connection(
+			server,
+			user=user['dn'],
+			password=pw,
+			raise_exceptions=True)
+		conn.bind()
+		conn.unbind()
+		return True
+	except ldap3.core.exceptions.LDAPInvalidCredentialsResult:
+		return False
+
 
 def get_mail_password(email, env):
-	# Gets the hashed password for a user. Passwords are stored in Dovecot's
-	# password format, with a prefixed scheme.
-	# http://wiki2.dovecot.org/Authentication/PasswordSchemes
-	# update the database
-	c = open_database(env)
-	c.execute('SELECT password FROM users WHERE email=?', (email,))
-	rows = c.fetchall()
-	if len(rows) != 1:
+	# Gets the hashed passwords for a user. In ldap, userPassword is
+	# multi-valued and each value can have different hash. This
+	# function returns all hashes as an array.
+	user = find_mail_user(env, email, attributes=["userPassword"])
+	if user is None:
 		raise ValueError("That's not a user (%s)." % email)
-	return rows[0][0]
+	if len(user['userPassword'])==0:
+		raise ValueError("The user has no password (%s)" % email)
+	return user['userPassword']
+
 
 def remove_mail_user(email, env):
-	# remove
-	conn, c = open_database(env, with_connection=True)
-	c.execute("DELETE FROM users WHERE email=?", (email,))
-	if c.rowcount != 1:
+	# Remove the user as a valid user of the system.
+	# If an error occurs, the function returns a tuple of (message,
+	# http-status).
+	#
+	# If successful, the string "OK" is returned.
+	conn = open_database(env)
+	
+	# find the user
+	user = find_mail_user(env, email, conn=conn)
+	if user is None:
 		return ("That's not a user (%s)." % email, 400)
-	conn.commit()
+	
+	# delete the user
+	conn.wait( conn.delete(user['dn']) )
+
+	# remove as a handled domain, if needed
+	remove_mail_domain(env, get_domain(email, as_unicode=False))
 
 	# Update things in case any domains are removed.
 	return kick(env, "mail user removed")
@@ -359,14 +750,19 @@ def parse_privs(value):
 	return [p for p in value.split("\n") if p.strip() != ""]
 
 def get_mail_user_privileges(email, env, empty_on_error=False):
-	# get privs
+	# Get an array of privileges held by the specified user.
 	c = open_database(env)
-	c.execute('SELECT privileges FROM users WHERE email=?', (email,))
-	rows = c.fetchall()
-	if len(rows) != 1:
+	try:
+		user = find_mail_user(env, email, ['mailaccess'], c)
+	except LookupError as e:
+		if empty_on_error: return []
+		raise e
+	
+	if user is None:
 		if empty_on_error: return []
 		return ("That's not a user (%s)." % email, 400)
-	return parse_privs(rows[0][0])
+	
+	return user['mailaccess']
 
 def validate_privilege(priv):
 	if "\n" in priv or priv.strip() == "":
@@ -374,38 +770,93 @@ def validate_privilege(priv):
 	return None
 
 def add_remove_mail_user_privilege(email, priv, action, env):
+	# Add or remove a privilege from a user.
+	# priv: the name of the privilege to add or remove
+	# action: "add" to add the privilege, or "remove" to remove it
+	# email: the user
+	#
+	# If an error occurs, the function returns a tuple of (message,
+	# http-status).
+	#
+	# If successful, the string "OK" is returned.
+	
 	# validate
 	validation = validate_privilege(priv)
 	if validation: return validation
 
 	# get existing privs, but may fail
-	privs = get_mail_user_privileges(email, env)
-	if isinstance(privs, tuple): return privs # error
+	user = find_mail_user(env, email, attributes=['mailaccess'])
+	if user is None:
+		return ("That's not a user (%s)." % email, 400)
+		
+	privs = user['mailaccess'].copy()
 
 	# update privs set
+	changed = False
 	if action == "add":
 		if priv not in privs:
 			privs.append(priv)
+			changed = True
+			
 	elif action == "remove":
-		privs = [p for p in privs if p != priv]
+		if priv in privs:
+			privs.remove(priv)
+			changed = True
 	else:
 		return ("Invalid action.", 400)
 
 	# commit to database
-	conn, c = open_database(env, with_connection=True)
-	c.execute("UPDATE users SET privileges=? WHERE email=?", ("\n".join(privs), email))
-	if c.rowcount != 1:
-		return ("Something went wrong.", 400)
-	conn.commit()
-
+	if changed:
+		conn = open_database(env)
+		conn.modify_record( user, {'mailaccess': privs} )
+		
 	return "OK"
 
+
+
+def convert_rfc822MailMember(env, conn, dn, mail):
+	# if a newly added alias or user exists as an rfc822MailMember,
+	# convert it to a member dn
+	# the new alias or user is specified by arguments mail and dn.
+	# mail is the new alias or user's email address
+	# dn is the new alias or user's distinguished name
+	# conn is an existing ldap database connection
+	id=conn.search(env.LDAP_ALIASES_BASE,
+				   "(&(objectClass=mailGroup)(rfc822MailMember=%s))" % mail,
+				   attributes=[ 'member','rfc822MailMember' ])
+	response = conn.wait(id)
+	for rec in response:
+		# remove mail from rfc822MailMember
+		changes={ "rfc822MailMember": [(ldap3.MODIFY_DELETE, [mail])] }
+		conn.wait( conn.modify(rec['dn'], changes) )
+
+		# add dn to member
+		rec['member'].append(dn)
+		changes={ "member": [(ldap3.MODIFY_ADD, rec['member'])] }
+		try:
+			conn.wait( conn.modify(rec['dn'], changes) )
+		except ldap3.core.exceptions.LDAPAttributeOrValueExistsResult:
+			pass
+
+		
 def add_mail_alias(address, forwards_to, permitted_senders, env, update_if_exists=False, do_kick=True):
+	# Add a new alias group with permitted senders.
+	#
+	# address: the email address of the alias
+	# forwards_to: a string of newline and comma separated email address
+	# where mail is delivered
+	# permitted_senders: a string of newline and comma separated email addresses of local users that are permitted to MAIL FROM the alias.
+	# update_if_exists: if False and the alias exists fail, otherwise update the existing alias with the new values
+	#
+	# If an error occurs, the function returns a tuple of (message,
+	# http-status).
+	#
+	# If successful, the string "OK" is returned.
+
 	# convert Unicode domain to IDNA
 	address = sanitize_idn_email_address(address)
 
-	# Our database is case sensitive (oops), which affects mail delivery
-	# (Postfix always queries in lowercase?), so force lowercase.
+	# for historical reasons, force the address to lowercase
 	address = address.lower()
 
 	# validate address
@@ -415,8 +866,11 @@ def add_mail_alias(address, forwards_to, permitted_senders, env, update_if_exist
 	if not validate_email(address, mode='alias'):
 		return ("Invalid email address (%s)." % address, 400)
 
+	# retrieve all logins as a map ( email.lower() => {mail,dn} )
+	valid_logins = get_mail_users( env, as_map=True )
+	
 	# validate forwards_to
-	validated_forwards_to = []
+	validated_forwards_to = [ ]
 	forwards_to = forwards_to.strip()
 
 	# extra checks for email addresses used in domain control validation
@@ -451,9 +905,8 @@ def add_mail_alias(address, forwards_to, permitted_senders, env, update_if_exist
 				validated_forwards_to.append(email)
 
 	# validate permitted_senders
-	valid_logins = get_mail_users(env)
-	validated_permitted_senders = []
-	permitted_senders = permitted_senders.strip()
+	validated_permitted_senders = [ ]  # list of dns
+	permitted_senders = permitted_senders.strip( )
 
 	# Parse comma and \n-separated sender logins & validate. The permitted_senders must be
 	# valid usernames.
@@ -461,51 +914,111 @@ def add_mail_alias(address, forwards_to, permitted_senders, env, update_if_exist
 		for login in line.split(","):
 			login = login.strip()
 			if login == "": continue
-			if login not in valid_logins:
+			if login.lower() not in valid_logins:
 				return ("Invalid permitted sender: %s is not a user on this system." % login, 400)
-			validated_permitted_senders.append(login)
+			validated_permitted_senders.append(valid_logins[login.lower()]['dn'])
 
 	# Make sure the alias has either a forwards_to or a permitted_sender.
 	if len(validated_forwards_to) + len(validated_permitted_senders) == 0:
 		return ("The alias must either forward to an address or have a permitted sender.", 400)
 
+
+	# break validated_forwards_to into 'local' where an email
+	# address is a local user, or 'remote' where the email doesn't
+	# exist on this system
+
+	vfwd_tos_local = []   # list of dn's
+	vfwd_tos_remote = []  # list of email
+	for fwd_to in validated_forwards_to:
+		fwd_to_lc = fwd_to.lower()
+		if fwd_to_lc in valid_logins:
+			dn = valid_logins[fwd_to_lc]['dn']
+			vfwd_tos_local.append(dn)
+		else:
+			vfwd_tos_remote.append(fwd_to)
+			
 	# save to db
 
-	forwards_to = ",".join(validated_forwards_to)
-
-	if len(validated_permitted_senders) == 0:
-		permitted_senders = None
+	conn = open_database(env)
+	existing_alias, existing_permitted_senders = find_mail_alias(env, address, ['member','rfc822MailMember'], conn)
+	if existing_alias and not update_if_exists:
+		return ("Alias already exists (%s)." % address, 400)
+	
+	cn="%s" % uuid.uuid4()
+	dn="cn=%s,%s" % (cn, env.LDAP_ALIASES_BASE)
+	if address.startswith('@') and \
+	   len(validated_forwards_to)==1 and \
+	   validated_forwards_to[0].startswith('@'):
+		description = "Domain alias %s->%s" % (address, validated_forwards_to[0])
+	elif address.startswith('@'):
+		description = "Catch-all for %s" % address
+	else:
+		description ="Mail group %s" % address
+	attrs = {
+		"mail": address,
+		"description": description,
+		"member": vfwd_tos_local,
+		"rfc822MailMember": vfwd_tos_remote
+	}
+	
+	op = conn.add_or_modify(dn, existing_alias,
+			['member', 'rfc822MailMember' ],
+			['mailGroup'],
+			attrs)
+	if op == 'modify':
+		return_status = "alias updated"
 	else:
-		permitted_senders = ",".join(validated_permitted_senders)
-
-	conn, c = open_database(env, with_connection=True)
-	try:
-		c.execute("INSERT INTO aliases (source, destination, permitted_senders) VALUES (?, ?, ?)", (address, forwards_to, permitted_senders))
 		return_status = "alias added"
-	except sqlite3.IntegrityError:
-		if not update_if_exists:
-			return ("Alias already exists (%s)." % address, 400)
-		else:
-			c.execute("UPDATE aliases SET destination = ?, permitted_senders = ? WHERE source = ?", (forwards_to, permitted_senders, address))
-			return_status = "alias updated"
-
-	conn.commit()
+		convert_rfc822MailMember(env, conn, dn, address)
+		
+	# add or modify permitted-senders group
+	
+	cn = '%s' % uuid.uuid4()
+	dn = "cn=%s,%s" % (cn, env.LDAP_PERMITTED_SENDERS_BASE)
+	attrs = {
+		"mail" : address,
+		"description": "Permitted to MAIL FROM this address",
+		"member" : validated_permitted_senders
+	}
+	if len(validated_permitted_senders)==0:
+		if existing_permitted_senders:
+			dn = existing_permitted_senders['dn']
+			conn.wait( conn.delete(dn) )
+	else:
+		op=conn.add_or_modify(dn, existing_permitted_senders,
+							  [ 'member' ], [ 'mailGroup' ],
+							  attrs)
 
 	if do_kick:
 		# Update things in case any new domains are added.
 		return kick(env, return_status)
+	
 
 def remove_mail_alias(address, env, do_kick=True):
+	# Remove an alias group and it's associated permitted senders
+	# group.
+	#
+	# address is the email address of the alias
+	#
+	# If an error occurs, the function returns a tuple of (message,
+	# http-status).
+	#
+	# If successful, the string "OK" is returned.
+	
 	# convert Unicode domain to IDNA
 	address = sanitize_idn_email_address(address)
 
 	# remove
-	conn, c = open_database(env, with_connection=True)
-	c.execute("DELETE FROM aliases WHERE source=?", (address,))
-	if c.rowcount != 1:
+	conn = open_database(env)
+	existing_alias, existing_permitted_senders = find_mail_alias(env, address, conn=conn)
+	if existing_alias:
+		conn.delete(existing_alias['dn'])
+	else:
 		return ("That's not an alias (%s)." % address, 400)
-	conn.commit()
 
+	if existing_permitted_senders:
+		conn.delete(existing_permitted_senders['dn'])
+		
 	if do_kick:
 		# Update things in case any domains are removed.
 		return kick(env, "alias removed")
@@ -515,6 +1028,8 @@ def get_system_administrator(env):
 
 def get_required_aliases(env):
 	# These are the aliases that must exist.
+	# Returns a set of email addresses.
+	
 	aliases = set()
 
 	# The system administrator alias is required.
@@ -555,9 +1070,8 @@ def kick(env, mail_result=None):
 
 	# Ensure every required alias exists.
 
-	existing_users = get_mail_users(env)
-	existing_alias_records = get_mail_aliases(env)
-	existing_aliases = set(a for a, *_ in existing_alias_records) # just first entry in tuple
+	existing_users = get_mail_users(env, as_map=True)
+	existing_aliases = get_mail_aliases(env, as_map=True)
 	required_aliases = get_required_aliases(env)
 
 	def ensure_admin_alias_exists(address):
@@ -581,8 +1095,9 @@ def kick(env, mail_result=None):
 
 	# Remove auto-generated postmaster/admin on domains we no
 	# longer have any other email addresses for.
-	for address, forwards_to, *_ in existing_alias_records:
+	for address in existing_aliases:
 		user, domain = address.split("@")
+		forwards_to = ",".join(existing_aliases[address]["forward_tos"])
 		if user in ("postmaster", "admin", "abuse") \
 			and address not in required_aliases \
 			and forwards_to == get_system_administrator(env):