Add support for bidirectional mail alias controls

This is an extension of #427. Building on that change it adds support in the
aliases table for flagging aliases as:
 1. Applicable to inbound and outbound mail.
 2. Applicable to inbound mail only.
 3. Applicable to outbound mail only.
 4. Disabled.

The aliases UI is also updated to allow administrators to set the direction of
each alias.

Using this extra information, the sqlite queries executed by Postfix are
updated so only the relevant alias types are checked.

The goal and result of this change is that outbound-only catch-all aliases can
now be defined (in fact catch-all aliases of any type can be defined).

This allow us to continue supporting relaying as described at
https://mailinabox.email/advanced-configuration.html#relay
without requiring that administrators either create regular aliases for each
outbound *relay* address, or that they create a catch-all alias and then face a
flood of spam.

I have tested the code as it is in this commit and fixed every issue I found,
so in that regard the change is complete. However I see room for improvement
in terms of updating terminology to make the UI etc. easier to understand.
I'll make those changes as subsequent commits so that this tested checkpoint is
not lost, but also so they can be rejected independently of the actual change
if not wanted.

management/daemon.py

@@ -45,7 +45,7 @@ def authorized_personnel_only(viewfunc):
 
 		# Authorized to access an API view?
 		if "admin" in privs:
-			# Call view func.	
+			# Call view func.
 			return viewfunc(*args, **kwargs)
 		elif not error:
 			error = "You are not an administrator."
@@ -179,7 +179,7 @@ def mail_aliases():
 	if request.args.get("format", "") == "json":
 		return json_response(get_mail_aliases_ex(env))
 	else:
-		return "".join(x+"\t"+y+"\n" for x, y in get_mail_aliases(env))
+		return "".join(source+"\t"+destination+"\t"+applies_inbound+"\t"+applies_outbound+"\n" for source, destination, applies_inbound, applies_outbound in get_mail_aliases(env))
 
 @app.route('/mail/aliases/add', methods=['POST'])
 @authorized_personnel_only
@@ -187,6 +187,8 @@ def mail_aliases_add():
 	return add_mail_alias(
 		request.form.get('source', ''),
 		request.form.get('destination', ''),
+		request.form.get('applies_inbound', '') == '1',
+		request.form.get('applies_outbound', '') == '1',
 		env,
 		update_if_exists=(request.form.get('update_if_exists', '') == '1')
 		)
@@ -283,7 +285,7 @@ def dns_set_record(qname, rtype="A"):
 				# make this action set (replace all records for this
 				# qname-rtype pair) rather than add (add a new record).
 				action = "set"
-		
+
 		elif request.method == "DELETE":
 			if value == '':
 				# Delete all records for this qname-type pair.

management/mailconfig.py

@@ -181,13 +181,13 @@ def get_admins(env):
 	return users
 
 def get_mail_aliases(env):
-	# Returns a sorted list of tuples of (alias, forward-to string).
+	# Returns a sorted list of tuples of (alias, forward-to string, applies-to-inbound-mail, applies-to-outbound-mail).
 	c = open_database(env)
-	c.execute('SELECT source, destination FROM aliases')
+	c.execute('SELECT source, destination, applies_inbound, applies_outbound FROM aliases')
-	aliases = { row[0]: row[1] for row in c.fetchall() } # make dict
+	aliases = { row[0]: row[1:4] for row in c.fetchall() } # make dict
 
 	# put in a canonical order: sort by domain, then by email address lexicographically
-	aliases = [ (source, aliases[source]) for source in utils.sort_email_addresses(aliases.keys(), env) ]
+	aliases = [ (source,) + aliases[source] for source in utils.sort_email_addresses(aliases.keys(), env) ]
 	return aliases
 
 def get_mail_aliases_ex(env):
@@ -202,6 +202,8 @@ def get_mail_aliases_ex(env):
 	#         source: "name@domain.tld", # IDNA-encoded
 	#         source_display: "name@domain.tld", # full Unicode
 	#         destination: ["target1@domain.com", "target2@domain.com", ...],
+	#         applies_inbound: True|False
+	#         applies_outbound: True|False
 	#         required: True|False
 	#       },
 	#       ...
@@ -212,7 +214,7 @@ def get_mail_aliases_ex(env):
 
 	required_aliases = get_required_aliases(env)
 	domains = {}
-	for source, destination in get_mail_aliases(env):
+	for source, destination, applies_inbound, applies_outbound in get_mail_aliases(env):
 		# get alias info
 		domain = get_domain(source)
 		required = (source in required_aliases)
@@ -227,6 +229,8 @@ def get_mail_aliases_ex(env):
 			"source": source,
 			"source_display": prettify_idn_email_address(source),
 			"destination": [prettify_idn_email_address(d.strip()) for d in destination.split(",")],
+			"applies_inbound": True if applies_inbound == 1 else False,
+			"applies_outbound": True if applies_outbound == 1 else False,
 			"required": required,
 		})
 
@@ -250,7 +254,7 @@ def get_mail_domains(env, filter_aliases=lambda alias : True):
 	# configured on the system.
 	return set(
 		   [get_domain(addr, as_unicode=False) for addr in get_mail_users(env)]
-		 + [get_domain(source, as_unicode=False) for source, target in get_mail_aliases(env) if filter_aliases((source, target)) ]
+		 + [get_domain(source, as_unicode=False) for source, *_ in get_mail_aliases(env) if filter_aliases(source) ]
 		 )
 
 def add_mail_user(email, pw, privs, env):
@@ -406,7 +410,7 @@ def add_remove_mail_user_privilege(email, priv, action, env):
 
 	return "OK"
 
-def add_mail_alias(source, destination, env, update_if_exists=False, do_kick=True):
+def add_mail_alias(source, destination, applies_inbound, applies_outbound, env, update_if_exists=False, do_kick=True):
 	# convert Unicode domain to IDNA
 	source = sanitize_idn_email_address(source)
 
@@ -460,13 +464,13 @@ def add_mail_alias(source, destination, env, update_if_exists=False, do_kick=Tru
 	# save to db
 	conn, c = open_database(env, with_connection=True)
 	try:
-		c.execute("INSERT INTO aliases (source, destination) VALUES (?, ?)", (source, destination))
+		c.execute("INSERT INTO aliases (source, destination, applies_inbound, applies_outbound) VALUES (?, ?, ?, ?)", (source, destination, 1 if applies_inbound else 0, 1 if applies_outbound else 0))
 		return_status = "alias added"
 	except sqlite3.IntegrityError:
 		if not update_if_exists:
 			return ("Alias already exists (%s)." % source, 400)
 		else:
-			c.execute("UPDATE aliases SET destination = ? WHERE source = ?", (destination, source))
+			c.execute("UPDATE aliases SET destination = ?, applies_inbound = ?, applies_outbound = ? WHERE source = ?", (destination, 1 if applies_inbound else 0, 1 if applies_outbound else 0, source))
 			return_status = "alias updated"
 
 	conn.commit()
@@ -507,8 +511,8 @@ def get_required_aliases(env):
 	# email on that domain are the required aliases or a catch-all/domain-forwarder.
 	real_mail_domains = get_mail_domains(env,
 		filter_aliases = lambda alias :
-			not alias[0].startswith("postmaster@") and not alias[0].startswith("admin@")
+			not alias.startswith("postmaster@") and not alias.startswith("admin@")
-			and not alias[0].startswith("@")
+			and not alias.startswith("@")
 			)
 
 	# Create postmaster@ and admin@ for all domains we serve mail on.
@@ -541,14 +545,14 @@ def kick(env, mail_result=None):
 			return
 
 		# Does this alias exists?
-		for s, t in existing_aliases:
+		for s, *_ in existing_aliases:
 			if s == source:
 				return
 
 		# Doesn't exist.
 		administrator = get_system_administrator(env)
 		if source == administrator: return # don't make an alias from the administrator to itself --- this alias must be created manually
-		add_mail_alias(source, administrator, env, do_kick=False)
+		add_mail_alias(source, administrator, True, True, env, do_kick=False)
 		results.append("added alias %s (=> %s)\n" % (source, administrator))
 
 	for alias in required_aliases:
@@ -556,7 +560,7 @@ def kick(env, mail_result=None):
 
 	# Remove auto-generated postmaster/admin on domains we no
 	# longer have any other email addresses for.
-	for source, target in existing_aliases:
+	for source, target, *_ in existing_aliases:
 		user, domain = source.split("@")
 		if user in ("postmaster", "admin") \
 			and source not in required_aliases \

management/status_checks.py

@@ -33,7 +33,7 @@ def run_checks(rounded_values, env, output, pool):
 	# (ignore errors; if bind9/rndc isn't running we'd already report
 	# that in run_services checks.)
 	shell('check_call', ["/usr/sbin/rndc", "flush"], trap=True)
-	
+
 	run_system_checks(rounded_values, env, output)
 
 	# perform other checks asynchronously
@@ -264,10 +264,10 @@ def run_domain_checks_on_domain(domain, rounded_time, env, dns_domains, dns_zone
 
 	if domain == env["PRIMARY_HOSTNAME"]:
 		check_primary_hostname_dns(domain, env, output, dns_domains, dns_zonefiles)
-		
+
 	if domain in dns_domains:
 		check_dns_zone(domain, env, output, dns_zonefiles)
-		
+
 	if domain in mail_domains:
 		check_mail_domain(domain, env, output)
 
@@ -351,11 +351,14 @@ def check_primary_hostname_dns(domain, env, output, dns_domains, dns_zonefiles):
 	check_alias_exists("Hostmaster contact address", "hostmaster@" + domain, env, output)
 
 def check_alias_exists(alias_name, alias, env, output):
-	mail_alises = dict(get_mail_aliases(env))
+	mail_aliases = dict([(source, (destination, applies_inbound)) for source, destination, applies_inbound, *_ in get_mail_aliases(env)])
-	if alias in mail_alises:
+	if alias in mail_aliases:
-		output.print_ok("%s exists as a mail alias. [%s ↦ %s]" % (alias_name, alias, mail_alises[alias]))
+		if mail_aliases[alias][1]:
+			output.print_ok("%s exists as an inbound mail alias. [%s ↦ %s]" % (alias_name, alias, mail_aliases[alias][0]))
+		else:
+			output.print_error("%s exists as a mail alias [%s ↦ %s] but is not enabled for inbound email." % (alias_name, alias, mail_aliases[alias][0]))
 	else:
-		output.print_error("""You must add a mail alias for %s and direct email to you or another administrator.""" % alias)
+		output.print_error("""You must add an inbound mail alias for %s which directs email to you or another administrator.""" % alias)
 
 def check_dns_zone(domain, env, output, dns_zonefiles):
 	# If a DS record is set at the registrar, check DNSSEC first because it will affect the NS query.
@@ -492,7 +495,7 @@ def check_mail_domain(domain, env, output):
 
 	# Check that the postmaster@ email address exists. Not required if the domain has a
 	# catch-all address or domain alias.
-	if "@" + domain not in dict(get_mail_aliases(env)):
+	if "@" + domain not in [source for source, *_ in get_mail_aliases(env)]:
 		check_alias_exists("Postmaster contact address", "postmaster@" + domain, env, output)
 
 	# Stop if the domain is listed in the Spamhaus Domain Block List.
@@ -884,7 +887,7 @@ def run_and_output_changes(env, pool, send_via_email):
 			if category not in cur_status:
 				out.add_heading(category)
 				out.print_warning("This section was removed.")
-	
+
 	if send_via_email:
 		# If there were changes, send off an email.
 		buf = out.buf.getvalue()
@@ -896,7 +899,7 @@ def run_and_output_changes(env, pool, send_via_email):
 			msg['To'] = "administrator@%s" % env['PRIMARY_HOSTNAME']
 			msg['Subject'] = "[%s] Status Checks Change Notice" % env['PRIMARY_HOSTNAME']
 			msg.set_payload(buf, "UTF-8")
-	
+
 			# send to administrator@
 			import smtplib
 			mailserver = smtplib.SMTP('localhost', 25)
@@ -906,7 +909,7 @@ def run_and_output_changes(env, pool, send_via_email):
 				"administrator@%s" % env['PRIMARY_HOSTNAME'], # RCPT TO
 				msg.as_string())
 			mailserver.quit()
-		
+
 	# Store the current status checks output for next time.
 	os.makedirs(os.path.dirname(cache_fn), exist_ok=True)
 	with open(cache_fn, "w") as f:

management/templates/aliases.html

@@ -13,7 +13,7 @@
   <div class="form-group">
     <div class="col-sm-offset-1 col-sm-11">
       <div id="alias_type_buttons" class="btn-group btn-group-xs">
-        <button type="button" class="btn btn-default active" data-mode="regular">Regular</button>
+        <button type="button" class="btn btn-default" data-mode="regular">Regular</button>
         <button type="button" class="btn btn-default" data-mode="catchall">Catch-All</button>
         <button type="button" class="btn btn-default" data-mode="domainalias">Domain Alias</button>
       </div>
@@ -30,6 +30,17 @@
       <div style="margin-top: 3px; padding-left: 3px; font-size: 90%" class="text-muted">You may use international (non-ASCII) characters for the domain part of the email address only.</div>
     </div>
   </div>
+  <div class="form-group">
+    <label for="addaliasDirection" class="col-sm-1 control-label">Direction</label>
+    <div class="col-sm-10">
+      <select class="form-control" id="addaliasDirection">
+        <option value="disabled">Disabled</option>
+        <option value="outbound">Outbound only</option>
+        <option value="inbound">Inbound only</option>
+        <option value="bidirectional">Both</option>
+      </select>
+    </div>
+  </div>
   <div class="form-group">
     <label for="addaliasTargets" class="col-sm-1 control-label">Forward To</label>
     <div class="col-sm-10">
@@ -50,6 +61,7 @@
     <tr>
       <th></th>
       <th>Alias<br></th>
+      <th>Direction</th>
       <th>Forwards To</th>
     </tr>
   </thead>
@@ -71,6 +83,7 @@
         </a>
     </td>
     <td class='email'> </td>
+    <td class='direction'> </td>
     <td class='target'> </td>
   </tr>
   </table>
@@ -100,6 +113,19 @@ function show_aliases() {
           if (alias.required) n.addClass('alias-required');
           n.attr('data-email', alias.source_display); // this is decoded from IDNA, but will get re-coded to IDNA on the backend
           n.find('td.email').text(alias.source_display)
+          if (!alias.applies_inbound && !alias.applies_outbound) {
+            n.find('td.direction').text('')
+            n.attr('data-direction', 'disabled');
+          } else if (!alias.applies_inbound && alias.applies_outbound) {
+            n.find('td.direction').text('↤')
+            n.attr('data-direction', 'outbound');
+          } else if (alias.applies_inbound && !alias.applies_outbound) {
+            n.find('td.direction').text('↦')
+            n.attr('data-direction', 'inbound');
+          } else if (alias.applies_inbound && alias.applies_outbound) {
+            n.find('td.direction').text('↮')
+            n.attr('data-direction', 'bidirectional');
+          }
           for (var j = 0; j < alias.destination.length; j++)
             n.find('td.target').append($("<div></div>").text(alias.destination[j]))
           $('#alias_table tbody').append(n);
@@ -114,18 +140,21 @@ function show_aliases() {
       if ($(this).attr('data-mode') == "regular") {
         $('#addaliasEmail').attr('type', 'email');
         $('#addaliasEmail').attr('placeholder', 'incoming email address (e.g. you@yourdomain.com)');
-	$('#addaliasTargets').attr('placeholder', 'forward to these email addresses (one per line or separated by commas)');
+        $('#addaliasDirection').val('bidirectional');
+      	$('#addaliasTargets').attr('placeholder', 'forward to these email addresses (one per line or separated by commas)');
         $('#alias_mode_info').slideUp();
       } else if ($(this).attr('data-mode') == "catchall") {
         $('#addaliasEmail').attr('type', 'text');
         $('#addaliasEmail').attr('placeholder', 'incoming catch-all address (e.g. @yourdomain.com)');
-	$('#addaliasTargets').attr('placeholder', 'forward to these email addresses (one per line or separated by commas)');
+        $('#addaliasDirection').val('outbound');
+        $('#addaliasTargets').attr('placeholder', 'forward to these email addresses (one per line or separated by commas)');
         $('#alias_mode_info').slideDown();
         $('#alias_mode_info span').addClass('hidden');
         $('#alias_mode_info span.catchall').removeClass('hidden');
       } else if ($(this).attr('data-mode') == "domainalias") {
         $('#addaliasEmail').attr('type', 'text');
         $('#addaliasEmail').attr('placeholder', 'incoming domain (@yourdomain.com)');
+        $('#addaliasDirection').val('inbound');
         $('#addaliasTargets').attr('placeholder', 'forward to domain (@yourdomain.com)');
         $('#alias_mode_info').slideDown();
         $('#alias_mode_info span').addClass('hidden');
@@ -140,6 +169,7 @@ var is_alias_add_update = false;
 function do_add_alias() {
   var title = (!is_alias_add_update) ? "Add Alias" : "Update Alias";
   var email = $("#addaliasEmail").val();
+  var direction = $("#addaliasDirection").val();
   var targets = $("#addaliasTargets").val();
   api(
     "/mail/aliases/add",
@@ -147,7 +177,9 @@ function do_add_alias() {
     {
       update_if_exists: is_alias_add_update ? '1' : '0',
       source: email,
-      destination: targets
+      destination: targets,
+      applies_inbound: (direction == 'bidirectional' || direction == 'inbound') ? '1' : '0',
+      applies_outbound: (direction == 'bidirectional' || direction == 'outbound') ? '1' : '0'
     },
     function(r) {
       // Responses are multiple lines of pre-formatted text.
@@ -164,6 +196,12 @@ function do_add_alias() {
 function aliases_reset_form() {
   $("#addaliasEmail").prop('disabled', false);
   $("#addaliasEmail").val('')
+  if ($('#alias_type_buttons button').attr('data-mode') == "regular")
+    $('#addaliasDirection').val('bidirectional');
+  else if ($('#alias_type_buttons button').attr('data-mode') == "catchall")
+    $('#alias_type_buttons').val('outbound');
+  else if ($('#addaliasDirection button').attr('data-mode') == "domainalias")
+    $('#addaliasDirection').val('inbound');
   $("#addaliasTargets").val('')
   $('#alias-cancel').addClass('hidden');
   $('#add-alias-button').text('Add Alias');
@@ -176,20 +214,21 @@ function aliases_edit(elem) {
   var targets = "";
   for (var i = 0; i < targetdivs.length; i++)
     targets += $(targetdivs[i]).text() + "\n";
-
+  var direction = $(elem).parents('tr').attr('data-direction')
-  is_alias_add_update = true;
-  $('#alias-cancel').removeClass('hidden');
-  $("#addaliasEmail").prop('disabled', true);
-  $("#addaliasEmail").val(email);
-  $("#addaliasTargets").val(targets);
-  $('#add-alias-button').text('Update');
   if (email.charAt(0) == '@' && targets.charAt(0) == '@')
     $('#alias_type_buttons button[data-mode="domainalias"]').click();
   else if (email.charAt(0) == '@')
     $('#alias_type_buttons button[data-mode="catchall"]').click();
   else
     $('#alias_type_buttons button[data-mode="regular"]').click();
+  $('#alias-cancel').removeClass('hidden');
+  $("#addaliasEmail").prop('disabled', true);
+  $("#addaliasEmail").val(email);
+  $('#addaliasDirection').val(direction);
+  $("#addaliasTargets").val(targets);
+  $('#add-alias-button').text('Update');
   $('body').animate({ scrollTop: 0 })
+  is_alias_add_update = true;
 }
 
 function aliases_remove(elem) {

setup/mail-users.sh

@@ -5,7 +5,7 @@
 #
 # This script configures user authentication for Dovecot
 # and Postfix (which relies on Dovecot) and destination
-# validation by quering an Sqlite3 database of mail users. 
+# validation by quering an Sqlite3 database of mail users.
 
 source setup/functions.sh # load our functions
 source /etc/mailinabox.conf # load global vars
@@ -21,7 +21,7 @@ db_path=$STORAGE_ROOT/mail/users.sqlite
 if [ ! -f $db_path ]; then
 	echo Creating new user database: $db_path;
 	echo "CREATE TABLE users (id INTEGER PRIMARY KEY AUTOINCREMENT, email TEXT NOT NULL UNIQUE, password TEXT NOT NULL, extra, privileges TEXT NOT NULL DEFAULT '');" | sqlite3 $db_path;
-	echo "CREATE TABLE aliases (id INTEGER PRIMARY KEY AUTOINCREMENT, source TEXT NOT NULL UNIQUE, destination TEXT NOT NULL);" | sqlite3 $db_path;
+	echo "CREATE TABLE aliases (id INTEGER PRIMARY KEY AUTOINCREMENT, source TEXT NOT NULL UNIQUE, destination TEXT NOT NULL, applies_inbound INTEGER NOT NULL DEFAULT 1, applies_outbound INTEGER NOT NULL DEFAULT 1);" | sqlite3 $db_path;
 fi
 
 # ### User Authentication
@@ -72,17 +72,17 @@ tools/editconf.py /etc/postfix/main.cf \
 # ### Sender Validation
 
 # Use a Sqlite3 database to set login maps. This is used with
-# reject_authenticated_sender_login_mismatch to see if user is
+# reject_authenticated_sender_login_mismatch to see if the user is
-# allowed to send mail using FROM field specified in the request.
+# allowed to send mail as the FROM address specified in the request.
 tools/editconf.py /etc/postfix/main.cf \
 	smtpd_sender_login_maps=sqlite:/etc/postfix/sender-login-maps.cf
 
-# SQL statement to set login map which includes the case when user is
+# SQL statement that returns a list of addresses/domains the logged in username
-# sending email using a valid alias.
+# is allowed to send as. This is similar to virtual-alias-maps.cf (see below).
-# This is the same as virtual-alias-maps.cf, See below
+# Matches from the users table take priority over (direct) aliases.
 cat > /etc/postfix/sender-login-maps.cf << EOF;
 dbpath=$db_path
-query = SELECT destination from (SELECT destination, 0 as priority FROM aliases WHERE source='%s' UNION SELECT email as destination, 1 as priority FROM users WHERE email='%s') ORDER BY priority LIMIT 1;
+query = SELECT destination from (SELECT destination, 0 as priority FROM aliases WHERE source='%s' AND applies_outbound=1 UNION SELECT email as destination, 1 as priority FROM users WHERE email='%s') ORDER BY priority LIMIT 1;
 EOF
 
 # ### Destination Validation
@@ -98,7 +98,7 @@ tools/editconf.py /etc/postfix/main.cf \
 # SQL statement to check if we handle mail for a domain, either for users or aliases.
 cat > /etc/postfix/virtual-mailbox-domains.cf << EOF;
 dbpath=$db_path
-query = SELECT 1 FROM users WHERE email LIKE '%%@%s' UNION SELECT 1 FROM aliases WHERE source LIKE '%%@%s'
+query = SELECT 1 FROM users WHERE email LIKE '%%@%s' UNION SELECT 1 FROM aliases WHERE source LIKE '%%@%s' AND applies_inbound=1
 EOF
 
 # SQL statement to check if we handle mail for a user.
@@ -129,7 +129,7 @@ EOF
 # postfix's preference for aliases for whole email addresses.
 cat > /etc/postfix/virtual-alias-maps.cf << EOF;
 dbpath=$db_path
-query = SELECT destination from (SELECT destination, 0 as priority FROM aliases WHERE source='%s' UNION SELECT email as destination, 1 as priority FROM users WHERE email='%s') ORDER BY priority LIMIT 1;
+query = SELECT destination from (SELECT destination, 0 as priority FROM aliases WHERE source='%s' AND applies_inbound=1 UNION SELECT email as destination, 1 as priority FROM users WHERE email='%s') ORDER BY priority LIMIT 1;
 EOF
 
 # Restart Services

setup/migrate.py

@@ -101,6 +101,11 @@ def migration_8(env):
 	# a new key, which will be 2048 bits.
 	os.unlink(os.path.join(env['STORAGE_ROOT'], 'mail/dkim/mail.private'))
 
+def migration_9(env):
+	db = os.path.join(env["STORAGE_ROOT"], 'mail/users.sqlite')
+	shell("check_call", ["sqlite3", db, "ALTER TABLE aliases ADD COLUMN applies_inbound INTEGER NOT NULL DEFAULT 1"])
+	shell("check_call", ["sqlite3", db, "ALTER TABLE aliases ADD COLUMN applies_outbound INTEGER NOT NULL DEFAULT 1"])
+
 def get_current_migration():
 	ver = 0
 	while True: