update features list in README

* zen.spamhaus.org
* dbl.spamhaus.org
* checks if a connection to Google's MTA on port 25 works

README.md

@@ -1,6 +1,8 @@
 Mail-in-a-Box
 =============
 
+By [@JoshData](https://github.com/JoshData) and contributors.
+
 Mail-in-a-Box helps individuals take back control of their email by defining a one-click, easy-to-deploy SMTP+everything else server: a mail server in a box.
 
 **This is a work in progress. I work on this in my limited free time.**
@@ -11,7 +13,7 @@ Why build this? Mass electronic surveillance by governments revealed over the la
 The Box
 -------
 
-Mail-in-a-Box turns a fresh Ubuntu 14.04 LTS 64-bit machine into a working mail server, including [SMTP](http://www.postfix.org/), [IMAP](http://dovecot.org/), [webmail](http://roundcube.net/), [spam filtering](https://spamassassin.apache.org/), [greylisting](http://postgrey.schweikert.ch/), DNS, [SPF](https://en.wikipedia.org/wiki/Sender_Policy_Framework), [DKIM](https://en.wikipedia.org/wiki/DomainKeys_Identified_Mail), [DMARC](https://en.wikipedia.org/wiki/DMARC), [DNSSEC](https://en.wikipedia.org/wiki/DNSSEC), [DANE TLSA](https://en.wikipedia.org/wiki/DNS-based_Authentication_of_Named_Entities), and basic system services like a firewall, intrusion protection, and setting the system clock.
+Mail-in-a-Box turns a fresh Ubuntu 14.04 LTS 64-bit machine into a working mail server, including SMTP ([postfix](http://www.postfix.org/)), IMAP ([dovecot](http://dovecot.org/)), Exchange ActiveSync ([z-push](https://github.com/fmbiete/Z-Push-contrib)), webmail ([Roundcube](http://roundcube.net/)), spam filtering ([spamassassin](https://spamassassin.apache.org/)), greylisting ([postgrey](http://postgrey.schweikert.ch/)), CardDAV/CalDAV ([ownCloud](http://owncloud.org/)), DNS, [SPF](https://en.wikipedia.org/wiki/Sender_Policy_Framework), DKIM ([OpenDKIM](http://www.opendkim.org/)), [DMARC](https://en.wikipedia.org/wiki/DMARC), [DNSSEC](https://en.wikipedia.org/wiki/DNSSEC), [DANE TLSA](https://en.wikipedia.org/wiki/DNS-based_Authentication_of_Named_Entities), and basic system services like a firewall, intrusion protection, and setting the system clock.
 
 This setup is what has been powering my own personal email since September 2013.
 
@@ -21,10 +23,12 @@ In short, it's like this:
 
 	# do this on a fresh install of Ubuntu 14.04 only!
 	sudo apt-get install -y git
-	git clone https://github.com/joshdata/mailinabox
+	git clone https://github.com/mail-in-a-box/mailinabox
 	cd mailinabox
 	sudo setup/start.sh
 
+Congratulations! You should now have a working setup. You'll be given the address of the administrative interface for further instructions.
+
 **Status**: This is a work in progress. It works for what it is, but it is missing such things as quotas, backup/restore, etc.
 
 The Goals
@@ -34,7 +38,7 @@ The Goals
 * Promote decentralization, innovation, and privacy on the web.
 * Have automated, auditable, and [idempotent](http://sharknet.us/2014/02/01/automated-configuration-management-challenges-with-idempotency/) configuration.
 
-For more background, see [The Rationale](https://github.com/JoshData/mailinabox/wiki).
+For more background, see [The Rationale](https://github.com/mail-in-a-box/mailinabox/wiki).
 
 What I am not trying to do:
 

Vagrantfile

@@ -22,6 +22,7 @@ Vagrant.configure("2") do |config|
     export PUBLIC_IPV6=auto
     export PRIMARY_HOSTNAME=auto-easy
     export CSR_COUNTRY=US
+    #export SKIP_NETWORK_CHECKS=1
 
     # Start the setup script.
     cd /vagrant

conf/nginx-primaryonly.conf

@@ -0,0 +1,41 @@
+	# ownCloud configuration.
+	rewrite ^/cloud$ /cloud/ redirect;
+	rewrite ^/cloud/$ /cloud/index.php;
+	rewrite ^(/cloud/core/doc/[^\/]+/)$ $1/index.html;
+	location /cloud/ {
+		alias /usr/local/lib/owncloud/;
+		location ~ ^/(data|config|\.ht|db_structure\.xml|README) {
+			deny all;
+	 	}
+	}
+	location ~ ^(/cloud)(/[^/]+\.php)(/.*)?$ {
+		# note: ~ has precendence over a regular location block
+		include fastcgi_params;
+		fastcgi_param SCRIPT_FILENAME /usr/local/lib/owncloud/$2;
+		fastcgi_param SCRIPT_NAME $1$2;
+		fastcgi_param PATH_INFO $3;
+		fastcgi_param MOD_X_ACCEL_REDIRECT_ENABLED on;
+		fastcgi_read_timeout 630;
+		fastcgi_pass php-fpm;
+		error_page 403 /cloud/core/templates/403.php;
+		error_page 404 /cloud/core/templates/404.php;
+		client_max_body_size 1G;
+		fastcgi_buffers 64 4K;
+	}
+	location ^~ /cloud/data {
+		# In order to support MOD_X_ACCEL_REDIRECT_ENABLED, we need to expose
+		# the data directory but only allow 'internal' redirects within nginx
+		# so that this is not exposed to the world.
+		internal;
+		alias $STORAGE_ROOT/owncloud;
+	}
+	location ~ ^/((caldav|carddav|webdav).*)$ {
+		# Z-Push doesn't like getting a redirect, and a plain rewrite didn't work either.
+		# Properly proxying like this seems to work fine.
+		proxy_pass https://$HOSTNAME/cloud/remote.php/$1;
+	}
+	rewrite ^/.well-known/host-meta /cloud/public.php?service=host-meta last;
+	rewrite ^/.well-known/host-meta.json /cloud/public.php?service=host-meta-json last;
+	rewrite ^/.well-known/carddav /cloud/remote.php/carddav/ redirect;
+	rewrite ^/.well-known/caldav /cloud/remote.php/caldav/ redirect;
+

conf/nginx-top.conf

@@ -0,0 +1,8 @@
+## NOTE: This file is automatically generated by Mail-in-a-Box.
+##       Do not edit this file. It will be replaced each time
+##       Mail-in-a-Box needs to update the web configuration.
+
+upstream php-fpm {
+	server unix:/var/run/php5-fpm.sock;
+}
+

conf/nginx.conf

@@ -1,6 +1,4 @@
-## NOTE: This file is automatically generated by Mail-in-a-Box.
-##       Do not edit this file. It will be replaced each time
-##       Mail-in-a-Box needs up update the web configuration.
+## $HOSTNAME
 
 # Redirect all HTTP to HTTPS.
 server {
@@ -26,6 +24,12 @@ server {
 	root $ROOT;
 	index index.html index.htm;
 
+	# Control Panel
+	rewrite ^/admin$ /admin/;
+	location /admin/ {
+		proxy_pass http://localhost:10222/;
+	}
+
 	# Roundcube Webmail configuration.
 	rewrite ^/mail$ /mail/ redirect;
 	rewrite ^/mail/$ /mail/index.php;
@@ -38,11 +42,12 @@ server {
 		return 403;
 	}
 	location ~ /mail/.*\.php {
+		# note: ~ has precendence over a regular location block
 		include fastcgi_params;
 		fastcgi_split_path_info ^/mail(/.*)()$;
 		fastcgi_index index.php;
 		fastcgi_param SCRIPT_FILENAME /usr/local/lib/roundcubemail/$fastcgi_script_name;
-		fastcgi_pass unix:/tmp/php-fastcgi.www-data.sock;
+		fastcgi_pass php-fpm;
 		client_max_body_size 20M;
 	}
 
@@ -50,22 +55,24 @@ server {
 	location = /.well-known/webfinger {
 		include fastcgi_params;
 		fastcgi_param SCRIPT_FILENAME /usr/local/bin/mailinabox-webfinger.php;
-		fastcgi_pass unix:/tmp/php-fastcgi.www-data.sock;
-	}
-
-	# Microsoft Exchange autodiscover.xml for email
-	location /autodiscover/autodiscover.xml {
-		include fastcgi_params;
-		fastcgi_param SCRIPT_FILENAME /usr/local/bin/mailinabox-exchange-autodiscover.php;
-		fastcgi_pass unix:/tmp/php-fastcgi.www-data.sock;
+		fastcgi_pass php-fpm;
 	}
 
 	# Z-Push (Microsoft Exchange ActiveSync)
 	location /Microsoft-Server-ActiveSync {
-	        include /etc/nginx/fastcgi_params;
+		include /etc/nginx/fastcgi_params;
 		fastcgi_param SCRIPT_FILENAME /usr/local/lib/z-push/index.php;
-		fastcgi_pass unix:/tmp/php-fastcgi.www-data.sock;
+		fastcgi_param PHP_VALUE "include_path=.:/usr/share/php:/usr/share/pear:/usr/share/awl/inc";
+		fastcgi_read_timeout 630;
+		fastcgi_pass php-fpm;
 	}
+	location /autodiscover/autodiscover.xml {
+		include fastcgi_params;
+		fastcgi_param SCRIPT_FILENAME /usr/local/lib/z-push/autodiscover/autodiscover.php;
+		fastcgi_param PHP_VALUE "include_path=.:/usr/share/php:/usr/share/pear:/usr/share/awl/inc";
+		fastcgi_pass php-fpm;
+	}
+
 
 	# ADDITIONAL DIRECTIVES HERE
 }

conf/phpfcgi-initscript

@@ -1,134 +0,0 @@
-#! /bin/sh
-### BEGIN INIT INFO
-# Provides:          php-fastcgi
-# Required-Start:    $all
-# Required-Stop:     $all
-# Default-Start:     2 3 4 5
-# Default-Stop:      0 1 6
-# Short-Description: Start and stop php-cgi in external FASTCGI mode
-# Description:       Start and stop php-cgi in external FASTCGI mode
-### END INIT INFO
-
-# Author: Kurt Zankl
-# via: http://blog.codefront.net/2007/06/11/nginx-php-and-a-php-fastcgi-daemon-init-script/
-# But modified by JT.
-
-# Do NOT "set -e"
-
-PATH=/sbin:/usr/sbin:/bin:/usr/bin
-DESC="php-fastcgi"
-NAME=php-fastcgi
-DAEMON=/usr/bin/php-cgi
-PIDFILE=/var/run/$NAME.pid
-SCRIPTNAME=/etc/init.d/$NAME
-PHP_CONFIG_FILE=/etc/php5/cgi/php.ini
-
-# Exit if the package is not installed
-[ -x "$DAEMON" ] || exit 0
-
-# Set defaults.
-START=yes
-EXEC_AS_USER=www-data
-#FCGI_SOCKET=localhost:9000
-FCGI_SOCKET=/tmp/php-fastcgi.$EXEC_AS_USER.sock
-PHP_FCGI_CHILDREN=4
-PHP_FCGI_MAX_REQUESTS=1000
-
-# Read configuration variable file if it is present
-[ -r /etc/default/$NAME ] && . /etc/default/$NAME
-
-# Load the VERBOSE setting and other rcS variables
-. /lib/init/vars.sh
-
-# Define LSB log_* functions.
-# Depend on lsb-base (>= 3.0-6) to ensure that this file is present.
-. /lib/lsb/init-functions
-
-# If the daemon is not enabled, give the user a warning and then exit,
-# unless we are stopping the daemon
-if [ "$START" != "yes" -a "$1" != "stop" ]; then
-        log_warning_msg "To enable $NAME, edit /etc/default/$NAME and set START=yes"
-        exit 0
-fi
-
-# Process configuration
-export PHP_FCGI_CHILDREN PHP_FCGI_MAX_REQUESTS
-DAEMON_ARGS="-q -b $FCGI_SOCKET -c $PHP_CONFIG_FILE"
-
-
-do_start()
-{
-        # Return
-        #   0 if daemon has been started
-        #   1 if daemon was already running
-        #   2 if daemon could not be started
-        start-stop-daemon --start --quiet --pidfile $PIDFILE --exec $DAEMON --test > /dev/null \
-                || return 1
-        start-stop-daemon --start --quiet --pidfile $PIDFILE --exec $DAEMON \
-                --background --make-pidfile --chuid $EXEC_AS_USER --startas $DAEMON -- \
-                $DAEMON_ARGS \
-                || return 2
-}
-
-do_stop()
-{
-        # Return
-        #   0 if daemon has been stopped
-        #   1 if daemon was already stopped
-        #   2 if daemon could not be stopped
-        #   other if a failure occurred
-        start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE > /dev/null # --name $DAEMON
-        RETVAL="$?"
-        [ "$RETVAL" = 2 ] && return 2
-        # Wait for children to finish too if this is a daemon that forks
-        # and if the daemon is only ever run from this initscript.
-        # If the above conditions are not satisfied then add some other code
-        # that waits for the process to drop all resources that could be
-        # needed by services started subsequently.  A last resort is to
-        # sleep for some time.
-        start-stop-daemon --stop --quiet --oknodo --retry=0/30/KILL/5 --exec $DAEMON
-        [ "$?" = 2 ] && return 2
-        # Many daemons don't delete their pidfiles when they exit.
-        rm -f $PIDFILE
-        return "$RETVAL"
-}
-case "$1" in
-  start)
-        [ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME"
-        do_start
-        case "$?" in
-                0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;;
-                2) [ "$VERBOSE" != no ] && log_end_msg 1 ;;
-        esac
-        ;;
-  stop)
-        [ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME"
-        do_stop
-        case "$?" in
-                0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;;
-                2) [ "$VERBOSE" != no ] && log_end_msg 1 ;;
-        esac
-        ;;
-  restart|force-reload)
-        log_daemon_msg "Restarting $DESC" "$NAME"
-        do_stop
-        case "$?" in
-          0|1)
-                do_start
-                case "$?" in
-                        0) log_end_msg 0 ;;
-                        1) log_end_msg 1 ;; # Old process is still running
-                        *) log_end_msg 1 ;; # Failed to start
-                esac
-                ;;
-          *)
-                # Failed to stop
-                log_end_msg 1
-                ;;
-        esac
-        ;;
-  *)
-        echo "Usage: $SCRIPTNAME {start|stop|restart|force-reload}" >&2
-        exit 3
-        ;;
-esac

conf/zpush/autodiscover_config.php

@@ -0,0 +1,24 @@
+<?php
+/***********************************************
+* File      :   config.php
+* Project   :   Z-Push
+* Descr     :   Autodiscover configuration file
+************************************************/
+
+// Defines the base path on the server
+define('BASE_PATH', dirname($_SERVER['SCRIPT_FILENAME']). '/');
+
+// The Z-Push server location for the autodiscover response
+define('SERVERURL', 'https://PRIMARY_HOSTNAME/Microsoft-Server-ActiveSync');
+
+define('USE_FULLEMAIL_FOR_LOGIN', true);
+
+define('LOGFILEDIR', '/var/log/z-push/');
+define('LOGFILE', LOGFILEDIR . 'autodiscover.log');
+define('LOGERRORFILE', LOGFILEDIR . 'autodiscover-error.log');
+define('LOGLEVEL', LOGLEVEL_INFO);
+define('LOGUSERLEVEL', LOGLEVEL);
+
+// the backend data provider
+define('BACKEND_PROVIDER', 'BackendCombined');
+?>

conf/zpush/backend_caldav.php

@@ -0,0 +1,18 @@
+<?php
+/***********************************************
+* File      :   config.php
+* Project   :   Z-Push
+* Descr     :   CalDAV backend configuration file
+************************************************/
+
+define('CALDAV_SERVER', 'https://localhost');
+define('CALDAV_PORT', '443');
+define('CALDAV_PATH', '/caldav/calendars/%u/');
+define('CALDAV_PERSONAL', '');
+
+// If the CalDAV server supports the sync-collection operation
+// DAViCal and SOGo support it
+// Setting this to false will work with most servers, but it will be slower
+define('CALDAV_SUPPORTS_SYNC', false);
+
+?>

conf/zpush/backend_carddav.php

@@ -0,0 +1,37 @@
+<?php
+/***********************************************
+* File      :   config.php
+* Project   :   Z-Push
+* Descr     :   CardDAV backend configuration file
+************************************************/
+
+
+define('CARDDAV_PROTOCOL', 'https'); /* http or https */
+define('CARDDAV_SERVER', 'localhost');
+define('CARDDAV_PORT', '443');
+define('CARDDAV_PATH', '/carddav/addressbooks/%u/');
+define('CARDDAV_DEFAULT_PATH', '/carddav/addressbooks/%u/contacts/'); /* subdirectory of the main path */
+define('CARDDAV_GAL_PATH', ''); /* readonly, searchable, not syncd */
+define('CARDDAV_GAL_MIN_LENGTH', 5);
+define('CARDDAV_CONTACTS_FOLDER_NAME', '%u Addressbook');
+
+
+// If the CardDAV server supports the sync-collection operation
+// DAViCal supports it, but SabreDav, Owncloud, SOGo don't
+// Setting this to false will work with most servers, but it will be slower: 1 petition for the href of vcards, and 1 petition for each vcard
+define('CARDDAV_SUPPORTS_SYNC', false);
+
+
+// If the CardDAV server supports the FN attribute for searches
+// DAViCal supports it, but SabreDav, Owncloud and SOGo don't
+// Setting this to true will search by FN. If false will search by sn, givenName and email
+// It's safe to leave it as false
+define('CARDDAV_SUPPORTS_FN_SEARCH', false);
+
+
+// If your carddav server needs to use file extension to recover a vcard.
+//    Davical needs it
+//    SOGo official demo online needs it, but some SOGo installation don't need it, so test it
+define('CARDDAV_URL_VCARD_EXTENSION', '.vcf');
+
+?>

conf/zpush/backend_combined.php

@@ -0,0 +1,49 @@
+<?php
+/***********************************************
+* File      :   backend/combined/config.php
+* Project   :   Z-Push
+* Descr     :   configuration file for the
+*               combined backend.
+************************************************/
+
+class BackendCombinedConfig {
+    public static function GetBackendCombinedConfig() {
+        return array(
+			'backends' => array(
+				'i' => array(
+					'name' => 'BackendIMAP',
+				),
+				'c' => array(
+					'name' => 'BackendCalDAV',
+				),
+				'd' => array(
+					'name' => 'BackendCardDAV',
+				),
+			),
+			'delimiter' => '/',
+			'folderbackend' => array(
+				SYNC_FOLDER_TYPE_INBOX => 'i',
+				SYNC_FOLDER_TYPE_DRAFTS => 'i',
+				SYNC_FOLDER_TYPE_WASTEBASKET => 'i',
+				SYNC_FOLDER_TYPE_SENTMAIL => 'i',
+				SYNC_FOLDER_TYPE_OUTBOX => 'i',
+				SYNC_FOLDER_TYPE_TASK => 'c',
+				SYNC_FOLDER_TYPE_APPOINTMENT => 'c',
+				SYNC_FOLDER_TYPE_CONTACT => 'd',
+				SYNC_FOLDER_TYPE_NOTE => 'c',
+				SYNC_FOLDER_TYPE_JOURNAL => 'c',
+				SYNC_FOLDER_TYPE_OTHER => 'i',
+				SYNC_FOLDER_TYPE_USER_MAIL => 'i',
+				SYNC_FOLDER_TYPE_USER_APPOINTMENT => 'c',
+				SYNC_FOLDER_TYPE_USER_CONTACT => 'd',
+				SYNC_FOLDER_TYPE_USER_TASK => 'c',
+				SYNC_FOLDER_TYPE_USER_JOURNAL => 'c',
+				SYNC_FOLDER_TYPE_USER_NOTE => 'c',
+				SYNC_FOLDER_TYPE_UNKNOWN => 'i',
+			),
+			'rootcreatefolderbackend' => 'i',
+		);
+    }
+}
+
+?>

conf/zpush/backend_imap.php

@@ -0,0 +1,42 @@
+<?php
+/***********************************************
+* File      :   config.php
+* Project   :   Z-Push
+* Descr     :   IMAP backend configuration file
+************************************************/
+
+define('IMAP_SERVER', 'localhost');
+define('IMAP_PORT', 993);
+define('IMAP_OPTIONS', '/ssl/norsh/novalidate-cert');
+define('IMAP_DEFAULTFROM', '');
+
+// not used
+define('IMAP_FROM_SQL_DSN', '');
+define('IMAP_FROM_SQL_USER', '');
+define('IMAP_FROM_SQL_PASSWORD', '');
+define('IMAP_FROM_SQL_OPTIONS', serialize(array(PDO::ATTR_PERSISTENT => true)));
+define('IMAP_FROM_SQL_QUERY', "select first_name, last_name, mail_address from users where mail_address = '#username@#domain'");
+define('IMAP_FROM_SQL_FIELDS', serialize(array('first_name', 'last_name', 'mail_address')));
+define('IMAP_FROM_SQL_FROM', '#first_name #last_name <#mail_address>');
+define('IMAP_FROM_LDAP_SERVER', '');
+define('IMAP_FROM_LDAP_SERVER_PORT', '389');
+define('IMAP_FROM_LDAP_USER', 'cn=zpush,ou=servers,dc=zpush,dc=org');
+define('IMAP_FROM_LDAP_PASSWORD', 'password');
+define('IMAP_FROM_LDAP_BASE', 'dc=zpush,dc=org');
+define('IMAP_FROM_LDAP_QUERY', '(mail=#username@#domain)');
+define('IMAP_FROM_LDAP_FIELDS', serialize(array('givenname', 'sn', 'mail')));
+define('IMAP_FROM_LDAP_FROM', '#givenname #sn <#mail>');
+
+
+// copy outgoing mail to this folder. If not set z-push will try the default folders
+define('IMAP_SENTFOLDER', '');
+define('IMAP_INLINE_FORWARD', true);
+define('IMAP_EXCLUDED_FOLDERS', '');
+define('IMAP_SMTP_METHOD', 'sendmail');
+
+global $imap_smtp_params;
+$imap_smtp_params = array('host' => 'ssl://localhost', 'port' => 587, 'auth' => true, 'username' => 'imap_username', 'password' => 'imap_password');
+
+define('MAIL_MIMEPART_CRLF', "\r\n");
+
+?>

management/auth.py

@@ -2,6 +2,9 @@ import base64, os, os.path
 
 from flask import make_response
 
+import utils
+from mailconfig import get_mail_user_privileges
+
 DEFAULT_KEY_PATH   = '/var/lib/mailinabox/api.key'
 DEFAULT_AUTH_REALM = 'Mail-in-a-Box Management Server'
 
@@ -37,32 +40,69 @@ class KeyAuthService:
 		with create_file_with_mode(self.key_path, 0o640) as key_file:
 			key_file.write(self.key + '\n')
 
-	def is_authenticated(self, request):
-		"""Test if the client key passed in HTTP header matches the service key"""
+	def is_authenticated(self, request, env):
+		"""Test if the client key passed in HTTP Authorization header matches the service key
+		or if the or username/password passed in the header matches an administrator user.
+		Returns 'OK' if the key is good or the user is an administrator, otherwise an error message."""
 
 		def decode(s):
-			return base64.b64decode(s.encode('utf-8')).decode('ascii')
-
-		def parse_api_key(header):
-			if header is None:
-				return
+			return base64.b64decode(s.encode('ascii')).decode('ascii')
 
+		def parse_basic_auth(header):
+			if " " not in header:
+				return None, None
 			scheme, credentials = header.split(maxsplit=1)
 			if scheme != 'Basic':
-				return
+				return None, None
 
-			username, password = decode(credentials).split(':', maxsplit=1)
-			return username
+			credentials = decode(credentials)
+			if ":" not in credentials:
+				return None, None
+			username, password = credentials.split(':', maxsplit=1)
+			return username, password
 
-		request_key = parse_api_key(request.headers.get('Authorization'))
+		header = request.headers.get('Authorization')
+		if not header:
+			return "No authorization header provided."
 
-		return request_key == self.key
+		username, password = parse_basic_auth(header)
 
-	def make_unauthorized_response(self):
-		return make_response(
-			'You must pass the API key from "{0}" as the username\n'.format(self.key_path),
-			401,
-			{ 'WWW-Authenticate': 'Basic realm="{0}"'.format(self.auth_realm) })
+		if username in (None, ""):
+			return "Authorization header invalid."
+		elif username == self.key:
+			return "OK"
+		else:
+			return self.check_imap_login( username, password, env)
+
+	def check_imap_login(self, email, pw, env):
+		# Validate a user's credentials.
+
+		# Sanity check.
+		if email == "" or pw == "":
+			return "Enter an email address and password."
+
+		# Authenticate.
+		try:
+			# Use doveadm to check credentials. doveadm will return
+			# a non-zero exit status if the credentials are no good,
+			# and check_call will raise an exception in that case.
+			utils.shell('check_call', [
+				"/usr/bin/doveadm",
+				"auth", "test",
+				email, pw
+				])
+		except:
+			# Login failed.
+			return "Invalid email address or password."
+
+		# Authorize.
+		# (This call should never fail on a valid user.)
+		privs = get_mail_user_privileges(email, env)
+		if isinstance(privs, tuple): raise Exception("Error getting privileges.")
+		if "admin" not in privs:
+			return "You are not an administrator for this system."
+
+		return "OK"
 
 	def _generate_key(self):
 		raw_key = os.urandom(32)

management/backup.py

@@ -10,13 +10,13 @@
 # 4) The backup directory is compressed into a single file using tar.
 # 5) That file is encrypted with a long password stored in backup/secret_key.txt.
 
-import sys, os, os.path, shutil
+import sys, os, os.path, shutil, glob
 
 from utils import exclusive_process, load_environment, shell
 
 # settings
 full_backup = "--full" in sys.argv
-keep_backups_for = "31D" # destroy backups older than 31 days
+keep_backups_for = "31D" # destroy backups older than 31 days except the most recent full backup
 
 env = load_environment()
 
@@ -27,6 +27,20 @@ backup_dir = os.path.join(env["STORAGE_ROOT"], 'backup')
 backup_duplicity_dir = os.path.join(backup_dir, 'duplicity')
 os.makedirs(backup_dir, exist_ok=True)
 
+# On the first run, always do a full backup. Incremental
+# will fail.
+if len(os.listdir(backup_duplicity_dir)) == 0:
+	full_backup = True
+else:
+	# When the size of incremental backups exceeds the size of existing
+	# full backups, take a new full backup. We want to avoid full backups
+	# because they are costly to synchronize off-site.
+	full_sz = sum(os.path.getsize(f) for f in glob.glob(backup_duplicity_dir + '/*-full.*'))
+	inc_sz = sum(os.path.getsize(f) for f in glob.glob(backup_duplicity_dir + '/*-inc.*'))
+	# (n.b. not counting size of new-signatures files because they are relatively small)
+	if inc_sz > full_sz * 1.5:
+		full_backup = True
+
 # Stop services.
 shell('check_call', ["/usr/sbin/service", "dovecot", "stop"])
 shell('check_call', ["/usr/sbin/service", "postfix", "stop"])
@@ -65,19 +79,6 @@ shell('check_call', [
 	"file://" + backup_duplicity_dir
 	])
 
-# Remove old increments. This deletes incremental data obsoleted by
-# any subsequent full backups.
-shell('check_call', [
-	"/usr/bin/duplicity",
-	"remove-all-inc-of-but-n-full",
-	"1",
-	"--archive-dir", "/tmp/duplicity-archive-dir",
-	"--name", "mailinabox",
-	"--force",
-	"--verbosity", "warning",
-	"file://" + backup_duplicity_dir
-	])
-
 # Remove duplicity's cache directory because it's redundant with our backup directory.
 shutil.rmtree("/tmp/duplicity-archive-dir")
 
@@ -108,3 +109,12 @@ for fn in os.listdir(backup_encrypted_dir):
 	fn2 = os.path.join(backup_duplicity_dir, fn.replace(".enc", ""))
 	if os.path.exists(fn2): continue
 	os.unlink(os.path.join(backup_encrypted_dir, fn))
+
+# Execute a post-backup script that does the copying to a remote server.
+# Run as the STORAGE_USER user, not as root. Pass our settings in
+# environment variables so the script has access to STORAGE_ROOT.
+post_script = os.path.join(backup_dir, 'after-backup')
+if os.path.exists(post_script):
+	shell('check_call',
+		['su', env['STORAGE_USER'], '-c', post_script],
+		env=env)

management/daemon.py

@@ -1,92 +1,214 @@
 #!/usr/bin/python3
 
-import os, os.path, re
+import os, os.path, re, json
 
-from flask import Flask, request, render_template, abort
-app = Flask(__name__)
+from functools import wraps
+
+from flask import Flask, request, render_template, abort, Response
 
 import auth, utils
-from mailconfig import get_mail_users, add_mail_user, set_mail_password, remove_mail_user, get_mail_aliases, get_mail_domains, add_mail_alias, remove_mail_alias
+from mailconfig import get_mail_users, add_mail_user, set_mail_password, remove_mail_user, get_archived_mail_users
+from mailconfig import get_mail_user_privileges, add_remove_mail_user_privilege
+from mailconfig import get_mail_aliases, get_mail_domains, add_mail_alias, remove_mail_alias
 
 env = utils.load_environment()
 
 auth_service = auth.KeyAuthService()
 
-@app.before_request
-def require_auth_key():
-	if not auth_service.is_authenticated(request):
-		abort(401)
+# We may deploy via a symbolic link, which confuses flask's template finding.
+me = __file__
+try:
+	me = os.readlink(__file__)
+except OSError:
+	pass
+
+app = Flask(__name__, template_folder=os.path.abspath(os.path.join(os.path.dirname(me), "templates")))
+
+# Decorator to protect views that require authentication.
+def authorized_personnel_only(viewfunc):
+	@wraps(viewfunc)
+	def newview(*args, **kwargs):
+		# Check if the user is authorized.
+		authorized_status = auth_service.is_authenticated(request, env)
+		if authorized_status == "OK":
+			# Authorized. Call view func.	
+			return viewfunc(*args, **kwargs)
+
+		# Not authorized. Return a 401 (send auth) and a prompt to authorize by default.
+		status = 401
+		headers = { 'WWW-Authenticate': 'Basic realm="{0}"'.format(auth_service.auth_realm) }
+
+		if request.headers.get('X-Requested-With') == 'XMLHttpRequest':
+			# Don't issue a 401 to an AJAX request because the user will
+			# be prompted for credentials, which is not helpful.
+			status = 403
+			headers = None
+
+		if request.headers.get('Accept') in (None, "", "*/*"):
+			# Return plain text output.
+			return Response(authorized_status+"\n", status=status, mimetype='text/plain', headers=headers)
+		else:
+			# Return JSON output.
+			return Response(json.dumps({
+				"status": "error",
+				"reason": authorized_status
+				}+"\n"), status=status, mimetype='application/json', headers=headers)
+
+	return newview
 
 @app.errorhandler(401)
 def unauthorized(error):
 	return auth_service.make_unauthorized_response()
 
+def json_response(data):
+	return Response(json.dumps(data), status=200, mimetype='application/json')
+
+###################################
+
+# Control Panel (unauthenticated views)
+
 @app.route('/')
 def index():
-    return render_template('index.html')
+	# Render the control panel. This route does not require user authentication
+	# so it must be safe!
+	return render_template('index.html',
+		hostname=env['PRIMARY_HOSTNAME'],
+	)
+
+@app.route('/me')
+def me():
+	# Is the caller authorized?
+	authorized_status = auth_service.is_authenticated(request, env)
+	if authorized_status != "OK":
+		return json_response({
+			"status": "not-authorized",
+			"reason": authorized_status,
+			})
+	return json_response({
+		"status": "authorized",
+		"api_key": auth_service.key,
+		})
 
 # MAIL
 
 @app.route('/mail/users')
+@authorized_personnel_only
 def mail_users():
-    return "".join(x+"\n" for x in get_mail_users(env))
+	if request.args.get("format", "") == "json":
+		return json_response(get_mail_users(env, as_json=True) + get_archived_mail_users(env))
+	else:
+		return "".join(x+"\n" for x in get_mail_users(env))
 
 @app.route('/mail/users/add', methods=['POST'])
+@authorized_personnel_only
 def mail_users_add():
-	return add_mail_user(request.form.get('email', ''), request.form.get('password', ''), env)
+	return add_mail_user(request.form.get('email', ''), request.form.get('password', ''), request.form.get('privileges', ''), env)
 
 @app.route('/mail/users/password', methods=['POST'])
+@authorized_personnel_only
 def mail_users_password():
 	return set_mail_password(request.form.get('email', ''), request.form.get('password', ''), env)
 
 @app.route('/mail/users/remove', methods=['POST'])
+@authorized_personnel_only
 def mail_users_remove():
 	return remove_mail_user(request.form.get('email', ''), env)
 
+
+@app.route('/mail/users/privileges')
+@authorized_personnel_only
+def mail_user_privs():
+	privs = get_mail_user_privileges(request.args.get('email', ''), env)
+	if isinstance(privs, tuple): return privs # error
+	return "\n".join(privs)
+
+@app.route('/mail/users/privileges/add', methods=['POST'])
+@authorized_personnel_only
+def mail_user_privs_add():
+	return add_remove_mail_user_privilege(request.form.get('email', ''), request.form.get('privilege', ''), "add", env)
+
+@app.route('/mail/users/privileges/remove', methods=['POST'])
+@authorized_personnel_only
+def mail_user_privs_remove():
+	return add_remove_mail_user_privilege(request.form.get('email', ''), request.form.get('privilege', ''), "remove", env)
+
+
 @app.route('/mail/aliases')
+@authorized_personnel_only
 def mail_aliases():
-    return "".join(x+"\t"+y+"\n" for x, y in get_mail_aliases(env))
+	if request.args.get("format", "") == "json":
+		return json_response(get_mail_aliases(env, as_json=True))
+	else:
+		return "".join(x+"\t"+y+"\n" for x, y in get_mail_aliases(env))
 
 @app.route('/mail/aliases/add', methods=['POST'])
+@authorized_personnel_only
 def mail_aliases_add():
-	return add_mail_alias(request.form.get('source', ''), request.form.get('destination', ''), env)
+	return add_mail_alias(
+		request.form.get('source', ''),
+		request.form.get('destination', ''),
+		env,
+		update_if_exists=(request.form.get('update_if_exists', '') == '1')
+		)
 
 @app.route('/mail/aliases/remove', methods=['POST'])
+@authorized_personnel_only
 def mail_aliases_remove():
 	return remove_mail_alias(request.form.get('source', ''), env)
 
 @app.route('/mail/domains')
+@authorized_personnel_only
 def mail_domains():
     return "".join(x+"\n" for x in get_mail_domains(env))
 
 # DNS
 
 @app.route('/dns/update', methods=['POST'])
+@authorized_personnel_only
 def dns_update():
 	from dns_update import do_dns_update
 	try:
-		return do_dns_update(env)
+		return do_dns_update(env, force=request.form.get('force', '') == '1')
 	except Exception as e:
 		return (str(e), 500)
 
-@app.route('/dns/ds')
-def dns_get_ds_records():
-	from dns_update import get_ds_records
-	try:
-		return get_ds_records(env).replace("\t", " ") # tabs confuse godaddy
-	except Exception as e:
-		return (str(e), 500)
+@app.route('/dns/dump')
+@authorized_personnel_only
+def dns_get_dump():
+	from dns_update import build_recommended_dns
+	return json_response(build_recommended_dns(env))
 
 # WEB
 
 @app.route('/web/update', methods=['POST'])
+@authorized_personnel_only
 def web_update():
 	from web_update import do_web_update
 	return do_web_update(env)
 
 # System
 
+@app.route('/system/status', methods=["POST"])
+@authorized_personnel_only
+def system_status():
+	from whats_next import run_checks
+	class WebOutput:
+		def __init__(self):
+			self.items = []
+		def add_heading(self, heading):
+			self.items.append({ "type": "heading", "text": heading, "extra": [] })
+		def print_ok(self, message):
+			self.items.append({ "type": "ok", "text": message, "extra": [] })
+		def print_error(self, message):
+			self.items.append({ "type": "error", "text": message, "extra": [] })
+		def print_line(self, message, monospace=False):
+			self.items[-1]["extra"].append({ "text": message, "monospace": monospace })
+	output = WebOutput()
+	run_checks(env, output)
+	return json_response(output.items)
+
 @app.route('/system/updates')
+@authorized_personnel_only
 def show_updates():
 	utils.shell("check_call", ["/usr/bin/apt-get", "-qq", "update"])
 	simulated_install = utils.shell("check_output", ["/usr/bin/apt-get", "-qq", "-s", "upgrade"])
@@ -98,6 +220,7 @@ def show_updates():
 	return "\n".join(pkgs)
 
 @app.route('/system/update-packages', methods=["POST"])
+@authorized_personnel_only
 def do_updates():
 	utils.shell("check_call", ["/usr/bin/apt-get", "-qq", "update"])
 	return utils.shell("check_output", ["/usr/bin/apt-get", "-y", "upgrade"], env={
@@ -108,6 +231,7 @@ def do_updates():
 
 if __name__ == '__main__':
 	if "DEBUG" in os.environ: app.debug = True
+	if "APIKEY" in os.environ: auth_service.key = os.environ["APIKEY"]
 
 	if not app.debug:
 		app.logger.addHandler(utils.create_syslog_handler())

management/dns_update.py

@@ -54,7 +54,7 @@ def get_custom_dns_config(env):
 	except:
 		return { }
 
-def do_dns_update(env):
+def do_dns_update(env, force=False):
 	# What domains (and their zone filenames) should we build?
 	domains = get_dns_domains(env)
 	zonefiles = get_dns_zones(env)
@@ -71,7 +71,7 @@ def do_dns_update(env):
 
 		# See if the zone has changed, and if so update the serial number
 		# and write the zone file.
-		if not write_nsd_zone(domain, "/etc/nsd/zones/" + zonefile, records, env):
+		if not write_nsd_zone(domain, "/etc/nsd/zones/" + zonefile, records, env, force):
 			# Zone was not updated. There were no changes.
 			continue
 
@@ -93,7 +93,7 @@ def do_dns_update(env):
 		# Thus we only sign a zone if write_nsd_zone returned True
 		# indicating the zone changed, and thus it got a new serial number.
 		# write_nsd_zone is smart enough to check if a zone's signature
-		# is nearing experiation and if so it'll bump the serial number
+		# is nearing expiration and if so it'll bump the serial number
 		# and return True so we get a chance to re-sign it.
 		sign_zone(domain, zonefile, env)
 
@@ -104,7 +104,7 @@ def do_dns_update(env):
 		zonefiles[i][1] += ".signed"
 
 	# Write the main nsd.conf file.
-	if write_nsd_conf(zonefiles):
+	if write_nsd_conf(zonefiles, env):
 		# Make sure updated_domains contains *something* if we wrote an updated
 		# nsd.conf so that we know to restart nsd.
 		if len(updated_domains) == 0:
@@ -115,10 +115,12 @@ def do_dns_update(env):
 		shell('check_call', ["/usr/sbin/service", "nsd", "restart"])
 
 	# Write the OpenDKIM configuration tables.
-	write_opendkim_tables(zonefiles, env)
-
-	# Kick opendkim.
-	shell('check_call', ["/usr/sbin/service", "opendkim", "restart"])
+	if write_opendkim_tables(zonefiles, env):
+		# Settings changed. Kick opendkim.
+		shell('check_call', ["/usr/sbin/service", "opendkim", "restart"])
+		if len(updated_domains) == 0:
+			# If this is the only thing that changed?
+			updated_domains.append("OpenDKIM configuration")
 
 	if len(updated_domains) == 0:
 		# if nothing was updated (except maybe OpenDKIM's files), don't show any output
@@ -158,11 +160,11 @@ def build_zone(domain, all_domains, additional_records, env, is_zone=True):
 		records.append(("_25._tcp", "TLSA", build_tlsa_record(env), "Recommended when DNSSEC is enabled. Advertises to mail servers connecting to the box that mandatory encryption should be used."))
 
 	# The MX record says where email for the domain should be delivered: Here!
-	records.append((None,  "MX",  "10 %s." % env["PRIMARY_HOSTNAME"], "Required. Specifies the hostname of the machine that handles @%s mail." % domain))
+	records.append((None,  "MX",  "10 %s." % env["PRIMARY_HOSTNAME"], "Required. Specifies the hostname (and priority) of the machine that handles @%s mail." % domain))
 
 	# SPF record: Permit the box ('mx', see above) to send mail on behalf of
 	# the domain, and no one else.
-	records.append((None,  "TXT", '"v=spf1 mx -all"', "Recomended. Specifies that only the box is permitted to send @%s mail." % domain))
+	records.append((None,  "TXT", '"v=spf1 mx -all"', "Recommended. Specifies that only the box is permitted to send @%s mail." % domain))
 
 	# Add DNS records for any subdomains of this domain. We should not have a zone for
 	# both a domain and one of its subdomains.
@@ -190,9 +192,9 @@ def build_zone(domain, all_domains, additional_records, env, is_zone=True):
 
 	# Add defaults if not overridden by the user's custom settings.
 	defaults = [
-		(None,  "A",    env["PUBLIC_IP"],       "Optional. Sets the IP address that %s resolves to, e.g. for web hosting." % domain),
+		(None,  "A",    env["PUBLIC_IP"],       "Optional. Sets the IP address that %s resolves to, e.g. for web hosting. (It is not necessary for receiving mail on this domain.)" % domain),
 		("www", "A",    env["PUBLIC_IP"],       "Optional. Sets the IP address that www.%s resolves to, e.g. for web hosting." % domain),
-		(None,  "AAAA", env.get('PUBLIC_IPV6'), "Optional. Sets the IPv6 address that %s resolves to, e.g. for web hosting." % domain),
+		(None,  "AAAA", env.get('PUBLIC_IPV6'), "Optional. Sets the IPv6 address that %s resolves to, e.g. for web hosting. (It is not necessary for receiving mail on this domain.)" % domain),
 		("www", "AAAA", env.get('PUBLIC_IPV6'), "Optional. Sets the IPv6 address that www.%s resolves to, e.g. for web hosting." % domain),
 	]
 	for qname, rtype, value, explanation in defaults:
@@ -207,7 +209,7 @@ def build_zone(domain, all_domains, additional_records, env, is_zone=True):
 		# Append the DKIM TXT record to the zone as generated by OpenDKIM, after string formatting above.
 		with open(opendkim_record_file) as orf:
 			m = re.match(r"(\S+)\s+IN\s+TXT\s+(\(.*\))\s*;", orf.read(), re.S)
-			records.append((m.group(1), "TXT", m.group(2), "Recommended. Specifies that only the box is permitted to send mail at this domain."))
+			records.append((m.group(1), "TXT", m.group(2), "Recommended. Provides a way for recipients to verify that this machine sent @%s mail." % domain))
 
 		# Append a DMARC record.
 		records.append(("_dmarc", "TXT", '"v=DMARC1; p=quarantine"', "Optional. Specifies that mail that does not originate from the box but claims to be from @%s is suspect and should be quarantined by the recipient's mail system." % domain))
@@ -288,7 +290,7 @@ def build_tlsa_record(env):
 
 ########################################################################
 
-def write_nsd_zone(domain, zonefile, records, env):
+def write_nsd_zone(domain, zonefile, records, env, force):
 	# We set the administrative email address for every domain to domain_contact@[domain.com].
 	# You should probably create an alias to your email address.
 
@@ -363,7 +365,7 @@ $TTL 86400           ; default time to live
 
 				# If the existing zone is the same as the new zone (modulo the serial number),
 				# there is no need to update the file. Unless we're forcing a bump.
-				if zone == existing_zone and not force_bump:
+				if zone == existing_zone and not force_bump and not force:
 					return False
 
 				# If the existing serial is not less than a serial number
@@ -383,7 +385,7 @@ $TTL 86400           ; default time to live
 
 ########################################################################
 
-def write_nsd_conf(zonefiles):
+def write_nsd_conf(zonefiles, env):
 	# Basic header.
 	nsdconf = """
 server:
@@ -397,15 +399,13 @@ server:
 """
 	
 	# Since we have bind9 listening on localhost for locally-generated
-	# DNS queries that require a recursive nameserver, we must have
-	# nsd listen only on public network interfaces. Those interfaces
-	# may have addresses different from the public IP address that the
-	# Internet sees this machine on. Get those interface addresses
-	# from `hostname -i` (which omits all localhost addresses).
-	for ipaddr in shell("check_output", ["/bin/hostname", "-I"]).strip().split(" "):
+	# DNS queries that require a recursive nameserver, and the system
+	# might have other network interfaces for e.g. tunnelling, we have
+	# to be specific about the network interfaces that nsd binds to.
+	for ipaddr in (env.get("PRIVATE_IP", "") + " " + env.get("PRIVATE_IPV6", "")).split(" "):
+		if ipaddr == "": continue
 		nsdconf += "  ip-address: %s\n" % ipaddr
 
-
 	# Append the zones.
 	for domain, zonefile in zonefiles:
 		nsdconf += """
@@ -480,13 +480,18 @@ def sign_zone(domain, zonefile, env):
 	# zone being signed, so we can't use the .ds files generated when we created the keys.
 	# The DS record points to the KSK only. Write this next to the zone file so we can
 	# get it later to give to the user with instructions on what to do with it.
-	rr_ds = shell('check_output', ["/usr/bin/ldns-key2ds",
-		"-n", # output to stdout
-		"-2", # SHA256
-		dnssec_keys["KSK"] + ".key"
-	])
+	#
+	# We want to be able to validate DS records too, but multiple forms may be valid depending
+	# on the digest type. So we'll write all (both) valid records. Only one DS record should
+	# actually be deployed. Preferebly the first.
 	with open("/etc/nsd/zones/" + zonefile + ".ds", "w") as f:
-		f.write(rr_ds)
+		for digest_type in ('2', '1'):
+			rr_ds = shell('check_output', ["/usr/bin/ldns-key2ds",
+				"-n", # output to stdout
+				"-" + digest_type, # 1=SHA1, 2=SHA256
+				dnssec_keys["KSK"] + ".key"
+			])
+			f.write(rr_ds)
 
 	# Remove our temporary file.
 	for fn in files_to_kill:
@@ -494,46 +499,55 @@ def sign_zone(domain, zonefile, env):
 	
 ########################################################################
 
-def get_ds_records(env):
-	zonefiles = get_dns_zones(env)
-	ret = ""
-	for domain, zonefile in zonefiles:
-		fn = "/etc/nsd/zones/" + zonefile + ".ds"
-		if os.path.exists(fn):
-			with open(fn, "r") as fr:
-				ret += fr.read().strip() + "\n"
-	return ret
-	
-	
-########################################################################
-
 def write_opendkim_tables(zonefiles, env):
 	# Append a record to OpenDKIM's KeyTable and SigningTable for each domain.
-	#
-	# The SigningTable maps email addresses to signing information. The KeyTable
-	# maps specify the hostname, the selector, and the path to the private key.
-	#
-	# DKIM ADSP and DMARC both only support policies where the signing domain matches
-	# the From address, so the KeyTable must specify that the signing domain for a
-	# sender matches the sender's domain.
-	#
-	# In SigningTable, we map every email address to a key record named after the domain.
-	# Then we specify for the key record its domain, selector, and key.
 
 	opendkim_key_file = os.path.join(env['STORAGE_ROOT'], 'mail/dkim/mail.private')
-	if not os.path.exists(opendkim_key_file): return
 
-	with open("/etc/opendkim/KeyTable", "w") as f:
-		f.write("\n".join(
-			"{domain} {domain}:mail:{key_file}".format(domain=domain, key_file=opendkim_key_file)
-			for domain, zonefile in zonefiles
-		))
+	if not os.path.exists(opendkim_key_file):
+		# Looks like OpenDKIM is not installed.
+		return False
 
-	with open("/etc/opendkim/SigningTable", "w") as f:
-		f.write("\n".join(
-			"*@{domain} {domain}".format(domain=domain)
-			for domain, zonefile in zonefiles
-		))
+	config = {
+		# The SigningTable maps email addresses to a key in the KeyTable that
+		# specifies signing information for matching email addresses. Here we
+		# map each domain to a same-named key.
+		#
+		# Elsewhere we set the DMARC policy for each domain such that mail claiming
+		# to be From: the domain must be signed with a DKIM key on the same domain.
+		# So we must have a separate KeyTable entry for each domain.
+		"SigningTable":
+			"".join(
+				"*@{domain} {domain}\n".format(domain=domain)
+				for domain, zonefile in zonefiles
+			),
+
+		# The KeyTable specifies the signing domain, the DKIM selector, and the
+		# path to the private key to use for signing some mail. Per DMARC, the
+		# signing domain must match the sender's From: domain.
+		"KeyTable":
+			"".join(
+				"{domain} {domain}:mail:{key_file}\n".format(domain=domain, key_file=opendkim_key_file)
+				for domain, zonefile in zonefiles
+			),
+	}
+
+	did_update = False
+	for filename, content in config.items():
+		# Don't write the file if it doesn't need an update.
+		if os.path.exists("/etc/opendkim/" + filename):
+			with open("/etc/opendkim/" + filename) as f:
+				if f.read() == content:
+					continue
+
+		# The contents needs to change.
+		with open("/etc/opendkim/" + filename, "w") as f:
+			f.write(content)
+		did_update = True
+
+	# Return whether the files changed. If they didn't change, there's
+	# no need to kick the opendkim process.
+	return did_update
 
 ########################################################################
 
@@ -578,9 +592,8 @@ def justtestingdotemail(domain, records):
 
 ########################################################################
 
-if __name__ == "__main__":
-	from utils import load_environment
-	env = load_environment()
+def build_recommended_dns(env):
+	ret = []
 	domains = get_dns_domains(env)
 	zonefiles = get_dns_zones(env)
 	for domain, zonefile in zonefiles:
@@ -589,15 +602,32 @@ if __name__ == "__main__":
 		# remove records that we don't dislay
 		records = [r for r in records if r[3] is not False]
 
-		# put Required at the top
+		# put Required at the top, then Recommended, then everythiing else
 		records.sort(key = lambda r : 0 if r[3].startswith("Required.") else (1 if r[3].startswith("Recommended.") else 2))
 
-		# print
-		for qname, rtype, value, explanation in records:
-			print("; " + explanation)
-			if qname == None:
+		# expand qnames
+		for i in range(len(records)):
+			if records[i][0] == None:
 				qname = domain
 			else:
-				qname = qname + "." + domain
-			print(qname, rtype, value)
+				qname = records[i][0] + "." + domain
+
+			records[i] = {
+				"qname": qname,
+				"rtype": records[i][1],
+				"value": records[i][2],
+				"explanation": records[i][3],
+			}
+
+		# return
+		ret.append((domain, records))
+	return ret
+
+if __name__ == "__main__":
+	from utils import load_environment
+	env = load_environment()
+	for zone, records in build_recommended_dns(env):
+		for record in records:
+			print("; " + record['explanation'])
+			print(record['qname'], record['rtype'], record['value'], sep="\t")
 			print()

management/mailconfig.py

@@ -46,15 +46,83 @@ def open_database(env, with_connection=False):
 	else:
 		return conn, conn.cursor()
 
-def get_mail_users(env):
+def get_mail_users(env, as_json=False):
 	c = open_database(env)
-	c.execute('SELECT email FROM users')
-	return [row[0] for row in c.fetchall()]
+	c.execute('SELECT email, privileges FROM users')
 
-def get_mail_aliases(env):
+	# turn into a list of tuples, but sorted by domain & email address
+	users = { row[0]: row[1] for row in c.fetchall() } # make dict
+	users = [ (email, users[email]) for email in utils.sort_email_addresses(users.keys(), env) ]
+
+	if not as_json:
+		return [email for email, privileges in users]
+	else:
+		aliases = get_mail_alias_map(env)
+		return [
+			{
+				"email": email,
+				"privileges": parse_privs(privileges),
+				"status": "active",
+				"aliases": [
+					(alias, sorted(evaluate_mail_alias_map(alias, aliases, env)))
+					for alias in aliases.get(email.lower(), [])
+					]
+			}
+			for email, privileges in users
+			 ]
+
+def get_archived_mail_users(env):
+	real_users = set(get_mail_users(env))
+	root = os.path.join(env['STORAGE_ROOT'], 'mail/mailboxes')
+	ret = []
+	for domain_enc in os.listdir(root):
+		for user_enc in os.listdir(os.path.join(root, domain_enc)):
+			email = utils.unsafe_domain_name(user_enc) + "@" + utils.unsafe_domain_name(domain_enc)
+			if email in real_users: continue
+			ret.append({
+				"email": email, 
+				"privileges": "",
+				"status": "inactive"
+			})
+	return ret
+
+def get_mail_aliases(env, as_json=False):
 	c = open_database(env)
 	c.execute('SELECT source, destination FROM aliases')
-	return [(row[0], row[1]) for row in c.fetchall()]
+	aliases = { row[0]: row[1] for row in c.fetchall() } # make dict
+
+	# put in a canonical order: sort by domain, then by email address lexicographically
+	aliases = [ (source, aliases[source]) for source in utils.sort_email_addresses(aliases.keys(), env) ] # sort
+
+	# but put automatic aliases to administrator@ last
+	aliases.sort(key = lambda x : x[1] == get_system_administrator(env))
+
+	if as_json:
+		required_aliases = get_required_aliases(env)
+		aliases = [
+			{
+				"source": alias[0],
+				"destination": [d.strip() for d in alias[1].split(",")],
+				"required": alias[0] in required_aliases or alias[0] == get_system_administrator(env),
+			}
+			for alias in aliases
+		]
+	return aliases
+
+def get_mail_alias_map(env):
+	aliases = { }
+	for alias, targets in get_mail_aliases(env):
+		for em in targets.split(","):
+			em = em.strip().lower()
+			aliases.setdefault(em, []).append(alias)
+	return aliases
+
+def evaluate_mail_alias_map(email, aliases,  env):
+	ret = set()
+	for alias in aliases.get(email.lower(), []):
+		ret.add(alias)
+		ret |= evaluate_mail_alias_map(alias, aliases, env)
+	return ret
 
 def get_mail_domains(env, filter_aliases=lambda alias : True):
 	def get_domain(emailaddr):
@@ -64,10 +132,30 @@ def get_mail_domains(env, filter_aliases=lambda alias : True):
 		 + [get_domain(source) for source, target in get_mail_aliases(env) if filter_aliases((source, target)) ]
 		 )
 
-def add_mail_user(email, pw, env):
+def add_mail_user(email, pw, privs, env):
+	# validate email
+	if email.strip() == "":
+		return ("No email address provided.", 400)
 	if not validate_email(email, mode='user'):
 		return ("Invalid email address.", 400)
 
+	# validate password
+	if pw.strip() == "":
+		return ("No password provided.", 400)
+	if re.search(r"[\s]", pw):
+		return ("Passwords cannot contain spaces.", 400)
+	if len(pw) < 4:
+		return ("Passwords must be at least four characters.", 400)
+
+	# validate privileges
+	if privs is None or privs.strip() == "":
+		privs = []
+	else:
+		privs = privs.split("\n")
+		for p in privs:
+			validation = validate_privilege(p)
+			if validation: return validation
+
 	# get the database
 	conn, c = open_database(env, with_connection=True)
 
@@ -76,14 +164,17 @@ def add_mail_user(email, pw, env):
 
 	# add the user to the database
 	try:
-		c.execute("INSERT INTO users (email, password) VALUES (?, ?)", (email, pw))
+		c.execute("INSERT INTO users (email, password, privileges) VALUES (?, ?, ?)",
+			(email, pw, "\n".join(privs)))
 	except sqlite3.IntegrityError:
 		return ("User already exists.", 400)
 
 	# write databasebefore next step
 	conn.commit()
 
-	# Create the user's INBOX and Spam folders and subscribe them.
+	# Create the user's INBOX, Spam, and Drafts folders, and subscribe them.
+	# K-9 mail will poll every 90 seconds if a Drafts folder does not exist, so create it
+	# to avoid unnecessary polling.
 
 	# Check if the mailboxes exist before creating them. When creating a user that had previously
 	# been deleted, the mailboxes will still exist because they are still on disk.
@@ -94,8 +185,9 @@ def add_mail_user(email, pw, env):
 		conn.commit()
 		return ("Failed to initialize the user: " + e.output.decode("utf8"), 400)
 
-	if "INBOX" not in existing_mboxes: utils.shell('check_call', ["doveadm", "mailbox", "create", "-u", email, "-s", "INBOX"])
-	if "Spam" not in existing_mboxes: utils.shell('check_call', ["doveadm", "mailbox", "create", "-u", email, "-s", "Spam"])
+	for folder in ("INBOX", "Spam", "Drafts"):
+		if folder not in existing_mboxes:
+			utils.shell('check_call', ["doveadm", "mailbox", "create", "-u", email, "-s", folder])
 
 	# Update things in case any new domains are added.
 	return kick(env, "mail user added")
@@ -122,20 +214,85 @@ def remove_mail_user(email, env):
 	# Update things in case any domains are removed.
 	return kick(env, "mail user removed")
 
-def add_mail_alias(source, destination, env, do_kick=True):
+def parse_privs(value):
+	return [p for p in value.split("\n") if p.strip() != ""]
+
+def get_mail_user_privileges(email, env):
+	c = open_database(env)
+	c.execute('SELECT privileges FROM users WHERE email=?', (email,))
+	rows = c.fetchall()
+	if len(rows) != 1:
+		return ("That's not a user (%s)." % email, 400)
+	return parse_privs(rows[0][0])
+
+def validate_privilege(priv):
+	if "\n" in priv or priv.strip() == "":
+		return ("That's not a valid privilege (%s)." % priv, 400)
+	return None
+
+def add_remove_mail_user_privilege(email, priv, action, env):
+	# validate
+	validation = validate_privilege(priv)
+	if validation: return validation
+
+	# get existing privs, but may fail
+	privs = get_mail_user_privileges(email, env)
+	if isinstance(privs, tuple): return privs # error
+
+	# update privs set
+	if action == "add":
+		if priv not in privs:
+			privs.append(priv)
+	elif action == "remove":
+		privs = [p for p in privs if p != priv]
+	else:
+		return ("Invalid action.", 400)
+
+	# commit to database
+	conn, c = open_database(env, with_connection=True)
+	c.execute("UPDATE users SET privileges=? WHERE email=?", ("\n".join(privs), email))
+	if c.rowcount != 1:
+		return ("Something went wrong.", 400)
+	conn.commit()
+
+	return "OK"
+
+def add_mail_alias(source, destination, env, update_if_exists=False, do_kick=True):
+	# validate source
+	if source.strip() == "":
+		return ("No incoming email address provided.", 400)
 	if not validate_email(source, mode='alias'):
-		return ("Invalid email address.", 400)
+		return ("Invalid incoming email address (%s)." % source, 400)
+
+	# parse comma and \n-separated destination emails & validate
+	dests = []
+	for line in destination.split("\n"):
+		for email in line.split(","):
+			email = email.strip()
+			if email == "": continue
+			if not validate_email(email, mode='alias'):
+				return ("Invalid destination email address (%s)." % email, 400)
+			dests.append(email)
+	if len(destination) == 0:
+		return ("No destination email address(es) provided.", 400)
+	destination = ",".join(dests)
 
 	conn, c = open_database(env, with_connection=True)
 	try:
 		c.execute("INSERT INTO aliases (source, destination) VALUES (?, ?)", (source, destination))
+		return_status = "alias added"
 	except sqlite3.IntegrityError:
-		return ("Alias already exists (%s)." % source, 400)
+		if not update_if_exists:
+			return ("Alias already exists (%s)." % source, 400)
+		else:
+			c.execute("UPDATE aliases SET destination = ? WHERE source = ?", (destination, source))
+			return_status = "alias updated"
+
 	conn.commit()
 
 	if do_kick:
 		# Update things in case any new domains are added.
-		return kick(env, "alias added")
+		return kick(env, return_status)
 
 def remove_mail_alias(source, env, do_kick=True):
 	conn, c = open_database(env, with_connection=True)
@@ -148,6 +305,35 @@ def remove_mail_alias(source, env, do_kick=True):
 		# Update things in case any domains are removed.
 		return kick(env, "alias removed")
 
+def get_system_administrator(env):
+	return "administrator@" + env['PRIMARY_HOSTNAME']
+
+def get_required_aliases(env):
+	# These are the aliases that must exist.
+	aliases = set()
+
+	# The hostmaster aliase is exposed in the DNS SOA for each zone.
+	aliases.add("hostmaster@" + env['PRIMARY_HOSTNAME'])
+
+	# Get a list of domains we serve mail for, except ones for which the only
+	# email on that domain is a postmaster/admin alias to the administrator.
+	real_mail_domains = get_mail_domains(env,
+		filter_aliases = lambda alias : \
+			(not alias[0].startswith("postmaster@") \
+			 and not alias[0].startswith("admin@")) \
+			or alias[1] != get_system_administrator(env) \
+			)
+
+	# Create postmaster@ and admin@ for all domains we serve mail on.
+	# postmaster@ is assumed to exist by our Postfix configuration. admin@
+	# isn't anything, but it might save the user some trouble e.g. when
+	# buying an SSL certificate.
+	for domain in real_mail_domains:
+		aliases.add("postmaster@" + domain)
+		aliases.add("admin@" + domain)
+
+	return aliases
+
 def kick(env, mail_result=None):
 	results = []
 
@@ -156,50 +342,32 @@ def kick(env, mail_result=None):
 	if mail_result is not None:
 		results.append(mail_result + "\n")
 
-	# Create hostmaster@ for the primary domain if it does not already exist.
-	# Default the target to administrator@ which the user is responsible for
-	# setting and keeping up to date.
+	# Ensure every required alias exists.
 
 	existing_aliases = get_mail_aliases(env)
-
-	administrator = "administrator@" + env['PRIMARY_HOSTNAME']
+	required_aliases = get_required_aliases(env)
 
 	def ensure_admin_alias_exists(source):
 		# Does this alias exists?
 		for s, t in existing_aliases:
 			if s == source:
 				return
-
 		# Doesn't exist.
+		administrator = get_system_administrator(env)
 		add_mail_alias(source, administrator, env, do_kick=False)
 		results.append("added alias %s (=> %s)\n" % (source, administrator))
 
-	ensure_admin_alias_exists("hostmaster@" + env['PRIMARY_HOSTNAME'])
 
-	# Get a list of domains we serve mail for, except ones for which the only
-	# email on that domain is a postmaster/admin alias to the administrator.
+	for alias in required_aliases:
+		ensure_admin_alias_exists(alias)
 
-	real_mail_domains = get_mail_domains(env,
-		filter_aliases = lambda alias : \
-			(not alias[0].startswith("postmaster@") \
-			 and not alias[0].startswith("admin@")) \
-			or alias[1] != administrator \
-			)
-
-	# Create postmaster@ and admin@ for all domains we serve mail on.
-	# postmaster@ is assumed to exist by our Postfix configuration. admin@
-	# isn't anything, but it might save the user some trouble e.g. when
-	# buying an SSL certificate.
-	for domain in real_mail_domains:
-		ensure_admin_alias_exists("postmaster@" + domain)
-		ensure_admin_alias_exists("admin@" + domain)
-
-	# Remove auto-generated hostmaster/postmaster/admin on domains we no
+	# Remove auto-generated postmaster/admin on domains we no
 	# longer have any other email addresses for.
 	for source, target in existing_aliases:
 		user, domain = source.split("@")
-		if user in ("postmaster", "admin") and domain not in real_mail_domains \
-			and target == administrator:
+		if user in ("postmaster", "admin") \
+			and source not in required_aliases \
+			and target == get_system_administrator(env):
 			remove_mail_alias(source, env, do_kick=False)
 			results.append("removed alias %s (was to %s; domain no longer used for email)\n" % (source, target))
 

management/templates/aliases.html

@@ -0,0 +1,159 @@
+<style>
+#alias_table .actions > * { padding-right: 3px; }
+#alias_table .alias-required .remove { display: none }
+</style>
+
+<h2>Aliases</h2>
+
+<h3>Add a mail alias</h3>
+
+<p>Aliases are email forwarders. An alias can forward email to a <a href="javascript:show_panel('users')">mail user</a> or to any email address.</p>
+
+<form class="form-horizontal" role="form" onsubmit="do_add_alias(); return false;">
+  <div class="form-group">
+    <label for="addaliasEmail" class="col-sm-2 control-label">Email Address</label>
+    <div class="col-sm-10">
+      <input type="email" class="form-control" id="addaliasEmail" placeholder="Incoming Email Address">
+    </div>
+  </div>
+  <div class="form-group">
+    <label for="addaliasTargets" class="col-sm-2 control-label">Forward To</label>
+    <div class="col-sm-10">
+      <textarea class="form-control" rows="3" id="addaliasTargets" placeholder="Forward to these email addresses (one per line or separated by commas)"></textarea>
+    </div>
+  </div>
+  <div class="form-group">
+    <div class="col-sm-offset-2 col-sm-10">
+      <button id="add-alias-button" type="submit" class="btn btn-primary">Add</button>
+      <button id="alias-cancel" class="btn btn-default hidden" onclick="aliases_reset_form(); return false;">Cancel</button>
+    </div>
+  </div>
+</form>
+
+<h3>Existing mail aliases</h3>
+<table id="alias_table" class="table" style="width: auto">
+  <thead>
+    <tr>
+      <th></th>
+      <th>Email Address<br></th>
+      <th>Forwards To</th>
+    </tr>
+  </thead>
+  <tbody>
+  </tbody>
+</table>
+
+<p style="margin-top: 1.5em"><small>Hostmaster@, postmaster@, and admin@ email addresses are required on some domains.</small></p>
+
+<div style="display: none">
+  <table>
+  <tr id="alias-template">
+    <td class='actions'>
+        <a href="#" onclick="aliases_edit(this); return false;" class='edit' title="Edit Alias">
+          <span class="glyphicon glyphicon-pencil"></span>
+        </a>
+        <a href="#" onclick="aliases_remove(this); return false;" class='remove' title="Remove Alias">
+          <span class="glyphicon glyphicon-trash"></span>
+        </a>
+    </td>
+    <td class='email'> </td>
+    <td class='target'> </td>
+  </tr>
+  </table>
+</div>
+
+
+<script>
+function show_aliases() {
+  $('#alias_table tbody').html("<tr><td colspan='2' class='text-muted'>Loading...</td></tr>")
+  api(
+    "/mail/aliases",
+    "GET",
+    { format: 'json' },
+    function(r) {
+      $('#alias_table tbody').html("");
+      for (var i = 0; i < r.length; i++) {
+        var n = $("#alias-template").clone();
+        n.attr('id', '');
+
+        if (r[i].required) n.addClass('alias-required');
+        n.attr('data-email', r[i].source);
+        n.find('td.email').text(r[i].source)
+        for (var j = 0; j < r[i].destination.length; j++)
+          n.find('td.target').append($("<div></div>").text(r[i].destination[j]))
+        $('#alias_table tbody').append(n);
+      }
+    })
+}
+
+var is_alias_add_update = false;
+function do_add_alias() {
+  var title = (!is_alias_add_update) ? "Add Alias" : "Update Alias";
+  var email = $("#addaliasEmail").val();
+  var targets = $("#addaliasTargets").val();
+  api(
+    "/mail/aliases/add",
+    "POST",
+    {
+      update_if_exists: is_alias_add_update ? '1' : '0',
+      source: email,
+      destination: targets
+    },
+    function(r) {
+      // Responses are multiple lines of pre-formatted text.
+      show_modal_error(title, $("<pre/>").text(r));
+      show_aliases()
+      aliases_reset_form();
+    },
+    function(r) {
+      show_modal_error(title, r);
+    });
+  return false;
+}
+
+function aliases_reset_form() {
+  $("#addaliasEmail").prop('disabled', false);
+  $("#addaliasEmail").val('')
+  $("#addaliasTargets").val('')
+  $('#alias-cancel').addClass('hidden');
+  $('#add-alias-button').text('Add');
+  is_alias_add_update = false;
+}
+
+function aliases_edit(elem) {
+  var email = $(elem).parents('tr').attr('data-email');
+  var targetdivs = $(elem).parents('tr').find('.target div');
+  var targets = "";
+  for (var i = 0; i < targetdivs.length; i++)
+    targets += $(targetdivs[i]).text() + "\n";
+
+  is_alias_add_update = true;
+  $('#alias-cancel').removeClass('hidden');
+  $("#addaliasEmail").prop('disabled', true);
+  $("#addaliasEmail").val(email);
+  $("#addaliasTargets").val(targets);
+  $('#add-alias-button').text('Update');
+  $('body').animate({ scrollTop: 0 })
+}
+
+function aliases_remove(elem) {
+  var email = $(elem).parents('tr').attr('data-email');
+  show_modal_confirm(
+    "Remove Alias",
+    "Remove " + email + "?",
+    "Remove",
+    function() {
+      api(
+        "/mail/aliases/remove",
+        "POST",
+        {
+          source: email
+        },
+        function(r) {
+          // Responses are multiple lines of pre-formatted text.
+          show_modal_error("Remove User", $("<pre/>").text(r));
+          show_aliases();
+        });
+    });
+}
+</script>

management/templates/index.html

@@ -1,11 +1,341 @@
-<!doctype html>
-<html>
-	<head>
-		<title>Mail-in-a-Box Management Server</title>
-	</head>
-	<body>
-		<h1>Mail-in-a-Box Management Server</h1>
+<!DOCTYPE html>
+<!--[if lt IE 7]>      <html class="no-js lt-ie9 lt-ie8 lt-ie7"> <![endif]-->
+<!--[if IE 7]>         <html class="no-js lt-ie9 lt-ie8"> <![endif]-->
+<!--[if IE 8]>         <html class="no-js lt-ie9"> <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js"> <!--<![endif]-->
+    <head>
+        <meta charset="utf-8">
+        <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
+        <meta name="viewport" content="width=device-width">
 
-		<p>Use this server to issue commands to the Mail-in-a-Box management daemon.</p>
-	</body>
+        <title>{{hostname}} - Mail-in-a-Box Control Panel</title>
+
+        <meta name="robots" content="noindex, nofollow">
+
+        <link rel="stylesheet" href="//maxcdn.bootstrapcdn.com/bootstrap/3.2.0/css/bootstrap.min.css">
+        <style>
+            @import url(https://fonts.googleapis.com/css?family=Raleway:400,700);
+            @import url(https://fonts.googleapis.com/css?family=Ubuntu:300);
+
+	    html {
+		overflow-y: scroll;
+	    }
+
+            body {
+                padding-top: 50px;
+                padding-bottom: 20px;
+            }
+
+            p {
+              margin-bottom: 1.25em;
+            }
+
+            h1, h2, h3 {
+              font-family: Raleway, sans-serif;
+              font-weight: bold;
+            }
+
+            h2 {
+              margin: 1em 0;
+            }
+
+            h3 {
+              font-size: 130%;
+              border-bottom: 1px solid black;
+              padding-bottom: 3px;
+              margin-bottom: 13px;
+              margin-top: 26px;
+            }
+
+            .panel {
+              display: none;
+            }
+
+            table.table {
+              margin: 1.5em 0;
+            }
+        </style>
+        <link rel="stylesheet" href="//maxcdn.bootstrapcdn.com/bootstrap/3.2.0/css/bootstrap-theme.min.css">
+        <style>
+        </style>
+    </head>
+    <body>
+        <!--[if lt IE 7]>
+            <p class="chromeframe">You are using an <strong>outdated</strong> browser. Please <a href="http://browsehappy.com/">upgrade your browser</a> or <a href="http://www.google.com/chromeframe/?redirect=true">activate Google Chrome Frame</a> to improve your experience.</p>
+        <![endif]-->
+    <div class="navbar navbar-inverse navbar-fixed-top">
+      <div class="container">
+        <div class="navbar-header">
+          <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+          </button>
+          <a class="navbar-brand" href="#">{{hostname}}</a>
+        </div>
+        <div class="navbar-collapse collapse">
+          <ul class="nav navbar-nav">
+            <li class="dropdown active">
+              <a href="#" class="dropdown-toggle" data-toggle="dropdown">System <b class="caret"></b></a>
+              <ul class="dropdown-menu">
+                <li><a href="#system_status" onclick="return show_panel(this);">Status Checks</a></li>
+                <li><a href="#system_external_dns" onclick="return show_panel(this);">External DNS (Advanced)</a></li>
+              </ul>
+            </li>
+            <li class="dropdown active">
+              <a href="#" class="dropdown-toggle" data-toggle="dropdown">Mail <b class="caret"></b></a>
+              <ul class="dropdown-menu">
+                <li><a href="#mail-guide" onclick="return show_panel(this);">Instructions</a></li>
+                <li><a href="#users" onclick="return show_panel(this);">Users</a></li>
+                <li><a href="#aliases" onclick="return show_panel(this);">Aliases</a></li>
+                <!--<li><a href="#">Another action</a></li>
+                <li><a href="#">Something else here</a></li>
+                <li class="divider"></li>
+                <li class="dropdown-header">Nav header</li>
+                <li><a href="#">Separated link</a></li>
+                <li><a href="#">One more separated link</a></li>-->
+              </ul>
+            </li>
+          </ul>
+          <ul class="nav navbar-nav navbar-right">
+            <li><a href="#" onclick="do_logout(); return false;" style="color: white">Log out?</a></li>
+          </ul>
+        </div><!--/.navbar-collapse -->
+      </div>
+    </div>
+
+    <div class="container-fluid">
+      <div id="panel_system_status" class="container panel">
+      {% include "system-status.html" %}
+      </div>
+
+      <div id="panel_system_external_dns" class="container panel">
+      {% include "system-external-dns.html" %}
+      </div>
+
+      <div id="panel_login" class="panel">
+      {% include "login.html" %}
+      </div>
+
+      <div id="panel_mail-guide" class="container panel">
+      {% include "mail-guide.html" %}
+      </div>
+
+      <div id="panel_users" class="container panel">
+      {% include "users.html" %}
+      </div>
+
+      <div id="panel_aliases" class="container panel">
+      {% include "aliases.html" %}
+      </div>
+
+      <hr>
+
+      <footer>
+        <p>This is a <a href="https://mailinabox.email">Mail-in-a-Box</a>.</p>
+      </footer>
+    </div> <!-- /container -->
+
+        <div id="ajax_loading_indicator" style="display: none; position: absolute; left: 0; top: 0; width: 100%; height: 100%; text-align: center; background-color: rgba(255,255,255,.75)">
+          <div style="margin: 20% auto">
+            <div><span class="glyphicon glyphicon-time"></span></div>
+            <div>Loading...</div>
+          </div>
+        </div>
+
+        <div id="global_modal" class="modal fade" tabindex="-1" role="dialog" aria-labelledby="errorModalTitle" aria-hidden="true">
+          <div class="modal-dialog modal-sm">
+            <div class="modal-content">
+              <div class="modal-header">
+                <button type="button" class="close" data-dismiss="modal" aria-hidden="true">&times;</button>
+                <h4 class="modal-title" id="errorModalTitle"> </h4>
+              </div>
+              <div class="modal-body">
+                <p> </p>
+              </div>
+              <div class="modal-footer">
+                <button type="button" class="btn btn-default" data-dismiss="modal">OK</button>
+                <button type="button" class="btn btn-danger" data-dismiss="modal">Yes</button>
+              </div>
+            </div>
+          </div>
+        </div>
+
+        <script src="//ajax.googleapis.com/ajax/libs/jquery/1.10.1/jquery.min.js"></script>
+        <script src="//maxcdn.bootstrapcdn.com/bootstrap/3.2.0/js/bootstrap.min.js"></script>
+
+        <script>
+var global_modal_state = null;
+var global_modal_funcs = null;
+
+$(function() {
+  $('#global_modal .btn-danger').click(function() {
+    // Don't take action now. Wait for the modal to be totally hidden
+    // so that we don't attempt to show another modal while this one
+    // is closing.
+    global_modal_state = 0; // OK
+  })
+  $('#global_modal .btn-default').click(function() {
+    global_modal_state = 1; // Cancel
+  })
+  $('#global_modal').on('hidden.bs.modal', function (e) {
+    // do the cancel function
+    if (global_modal_state == null) global_modal_state = 1; // cancel if the user hit ESC or clicked outside of the modal
+    if (global_modal_funcs && global_modal_funcs[global_modal_state])
+      global_modal_funcs[global_modal_state]();
+  })  
+})
+
+function show_modal_error(title, message, callback) {
+  $('#global_modal h4').text(title);
+  $('#global_modal .modal-body').html("<p/>");
+  if (typeof question == String) {
+    $('#global_modal p').text(message);
+    $('#global_modal .modal-dialog').addClass("modal-sm");
+  } else {
+    $('#global_modal p').html("").append(message);
+    $('#global_modal .modal-dialog').removeClass("modal-sm");
+  }
+  $('#global_modal .btn-default').show().text("OK");
+  $('#global_modal .btn-danger').hide();
+  global_modal_funcs = [callback, callback];
+  global_modal_state = null;
+  $('#global_modal').modal({});
+}
+
+function show_modal_confirm(title, question, verb, yes_callback, cancel_callback) {
+  $('#global_modal h4').text(title);
+  if (typeof question == String) {
+    $('#global_modal .modal-dialog').addClass("modal-sm");
+    $('#global_modal .modal-body').html("<p/>");
+    $('#global_modal p').text(question);
+  } else {
+    $('#global_modal .modal-dialog').removeClass("modal-sm");
+    $('#global_modal .modal-body').html("").append(question);
+  }
+  $('#global_modal .btn-default').show().text("Cancel");
+  $('#global_modal .btn-danger').show().text(verb);
+  global_modal_funcs = [yes_callback, cancel_callback];
+  global_modal_state = null;
+  $('#global_modal').modal({});
+}
+
+var is_ajax_loading = false;
+function ajax(options) {
+  setTimeout("if (is_ajax_loading) $('#ajax_loading_indicator').fadeIn()", 100);
+  function hide_loading_indicator() {
+    is_ajax_loading = false;
+    $('#ajax_loading_indicator').hide();
+  }
+  var old_success = options.success;
+  var old_error = options.error;
+  options.success = function(data) {
+    hide_loading_indicator();
+    if (data.status == "error")
+      show_modal_error("Error", data.message);
+    else if (old_success)
+      old_success(data);
+  };
+  options.error = function(jqxhr) {
+    hide_loading_indicator();
+    if (!old_error) 
+      show_modal_error("Error", "Something went wrong, sorry.")
+    else
+      old_error(jqxhr.responseText);
+  };
+  is_ajax_loading = true;
+  $.ajax(options);
+}
+
+var api_credentials = ["", ""];
+function api(url, method, data, callback, callback_error) {
+  // from http://www.webtoolkit.info/javascript-base64.html
+  function base64encode(input) {
+    _keyStr = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=";
+    var output = "";
+    var chr1, chr2, chr3, enc1, enc2, enc3, enc4;
+    var i = 0;
+    while (i < input.length) {
+        chr1 = input.charCodeAt(i++);
+        chr2 = input.charCodeAt(i++);
+        chr3 = input.charCodeAt(i++);
+        enc1 = chr1 >> 2;
+        enc2 = ((chr1 & 3) << 4) | (chr2 >> 4);
+        enc3 = ((chr2 & 15) << 2) | (chr3 >> 6);
+        enc4 = chr3 & 63;
+        if (isNaN(chr2)) {
+            enc3 = enc4 = 64;
+        } else if (isNaN(chr3)) {
+            enc4 = 64;
+        }
+        output = output +
+        _keyStr.charAt(enc1) + _keyStr.charAt(enc2) +
+        _keyStr.charAt(enc3) + _keyStr.charAt(enc4);
+    }
+
+    return output;
+  }
+
+  ajax({
+    url: "/admin" + url,
+    method: method,
+    data: data,
+    beforeSend: function(xhr) {
+      // We don't store user credentials in a cookie to avoid the hassle of CSRF
+      // attacks. The Authorization header only gets set in our AJAX calls triggered
+      // by user actions.
+      xhr.setRequestHeader(
+        'Authorization',
+        'Basic ' + base64encode(api_credentials[0] + ':' + api_credentials[1]));
+    },
+    success: callback,
+    error: callback_error,
+    statusCode: {
+      403: function(xhr) {
+        // Credentials are no longer valid. Try to login again.
+        var p = current_panel;
+        show_panel('login');
+        switch_back_to_panel = p;
+      }
+    }
+  })
+}
+
+var current_panel = null;
+var switch_back_to_panel = null;
+function show_panel(panelid) {
+  if (panelid.getAttribute)
+    // we might be passed an HTMLElement <a>.
+    panelid = panelid.getAttribute('href').substring(1);
+
+  $('.panel').hide();
+  $('#panel_' + panelid).show();
+  if (typeof localStorage != 'undefined')
+    localStorage.setItem("miab-cp-lastpanel", panelid);
+  if (window["show_" + panelid])
+    window["show_" + panelid]();
+
+  current_panel = panelid;
+  switch_back_to_panel = null;
+
+  return false; // when called from onclick, cancel navigation
+}
+
+$(function() {
+  // Recall saved user credentials.
+  if (typeof sessionStorage != 'undefined' && sessionStorage.getItem("miab-cp-credentials"))
+    api_credentials = sessionStorage.getItem("miab-cp-credentials").split(":");
+  else if (typeof localStorage != 'undefined' && localStorage.getItem("miab-cp-credentials"))
+    api_credentials = localStorage.getItem("miab-cp-credentials").split(":");
+
+  // Recall what the user was last looking at.
+  if (typeof localStorage != 'undefined' && localStorage.getItem("miab-cp-lastpanel")) {
+    show_panel(localStorage.getItem("miab-cp-lastpanel"));
+  } else {
+    show_panel('login');
+  }
+})
+
+        </script>
+    </body>
 </html>

management/templates/login.html

@@ -0,0 +1,102 @@
+<div class="row">
+  <div class="col-sm-offset-2 col-sm-8 col-md-offset-3 col-md-6 col-lg-offset-4 col-lg-4">
+    <center>
+      <h1 style="margin: 1em">{{hostname}}</h1>
+      <p style="margin: 2em">Log in here for your Mail-in-a-Box control panel.</p>
+    </center>
+
+  <form class="form-horizontal" role="form" onsubmit="do_login(); return false;">
+    <div class="form-group">
+      <label for="inputEmail3" class="col-sm-2 control-label">Email</label>
+      <div class="col-sm-10">
+        <input name="email" type="email" class="form-control" id="loginEmail" placeholder="Email">
+      </div>
+    </div>
+    <div class="form-group">
+      <label for="inputPassword3" class="col-sm-2 control-label">Password</label>
+      <div class="col-sm-10">
+        <input name="password" type="password" class="form-control" id="loginPassword" placeholder="Password">
+      </div>
+    </div>
+    <div class="form-group">
+      <div class="col-sm-offset-2 col-sm-10">
+        <div class="checkbox">
+          <label>
+            <input name='remember' type="checkbox" id="loginRemember"> Remember me
+          </label>
+        </div>
+      </div>
+    </div>
+    <div class="form-group">
+      <div class="col-sm-offset-2 col-sm-10">
+        <button type="submit" class="btn btn-default">Sign in</button>
+      </div>
+    </div>
+  </form>
+  </div>
+</div>
+
+<script>
+function do_login() {
+  if ($('#loginEmail').val() == "") {
+    show_modal_error("Login Failed", "Enter your email address.")
+    return false;
+  }
+  if ($('#loginPassword').val() == "") {
+    show_modal_error("Login Failed", "Enter your email password.")
+    return false;
+  }
+
+  // Exchange the email address & password for an API key.
+  api_credentials = [$('#loginEmail').val(), $('#loginPassword').val()]
+
+  api(
+  "/me",
+  "GET",
+  { },
+  function(response){ 
+    // This API call always succeeds. It returns a JSON object indicating
+    // whether the request was authenticated or not.
+    if (response.status != "authorized") {
+      // Show why the login failed.
+      show_modal_error("Login Failed", response.reason)
+
+      // Reset any saved credentials.
+      do_logout();
+
+    } else {
+      // Login succeeded.
+
+      // Save the new credentials.
+      api_credentials = [response.api_key, ""];
+
+      // Try to wipe the username/password information.
+      $('#loginEmail').val('');
+      $('#loginPassword').val('');
+
+      // Remember the credentials.
+      if (typeof localStorage != 'undefined' && typeof sessionStorage != 'undefined') {
+        if ($('#loginRemember').val()) {
+          localStorage.setItem("miab-cp-credentials", api_credentials.join(":"));
+          sessionStorage.removeItem("miab-cp-credentials");
+        } else {
+          localStorage.removeItem("miab-cp-credentials");
+          sessionStorage.setItem("miab-cp-credentials", api_credentials.join(":"));
+        }
+      }
+
+      // Open the next panel the user wants to go to.
+      show_panel(!switch_back_to_panel ? 'system_status' : switch_back_to_panel)
+    }
+  })
+}
+
+function do_logout() {
+  api_credentials = ["", ""];
+  if (typeof localStorage != 'undefined')
+    localStorage.removeItem("miab-cp-credentials");
+  if (typeof sessionStorage != 'undefined')
+    sessionStorage.removeItem("miab-cp-credentials");
+  show_panel('login');
+}
+</script>

management/templates/mail-guide.html

@@ -0,0 +1,41 @@
+<div class="container">
+	<h2>Checking and Sending Mail</h2>
+
+	<h4>App Configuration</h4>
+
+	<p>You can access your email using webmail, desktop mail clients, or mobile apps.</p>
+
+	<p>Here is what you need to know for webmail:</p>
+
+	<style>#panel_mail-guide table.table { width: auto; margin-left: 1.5em; }</style>
+
+	<table class="table">
+	<tr><th>Webmail Address:</th> <td><a href="https://{{hostname}}/mail"><b>https://{{hostname}}/mail</b></a></td></tr>
+	<tr><th>Username:</th> <td>Your whole email address.</td></tr>
+	<tr><th>Password:</th> <td>Your mail password.</td></tr>
+	</table>
+
+	<p>On mobile devices you might need to install a &ldquo;mail client&rdquo; app. We recommend <a href="https://play.google.com/store/apps/details?id=com.fsck.k9">K-9 Mail</a>. On a desktop you could try <a href="https://www.mozilla.org/en-US/thunderbird/">Mozilla Thunderbird</a>.</p>
+
+	<p>Configure your device or desktop mail client as follows:</p>
+
+	<table class="table" style="max-width: 30em">
+	<tr><th>Server Name:</th> <td>{{hostname}}</td></tr>
+	<tr><th>Username:</th> <td>Your whole email address.</td></tr>
+	<tr><th>Password:</th> <td>Your mail password.</td></tr>
+	</table>
+
+
+	<table class="table">
+	<thead><tr><th>Protocol</th> <th>Port</th> <th>Options</th></tr></thead>
+	<tr><th>IMAP</th> <td>993</td> <td>SSL</td></tr>
+	<tr><th>SMTP</th> <td>587</td> <td>STARTTLS</td></tr>
+	<tr><th>Exchange ActiveSync</th> <td>n/a</td> <td>Secure Connection</td></tr>
+	</table>
+
+	<p>Depending on your mail program, you will use either IMAP &amp; SMTP or Exchange ActiveSync. See this <a href="http://z-push.org/compatibility/">list of compatible devices</a> for Exchange ActiveSync.</p>
+
+	<h4>Notes</h4>
+
+	<p>Mail-in-a-Box uses <a href="http://en.wikipedia.org/wiki/Greylisting">greylisting</a> to cut down on spam. The first time you receive an email from a recipient, it may be delayed for ten minutes.</p>
+</div>

management/templates/system-external-dns.html

@@ -0,0 +1,81 @@
+<style>
+#external_dns_settings .heading td {
+  font-weight: bold;
+  font-size: 120%;
+  padding-top: 1.5em;
+}
+#external_dns_settings .heading.first td {
+  border-top: none;
+  padding-top: 0;
+}
+#external_dns_settings .values td {
+	padding-top: .75em;
+	padding-bottom: 0;
+	max-width: 50vw;
+	word-wrap: break-word;
+}
+#external_dns_settings .explanation td {
+	border: 0;
+	padding-top: .5em;
+	padding-bottom: .75em;
+	font-style: italic;
+	color: #777;
+}
+</style>
+
+<h2>External DNS</h2>
+
+<p class="text-danger">This is for advanced configurations.</p>
+
+<h3>Overview</h3>
+
+<p>Although your box is configured to serve its own DNS, it is possible to host your DNS elsewhere. We do not recommend this.</p>
+
+<p>If you do so, you are responsible for keeping your DNS entries up to date. In particular DNSSEC entries must be re-signed periodically. Do not set a DS record at your registrar or publish DNSSEC entries in your DNS zones if you do not intend to keep them up to date.</p>
+
+<h3>DNS Settings</h3>
+
+<p>Enter the following DNS entries at your DNS provider:</p>
+
+<table id="external_dns_settings" class="table">
+	<thead>
+		<tr>
+			<th>QName</th>
+			<th>Type</th>
+			<th>Value</th>
+		</tr>
+	</thead>
+	<tbody>
+	</tbody>
+</table>
+
+<script>
+function show_system_external_dns() {
+  $('#external_dns_settings tbody').html("<tr><td colspan='2' class='text-muted'>Loading...</td></tr>")
+  api(
+    "/dns/dump",
+    "GET",
+    { },
+    function(zones) {
+      $('#external_dns_settings tbody').html("");
+      for (var j = 0; j < zones.length; j++) {
+        var h = $("<tr class='heading'><td colspan='3'></td></tr>");
+        h.find("td").text(zones[j][0]);
+        $('#external_dns_settings tbody').append(h);
+
+        var r = zones[j][1];
+        for (var i = 0; i < r.length; i++) {
+          var n = $("<tr class='values'><td class='qname'/><td class='rtype'/><td class='value'/></tr>");
+          n.find('.qname').text(r[i].qname);
+          n.find('.rtype').text(r[i].rtype);
+          n.find('.value').text(r[i].value);
+          $('#external_dns_settings tbody').append(n);
+
+          var n = $("<tr class='explanation'><td colspan='3'/></tr>");
+          n.find('td').text(r[i].explanation);
+          $('#external_dns_settings tbody').append(n);
+        }
+      }
+    })
+}
+</script>

management/templates/system-status.html

@@ -0,0 +1,79 @@
+<h2>System Status Checks</h2>
+
+<style>
+#system-checks .heading td {
+  font-weight: bold;
+  font-size: 120%;
+  padding-top: 1.5em;
+}
+#system-checks .heading.first td {
+  border-top: none;
+  padding-top: 0;
+}
+#system-checks .error td {
+  color: #733;
+}
+#system-checks .ok td {
+  color: #030;
+}
+#system-checks div.extra {
+  display: none;
+  margin-top: 1em;
+  max-width: 50em;
+  word-wrap: break-word;
+}
+#system-checks a.showhide {
+  display: none;
+  font-size: 85%;
+}
+#system-checks .pre {
+  margin: 1em;
+  font-family: monospace;
+  white-space: pre-wrap;
+}
+</style>
+
+<table id="system-checks" class="table" style="max-width: 60em">
+  <thead>
+  </thead>
+  <tbody>
+  </tbody>
+</table>
+
+<script>
+function show_system_status() {
+  $('#system-checks tbody').html("<tr><td colspan='2' class='text-muted'>Loading...</td></tr>")
+  api(
+    "/system/status",
+    "POST",
+    { },
+    function(r) {
+      $('#system-checks tbody').html("");
+      for (var i = 0; i < r.length; i++) {
+        var n = $("<tr><td class='status'/><td class='message'><p style='margin: 0'/><div class='extra'/><a class='showhide' href='#'/></tr>");
+        if (i == 0) n.addClass('first')
+        n.addClass(r[i].type)
+        if (r[i].type == "ok") n.find('td.status').text("✓")
+        if (r[i].type == "error") n.find('td.status').text("✖")
+        n.find('td.message p').text(r[i].text)
+        $('#system-checks tbody').append(n);
+
+        if (r[i].extra.length > 0) {
+          n.find('a.showhide').show().text("show more").click(function() {
+            $(this).hide();
+            $(this).parent().find('.extra').fadeIn();
+            return false;
+          });
+        }
+
+        for (var j = 0; j < r[i].extra.length; j++) {
+
+          var m = $("<div/>").text(r[i].extra[j].text)
+          if (r[i].extra[j].monospace)
+            m.addClass("pre");
+          n.find('> td.message > div').append(m);
+        }
+      }
+    })
+}
+</script>

management/templates/users.html

@@ -0,0 +1,196 @@
+<h2>Users</h2>
+
+<style>
+#user_table tr.account_inactive td .address { color: #888; text-decoration: line-through; }
+#user_table .aliases { margin: .25em 0 0 1em; font-size: 95%; }
+#user_table .aliases div:before { content: "⇖ "; }
+#user_table .aliases div {  }
+#user_table .actions { margin: .25em 0 0 1em; font-size: 95%; }
+#user_table .actions > * { display: none; }
+#user_table .account_active .actions a.archive { display: inline; }
+#user_table .account_inactive .actions .restore { display: inline; }
+</style>
+
+<h3>Add a mail user</h3>
+
+<p>Add an email address to this system. This will create a new login username/password. (Use <a href="javascript:show_panel('aliases')">aliases</a> to create email addresses that forward to existing accounts.)</p>
+
+<form class="form-inline" role="form" onsubmit="return do_add_user(); return false;">
+  <div class="form-group">
+    <label class="sr-only" for="adduserEmail">Email address</label>
+    <input type="email" class="form-control" id="adduserEmail" placeholder="Email Address">
+  </div>
+  <div class="form-group">
+    <label class="sr-only" for="adduserPassword">Password</label>
+    <input type="password" class="form-control" id="adduserPassword" placeholder="Password">
+  </div>
+  <div class="form-group">
+    <select class="form-control" id="adduserPrivs">
+      <option value="">Normal User</option>
+      <option value="admin">Administrator</option>
+    </select>
+  </div>
+  <button type="submit" class="btn btn-primary">Add User</button>
+</form>
+<p style="margin-top: .5em"><small>
+  Passwords must be at least four characters and may not contain spaces.
+  Administrators get access to this control panel.
+</small></p>
+
+<h3>Existing mail users</h3>
+<table id="user_table" class="table" style="width: auto">
+  <thead>
+    <tr>
+      <th></th>
+      <th>Email Address<br><small style="font-weight: normal">(Also the user&rsquo;s login username.)</small></th>
+      <th>Privileges</th>
+    </tr>
+  </thead>
+  <tbody>
+  </tbody>
+</table>
+
+<div style="display: none">
+  <table>
+  <tr id="user-template">
+    <td class='actions'>
+        <a href="#" onclick="users_remove(this); return false;" class='archive' title="Archive Account">
+          <span class="glyphicon glyphicon-trash"></span>
+        </a>
+    </td>
+    <td class='email'>
+      <div class='address'> </div>
+      <div class='aliases' style='display: none'> </div>
+      <div class='actions'>
+        <span class='restore'>To restore account, create a new account with this email address.</span>
+      </div>
+    </td>
+    <td class='privs'> </td>
+  </tr>
+  </table>
+</div>
+
+
+<script>
+function show_users() {
+  $('#user_table tbody').html("<tr><td colspan='2' class='text-muted'>Loading...</td></tr>")
+  api(
+    "/mail/users",
+    "GET",
+    { format: 'json' },
+    function(r) {
+      $('#user_table tbody').html("");
+      for (var i = 0; i < r.length; i++) {
+        var n = $("#user-template").clone();
+        n.attr('id', '');
+
+        n.addClass("account_" + r[i].status);
+        n.attr('data-email', r[i].email);
+        n.find('td.email .address').text(r[i].email)
+
+        var add_privs = ["admin"];
+
+        for (var j = 0; j < r[i].privileges.length; j++) {
+          var p = $("<div><span class='name'></span> <a href='#' onclick='mod_priv(this, \"remove\"); return false;'><span class=\"glyphicon glyphicon-trash\" style='font-size: 90%'></span></a></div>");
+          p.find('span.name').text(r[i].privileges[j]);
+          n.find('td.privs').append(p);
+          if (add_privs.indexOf(r[i].privileges[j]) >= 0)
+            add_privs.splice(add_privs.indexOf(r[i].privileges[j]), 1);
+        }
+
+        for (var j = 0; j < add_privs.length; j++) {
+          var p = $("<div><small><a href='#' onclick='mod_priv(this, \"add\"); return false;'><span class=\"glyphicon glyphicon-plus\" style='font-size: 90%'></span> <span class='name'></span>?</a></small></div>");
+          p.find('span.name').text(add_privs[j]);
+          n.find('td.privs').append(p);
+        }
+
+        if (r[i].aliases && r[i].aliases.length > 0) {
+          n.find('.aliases').show();
+          for (var j = 0; j < r[i].aliases.length; j++) {
+            n.find('td.email .aliases').append($("<div/>").text(
+              r[i].aliases[j][0]
+              + (r[i].aliases[j][1].length > 0 ? " ⇐ " + r[i].aliases[j][1].join(", ") : "")
+              ))
+          }
+        }
+        $('#user_table tbody').append(n);
+      }
+    })
+}
+
+function do_add_user() {
+  var email = $("#adduserEmail").val();
+  var pw = $("#adduserPassword").val();
+  var privs = $("#adduserPrivs").val();
+  api(
+    "/mail/users/add",
+    "POST",
+    {
+      email: email,
+      password: pw,
+      privileges: privs
+    },
+    function(r) {
+      // Responses are multiple lines of pre-formatted text.
+      show_modal_error("Add User", $("<pre/>").text(r));
+      show_users()
+    },
+    function(r) {
+      show_modal_error("Add User", r);
+    });
+  return false;
+}
+
+function users_remove(elem) {
+  var email = $(elem).parents('tr').attr('data-email');
+  show_modal_confirm(
+    "Archive User",
+    $("<p>Are you sure you want to archive " + email + "?</p> <p>The user's mailboxes will not be deleted (you can do that later), but the user will no longer be able to log into any services on this machine.</p>"),
+    "Archive",
+    function() {
+      api(
+        "/mail/users/remove",
+        "POST",
+        {
+          email: email
+        },
+        function(r) {
+          // Responses are multiple lines of pre-formatted text.
+          show_modal_error("Remove User", $("<pre/>").text(r));
+          show_users();
+        },
+        function(r) {
+          show_modal_error("Remove User", r);
+        });
+    });
+}
+
+function mod_priv(elem, add_remove) {
+  var email = $(elem).parents('tr').attr('data-email');
+  var priv = $(elem).parents('td').find('.name').text();
+
+  // can't remove your own admin access
+  if (priv == "admin" && add_remove == "remove" && api_credentials != null && email == api_credentials[0]) {
+    show_modal_error("Modify Privileges", "You cannot remove the admin privilege from yourself.");
+    return;
+  }
+
+  var add_remove1 = add_remove.charAt(0).toUpperCase() + add_remove.substring(1);
+  show_modal_confirm(
+    "Modify Privileges",
+    "Are you sure you want to " + add_remove + " the " + priv + " privilege for " + email + "?",
+    add_remove1,
+    function() {
+      api(
+        "/mail/users/privileges/" + add_remove,
+        "POST",
+        {
+          email: email,
+          privilege: priv
+        },
+        function(r) {
+          show_users();
+        });
+    });
+}
+</script>

management/utils.py

@@ -23,6 +23,10 @@ def safe_domain_name(name):
     import urllib.parse
     return urllib.parse.quote(name, safe='')
 
+def unsafe_domain_name(name_encoded):
+    import urllib.parse
+    return urllib.parse.unquote(name_encoded)
+
 def sort_domains(domain_names, env):
     # Put domain names in a nice sorted order. For web_update, PRIMARY_HOSTNAME
     # must appear first so it becomes the nginx default server.
@@ -51,6 +55,17 @@ def sort_domains(domain_names, env):
 
     return groups[0] + groups[1] + groups[2]
 
+def sort_email_addresses(email_addresses, env):
+    email_addresses = set(email_addresses)
+    domains = set(email.split("@", 1)[1] for email in email_addresses if "@" in email)
+    ret = []
+    for domain in sort_domains(domains, env):
+        domain_emails = set(email for email in email_addresses if email.endswith("@" + domain))
+        ret.extend(sorted(domain_emails))
+        email_addresses -= domain_emails
+    ret.extend(sorted(email_addresses)) # whatever is left
+    return ret
+
 def exclusive_process(name):
     # Ensure that a process named `name` does not execute multiple
     # times concurrently.

management/web_update.py

@@ -40,10 +40,13 @@ def get_web_domains(env):
 	
 def do_web_update(env):
 	# Build an nginx configuration file.
-	nginx_conf = ""
-	template = open(os.path.join(os.path.dirname(__file__), "../conf/nginx.conf")).read()
+	nginx_conf = open(os.path.join(os.path.dirname(__file__), "../conf/nginx-top.conf")).read()
+
+	# Add configuration for each web domain.
+	template1 = open(os.path.join(os.path.dirname(__file__), "../conf/nginx.conf")).read()
+	template2 = open(os.path.join(os.path.dirname(__file__), "../conf/nginx-primaryonly.conf")).read()
 	for domain in get_web_domains(env):
-		nginx_conf += make_domain_config(domain, template, env)
+		nginx_conf += make_domain_config(domain, template1, template2, env)
 
 	# Did the file change? If not, don't bother writing & restarting nginx.
 	nginx_conf_fn = "/etc/nginx/conf.d/local.conf"
@@ -56,12 +59,15 @@ def do_web_update(env):
 	with open(nginx_conf_fn, "w") as f:
 		f.write(nginx_conf)
 
-	# Kick nginx.
-	shell('check_call', ["/usr/sbin/service", "nginx", "restart"])
+	# Kick nginx. Since this might be called from the web admin
+	# don't do a 'restart'. That would kill the connection before
+	# the API returns its response. A 'reload' should be good
+	# enough and doesn't break any open connections.
+	shell('check_call', ["/usr/sbin/service", "nginx", "reload"])
 
 	return "web updated\n"
 
-def make_domain_config(domain, template, env):
+def make_domain_config(domain, template, template_for_primaryhost, env):
 	# How will we configure this domain.
 
 	# Where will its root directory be for static files?
@@ -75,25 +81,30 @@ def make_domain_config(domain, template, env):
 	# available. Make a self-signed one now if one doesn't exist.
 	ensure_ssl_certificate_exists(domain, ssl_key, ssl_certificate, csr_path, env)
 
+	# Put pieces together.
+	nginx_conf_parts = re.split("\s*# ADDITIONAL DIRECTIVES HERE\s*", template)
+	nginx_conf = nginx_conf_parts[0] + "\n"
+	if domain == env['PRIMARY_HOSTNAME']:
+		nginx_conf += template_for_primaryhost + "\n"
+
 	# Replace substitution strings in the template & return.
-	nginx_conf = template
+	nginx_conf = nginx_conf.replace("$STORAGE_ROOT", env['STORAGE_ROOT'])
 	nginx_conf = nginx_conf.replace("$HOSTNAME", domain)
 	nginx_conf = nginx_conf.replace("$ROOT", root)
 	nginx_conf = nginx_conf.replace("$SSL_KEY", ssl_key)
 	nginx_conf = nginx_conf.replace("$SSL_CERTIFICATE", ssl_certificate)
 
 	# Add in any user customizations.
-	nginx_conf_parts = re.split("(# ADDITIONAL DIRECTIVES HERE\n)", nginx_conf)
 	nginx_conf_custom_fn = os.path.join(env["STORAGE_ROOT"], "www/custom.yaml")
 	if os.path.exists(nginx_conf_custom_fn):
 		yaml = rtyaml.load(open(nginx_conf_custom_fn))
 		if domain in yaml:
 			yaml = yaml[domain]
-			if "proxy" in yaml:
-				nginx_conf_parts[1] += "\tlocation / {\n\t\tproxy_pass %s;\n\t}\n" % yaml["proxy"]
+			for path, url in yaml.get("proxies", {}).items():
+				nginx_conf += "\tlocation %s {\n\t\tproxy_pass %s;\n\t}\n" % (path, url)
 
-	# Put it all together.	
-	nginx_conf = "".join(nginx_conf_parts)
+	# Ending.
+	nginx_conf += nginx_conf_parts[1]
 
 	return nginx_conf
 

management/whats_next.py

@@ -10,36 +10,64 @@ import os, os.path, re, subprocess
 
 import dns.reversename, dns.resolver
 
-from dns_update import get_dns_zones
+from dns_update import get_dns_zones, build_tlsa_record
 from web_update import get_web_domains, get_domain_ssl_files
 from mailconfig import get_mail_domains, get_mail_aliases
 
-from utils import shell, sort_domains
+from utils import shell, sort_domains, load_env_vars_from_file
 
-def run_checks(env):
+def run_checks(env, output):
+	env["out"] = output
 	run_system_checks(env)
+	run_network_checks(env)
 	run_domain_checks(env)
 
 def run_system_checks(env):
-	print("System")
-	print("======")
+	env["out"].add_heading("System")
 
 	# Check that SSH login with password is disabled.
 	sshd = open("/etc/ssh/sshd_config").read()
 	if re.search("\nPasswordAuthentication\s+yes", sshd) \
 		or not re.search("\nPasswordAuthentication\s+no", sshd):
-		print_error("""The SSH server on this machine permits password-based login. A more secure
+		env['out'].print_error("""The SSH server on this machine permits password-based login. A more secure
 			way to log in is using a public key. Add your SSH public key to $HOME/.ssh/authorized_keys, check
 			that you can log in without a password, set the option 'PasswordAuthentication no' in
 			/etc/ssh/sshd_config, and then restart the openssh via 'sudo service ssh restart'.""")
 	else:
-		print_ok("SSH disallows password-based login.")
+		env['out'].print_ok("SSH disallows password-based login.")
 
 	# Check that the administrator alias exists since that's where all
 	# admin email is automatically directed.
 	check_alias_exists("administrator@" + env['PRIMARY_HOSTNAME'], env)
 
-	print()
+def run_network_checks(env):
+	# Also see setup/network-checks.sh.
+
+	env["out"].add_heading("Network")
+
+	# Stop if we cannot make an outbound connection on port 25. Many residential
+	# networks block outbound port 25 to prevent their network from sending spam.
+	# See if we can reach one of Google's MTAs with a 5-second timeout.
+	code, ret = shell("check_call", ["/bin/nc", "-z", "-w5", "aspmx.l.google.com", "25"], trap=True)
+	if ret == 0:
+		env['out'].print_ok("Outbound mail (SMTP port 25) is not blocked.")
+	else:
+		env['out'].print_error("""Outbound mail (SMTP port 25) seems to be blocked by your network. You
+			will not be able to send any mail. Many residential networks block port 25 to prevent hijacked
+			machines from being able to send spam. A quick connection test to Google's mail server on port 25
+			failed.""")
+
+	# Stop if the IPv4 address is listed in the ZEN Spamhouse Block List.
+	# The user might have ended up on an IP address that was previously in use
+	# by a spammer, or the user may be deploying on a residential network. We
+	# will not be able to reliably send mail in these cases.
+	rev_ip4 = ".".join(reversed(env['PUBLIC_IP'].split('.')))
+	if not query_dns(rev_ip4+'.zen.spamhaus.org', 'A', nxdomain=None):
+		env['out'].print_ok("IP address is not blacklisted by zen.spamhaus.org.")
+	else:
+		env['out'].print_error("""The IP address of this machine %s is listed in the Spamhaus Block List,
+			which may prevent recipients from receiving your email. See http://www.spamhaus.org/query/ip/%s."""
+			% (env['PUBLIC_IP'], env['PUBLIC_IP']))
 
 def run_domain_checks(env):
 	# Get the list of domains we handle mail for.
@@ -54,8 +82,7 @@ def run_domain_checks(env):
 
 	# Check the domains.
 	for domain in sort_domains(mail_domains | dns_domains | web_domains, env):
-		print(domain)
-		print("=" * len(domain))
+		env["out"].add_heading(domain)
 
 		if domain == env["PRIMARY_HOSTNAME"]:
 			check_primary_hostname_dns(domain, env)
@@ -69,16 +96,14 @@ def run_domain_checks(env):
 		if domain in web_domains:
 			check_web_domain(domain, env)
 
-		print()
-
 def check_primary_hostname_dns(domain, env):
 	# Check that the ns1/ns2 hostnames resolve to A records. This information probably
 	# comes from the TLD since the information is set at the registrar.
 	ip = query_dns("ns1." + domain, "A") + '/' + query_dns("ns2." + domain, "A")
 	if ip == env['PUBLIC_IP'] + '/' + env['PUBLIC_IP']:
-		print_ok("Nameserver IPs are correct at registrar. [ns1/ns2.%s => %s]" % (env['PRIMARY_HOSTNAME'], env['PUBLIC_IP']))
+		env['out'].print_ok("Nameserver glue records are correct at registrar. [ns1/ns2.%s => %s]" % (env['PRIMARY_HOSTNAME'], env['PUBLIC_IP']))
 	else:
-		print_error("""Nameserver IP addresses are incorrect. The ns1.%s and ns2.%s nameservers must be configured at your domain name
+		env['out'].print_error("""Nameserver glue records are incorrect. The ns1.%s and ns2.%s nameservers must be configured at your domain name
 			registrar as having the IP address %s. They currently report addresses of %s. It may take several hours for
 			public DNS to update after a change."""
 			% (env['PRIMARY_HOSTNAME'], env['PRIMARY_HOSTNAME'], env['PUBLIC_IP'], ip))
@@ -86,9 +111,9 @@ def check_primary_hostname_dns(domain, env):
 	# Check that PRIMARY_HOSTNAME resolves to PUBLIC_IP in public DNS.
 	ip = query_dns(domain, "A")
 	if ip == env['PUBLIC_IP']:
-		print_ok("Domain resolves to box's IP address. [%s => %s]" % (env['PRIMARY_HOSTNAME'], env['PUBLIC_IP']))
+		env['out'].print_ok("Domain resolves to box's IP address. [%s => %s]" % (env['PRIMARY_HOSTNAME'], env['PUBLIC_IP']))
 	else:
-		print_error("""This domain must resolve to your box's IP address (%s) in public DNS but it currently resolves
+		env['out'].print_error("""This domain must resolve to your box's IP address (%s) in public DNS but it currently resolves
 			to %s. It may take several hours for public DNS to update after a change. This problem may result from other
 			issues listed here."""
 			% (env['PUBLIC_IP'], ip))
@@ -98,53 +123,90 @@ def check_primary_hostname_dns(domain, env):
 	ipaddr_rev = dns.reversename.from_address(env['PUBLIC_IP'])
 	existing_rdns = query_dns(ipaddr_rev, "PTR")
 	if existing_rdns == domain:
-		print_ok("Reverse DNS is set correctly at ISP. [%s => %s]" % (env['PUBLIC_IP'], env['PRIMARY_HOSTNAME']))
+		env['out'].print_ok("Reverse DNS is set correctly at ISP. [%s => %s]" % (env['PUBLIC_IP'], env['PRIMARY_HOSTNAME']))
 	else:
-		print_error("""Your box's reverse DNS is currently %s, but it should be %s. Your ISP or cloud provider will have instructions
+		env['out'].print_error("""Your box's reverse DNS is currently %s, but it should be %s. Your ISP or cloud provider will have instructions
 			on setting up reverse DNS for your box at %s.""" % (existing_rdns, domain, env['PUBLIC_IP']) )
 
+	# Check the TLSA record.
+	tlsa_qname = "_25._tcp." + domain
+	tlsa25 = query_dns(tlsa_qname, "TLSA", nxdomain=None)
+	tlsa25_expected = build_tlsa_record(env)
+	if tlsa25 == tlsa25_expected:
+		env['out'].print_ok("""The DANE TLSA record for incoming mail is correct (%s).""" % tlsa_qname,)
+	elif tlsa25 is None:
+		env['out'].print_error("""The DANE TLSA record for incoming mail is not set. This is optional.""")
+	else:
+		env['out'].print_error("""The DANE TLSA record for incoming mail (%s) is not correct. It is '%s' but it should be '%s'. Try running tools/dns_update to
+			regenerate the record. It may take several hours for
+                        public DNS to update after a change."""
+                        % (tlsa_qname, tlsa25, tlsa25_expected))
+
 	# Check that the hostmaster@ email address exists.
 	check_alias_exists("hostmaster@" + domain, env)
 
 def check_alias_exists(alias, env):
 	mail_alises = dict(get_mail_aliases(env))
 	if alias in mail_alises:
-		print_ok("%s exists as a mail alias [=> %s]" % (alias, mail_alises[alias]))
+		env['out'].print_ok("%s exists as a mail alias [=> %s]" % (alias, mail_alises[alias]))
 	else:
-		print_error("""You must add a mail alias for %s and direct email to you or another administrator.""" % alias)
+		env['out'].print_error("""You must add a mail alias for %s and direct email to you or another administrator.""" % alias)
 
 def check_dns_zone(domain, env, dns_zonefiles):
 	# We provide a DNS zone for the domain. It should have NS records set up
 	# at the domain name's registrar pointing to this box.
 	existing_ns = query_dns(domain, "NS")
 	correct_ns = "ns1.BOX; ns2.BOX".replace("BOX", env['PRIMARY_HOSTNAME'])
-	if existing_ns == correct_ns:
-		print_ok("Nameservers are set correctly at registrar. [%s]" % correct_ns)
+	if existing_ns.lower() == correct_ns.lower():
+		env['out'].print_ok("Nameservers are set correctly at registrar. [%s]" % correct_ns)
 	else:
-		print_error("""The nameservers set on this domain are incorrect. They are currently %s. Use your domain name registar's
+		env['out'].print_error("""The nameservers set on this domain are incorrect. They are currently %s. Use your domain name registar's
 			control panel to set the nameservers to %s."""
 				% (existing_ns, correct_ns) )
 
-	# See if the domain has a DS record set.
+	# See if the domain has a DS record set at the registrar. The DS record may have
+	# several forms. We have to be prepared to check for any valid record. We've
+	# pre-generated all of the valid digests --- read them in.
+	ds_correct = open('/etc/nsd/zones/' + dns_zonefiles[domain] + '.ds').read().strip().split("\n")
+	digests = { }
+	for rr_ds in ds_correct:
+		ds_keytag, ds_alg, ds_digalg, ds_digest = rr_ds.split("\t")[4].split(" ")
+		digests[ds_digalg] = ds_digest
+
+	# Some registrars may want the public key so they can compute the digest. The DS
+	# record that we suggest using is for the KSK (and that's how the DS records were generated).
+	dnssec_keys = load_env_vars_from_file(os.path.join(env['STORAGE_ROOT'], 'dns/dnssec/keys.conf'))
+	dnsssec_pubkey = open(os.path.join(env['STORAGE_ROOT'], 'dns/dnssec/' + dnssec_keys['KSK'] + '.key')).read().split("\t")[3].split(" ")[3]
+
+	# Query public DNS for the DS record at the registrar.
 	ds = query_dns(domain, "DS", nxdomain=None)
-	ds_correct = open('/etc/nsd/zones/' + dns_zonefiles[domain] + '.ds').read().strip()
-	ds_expected = re.sub(r"\S+\.\s+3600\s+IN\s+DS\s*", "", ds_correct)
-	if ds == ds_expected:
-		print_ok("DNS 'DS' record is set correctly at registrar.")
-	elif ds == None:
-		print_error("""This domain's DNS DS record is not set. The DS record is optional. The DS record activates DNSSEC.
-			To set a DS record, you must follow the instructions provided by your domain name registrar and provide to them this information:""")
-		print("")
-		print("   " + ds_correct)
-		print("")
+	ds_looks_valid = ds and len(ds.split(" ")) == 4
+	if ds_looks_valid: ds = ds.split(" ")
+	if ds_looks_valid and ds[0] == ds_keytag and ds[1] == '7' and ds[3] == digests.get(ds[2]):
+		env['out'].print_ok("DNS 'DS' record is set correctly at registrar.")
 	else:
-		print_error("""This domain's DNS DS record is incorrect. The chain of trust is broken between the public DNS system
-			and this machine's DNS server. It may take several hours for public DNS to update after a change. If you did not recently
-			make a change, you must resolve this immediately by following the instructions provided by your domain name registrar and
-			provide to them this information:""")
-		print("")
-		print("   " + ds_correct)
-		print("")
+		if ds == None:
+			env['out'].print_error("""This domain's DNS DS record is not set. The DS record is optional. The DS record activates DNSSEC.
+				To set a DS record, you must follow the instructions provided by your domain name registrar and provide to them this information:""")
+		else:
+			env['out'].print_error("""This domain's DNS DS record is incorrect. The chain of trust is broken between the public DNS system
+				and this machine's DNS server. It may take several hours for public DNS to update after a change. If you did not recently
+				make a change, you must resolve this immediately by following the instructions provided by your domain name registrar and
+				provide to them this information:""")
+		env['out'].print_line("")
+		env['out'].print_line("Key Tag: " + ds_keytag + ("" if not ds_looks_valid or ds[0] == ds_keytag else " (Got '%s')" % ds[0]))
+		env['out'].print_line("Key Flags: KSK")
+		env['out'].print_line("Algorithm: 7 / RSASHA1-NSEC3-SHA1" + ("" if not ds_looks_valid or ds[1] == '7' else " (Got '%s')" % ds[1]))
+		env['out'].print_line("Digest Type: 2 / SHA-256")
+		env['out'].print_line("Digest: " + digests['2'])
+		if ds_looks_valid and ds[3] != digests.get(ds[2]):
+			env['out'].print_line("(Got digest type %s and digest %s which do not match.)" % (ds[2], ds[3]))
+		env['out'].print_line("Public Key: ")
+		env['out'].print_line(dnsssec_pubkey, monospace=True)
+		env['out'].print_line("")
+		env['out'].print_line("Bulk/Record Format:")
+		env['out'].print_line("" + ds_correct[0])
+		env['out'].print_line("")
 
 def check_mail_domain(domain, env):
 	# Check the MX record.
@@ -153,14 +215,14 @@ def check_mail_domain(domain, env):
 	expected_mx = "10 " + env['PRIMARY_HOSTNAME']
 
 	if mx == expected_mx:
-		print_ok("Domain's email is directed to this domain. [%s => %s]" % (domain, mx))
+		env['out'].print_ok("Domain's email is directed to this domain. [%s => %s]" % (domain, mx))
 
 	elif mx == None:
 		# A missing MX record is okay on the primary hostname because
 		# the primary hostname's A record (the MX fallback) is... itself,
 		# which is what we want the MX to be.
 		if domain == env['PRIMARY_HOSTNAME']:
-			print_ok("Domain's email is directed to this domain. [%s has no MX record, which is ok]" % (domain,))
+			env['out'].print_ok("Domain's email is directed to this domain. [%s has no MX record, which is ok]" % (domain,))
 
 		# And a missing MX record is okay on other domains if the A record
 		# matches the A record of the PRIMARY_HOSTNAME. Actually this will
@@ -169,20 +231,29 @@ def check_mail_domain(domain, env):
 			domain_a = query_dns(domain, "A", nxdomain=None)
 			primary_a = query_dns(env['PRIMARY_HOSTNAME'], "A", nxdomain=None)
 			if domain_a != None and domain_a == primary_a:
-				print_ok("Domain's email is directed to this domain. [%s has no MX record but its A record is OK]" % (domain,))
+				env['out'].print_ok("Domain's email is directed to this domain. [%s has no MX record but its A record is OK]" % (domain,))
 			else:
-				print_error("""This domain's DNS MX record is not set. It should be '%s'. Mail will not
+				env['out'].print_error("""This domain's DNS MX record is not set. It should be '%s'. Mail will not
 					be delivered to this box. It may take several hours for public DNS to update after a
 					change. This problem may result from other issues listed here.""" % (expected_mx,))
 
 	else:
-		print_error("""This domain's DNS MX record is incorrect. It is currently set to '%s' but should be '%s'. Mail will not
+		env['out'].print_error("""This domain's DNS MX record is incorrect. It is currently set to '%s' but should be '%s'. Mail will not
 			be delivered to this box. It may take several hours for public DNS to update after a change. This problem may result from
 			other issues listed here.""" % (mx, expected_mx))
 
 	# Check that the postmaster@ email address exists.
 	check_alias_exists("postmaster@" + domain, env)
 
+	# Stop if the domain is listed in the Spamhaus Domain Block List.
+	# The user might have chosen a domain that was previously in use by a spammer
+	# and will not be able to reliably send mail.
+	if not query_dns(domain+'.dbl.spamhaus.org', "A", nxdomain=None):
+		env['out'].print_ok("Domain is not blacklisted by dbl.spamhaus.org.")
+	else:
+		env['out'].print_error("""This domain is listed in the Spamhaus Domain Block List, which may prevent recipients from receiving your mail.
+			See http://www.spamhaus.org/dbl/ and http://www.spamhaus.org/query/domain/%s.""" % domain)
+
 def check_web_domain(domain, env):
 	# See if the domain's A record resolves to our PUBLIC_IP. This is already checked
 	# for PRIMARY_HOSTNAME, for which it is required for mail specifically. For it and
@@ -190,9 +261,9 @@ def check_web_domain(domain, env):
 	if domain != env['PRIMARY_HOSTNAME']:
 		ip = query_dns(domain, "A")
 		if ip == env['PUBLIC_IP']:
-			print_ok("Domain resolves to this box's IP address. [%s => %s]" % (domain, env['PUBLIC_IP']))
+			env['out'].print_ok("Domain resolves to this box's IP address. [%s => %s]" % (domain, env['PUBLIC_IP']))
 		else:
-			print_error("""This domain should resolve to your box's IP address (%s) if you would like the box to serve
+			env['out'].print_error("""This domain should resolve to your box's IP address (%s) if you would like the box to serve
 				webmail or a website on this domain. The domain currently resolves to %s in public DNS. It may take several hours for
 				public DNS to update after a change. This problem may result from other issues listed here.""" % (env['PUBLIC_IP'], ip))
 
@@ -212,20 +283,21 @@ def query_dns(qname, rtype, nxdomain='[Not Set]'):
 
 	# There may be multiple answers; concatenate the response. Remove trailing
 	# periods from responses since that's how qnames are encoded in DNS but is
-	# confusing for us.
-	return "; ".join(str(r).rstrip('.') for r in response)
+	# confusing for us. The order of the answers doesn't matter, so sort so we
+	# can compare to a well known order.
+	return "; ".join(sorted(str(r).rstrip('.') for r in response))
 
 def check_ssl_cert(domain, env):
 	# Check that SSL certificate is signed.
 
 	# Skip the check if the A record is not pointed here.
-	if query_dns(domain, "A") != env['PUBLIC_IP']: return
+	if query_dns(domain, "A", None) not in (env['PUBLIC_IP'], None): return
 
 	# Where is the SSL stored?
 	ssl_key, ssl_certificate, ssl_csr_path = get_domain_ssl_files(domain, env)
 
 	if not os.path.exists(ssl_certificate):
-		print_error("The SSL certificate file for this domain is missing.")
+		env['out'].print_error("The SSL certificate file for this domain is missing.")
 		return
 
 	# Check that the certificate is good.
@@ -243,34 +315,34 @@ def check_ssl_cert(domain, env):
 		fingerprint = re.sub(".*Fingerprint=", "", fingerprint).strip()
 
 		if domain == env['PRIMARY_HOSTNAME']:
-			print_error("""The SSL certificate for this domain is currently self-signed. You will get a security
+			env['out'].print_error("""The SSL certificate for this domain is currently self-signed. You will get a security
 			warning when you check or send email and when visiting this domain in a web browser (for webmail or
 			static site hosting). You may choose to confirm the security exception, but check that the certificate
 			fingerprint matches the following:""")
-			print()
-			print("   " + fingerprint)
+			env['out'].print_line("")
+			env['out'].print_line("   " + fingerprint, monospace=True)
 		else:
-			print_error("""The SSL certificate for this domain is currently self-signed. Visitors to a website on
+			env['out'].print_error("""The SSL certificate for this domain is currently self-signed. Visitors to a website on
 			this domain will get a security warning. If you are not serving a website on this domain, then it is
 			safe to leave the self-signed certificate in place.""")
-		print()
-		print_block("""You can purchase a signed certificate from many places. You will need to provide this Certificate Signing Request (CSR)
+		env['out'].print_line("")
+		env['out'].print_line("""You can purchase a signed certificate from many places. You will need to provide this Certificate Signing Request (CSR)
 			to whoever you purchase the SSL certificate from:""")
-		print()
-		print(open(ssl_csr_path).read().strip())
-		print()
-		print_block("""When you purchase an SSL certificate you will receive a certificate in PEM format and possibly a file containing intermediate certificates in PEM format.
+		env['out'].print_line("")
+		env['out'].print_line(open(ssl_csr_path).read().strip(), monospace=True)
+		env['out'].print_line("")
+		env['out'].print_line("""When you purchase an SSL certificate you will receive a certificate in PEM format and possibly a file containing intermediate certificates in PEM format.
 			If you receive intermediate certificates, use a text editor and paste your certificate on top and then the intermediate certificates
 			below it. Save the file and place it onto this machine at %s. Then run "service nginx restart".""" % ssl_certificate)
 
 	elif cert_status == "OK":
-		print_ok("SSL certificate is signed & valid.")
+		env['out'].print_ok("SSL certificate is signed & valid.")
 
 	else:
-		print_error("The SSL certificate has a problem:")
-		print("")
-		print(cert_status)
-		print("")
+		env['out'].print_error("The SSL certificate has a problem:")
+		env['out'].print_line("")
+		env['out'].print_line(cert_status)
+		env['out'].print_line("")
 
 def check_certificate(domain, ssl_certificate, ssl_private_key):
 	# Use openssl verify to check the status of a certificate.
@@ -361,32 +433,56 @@ def check_certificate(domain, ssl_certificate, ssl_private_key):
 	else:
 		return verifyoutput.strip()
 
-def print_ok(message):
-	print_block(message, first_line="✓  ")
-
-def print_error(message):
-	print_block(message, first_line="✖  ")
-
 try:
 	terminal_columns = int(shell('check_output', ['stty', 'size']).split()[1])
 except:
 	terminal_columns = 76
-def print_block(message, first_line="   "):
-	print(first_line, end='')
-	message = re.sub("\n\s*", " ", message)
-	words = re.split("(\s+)", message)
-	linelen = 0
-	for w in words:
-		if linelen + len(w) > terminal_columns-1-len(first_line):
-			print()
-			print("   ", end="")
-			linelen = 0
-		if linelen == 0 and w.strip() == "": continue
-		print(w, end="")
-		linelen += len(w)
-	if linelen > 0:
+class ConsoleOutput:
+	def add_heading(self, heading):
+		print()
+		print(heading)
+		print("=" * len(heading))
+
+	def print_ok(self, message):
+		self.print_block(message, first_line="✓  ")
+
+	def print_error(self, message):
+		self.print_block(message, first_line="✖  ")
+
+	def print_block(self, message, first_line="   "):
+		print(first_line, end='')
+		message = re.sub("\n\s*", " ", message)
+		words = re.split("(\s+)", message)
+		linelen = 0
+		for w in words:
+			if linelen + len(w) > terminal_columns-1-len(first_line):
+				print()
+				print("   ", end="")
+				linelen = 0
+			if linelen == 0 and w.strip() == "": continue
+			print(w, end="")
+			linelen += len(w)
 		print()
 
+	def print_line(self, message, monospace=False):
+		for line in message.split("\n"):
+			self.print_block(line)
+
 if __name__ == "__main__":
+	import sys
 	from utils import load_environment
-	run_checks(load_environment())
+	env = load_environment()
+	if len(sys.argv) == 1:
+		run_checks(env, ConsoleOutput())
+	elif sys.argv[1] == "--check-primary-hostname":
+		# See if the primary hostname appears resolvable and has a signed certificate.
+		domain = env['PRIMARY_HOSTNAME']
+		if query_dns(domain, "A") != env['PUBLIC_IP']:
+			sys.exit(1)
+		ssl_key, ssl_certificate, ssl_csr_path = get_domain_ssl_files(domain, env)
+		if not os.path.exists(ssl_certificate):
+			sys.exit(1)
+		cert_status = check_certificate(domain, ssl_certificate, ssl_key)
+		if cert_status != "OK":
+			sys.exit(1)
+		sys.exit(0)

notes/dspam.sh

@@ -1,62 +0,0 @@
-# Spam filtering with dspam.
-#
-# This mostly works. But dspam crashes. So..... we're not using this.
-
-apt-get -q -y install dspam libdspam7-drv-sqlite3 dovecot-antispam dovecot-sieve
-
-# Let it turn on.
-sed -i "s/START=no/START=yes/" /etc/default/dspam 
-
-# Override some of the basic settings that have default values we don't like.
-# Listen as an SMTP server, and pass messages back directly to dovecot.
-tools/editconf.py /etc/dspam/dspam.conf -s \
-	Home=$STORAGE_ROOT/mail/dspam \
-	ServerMode=standard \
-	ServerHost=127.0.0.1 \
-	ServerParameters=--deliver=innocent \
-	DeliveryProto=LMTP \
-	DeliveryHost=/var/run/dovecot/lmtp \
-	Tokenizer=osb
-
-# Put other settings into a local configuration file.
-cat > /etc/dspam/dspam.d/local.conf << EOF;
-IgnoreHeader X-Spam-Status
-IgnoreHeader X-Spam-Scanned
-IgnoreHeader X-Virus-Scanner-Result
-IgnoreHeader X-Virus-Scanned
-IgnoreHeader X-DKIM
-IgnoreHeader DKIM-Signature
-IgnoreHeader DomainKey-Signature
-IgnoreHeader X-Google-Dkim-Signature
-EOF
-
-# Global preferences.
-tools/editconf.py /etc/dspam/default.prefs \
-	spamAction=deliver \
-	signatureLocation=headers \
-	showFactors=on
-
-# Hook into postfix. Replace dovecot with dspam as the mail delivery agent.
-# dspam is configured above to pass mail on to dovecot next.
-tools/editconf.py /etc/postfix/main.cf virtual_transport=lmtp:[127.0.0.1]:2424
-
-# Hook into dovecot... these aren't tested.
-
-# Automatically move spam into a folder called Spam. Enable the sieve plugin.
-# (Note: Be careful if we want to use multiple plugins later.)
-sudo sed -i "s/#mail_plugins = .*/mail_plugins = \$mail_plugins sieve/" /etc/dovecot/conf.d/20-lmtp.conf
-
-# The sieve scripts are installed by users_update.sh.
-
-# to detect when a message moves between folders so we can
-# pass it to dspam for training. (Be careful if we use multiple plugins later.)
-# This is not finished.
-sudo sed -i "s/#mail_plugins = .*/mail_plugins = \$mail_plugins antispam/" /etc/dovecot/conf.d/20-imap.conf
-	
-# Create storage space.
-mkdir -p $STORAGE_ROOT/mail/dspam
-chown dspam:dspam $STORAGE_ROOT/mail/dspam
-
-service dspam restart
-service postfix restart
-

setup/bootstrap.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+#########################################################
+# This script is intended to be run like this:
+#
+#   wget https://raw.githubusercontent.com/mail-in-a-box/mailinabox/master/setup/bootstrap.sh
+#   sudo bash bootstrap.sh
+#
+# We can't pipe directly to bash because setup/start.sh
+# asks for user input on stdin.
+#
+#########################################################
+
+# Are we running as root?
+if [[ $EUID -ne 0 ]]; then
+	echo "This script must be run as root. Did you leave out sudo?"
+	exit
+fi
+
+# Go to root's home directory.
+cd
+
+# Clone the Mail-in-a-Box repository if it doesn't exist.
+if [ ! -d mailinabox ]; then
+	echo Downloading Mail-in-a-Box . . .
+	apt-get -q -q install -y git
+	git clone -q --depth 1 -b master https://github.com/mail-in-a-box/mailinabox
+	cd mailinabox
+
+# If it does exist, update it.
+else
+	echo Updating Mail-in-a-Box . . .
+	cd mailinabox
+	if ! git pull -q --ff-only; then
+		echo "Update failed. Did you modify something in `pwd`?"
+		exit
+	fi
+fi
+
+# Start setup script.
+setup/start.sh

setup/dns.sh

@@ -33,7 +33,7 @@ sudo mkdir -p /var/run/nsd
 
 mkdir -p "$STORAGE_ROOT/dns/dnssec";
 if [ ! -f "$STORAGE_ROOT/dns/dnssec/keys.conf" ]; then
-	# These two steps take a while.
+	echo "Generating DNSSEC signing keys. This may take a few minutes..."
 
 	# Create the Key-Signing Key (KSK) (-k) which is the so-called
 	# Secure Entry Point. Use a NSEC3-compatible algorithm (best

setup/functions.sh

@@ -70,69 +70,51 @@ function get_default_hostname {
 	printf '%s\n' "$1" # return this value
 }
 
-function get_default_publicip {
-	# Get the machine's public IP address. The machine might have
-	# an IP on a private network, but the IP address that we put
-	# into DNS must be one on the public Internet. Try a public
-	# API, but if that fails (maybe we don't have Internet access
-	# right now) then use the IP address that this machine knows
-	# itself as.
-	get_publicip_from_web_service || get_publicip_fallback
-}
-
-function get_default_publicipv6 {
-	get_publicipv6_from_web_service || get_publicipv6_fallback
-}
-
 function get_publicip_from_web_service {
 	# This seems to be the most reliable way to determine the
 	# machine's public IP address: asking a very nice web API
 	# for how they see us. Thanks go out to icanhazip.com.
-	curl -4 --fail --silent icanhazip.com 2>/dev/null
+	# See: https://major.io/icanhazip-com-faq/
+	#
+	# Pass '4' or '6' as an argument to this function to specify
+	# what type of address to get (IPv4, IPv6).
+	curl -$1 --fail --silent --max-time 15 icanhazip.com 2>/dev/null
 }
 
-function get_publicipv6_from_web_service {
-	curl -6 --fail --silent icanhazip.com 2>/dev/null
-}
+function get_default_privateip {
+	# Return the IP address of the network interface connected
+	# to the Internet.
+	#
+	# We used to use `hostname -I` and then filter for either
+	# IPv4 or IPv6 addresses. However if there are multiple
+	# network interfaces on the machine, not all may be for
+	# reaching the Internet.
+	#
+	# Instead use `ip route get` which asks the kernel to use
+	# the system's routes to select which interface would be
+	# used to reach a public address. We'll use 8.8.8.8 as
+	# the destination. It happens to be Google Public DNS, but
+	# no connection is made. We're just seeing how the box
+	# would connect to it. There many be multiple IP addresses
+	# assigned to an interface. `ip route get` reports the
+	# preferred. That's good enough for us. See issue #121.
+	#
+	# Also see ae67409603c49b7fa73c227449264ddd10aae6a9 and
+	# issue #3 for why/how we originally added IPv6.
+	#
+	# Pass '4' or '6' as an argument to this function to specify
+	# what type of address to get (IPv4, IPv6).
 
-function get_publicip_fallback {
-	# Return the IP address that this machine knows itself as.
-	# It certainly may not be the IP address that this machine
-	# operates as on the public Internet. The machine might
-	# have multiple addresses if it has multiple network adapters.
-	set -- $(hostname --ip-address       2>/dev/null) \
-	       $(hostname --all-ip-addresses 2>/dev/null)
-	while (( $# )) && { ! is_ipv4 "$1" || is_loopback_ip "$1"; }; do
-		shift
-	done
-	printf '%s\n' "$1" # return this value
-}
+	target=8.8.8.8
 
-function get_publicipv6_fallback {
-	set -- $(hostname --ip-address       2>/dev/null) \
-	       $(hostname --all-ip-addresses 2>/dev/null)
-	while (( $# )) && { ! is_ipv6 "$1" || is_loopback_ipv6 "$1"; }; do
-		shift
-	done
-	printf '%s\n' "$1" # return this value
-}
+	# For the IPv6 route, use the corresponding IPv6 address
+	# of Google Public DNS. Again, it doesn't matter so long
+	# as it's an address on the public Internet.
+	if [ "$1" == "6" ]; then target=2001:4860:4860::8888; fi
 
-function is_ipv4 {
-	# helper for get_publicip_fallback
-	[[ "$1" == *.*.*.* ]]
-}
-
-function is_ipv6 {
-	[[ "$1" == *:*:* ]]
-}
-
-function is_loopback_ip {
-	# helper for get_publicip_fallback
-	[[ "$1" == 127.* ]]
-}
-
-function is_loopback_ipv6 {
-	[[ "$1" == ::1 ]]
+	ip -$1 -o route get $target \
+		| grep -v unreachable \
+		| sed "s/.* src \([^ ]*\).*/\1/"
 }
 
 function ufw_allow {

setup/mail-dovecot.sh

@@ -53,6 +53,15 @@ tools/editconf.py /etc/dovecot/conf.d/10-ssl.conf \
 sed -i "s/#port = 143/port = 0/" /etc/dovecot/conf.d/10-master.conf
 sed -i "s/#port = 110/port = 0/" /etc/dovecot/conf.d/10-master.conf
 
+# Make IMAP IDLE slightly more efficient. By default, Dovecot says "still here"
+# every two minutes. With K-9 mail, the bandwidth and battery usage due to
+# this are minimal. But for good measure, let's go to 4 minutes to halve the
+# bandwidth and number of times the device's networking might be woken up.
+# The risk is that if the connection is silent for too long it might be reset
+# by a peer. See #129 and http://razor.occams.info/blog/2014/08/09/how-bad-is-imap-idle/.
+tools/editconf.py /etc/dovecot/conf.d/20-imap.conf \
+	imap_idle_notify_interval="4 mins"
+
 # LDA (LMTP)
 
 # Enable Dovecot's LDA service with the LMTP protocol. It will listen

setup/mail-postfix.sh

@@ -31,7 +31,7 @@ source /etc/mailinabox.conf # load global vars
 
 # Install packages.
 
-apt_install postfix postgrey postfix-pcre
+apt_install postfix postgrey postfix-pcre ca-certificates
 
 # Basic Settings
 
@@ -41,7 +41,7 @@ apt_install postfix postgrey postfix-pcre
 tools/editconf.py /etc/postfix/main.cf \
 	inet_interfaces=all \
 	myhostname=$PRIMARY_HOSTNAME\
-	smtpd_banner="\$myhostname ESMTP Hi, I'm a Mail-in-a-Box (Ubuntu/Postfix; see https://github.com/joshdata/mailinabox)" \
+	smtpd_banner="\$myhostname ESMTP Hi, I'm a Mail-in-a-Box (Ubuntu/Postfix; see https://mailinabox.email/)" \
 	mydestination=localhost
 
 # Outgoing Mail
@@ -75,15 +75,27 @@ tools/editconf.py /etc/postfix/main.cf \
 	smtpd_tls_received_header=yes
 
 # When connecting to remote SMTP servers, prefer TLS and use DANE if available.
-# Postfix queries for the TLSA record on the destination MX host. If no TLSA records are found,
+#
+# Prefering ("opportunistic") TLS means Postfix will accept whatever SSL certificate the remote
+# end provides, if the remote end offers STARTTLS during the connection. DANE takes this a
+# step further:
+#
+# Postfix queries DNS for the TLSA record on the destination MX host. If no TLSA records are found,
 # then opportunistic TLS is used. Otherwise the server certificate must match the TLSA records
 # or else the mail bounces. TLSA also requires DNSSEC on the MX host. Postfix doesn't do DNSSEC
 # itself but assumes the system's nameserver does and reports DNSSEC status. Thus this also
 # relies on our local bind9 server being present and smtp_dns_support_level being set to dnssec
 # to use it.
+#
+# The smtp_tls_CAfile is superflous, but it turns warnings in the logs about untrusted certs
+# into notices about trusted certs. Since in these cases Postfix is doing opportunistic TLS,
+# it does not care about whether the remote certificate is trusted. But, looking at the logs,
+# it's nice to be able to see that the connection was in fact encrypted for the right party.
+# The CA file is provided by the package ca-certificates.
 tools/editconf.py /etc/postfix/main.cf \
 	smtp_tls_security_level=dane \
 	smtp_dns_support_level=dnssec \
+	smtp_tls_CAfile=/etc/ssl/certs/ca-certificates.crt \
 	smtp_tls_loglevel=2
 
 # Incoming Mail
@@ -112,9 +124,11 @@ tools/editconf.py /etc/postfix/main.cf \
 # reject_non_fqdn_sender: Reject not-nice-looking return paths.
 # reject_unknown_sender_domain: Reject return paths with invalid domains.
 # reject_rhsbl_sender: Reject return paths that use blacklisted domains.
-# permit_sasl_authenticated: Authenticated users (i.e. on port 587).
-# permit_mynetworks: Mail that originates locally.
+#
+# permit_sasl_authenticated: Authenticated users (i.e. on port 587) can skip further checks.
+# permit_mynetworks: Mail that originates locally can skip further checks.
 # reject_rbl_client: Reject connections from IP addresses blacklisted in zen.spamhaus.org
+# reject_unlisted_recipient: Although Postfix will reject mail to unknown recipients, it's nicer to reject such mail ahead of greylisting rather than after.
 # check_policy_service: Apply greylisting using postgrey.
 #
 # Notes:
@@ -124,7 +138,7 @@ tools/editconf.py /etc/postfix/main.cf \
 # "450 4.7.1 Client host rejected: Service unavailable". This is a retry code, so the mail doesn't properly bounce.
 tools/editconf.py /etc/postfix/main.cf \
 	smtpd_sender_restrictions="reject_non_fqdn_sender,reject_unknown_sender_domain,reject_rhsbl_sender dbl.spamhaus.org" \
-	smtpd_recipient_restrictions=permit_sasl_authenticated,permit_mynetworks,"reject_rbl_client zen.spamhaus.org","check_policy_service inet:127.0.0.1:10023"
+	smtpd_recipient_restrictions=permit_sasl_authenticated,permit_mynetworks,"reject_rbl_client zen.spamhaus.org",reject_unlisted_recipient,"check_policy_service inet:127.0.0.1:10023"
 
 # Increase the message size limit from 10MB to 128MB.
 tools/editconf.py /etc/postfix/main.cf \

setup/mail-users.sh

@@ -17,7 +17,7 @@ db_path=$STORAGE_ROOT/mail/users.sqlite
 # Create an empty database if it doesn't yet exist.
 if [ ! -f $db_path ]; then
 	echo Creating new user database: $db_path;
-	echo "CREATE TABLE users (id INTEGER PRIMARY KEY AUTOINCREMENT, email TEXT NOT NULL UNIQUE, password TEXT NOT NULL, extra);" | sqlite3 $db_path;
+	echo "CREATE TABLE users (id INTEGER PRIMARY KEY AUTOINCREMENT, email TEXT NOT NULL UNIQUE, password TEXT NOT NULL, extra, privileges TEXT NOT NULL DEFAULT '');" | sqlite3 $db_path;
 	echo "CREATE TABLE aliases (id INTEGER PRIMARY KEY AUTOINCREMENT, source TEXT NOT NULL UNIQUE, destination TEXT NOT NULL);" | sqlite3 $db_path;
 fi
 

setup/migrate.py

@@ -8,7 +8,7 @@
 import sys, os, os.path, glob, re, shutil
 
 sys.path.insert(0, 'management')
-from utils import load_environment, save_environment, safe_domain_name
+from utils import load_environment, save_environment, shell
 
 def migration_1(env):
 	# Re-arrange where we store SSL certificates. There was a typo also.
@@ -45,6 +45,17 @@ def migration_2(env):
 	for fn in glob.glob(os.path.join(env["STORAGE_ROOT"], 'mail/mailboxes/*/*/.dovecot.svbin')):
 		os.unlink(fn)
 
+def migration_3(env):
+	# Move the migration ID from /etc/mailinabox.conf to $STORAGE_ROOT/mailinabox.version
+	# so that the ID stays with the data files that it describes the format of. The writing
+	# of the file will be handled by the main function.
+	pass
+
+def migration_4(env):
+	# Add a new column to the mail users table where we can store administrative privileges.
+	db = os.path.join(env["STORAGE_ROOT"], 'mail/users.sqlite')
+	shell("check_call", ["sqlite3", db, "ALTER TABLE users ADD privileges TEXT NOT NULL DEFAULT ''"])
+
 def get_current_migration():
 	ver = 0
 	while True:
@@ -61,7 +72,14 @@ def run_migrations():
 
 	env = load_environment()
 
-	ourver = int(env.get("MIGRATIONID", "0"))
+	migration_id_file = os.path.join(env['STORAGE_ROOT'], 'mailinabox.version')
+	if os.path.exists(migration_id_file):
+		with open(migration_id_file) as f:
+			ourver = int(f.read().strip())
+	else:
+		# Load the legacy location of the migration ID. We'll drop support
+		# for this eventually.
+		ourver = int(env.get("MIGRATIONID", "0"))
 
 	while True:
 		next_ver = (ourver + 1)
@@ -71,6 +89,7 @@ def run_migrations():
 			# No more migrations to run.
 			break
 
+		print()
 		print("Running migration to Mail-in-a-Box #%d..." % next_ver)
 
 		try:
@@ -88,8 +107,13 @@ def run_migrations():
 
 		# Write out our current version now. Do this sooner rather than later
 		# in case of any problems.
-		env["MIGRATIONID"] = ourver
-		save_environment(env)
+		with open(migration_id_file, "w") as f:
+			f.write(str(ourver) + "\n")
+
+		# Delete the legacy location of this field.
+		if "MIGRATIONID" in env:
+			del env["MIGRATIONID"]
+			save_environment(env)
 
 		# iterate and try next version...
 

setup/network-checks.sh

@@ -0,0 +1,53 @@
+# Stop if the PRIMARY_HOSTNAME is listed in the Spamhaus Domain Block List.
+# The user might have chosen a name that was previously in use by a spammer
+# and will not be able to reliably send mail. Do this after any automatic
+# choices made above.
+if host $PRIMARY_HOSTNAME.dbl.spamhaus.org > /dev/null; then
+	echo
+	echo "The hostname you chose '$PRIMARY_HOSTNAME' is listed in the"
+	echo "Spamhaus Domain Block List. See http://www.spamhaus.org/dbl/"
+	echo "and http://www.spamhaus.org/query/domain/$PRIMARY_HOSTNAME."
+	echo
+	echo "You will not be able to send mail using this domain name, so"
+	echo "setup cannot continue."
+	echo
+	exit 1
+fi
+
+# Stop if the IPv4 address is listed in the ZEN Spamhouse Block List.
+# The user might have ended up on an IP address that was previously in use
+# by a spammer, or the user may be deploying on a residential network. We
+# will not be able to reliably send mail in these cases.
+REVERSED_IPV4=$(echo $PUBLIC_IP | sed "s/\([0-9]*\).\([0-9]*\).\([0-9]*\).\([0-9]*\)/\4.\3.\2.\1/")
+if host $REVERSED_IPV4.zen.spamhaus.org > /dev/null; then
+	echo
+	echo "The IP address $PUBLIC_IP is listed in the Spamhaus Block List."
+	echo "See http://www.spamhaus.org/query/ip/$PUBLIC_IP."
+	echo
+	echo "You will not be able to send mail using this machine, so setup"
+	echo "cannot continue."
+	echo
+	echo "Associate a different IP address with this machine if possible."
+	echo "Many residential network IP addresses are listed, so Mail-in-a-Box"
+	echo "typically cannot be used on a residential Internet connection."
+	echo
+	exit 1
+fi
+
+# Stop if we cannot make an outbound connection on port 25. Many residential
+# networks block outbound port 25 to prevent their network from sending spam.
+# See if we can reach one of Google's MTAs with a 5-second timeout.
+if ! nc -z -w5 aspmx.l.google.com 25; then
+	echo
+	echo "Outbound mail (port 25) seems to be blocked by your network."
+	echo
+	echo "You will not be able to send mail using this machine, so setup"
+	echo "cannot continue."
+	echo
+	echo "Many residential networks block port 25 to prevent hijacked"
+	echo "machines from being able to send spam. I just tried to connect"
+	echo "to Google's mail server on port 25 but the connection did not"
+	echo "succeed."
+	echo
+	exit 1
+fi

setup/owncloud.sh

@@ -0,0 +1,130 @@
+#!/bin/bash
+# Owncloud
+##########################
+
+source setup/functions.sh # load our functions
+source /etc/mailinabox.conf # load global vars
+
+apt_install \
+	dbconfig-common \
+	php5-cli php5-sqlite php5-gd php5-imap php5-curl php-pear php-apc curl libapr1 libtool libcurl4-openssl-dev php-xml-parser \
+	php5 php5-dev php5-gd php5-fpm memcached php5-memcache unzip
+
+apt-get purge -qq -y owncloud*
+
+# Install ownCloud from source if it is not already present
+# TODO: Check version?
+if [ ! -d /usr/local/lib/owncloud ]; then
+	echo installing ownCloud...
+	rm -f /tmp/owncloud.zip
+	wget -qO /tmp/owncloud.zip https://download.owncloud.org/community/owncloud-7.0.1.zip
+	unzip -q /tmp/owncloud.zip -d /usr/local/lib
+	rm -f /tmp/owncloud.zip
+fi
+
+# Setup ownCloud if the ownCloud database does not yet exist. Running setup when
+# the database does exist wipes the database and user data.
+if [ ! -f $STORAGE_ROOT/owncloud/owncloud.db ]; then
+	# Create a configuration file.
+	TIMEZONE=`cat /etc/timezone`
+	instanceid=oc$(echo $PRIMARY_HOSTNAME | sha1sum | fold -w 10 | head -n 1)
+	cat - > /usr/local/lib/owncloud/config/config.php <<EOF;
+<?php
+\$CONFIG = array (
+  'datadirectory' => '$STORAGE_ROOT/owncloud',
+
+  'instanceid' => '$instanceid',
+
+  'trusted_domains' => 
+    array (
+      0 => '$PRIMARY_HOSTNAME',
+    ),
+  'forcessl' => true, # if unset/false, ownCloud sends a HSTS=0 header, which conflicts with nginx config
+
+  'overwritewebroot' => '/cloud',
+  'user_backends' => array(
+    array(
+      'class'=>'OC_User_IMAP',
+      'arguments'=>array('{localhost:993/imap/ssl/novalidate-cert}')
+    )
+  ),
+  "memcached_servers" => array (
+    array('localhost', 11211),
+  ),
+  'mail_smtpmode' => 'sendmail',
+  'mail_smtpsecure' => '',
+  'mail_smtpauthtype' => 'LOGIN',
+  'mail_smtpauth' => false,
+  'mail_smtphost' => '',
+  'mail_smtpport' => '',
+  'mail_smtpname' => '',
+  'mail_smtppassword' => '',
+  'mail_from_address' => 'owncloud',
+  'mail_domain' => '$PRIMARY_HOSTNAME',
+  'logtimezone' => '$TIMEZONE',
+);
+?>
+EOF
+
+	# Create an auto-configuration file to fill in database settings
+	# when the install script is run. Make an administrator account
+	# here or else the install can't finish.
+	adminpassword=$(dd if=/dev/random bs=40 count=1 2>/dev/null | sha1sum | fold -w 30 | head -n 1)
+	cat - > /usr/local/lib/owncloud/config/autoconfig.php <<EOF;
+<?php
+\$AUTOCONFIG = array (
+  # storage/database
+  'directory' => '$STORAGE_ROOT/owncloud',
+  'dbtype' => 'sqlite3',
+
+  # create an administrator account with a random password so that
+  # the user does not have to enter anything on first load of ownCloud
+  'adminlogin'    => 'root',
+  'adminpass'     => '$adminpassword',
+);
+?>
+EOF
+
+	# Create user data directory and set permissions
+	mkdir -p $STORAGE_ROOT/owncloud
+	chown -R www-data.www-data $STORAGE_ROOT/owncloud /usr/local/lib/owncloud
+
+	# Execute ownCloud's setup step, which creates the ownCloud sqlite database.
+	# It also wipes it if it exists. And it deletes the autoconfig.php file.
+	(cd /usr/local/lib/owncloud; sudo -u www-data php /usr/local/lib/owncloud/index.php;)
+fi
+
+# Enable/disable apps. Note that this must be done after the ownCloud setup.
+# The firstrunwizard gave Josh all sorts of problems, so disabling that.
+# user_external is what allows ownCloud to use IMAP for login.
+hide_output php /usr/local/lib/owncloud/console.php app:disable firstrunwizard
+hide_output php /usr/local/lib/owncloud/console.php app:enable user_external
+
+# Set PHP FPM values to support large file uploads
+# (semicolon is the comment character in this file, hashes produce deprecation warnings)
+tools/editconf.py /etc/php5/fpm/php.ini -c ';' \
+	upload_max_filesize=16G \
+	post_max_size=16G \
+	output_buffering=16384 \
+	memory_limit=512M \
+	max_execution_time=600 \
+	short_open_tag=On
+
+# Set up a cron job for owncloud.
+cat > /etc/cron.hourly/mailinabox-owncloud << EOF;
+#!/bin/bash
+# Mail-in-a-Box
+sudo -u www-data php -f /usr/local/lib/owncloud/cron.php
+EOF
+chmod +x /etc/cron.hourly/mailinabox-owncloud
+
+## Ensure all system admins are ownCloud admins.
+## Actually we don't do this. There's nothing much of interest that the user could
+## change from the ownCloud admin, and there's a lot they could mess up.
+#for user in $(tools/mail.py user admins); do
+#	sqlite3 $STORAGE_ROOT/owncloud/owncloud.db "INSERT OR IGNORE INTO oc_group_user VALUES ('admin', '$user')"
+#done
+
+# Finished.
+php5enmod imap
+restart_service php5-fpm

setup/ssl.sh

@@ -21,7 +21,7 @@ source /etc/mailinabox.conf # load global vars
 apt_install openssl
 
 mkdir -p $STORAGE_ROOT/ssl
-if [ ! -f $STORAGE_ROOT/ssl/ssl_certificate.pem ]; then
+if [ ! -f $STORAGE_ROOT/ssl/ssl_private_key.pem ]; then
 	# Generate a new private key if one doesn't already exist.
 	# Set the umask so the key file is not world-readable.
 	(umask 077; hide_output \
@@ -40,8 +40,3 @@ if [ ! -f $STORAGE_ROOT/ssl/ssl_certificate.pem ]; then
 	  -in $STORAGE_ROOT/ssl/ssl_cert_sign_req.csr -signkey $STORAGE_ROOT/ssl/ssl_private_key.pem -out $STORAGE_ROOT/ssl/ssl_certificate.pem
 fi
 
-echo
-echo "Your SSL certificate's fingerpint is:"
-openssl x509 -in $STORAGE_ROOT/ssl/ssl_certificate.pem -noout -fingerprint \
-	| sed "s/SHA1 Fingerprint=//"
-echo

setup/start.sh

@@ -50,16 +50,13 @@ fi
 if [ -f /etc/mailinabox.conf ]; then
 	# Run any system migrations before proceeding. Since this is a second run,
 	# we assume we have Python already installed.
-	echo
 	setup/migrate.py --migrate
 
-	# Okay now load the old .conf file to get existing configuration options.
+	# Load the old .conf file to get existing configuration options loaded
+	# into variables with a DEFAULT_ prefix.
 	cat /etc/mailinabox.conf | sed s/^/DEFAULT_/ > /tmp/mailinabox.prev.conf
 	source /tmp/mailinabox.prev.conf
-	MIGRATIONID=$DEFAULT_MIGRATIONID
-else
-	# What migration are we at for new installs?
-	MIGRATIONID=$(setup/migrate.py --current)
+	rm -f /tmp/mailinabox.prev.conf
 fi
 
 # The box needs a name.
@@ -106,34 +103,82 @@ if [ -z "$PRIMARY_HOSTNAME" ]; then
 fi
 
 # If the machine is behind a NAT, inside a VM, etc., it may not know
-# its IP address on the public network / the Internet. We need to
-# confirm our best guess with the user.
+# its IP address on the public network / the Internet. Ask the Internet
+# and possibly confirm with user.
 if [ -z "$PUBLIC_IP" ]; then
-	if [ -z "$DEFAULT_PUBLIC_IP" ]; then
-		# set a default on first run
-		DEFAULT_PUBLIC_IP=`get_default_publicip`
+	# Ask the Internet.
+	GUESSED_IP=$(get_publicip_from_web_service 4)
+
+	# On the first run, if we got an answer from the Internet then don't
+	# ask the user.
+	if [[ -z "$DEFAULT_PUBLIC_IP" && ! -z "$GUESSED_IP" ]]; then
+		PUBLIC_IP=$GUESSED_IP
+
+	# Otherwise on the first run at least provide a default.
+	elif [[ -z "$DEFAULT_PUBLIC_IP" ]]; then
+		DEFAULT_PUBLIC_IP=$(get_default_privateip 4)
+
+	# On later runs, if the previous value matches the guessed value then
+	# don't ask the user either.
+	elif [ "$DEFAULT_PUBLIC_IP" == "$GUESSED_IP" ]; then
+		PUBLIC_IP=$GUESSED_IP
 	fi
 
-	echo
-	echo "Enter the public IP address of this machine, as given to you by your"
-	echo "ISP. We've guessed a value, but just backspace it if it's wrong."
-	echo
+	if [ -z "$PUBLIC_IP" ]; then
+		echo
+		echo "Enter the public IP address of this machine, as given to you by your ISP."
+		echo
 
-	read -e -i "$DEFAULT_PUBLIC_IP" -p "Public IP: " PUBLIC_IP
+		read -e -i "$DEFAULT_PUBLIC_IP" -p "Public IP: " PUBLIC_IP
+	fi
 fi
 
-# Same for IPv6.
+# Same for IPv6. But it's optional. Also, if it looks like the system
+# doesn't have an IPv6, don't ask for one.
 if [ -z "$PUBLIC_IPV6" ]; then
-	if [ -z "$DEFAULT_PUBLIC_IPV6" ]; then
-		# set a default on first run
-		DEFAULT_PUBLIC_IPV6=`get_default_publicipv6`
+	# Ask the Internet.
+	GUESSED_IP=$(get_publicip_from_web_service 6)
+	MATCHED=0
+	if [[ -z "$DEFAULT_PUBLIC_IPV6" && ! -z "$GUESSED_IP" ]]; then
+		PUBLIC_IPV6=$GUESSED_IP
+	elif [[ "$DEFAULT_PUBLIC_IPV6" == "$GUESSED_IP" ]]; then
+		# No IPv6 entered and machine seems to have none, or what
+		# the user entered matches what the Internet tells us.
+		PUBLIC_IPV6=$GUESSED_IP
+		MATCHED=1
+	elif [[ -z "$DEFAULT_PUBLIC_IPV6" ]]; then
+		DEFAULT_PUBLIC_IP=$(get_default_privateip 6)
 	fi
 
-	echo
-	echo "(Optional) Enter the IPv6 address of this machine. Leave blank"
-	echo "           if the machine does not have an IPv6 address."
+	if [[ -z "$PUBLIC_IPV6" && $MATCHED == 0 ]]; then
+		echo
+		echo "Optional:"
+		echo "Enter the public IPv6 address of this machine, as given to you by your ISP."
+		echo "Leave blank if the machine does not have an IPv6 address."
+		echo
 
-	read -e -i "$DEFAULT_PUBLIC_IPV6" -p "Public IPv6: " PUBLIC_IPV6
+		read -e -i "$DEFAULT_PUBLIC_IPV6" -p "Public IPv6: " PUBLIC_IPV6
+	fi
+fi
+
+# Get the IP addresses of the local network interface(s) that are connected
+# to the Internet. We need these when we want to have services bind only to
+# the public network interfaces (not loopback, not tunnel interfaces).
+if [ -z "$PRIVATE_IP" ]; then
+	PRIVATE_IP=$(get_default_privateip 4)
+fi
+if [ -z "$PRIVATE_IPV6" ]; then
+	PRIVATE_IPV6=$(get_default_privateip 6)
+fi
+if [[ -z "$PRIVATE_IP" && -z "$PRIVATE_IPV6" ]]; then
+	echo
+	echo "I could not determine the IP or IPv6 address of the network inteface"
+	echo "for connecting to the Internet. Setup must stop."
+	echo
+	hostname -I
+	route
+	echo
+	exit
 fi
 
 # We need a country code to generate a certificate signing request. However
@@ -162,21 +207,37 @@ fi
 
 # Automatic configuration, e.g. as used in our Vagrant configuration.
 if [ "$PUBLIC_IP" = "auto" ]; then
-	# Use a public API to get our public IP address.
-	PUBLIC_IP=`get_default_publicip`
-	echo "IP Address: $PUBLIC_IP"
+	# Use a public API to get our public IP address, or fall back to local network configuration.
+	PUBLIC_IP=$(get_publicip_from_web_service 4 || get_default_privateip 4)
 fi
 if [ "$PUBLIC_IPV6" = "auto" ]; then
-	# Use a public API to get our public IP address.
-	PUBLIC_IPV6=`get_default_publicipv6`
-	echo "IPv6 Address: $PUBLIC_IPV6"
+	# Use a public API to get our public IPv6 address, or fall back to local network configuration.
+	PUBLIC_IPV6=$(get_publicip_from_web_service 6 || get_default_privateip 6)
 fi
 if [ "$PRIMARY_HOSTNAME" = "auto-easy" ]; then
 	# Generate a probably-unique subdomain under our justtesting.email domain.
-	PRIMARY_HOSTNAME=m`get_default_publicip | sha1sum | cut -c1-5`.justtesting.email
-	echo "Primary Hostname: $PRIMARY_HOSTNAME"
+	PRIMARY_HOSTNAME=`echo $PUBLIC_IP | sha1sum | cut -c1-5`.justtesting.email
 fi
 
+# Show the configuration, since the user may have not entered it manually.
+echo
+echo "Primary Hostname: $PRIMARY_HOSTNAME"
+echo "Public IP Address: $PUBLIC_IP"
+if [ ! -z "$PUBLIC_IPV6" ]; then
+	echo "Public IPv6 Address: $PUBLIC_IPV6"
+fi
+if [ "$PRIVATE_IP" != "$PUBLIC_IP" ]; then
+	echo "Private IP Address: $PRIVATE_IP"
+fi
+if [ "$PRIVATE_IPV6" != "$PUBLIC_IPV6" ]; then
+	echo "Private IPv6 Address: $PRIVATE_IPV6"
+fi
+echo
+
+# Run some network checks to make sure setup on this machine makes sense.
+if [ -z "$SKIP_NETWORK_CHECKS" ]; then
+	. setup/network-checks.sh
+fi
 
 # Create the user named "user-data" and store all persistent user
 # data (mailboxes, etc.) in that user's home directory.
@@ -185,6 +246,8 @@ if [ -z "$STORAGE_ROOT" ]; then
 	if [ ! -d /home/$STORAGE_USER ]; then useradd -m $STORAGE_USER; fi
 	STORAGE_ROOT=/home/$STORAGE_USER
 	mkdir -p $STORAGE_ROOT
+	echo $(setup/migrate.py --current) > $STORAGE_ROOT/mailinabox.version
+	chown $STORAGE_USER.$STORAGE_USER $STORAGE_ROOT/mailinabox.version
 fi
 
 # Save the global options in /etc/mailinabox.conf so that standalone
@@ -195,8 +258,9 @@ STORAGE_ROOT=$STORAGE_ROOT
 PRIMARY_HOSTNAME=$PRIMARY_HOSTNAME
 PUBLIC_IP=$PUBLIC_IP
 PUBLIC_IPV6=$PUBLIC_IPV6
+PRIVATE_IP=$PRIVATE_IP
+PRIVATE_IPV6=$PRIVATE_IPV6
 CSR_COUNTRY=$CSR_COUNTRY
-MIGRATIONID=$MIGRATIONID
 EOF
 
 # Start service configuration.
@@ -210,6 +274,7 @@ EOF
 . setup/spamassassin.sh
 . setup/web.sh
 . setup/webmail.sh
+. setup/owncloud.sh
 . setup/zpush.sh
 . setup/management.sh
 
@@ -238,18 +303,47 @@ if [ -z "`tools/mail.py user`" ]; then
 			EMAIL_ADDR=me@$PRIMARY_HOSTNAME
 			EMAIL_PW=1234
 			echo
-			echo "Creating a new mail account for $EMAIL_ADDR with password $EMAIL_PW."
+			echo "Creating a new administrative mail account for $EMAIL_ADDR with password $EMAIL_PW."
 			echo
 		fi
 	else
 		echo
-		echo "Okay. I'm about to set up $EMAIL_ADDR for you."
+		echo "Okay. I'm about to set up $EMAIL_ADDR for you. This account will also"
+		echo "have access to the box's control panel."
 	fi
 
 	# Create the user's mail account. This will ask for a password if none was given above.
 	tools/mail.py user add $EMAIL_ADDR $EMAIL_PW
 
+	# Make it an admin.
+	hide_output tools/mail.py user make-admin $EMAIL_ADDR
+
 	# Create an alias to which we'll direct all automatically-created administrative aliases.
 	tools/mail.py alias add administrator@$PRIMARY_HOSTNAME $EMAIL_ADDR
 fi
 
+echo
+echo "-----------------------------------------------"
+echo
+echo Your Mail-in-a-Box is running.
+echo
+echo Please log in to the control panel for further instructions at:
+echo
+if management/whats_next.py --check-primary-hostname; then
+	# Show the nice URL if it appears to be resolving and has a valid certificate.
+	echo https://$PRIMARY_HOSTNAME/admin
+	echo
+	echo If there are problems with this URL, instead use:
+	echo
+fi
+echo https://$PUBLIC_IP/admin
+echo
+echo You will be alerted that the website has an invalid certificate. Check that
+echo the certificate fingerprint matches:
+echo
+openssl x509 -in $STORAGE_ROOT/ssl/ssl_certificate.pem -noout -fingerprint \
+        | sed "s/SHA1 Fingerprint=//"
+echo
+echo Then you can confirm the security exception and continue.
+echo
+

setup/system.sh

@@ -56,7 +56,7 @@ fi
 #   name server, on IPV6.
 # * The listen-on directive in named.conf.options restricts bind9 to
 #   binding to the loopback interface instead of all interfaces.
-apt_install bind9
+apt_install bind9 resolvconf
 tools/editconf.py /etc/default/bind9 \
 	RESOLVCONF=yes \
 	"OPTIONS=\"-u bind -4\""
@@ -64,5 +64,10 @@ if ! grep -q "listen-on " /etc/bind/named.conf.options; then
 	# Add a listen-on directive if it doesn't exist inside the options block.
 	sed -i "s/^}/\n\tlisten-on { 127.0.0.1; };\n}/" /etc/bind/named.conf.options
 fi
+if [ -f /etc/resolvconf/resolv.conf.d/original ]; then
+	echo "Archiving old resolv.conf (was /etc/resolvconf/resolv.conf.d/original, now /etc/resolvconf/resolv.conf.original)."
+	mv /etc/resolvconf/resolv.conf.d/original /etc/resolvconf/resolv.conf.original
+fi
 
 restart_service bind9
+restart_service resolvconf

setup/web.sh

@@ -5,7 +5,7 @@
 source setup/functions.sh # load our functions
 source /etc/mailinabox.conf # load global vars
 
-apt_install nginx php5-cgi
+apt_install nginx php5-fpm
 
 rm -f /etc/nginx/sites-enabled/default
 
@@ -30,26 +30,34 @@ if [ ! -f $STORAGE_ROOT/www/default/index.html ]; then
 fi
 chown -R $STORAGE_USER $STORAGE_ROOT/www
 
-# Create an init script to start the PHP FastCGI daemon and keep it
-# running after a reboot. Allows us to serve Roundcube for webmail.
-rm -f /etc/init.d/php-fastcgi
-ln -s $(pwd)/conf/phpfcgi-initscript /etc/init.d/php-fastcgi
-hide_output update-rc.d php-fastcgi defaults
+# We previously installed a custom init script to start the PHP FastCGI daemon.
+# Remove it now that we're using php5-fpm.
+if [ -L /etc/init.d/php-fastcgi ]; then
+	echo "Removing /etc/init.d/php-fastcgi, php5-cgi..."
+	rm -f /etc/init.d/php-fastcgi
+	hide_output update-rc.d php-fastcgi remove
+	apt-get -y purge php5-cgi
+fi
 
-# Put our webfinger and Exchange autodiscover.xml server scripts
-# into a well-known location.
-for f in webfinger exchange-autodiscover; do
+# Put our webfinger script into a well-known location.
+for f in webfinger; do
 	cp tools/$f.php /usr/local/bin/mailinabox-$f.php
 	chown www-data.www-data /usr/local/bin/mailinabox-$f.php
 done
 
+# Remove obsoleted scripts.
+# exchange-autodiscover is now handled by Z-Push.
+for f in exchange-autodiscover; do
+	rm /usr/local/bin/mailinabox-$f.php
+done
+
 # Make some space for users to customize their webfinger responses.
 mkdir -p $STORAGE_ROOT/webfinger/acct;
 chown -R $STORAGE_USER $STORAGE_ROOT/webfinger
 
 # Start services.
 restart_service nginx
-restart_service php-fastcgi
+restart_service php5-fpm
 
 # Open ports.
 ufw_allow http

setup/webmail.sh

@@ -100,4 +100,4 @@ chmod 664 $STORAGE_ROOT/mail/users.sqlite
 
 # Enable PHP modules.
 php5enmod mcrypt
-restart_service php-fastcgi
+restart_service php5-fpm

setup/zpush.sh

@@ -14,30 +14,56 @@ source /etc/mailinabox.conf # load global vars
 # Prereqs.
 
 apt_install \
-	php-soap php5-imap
+	php-soap php5-imap libawl-php php5-xsl
 
 php5enmod imap
 
 # Copy Z-Push into place.
-
-if [ ! -d /usr/local/lib/z-push ]; then
-	ZPUSH=z-push-2.1.3-1892
-	wget -qO /tmp/zpush.tgz http://download.z-push.org/final/2.1/$ZPUSH.tar.gz
-	tar -C /tmp -zxf /tmp/zpush.tgz
-	mv /tmp/$ZPUSH /usr/local/lib/z-push
+needs_update=0
+if [ ! -f /usr/local/lib/z-push/version ]; then
+	needs_update=1
+elif [[ `curl -s https://api.github.com/repos/fmbiete/Z-Push-contrib/git/refs/heads/master` != `cat /usr/local/lib/z-push/version` ]]; then
+	# checks if the version 
+	needs_update=1
+fi
+if [ $needs_update == 1 ]; then
+	rm -rf /usr/local/lib/z-push
+	rm -f /tmp/zpush.zip
+	echo installing z-push \(fmbiete fork\)...
+	wget -qO /tmp/zpush.zip https://github.com/fmbiete/Z-Push-contrib/archive/master.zip
+	unzip -q /tmp/zpush.zip -d /usr/local/lib/
+	mv /usr/local/lib/Z-Push-contrib-master /usr/local/lib/z-push
+	rm -f /usr/sbin/z-push-{admin,top}
 	ln -s /usr/local/lib/z-push/z-push-admin.php /usr/sbin/z-push-admin
 	ln -s /usr/local/lib/z-push/z-push-top.php /usr/sbin/z-push-top
-	rm /tmp/zpush.tgz;
+	rm /tmp/zpush.zip;
+	curl -s https://api.github.com/repos/fmbiete/Z-Push-contrib/git/refs/heads/master > /usr/local/lib/z-push/version
 fi
 
-# Configure. Tell is to connect to email via IMAP using SSL. Since we connect on
-# localhost, the certificate won't match (it may be self-signed and invalid anyway)
-# so don't check the cert.
-sed -i "s/define('BACKEND_PROVIDER', .*/define('BACKEND_PROVIDER', 'BackendIMAP');/" /usr/local/lib/z-push/config.php
-#sed -i "s/define('IMAP_SERVER', .*/define('IMAP_SERVER', '$PRIMARY_HOSTNAME');/" /usr/local/lib/z-push/backend/imap/config.php
-sed -i "s/define('IMAP_PORT', .*/define('IMAP_PORT', 993);/" /usr/local/lib/z-push/backend/imap/config.php
-sed -i "s/define('IMAP_OPTIONS', .*/define('IMAP_OPTIONS', '\/ssl\/norsh\/novalidate-cert');/" /usr/local/lib/z-push/backend/imap/config.php
+# Configure default config.
+sed -i "s/define('TIMEZONE', .*/define('TIMEZONE', 'Etc\/UTC');/" /usr/local/lib/z-push/config.php
+sed -i "s/define('BACKEND_PROVIDER', .*/define('BACKEND_PROVIDER', 'BackendCombined');/" /usr/local/lib/z-push/config.php
 
+# Configure BACKEND
+rm -f /usr/local/lib/z-push/backend/combined/config.php
+cp conf/zpush/backend_combined.php /usr/local/lib/z-push/backend/combined/config.php
+
+# Configure IMAP
+rm -f /usr/local/lib/z-push/backend/imap/config.php
+cp conf/zpush/backend_imap.php /usr/local/lib/z-push/backend/imap/config.php
+
+# Configure CardDav
+rm -f /usr/local/lib/z-push/backend/carddav/config.php
+cp conf/zpush/backend_carddav.php /usr/local/lib/z-push/backend/carddav/config.php
+
+# Configure CalDav
+rm -f /usr/local/lib/z-push/backend/caldav/config.php
+cp conf/zpush/backend_caldav.php /usr/local/lib/z-push/backend/caldav/config.php
+
+# Configure Autodiscover
+rm -f /usr/local/lib/z-push/autodiscover/config.php
+cp conf/zpush/autodiscover_config.php /usr/local/lib/z-push/autodiscover/config.php
+sed -i "s/PRIMARY_HOSTNAME/$PRIMARY_HOSTNAME/" /usr/local/lib/z-push/autodiscover/config.php
 
 # Some directories it will use.
 
@@ -50,4 +76,4 @@ chown www-data:www-data /var/lib/z-push
 
 # Restart service.
 
-restart_service php-fastcgi
+restart_service php5-fpm

tools/dns_update

@@ -1,2 +1,6 @@
 #!/bin/bash
-curl -s -d POSTDATA --user $(</var/lib/mailinabox/api.key): http://127.0.0.1:10222/dns/update
+POSTDATA=dummy
+if [ "$1" == "--force" ]; then
+	POSTDATA=force=1
+fi
+curl -s -d $POSTDATA --user $(</var/lib/mailinabox/api.key): http://127.0.0.1:10222/dns/update

tools/editconf.py

@@ -33,6 +33,7 @@ settings = sys.argv[2:]
 
 delimiter = "="
 delimiter_re = r"\s*=\s*"
+comment_char = "#"
 folded_lines = False
 testing = False
 while settings[0][0] == "-" and settings[0] != "--":
@@ -42,7 +43,11 @@ while settings[0][0] == "-" and settings[0] != "--":
 		delimiter = " "
 		delimiter_re = r"\s+"
 	elif opt == "-w":
+		# Line folding is possible in this file.
 		folded_lines = True
+	elif opt == "-c":
+		# Specifies a different comment character.
+		comment_char = settings.pop(0)
 	elif opt == "-t":
 		testing = True
 	else:
@@ -60,7 +65,7 @@ while len(input_lines) > 0:
 
 	# If this configuration file uses folded lines, append any folded lines
 	# into our input buffer.
-	if folded_lines and line[0] not in ("#", " ", ""):
+	if folded_lines and line[0] not in (comment_char, " ", ""):
 		while len(input_lines) > 0 and input_lines[0][0] in " \t":
 			line += input_lines.pop(0)
 
@@ -68,7 +73,11 @@ while len(input_lines) > 0:
 	for i in range(len(settings)):
 		# Check that this line contain this setting from the command-line arguments.
 		name, val = settings[i].split("=", 1)
-		m = re.match("(\s*)(#\s*)?" + re.escape(name) + delimiter_re + "(.*?)\s*$", line, re.S)
+		m = re.match(
+			   "(\s*)"
+			 + "(" + re.escape(comment_char) + "\s*)?"
+			 + re.escape(name) + delimiter_re + "(.*?)\s*$",
+			 line, re.S)
 		if not m: continue
 		indent, is_comment, existing_val = m.groups()
 
@@ -83,7 +92,7 @@ while len(input_lines) > 0:
 		
 		# comment-out the existing line (also comment any folded lines)
 		if is_comment is None:
-			buf += "#" + line.rstrip().replace("\n", "\n#") + "\n"
+			buf += comment_char + line.rstrip().replace("\n", "\n" + comment_char) + "\n"
 		else:
 			# the line is already commented, pass it through
 			buf += line

tools/exchange-autodiscover.php

@@ -1,92 +0,0 @@
-<?php
-	// Parse our configuration file to get the PRIMARY_HOSTNAME.
-	$PRIMARY_HOSTNAME = NULL;
-	foreach (file("/etc/mailinabox.conf") as $line) {
-		$line = explode("=", rtrim($line), 2);
-		if ($line[0] == "PRIMARY_HOSTNAME") {
-			$PRIMARY_HOSTNAME = $line[1];
-		}
-	}
-	if ($PRIMARY_HOSTNAME == NULL) exit("no PRIMARY_HOSTNAME");
-
-	// We might get two kinds of requests.
-	$post_body = file_get_contents('php://input');
-	preg_match('/<AcceptableResponseSchema>(.*?)<\/AcceptableResponseSchema>/', $post_body, $match);
-	$AcceptableResponseSchema = $match[1];
-
-	if ($AcceptableResponseSchema == "http://schemas.microsoft.com/exchange/autodiscover/mobilesync/responseschema/2006") {
-		// There is no way to convey the user's login name with this?
-		?>
-<?xml version="1.0" encoding="utf-8"?>
-<Autodiscover
-xmlns:autodiscover="http://schemas.microsoft.com/exchange/autodiscover/mobilesync/responseschema/2006">
-    <autodiscover:Response>
-        <autodiscover:Action>
-            <autodiscover:Settings>
-                <autodiscover:Server>
-                    <autodiscover:Type>MobileSync</autodiscover:Type>
-                    <autodiscover:Url>https://<?php echo $PRIMARY_HOSTNAME ?></autodiscover:Url>
-                    <autodiscover:Name>https://<?php echo $PRIMARY_HOSTNAME ?></autodiscover:Name>
-                </autodiscover:Server>
-            </autodiscover:Settings>
-        </autodiscover:Action>
-    </autodiscover:Response>
-</Autodiscover>
-<?php
-	} else {
-
-	// I don't know when this is actually used. I implemented this before seeing that
-	// it is not what my phone wanted.
-
-	// Parse the email address out of the POST request, which
-	// we pass back as the login name.
-	preg_match('/<EMailAddress>(.*?)<\/EMailAddress>/', $post_body, $match);
-	$LOGIN = $match[1];
-
-	header("Content-type: text/xml");
-?>
-<?xml version="1.0" encoding="utf-8" ?>
-<Autodiscover xmlns="http://schemas.microsoft.com/exchange/autodiscover/responseschema/2006">
-	<Response xmlns="http://schemas.microsoft.com/exchange/autodiscover/outlook/responseschema/2006a">
-		<ServiceHome>https://<?php echo $PRIMARY_HOSTNAME ?></ServiceHome>
-		<Account>
-			<AccountType>email</AccountType>
-			<Action>settings</Action>
-
-			<Protocol>
-				<Type>IMAP</Type>
-				<Server><?php echo $PRIMARY_HOSTNAME ?></Server>
-				<Port>993</Port>
-				<SSL>on</SSL>
-				<LoginName><?php echo $LOGIN ?></LoginName>
-			</Protocol>
-
-			<Protocol>
-				<Type>SMTP</Type>
-				<Server><?php echo $PRIMARY_HOSTNAME ?></Server>
-				<Port>587</Port>
-				<SSL>on</SSL>
-				<LoginName><?php echo $LOGIN ?></LoginName>
-			</Protocol>
-
-			<Protocol>
-				<Type>DAV</Type>
-				<Server>https://<?php echo $PRIMARY_HOSTNAME ?></Server>
-				<SSL>on</SSL>
-				<DomainRequired>on</DomainRequired>
-				<LoginName><?php echo $LOGIN ?></LoginName>
-			</Protocol>
-
-			<Protocol>
-				<Type>WEB</Type>
-				<Server>https://<?php echo $PRIMARY_HOSTNAME ?>/mail</Server>
-				<SSL>on</SSL>
-			</Protocol>
-		</Account>
-	</Response>
-</Autodiscover>
-
-	<?php
-	}
-	?>
-

tools/mail.py

@@ -1,8 +1,8 @@
 #!/usr/bin/python3
 
-import sys, getpass, urllib.request, urllib.error
+import sys, getpass, urllib.request, urllib.error, json
 
-def mgmt(cmd, data=None):
+def mgmt(cmd, data=None, is_json=False):
 	mgmt_uri = 'http://localhost:10222'
 
 	setup_key_auth(mgmt_uri)
@@ -11,9 +11,20 @@ def mgmt(cmd, data=None):
 	try:
 		response = urllib.request.urlopen(req)
 	except urllib.error.HTTPError as e:
-		print(e.read().decode('utf8'))
+		if e.code == 401:
+			try:
+				print(e.read().decode("utf8"))
+			except:
+				pass
+			print("The management daemon refused access. The API key file may be out of sync. Try 'service mailinabox restart'.", file=sys.stderr)
+		elif hasattr(e, 'read'):
+			print(e.read().decode('utf8'), file=sys.stderr)
+		else:
+			print(e, file=sys.stderr)
 		sys.exit(1)
-	return response.read().decode('utf8')
+	resp = response.read().decode('utf8')
+	if is_json: resp = json.loads(resp)
+	return resp
 
 def read_password():
 	first  = getpass.getpass('password: ')
@@ -42,6 +53,9 @@ if len(sys.argv) < 2:
 	print("  tools/mail.py user add user@domain.com [password]")
 	print("  tools/mail.py user password user@domain.com [password]")
 	print("  tools/mail.py user remove user@domain.com")
+	print("  tools/mail.py user make-admin user@domain.com")
+	print("  tools/mail.py user remove-admin user@domain.com")
+	print("  tools/mail.py user admins (lists admins)")
 	print("  tools/mail.py alias  (lists aliases)")
 	print("  tools/mail.py alias add incoming.name@domain.com sent.to@other.domain.com")
 	print("  tools/mail.py alias remove incoming.name@domain.com")
@@ -50,7 +64,13 @@ if len(sys.argv) < 2:
 	print()
 
 elif sys.argv[1] == "user" and len(sys.argv) == 2:
-	print(mgmt("/mail/users"))
+	# Dump a list of users, one per line. Mark admins with an asterisk.
+	users = mgmt("/mail/users?format=json", is_json=True)
+	for user in users:
+		print(user['email'], end='')
+		if "admin" in user['privileges']:
+			print("*", end='')
+		print()
 
 elif sys.argv[1] == "user" and sys.argv[2] in ("add", "password"):
 	if len(sys.argv) < 5:
@@ -70,6 +90,20 @@ elif sys.argv[1] == "user" and sys.argv[2] in ("add", "password"):
 elif sys.argv[1] == "user" and sys.argv[2] == "remove" and len(sys.argv) == 4:
 	print(mgmt("/mail/users/remove", { "email": sys.argv[3] }))
 
+elif sys.argv[1] == "user" and sys.argv[2] in ("make-admin", "remove-admin") and len(sys.argv) == 4:
+	if sys.argv[2] == "make-admin":
+		action = "add"
+	else:
+		action = "remove"
+	print(mgmt("/mail/users/privileges/" + action, { "email": sys.argv[3], "privilege": "admin" }))
+
+elif sys.argv[1] == "user" and sys.argv[2] == "admins":
+	# Dump a list of admin users.
+	users = mgmt("/mail/users?format=json", is_json=True)
+	for user in users:
+		if "admin" in user['privileges']:
+			print(user['email'])
+
 elif sys.argv[1] == "alias" and len(sys.argv) == 2:
 	print(mgmt("/mail/aliases"))
 
@@ -81,4 +115,5 @@ elif sys.argv[1] == "alias" and sys.argv[2] == "remove" and len(sys.argv) == 4:
 
 else:
 	print("Invalid command-line arguments.")
+	sys.exit(1)
 

tools/webfinger.php

@@ -1,5 +1,9 @@
 <?php
-	$resource = $_GET['resource'];
+	$resource = '';
+
+	if(isset($_GET['resource'])){
+		$resource = $_GET['resource'];
+	}
 
 	// Parse our configuration file to get the STORAGE_ROOT.
 	$STORAGE_ROOT = NULL;