Merge branch 'master' of supplee.net:mailinabox-quota

.gitignore

@@ -5,3 +5,4 @@ tools/__pycache__/
 externals/
 .env
 .vagrant
+.idea/

CHANGELOG.md

@@ -1,6 +1,49 @@
 CHANGELOG
 =========
 
+v0.43 (September 1, 2019)
+-------------------------
+
+Security fixes:
+
+* A security issue was discovered in rsync backups. If you have enabled rsync backups, the file `id_rsa_miab` may have been copied to your backup destination. This file can be used to access your backup destination. If the file was copied to your backup destination, we recommend that you delete the file on your backup destination, delete `/root/.ssh/id_rsa_miab` on your Mail-in-a-Box, then re-run Mail-in-a-Box setup, and re-configure your SSH public key at your backup destination according to the instructions in the Mail-in-a-Box control panel.
+* Brute force attack prevention was missing for the managesieve service.
+
+Setup:
+
+* Nextcloud was not upgraded properly after restoring Mail-in-a-Box from a backup from v0.40 or earlier.
+
+Mail:
+
+* Upgraded Roundcube to 1.3.10.
+* Fetch an updated whitelist for greylisting on a monthly basis to reduce the number of delayed incoming emails.
+
+Control panel:
+
+* When using secondary DNS, it is now possible to specify a subnet range with the `xfr:` option.
+* Fixed an issue when the secondary DNS option is used and the secondary DNS hostname resolves to multiple IP addresses.
+* Fix a bug in how a backup configuration error is shown.
+
+v0.42b (August 3, 2019)
+-----------------------
+
+Changes:
+
+* Decreased the minimum supported RAM to 502 Mb.
+* Improved mail client autoconfiguration.
+* Added support for S3-compatible backup services besides Amazon S3.
+* Fixed the control panel login page to let LastPass save passwords.
+* Fixed an error in the user privileges API.
+* Silenced some spurrious messages.
+
+Software updates:
+
+* Upgraded Roundcube from 1.3.8 to 1.3.9.
+* Upgraded Nextcloud from 14.0.6 to 15.0.8 (with Contacts from 2.1.8 to 3.1.1 and Calendar from 1.6.4 to 1.6.5).
+* Upgraded Z-Push from 2.4.4 to 2.5.0.
+
+Note that v0.42 (July 4, 2019) was pulled shortly after it was released to fix a Nextcloud upgrade issue.
+
 v0.41 (February 26, 2019)
 -------------------------
 

README.md

@@ -20,8 +20,8 @@ Mailbox size recalculation by Dovecot can be forced using the command:
 Please report any bugs on github.
 
 
-Installing v0.41-quota
+Installing v0.42b-quota
-----------------------
+-----------------------
 
 To install the latest version, log into your box and execute the following commands:
 
@@ -34,8 +34,8 @@ Follow the standard directions for setting up an MiaB installation.  There are n
 The default quota is set to `0` which means unlimited.  If you want to set a different default quota, follow the directions above.
 
 
-Upgrading v0.41 to v.0.41-quota
+Upgrading v0.4x to v.0.42b-quota
--------------------------------
+--------------------------------
 
 This is experimental software.  You have been warned.
 
@@ -51,7 +51,7 @@ This is experimental software.  You have been warned.
 
 
 Upgrading MiaB with quotas to a New Version
----------------------------------------
+-------------------------------------------
 
 * `cd` into the `mailinabox` directory.
 
@@ -69,6 +69,18 @@ Issues
 Changes
 -------
 
+### v0.43-quota-0.19-beta
+
+* Add user quota API documentation to the mail users page
+
+### v0.43-quota-0.18-beta
+
+* Update to v0.43 of Mail-in-a-Box
+
+### v0.42b-quota-0.18-beta
+
+* Update to v0.42b of Mail-in-a-Box
+
 ### v0.41-quota-0.18-beta
 
 * Bump version to add a new annotated tag.  The last version had a plain tag which is not seen when checking for the latest version.
@@ -198,7 +210,7 @@ by him:
 	$ curl -s https://keybase.io/joshdata/key.asc | gpg --import
 	gpg: key C10BDD81: public key "Joshua Tauberer <jt@occams.info>" imported
 
-	$ git verify-tag v0.41
+	$ git verify-tag v0.43
 	gpg: Signature made ..... using RSA key ID C10BDD81
 	gpg: Good signature from "Joshua Tauberer <jt@occams.info>"
 	gpg: WARNING: This key is not certified with a trusted signature!
@@ -211,7 +223,7 @@ and on his [personal homepage](https://razor.occams.info/). (Of course, if this
 
 Checkout the tag corresponding to the most recent release:
 
-	$ git checkout v0.41
+	$ git checkout v0.43
 
 Begin the installation.
 

conf/cronjob/dovecot

@@ -0,0 +1 @@
+/usr/bin/doveadm fts rescan -A

conf/cronjob/solr

@@ -0,0 +1,2 @@
+*/1 * * * * root /usr/bin/curl http://127.0.0.1:8080/solr/update?commit=true &>/dev/null
+30 3 * * * root /usr/bin/curl http://127.0.0.1:8080/solr/update?optimize=true &>/dev/null

conf/fail2ban/filter.d/dovecotimap.conf

@@ -1,4 +1,4 @@
-# Fail2Ban filter Dovecot authentication and pop3/imap server
+# Fail2Ban filter Dovecot authentication and pop3/imap/managesieve server
 # For Mail-in-a-Box
 
 [INCLUDES]
@@ -9,7 +9,7 @@ before = common.conf
 
 _daemon = (auth|dovecot(-auth)?|auth-worker)
 
-failregex = ^%(__prefix_line)s(pop3|imap)-login: (Info: )?(Aborted login|Disconnected)(: Inactivity)? \(((no auth attempts|auth failed, \d+ attempts)( in \d+ secs)?|tried to use (disabled|disallowed) \S+ auth)\):( user=<\S*>,)?( method=\S+,)? rip=<HOST>, lip=(\d{1,3}\.){3}\d{1,3}(, TLS( handshaking)?(: Disconnected)?)?(, session=<\S+>)?\s*$
+failregex = ^%(__prefix_line)s(pop3|imap|managesieve)-login: (Info: )?(Aborted login|Disconnected)(: Inactivity)? \(((no auth attempts|auth failed, \d+ attempts)( in \d+ secs)?|tried to use (disabled|disallowed) \S+ auth)\):( user=<\S*>,)?( method=\S+,)? rip=<HOST>, lip=(\d{1,3}\.){3}\d{1,3}(, TLS( handshaking)?(: Disconnected)?)?(, session=<\S+>)?\s*$
 
 ignoreregex =
 

conf/nginx-alldomains.conf

@@ -18,6 +18,9 @@
 	location = /.well-known/autoconfig/mail/config-v1.1.xml {
 		alias /var/lib/mailinabox/mozilla-autoconfig.xml;
 	}
+	location = /mail/config-v1.1.xml {
+		alias /var/lib/mailinabox/mozilla-autoconfig.xml;
+	}
 
 	# Roundcube Webmail configuration.
 	rewrite ^/mail$ /mail/ redirect;

conf/nginx-primaryonly.conf

@@ -19,6 +19,7 @@
 	rewrite ^/cloud/$ /cloud/index.php;
 	rewrite ^/cloud/(contacts|calendar|files)$ /cloud/index.php/apps/$1/ redirect;
 	rewrite ^(/cloud/core/doc/[^\/]+/)$ $1/index.html;
+	rewrite ^(/cloud/oc[sm]-provider)/$ $1/index.php redirect;
 	location /cloud/ {
 		alias /usr/local/lib/owncloud/;
 	 	location ~ ^/cloud/(build|tests|config|lib|3rdparty|templates|data|README)/ {
@@ -27,6 +28,14 @@
 	 	location ~ ^/cloud/(?:\.|autotest|occ|issue|indie|db_|console) {
 	 		deny all;
 	 	}
+		# Enable paths for service and cloud federation discovery
+		# Resolves warning in Nextcloud Settings panel
+                location ~ ^/cloud/(oc[sm]-provider)?/([^/]+\.php)$ {
+                        index index.php;
+                        include fastcgi_params;
+                        fastcgi_param SCRIPT_FILENAME /usr/local/lib/owncloud/$1/$2;
+                        fastcgi_pass php-fpm;
+                }
 	}
 	location ~ ^(/cloud)((?:/ocs)?/[^/]+\.php)(/.*)?$ {
 		# note: ~ has precendence over a regular location block

conf/nginx.conf

@@ -1,5 +1,6 @@
 ## $HOSTNAME
 
+#BEGIN_HTTP
 # Redirect all HTTP to HTTPS *except* the ACME challenges (Let's Encrypt TLS certificate
 # domain validation challenges) path, which must be served over HTTP per the ACME spec
 # (due to some Apache vulnerability).
@@ -28,11 +29,12 @@ server {
 		alias $STORAGE_ROOT/ssl/lets_encrypt/webroot/.well-known/acme-challenge/;
 	}
 }
+#END_HTTP
 
 # The secure HTTPS server.
 server {
-	listen 443 ssl http2;
+	listen $HTTP_SSL_PORT ssl http2;
-	listen [::]:443 ssl http2;
+	listen [::]:$HTTP_SSL_PORT ssl http2;
 
 	server_name $HOSTNAME;
 

management/auth.py

@@ -59,7 +59,7 @@ class KeyAuthService:
 
 			credentials = decode(credentials)
 			if ":" not in credentials:
-				return None, None
+				return credentials, None
 			username, password = credentials.split(':', maxsplit=1)
 			return username, password
 

management/backup.py

@@ -15,7 +15,7 @@ from exclusiveprocess import Lock
 from utils import load_environment, shell, wait_for_service, fix_boto
 
 rsync_ssh_options = [
-	"--ssh-options='-i /root/.ssh/id_rsa_miab'",
+	"--ssh-options= -i /root/.ssh/id_rsa_miab",
 	"--rsync-options= -e \"/usr/bin/ssh -oStrictHostKeyChecking=no -oBatchMode=yes -p 22 -i /root/.ssh/id_rsa_miab\"",
 ]
 
@@ -406,7 +406,7 @@ def list_target_files(config):
 				reason = "Provided path {} is invalid.".format(target_path)
 			elif 'Network is unreachable' in listing:
 				reason = "The IP address {} is unreachable.".format(target.hostname)
-			elif 'Could not resolve hostname':
+			elif 'Could not resolve hostname' in listing:
 				reason = "The hostname {} cannot be resolved.".format(target.hostname)
 			else:
 				reason = "Unknown error." \
@@ -419,15 +419,22 @@ def list_target_files(config):
 		fix_boto() # must call prior to importing boto
 		import boto.s3
 		from boto.exception import BotoServerError
+		custom_region = False
 		for region in boto.s3.regions():
 			if region.endpoint == target.hostname:
 				break
 		else:
-			raise ValueError("Invalid S3 region/host.")
+			# If region is not found this is a custom region
+			custom_region = True
 
 		bucket = target.path[1:].split('/')[0]
 		path = '/'.join(target.path[1:].split('/')[1:]) + '/'
 
+		# Create a custom region with custom endpoint
+		if custom_region:
+			from boto.s3.connection import S3Connection
+			region = boto.s3.S3RegionInfo(name=bucket, endpoint=target.hostname, connection_cls=S3Connection)
+
 		# If no prefix is specified, set the path to '', otherwise boto won't list the files
 		if path == '/':
 			path = ''

management/daemon.py

@@ -1,5 +1,7 @@
 import os, os.path, re, json, time
 import subprocess
+import base64
+import sys
 
 from functools import wraps
 
@@ -347,6 +349,34 @@ def dns_get_dump():
 	from dns_update import build_recommended_dns
 	return json_response(build_recommended_dns(env))
 
+@app.route('/letsencrypt/dns-auth/<domain>/<token>', methods=['GET'])
+@authorized_personnel_only
+def letsencrypt_dns_auth(domain, token):
+	from dns_update import do_dns_update, set_custom_dns_record
+	try:
+		qname = '_acme-challenge.' + domain
+		if set_custom_dns_record(qname, 'TXT', token, 'add', env):
+			if not do_dns_update(env):
+				return ("Error updating DNS", 400)
+		return "OK"
+
+	except ValueError as e:
+		return (str(e), 400)
+
+@app.route('/letsencrypt/dns-cleanup/<domain>', methods=['GET'])
+@authorized_personnel_only
+def letsencrypt_dns_cleanup(domain):
+	from dns_update import do_dns_update, set_custom_dns_record
+	try:
+		qname = '_acme-challenge.' + domain
+		if set_custom_dns_record(qname, 'TXT', None, 'remove', env):
+			if not do_dns_update(env):
+				return ("Error updating DNS", 400)
+		return "OK"
+
+	except ValueError as e:
+		return (str(e), 400)
+
 # SSL
 
 @app.route('/ssl/status')
@@ -543,6 +573,9 @@ def privacy_status_set():
 	utils.write_settings(config, env)
 	return "OK"
 
+
+# Quotas
+
 @app.route('/system/default-quota', methods=["GET"])
 @authorized_personnel_only
 def default_quota_get():
@@ -566,6 +599,37 @@ def default_quota_set():
 
     return "OK"
 
+
+# Mailgraph
+
+@app.route('/mailgraph/image.cgi', methods=['GET'])
+@authorized_personnel_only
+def mailgraph():
+	if request.query_string:
+		query = request.query_string.decode('utf-8', 'ignore')
+		if '&' in query:
+			query = query.split('&')[0]
+
+		print("QUERY_STRING=%s" % query, file=sys.stderr)
+
+		code, bin_out = utils.shell(
+			"check_output",
+			["/usr/share/mailgraph/mailgraph.cgi"],
+			env={"QUERY_STRING": query},
+			return_bytes=True,
+			trap=True
+		)
+
+		if code != 0:
+			return ('Error generating mailgraph image: %s' % query, 500)
+
+		headers, image_bytes = bin_out.split(b'\n\n', 1)
+
+		return base64.b64encode(image_bytes)
+
+	return ('Mailgraph: no image requested', 500)
+
+
 # MUNIN
 
 @app.route('/munin/')

management/dns_update.py

@@ -288,6 +288,21 @@ def build_zone(domain, all_domains, additional_records, www_redirect_domains, en
 			if not has_rec(qname, "SRV"):
 				records.append((qname, "SRV", "0 0 443 " + env["PRIMARY_HOSTNAME"] + ".", "Recommended. Specifies the hostname of the server that handles CardDAV/CalDAV services for email addresses on this domain."))
 
+	# Adds autoconfiguration A records for all domains.
+	# This allows the following clients to automatically configure email addresses in the respective applications.
+	# autodiscover.* - Z-Push ActiveSync Autodiscover
+	# autoconfig.* - Thunderbird Autoconfig
+	autodiscover_records = [
+		("autodiscover", "A", env["PUBLIC_IP"], "Provides email configuration autodiscovery support for Z-Push ActiveSync Autodiscover."),
+		("autodiscover", "AAAA", env["PUBLIC_IPV6"], "Provides email configuration autodiscovery support for Z-Push ActiveSync Autodiscover."),
+		("autoconfig", "A", env["PUBLIC_IP"], "Provides email configuration autodiscovery support for Thunderbird Autoconfig."),
+		("autoconfig", "AAAA", env["PUBLIC_IPV6"], "Provides email configuration autodiscovery support for Thunderbird Autoconfig.")
+	]
+	for qname, rtype, value, explanation in autodiscover_records:
+		if value is None or value.strip() == "": continue # skip IPV6 if not set
+		if not has_rec(qname, rtype):
+			records.append((qname, rtype, value, explanation))
+
 	# Sort the records. The None records *must* go first in the nsd zone file. Otherwise it doesn't matter.
 	records.sort(key = lambda rec : list(reversed(rec[0].split(".")) if rec[0] is not None else ""))
 
@@ -888,10 +903,14 @@ def set_secondary_dns(hostnames, env):
 			else:
 				# Validate IP address.
 				try:
+					if "/" in item[4:]:
+						v = ipaddress.ip_network(item[4:]) # raises a ValueError if there's a problem
+						if not isinstance(v, ipaddress.IPv4Network): raise ValueError("That's an IPv6 subnet.")
+					else:
 						v = ipaddress.ip_address(item[4:]) # raises a ValueError if there's a problem
 						if not isinstance(v, ipaddress.IPv4Address): raise ValueError("That's an IPv6 address.")
 				except ValueError:
-					raise ValueError("'%s' is not an IPv4 address." % item[4:])
+					raise ValueError("'%s' is not an IPv4 address or subnet." % item[4:])
 
 		# Set.
 		set_custom_dns_record("_secondary_nameserver", "A", " ".join(hostnames), "set", env)

management/mailconfig.py

@@ -195,7 +195,7 @@ def get_mail_users_ex(env, with_archived=False):
 					if email in active_accounts: continue
 					user = {
 						"email": email,
-						"privileges": "",
+						"privileges": [],
 						"status": "inactive",
 						"mailbox": mbox,
 					}

management/ssl_certificates.py

@@ -313,6 +313,7 @@ def provision_certificates(env, limit_domains):
 				webroot = os.path.join(account_path, 'webroot')
 				os.makedirs(webroot, exist_ok=True)
 				with tempfile.TemporaryDirectory() as d:
+					miab_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 					cert_file = os.path.join(d, 'cert_and_chain.pem')
 					print("Provisioning TLS certificates for " + ", ".join(domain_list) + ".")
 					certbotret = subprocess.check_output([
@@ -328,7 +329,10 @@ def provision_certificates(env, limit_domains):
 						"--chain-path", os.path.join(d, 'chain'), # we only use the full chain
 						"--fullchain-path", cert_file,
 
-						"--webroot", "--webroot-path", webroot,
+						"--manual",
+						"--preferred-challenge", "dns",
+						"--manual-auth-hook", os.path.join(miab_dir, "tools/dns-auth.sh"),
+						"--manual-cleanup-hook", os.path.join(miab_dir, "tools/dns-cleanup.sh"),
 
 						"--config-dir", account_path,
 						#"--staging",

management/status_checks.py

@@ -39,6 +39,7 @@ def get_services():
 		{ "name": "Mail Filters (Sieve/dovecot)", "port": 4190, "public": True, },
 		{ "name": "HTTP Web (nginx)", "port": 80, "public": True, },
 		{ "name": "HTTPS Web (nginx)", "port": 443, "public": True, },
+	    { "name": "Solr Full Text Search (tomcat)", "port": 8080, "public": False, },
 	]
 
 def run_checks(rounded_values, env, output, pool):
@@ -487,10 +488,12 @@ def check_dns_zone(domain, env, output, dns_zonefiles):
 	if custom_secondary_ns and not probably_external_dns:
 		for ns in custom_secondary_ns:
 			# We must first resolve the nameserver to an IP address so we can query it.
-			ns_ip = query_dns(ns, "A")
+			ns_ips = query_dns(ns, "A")
-			if not ns_ip:
+			if not ns_ips:
 				output.print_error("Secondary nameserver %s is not valid (it doesn't resolve to an IP address)." % ns)
 				continue
+			# Choose the first IP if nameserver returns multiple
+			ns_ip = ns_ips.split('; ')[0]
 
 			# Now query it to see what it says about this domain.
 			ip = query_dns(domain, "A", at=ns_ip, nxdomain=None)

management/templates/custom-dns.html

@@ -90,7 +90,7 @@
     <div class="col-sm-offset-1 col-sm-11">
       <p class="small">
         Multiple secondary servers can be separated with commas or spaces (i.e., <code>ns2.hostingcompany.com ns3.hostingcompany.com</code>). 
-        To enable zone transfers to additional servers without listing them as secondary nameservers, add <code>xfr:IPADDRESS</code>.
+        To enable zone transfers to additional servers without listing them as secondary nameservers, add an IP address or subnet using <code>xfr:10.20.30.40</code> or <code>xfr:10.20.30.40/24</code>.
       </p>
       <p id="secondarydns-clear-instructions" style="display: none" class="small">
         Clear the input field above and click Update to use this machine itself as secondary DNS, which is the default/normal setup.

management/templates/index.html

@@ -102,6 +102,7 @@
                 <li><a href="#mail-guide" onclick="return show_panel(this);">Instructions</a></li>
                 <li><a href="#users" onclick="return show_panel(this);">Users</a></li>
                 <li><a href="#aliases" onclick="return show_panel(this);">Aliases</a></li>
+                <li><a href="#mailgraph" onclick="return show_panel(this);">Mailgraph</a></li>
               </ul>
             </li>
             <li><a href="#sync_guide" onclick="return show_panel(this);">Contacts/Calendar</a></li>
@@ -151,6 +152,10 @@
       {% include "sync-guide.html" %}
       </div>
 
+      <div id="panel_mailgraph" class="admin_panel">
+      {% include "mailgraph.html" %}
+      </div>
+
       <div id="panel_web" class="admin_panel">
       {% include "web.html" %}
       </div>

management/templates/login.html

@@ -23,7 +23,7 @@ sudo tools/mail.py user make-admin me@{{hostname}}</pre>
 <p style="margin: 2em; text-align: center;">Log in here for your Mail-in-a-Box control panel.</p>
 
 <div style="margin: 0 auto; max-width: 32em;">
-  <form class="form-horizontal" role="form" onsubmit="do_login(); return false;">
+  <form class="form-horizontal" role="form" onsubmit="do_login(); return false;" method="get">
     <div class="form-group">
       <label for="inputEmail3" class="col-sm-3 control-label">Email</label>
       <div class="col-sm-9">

management/templates/mailgraph.html

@@ -0,0 +1,48 @@
+<h2>Mail statistics</h2>
+<ul id="jump">
+  <li><a href="#G0">Day</a>&nbsp;</li>
+  <li><a href="#G1">Week</a>&nbsp;</li>
+  <li><a href="#G2">Month</a>&nbsp;</li>
+  <li><a href="#G3">Year</a>&nbsp;</li>
+</ul>
+
+<h3 id="G0">Last Day</h3>
+<p><img src="" data-src="/mailgraph/image.cgi?0-n" alt="mailgraph"/></p>
+<p><img src="" data-src="/mailgraph/image.cgi?0-e" alt="mailgraph"/></p>
+<p><img src="" data-src="/mailgraph/image.cgi?0-g" alt="mailgraph"/></p>
+
+<h3 id="G1">Last Week</h3>
+<p><img src="" data-src="/mailgraph/image.cgi?1-n" alt="mailgraph"/></p>
+<p><img src="" data-src="/mailgraph/image.cgi?1-e" alt="mailgraph"/></p>
+<p><img src="" data-src="/mailgraph/image.cgi?1-g" alt="mailgraph"/></p>
+
+<h3 id="G2">Last Month</h3>
+<p><img src="" data-src="/mailgraph/image.cgi?2-n" alt="mailgraph"/></p>
+<p><img src="" data-src="/mailgraph/image.cgi?2-e" alt="mailgraph"/></p>
+<p><img src="" data-src="/mailgraph/image.cgi?2-g" alt="mailgraph"/></p>
+
+<h3 id="G3">Last Year</h3>
+<p><img src="" data-src="/mailgraph/image.cgi?3-n" alt="mailgraph"/></p>
+<p><img src="" data-src="/mailgraph/image.cgi?3-e" alt="mailgraph"/></p>
+<p><img src="" data-src="/mailgraph/image.cgi?3-g" alt="mailgraph"/></p>
+
+<hr/>
+
+<p><a href="http://mailgraph.schweikert.ch/">Mailgraph</a> 1.14 by <a href="http://david.schweikert.ch/">David Schweikert</a>
+(built on Tobi Oetiker's <a href="http://oss.oetiker.ch/rrdtool/">RRDtool</a>)</p>
+
+<script type="text/javascript">
+    function show_mailgraph() {
+        $('[data-src]').each(function() {
+            var that = this;
+            api(
+                $(that).attr('data-src'),
+                'GET',
+                '',
+                function(data) {
+                    $(that).attr('src', 'data:image/gif;base64,' + data);
+                }
+            );
+        });
+    }
+</script>

management/templates/system-backup.html

@@ -77,15 +77,22 @@
     </div>
   </div>
   <div class="form-group backup-target-s3">
-    <label for="backup-target-s3-host" class="col-sm-2 control-label">S3 Region</label>
+    <label for="backup-target-s3-host-select" class="col-sm-2 control-label">S3 Region</label>
     <div class="col-sm-8">
-      <select class="form-control" rows="1" id="backup-target-s3-host">
+      <select class="form-control" rows="1" id="backup-target-s3-host-select">
         {% for name, host in backup_s3_hosts %}
           <option value="{{host}}">{{name}}</option>
         {% endfor %}
+        <option value="other">Other</option>
       </select>
     </div>
   </div>
+  <div class="form-group backup-target-s3">
+    <label for="backup-target-s3-host" class="col-sm-2 control-label">S3 Host / Endpoint</label>
+    <div class="col-sm-8">
+      <input type="text" placeholder="Endpoint" class="form-control" rows="1" id="backup-target-s3-host">
+    </div>
+  </div>
   <div class="form-group backup-target-s3">
     <label for="backup-target-s3-path" class="col-sm-2 control-label">S3 Path</label>
     <div class="col-sm-8">
@@ -139,6 +146,8 @@ function toggle_form() {
   var target_type = $("#backup-target-type").val();
   $(".backup-target-local, .backup-target-rsync, .backup-target-s3").hide();
   $(".backup-target-" + target_type).show();
+
+  init_inputs(target_type);
 }
 
 function nice_size(bytes) {
@@ -278,4 +287,20 @@ function set_custom_backup() {
     });
   return false;
 }
+
+function init_inputs(target_type) {
+  function set_host(host) {
+    if(host !== 'other') {
+      $("#backup-target-s3-host").val(host);
+    } else {
+      $("#backup-target-s3-host").val('');
+    }
+  }
+  if (target_type == "s3") {
+    $('#backup-target-s3-host-select').off('change').on('change', function() {
+      set_host($('#backup-target-s3-host-select').val());
+    });
+    set_host($('#backup-target-s3-host-select').val());
+  }
+}
 </script>

management/templates/users.html

@@ -117,10 +117,24 @@
 <table class="table" style="margin-top: .5em">
 <thead><th>Verb</th> <th>Action</th><th></th></thead>
 <tr><td>GET</td><td><i>(none)</i></td> <td>Returns a list of existing mail users. Adding <code>?format=json</code> to the URL will give JSON-encoded results.</td></tr>
-<tr><td>POST</td><td>/add</td> <td>Adds a new mail user. Required POST-body parameters are <code>email</code> and <code>password</code>.</td></tr>
+<tr>
+    <td>POST</td>
+    <td>/add</td>
+    <td>Adds a new mail user. Required POST-body parameters are <code>email</code> and <code>password</code>. Optional parameters: <code>privilege=admin</code> and <code>quota</code></td>
+</tr>
 <tr><td>POST</td><td>/remove</td> <td>Removes a mail user. Required POST-by parameter is <code>email</code>.</td></tr>
 <tr><td>POST</td><td>/privileges/add</td> <td>Used to make a mail user an admin. Required POST-body parameters are <code>email</code> and <code>privilege=admin</code>.</td></tr>
 <tr><td>POST</td><td>/privileges/remove</td> <td>Used to remove the admin privilege from a mail user. Required POST-body parameter is <code>email</code>.</td></tr>
+<tr>
+    <td>GET</td>
+    <td>/quota</td>
+    <td>Get the quota for a mail user. Required POST-body parameters are <code>email</code> and will return JSON result</td>
+</tr>
+<tr>
+    <td>POST</td>
+    <td>/quota</td>
+    <td>Set the quota for a mail user. Required POST-body parameters are <code>email</code> and <code>quota</code>.</td>
+</tr>
 </table>
 
 <h4>Examples:</h4>

management/web_update.py

@@ -29,6 +29,12 @@ def get_web_domains(env, include_www_redirects=True, exclude_dns_elsewhere=True)
 		# IP address than this box. Remove those domains from our list.
 		domains -= get_domains_with_a_records(env)
 
+	# Add Autoconfiguration domains, allowing us to serve correct SSL certs.
+	# 'autoconfig.' for Mozilla Thunderbird auto setup.
+	# 'autodiscover.' for Activesync autodiscovery.
+	domains |= set('autoconfig.' + maildomain for maildomain in get_mail_domains(env))
+	domains |= set('autodiscover.' + maildomain for maildomain in get_mail_domains(env))
+
 	# Ensure the PRIMARY_HOSTNAME is in the list so we can serve webmail
 	# as well as Z-Push for Exchange ActiveSync. This can't be removed
 	# by a custom A/AAAA record and is never a 'www.' redirect.
@@ -94,6 +100,20 @@ def do_web_update(env):
 			# Add default 'www.' redirect.
 			nginx_conf += make_domain_config(domain, [template0, template3], ssl_certificates, env)
 
+	if str(env['HTTP_SSL_PORT']) != "443":
+		in_http = False
+		new_conf = ''
+		for line in nginx_conf.split('\n'):
+			if line.strip() == '#BEGIN_HTTP':
+				in_http = True
+			elif line.strip() == '#END_HTTP':
+				in_http = False
+
+			if not in_http:
+				new_conf += line + '\n'
+
+		nginx_conf = new_conf
+
 	# Did the file change? If not, don't bother writing & restarting nginx.
 	nginx_conf_fn = "/etc/nginx/conf.d/local.conf"
 	if os.path.exists(nginx_conf_fn):
@@ -178,8 +198,12 @@ def make_domain_config(domain, templates, ssl_certificates, env):
 		nginx_conf = re.sub("[ \t]*# ADDITIONAL DIRECTIVES HERE *\n", t, nginx_conf)
 
 	# Replace substitution strings in the template & return.
+	if int(env['HTTP_SSL_PORT']) != 443:
+		# disable the regular HTTP server
+		nginx_conf = re.sub(r'#BEGIN_HTTP.*?#END_HTTP', repl='', string=nginx_conf, flags=re.MULTILINE)
 	nginx_conf = nginx_conf.replace("$STORAGE_ROOT", env['STORAGE_ROOT'])
 	nginx_conf = nginx_conf.replace("$HOSTNAME", domain)
+	nginx_conf = nginx_conf.replace("$HTTP_SSL_PORT", env['HTTP_SSL_PORT'])
 	nginx_conf = nginx_conf.replace("$ROOT", root)
 	nginx_conf = nginx_conf.replace("$SSL_KEY", tls_cert["private-key"])
 	nginx_conf = nginx_conf.replace("$SSL_CERTIFICATE", tls_cert["certificate"])

requirements.txt

@@ -0,0 +1,11 @@
+rtyaml
+email_validator>=1.0.0
+exclusiveprocess
+flask
+dnspython
+python-dateutil
+idna>=2.0.0
+cryptography==2.2.2
+boto
+psutil
+npyscreen

setup/bootstrap.sh

@@ -20,7 +20,7 @@ if [ -z "$TAG" ]; then
 	# want to display in status checks.
 	if [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/18\.04\.[0-9]/18.04/' `" == "Ubuntu 18.04 LTS" ]; then
 		# This machine is running Ubuntu 18.04.
-		TAG=v0.41-quota-0.18-beta
+		TAG=v0.43-quota-0.19-beta
 
 	elif [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/14\.04\.[0-9]/14.04/' `" == "Ubuntu 14.04 LTS" ]; then
 		# This machine is running Ubuntu 14.04.

setup/functions.sh

@@ -127,7 +127,7 @@ function get_default_privateip {
 	if [ "$1" == "6" ]; then target=2001:4860:4860::8888; fi
 
 	# Get the route information.
-	route=$(ip -$1 -o route get $target | grep -v unreachable)
+	route=$(ip -$1 -o route get $target 2>/dev/null | grep -v unreachable)
 
 	# Parse the address out of the route information.
 	address=$(echo $route | sed "s/.* src \([^ ]*\).*/\1/")

setup/mail-dovecot.sh

@@ -82,11 +82,12 @@ tools/editconf.py /etc/dovecot/conf.d/10-auth.conf \
 # Enable SSL, specify the location of the SSL certificate and private key files.
 # Disable obsolete SSL protocols and allow only good ciphers per http://baldric.net/2013/12/07/tls-ciphers-in-postfix-and-dovecot/.
 # Enable strong ssl dh parameters
+
 tools/editconf.py /etc/dovecot/conf.d/10-ssl.conf \
 	ssl=required \
 	"ssl_cert=<$STORAGE_ROOT/ssl/ssl_certificate.pem" \
 	"ssl_key=<$STORAGE_ROOT/ssl/ssl_private_key.pem" \
-	"ssl_protocols=!SSLv3 !SSLv2" \
+	"ssl_protocols=!SSLv3" \
 	"ssl_cipher_list=ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS" \
 	"ssl_prefer_server_ciphers = yes" \
 	"ssl_dh_parameters_length = 2048"
@@ -137,6 +138,14 @@ service lmtp {
   }
 }
 
+# Enable imap-login on localhost to allow the user_external plugin
+# for Nextcloud to do imap authentication. (See #1577)
+service imap-login {
+  inet_listener imap {
+    address = 127.0.0.1
+    port = 143
+  }
+}
 protocol imap {
   mail_max_userip_connections = 20
 }

setup/mail-postfix.sh

@@ -42,7 +42,8 @@ source /etc/mailinabox.conf # load global vars
 # * `ca-certificates`: A trust store used to squelch postfix warnings about
 #   untrusted opportunistically-encrypted connections.
 echo "Installing Postfix (SMTP server)..."
-apt_install postfix postfix-sqlite postfix-pcre postgrey ca-certificates
+apt_install postfix postfix-sqlite postfix-pcre postgrey ca-certificates \
+	postfix-policyd-spf-python postsrsd
 
 # ### Basic Settings
 
@@ -97,7 +98,9 @@ tools/editconf.py /etc/postfix/master.cf -s -w \
 	  -o cleanup_service_name=authclean" \
 	"authclean=unix  n       -       -       -       0       cleanup
 	  -o header_checks=pcre:/etc/postfix/outgoing_mail_header_filters
-	  -o nested_header_checks="
+	  -o nested_header_checks=" \
+	"policy-spf=unix  -       n       n       -       -       spawn
+	  user=nobody argv=/usr/bin/policyd-spf"
 
 # Install the `outgoing_mail_header_filters` file required by the new 'authclean' service.
 cp conf/postfix_outgoing_mail_header_filters /etc/postfix/outgoing_mail_header_filters
@@ -196,9 +199,23 @@ tools/editconf.py /etc/postfix/main.cf lmtp_destination_recipient_limit=1
 # so these IPs get mail delivered quickly. But when an IP is not listed in the permit_dnswl_client list (i.e. it is not #NODOC
 # whitelisted) then postfix does a DEFER_IF_REJECT, which results in all "unknown user" sorts of messages turning into #NODOC
 # "450 4.7.1 Client host rejected: Service unavailable". This is a retry code, so the mail doesn't properly bounce. #NODOC
-tools/editconf.py /etc/postfix/main.cf \
+
-	smtpd_sender_restrictions="reject_non_fqdn_sender,reject_unknown_sender_domain,reject_authenticated_sender_login_mismatch,reject_rhsbl_sender dbl.spamhaus.org" \
+postconf -e smtpd_sender_restrictions="reject_non_fqdn_sender,reject_unknown_sender_domain,reject_authenticated_sender_login_mismatch,reject_rhsbl_sender dbl.spamhaus.org"
-	smtpd_recipient_restrictions=permit_sasl_authenticated,permit_mynetworks,"reject_rbl_client zen.spamhaus.org",reject_unlisted_recipient,"check_policy_service inet:127.0.0.1:10023","check_policy_service inet:127.0.0.1:12340"
+
+RECIPIENT_RESTRICTIONS="permit_sasl_authenticated,permit_mynetworks,reject_rbl_client zen.spamhaus.org,reject_unlisted_recipient"
+
+if [ $POSTGREY == 1 ]; then
+    RECIPIENT_RESTRICTIONS="${RECIPIENT_RESTRICTIONS},check_policy_service inet:127.0.0.1:10023"
+fi
+
+if [ $POLICY_SPF == 1 ]; then
+    RECIPIENT_RESTRICTIONS="${RECIPIENT_RESTRICTIONS},check_policy_service unix:private/policy-spf"
+fi
+
+# Add quota check
+RECIPIENT_RESTRICTIONS="${RECIPIENT_RESTRICTIONS},check_policy_service inet:127.0.0.1:12340"
+
+postconf -e smtpd_recipient_restrictions="$RECIPIENT_RESTRICTIONS"
 
 # Postfix connects to Postgrey on the 127.0.0.1 interface specifically. Ensure that
 # Postgrey listens on the same interface (and not IPv6, for instance).
@@ -208,13 +225,63 @@ tools/editconf.py /etc/postfix/main.cf \
 # e-mails really latter, delay of greylisting has been set to
 # 180 seconds (default is 300 seconds).
 tools/editconf.py /etc/default/postgrey \
-	POSTGREY_OPTS=\"'--inet=127.0.0.1:10023 --delay=180'\"
+	POSTGREY_OPTS=\"'--inet=127.0.0.1:10023 --delay=180 --whitelist-recipients=/etc/postgrey/whitelist_clients'\"
+
+
+# We are going to setup a newer whitelist for postgrey, the version included in the distribution is old
+cat > /etc/cron.daily/mailinabox-postgrey-whitelist << EOF;
+#!/bin/bash
+
+# Mail-in-a-Box
+
+# check we have a postgrey_whitelist_clients file and that it is not older than 28 days
+if [ ! -f /etc/postgrey/whitelist_clients ] || find /etc/postgrey/whitelist_clients -mtime +28 > /dev/null ; then
+    # ok we need to update the file, so lets try to fetch it
+    if curl https://postgrey.schweikert.ch/pub/postgrey_whitelist_clients --output /tmp/postgrey_whitelist_clients -sS --fail > /dev/null 2>&1 ; then
+        # if fetching hasn't failed yet then check it is a plain text file
+        # curl manual states that --fail sometimes still produces output
+        # this final check will at least check the output is not html
+        # before moving it into place
+        if [ "\$(file -b --mime-type /tmp/postgrey_whitelist_clients)" == "text/plain" ]; then
+            mv /tmp/postgrey_whitelist_clients /etc/postgrey/whitelist_clients
+            service postgrey restart
+	else
+            rm /tmp/postgrey_whitelist_clients
+        fi
+    fi
+fi
+EOF
+chmod +x /etc/cron.daily/mailinabox-postgrey-whitelist
+/etc/cron.daily/mailinabox-postgrey-whitelist
 
 # Increase the message size limit from 10MB to 128MB.
 # The same limit is specified in nginx.conf for mail submitted via webmail and Z-Push.
 tools/editconf.py /etc/postfix/main.cf \
 	message_size_limit=134217728
 
+if [ $POSTSRSD == 1 ]; then
+    # Setup SRS
+    postconf -e \
+        sender_canonical_maps=tcp:localhost:10001 \
+        sender_canonical_classes=envelope_sender \
+        recipient_canonical_maps=tcp:localhost:10002 \
+        recipient_canonical_classes=envelope_recipient,header_recipient
+
+    hide_output systemctl enable postsrsd
+    hide_output systemctl restart postsrsd
+
+else
+    postconf -e \
+        sender_canonical_maps= \
+        sender_canonical_classes= \
+        recipient_canonical_maps= \
+        recipient_canonical_classes=
+
+    hide_output systemctl disable postsrsd
+    hide_output systemctl stop postsrsd
+fi
+
+
 # Allow the two SMTP ports in the firewall.
 
 ufw_allow smtp
@@ -223,4 +290,11 @@ ufw_allow submission
 # Restart services
 
 restart_service postfix
-restart_service postgrey
+
+if [ $POSTGREY == 1 ]; then
+    hide_output systemctl enable postgrey
+    hide_output systemctl restart postgrey
+else
+    hide_output systemctl disable postgrey
+    hide_output systemctl stop postgrey
+fi

setup/management.sh

@@ -38,7 +38,7 @@ inst_dir=/usr/local/lib/mailinabox
 mkdir -p $inst_dir
 venv=$inst_dir/env
 if [ ! -d $venv ]; then
-	virtualenv -ppython3 $venv
+	hide_output virtualenv -ppython3 $venv
 fi
 
 # Upgrade pip because the Ubuntu-packaged version is out of date.

setup/munin.sh

@@ -76,4 +76,8 @@ restart_service munin-node
 # generate initial statistics so the directory isn't empty
 # (We get "Pango-WARNING **: error opening config file '/root/.config/pango/pangorc': Permission denied"
 # if we don't explicitly set the HOME directory when sudo'ing.)
+# We check to see if munin-cron is already running, if it is, there is no need to run it simultaneously
+# generating an error.
+if [ ! -f /var/run/munin/munin-update.lock ]; then
 	sudo -H -u munin munin-cron
+fi

setup/nextcloud.sh

@@ -13,7 +13,8 @@ apt-get purge -qq -y owncloud* # we used to use the package manager
 
 apt_install php php-fpm \
 	php-cli php-sqlite3 php-gd php-imap php-curl php-pear curl \
-	php-dev php-gd php-xml php-mbstring php-zip php-apcu php-json php-intl
+	php-dev php-gd php-xml php-mbstring php-zip php-apcu php-json \
+	php-intl php-imagick
 
 InstallNextcloud() {
 
@@ -24,12 +25,12 @@ InstallNextcloud() {
 	echo "Upgrading to Nextcloud version $version"
 	echo
 
-	# Remove the current owncloud/Nextcloud
-	rm -rf /usr/local/lib/owncloud
-
         # Download and verify
         wget_verify https://download.nextcloud.com/server/releases/nextcloud-$version.zip $hash /tmp/nextcloud.zip
 
+	# Remove the current owncloud/Nextcloud
+	rm -rf /usr/local/lib/owncloud
+
 	# Extract ownCloud/Nextcloud
 	unzip -q /tmp/nextcloud.zip -d /usr/local/lib
 	mv /usr/local/lib/nextcloud /usr/local/lib/owncloud
@@ -39,14 +40,22 @@ InstallNextcloud() {
 	# their github repositories.
 	mkdir -p /usr/local/lib/owncloud/apps
 
-	wget_verify https://github.com/nextcloud/contacts/releases/download/v2.1.8/contacts.tar.gz b5d5bbee33f0c32b124b46cb6aaab90c695ac170 /tmp/contacts.tgz
+	wget_verify https://github.com/nextcloud/contacts/releases/download/v3.1.1/contacts.tar.gz a06bd967197dcb03c94ec1dbd698c037018669e5 /tmp/contacts.tgz
 	tar xf /tmp/contacts.tgz -C /usr/local/lib/owncloud/apps/
 	rm /tmp/contacts.tgz
 
-	wget_verify https://github.com/nextcloud/calendar/releases/download/v1.6.4/calendar.tar.gz d8a7950dba14803472b6c19625a8ceb23d6fd4ef /tmp/calendar.tgz
+	wget_verify https://github.com/nextcloud/calendar/releases/download/v1.6.5/calendar.tar.gz 79941255521a5172f7e4ce42dc7773838b5ede2f /tmp/calendar.tgz
 	tar xf /tmp/calendar.tgz -C /usr/local/lib/owncloud/apps/
 	rm /tmp/calendar.tgz
 
+	# Starting with Nextcloud 15, the app user_external is no longer included in Nextcloud core,
+	# we will install from their github repository.
+	if [[ $version =~ ^15 ]]; then
+		wget_verify https://github.com/nextcloud/user_external/releases/download/v0.6.3/user_external-0.6.3.tar.gz 0f756d35fef6b64a177d6a16020486b76ea5799c /tmp/user_external.tgz
+		tar -xf /tmp/user_external.tgz -C /usr/local/lib/owncloud/apps/
+		rm /tmp/user_external.tgz
+	fi
+
 	# Fix weird permissions.
 	chmod 750 /usr/local/lib/owncloud/{apps,config}
 
@@ -75,15 +84,32 @@ InstallNextcloud() {
 
 		# Add missing indices. NextCloud didn't include this in the normal upgrade because it might take some time.
 		sudo -u www-data php /usr/local/lib/owncloud/occ db:add-missing-indices
+
+		# Run conversion to BigInt identifiers, this process may take some time on large tables.
+		sudo -u www-data php /usr/local/lib/owncloud/occ db:convert-filecache-bigint --no-interaction
 	fi
 }
 
-nextcloud_ver=14.0.6
+# Nextcloud Version to install. Checks are done down below to step through intermediate versions.
-nextcloud_hash=4e43a57340f04c2da306c8eea98e30040399ae5a
+nextcloud_ver=15.0.8
+nextcloud_hash=4129d8d4021c435f2e86876225fb7f15adf764a3
 
-# Check if Nextcloud dir exist, and check if version matches nextcloud_ver (if either doesn't - install/upgrade)
+# Current Nextcloud Version, #1623
-if [ ! -d /usr/local/lib/owncloud/ ] \
+# Checking /usr/local/lib/owncloud/version.php shows version of the Nextcloud application, not the DB
-		|| ! grep -q $nextcloud_ver /usr/local/lib/owncloud/version.php; then
+# $STORAGE_ROOT/owncloud is kept together even during a backup.  It is better to rely on config.php than
+# version.php since the restore procedure can leave the system in a state where you have a newer Nextcloud
+# application version than the database.
+
+# If config.php exists, get version number, otherwise CURRENT_NEXTCLOUD_VER is empty.
+if [ -f "$STORAGE_ROOT/owncloud/config.php" ]; then
+	CURRENT_NEXTCLOUD_VER=$(php -r "include(\"$STORAGE_ROOT/owncloud/config.php\"); echo(\$CONFIG['version']);")
+else
+	CURRENT_NEXTCLOUD_VER=""
+fi
+
+# If the Nextcloud directory is missing (never been installed before, or the nextcloud version to be installed is different
+# from the version currently installed, do the install/upgrade
+if [ ! -d /usr/local/lib/owncloud/ ] || [[ ! ${CURRENT_NEXTCLOUD_VER} =~ ^$nextcloud_ver ]]; then
 
 	# Stop php-fpm if running. If theyre not running (which happens on a previously failed install), dont bail.
 	service php7.2-fpm stop &> /dev/null || /bin/true
@@ -104,16 +130,22 @@ if [ ! -d /usr/local/lib/owncloud/ ] \
 	fi
 
 	# If ownCloud or Nextcloud was previously installed....
-	if [ -e /usr/local/lib/owncloud/version.php ]; then
+	if [ ! -z ${CURRENT_NEXTCLOUD_VER} ]; then
 		# Database migrations from ownCloud are no longer possible because ownCloud cannot be run under
 		# PHP 7.
-		if grep -q "OC_VersionString = '[89]\." /usr/local/lib/owncloud/version.php; then
+		if [[ ${CURRENT_NEXTCLOUD_VER} =~ ^[89] ]]; then
 			echo "Upgrades from Mail-in-a-Box prior to v0.28 (dated July 30, 2018) with Nextcloud < 13.0.6 (you have ownCloud 8 or 9) are not supported. Upgrade to Mail-in-a-Box version v0.30 first. Setup aborting."
 			exit 1
-		fi
+		elif [[ ${CURRENT_NEXTCLOUD_VER} =~ ^1[012] ]]; then
-		if grep -q "OC_VersionString = '1[012]\." /usr/local/lib/owncloud/version.php; then
 			echo "Upgrades from Mail-in-a-Box prior to v0.28 (dated July 30, 2018) with Nextcloud < 13.0.6 (you have ownCloud 10, 11 or 12) are not supported. Upgrade to Mail-in-a-Box version v0.30 first. Setup aborting."
 			exit 1
+		elif [[ ${CURRENT_NEXTCLOUD_VER} =~ ^13 ]]; then
+			# If we are running Nextcloud 13, upgrade to Nextcloud 14
+			InstallNextcloud 14.0.6 4e43a57340f04c2da306c8eea98e30040399ae5a
+		elif [[ ${CURRENT_NEXTCLOUD_VER} =~ ^14 ]]; then
+			# During the upgrade from Nextcloud 14 to 15, user_external may cause the upgrade to fail.
+			# We will disable it here before the upgrade and install it again after the upgrade.
+			hide_output sudo -u www-data php /usr/local/lib/owncloud/console.php app:disable user_external
 		fi
 	fi
 
@@ -144,8 +176,10 @@ if [ ! -f $STORAGE_ROOT/owncloud/owncloud.db ]; then
   'user_backends' => array(
     array(
       'class' => 'OC_User_IMAP',
-	  'arguments'=>array('{127.0.0.1:993/imap/ssl/novalidate-cert}')
+        'arguments' => array(
-	)
+          '127.0.0.1', 143, null
+         ),
+    ),
   ),
   'memcache.local' => '\OC\Memcache\APCu',
   'mail_smtpmode' => 'sendmail',
@@ -217,6 +251,8 @@ include("$STORAGE_ROOT/owncloud/config.php");
 
 \$CONFIG['mail_domain'] = '$PRIMARY_HOSTNAME';
 
+\$CONFIG['user_backends'] = array(array('class' => 'OC_User_IMAP','arguments' => array('127.0.0.1', 143, null),),);
+
 echo "<?php\n\\\$CONFIG = ";
 var_export(\$CONFIG);
 echo ";";

setup/options-dialog.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python
+# encoding: utf-8
+
+import npyscreen
+import sys
+import os
+
+
+class OptionsApp(npyscreen.NPSApp):
+    def main(self):
+        # These lines create the form and populate it with widgets.
+        # A fairly complex screen in only 8 or so lines of code - a line for each control.
+        npyscreen.setTheme(npyscreen.Themes.BlackOnWhiteTheme)
+
+        form  = npyscreen.Form(name = "Mail-in-a-Box Options",)
+        form.add(
+            npyscreen.TitleFixedText,
+            name="POSTGREY",
+            value="",
+            editable=False
+        )
+        form.add(
+            npyscreen.MultiLineEdit,
+            value="The Postgrey service greylists incoming messages from unknown senders.\n"
+                  "It can be useful for fighting spam but often causes message delivery\n"
+                  "delays of several minutes.",
+            max_height=4,
+            editable=False
+        )
+
+        form.add(
+            npyscreen.TitleFixedText,
+            name="POSTSRSD",
+            value="",
+            editable=False
+        )
+        form.add(
+            npyscreen.MultiLineEdit,
+            value="The PostSRSd daemon performs return path rewriting using the SRS protocol.\n"
+                  "Not that all messages, including locally delivered mail will have their return\n"
+                  "paths rewritten",
+            max_height=4,
+            editable=False
+        )
+
+        form.add(
+            npyscreen.TitleFixedText,
+            name="POLICY_SPF",
+            value="",
+            editable=False
+        )
+        form.add(
+            npyscreen.MultiLineEdit,
+            value=""
+                  "The policy SPF service checks the SPF of incoming mails and rejects those\n"
+                  "that do not qualify.  This helps to prevent spoofing, but if valid mail does\n"
+                  "not have SPF configured properly it will be rejected.",
+            max_height=4,
+            editable=False
+        )
+
+        init_values = []
+        if int(os.getenv('POSTGREY', 1)) == 1:
+            init_values.append(0)
+
+        if int(os.getenv('POSTSRSD', 0)) == 1:
+            init_values.append(1)
+
+        if int(os.getenv('POLICY_SPF', 0)) == 1:
+            init_values.append(2)
+
+        options = form.add(
+            npyscreen.TitleMultiSelect,
+            max_height=-2,
+            value=init_values,
+            name="Options",
+            values= ["POSTGREY","POSTSRSD","POLICY_SPF"],
+            scroll_exit=True
+        )
+
+        # This lets the user interact with the Form.
+        form.edit()
+
+        with open('_options.sh', 'w') as output:
+            print('POSTGREY=%i' % (1 if 0 in options.value else 0), file=output)
+            print('POSTSRSD=%i' % (1 if 1 in options.value else 0), file=output)
+            print('POLICY_SPF=%i' % (1 if 2 in options.value else 0), file=output)
+            # print(npyscreen.ThemeManager.default_colors, file=output)
+
+
+if __name__ == "__main__":
+    App = OptionsApp()
+    App.run()

setup/preflight.sh

@@ -26,7 +26,7 @@ fi
 #
 # Skip the check if we appear to be running inside of Vagrant, because that's really just for testing.
 TOTAL_PHYSICAL_MEM=$(head -n 1 /proc/meminfo | awk '{print $2}')
-if [ $TOTAL_PHYSICAL_MEM -lt 500000 ]; then
+if [ $TOTAL_PHYSICAL_MEM -lt 490000 ]; then
 if [ ! -d /vagrant ]; then
 	TOTAL_PHYSICAL_MEM=$(expr \( \( $TOTAL_PHYSICAL_MEM \* 1024 \) / 1000 \) / 1000)
 	echo "Your Mail-in-a-Box needs more memory (RAM) to function properly."

setup/questions.sh

@@ -16,6 +16,7 @@ if [ -z "${NONINTERACTIVE:-}" ]; then
 	# we install it inside a virtualenv. In this script, we don't have the virtualenv yet
 	# so we install the python package globally.
 	hide_output pip3 install "email_validator>=1.0.0" || exit 1
+	hide_output pip3 install npyscreen || exit 1
 
 	message_box "Mail-in-a-Box Installation" \
 		"Hello and thanks for deploying a Mail-in-a-Box!
@@ -193,6 +194,16 @@ if [ -z "${STORAGE_ROOT:-}" ]; then
 	STORAGE_ROOT=$([[ -z "${DEFAULT_STORAGE_ROOT:-}" ]] && echo "/home/$STORAGE_USER" || echo "$DEFAULT_STORAGE_ROOT")
 fi
 
+# export options variables so they are visible to the options program
+export POSTGREY
+export POSTSRSD
+export POLICY_SPF
+
+python3 setup/options-dialog.py
+source ./_options.sh
+rm _options.sh
+
+
 # Show the configuration, since the user may have not entered it manually.
 echo
 echo "Primary Hostname: $PRIMARY_HOSTNAME"

setup/solr.sh

@@ -0,0 +1,85 @@
+#!/bin/bash
+#
+# Inspired by the solr.sh from jkaberg (https://github.com/jkaberg/mailinabox-sogo)
+# with some modifications
+#
+# IMAP search with lucene via solr
+# --------------------------------
+#
+# By default dovecot uses its own Squat search index that has awful performance
+# on large mailboxes. Dovecot 2.1+ has support for using Lucene internally but
+# this didn't make it into the Ubuntu packages, so we use Solr instead to run
+# Lucene for us.
+#
+# Solr runs as a tomcat process. The dovecot solr plugin talks to solr via its
+# HTTP interface, causing mail to be indexed when searches occur, and getting
+# results back.
+
+source setup/functions.sh # load our functions
+source /etc/mailinabox.conf # load global vars
+
+# Install packages and basic configuation
+# ---------------------------------------
+
+echo "Installing Solr..."
+
+# Install packages
+apt_install solr-tomcat dovecot-solr
+
+# Solr requires a schema to tell it how to index data, this is provided by dovecot
+cp /usr/share/dovecot/solr-schema.xml /etc/solr/conf/schema.xml
+
+# Update the dovecot plugin configuration
+#
+# Break-imap-search makes search work the way users expect, rather than the way
+# the IMAP specification expects
+tools/editconf.py /etc/dovecot/conf.d/10-mail.conf \
+        mail_plugins="fts fts_solr"
+
+cat > /etc/dovecot/conf.d/90-plugin-fts.conf << EOF;
+plugin {
+  fts = solr
+  fts_autoindex = yes
+  fts_solr = break-imap-search url=http://127.0.0.1:8080/solr/
+}
+EOF
+
+# Bump memory allocation for Solr.
+# Not needed? I'll let it sit here for a while.
+#echo 'export JAVA_OPTS=-Xms512M -Xmx1024M' > /usr/share/tomcat7/bin/setenv.sh
+
+# Install cronjobs to keep FTS up to date
+hide_output install -m 755 conf/cronjob/dovecot /etc/cron.daily/
+hide_output install -m 644 conf/cronjob/solr /etc/cron.d/
+
+# PERMISSIONS
+
+# Ensure configuration files are owned by dovecot and not world readable.
+chown -R mail:dovecot /etc/dovecot
+chmod -R o-rwx /etc/dovecot
+
+mkdir -p /etc/systemd/system/tomcat9.service.d
+cat > /etc/systemd/system/tomcat9.service.d/solr-permissions.conf << EOF
+[Service]
+ReadWritePaths=/var/lib/solr/
+ReadWritePaths=/var/lib/solr/data/
+EOF
+
+# Restart services to reload solr schema & dovecot plugins
+restart_service tomcat9
+restart_service dovecot
+
+
+# Kickoff building the index
+
+# Per doveadm-fts manpage: Scan what mails exist in the full text search index
+# and compare those to what actually exist in mailboxes.
+# This removes mails from the index that have already been expunged  and  makes
+# sure that the next doveadm index will index all the missing mails (if any).
+doveadm fts rescan -A
+
+# Adds unindexed files to the fts database
+# * `-q`: Queues the indexing to be run by indexer process. (will background the indexing)
+# * `-A`: All users
+# * `'*'`: All folders
+doveadm index -q -A '*'

setup/start.sh

@@ -42,6 +42,22 @@ else
 	FIRST_TIME_SETUP=1
 fi
 
+if [ -z "${HTTP_SSL_PORT:-}" ]; then
+	HTTP_SSL_PORT=$([[ -z "${DEFAULT_HTTP_SSL_PORT:-}" ]] && echo "443" || echo "$DEFAULT_HTTP_SSL_PORT")
+fi
+
+if [ -z "${POSTGREY:-}" ]; then
+	POSTGREY=$([[ -z "${DEFAULT_POSTGREY:-}" ]] && echo "1" || echo "$DEFAULT_POSTGREY")
+fi
+
+if [ -z "${POSTSRSD:-}" ]; then
+	POSTSRSD=$([[ -z "${DEFAULT_POSTSRSD:-}" ]] && echo "0" || echo "$DEFAULT_POSTSRSD")
+fi
+
+if [ -z "${POLICY_SPF:-}" ]; then
+	POLICY_SPF=$([[ -z "${DEFAULT_POLICY_SPF:-}" ]] && echo "0" || echo "$DEFAULT_POLICY_SPF")
+fi
+
 # Put a start script in a global location. We tell the user to run 'mailinabox'
 # in the first dialog prompt, so we should do this before that starts.
 cat > /usr/local/bin/mailinabox << EOF;
@@ -93,6 +109,10 @@ PUBLIC_IP=$PUBLIC_IP
 PUBLIC_IPV6=$PUBLIC_IPV6
 PRIVATE_IP=$PRIVATE_IP
 PRIVATE_IPV6=$PRIVATE_IPV6
+HTTP_SSL_PORT=$HTTP_SSL_PORT
+POSTGREY=$POSTGREY
+POSTSRSD=$POSTSRSD
+POLICY_SPF=$POLICY_SPF
 EOF
 
 # Start service configuration.
@@ -102,6 +122,7 @@ source setup/dns.sh
 source setup/mail-postfix.sh
 source setup/mail-dovecot.sh
 source setup/mail-users.sh
+source setup/solr.sh
 source setup/dkim.sh
 source setup/spamassassin.sh
 source setup/web.sh

setup/web.sh

@@ -19,7 +19,7 @@ fi
 
 echo "Installing Nginx (web server)..."
 
-apt_install nginx php-cli php-fpm
+apt_install nginx php-cli php-fpm fcgiwrap mailgraph
 
 rm -f /etc/nginx/sites-enabled/default
 
@@ -48,6 +48,12 @@ tools/editconf.py /etc/php/7.2/fpm/php.ini -c ';' \
 tools/editconf.py /etc/php/7.2/fpm/php.ini -c ';' \
         default_charset="UTF-8"
 
+# Set higher timeout since searches with Roundcube and Solr may take longer
+# than the default 60 seconds. We will also match Roundcube's timeout to the
+# same value
+tools/editconf.py /etc/php/7.2/fpm/php.ini -c ';' \
+        default_socket_timeout=180
+
 # Switch from the dynamic process manager to the ondemand manager see #1216
 tools/editconf.py /etc/php/7.2/fpm/pool.d/www.conf -c ';' \
 	pm=ondemand
@@ -96,6 +102,9 @@ restart_service nginx
 restart_service php7.2-fpm
 
 # Open ports.
+if [ $HTTP_SSL_PORT == 443 ]; then
     ufw_allow http
     ufw_allow https
-
+else
+    ufw_allow $HTTP_SSL_PORT
+fi

setup/webmail.sh

@@ -28,8 +28,8 @@ apt_install \
 # Install Roundcube from source if it is not already present or if it is out of date.
 # Combine the Roundcube version number with the commit hash of plugins to track
 # whether we have the latest version of everything.
-VERSION=1.3.8
+VERSION=1.3.10
-HASH=90c7900ccf7b2f46fe49c650d5adb9b85ee9cc22
+HASH=431625fc737e301f9b7e502cccc61e50a24786b8
 PERSISTENT_LOGIN_VERSION=dc5ca3d3f4415cc41edb2fde533c8a8628a94c76
 HTML5_NOTIFIER_VERSION=4b370e3cd60dabd2f428a26f45b677ad1b7118d5
 CARDDAV_VERSION=3.0.3
@@ -108,7 +108,7 @@ cat > $RCM_CONFIG <<EOF;
      'verify_peer_name'  => false,
    ),
  );
-\$config['imap_timeout'] = 15;
+\$config['imap_timeout'] = 180;
 \$config['smtp_server'] = 'tls://127.0.0.1';
 \$config['smtp_port'] = 587;
 \$config['smtp_user'] = '%u';

setup/zpush.sh

@@ -22,8 +22,8 @@ apt_install \
 phpenmod -v php imap
 
 # Copy Z-Push into place.
-VERSION=2.4.4
+VERSION=2.5.0
-TARGETHASH=104d44426852429dac8ec2783a4e9ad7752d4682
+TARGETHASH=30ce5c1af3f10939036361b6032d1187651b621e
 needs_update=0 #NODOC
 if [ ! -f /usr/local/lib/z-push/version ]; then
 	needs_update=1 #NODOC

tests/fail2ban.py

@@ -90,6 +90,26 @@ def pop_test():
 		if M:
 			M.quit()
 
+def managesieve_test():
+	# We don't have a Python sieve client, so we'll
+	# just run the IMAP client and see what happens.
+	import imaplib
+
+	try:
+		M = imaplib.IMAP4(hostname, 4190)
+	except ConnectionRefusedError:
+		# looks like fail2ban worked
+		raise IsBlocked()
+
+	try:
+		M.login("fakeuser", "fakepassword")
+		raise Exception("authentication didn't fail")
+	except imaplib.IMAP4.error:
+		# authentication should fail
+		pass
+	finally:
+		M.logout() # shuts down connection, has nothing to do with login()
+
 def http_test(url, expected_status, postdata=None, qsargs=None, auth=None):
 	import urllib.parse
 	import requests
@@ -208,6 +228,9 @@ if __name__ == "__main__":
 	# POP
 	run_test(pop_test, [], 20, 30, 4)
 
+	# Managesieve
+	run_test(managesieve_test, [], 20, 30, 4)
+
 	# Mail-in-a-Box control panel
 	run_test(http_test, ["/admin/me", 200], 20, 30, 1)
 

tools/dns-auth.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+# TODO: Make work with port other than 443
+
+API_KEY=`cat /var/lib/mailinabox/api.key`
+HOSTNAME=`hostname`
+
+curl -s -X PUT -d "$CERTBOT_VALIDATION" --user "$API_KEY:" https://$HOSTNAME/admin/dns/custom/_acme-challenge.$CERTBOT_DOMAIN/TXT
+
+sleep 15

tools/dns-cleanup.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+# TODO: Make work with port other than 443
+
+API_KEY=`cat /var/lib/mailinabox/api.key`
+HOSTNAME=`hostname`
+
+curl -s -X DELETE --user "$API_KEY:" https://$HOSTNAME/admin/dns/custom/_acme-challenge.$CERTBOT_DOMAIN/TXT