v0.30 - the last Ubuntu 14.04 release

* roundcube:  upgrade carddav plugin to 3.0.3 & updated migrate.py

* Check for db first and clear sessions to force re-login

CHANGELOG.md

@@ -1,6 +1,59 @@
 CHANGELOG
 =========
 
+v0.30 (January 9, 2019)
+-----------------------
+
+Setup:
+
+* Update to Roundcube 1.3.8 and the CardDAV plugin to 3.0.3.
+* Add missing rsyslog package to install line since some OS images don't have it installed by default.
+* A log file for nsd was added.
+
+Control Panel:
+
+* The users page now documents that passwords should only have ASCII characters to prevent character encoding mismaches between clients and the server.
+* The users page no longer shows user mailbox sizes because this was extremely slow for very large mailboxes.
+* The Mail-in-a-Box version is now shown in the system status checks even when the new-version check is disabled.
+* The alises page now warns that alises should not be used to forward mail off of the box. Mail filters within Roundcube are better for that.
+* The explanation of greylisting has been improved.
+
+v0.29 (October 25, 2018)
+------------------------
+
+* Starting with v0.28, TLS certificate provisioning wouldn't work on new boxes until the mailinabox setup command was run a second time because of a problem with the non-interactive setup.
+* Update to Nextcloud 13.0.6.
+* Update to Roundcube 1.3.7.
+* Update to Z-Push 2.4.4.
+* Backup dates listed in the control panel now use an internationalized format.
+
+v0.28 (July 30, 2018)
+---------------------
+
+System:
+
+* We now use EFF's `certbot` to provision TLS certificates (from Let's Encrypt) instead of our home-grown ACME library.
+
+Contacts/Calendar:
+
+* Fix for Mac OS X autoconfig of the calendar.
+
+Setup:
+
+* Installing Z-Push broke because of what looks like a change or problem in their git server HTTPS certificate. That's fixed.
+
+v0.27 (June 14, 2018)
+---------------------
+
+Mail:
+
+* A report of box activity, including sent/received mail totals and logins by user, is now emailed to the box's administrator user each week.
+* Update Roundcube to version 1.3.6 and Z-Push to version 2.3.9.
+
+Control Panel:
+
+* The undocumented feature for proxying web requests to another server now sets X-Forwarded-For.
+
 v0.26c (February 13, 2018)
 --------------------------
 

README.md

@@ -59,7 +59,7 @@ by me:
 	$ curl -s https://keybase.io/joshdata/key.asc | gpg --import
 	gpg: key C10BDD81: public key "Joshua Tauberer <jt@occams.info>" imported
 
-	$ git verify-tag v0.26c
+	$ git verify-tag v0.30
 	gpg: Signature made ..... using RSA key ID C10BDD81
 	gpg: Good signature from "Joshua Tauberer <jt@occams.info>"
 	gpg: WARNING: This key is not certified with a trusted signature!
@@ -72,7 +72,7 @@ and on my [personal homepage](https://razor.occams.info/). (Of course, if this r
 
 Checkout the tag corresponding to the most recent release:
 
-	$ git checkout v0.26c
+	$ git checkout v0.30
 
 Begin the installation.
 

Vagrantfile

@@ -19,9 +19,9 @@ Vagrant.configure("2") do |config|
   config.vm.network "private_network", ip: "192.168.50.4"
 
   config.vm.provision :shell, :inline => <<-SH
-	# Set environment variables so that the setup script does
+    # Set environment variables so that the setup script does
-	# not ask any questions during provisioning. We'll let the
+    # not ask any questions during provisioning. We'll let the
-	# machine figure out its own public IP.
+    # machine figure out its own public IP.
     export NONINTERACTIVE=1
     export PUBLIC_IP=auto
     export PUBLIC_IPV6=auto

conf/ios-profile.xml

@@ -18,8 +18,6 @@
       <string>PRIMARY_HOSTNAME</string>
       <key>CalDAVPort</key>
       <real>443</real>
-      <key>CalDAVPrincipalURL</key>
-      <string>/cloud/remote.php/caldav/calendars/</string>
       <key>CalDAVUseSSL</key>
       <true/>
       <key>PayloadDescription</key>

conf/nginx.conf

@@ -25,7 +25,7 @@ server {
 		# This path must be served over HTTP for ACME domain validation.
 		# We map this to a special path where our TLS cert provisioning
 		# tool knows to store challenge response files.
-		alias $STORAGE_ROOT/ssl/lets_encrypt/acme_challenges/;
+		alias $STORAGE_ROOT/ssl/lets_encrypt/webroot/.well-known/acme-challenge/;
 	}
 }
 

management/backup.py

@@ -54,7 +54,7 @@ def backup_status(env):
 		date = dateutil.parser.parse(keys[1]).astimezone(dateutil.tz.tzlocal())
 		return {
 			"date": keys[1],
-			"date_str": date.strftime("%x %X") + " " + now.tzname(),
+			"date_str": date.strftime("%Y-%m-%d %X") + " " + now.tzname(),
 			"date_delta": reldate(date, now, "the future?"),
 			"full": keys[0] == "full",
 			"size": 0, # collection-status doesn't give us the size

management/daemon.py

@@ -146,7 +146,7 @@ def me():
 @authorized_personnel_only
 def mail_users():
 	if request.args.get("format", "") == "json":
-		return json_response(get_mail_users_ex(env, with_archived=True, with_slow_info=True))
+		return json_response(get_mail_users_ex(env, with_archived=True))
 	else:
 		return "".join(x+"\n" for x in get_mail_users(env))
 
@@ -333,11 +333,16 @@ def ssl_get_status():
 	from web_update import get_web_domains_info, get_web_domains
 
 	# What domains can we provision certificates for? What unexpected problems do we have?
-	provision, cant_provision = get_certificates_to_provision(env, show_extended_problems=False)
+	provision, cant_provision = get_certificates_to_provision(env, show_valid_certs=False)
 	
 	# What's the current status of TLS certificates on all of the domain?
 	domains_status = get_web_domains_info(env)
-	domains_status = [{ "domain": d["domain"], "status": d["ssl_certificate"][0], "text": d["ssl_certificate"][1] } for d in domains_status ]
+	domains_status = [
+		{
+			"domain": d["domain"],
+			"status": d["ssl_certificate"][0],
+			"text": d["ssl_certificate"][1] + ((" " + cant_provision[d["domain"]] if d["domain"] in cant_provision else ""))
+		} for d in domains_status ]
 
 	# Warn the user about domain names not hosted here because of other settings.
 	for domain in set(get_web_domains(env, exclude_dns_elsewhere=False)) - set(get_web_domains(env)):
@@ -349,7 +354,6 @@ def ssl_get_status():
 
 	return json_response({
 		"can_provision": utils.sort_domains(provision, env),
-		"cant_provision": [{ "domain": domain, "problem": cant_provision[domain] } for domain in utils.sort_domains(cant_provision, env) ],
 		"status": domains_status,
 	})
 
@@ -376,11 +380,8 @@ def ssl_install_cert():
 @authorized_personnel_only
 def ssl_provision_certs():
 	from ssl_certificates import provision_certificates
-	agree_to_tos_url = request.form.get('agree_to_tos_url')
+	requests = provision_certificates(env, limit_domains=None)
-	status = provision_certificates(env,
+	return json_response({ "requests": requests })
-		agree_to_tos_url=agree_to_tos_url,
-		jsonable=True)
-	return json_response(status)
 
 
 # WEB

management/daily_tasks.sh

@@ -9,11 +9,17 @@ export LC_ALL=en_US.UTF-8
 export LANG=en_US.UTF-8
 export LC_TYPE=en_US.UTF-8
 
+# On Mondays, i.e. once a week, send the administrator a report of total emails
+# sent and received so the admin might notice server abuse.
+if [ `date "+%u"` -eq 1 ]; then
+    management/mail_log.py -t week | management/email_administrator.py "Mail-in-a-Box Usage Report"
+fi
+
 # Take a backup.
 management/backup.py | management/email_administrator.py "Backup Status"
 
 # Provision any new certificates for new domains or domains with expiring certificates.
-management/ssl_certificates.py -q --headless | management/email_administrator.py "Error Provisioning TLS Certificate"
+management/ssl_certificates.py -q | management/email_administrator.py "Error Provisioning TLS Certificate"
 
 # Run status checks and email the administrator if anything changed.
 management/status_checks.py --show-changes | management/email_administrator.py "Status Checks Change Notice"

management/email_administrator.py

@@ -4,8 +4,14 @@
 
 import sys
 
+import html
 import smtplib
-from email.message import Message
+
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+
+# In Python 3.6:
+#from email.message import Message
 
 from utils import load_environment
 
@@ -26,11 +32,23 @@ if content == "":
 	sys.exit(0)
 
 # create MIME message
-msg = Message()
+msg = MIMEMultipart('alternative')
+
+# In Python 3.6:
+#msg = Message()
+
 msg['From'] = "\"%s\" <%s>" % (env['PRIMARY_HOSTNAME'], admin_addr)
 msg['To'] = admin_addr
 msg['Subject'] = "[%s] %s" % (env['PRIMARY_HOSTNAME'], subject)
-msg.set_payload(content, "UTF-8")
+
+content_html = "<html><body><pre>{}</pre></body></html>".format(html.escape(content))
+
+msg.attach(MIMEText(content, 'plain'))
+msg.attach(MIMEText(content_html, 'html'))
+
+# In Python 3.6:
+#msg.set_content(content)
+#msg.add_alternative(content_html, "html")
 
 # send
 smtpclient = smtplib.SMTP('127.0.0.1', 25)

management/mail_log.py

@@ -53,10 +53,10 @@ VERBOSE = False
 # List of strings to filter users with
 FILTERS = None
 
-# What to show by default
+# What to show (with defaults)
 SCAN_OUT = True  # Outgoing email
 SCAN_IN = True  # Incoming email
-SCAN_CONN = False  # IMAP and POP3 logins
+SCAN_DOVECOT_LOGIN = True  # Dovecot Logins
 SCAN_GREY = False  # Greylisted email
 SCAN_BLOCKED = False  # Rejected email
 
@@ -76,7 +76,8 @@ def scan_files(collector):
             tmp_file = tempfile.NamedTemporaryFile()
             shutil.copyfileobj(gzip.open(fn), tmp_file)
 
-        print("Processing file", fn, "...")
+        if VERBOSE:
+            print("Processing file", fn, "...")
         fn = tmp_file.name if tmp_file else fn
 
         for line in reverse_readline(fn):
@@ -105,7 +106,7 @@ def scan_mail_log(env):
         "scan_time": time.time(),  # The time in seconds the scan took
         "sent_mail": OrderedDict(),  # Data about email sent by users
         "received_mail": OrderedDict(),  # Data about email received by users
-        "dovecot": OrderedDict(),  # Data about Dovecot activity
+        "logins": OrderedDict(),  # Data about login activity
         "postgrey": {},  # Data about greylisting of email addresses
         "rejected": OrderedDict(),  # Emails that were blocked
         "known_addresses": None,  # Addresses handled by the Miab installation
@@ -119,8 +120,8 @@ def scan_mail_log(env):
     except ImportError:
         pass
 
-    print("Scanning from {:%Y-%m-%d %H:%M:%S} back to {:%Y-%m-%d %H:%M:%S}".format(
+    print("Scanning logs from {:%Y-%m-%d %H:%M:%S} to {:%Y-%m-%d %H:%M:%S}".format(
-        START_DATE, END_DATE)
+        END_DATE, START_DATE)
     )
 
     # Scan the lines in the log files until the date goes out of range
@@ -138,8 +139,8 @@ def scan_mail_log(env):
     # Print Sent Mail report
 
     if collector["sent_mail"]:
-        msg = "Sent email between {:%Y-%m-%d %H:%M:%S} and {:%Y-%m-%d %H:%M:%S}"
+        msg = "Sent email"
-        print_header(msg.format(END_DATE, START_DATE))
+        print_header(msg)
 
         data = OrderedDict(sorted(collector["sent_mail"].items(), key=email_sort))
 
@@ -173,8 +174,8 @@ def scan_mail_log(env):
     # Print Received Mail report
 
     if collector["received_mail"]:
-        msg = "Received email between {:%Y-%m-%d %H:%M:%S} and {:%Y-%m-%d %H:%M:%S}"
+        msg = "Received email"
-        print_header(msg.format(END_DATE, START_DATE))
+        print_header(msg)
 
         data = OrderedDict(sorted(collector["received_mail"].items(), key=email_sort))
 
@@ -199,43 +200,55 @@ def scan_mail_log(env):
             [accum]
         )
 
-    # Print Dovecot report
+    # Print login report
 
-    if collector["dovecot"]:
+    if collector["logins"]:
-        msg = "Email client logins between {:%Y-%m-%d %H:%M:%S} and {:%Y-%m-%d %H:%M:%S}"
+        msg = "User logins per hour"
-        print_header(msg.format(END_DATE, START_DATE))
+        print_header(msg)
 
-        data = OrderedDict(sorted(collector["dovecot"].items(), key=email_sort))
+        data = OrderedDict(sorted(collector["logins"].items(), key=email_sort))
+
+        # Get a list of all of the protocols seen in the logs in reverse count order.
+        all_protocols = defaultdict(int)
+        for u in data.values():
+            for protocol_name, count in u["totals_by_protocol"].items():
+                all_protocols[protocol_name] += count
+        all_protocols = [k for k, v in sorted(all_protocols.items(), key=lambda kv : -kv[1])]
 
         print_user_table(
             data.keys(),
             data=[
-                ("imap", [u["imap"] for u in data.values()]),
+                (protocol_name, [
-                ("pop3", [u["pop3"] for u in data.values()]),
+                    round(u["totals_by_protocol"][protocol_name] / (u["latest"]-u["earliest"]).total_seconds() * 60*60, 1)
+                    if (u["latest"]-u["earliest"]).total_seconds() > 0
+                    else 0 # prevent division by zero
+                  for u in data.values()])
+                for protocol_name in all_protocols
             ],
             sub_data=[
-                ("IMAP IP addresses", [[k + " (%d)" % v for k, v in u["imap-logins"].items()]
+                ("Protocol and Source", [[
-                                       for u in data.values()]),
+                    "{} {}: {} times".format(protocol_name, host, count)
-                ("POP3 IP addresses", [[k + " (%d)" % v for k, v in u["pop3-logins"].items()]
+                    for (protocol_name, host), count
-                                       for u in data.values()]),
+                    in sorted(u["totals_by_protocol_and_host"].items(), key=lambda kv:-kv[1])
+                  ] for u in data.values()])
             ],
             activity=[
-                ("imap", [u["activity-by-hour"]["imap"] for u in data.values()]),
+                (protocol_name, [u["activity-by-hour"][protocol_name] for u in data.values()])
-                ("pop3", [u["activity-by-hour"]["pop3"] for u in data.values()]),
+                for protocol_name in all_protocols
             ],
             earliest=[u["earliest"] for u in data.values()],
             latest=[u["latest"] for u in data.values()],
+            numstr=lambda n : str(round(n, 1)),
         )
 
-        accum = {"imap": defaultdict(int), "pop3": defaultdict(int), "both": defaultdict(int)}
+        accum = { protocol_name: defaultdict(int) for protocol_name in all_protocols }
         for h in range(24):
-            accum["imap"][h] = sum(d["activity-by-hour"]["imap"][h] for d in data.values())
+            for protocol_name in all_protocols:
-            accum["pop3"][h] = sum(d["activity-by-hour"]["pop3"][h] for d in data.values())
+              accum[protocol_name][h] = sum(d["activity-by-hour"][protocol_name][h] for d in data.values())
-            accum["both"][h] = accum["imap"][h] + accum["pop3"][h]
 
         print_time_table(
-            ["imap", "pop3", "   +"],
+            all_protocols,
-            [accum["imap"], accum["pop3"], accum["both"]]
+            [accum[protocol_name] for protocol_name in all_protocols]
         )
 
     if collector["postgrey"]:
@@ -244,7 +257,7 @@ def scan_mail_log(env):
 
         print(textwrap.fill(
             "The following mail was greylisted, meaning the emails were temporarily rejected. "
-            "Legitimate senders will try again within ten minutes.",
+            "Legitimate senders must try again after three minutes.",
             width=80, initial_indent=" ", subsequent_indent=" "
         ), end='\n\n')
 
@@ -348,9 +361,9 @@ def scan_mail_log_line(line, collector):
     elif service == "postfix/lmtp":
         if SCAN_IN:
             scan_postfix_lmtp_line(date, log, collector)
-    elif service in ("imap-login", "pop3-login"):
+    elif service.endswith("-login"):
-        if SCAN_CONN:
+        if SCAN_DOVECOT_LOGIN:
-            scan_dovecot_line(date, log, collector, service[:4])
+            scan_dovecot_login_line(date, log, collector, service[:4])
     elif service == "postgrey":
         if SCAN_GREY:
             scan_postgrey_line(date, log, collector)
@@ -448,44 +461,43 @@ def scan_postfix_smtpd_line(date, log, collector):
                 collector["rejected"][user] = data
 
 
-def scan_dovecot_line(date, log, collector, prot):
+def scan_dovecot_login_line(date, log, collector, protocol_name):
-    """ Scan a dovecot log line and extract interesting data """
+    """ Scan a dovecot login log line and extract interesting data """
 
     m = re.match("Info: Login: user=<(.*?)>, method=PLAIN, rip=(.*?),", log)
 
     if m:
         # TODO: CHECK DIT
-        user, rip = m.groups()
+        user, host = m.groups()
 
         if user_match(user):
+            add_login(user, date, protocol_name, host, collector)
+
+
+def add_login(user, date, protocol_name, host, collector):
             # Get the user data, or create it if the user is new
-            data = collector["dovecot"].get(
+            data = collector["logins"].get(
                 user,
                 {
-                    "imap": 0,
-                    "pop3": 0,
                     "earliest": None,
                     "latest": None,
-                    "imap-logins": defaultdict(int),
+                    "totals_by_protocol": defaultdict(int),
-                    "pop3-logins": defaultdict(int),
+                    "totals_by_protocol_and_host": defaultdict(int),
-                    "activity-by-hour": {
+                    "activity-by-hour": defaultdict(lambda : defaultdict(int)),
-                        "imap": defaultdict(int),
-                        "pop3": defaultdict(int),
-                    },
                 }
             )
 
-            data[prot] += 1
-            data["activity-by-hour"][prot][date.hour] += 1
-
             if data["latest"] is None:
                 data["latest"] = date
             data["earliest"] = date
 
-            if rip not in ("127.0.0.1", "::1") or True:
+            data["totals_by_protocol"][protocol_name] += 1
-                data["%s-logins" % prot][rip] += 1
+            data["totals_by_protocol_and_host"][(protocol_name, host)] += 1
 
-            collector["dovecot"][user] = data
+            if host not in ("127.0.0.1", "::1") or True:
+                data["activity-by-hour"][protocol_name][date.hour] += 1
+
+            collector["logins"][user] = data
 
 
 def scan_postfix_lmtp_line(date, log, collector):
@@ -561,6 +573,8 @@ def scan_postfix_submission_line(date, log, collector):
 
             collector["sent_mail"][user] = data
 
+            # Also log this as a login.
+            add_login(user, date, "smtp", client, collector)
 
 # Utility functions
 
@@ -640,7 +654,7 @@ def print_time_table(labels, data, do_print=True):
         for i, d in enumerate(data):
             lines[i] += base.format(d[h])
 
-    lines.insert(0, "┬")
+    lines.insert(0, "┬ totals by time of day:")
     lines.append("└" + (len(lines[-1]) - 2) * "─")
 
     if do_print:
@@ -650,7 +664,7 @@ def print_time_table(labels, data, do_print=True):
 
 
 def print_user_table(users, data=None, sub_data=None, activity=None, latest=None, earliest=None,
-                     delimit=False):
+                     delimit=False, numstr=str):
     str_temp = "{:<32} "
     lines = []
     data = data or []
@@ -764,7 +778,7 @@ def print_user_table(users, data=None, sub_data=None, activity=None, latest=None
 
     # Print totals
 
-    data_accum = [str(a) for a in data_accum]
+    data_accum = [numstr(a) for a in data_accum]
     footer = str_temp.format("Totals:" if do_accum else " ")
     for row, (l, _) in enumerate(data):
         temp = "{:>%d}" % max(5, len(l) + 1)
@@ -818,7 +832,7 @@ if __name__ == "__main__":
                         action="store_true")
     parser.add_argument("-s", "--sent", help="Scan for sent emails.",
                         action="store_true")
-    parser.add_argument("-l", "--logins", help="Scan for IMAP/POP logins.",
+    parser.add_argument("-l", "--logins", help="Scan for user logins to IMAP/POP3.",
                         action="store_true")
     parser.add_argument("-g", "--grey", help="Scan for greylisted emails.",
                         action="store_true")
@@ -863,8 +877,8 @@ if __name__ == "__main__":
         if not SCAN_OUT:
             print("Ignoring sent emails")
 
-        SCAN_CONN = args.logins
+        SCAN_DOVECOT_LOGIN = args.logins
-        if not SCAN_CONN:
+        if not SCAN_DOVECOT_LOGIN:
             print("Ignoring logins")
 
         SCAN_GREY = args.grey

management/mailconfig.py

@@ -105,7 +105,7 @@ def get_mail_users(env):
 	users = [ row[0] for row in c.fetchall() ]
 	return utils.sort_email_addresses(users, env)
 
-def get_mail_users_ex(env, with_archived=False, with_slow_info=False):
+def get_mail_users_ex(env, with_archived=False):
 	# Returns a complex data structure of all user accounts, optionally
 	# including archived (status="inactive") accounts.
 	#
@@ -139,9 +139,6 @@ def get_mail_users_ex(env, with_archived=False, with_slow_info=False):
 		}
 		users.append(user)
 
-		if with_slow_info:
-			user["mailbox_size"] = utils.du(os.path.join(env['STORAGE_ROOT'], 'mail/mailboxes', *reversed(email.split("@"))))
-
 	# Add in archived accounts.
 	if with_archived:
 		root = os.path.join(env['STORAGE_ROOT'], 'mail/mailboxes')
@@ -158,8 +155,6 @@ def get_mail_users_ex(env, with_archived=False, with_slow_info=False):
 						"mailbox": mbox,
 					}
 					users.append(user)
-					if with_slow_info:
-						user["mailbox_size"] = utils.du(mbox)
 
 	# Group by domain.
 	domains = { }

management/ssl_certificates.py

@@ -1,7 +1,7 @@
 #!/usr/local/lib/mailinabox/env/bin/python
 # Utilities for installing and selecting SSL certificates.
 
-import os, os.path, re, shutil
+import os, os.path, re, shutil, subprocess, tempfile
 
 from utils import shell, safe_domain_name, sort_domains
 import idna
@@ -24,6 +24,16 @@ def get_ssl_certificates(env):
 		if not os.path.exists(ssl_root):
 			return
 		for fn in os.listdir(ssl_root):
+			if fn == 'ssl_certificate.pem':
+				# This is always a symbolic link
+				# to the certificate to use for
+				# PRIMARY_HOSTNAME. Don't let it
+				# be eligible for use because we
+				# could end up creating a symlink
+				# to itself --- we want to find
+				# the cert that it should be a
+				# symlink to.
+				continue
 			fn = os.path.join(ssl_root, fn)
 			if os.path.isfile(fn):
 				yield fn
@@ -74,6 +84,12 @@ def get_ssl_certificates(env):
 
 		# Add this cert to the list of certs usable for the domains.
 		for domain in cert_domains:
+			# The primary hostname can only use a certificate mapped
+			# to the system private key.
+			if domain == env['PRIMARY_HOSTNAME']:
+				if cert._private_key._filename != os.path.join(env['STORAGE_ROOT'], 'ssl', 'ssl_private_key.pem'):
+					continue
+
 			domains.setdefault(domain, []).append(cert)
 
 	# Sort the certificates to prefer good ones.
@@ -81,6 +97,7 @@ def get_ssl_certificates(env):
 	now = datetime.datetime.utcnow()
 	ret = { }
 	for domain, cert_list in domains.items():
+		#for c in cert_list: print(domain, c.not_valid_before, c.not_valid_after, "("+str(now)+")", c.issuer, c.subject, c._filename)
 		cert_list.sort(key = lambda cert : (
 			# must be valid NOW
 			cert.not_valid_before <= now <= cert.not_valid_after,
@@ -124,21 +141,23 @@ def get_ssl_certificates(env):
 
 	return ret
 
-def get_domain_ssl_files(domain, ssl_certificates, env, allow_missing_cert=False, raw=False):
+def get_domain_ssl_files(domain, ssl_certificates, env, allow_missing_cert=False, use_main_cert=True):
-	# Get the system certificate info.
+	if use_main_cert or not allow_missing_cert:
-	ssl_private_key = os.path.join(os.path.join(env["STORAGE_ROOT"], 'ssl', 'ssl_private_key.pem'))
+		# Get the system certificate info.
-	ssl_certificate = os.path.join(os.path.join(env["STORAGE_ROOT"], 'ssl', 'ssl_certificate.pem'))
+		ssl_private_key = os.path.join(os.path.join(env["STORAGE_ROOT"], 'ssl', 'ssl_private_key.pem'))
-	system_certificate = {
+		ssl_certificate = os.path.join(os.path.join(env["STORAGE_ROOT"], 'ssl', 'ssl_certificate.pem'))
-		"private-key": ssl_private_key,
+		system_certificate = {
-		"certificate": ssl_certificate,
+			"private-key": ssl_private_key,
-		"primary-domain": env['PRIMARY_HOSTNAME'],
+			"certificate": ssl_certificate,
-		"certificate_object": load_pem(load_cert_chain(ssl_certificate)[0]),
+			"primary-domain": env['PRIMARY_HOSTNAME'],
-	}
+			"certificate_object": load_pem(load_cert_chain(ssl_certificate)[0]),
+		}
 
-	if domain == env['PRIMARY_HOSTNAME']:
+	if use_main_cert:
-		# The primary domain must use the server certificate because
+		if domain == env['PRIMARY_HOSTNAME']:
-		# it is hard-coded in some service configuration files.
+			# The primary domain must use the server certificate because
-		return system_certificate
+			# it is hard-coded in some service configuration files.
+			return system_certificate
 
 	wildcard_domain = re.sub("^[^\.]+", "*", domain)
 	if domain in ssl_certificates:
@@ -155,136 +174,97 @@ def get_domain_ssl_files(domain, ssl_certificates, env, allow_missing_cert=False
 
 # PROVISIONING CERTIFICATES FROM LETSENCRYPT
 
-def get_certificates_to_provision(env, show_extended_problems=True, force_domains=None):
+def get_certificates_to_provision(env, limit_domains=None, show_valid_certs=True):
-	# Get a set of domain names that we should now provision certificates
+	# Get a set of domain names that we can provision certificates for
-	# for. Provision if a domain name has no valid certificate or if any
+	# using certbot. We start with domains that the box is serving web
-	# certificate is expiring in 14 days. If provisioning anything, also
+	# for and subtract:
-	# provision certificates expiring within 30 days. The period between
+	# * domains not in limit_domains if limit_domains is not empty
-	# 14 and 30 days allows us to consolidate domains into multi-domain
+	# * domains with custom "A" records, i.e. they are hosted elsewhere
-	# certificates for domains expiring around the same time.
+	# * domains with actual "A" records that point elsewhere
+	# * domains that already have certificates that will be valid for a while
 
 	from web_update import get_web_domains
+	from status_checks import query_dns, normalize_ip
 
-	import datetime
+	existing_certs = get_ssl_certificates(env)
-	now = datetime.datetime.utcnow()
 
-	# Get domains with missing & expiring certificates.
+	plausible_web_domains = get_web_domains(env, exclude_dns_elsewhere=False)
-	certs = get_ssl_certificates(env)
+	actual_web_domains = get_web_domains(env)
-	domains = set()
+
-	domains_if_any = set()
+	domains_to_provision = set()
-	problems = { }
+	domains_cant_provision = { }
-	for domain in get_web_domains(env):
+
-		# If the user really wants a cert for certain domains, include it.
+	for domain in plausible_web_domains:
-		if force_domains:
+		# Skip domains that the user doesn't want to provision now.
-			if force_domains == "ALL" or (isinstance(force_domains, list) and domain in force_domains):
+		if limit_domains and domain not in limit_domains:
-				domains.add(domain)
 			continue
 
-		# Include this domain if its certificate is missing, self-signed, or expiring soon.
+		# Check that there isn't an explicit A/AAAA record.
-		try:
+		if domain not in actual_web_domains:
-			cert = get_domain_ssl_files(domain, certs, env, allow_missing_cert=True)
+			domains_cant_provision[domain] = "The domain has a custom DNS A/AAAA record that points the domain elsewhere, so there is no point to installing a TLS certificate here and we could not automatically provision one anyway because provisioning requires access to the website (which isn't here)."
-		except FileNotFoundError as e:
+
-			# system certificate is not present
+		# Check that the DNS resolves to here.
-			problems[domain] = "Error: " + str(e)
-			continue
-		if cert is None:
-			# No valid certificate available.
-			domains.add(domain)
 		else:
-			cert = cert["certificate_object"]
+
-			if cert.issuer == cert.subject:
+			# Does the domain resolve to this machine in public DNS? If not,
-				# This is self-signed. Get a real one.
+			# we can't do domain control validation. For IPv6 is configured,
-				domains.add(domain)
+			# make sure both IPv4 and IPv6 are correct because we don't know
+			# how Let's Encrypt will connect.
+			bad_dns = []
+			for rtype, value in [("A", env["PUBLIC_IP"]), ("AAAA", env.get("PUBLIC_IPV6"))]:
+				if not value: continue # IPv6 is not configured
+				response = query_dns(domain, rtype)
+				if response != normalize_ip(value):
+					bad_dns.append("%s (%s)" % (response, rtype))
+	
+			if bad_dns:
+				domains_cant_provision[domain] = "The domain name does not resolve to this machine: " \
+					+ (", ".join(bad_dns)) \
+					+ "."
 			
-			# Valid certificate today, but is it expiring soon?
+			else:
-			elif cert.not_valid_after-now < datetime.timedelta(days=14):
+				# DNS is all good.
-				domains.add(domain)
-			elif cert.not_valid_after-now < datetime.timedelta(days=30):
-				domains_if_any.add(domain)
 
-			# It's valid. Should we report its validness?
+				# Check for a good existing cert.
-			elif show_extended_problems:
+				existing_cert = get_domain_ssl_files(domain, existing_certs, env, use_main_cert=False, allow_missing_cert=True)
-				problems[domain] = "The certificate is valid for at least another 30 days --- no need to replace."
+				if existing_cert:
+					existing_cert_check = check_certificate(domain, existing_cert['certificate'], existing_cert['private-key'],
+						warn_if_expiring_soon=14)
+					if existing_cert_check[0] == "OK":
+						if show_valid_certs:
+							domains_cant_provision[domain] = "The domain has a valid certificate already. ({} Certificate: {}, private key {})".format(
+								existing_cert_check[1],
+								existing_cert['certificate'],
+								existing_cert['private-key'])
+						continue
 
-	# Warn the user about domains hosted elsewhere.
+				domains_to_provision.add(domain)
-	if not force_domains and show_extended_problems:
-		for domain in set(get_web_domains(env, exclude_dns_elsewhere=False)) - set(get_web_domains(env)):
-			problems[domain] = "The domain's DNS is pointed elsewhere, so there is no point to installing a TLS certificate here and we could not automatically provision one anyway because provisioning requires access to the website (which isn't here)."
 
-	# Filter out domains that we can't provision a certificate for.
+	return (domains_to_provision, domains_cant_provision)
-	def can_provision_for_domain(domain):
-		from status_checks import normalize_ip
-
-		# Does the domain resolve to this machine in public DNS? If not,
-		# we can't do domain control validation. For IPv6 is configured,
-		# make sure both IPv4 and IPv6 are correct because we don't know
-		# how Let's Encrypt will connect.
-		import dns.resolver
-		for rtype, value in [("A", env["PUBLIC_IP"]), ("AAAA", env.get("PUBLIC_IPV6"))]:
-			if not value: continue # IPv6 is not configured
-			try:
-				# Must make the qname absolute to prevent a fall-back lookup with a
-				# search domain appended, by adding a period to the end.
-				response = dns.resolver.query(domain + ".", rtype)
-			except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer) as e:
-				problems[domain] = "DNS isn't configured properly for this domain: DNS resolution failed (%s: %s)." % (rtype, str(e) or repr(e)) # NoAnswer's str is empty
-				return False
-			except Exception as e:
-				problems[domain] = "DNS isn't configured properly for this domain: DNS lookup had an error: %s." % str(e)
-				return False
-
-			# Unfortunately, the response.__str__ returns bytes
-			# instead of string, if it resulted from an AAAA-query.
-			# We need to convert manually, until this is fixed:
-			# https://github.com/rthalley/dnspython/issues/204
-			#
-			# BEGIN HOTFIX
-			def rdata__str__(r):
-				s = r.to_text()
-				if isinstance(s, bytes):
-					 s = s.decode('utf-8')
-				return s
-			# END HOTFIX
-
-			if len(response) != 1 or normalize_ip(rdata__str__(response[0])) != normalize_ip(value):
-				problems[domain] = "Domain control validation cannot be performed for this domain because DNS points the domain to another machine (%s %s)." % (rtype, ", ".join(rdata__str__(r) for r in response))
-				return False
-
-		return True
-
-	domains = set(filter(can_provision_for_domain, domains))
-
-	# If there are any domains we definitely will provision for, add in
-	# additional domains to do at this time.
-	if len(domains) > 0:
-		domains |= set(filter(can_provision_for_domain, domains_if_any))
-
-	return (domains, problems)
-
-def provision_certificates(env, agree_to_tos_url=None, logger=None, show_extended_problems=True, force_domains=None, jsonable=False):
-	import requests.exceptions
-	import acme.messages
-
-	from free_tls_certificates import client
 
+def provision_certificates(env, limit_domains):
 	# What domains should we provision certificates for? And what
 	# errors prevent provisioning for other domains.
-	domains, problems = get_certificates_to_provision(env, force_domains=force_domains, show_extended_problems=show_extended_problems)
+	domains, domains_cant_provision = get_certificates_to_provision(env, limit_domains=limit_domains)
+
+	# Build a list of what happened on each domain or domain-set.
+	ret = []
+	for domain, error in domains_cant_provision.items():
+		ret.append({
+			"domains": [domain],
+			"log": [error],
+			"result": "skipped",
+		})
 
-	# Exit fast if there is nothing to do.
-	if len(domains) == 0:
-		return {
-			"requests": [],
-			"problems": problems,
-		}
 
 	# Break into groups of up to 100 certificates at a time, which is Let's Encrypt's
 	# limit for a single certificate. We'll sort to put related domains together.
+	max_domains_per_group = 100
 	domains = sort_domains(domains, env)
 	certs = []
 	while len(domains) > 0:
-		certs.append( domains[0:100] )
+		certs.append( domains[:max_domains_per_group] )
-		domains = domains[100:]
+		domains = domains[max_domains_per_group:]
 
 	# Prepare to provision.
 
@@ -293,115 +273,82 @@ def provision_certificates(env, agree_to_tos_url=None, logger=None, show_extende
 	if not os.path.exists(account_path):
 		os.mkdir(account_path)
 
-	# Where should we put ACME challenge files. This is mapped to /.well-known/acme_challenge
-	# by the nginx configuration.
-	challenges_path = os.path.join(account_path, 'acme_challenges')
-	if not os.path.exists(challenges_path):
-		os.mkdir(challenges_path)
-
-	# Read in the private key that we use for all TLS certificates. We'll need that
-	# to generate a CSR (done by free_tls_certificates).
-	with open(os.path.join(env['STORAGE_ROOT'], 'ssl/ssl_private_key.pem'), 'rb') as f:
-		private_key = f.read()
-
 	# Provision certificates.
-
-	ret = []
 	for domain_list in certs:
-		# For return.
+		ret.append({
-		ret_item = {
 			"domains": domain_list,
 			"log": [],
-		}
+		})
-		ret.append(ret_item)
-
-		# Logging for free_tls_certificates.
-		def my_logger(message):
-			if logger: logger(message)
-			ret_item["log"].append(message)
-
-		# Attempt to provision a certificate.
 		try:
-			try:
+			# Create a CSR file for our master private key so that certbot
-				cert = client.issue_certificate(
+			# uses our private key.
-					domain_list,
+			key_file = os.path.join(env['STORAGE_ROOT'], 'ssl', 'ssl_private_key.pem')
-					account_path,
+			with tempfile.NamedTemporaryFile() as csr_file:
-					agree_to_tos_url=agree_to_tos_url,
+				# We could use openssl, but certbot requires
-					private_key=private_key,
+				# that the CN domain and SAN domains match
-					logger=my_logger)
+				# the domain list passed to certbot, and adding
+				# SAN domains openssl req is ridiculously complicated.
+				# subprocess.check_output([
+				# 	"openssl", "req", "-new",
+				# 	"-key", key_file,
+				# 	"-out", csr_file.name,
+				# 	"-subj", "/CN=" + domain_list[0],
+				# 	"-sha256" ])
+				from cryptography import x509
+				from cryptography.hazmat.backends import default_backend
+				from cryptography.hazmat.primitives.serialization import Encoding
+				from cryptography.hazmat.primitives import hashes
+				from cryptography.x509.oid import NameOID
+				builder = x509.CertificateSigningRequestBuilder()
+				builder = builder.subject_name(x509.Name([ x509.NameAttribute(NameOID.COMMON_NAME, domain_list[0]) ]))
+				builder = builder.add_extension(x509.BasicConstraints(ca=False, path_length=None), critical=True)
+				builder = builder.add_extension(x509.SubjectAlternativeName(
+					[x509.DNSName(d) for d in domain_list]
+				), critical=False)
+				request = builder.sign(load_pem(load_cert_chain(key_file)[0]), hashes.SHA256(), default_backend())
+				with open(csr_file.name, "wb") as f:
+					f.write(request.public_bytes(Encoding.PEM))
 
-			except client.NeedToTakeAction as e:
+				# Provision, writing to a temporary file.
-				# Write out the ACME challenge files.
+				webroot = os.path.join(account_path, 'webroot')
-				for action in e.actions:
+				os.makedirs(webroot, exist_ok=True)
-					if isinstance(action, client.NeedToInstallFile):
+				with tempfile.TemporaryDirectory() as d:
-						fn = os.path.join(challenges_path, action.file_name)
+					cert_file = os.path.join(d, 'cert_and_chain.pem')
-						with open(fn, 'w') as f:
+					print("Provisioning TLS certificates for " + ", ".join(domain_list) + ".")
-							f.write(action.contents)
+					certbotret = subprocess.check_output([
-					else:
+						"certbot",
-						raise ValueError(str(action))
+						"certonly",
+						#"-v", # just enough to see ACME errors
+						"--non-interactive", # will fail if user hasn't registered during Mail-in-a-Box setup
 
-				# Try to provision now that the challenge files are installed.
+						"-d", ",".join(domain_list), # first will be main domain
 
-				cert = client.issue_certificate(
+						"--csr", csr_file.name, # use our private key; unfortunately this doesn't work with auto-renew so we need to save cert manually
-					domain_list,
+						"--cert-path", os.path.join(d, 'cert'), # we only use the full chain
-					account_path,
+						"--chain-path", os.path.join(d, 'chain'), # we only use the full chain
-					private_key=private_key,
+						"--fullchain-path", cert_file,
-					logger=my_logger)
 
-		except client.NeedToAgreeToTOS as e:
+						"--webroot", "--webroot-path", webroot,
-			# The user must agree to the Let's Encrypt terms of service agreement
-			# before any further action can be taken.
-			ret_item.update({
-				"result": "agree-to-tos",
-				"url": e.url,
-			})
 
-		except client.WaitABit as e:
+						"--config-dir", account_path,
-			# We need to hold on for a bit before querying again to see if we can
+						#"--staging",
-			# acquire a provisioned certificate.
+					], stderr=subprocess.STDOUT).decode("utf8")
-			import time, datetime
+					install_cert_copy_file(cert_file, env)
-			ret_item.update({
-				"result": "wait",
-				"until": e.until_when if not jsonable else e.until_when.isoformat(),
-				"seconds": (e.until_when - datetime.datetime.now()).total_seconds()
-			})
 
-		except client.AccountDataIsCorrupt as e:
+			ret[-1]["log"].append(certbotret)
-			# This is an extremely rare condition.
+			ret[-1]["result"] = "installed"
-			ret_item.update({
+		except subprocess.CalledProcessError as e:
-				"result": "error",
+			ret[-1]["log"].append(e.output.decode("utf8"))
-				"message": "Something unexpected went wrong. It looks like your local Let's Encrypt account data is corrupted. There was a problem with the file " + e.account_file_path + ".",
+			ret[-1]["result"] = "error"
-			})
+		except Exception as e:
+			ret[-1]["log"].append(str(e))
+			ret[-1]["result"] = "error"
 
-		except (client.InvalidDomainName, client.NeedToTakeAction, client.ChallengeFailed, client.RateLimited, acme.messages.Error, requests.exceptions.RequestException) as e:
+	# Run post-install steps.
-			ret_item.update({
+	ret.extend(post_install_func(env))
-				"result": "error",
-				"message": "Something unexpected went wrong: " + str(e),
-			})
-
-		else:
-			# A certificate was issued.
-
-			install_status = install_cert(domain_list[0], cert['cert'].decode("ascii"), b"\n".join(cert['chain']).decode("ascii"), env, raw=True)
-
-			# str indicates the certificate was not installed.
-			if isinstance(install_status, str):
-				ret_item.update({
-					"result": "error",
-					"message": "Something unexpected was wrong with the provisioned certificate: " + install_status,
-				})
-			else:
-				# A list indicates success and what happened next.
-				ret_item["log"].extend(install_status)
-				ret_item.update({
-					"result": "installed",
-				})
 
 	# Return what happened with each certificate request.
-	return {
+	return ret
-		"requests": ret,
-		"problems": problems,
-	}
 
 def provision_certificates_cmdline():
 	import sys
@@ -412,151 +359,39 @@ def provision_certificates_cmdline():
 	Lock(die=True).forever()
 	env = load_environment()
 
-	verbose = False
+	quiet = False
-	headless = False
+	domains = []
-	force_domains = None
-	show_extended_problems = True
-	
-	args = list(sys.argv)
-	args.pop(0) # program name
-	if args and args[0] == "-v":
-		verbose = True
-		args.pop(0)
-	if args and args[0] == "-q":
-		show_extended_problems = False
-		args.pop(0)
-	if args and args[0] == "--headless":
-		headless = True
-		args.pop(0)
-	if args and args[0] == "--force":
-		force_domains = "ALL"
-		args.pop(0)
-	else:
-		force_domains = args
 
-	agree_to_tos_url = None
+	for arg in sys.argv[1:]:
-	while True:
+		if arg == "-q":
-		# Run the provisioning script. This installs certificates. If there are
+			quiet = True
-		# a very large number of domains on this box, it issues separate
+		else:
-		# certificates for groups of domains. We have to check the result for
+			domains.append(arg)
-		# each group.
-		def my_logger(message):
-			if verbose:
-				print(">", message)
-		status = provision_certificates(env, agree_to_tos_url=agree_to_tos_url, logger=my_logger, force_domains=force_domains, show_extended_problems=show_extended_problems)
-		agree_to_tos_url = None # reset to prevent infinite looping
 
-		if not status["requests"]:
+	# Go.
-			# No domains need certificates.
+	status = provision_certificates(env, limit_domains=domains)
-			if not headless or verbose:
-				if len(status["problems"]) == 0:
-					print("No domains hosted on this box need a new TLS certificate at this time.")
-				elif len(status["problems"]) > 0:
-					print("No TLS certificates could be provisoned at this time:")
-					print()
-					for domain in sort_domains(status["problems"], env):
-						print("%s: %s" % (domain, status["problems"][domain]))
 
-			sys.exit(0)
+	# Show what happened.
-
+	for request in status:
-		# What happened?
+		if isinstance(request, str):
-		wait_until = None
+			print(request)
-		wait_domains = []
+		else:
-		for request in status["requests"]:
+			if quiet and request['result'] == 'skipped':
-			if request["result"] == "agree-to-tos":
+				continue
-				# We may have asked already in a previous iteration.
+			print(request['result'] + ":", ", ".join(request['domains']) + ":")
-				if agree_to_tos_url is not None:
+			for line in request["log"]:
-					continue
+				print(line)
-
-				# Can't ask the user a question in this mode. Warn the user that something
-				# needs to be done.
-				if headless:
-					print(", ".join(request["domains"]) + " need a new or renewed TLS certificate.")
-					print()
-					print("This box can't do that automatically for you until you agree to Let's Encrypt's")
-					print("Terms of Service agreement. Use the Mail-in-a-Box control panel to provision")
-					print("certificates for these domains.")
-					sys.exit(1)
-
-				print("""
-I'm going to provision a TLS certificate (formerly called a SSL certificate)
-for you from Let's Encrypt (letsencrypt.org).
-
-TLS certificates are cryptographic keys that ensure communication between
-you and this box are secure when getting and sending mail and visiting
-websites hosted on this box. Let's Encrypt is a free provider of TLS
-certificates.
-
-Please open this document in your web browser:
-
-%s
-
-It is Let's Encrypt's terms of service agreement. If you agree, I can
-provision that TLS certificate. If you don't agree, you will have an
-opportunity to install your own TLS certificate from the Mail-in-a-Box
-control panel.
-
-Do you agree to the agreement? Type Y or N and press <ENTER>: """
-				 % request["url"], end='', flush=True)
-			
-				if sys.stdin.readline().strip().upper() != "Y":
-					print("\nYou didn't agree. Quitting.")
-					sys.exit(1)
-
-				# Okay, indicate agreement on next iteration.
-				agree_to_tos_url = request["url"]
-
-			if request["result"] == "wait":
-				# Must wait. We'll record until when. The wait occurs below.
-				if wait_until is None:
-					wait_until = request["until"]
-				else:
-					wait_until = max(wait_until, request["until"])
-				wait_domains += request["domains"]
-
-			if request["result"] == "error":
-				print(", ".join(request["domains"]) + ":")
-				print(request["message"])
-
-			if request["result"] == "installed":
-				print("A TLS certificate was successfully installed for " + ", ".join(request["domains"]) + ".")
-
-		if wait_until:
-			# Wait, then loop.
-			import time, datetime
 			print()
-			print("A TLS certificate was requested for: " + ", ".join(wait_domains) + ".")
-			first = True
-			while wait_until > datetime.datetime.now():
-				if not headless or first:
-					print ("We have to wait", int(round((wait_until - datetime.datetime.now()).total_seconds())), "seconds for the certificate to be issued...")
-				time.sleep(10)
-				first = False
 
-			continue # Loop!
-
-		if agree_to_tos_url:
-			# The user agrees to the TOS. Loop to try again by agreeing.
-			continue # Loop!
-
-		# Unless we were instructed to wait, or we just agreed to the TOS,
-		# we're done for now.
-		break
-
-	# And finally show the domains with problems.
-	if len(status["problems"]) > 0:
-		print("TLS certificates could not be provisoned for:")
-		for domain in sort_domains(status["problems"], env):
-			print("%s: %s" % (domain, status["problems"][domain]))
 
 # INSTALLING A NEW CERTIFICATE FROM THE CONTROL PANEL
 
 def create_csr(domain, ssl_key, country_code, env):
 	return shell("check_output", [
-                "openssl", "req", "-new",
+				"openssl", "req", "-new",
-                "-key", ssl_key,
+				"-key", ssl_key,
-                "-sha256",
+				"-sha256",
-                "-subj", "/C=%s/CN=%s" % (country_code, domain)])
+				"-subj", "/C=%s/CN=%s" % (country_code, domain)])
 
 def install_cert(domain, ssl_cert, ssl_chain, env, raw=False):
 	# Write the combined cert+chain to a temporary path and validate that it is OK.
@@ -577,6 +412,16 @@ def install_cert(domain, ssl_cert, ssl_chain, env, raw=False):
 			cert_status += " " + cert_status_details
 		return cert_status
 
+	# Copy certifiate into ssl directory.
+	install_cert_copy_file(fn, env)
+
+	# Run post-install steps.
+	ret = post_install_func(env)
+	if raw: return ret
+	return "\n".join(ret)
+
+
+def install_cert_copy_file(fn, env):
 	# Where to put it?
 	# Make a unique path for the certificate.
 	from cryptography.hazmat.primitives import hashes
@@ -594,14 +439,26 @@ def install_cert(domain, ssl_cert, ssl_chain, env, raw=False):
 	os.makedirs(os.path.dirname(ssl_certificate), exist_ok=True)
 	shutil.move(fn, ssl_certificate)
 
-	ret = ["OK"]
 
-	# When updating the cert for PRIMARY_HOSTNAME, symlink it from the system
+def post_install_func(env):
+	ret = []
+
+	# Get the certificate to use for PRIMARY_HOSTNAME.
+	ssl_certificates = get_ssl_certificates(env)
+	cert = get_domain_ssl_files(env['PRIMARY_HOSTNAME'], ssl_certificates, env, use_main_cert=False)
+	if not cert:
+		# Ruh-row, we don't have any certificate usable
+		# for the primary hostname.
+		ret.append("there is no valid certificate for " + env['PRIMARY_HOSTNAME'])
+
+	# Symlink the best cert for PRIMARY_HOSTNAME to the system
 	# certificate path, which is hard-coded for various purposes, and then
 	# restart postfix and dovecot.
-	if domain == env['PRIMARY_HOSTNAME']:
+	system_ssl_certificate = os.path.join(os.path.join(env["STORAGE_ROOT"], 'ssl', 'ssl_certificate.pem'))
+	if cert and os.readlink(system_ssl_certificate) != cert['certificate']:
 		# Update symlink.
-		system_ssl_certificate = os.path.join(os.path.join(env["STORAGE_ROOT"], 'ssl', 'ssl_certificate.pem'))
+		ret.append("updating primary certificate")
+		ssl_certificate = cert['certificate']
 		os.unlink(system_ssl_certificate)
 		os.symlink(ssl_certificate, system_ssl_certificate)
 
@@ -617,12 +474,12 @@ def install_cert(domain, ssl_cert, ssl_chain, env, raw=False):
 	# Update the web configuration so nginx picks up the new certificate file.
 	from web_update import do_web_update
 	ret.append( do_web_update(env) )
-	if raw: return ret
+
-	return "\n".join(ret)
+	return ret
 
 # VALIDATION OF CERTIFICATES
 
-def check_certificate(domain, ssl_certificate, ssl_private_key, warn_if_expiring_soon=True, rounded_time=False, just_check_domain=False):
+def check_certificate(domain, ssl_certificate, ssl_private_key, warn_if_expiring_soon=10, rounded_time=False, just_check_domain=False):
 	# Check that the ssl_certificate & ssl_private_key files are good
 	# for the provided domain.
 
@@ -728,7 +585,7 @@ def check_certificate(domain, ssl_certificate, ssl_private_key, warn_if_expiring
 			# We'll renew it with Lets Encrypt.
 			expiry_info = "The certificate expires on %s." % cert_expiration_date.strftime("%x")
 
-		if ndays <= 10 and warn_if_expiring_soon:
+		if warn_if_expiring_soon and ndays <= warn_if_expiring_soon:
 			# Warn on day 10 to give 4 days for us to automatically renew the
 			# certificate, which occurs on day 14.
 			return ("The certificate is expiring soon: " + expiry_info, None)

management/status_checks.py

@@ -393,7 +393,7 @@ def check_primary_hostname_dns(domain, env, output, dns_domains, dns_zonefiles):
 
 	# Check that PRIMARY_HOSTNAME resolves to PUBLIC_IP[V6] in public DNS.
 	ipv6 = query_dns(domain, "AAAA") if env.get("PUBLIC_IPV6") else None
-	if ip == env['PUBLIC_IP'] and not (ipv6 and env['PUBLIC_IPV6'] and normalize_ip(ipv6) != normalize_ip(env['PUBLIC_IPV6'])):
+	if ip == env['PUBLIC_IP'] and not (ipv6 and env['PUBLIC_IPV6'] and ipv6 != normalize_ip(env['PUBLIC_IPV6'])):
 		output.print_ok("Domain resolves to box's IP address. [%s ↦ %s]" % (env['PRIMARY_HOSTNAME'], my_ips))
 	else:
 		output.print_error("""This domain must resolve to your box's IP address (%s) in public DNS but it currently resolves
@@ -640,7 +640,7 @@ def check_web_domain(domain, rounded_time, ssl_certificates, env, output):
 		for (rtype, expected) in (("A", env['PUBLIC_IP']), ("AAAA", env.get('PUBLIC_IPV6'))):
 			if not expected: continue # IPv6 is not configured
 			value = query_dns(domain, rtype)
-			if normalize_ip(value) == normalize_ip(expected):
+			if value == normalize_ip(expected):
 				ok_values.append(value)
 			else:
 				output.print_error("""This domain should resolve to your box's IP address (%s %s) if you would like the box to serve
@@ -687,27 +687,17 @@ def query_dns(qname, rtype, nxdomain='[Not Set]', at=None):
 	except dns.exception.Timeout:
 		return "[timeout]"
 
+	# Normalize IP addresses. IP address --- especially IPv6 addresses --- can
+	# be expressed in equivalent string forms. Canonicalize the form before
+	# returning them. The caller should normalize any IP addresses the result
+	# of this method is compared with.
+	if rtype in ("A", "AAAA"):
+		response = [normalize_ip(str(r)) for r in response]
+
 	# There may be multiple answers; concatenate the response. Remove trailing
 	# periods from responses since that's how qnames are encoded in DNS but is
 	# confusing for us. The order of the answers doesn't matter, so sort so we
 	# can compare to a well known order.
-
-	# Unfortunately, the response.__str__ returns bytes
-	# instead of string, if it resulted from an AAAA-query.
-	# We need to convert manually, until this is fixed:
-	# https://github.com/rthalley/dnspython/issues/204
-	#
-	# BEGIN HOTFIX
-	response_new = []
-	for r in response:
-		s = r.to_text()
-		if isinstance(s, bytes):
-			s = s.decode('utf-8')
-		response_new.append(s)
-	
-	response = response_new
-	# END HOTFIX
-
 	return "; ".join(sorted(str(r).rstrip('.') for r in response))
 
 def check_ssl_cert(domain, rounded_time, ssl_certificates, env, output):
@@ -805,14 +795,14 @@ def get_latest_miab_version():
 def check_miab_version(env, output):
 	config = load_settings(env)
 
-	if config.get("privacy", True):
+	try:
-		output.print_warning("Mail-in-a-Box version check disabled by privacy setting.")
+		this_ver = what_version_is_this(env)
-	else:
+	except:
-		try:
+		this_ver = "Unknown"
-			this_ver = what_version_is_this(env)
-		except:
-			this_ver = "Unknown"
 
+	if config.get("privacy", True):
+		output.print_warning("You are running version Mail-in-a-Box %s. Mail-in-a-Box version check disabled by privacy setting." % this_ver)
+	else:
 		latest_ver = get_latest_miab_version()
 
 		if this_ver == latest_ver:
@@ -892,7 +882,9 @@ def run_and_output_changes(env, pool):
 		json.dump(cur.buf, f, indent=True)
 
 def normalize_ip(ip):
-	# Use ipaddress module to normalize the IPv6 notation and ensure we are matching IPv6 addresses written in different representations according to rfc5952.
+	# Use ipaddress module to normalize the IPv6 notation and
+	# ensure we are matching IPv6 addresses written in different
+	# representations according to rfc5952.
 	import ipaddress
 	try:
 		return str(ipaddress.ip_address(ip))

management/templates/aliases.html

@@ -39,8 +39,9 @@
     <label for="addaliasForwardsTo" class="col-sm-1 control-label">Forwards To</label>
     <div class="col-sm-10">
       <textarea class="form-control" rows="3" id="addaliasForwardsTo"></textarea>
-      <div style="margin-top: 3px; padding-left: 3px; font-size: 90%" class="text-muted">
+      <div style="margin-top: 3px; padding-left: 3px; font-size: 90%">
-        <span class="domainalias">Enter just the part of an email address starting with the @-sign.</span>
+        <span class="domainalias text-muted">Enter just the part of an email address starting with the @-sign.</span>
+        <span class="text-danger">Only forward mail to addresses handled by this Mail-in-a-Box, since mail forwarded by aliases to other domains may be rejected or filtered by the receiver. To forward mail to other domains, create a mail user and then log into webmail for the user and create a filter rule to forward mail.</span>
       </div>
     </div>
   </div>

management/templates/mail-guide.html

@@ -59,7 +59,7 @@
 		</div>
 		<div class="panel-body">
 			<h4>Greylisting</h4>
-			<p>Your box using a technique called greylisting to cut down on spam. Greylisting works by delaying mail from people you haven&rsquo;t received mail from before for up to about 10 minutes. The vast majority of spam gets tricked by this. If you are waiting for an email from someone new, such as if you are registering on a new website and are waiting for an email confirmation, please give it up to 10-15 minutes to arrive.</p>
+			<p>Your box uses a technique called greylisting to cut down on spam. Greylisting works by initially rejecting mail from people you haven&rsquo;t received mail from before. Legitimate mail servers will attempt redelivery shortly afterwards, but the vast majority of spam gets tricked by this. If you are waiting for an email from someone new, such as if you are registering on a new website and are waiting for an email confirmation, please be aware there will be a minimum of 3 minutes delay, depending how soon the remote server attempts redelivery.</p>
 
 			<h4>+tag addresses</h4>
 			<p>Every incoming email address also receives mail for <code>+tag</code> addresses. If your email address is <code>you@yourdomain.com</code>, you&rsquo;ll also automatically get mail sent to <code>you+anythinghere@yourdomain.com</code>. Use this as a fast way to segment incoming mail for your own filtering rules without having to create aliases in this control panel.</p>

management/templates/ssl.html

@@ -8,7 +8,7 @@
 <p>You need a TLS certificate for this box&rsquo;s hostname ({{hostname}}) and every other domain name and subdomain that this box is hosting a website for (see the list below).</p>
 
 <div id="ssl_provision">
-<h3>Provision a certificate</h3>
+<h3>Provision certificates</h3>
 
 <div id="ssl_provision_p" style="display: none; margin-top: 1.5em">
   <button onclick='return provision_tls_cert();' class='btn btn-primary' style="float: left; margin: 0 1.5em 1em 0;">Provision</button>
@@ -19,21 +19,6 @@
 <div class="clearfix"> </div>
 
 <div id="ssl_provision_result"></div>
-
-<div id="ssl_provision_problems_div" style="display: none;">
-<p style="margin-bottom: .5em;">Certificates cannot be automatically provisioned for:</p>
-<table id="ssl_provision_problems" style="margin-top: 0;" class="table">
-  <thead>
-    <tr>
-      <th>Domain</th>
-      <th>Problem</th>
-    </tr>
-  </thead>
-  <tbody>
-  </tbody>
-</table>
-<p>Use the <em>Install Certificate</em> button below for these domains.</p>
-</div>
 </div>
 
 <h3>Certificate status</h3>
@@ -103,24 +88,12 @@ function show_tls(keep_provisioning_shown) {
       // provisioning status
 
       if (!keep_provisioning_shown)
-        $('#ssl_provision').toggle(res.can_provision.length + res.cant_provision.length > 0)
+        $('#ssl_provision').toggle(res.can_provision.length > 0)
 
       $('#ssl_provision_p').toggle(res.can_provision.length > 0);
       if (res.can_provision.length > 0)
           $('#ssl_provision_p span').text(res.can_provision.join(", "));
 
-      $('#ssl_provision_problems_div').toggle(res.cant_provision.length > 0);
-      $('#ssl_provision_problems tbody').text("");
-      for (var i = 0; i < res.cant_provision.length; i++) {
-        var domain = res.cant_provision[i];
-        var row = $("<tr><th class='domain'><a href=''></a></th><td class='status'></td></tr>");
-        $('#ssl_provision_problems tbody').append(row);
-        row.attr('data-domain', domain.domain);
-        row.find('.domain a').text(domain.domain);
-        row.find('.domain a').attr('href', 'https://' + domain.domain);
-        row.find('.status').text(domain.problem);
-      }
-
       // certificate status
       var domains = res.status;
       var tb = $('#ssl_domains tbody');
@@ -196,20 +169,15 @@ function install_cert() {
     });
 }
 
-var agree_to_tos_url_prompt = null;
-var agree_to_tos_url = null;
 function provision_tls_cert() {
   // Automatically provision any certs.
   $('#ssl_provision_p .btn').attr('disabled', '1'); // prevent double-clicks
   api(
     "/ssl/provision",
     "POST",
-    {
+    { },
-      agree_to_tos_url: agree_to_tos_url
-    },
     function(status) {
       // Clear last attempt.
-      agree_to_tos_url = null;
       $('#ssl_provision_result').text("");
       may_reenable_provision_button = true;
 
@@ -225,52 +193,33 @@ function provision_tls_cert() {
       for (var i = 0; i < status.requests.length; i++) {
         var r = status.requests[i];
 
+        if (r.result == "skipped") {
+          // not interested --- this domain wasn't in the table
+          // to begin with
+          continue;
+        }
+
         // create an HTML block to display the results of this request
         var n = $("<div><h4/><p/></div>");
         $('#ssl_provision_result').append(n);
 
+        // plain log line
+        if (typeof r === "string") {
+          n.find("p").text(r);
+          continue;
+        }
+
         // show a header only to disambiguate request blocks
         if (status.requests.length > 0)
           n.find("h4").text(r.domains.join(", "));
 
-        if (r.result == "agree-to-tos") {
+        if (r.result == "error") {
-          // user needs to agree to Let's Encrypt's TOS
-          agree_to_tos_url_prompt = r.url;
-          $('#ssl_provision_p .btn').attr('disabled', '1');
-          n.find("p").html("Please open and review <a href='" + r.url + "' target='_blank'>Let's Encrypt's terms of service agreement</a>. You must agree to their terms for a certificate to be automatically provisioned from them.");
-          n.append($('<button onclick="agree_to_tos_url = agree_to_tos_url_prompt; return provision_tls_cert();" class="btn btn-success" style="margin-left: 2em">Agree &amp; Try Again</button>'));
-
-          // don't re-enable the Provision button -- user must use the Agree button
-          may_reenable_provision_button = false;
-
-        } else if (r.result == "error") {
           n.find("p").addClass("text-danger").text(r.message);
 
-        } else if (r.result == "wait") {
-          // Show a button that counts down to zero, at which point it becomes enabled.
-          n.find("p").text("A certificate is now in the process of being provisioned, but it takes some time. Please wait until the Finish button is enabled, and then click it to acquire the certificate.");
-          var b = $('<button onclick="return provision_tls_cert();" class="btn btn-success" style="margin-left: 2em">Finish</button>');
-          b.attr("disabled", "1");
-          var now = new Date();
-          n.append(b);
-          function ready_to_finish() {
-            var remaining = Math.round(r.seconds - (new Date() - now)/1000);
-            if (remaining > 0) {
-              setTimeout(ready_to_finish, 1000);
-              b.text("Finish (" + remaining + "...)")
-            } else {
-              b.text("Finish (ready)")
-              b.removeAttr("disabled");
-            }
-          }
-          ready_to_finish();
-
-          // don't re-enable the Provision button -- user must use the Retry button when it becomes enabled
-          may_reenable_provision_button = false;
-
         } else if (r.result == "installed") {
           n.find("p").addClass("text-success").text("The TLS certificate was provisioned and installed.");
           setTimeout("show_tls(true)", 1); // update main table of certificate statuses, call with arg keep_provisioning_shown true so that we don't clear what we just outputted
+
         }
 
         // display the detailed log info in case of problems
@@ -278,7 +227,6 @@ function provision_tls_cert() {
         n.append(trace);
         for (var j = 0; j < r.log.length; j++)
           trace.append($("<div/>").text(r.log[j]));
-
       }
 
       if (may_reenable_provision_button)

management/templates/users.html

@@ -31,7 +31,7 @@
   <button type="submit" class="btn btn-primary">Add User</button>
 </form>
 <ul style="margin-top: 1em; padding-left: 1.5em; font-size: 90%;">
-  <li>Passwords must be at least eight characters and may not contain spaces. For best results, <a href="#" onclick="return generate_random_password()">generate a random password</a>.</li>
+  <li>Passwords must be at least eight characters consisting of English lettters and numbers only. For best results, <a href="#" onclick="return generate_random_password()">generate a random password</a>.</li>
   <li>Use <a href="#" onclick="return show_panel('aliases')">aliases</a> to create email addresses that forward to existing accounts.</li>
   <li>Administrators get access to this control panel.</li>
   <li>User accounts cannot contain any international (non-ASCII) characters, but <a href="#" onclick="return show_panel('aliases');">aliases</a> can.</li>
@@ -43,7 +43,6 @@
     <tr>
       <th width="50%">Email Address</th>
       <th>Actions</th>
-      <th>Mailbox Size</th>
     </tr>
   </thead>
   <tbody>
@@ -73,8 +72,6 @@
           archive account
         </a>
     </td>
-    <td class='mailboxsize'>
-    </td>
   </tr>
   <tr id="user-extra-template" class="if_inactive">
     <td colspan="3" style="border: 0; padding-top: 0">
@@ -156,7 +153,6 @@ function show_users() {
 
           n.attr('data-email', user.email);
           n.find('.address').text(user.email)
-          n.find('.mailboxsize').text(nice_size(user.mailbox_size))
           n2.find('.restore_info tt').text(user.mailbox);
 
           if (user.status == 'inactive') continue;

management/web_update.py

@@ -149,7 +149,10 @@ def make_domain_config(domain, templates, ssl_certificates, env):
 
 			# any proxy or redirect here?
 			for path, url in yaml.get("proxies", {}).items():
-				nginx_conf_extra += "\tlocation %s {\n\t\tproxy_pass %s;\n\t}\n" % (path, url)
+				nginx_conf_extra += "\tlocation %s {" % path
+				nginx_conf_extra += "\n\t\tproxy_pass %s;" % url
+				nginx_conf_extra += "\n\t\tproxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;"
+				nginx_conf_extra += "\n\t}\n"
 			for path, url in yaml.get("redirects", {}).items():
 				nginx_conf_extra += "\trewrite %s %s permanent;\n" % (path, url)
 
@@ -198,8 +201,11 @@ def get_web_domains_info(env):
 
 	# for the SSL config panel, get cert status
 	def check_cert(domain):
-		tls_cert = get_domain_ssl_files(domain, ssl_certificates, env, allow_missing_cert=True)
+		try:
-		if tls_cert is None: return ("danger", "No Certificate Installed")
+			tls_cert = get_domain_ssl_files(domain, ssl_certificates, env, allow_missing_cert=True)
+		except OSError: # PRIMARY_HOSTNAME cert is missing
+			tls_cert = None
+		if tls_cert is None: return ("danger", "No certificate installed.")
 		cert_status, cert_status_details = check_certificate(domain, tls_cert["certificate"], tls_cert["private-key"])
 		if cert_status == "OK":
 			return ("success", "Signed & valid. " + cert_status_details)

setup/bootstrap.sh

@@ -7,7 +7,7 @@
 #########################################################
 
 if [ -z "$TAG" ]; then
-	TAG=v0.26c
+	TAG=v0.30
 fi
 
 # Are we running as root?

setup/dns.sh

@@ -26,6 +26,7 @@ cat > /etc/nsd/nsd.conf << EOF;
 # Do not edit. Overwritten by Mail-in-a-Box setup.
 server:
   hide-version: yes
+  logfile: "/var/log/nsd.log"
 
   # identify the server (CH TXT ID.SERVER entry).
   identity: ""
@@ -41,6 +42,18 @@ server:
 
 EOF
 
+# Add log rotation
+cat > /etc/logrotate.d/nsd <<EOF;
+/var/log/nsd.log {
+  weekly
+  missingok
+  rotate 12
+  compress
+  delaycompress
+  notifempty
+}
+EOF
+
 # Since we have bind9 listening on localhost for locally-generated
 # DNS queries that require a recursive nameserver, and the system
 # might have other network interfaces for e.g. tunnelling, we have

setup/management.sh

@@ -6,18 +6,32 @@ echo "Installing Mail-in-a-Box system management daemon..."
 
 # DEPENDENCIES
 
+# We used to install management daemon-related Python packages
+# directly to /usr/local/lib. We moved to a virtualenv because
+# these packages might conflict with apt-installed packages.
+# We may have a lingering version of acme that conflcits with
+# certbot, which we're about to install below, so remove it
+# first. Once acme is installed by an apt package, this might
+# break the package version and `apt-get install --reinstall python3-acme`
+# might be needed in that case.
+while [ -d /usr/local/lib/python3.4/dist-packages/acme ]; do
+	pip3 uninstall -y acme;
+done
+
 # duplicity is used to make backups of user data. It uses boto
 # (via Python 2) to do backups to AWS S3. boto from the Ubuntu
 # package manager is too out-of-date -- it doesn't support the newer
 # S3 api used in some regions, which breaks backups to those regions.
 # See #627, #653.
-apt_install duplicity python-pip
+#
+# python-virtualenv is used to isolate the Python 3 packages we
+# install via pip from the system-installed packages.
+#
+# certbot installs EFF's certbot which we use to
+# provision free TLS certificates.
+apt_install duplicity python-pip python-virtualenv certbot
 hide_output pip2 install --upgrade boto
 
-# These are required to build/install the cryptography Python package
-# used by our management daemon.
-apt_install python-virtualenv build-essential libssl-dev libffi-dev python3-dev
-
 # Create a virtualenv for the installation of Python 3 packages
 # used by the management daemon.
 inst_dir=/usr/local/lib/mailinabox
@@ -27,39 +41,16 @@ if [ ! -d $venv ]; then
 	virtualenv -ppython3 $venv
 fi
 
-# pip<6.1 + setuptools>=34 had a problem with packages that
+# Upgrade pip because the Ubuntu-packaged version is out of date.
-# try to update setuptools during installation, like cryptography.
-# See https://github.com/pypa/pip/issues/4253. The Ubuntu 14.04
-# package versions are pip 1.5.4 and setuptools 3.3. When we used to
-# instal cryptography system-wide under those versions, it updated
-# setuptools to version 34, which created the conflict, and
-# then pip gets permanently broken with errors like
-# "ImportError: No module named 'packaging'".
-#
-# Let's test for the error:
-if ! python3 -c "from pkg_resources import load_entry_point" 2&> /dev/null; then
-	# This system seems to be broken already.
-	echo "Fixing broken pip and setuptools..."
-	rm -rf /usr/local/lib/python3.4/dist-packages/{pkg_resources,setuptools}*
-	apt-get install --reinstall python3-setuptools python3-pip python3-pkg-resources
-fi
-#
-# The easiest work-around on systems that aren't already broken is
-# to upgrade pip (to >=9.0.1) and setuptools (to >=34.1) individually
-# before we install any package that tries to update setuptools.
 hide_output $venv/bin/pip install --upgrade pip
-hide_output $venv/bin/pip install --upgrade setuptools
 
 # Install other Python 3 packages used by the management daemon.
 # The first line is the packages that Josh maintains himself!
 # NOTE: email_validator is repeated in setup/questions.sh, so please keep the versions synced.
-# Force acme to be updated because it seems to need it after the
-# pip/setuptools breakage (see above) and the ACME protocol may
-# have changed (I got an error on one of my systems).
 hide_output $venv/bin/pip install --upgrade \
-	rtyaml "email_validator>=1.0.0" "free_tls_certificates>=0.1.3" "exclusiveprocess" \
+	rtyaml "email_validator>=1.0.0" "exclusiveprocess" \
 	flask dnspython python-dateutil \
-	"idna>=2.0.0" "cryptography>=1.0.2" "acme==0.20.0" boto psutil
+	"idna>=2.0.0" "cryptography==2.2.2" boto psutil
 
 # CONFIGURATION
 

setup/migrate.py

@@ -137,6 +137,51 @@ def migration_10(env):
 				shutil.move(sslcert, newname)
 				os.rmdir(d)
 
+def migration_11(env):
+	# Archive the old Let's Encrypt account directory managed by free_tls_certificates
+	# because we'll use that path now for the directory managed by certbot.
+	try:
+		old_path = os.path.join(env["STORAGE_ROOT"], 'ssl', 'lets_encrypt')
+		new_path = os.path.join(env["STORAGE_ROOT"], 'ssl', 'lets_encrypt-old')
+		shutil.move(old_path, new_path)
+	except:
+		# meh
+		pass
+
+def migration_12(env):
+	# Upgrading to Carddav Roundcube plugin to version 3+, it requires the carddav_*
+        # tables to be dropped.
+        # Checking that the roundcube database already exists.
+        if os.path.exists(os.path.join(env["STORAGE_ROOT"], "mail/roundcube/roundcube.sqlite")):
+            import sqlite3
+            conn = sqlite3.connect(os.path.join(env["STORAGE_ROOT"], "mail/roundcube/roundcube.sqlite"))
+            c = conn.cursor()
+            # Get a list of all the tables that begin with 'carddav_'
+            c.execute("SELECT name FROM sqlite_master WHERE type = ? AND name LIKE ?", ('table', 'carddav_%'))
+            carddav_tables = c.fetchall()
+            # If there were tables that begin with 'carddav_', drop them
+            if carddav_tables:
+                for table in carddav_tables:
+                    try:
+                        table = table[0]
+                        c = conn.cursor()
+                        dropcmd = "DROP TABLE %s" % table
+                        c.execute(dropcmd)
+                    except:
+                        print("Failed to drop table", table, e)
+            # Save.
+            conn.commit()
+            conn.close()
+
+            # Delete all sessions, requring users to login again to recreate carddav_*
+            # databases
+            conn = sqlite3.connect(os.path.join(env["STORAGE_ROOT"], "mail/roundcube/roundcube.sqlite"))
+            c = conn.cursor()
+            c.execute("delete from session;")
+            conn.commit()
+            conn.close()
+
+
 def get_current_migration():
 	ver = 0
 	while True:

setup/owncloud.sh

@@ -19,7 +19,7 @@ apt-get purge -qq -y owncloud*
 
 apt_install php7.0 php7.0-fpm \
 	php7.0-cli php7.0-sqlite php7.0-gd php7.0-imap php7.0-curl php-pear php-apc curl \
-        php7.0-dev php7.0-gd memcached php7.0-memcached php7.0-xml php7.0-mbstring php7.0-zip php7.0-apcu
+        php7.0-dev php7.0-gd php7.0-xml php7.0-mbstring php7.0-zip php7.0-apcu php7.0-json php7.0-intl
 
 # Migrate <= v0.10 setups that stored the ownCloud config.php in /usr/local rather than
 # in STORAGE_ROOT. Move the file to STORAGE_ROOT.
@@ -57,11 +57,11 @@ InstallNextcloud() {
 	# their github repositories.
 	mkdir -p /usr/local/lib/owncloud/apps
 
-	wget_verify https://github.com/nextcloud/contacts/releases/download/v1.5.3/contacts.tar.gz 78c4d49e73f335084feecd4853bd8234cf32615e /tmp/contacts.tgz
+	wget_verify https://github.com/nextcloud/contacts/releases/download/v2.1.5/contacts.tar.gz b7460d15f1b78d492ed502d778c0c458d503ba17 /tmp/contacts.tgz
 	tar xf /tmp/contacts.tgz -C /usr/local/lib/owncloud/apps/
 	rm /tmp/contacts.tgz
 
-	wget_verify https://github.com/nextcloud/calendar/releases/download/v1.5.3/calendar.tar.gz b370352d1f280805cc7128f78af4615f623827f8 /tmp/calendar.tgz
+	wget_verify https://github.com/nextcloud/calendar/releases/download/v1.6.1/calendar.tar.gz f93a247cbd18bc624f427ba2a967d93ebb941f21 /tmp/calendar.tgz
 	tar xf /tmp/calendar.tgz -C /usr/local/lib/owncloud/apps/
 	rm /tmp/calendar.tgz
 
@@ -154,8 +154,8 @@ InstallOwncloud() {
 	fi
 }
 
-owncloud_ver=12.0.5
+owncloud_ver=13.0.6
-owncloud_hash=d25afbac977a4e331f5e38df50aed0844498ca86
+owncloud_hash=33e41f476f0e2be5dc7cdb9d496673d9647aa3d6
 
 # Check if Nextcloud dir exist, and check if version matches owncloud_ver (if either doesn't - install/upgrade)
 if [ ! -d /usr/local/lib/owncloud/ ] \
@@ -245,6 +245,12 @@ EOF
 			echo "We are running Nextcloud 10.0.x, upgrading to Nextcloud 11.0.7 first"
 			InstallNextcloud 11.0.7 f936ddcb2ae3dbb66ee4926eb8b2ebbddc3facbe
 		fi
+
+		# If we are upgrading from Nextcloud 11 we should go to Nextcloud 12 first.
+		if grep -q "OC_VersionString = '11\." /usr/local/lib/owncloud/version.php; then
+			echo "We are running Nextcloud 11, upgrading to Nextcloud 12.0.5 first"
+			InstallNextcloud 12.0.5 d25afbac977a4e331f5e38df50aed0844498ca86
+		fi
 	fi
 
 	InstallNextcloud $owncloud_ver $owncloud_hash

setup/start.sh

@@ -14,7 +14,7 @@ source setup/preflight.sh
 # Python may not be able to read/write files. This is also
 # in the management daemon startup script and the cron script.
 
-if [ -z `locale -a | grep en_US.utf8` ]; then
+if ! locale -a | grep en_US.utf8 > /dev/null; then
     # Generate locale if not exists
     hide_output locale-gen en_US.UTF-8
 fi
@@ -127,13 +127,23 @@ tools/web_update
 # fail2ban was first configured, but they should exist now.
 restart_service fail2ban
 
-# If DNS is already working, try to provision TLS certficates from Let's Encrypt.
-# Suppress extra reasons why domains aren't getting a new certificate.
-management/ssl_certificates.py -q
-
 # If there aren't any mail users yet, create one.
 source setup/firstuser.sh
 
+# Register with Let's Encrypt, including agreeing to the Terms of Service.
+# We'd let certbot ask the user interactively, but when this script is
+# run in the recommended curl-pipe-to-bash method there is no TTY and
+# certbot will fail if it tries to ask.
+if [ ! -d $STORAGE_ROOT/ssl/lets_encrypt/accounts/acme-v02.api.letsencrypt.org/ ]; then
+echo
+echo "-----------------------------------------------"
+echo "Mail-in-a-Box uses Let's Encrypt to provision free SSL/TLS certificates"
+echo "to enable HTTPS connections to your box. We're automatically"
+echo "agreeing you to their subscriber agreement. See https://letsencrypt.org."
+echo
+certbot register --register-unsafely-without-email --agree-tos --config-dir $STORAGE_ROOT/ssl/lets_encrypt
+fi
+
 # Done.
 echo
 echo "-----------------------------------------------"

setup/system.sh

@@ -68,17 +68,10 @@ then
 	fi
 fi
 
-# ### Add Mail-in-a-Box's PPA.
+# ### Add PPAs.
-
-# We've built several .deb packages on our own that we want to include.
-# One is a replacement for Ubuntu's stock postgrey package that makes
-# some enhancements. The other is dovecot-lucene, a Lucene-based full
-# text search plugin for (and by) dovecot, which is not available in
-# Ubuntu currently.
-#
-# So, first ensure add-apt-repository is installed, then use it to install
-# the [mail-in-a-box ppa](https://launchpad.net/~mail-in-a-box/+archive/ubuntu/ppa).
 
+# We install some non-standard Ubuntu packages maintained by us and other
+# third-party providers. First ensure add-apt-repository is installed.
 
 if [ ! -f /usr/bin/add-apt-repository ]; then
 	echo "Installing add-apt-repository..."
@@ -86,11 +79,21 @@ if [ ! -f /usr/bin/add-apt-repository ]; then
 	apt_install software-properties-common
 fi
 
+# [Main-in-a-Box's own PPA](https://launchpad.net/~mail-in-a-box/+archive/ubuntu/ppa)
+# holds several .deb packages that we built on our own.
+# One is a replacement for Ubuntu's stock postgrey package that makes
+# some enhancements. The other is dovecot-lucene, a Lucene-based full
+# text search plugin for (and by) dovecot, which is not available in
+# Ubuntu currently.
+
 hide_output add-apt-repository -y ppa:mail-in-a-box/ppa
+hide_output add-apt-repository -y ppa:certbot/certbot
 
 # ### Update Packages
 
-# Update system packages to make sure we have the latest upstream versions of things from Ubuntu.
+# Update system packages to make sure we have the latest upstream versions
+# of things from Ubuntu, as well as the directory of packages provide by the
+# PPAs so we can install those packages later.
 
 echo Updating system packages...
 hide_output apt-get update
@@ -123,7 +126,7 @@ echo Installing system packages...
 apt_install python3 python3-dev python3-pip \
 	netcat-openbsd wget curl git sudo coreutils bc \
 	haveged pollinate unzip \
-	unattended-upgrades cron ntp fail2ban
+	unattended-upgrades cron ntp fail2ban rsyslog
 
 # ### Add PHP7 PPA
 

setup/webmail.sh

@@ -22,7 +22,7 @@ source /etc/mailinabox.conf # load global vars
 echo "Installing Roundcube (webmail)..."
 apt_install \
 	dbconfig-common \
-	php7.0-cli php7.0-sqlite php7.0-mcrypt php7.0-intl php7.0-json php7.0-common \
+	php7.0-cli php7.0-sqlite php7.0-mcrypt php7.0-intl php7.0-json php7.0-common php7.0-curl \
 	php7.0-gd php7.0-pspell tinymce libjs-jquery libjs-jquery-mousewheel libmagic1 php7.0-mbstring
 
 apt_get_quiet remove php-mail-mimedecode # no longer needed since Roundcube 1.1.3
@@ -35,12 +35,12 @@ apt-get purge -qq -y roundcube* #NODOC
 # Install Roundcube from source if it is not already present or if it is out of date.
 # Combine the Roundcube version number with the commit hash of plugins to track
 # whether we have the latest version of everything.
-VERSION=1.3.3
+VERSION=1.3.8
-HASH=903a4eb1bfc25e9a08d782a7f98502cddfa579de
+HASH=90c7900ccf7b2f46fe49c650d5adb9b85ee9cc22
 PERSISTENT_LOGIN_VERSION=dc5ca3d3f4415cc41edb2fde533c8a8628a94c76
 HTML5_NOTIFIER_VERSION=4b370e3cd60dabd2f428a26f45b677ad1b7118d5
-CARDDAV_VERSION=2.0.4
+CARDDAV_VERSION=3.0.3
-CARDDAV_HASH=d93f3cfb3038a519e71c7c3212c1d16f5da609a4
+CARDDAV_HASH=d1e3b0d851ffa2c6bd42bf0c04f70d0e1d0d78f8
 
 UPDATE_KEY=$VERSION:$PERSISTENT_LOGIN_VERSION:$HTML5_NOTIFIER_VERSION:$CARDDAV_VERSION
 
@@ -155,6 +155,7 @@ cat > ${RCM_PLUGIN_DIR}/carddav/config.inc.php <<EOF;
 	 'preemptive_auth' => '1',
 	 'hide'        =>  false,
 );
+?>
 EOF
 
 # Create writable directories.

setup/zpush.sh

@@ -22,7 +22,8 @@ apt_install \
 phpenmod -v php7.0 imap
 
 # Copy Z-Push into place.
-VERSION=2.3.8
+VERSION=2.4.4
+TARGETHASH=104d44426852429dac8ec2783a4e9ad7752d4682
 needs_update=0 #NODOC
 if [ ! -f /usr/local/lib/z-push/version ]; then
 	needs_update=1 #NODOC
@@ -31,13 +32,14 @@ elif [[ $VERSION != `cat /usr/local/lib/z-push/version` ]]; then
 	needs_update=1 #NODOC
 fi
 if [ $needs_update == 1 ]; then
-	rm -rf /usr/local/lib/z-push
+	# Download
+	wget_verify "https://stash.z-hub.io/rest/api/latest/projects/ZP/repos/z-push/archive?at=refs%2Ftags%2F$VERSION&format=zip" $TARGETHASH /tmp/z-push.zip
 
-	git_clone https://stash.z-hub.io/scm/zp/z-push.git $VERSION '' /tmp/z-push
+	# Extract into place.
-
+	rm -rf /usr/local/lib/z-push /tmp/z-push
-	mkdir /usr/local/lib/z-push
+	unzip -q /tmp/z-push.zip -d /tmp/z-push
-	cp -r /tmp/z-push/src/* /usr/local/lib/z-push
+	mv /tmp/z-push/src /usr/local/lib/z-push
-	rm -rf /tmp/z-push
+	rm -rf /tmp/z-push.zip /tmp/z-push
 
 	rm -f /usr/sbin/z-push-{admin,top}
 	ln -s /usr/local/lib/z-push/z-push-admin.php /usr/sbin/z-push-admin

tools/owncloud-unlockadmin.sh

@@ -20,4 +20,4 @@ echo
 echo Press enter to continue.
 read
 
-sqlite3 $STORAGE_ROOT/owncloud/owncloud.db "INSERT OR IGNORE INTO oc_group_user VALUES ('admin', '$ADMIN')" && echo Done.
+sudo -u www-data php /usr/local/lib/owncloud/occ group:adduser admin $ADMIN && echo Done.

tools/update-subresource-integrity.py

@@ -1,24 +0,0 @@
-#!/usr/bin/python3
-# Updates subresource integrity attributes in management/templates/index.html
-# to prevent CDN-hosted resources from being used as an attack vector. Run this
-# after updating the Bootstrap and jQuery <link> and <script> to compute the
-# appropriate hash and insert it into the template.
-
-import re, urllib.request, hashlib, base64
-
-fn = "management/templates/index.html"
-
-with open(fn, 'r') as f:
-	content = f.read()
-
-def make_integrity(url):
-	resource = urllib.request.urlopen(url).read()
-	return "sha256-" + base64.b64encode(hashlib.sha256(resource).digest()).decode('ascii')
-
-content = re.sub(
-	r'<(link rel="stylesheet" href|script src)="(.*?)" integrity="(.*?)"',
-	lambda m : '<' + m.group(1) + '="' + m.group(2) + '" integrity="' + make_integrity(m.group(2)) + '"',
-	content)
-
-with open(fn, 'w') as f:
-	f.write(content)