v0.22 released

Since it says "RCMCardDAV requires at least PHP 5.6.18. Older versions might work", let's hope for the best.

Also hiding its preferences panel in settings since if it doesn't work, we don't want folks using it for anything but connecting to ownCloud contacts.

.editorconfig

@@ -0,0 +1,30 @@
+# EditorConfig helps developers define and maintain consistent
+# coding styles between different editors and IDEs
+# editorconfig.org
+
+root = true
+
+[*]
+indent_style = space
+indent_size = 4
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[Makefile]
+indent_style = tabs
+indent_size = 4
+
+[Vagrantfile]
+indent_size = 2
+
+[*.rb]
+indent_size = 2
+
+[*.py]
+indent_style = tabs
+
+[*.js]
+indent_size = 2
+

.gitignore

@@ -4,3 +4,4 @@ management/__pycache__/
 tools/__pycache__/
 externals/
 .env
+.vagrant

CHANGELOG.md

@@ -1,6 +1,49 @@
 CHANGELOG
 =========
 
+v0.22 (April 2, 2017)
+---------------------
+
+Mail:
+
+* The CardDAV plugin has been added to Roundcube so that your ownCloud contacts are available in webmail.
+* Upgraded to Roundcube 1.2.4 and updated the persistent login plugin.
+* Allow larger messages to be checked by SpamAssassin.
+* Dovecot's vsz memory limit has been increased proportional to system memory.
+* Newly set user passwords must be at least eight characters.
+
+ownCloud:
+
+* Upgraded to ownCloud 9.1.4.
+
+Control Panel/Management:
+
+* The status checks page crashed when the mailinabox.email website was down - that's fixed.
+* Made nightly re-provisioning of TLS certificates less noisy.
+* Fixed bugs in rsync backup method and in the list of recent backups.
+* Fixed incorrect status checks errors about IPv6 addresses.
+* Fixed incorrect status checks errors for secondary nameservers if round-robin custom A records are set.
+* The management mail_log.py tool has been rewritten.
+
+DNS:
+
+* Added support for DSA, ED25519, and custom SSHFP records.
+
+System:
+
+* The SSH fail2ban jail was not activated.
+
+Installation:
+
+* At the end of installation, the SHA256 -- rather than SHA1 -- hash of the system's TLS certificate is shown.
+
+v0.21c (February 1, 2017)
+-------------------------
+
+Installations and upgrades started failing about 10 days ago with the error "ImportError: No module named 'packaging'" after an upstream package (Python's setuptools) was updated by its maintainers. The updated package conflicted with Ubuntu 14.04's version of another package (Python's pip). This update upgrades both packages to remove the conflict.
+
+If you already encountered the error during installation or upgrade of Mail-in-a-Box, this update may not correct the problem on your existing system. See https://discourse.mailinabox.email/t/v0-21c-release-fixes-python-package-installation-issue/1881 for help if the problem persists after upgrading to this version of Mail-in-a-Box.
+
 v0.21b (December 4, 2016)
 -------------------------
 

README.md

@@ -59,7 +59,7 @@ by me:
 	$ curl -s https://keybase.io/joshdata/key.asc | gpg --import
 	gpg: key C10BDD81: public key "Joshua Tauberer <jt@occams.info>" imported
 
-	$ git verify-tag v0.21b
+	$ git verify-tag v0.22
 	gpg: Signature made ..... using RSA key ID C10BDD81
 	gpg: Good signature from "Joshua Tauberer <jt@occams.info>"
 	gpg: WARNING: This key is not certified with a trusted signature!
@@ -72,7 +72,7 @@ and on my [personal homepage](https://razor.occams.info/). (Of course, if this r
 
 Checkout the tag corresponding to the most recent release:
 
-	$ git checkout v0.21b
+	$ git checkout v0.22
 
 Begin the installation.
 

Vagrantfile

@@ -5,23 +5,27 @@ Vagrant.configure("2") do |config|
   config.vm.box = "ubuntu14.04"
   config.vm.box_url = "http://cloud-images.ubuntu.com/vagrant/trusty/current/trusty-server-cloudimg-amd64-vagrant-disk1.box"
 
+  if Vagrant.has_plugin?("vagrant-cachier")
+    # Configure cached packages to be shared between instances of the same base box.
+    # More info on http://fgrehm.viewdocs.io/vagrant-cachier/usage
+    config.cache.scope = :box
+  end
+
   # Network config: Since it's a mail server, the machine must be connected
   # to the public web. However, we currently don't want to expose SSH since
   # the machine's box will let anyone log into it. So instead we'll put the
   # machine on a private network.
-  config.vm.hostname = "mailinabox"
+  config.vm.hostname = "mailinabox.lan"
   config.vm.network "private_network", ip: "192.168.50.4"
 
   config.vm.provision :shell, :inline => <<-SH
 	# Set environment variables so that the setup script does
 	# not ask any questions during provisioning. We'll let the
-	# machine figure out its own public IP and it'll take a
+	# machine figure out its own public IP.
-	# subdomain on our justtesting.email domain so we can get
-	# started quickly.
     export NONINTERACTIVE=1
     export PUBLIC_IP=auto
     export PUBLIC_IPV6=auto
-    export PRIMARY_HOSTNAME=auto-easy
+    export PRIMARY_HOSTNAME=auto
     #export SKIP_NETWORK_CHECKS=1
 
     # Start the setup script.

conf/fail2ban/jails.conf

@@ -73,6 +73,7 @@ action   = iptables-allports[name=recidive]
 enabled  = true
 
 [ssh]
+enabled = true
 maxretry = 7
 bantime = 3600
 

management/backup.py

@@ -10,8 +10,9 @@
 import os, os.path, shutil, glob, re, datetime, sys
 import dateutil.parser, dateutil.relativedelta, dateutil.tz
 import rtyaml
+from exclusiveprocess import Lock
 
-from utils import exclusive_process, load_environment, shell, wait_for_service, fix_boto
+from utils import load_environment, shell, wait_for_service, fix_boto
 
 rsync_ssh_options = [
 	"--ssh-options='-i /root/.ssh/id_rsa_miab'",
@@ -38,6 +39,8 @@ def backup_status(env):
 	def reldate(date, ref, clip):
 		if ref < date: return clip
 		rd = dateutil.relativedelta.relativedelta(ref, date)
+		if rd.years > 1: return "%d years, %d months" % (rd.years, rd.months)
+		if rd.years == 1: return "%d year, %d months" % (rd.years, rd.months)
 		if rd.months > 1: return "%d months, %d days" % (rd.months, rd.days)
 		if rd.months == 1: return "%d month, %d days" % (rd.months, rd.days)
 		if rd.days >= 7: return "%d days" % rd.days
@@ -204,7 +207,10 @@ def get_target_type(config):
 def perform_backup(full_backup):
 	env = load_environment()
 
-	exclusive_process("backup")
+	# Create an global exclusive lock so that the backup script
+	# cannot be run more than one.
+	Lock(die=True).forever()
+
 	config = get_backup_config(env)
 	backup_root = os.path.join(env["STORAGE_ROOT"], 'backup')
 	backup_cache_dir = os.path.join(backup_root, 'cache')
@@ -382,21 +388,21 @@ def run_duplicity_restore(args):
 def list_target_files(config):
 	import urllib.parse
 	try:
-		p = urllib.parse.urlparse(config["target"])
+		target = urllib.parse.urlparse(config["target"])
 	except ValueError:
 		return "invalid target"
 
-	if p.scheme == "file":
+	if target.scheme == "file":
-		return [(fn, os.path.getsize(os.path.join(p.path, fn))) for fn in os.listdir(p.path)]
+		return [(fn, os.path.getsize(os.path.join(target.path, fn))) for fn in os.listdir(target.path)]
 
-	elif p.scheme == "rsync":
+	elif target.scheme == "rsync":
 		rsync_fn_size_re = re.compile(r'.*    ([^ ]*) [^ ]* [^ ]* (.*)')
 		rsync_target = '{host}:{path}'
 
-		_, target_host, target_path = config['target'].split('//')
+		if not target.path.endswith('/'):
-		target_path = '/' + target_path
+			target_path = target.path + '/'
-		if not target_path.endswith('/'):
+		if target.path.startswith('/'):
-			target_path += '/'
+			target_path = target.path[1:]
 
 		rsync_command = [ 'rsync',
 					'-e',
@@ -404,11 +410,11 @@ def list_target_files(config):
 					'--list-only',
 					'-r',
 					rsync_target.format(
-						host=target_host,
+						host=target.netloc,
 						path=target_path)
 				]
 
-		code, listing = shell('check_output', rsync_command, trap=True)
+		code, listing = shell('check_output', rsync_command, trap=True, capture_stderr=True)
 		if code == 0:
 			ret = []
 			for l in listing.split('\n'):
@@ -417,21 +423,33 @@ def list_target_files(config):
 					ret.append( (match.groups()[1], int(match.groups()[0].replace(',',''))) )
 			return ret
 		else:
-			raise ValueError("Connection to rsync host failed")
+			if 'Permission denied (publickey).' in listing:
+				reason = "Invalid user or check you correctly copied the SSH key."
+			elif 'No such file or directory' in listing:
+				reason = "Provided path {} is invalid.".format(target_path)
+			elif 'Network is unreachable' in listing:
+				reason = "The IP address {} is unreachable.".format(target.hostname)
+			elif 'Could not resolve hostname':
+				reason = "The hostname {} cannot be resolved.".format(target.hostname)
+			else:
+				reason = "Unknown error." \
+						"Please check running 'python management/backup.py --verify'" \
+						"from mailinabox sources to debug the issue."
+			raise ValueError("Connection to rsync host failed: {}".format(reason))
 
-	elif p.scheme == "s3":
+	elif target.scheme == "s3":
 		# match to a Region
 		fix_boto() # must call prior to importing boto
 		import boto.s3
 		from boto.exception import BotoServerError
 		for region in boto.s3.regions():
-			if region.endpoint == p.hostname:
+			if region.endpoint == target.hostname:
 				break
 		else:
 			raise ValueError("Invalid S3 region/host.")
 
-		bucket = p.path[1:].split('/')[0]
+		bucket = target.path[1:].split('/')[0]
-		path = '/'.join(p.path[1:].split('/')[1:]) + '/'
+		path = '/'.join(target.path[1:].split('/')[1:]) + '/'
 
 		# If no prefix is specified, set the path to '', otherwise boto won't list the files
 		if path == '/':
@@ -536,6 +554,12 @@ if __name__ == "__main__":
 		# are readable, and b) report if they are up to date.
 		run_duplicity_verification()
 
+	elif sys.argv[-1] == "--list":
+		# Run duplicity's verification command to check a) the backup files
+		# are readable, and b) report if they are up to date.
+		for fn, size in list_target_files(get_backup_config(load_environment())):
+			print("{}\t{}".format(fn, size))
+
 	elif sys.argv[-1] == "--status":
 		# Show backup status.
 		ret = backup_status(load_environment())

management/daily_tasks.sh

@@ -13,7 +13,7 @@ export LC_TYPE=en_US.UTF-8
 management/backup.py | management/email_administrator.py "Backup Status"
 
 # Provision any new certificates for new domains or domains with expiring certificates.
-management/ssl_certificates.py --headless | management/email_administrator.py "Error Provisioning TLS Certificate"
+management/ssl_certificates.py -q --headless | management/email_administrator.py "Error Provisioning TLS Certificate"
 
 # Run status checks and email the administrator if anything changed.
 management/status_checks.py --show-changes | management/email_administrator.py "Status Checks Change Notice"

management/dns_update.py

@@ -342,6 +342,7 @@ def build_sshfp_records():
 		"ssh-rsa": 1,
 		"ssh-dss": 2,
 		"ecdsa-sha2-nistp256": 3,
+		"ssh-ed25519": 4,
 	}
 
 	# Get our local fingerprints by running ssh-keyscan. The output looks
@@ -359,7 +360,7 @@ def build_sshfp_records():
 				ports = ports + [s[1]]
 	# the keys are the same at each port, so we only need to get
 	# them at the first port found (may not be port 22)
-	keys = shell("check_output", ["ssh-keyscan", "-p", ports[0], "localhost"])
+	keys = shell("check_output", ["ssh-keyscan", "-t", "rsa,dsa,ecdsa,ed25519", "-p", ports[0], "localhost"])
 	for key in sorted(keys.split("\n")):
 		if key.strip() == "" or key[0] == "#": continue
 		try:
@@ -766,7 +767,7 @@ def set_custom_dns_record(qname, rtype, value, action, env):
 				v = ipaddress.ip_address(value) # raises a ValueError if there's a problem
 				if rtype == "A" and not isinstance(v, ipaddress.IPv4Address): raise ValueError("That's an IPv6 address.")
 				if rtype == "AAAA" and not isinstance(v, ipaddress.IPv6Address): raise ValueError("That's an IPv4 address.")
-		elif rtype in ("CNAME", "TXT", "SRV", "MX"):
+		elif rtype in ("CNAME", "TXT", "SRV", "MX", "SSHFP"):
 			# anything goes
 			pass
 		else:
@@ -881,10 +882,10 @@ def set_secondary_dns(hostnames, env):
 	return do_dns_update(env)
 
 
-def get_custom_dns_record(custom_dns, qname, rtype):
+def get_custom_dns_records(custom_dns, qname, rtype):
 	for qname1, rtype1, value in custom_dns:
 		if qname1 == qname and rtype1 == rtype:
-			return value
+			yield value
 	return None
 
 ########################################################################

management/mailconfig.py

@@ -599,8 +599,8 @@ def validate_password(pw):
 		raise ValueError("No password provided.")
 	if re.search(r"[\s]", pw):
 		raise ValueError("Passwords cannot contain spaces.")
-	if len(pw) < 4:
+	if len(pw) < 8:
-		raise ValueError("Passwords must be at least four characters.")
+		raise ValueError("Passwords must be at least eight characters.")
 
 
 if __name__ == "__main__":

management/ssl_certificates.py

@@ -4,7 +4,6 @@
 import os, os.path, re, shutil
 
 from utils import shell, safe_domain_name, sort_domains
-
 import idna
 
 # SELECTING SSL CERTIFICATES FOR USE IN WEB
@@ -214,6 +213,7 @@ def get_certificates_to_provision(env, show_extended_problems=True, force_domain
 
 	# Filter out domains that we can't provision a certificate for.
 	def can_provision_for_domain(domain):
+		from status_checks import normalize_ip
 		# Let's Encrypt doesn't yet support IDNA domains.
 		# We store domains in IDNA (ASCII). To see if this domain is IDNA,
 		# we'll see if its IDNA-decoded form is different.
@@ -252,7 +252,7 @@ def get_certificates_to_provision(env, show_extended_problems=True, force_domain
 				return s
 			# END HOTFIX
 
-			if len(response) != 1 or rdata__str__(response[0]) != value:
+			if len(response) != 1 or normalize_ip(rdata__str__(response[0])) != normalize_ip(value):
 				problems[domain] = "Domain control validation cannot be performed for this domain because DNS points the domain to another machine (%s %s)." % (rtype, ", ".join(rdata__str__(r) for r in response))
 				return False
 
@@ -411,9 +411,11 @@ def provision_certificates(env, agree_to_tos_url=None, logger=None, show_extende
 
 def provision_certificates_cmdline():
 	import sys
-	from utils import load_environment, exclusive_process
+	from exclusiveprocess import Lock
 
-	exclusive_process("update_tls_certificates")
+	from utils import load_environment
+
+	Lock(die=True).forever()
 	env = load_environment()
 
 	verbose = False
@@ -426,7 +428,7 @@ def provision_certificates_cmdline():
 	if args and args[0] == "-v":
 		verbose = True
 		args.pop(0)
-	if args and args[0] == "q":
+	if args and args[0] == "-q":
 		show_extended_problems = False
 		args.pop(0)
 	if args and args[0] == "--headless":

management/status_checks.py

@@ -11,7 +11,7 @@ import dateutil.parser, dateutil.tz
 import idna
 import psutil
 
-from dns_update import get_dns_zones, build_tlsa_record, get_custom_dns_config, get_secondary_dns, get_custom_dns_record
+from dns_update import get_dns_zones, build_tlsa_record, get_custom_dns_config, get_secondary_dns, get_custom_dns_records
 from web_update import get_web_domains, get_domains_with_a_records
 from ssl_certificates import get_ssl_certificates, get_domain_ssl_files, check_certificate
 from mailconfig import get_mail_domains, get_mail_aliases
@@ -393,7 +393,7 @@ def check_primary_hostname_dns(domain, env, output, dns_domains, dns_zonefiles):
 
 	# Check that PRIMARY_HOSTNAME resolves to PUBLIC_IP[V6] in public DNS.
 	ipv6 = query_dns(domain, "AAAA") if env.get("PUBLIC_IPV6") else None
-	if ip == env['PUBLIC_IP'] and ipv6 in (None, env['PUBLIC_IPV6']):
+	if ip == env['PUBLIC_IP'] and not (ipv6 and env['PUBLIC_IPV6'] and normalize_ip(ipv6) != normalize_ip(env['PUBLIC_IPV6'])):
 		output.print_ok("Domain resolves to box's IP address. [%s ↦ %s]" % (env['PRIMARY_HOSTNAME'], my_ips))
 	else:
 		output.print_error("""This domain must resolve to your box's IP address (%s) in public DNS but it currently resolves
@@ -459,7 +459,7 @@ def check_dns_zone(domain, env, output, dns_zonefiles):
 	# half working.)
 
 	custom_dns_records = list(get_custom_dns_config(env)) # generator => list so we can reuse it
-	correct_ip = get_custom_dns_record(custom_dns_records, domain, "A") or env['PUBLIC_IP']
+	correct_ip = "; ".join(sorted(get_custom_dns_records(custom_dns_records, domain, "A"))) or env['PUBLIC_IP']
 	custom_secondary_ns = get_secondary_dns(custom_dns_records, mode="NS")
 	secondary_ns = custom_secondary_ns or ["ns2." + env['PRIMARY_HOSTNAME']]
 
@@ -700,10 +700,11 @@ def query_dns(qname, rtype, nxdomain='[Not Set]', at=None):
 	# BEGIN HOTFIX
 	response_new = []
 	for r in response:
-	        if isinstance(r.to_text(), bytes):
+		s = r.to_text()
-	                response_new.append(r.to_text().decode('utf-8'))
+		if isinstance(s, bytes):
-	        else:
+			s = s.decode('utf-8')
-	                response_new.append(r)
+		response_new.append(s)
+	
 	response = response_new
 	# END HOTFIX
 
@@ -793,8 +794,13 @@ def what_version_is_this(env):
 def get_latest_miab_version():
 	# This pings https://mailinabox.email/setup.sh and extracts the tag named in
 	# the script to determine the current product version.
-	import urllib.request
+    from urllib.request import urlopen, HTTPError, URLError
-	return re.search(b'TAG=(.*)', urllib.request.urlopen("https://mailinabox.email/setup.sh?ping=1").read()).group(1).decode("utf8")
+    from socket import timeout
+
+    try:
+        return re.search(b'TAG=(.*)', urlopen("https://mailinabox.email/setup.sh?ping=1", timeout=5).read()).group(1).decode("utf8")
+    except (HTTPError, URLError, timeout):
+        return None
 
 def check_miab_version(env, output):
 	config = load_settings(env)
@@ -811,6 +817,8 @@ def check_miab_version(env, output):
 
 		if this_ver == latest_ver:
 			output.print_ok("Mail-in-a-Box is up to date. You are running version %s." % this_ver)
+		elif latest_ver is None:
+			output.print_error("Latest Mail-in-a-Box version could not be determined. You are running version %s." % this_ver)
 		else:
 			output.print_error("A new version of Mail-in-a-Box is available. You are running version %s. The latest version is %s. For upgrade instructions, see https://mailinabox.email. "
 				% (this_ver, latest_ver))
@@ -883,6 +891,11 @@ def run_and_output_changes(env, pool):
 	with open(cache_fn, "w") as f:
 		json.dump(cur.buf, f, indent=True)
 
+def normalize_ip(ip):
+	# Use ipaddress module to normalize the IPv6 notation and ensure we are matching IPv6 addresses written in different representations according to rfc5952.
+	import ipaddress
+	return str(ipaddress.ip_address(ip))
+
 class FileOutput:
 	def __init__(self, buf, width):
 		self.buf = buf

management/templates/custom-dns.html

@@ -37,6 +37,7 @@
         <option value="TXT" data-hint="Enter arbitrary text.">TXT (text record)</option>
         <option value="MX" data-hint="Enter record in the form of PRIORITY DOMAIN., including trailing period (e.g. 20 mx.example.com.).">MX (mail exchanger)</option>
         <option value="SRV" data-hint="Enter record in the form of PRIORITY WEIGHT PORT TARGET., including trailing period (e.g. 10 10 5060 sip.example.com.).">SRV (service record)</option>
+        <option value="SSHFP" data-hint="Enter record in the form of ALGORITHM TYPE FINGERPRINT.">SSHFP (SSH fingerprint record)</option>
       </select>
     </div>
   </div>
@@ -124,7 +125,7 @@
 <tr><td>email</td> <td>The email address of any administrative user here.</td></tr>
 <tr><td>password</td> <td>That user&rsquo;s password.</td></tr>
 <tr><td>qname</td> <td>The fully qualified domain name for the record you are trying to set. It must be one of the domain names or a subdomain of one of the domain names hosted on this box. (Add mail users or aliases to add new domains.)</td></tr>
-<tr><td>rtype</td> <td>The resource type. Defaults to <code>A</code> if omitted. Possible values: <code>A</code> (an IPv4 address), <code>AAAA</code> (an IPv6 address), <code>TXT</code> (a text string), <code>CNAME</code> (an alias, which is a fully qualified domain name &mdash; don&rsquo;t forget the final period), <code>MX</code>, or <code>SRV</code>.</td></tr>
+<tr><td>rtype</td> <td>The resource type. Defaults to <code>A</code> if omitted. Possible values: <code>A</code> (an IPv4 address), <code>AAAA</code> (an IPv6 address), <code>TXT</code> (a text string), <code>CNAME</code> (an alias, which is a fully qualified domain name &mdash; don&rsquo;t forget the final period), <code>MX</code>, <code>SRV</code>, or <code>SSHFP</code>.</td></tr>
 <tr><td>value</td> <td>For PUT, POST, and DELETE, the record&rsquo;s value. If the <code>rtype</code> is <code>A</code> or <code>AAAA</code> and <code>value</code> is empty or omitted, the IPv4 or IPv6 address of the remote host is used (be sure to use the <code>-4</code> or <code>-6</code> options to curl). This is handy for dynamic DNS!</td></tr>
 </table>
 

management/templates/mail-guide.html

@@ -42,7 +42,7 @@
 
 				<h4>Exchange/ActiveSync settings</h4>
 
-					<p>On iOS devices, devices on this <a href="http://z-push.org/compatibility/">compatibility list</a>, or using Outlook 2007 or later on Windows 7 and later, you may set up your mail as an Exchange or ActiveSync server. However, we&rsquo;ve found this to be more buggy than using IMAP as described above. If you encounter any problems, please use the manual settings above.</p>
+					<p>On iOS devices, devices on this <a href="https://wiki.z-hub.io/display/ZP/Compatibility">compatibility list</a>, or using Outlook 2007 or later on Windows 7 and later, you may set up your mail as an Exchange or ActiveSync server. However, we&rsquo;ve found this to be more buggy than using IMAP as described above. If you encounter any problems, please use the manual settings above.</p>
 
 					<table class="table">
 					<tr><th>Server</th> <td>{{hostname}}</td></tr>

management/templates/users.html

@@ -31,7 +31,7 @@
   <button type="submit" class="btn btn-primary">Add User</button>
 </form>
 <ul style="margin-top: 1em; padding-left: 1.5em; font-size: 90%;">
-  <li>Passwords must be at least four characters and may not contain spaces. For best results, <a href="#" onclick="return generate_random_password()">generate a random password</a>.</li>
+  <li>Passwords must be at least eight characters and may not contain spaces. For best results, <a href="#" onclick="return generate_random_password()">generate a random password</a>.</li>
   <li>Use <a href="#" onclick="return show_panel('aliases')">aliases</a> to create email addresses that forward to existing accounts.</li>
   <li>Administrators get access to this control panel.</li>
   <li>User accounts cannot contain any international (non-ASCII) characters, but <a href="#" onclick="return show_panel('aliases');">aliases</a> can.</li>
@@ -296,7 +296,7 @@ function mod_priv(elem, add_remove) {
 function generate_random_password() {
   var pw = "";
   var charset = "ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz23456789"; // confusable characters skipped
-  for (var i = 0; i < 10; i++)
+  for (var i = 0; i < 12; i++)
     pw += charset.charAt(Math.floor(Math.random() * charset.length));
   show_modal_error("Random Password", "<p>Here, try this:</p> <p><code style='font-size: 110%'>" + pw + "</code></pr");
   return false; // cancel click

management/utils.py

@@ -106,76 +106,6 @@ def sort_email_addresses(email_addresses, env):
     ret.extend(sorted(email_addresses)) # whatever is left
     return ret
 
-def exclusive_process(name):
-    # Ensure that a process named `name` does not execute multiple
-    # times concurrently.
-    import os, sys, atexit
-    pidfile = '/var/run/mailinabox-%s.pid' % name
-    mypid = os.getpid()
-
-    # Attempt to get a lock on ourself so that the concurrency check
-    # itself is not executed in parallel.
-    with open(__file__, 'r+') as flock:
-        # Try to get a lock. This blocks until a lock is acquired. The
-        # lock is held until the flock file is closed at the end of the
-        # with block.
-        os.lockf(flock.fileno(), os.F_LOCK, 0)
-
-        # While we have a lock, look at the pid file. First attempt
-        # to write our pid to a pidfile if no file already exists there.
-        try:
-            with open(pidfile, 'x') as f:
-                # Successfully opened a new file. Since the file is new
-                # there is no concurrent process. Write our pid.
-                f.write(str(mypid))
-                atexit.register(clear_my_pid, pidfile)
-                return
-        except FileExistsError:
-            # The pid file already exixts, but it may contain a stale
-            # pid of a terminated process.
-            with open(pidfile, 'r+') as f:
-                # Read the pid in the file.
-                existing_pid = None
-                try:
-                    existing_pid = int(f.read().strip())
-                except ValueError:
-                    pass # No valid integer in the file.
-
-                # Check if the pid in it is valid.
-                if existing_pid:
-                    if is_pid_valid(existing_pid):
-                        print("Another %s is already running (pid %d)." % (name, existing_pid), file=sys.stderr)
-                        sys.exit(1)
-
-                # Write our pid.
-                f.seek(0)
-                f.write(str(mypid))
-                f.truncate()
-                atexit.register(clear_my_pid, pidfile)
- 
-
-def clear_my_pid(pidfile):
-    import os
-    os.unlink(pidfile)
-
-
-def is_pid_valid(pid):
-    """Checks whether a pid is a valid process ID of a currently running process."""
-    # adapted from http://stackoverflow.com/questions/568271/how-to-check-if-there-exists-a-process-with-a-given-pid
-    import os, errno
-    if pid <= 0: raise ValueError('Invalid PID.')
-    try:
-        os.kill(pid, 0)
-    except OSError as err:
-        if err.errno == errno.ESRCH: # No such process
-            return False
-        elif err.errno == errno.EPERM: # Not permitted to send signal
-            return True
-        else: # EINVAL
-            raise
-    else:
-        return True
-
 def shell(method, cmd_args, env={}, capture_stderr=False, return_bytes=False, trap=False, input=None):
     # A safe way to execute processes.
     # Some processes like apt-get require being given a sane PATH.

setup/bootstrap.sh

@@ -7,7 +7,7 @@
 #########################################################
 
 if [ -z "$TAG" ]; then
-	TAG=v0.21b
+	TAG=v0.22
 fi
 
 # Are we running as root?

setup/firstuser.sh

@@ -35,7 +35,7 @@ if [ -z "`tools/mail.py user`" ]; then
 		else
 			# Use me@PRIMARY_HOSTNAME
 			EMAIL_ADDR=me@$PRIMARY_HOSTNAME
-			EMAIL_PW=1234
+			EMAIL_PW=12345678
 			echo
 			echo "Creating a new administrative mail account for $EMAIL_ADDR with password $EMAIL_PW."
 			echo
@@ -54,4 +54,4 @@ if [ -z "`tools/mail.py user`" ]; then
 
 	# Create an alias to which we'll direct all automatically-created administrative aliases.
 	tools/mail.py alias add administrator@$PRIMARY_HOSTNAME $EMAIL_ADDR > /dev/null
-fi
+fi

setup/mail-dovecot.sh

@@ -37,8 +37,16 @@ apt_install \
 # of active IMAP connections (at, say, 5 open connections per user that
 # would be 20 users). Set it to 250 times the number of cores this
 # machine has, so on a two-core machine that's 500 processes/100 users).
+# The `default_vsz_limit` is the maximum amount of virtual memory that
+# can be allocated. It should be set *reasonably high* to avoid allocation
+# issues with larger mailboxes. We're setting it to 1/3 of the total
+# available memory (physical mem + swap) to be sure.
+# See here for discussion:
+# - https://www.dovecot.org/list/dovecot/2012-August/137569.html
+# - https://www.dovecot.org/list/dovecot/2011-December/132455.html
 tools/editconf.py /etc/dovecot/conf.d/10-master.conf \
 	default_process_limit=$(echo "`nproc` * 250" | bc) \
+	default_vsz_limit=$(echo "`free -tom  | tail -1 | awk '{print $2}'` / 3" | bc)M \
 	log_path=/var/log/mail.log
 
 # The inotify `max_user_instances` default is 128, which constrains

setup/management.sh

@@ -4,7 +4,10 @@ source setup/functions.sh
 
 echo "Installing Mail-in-a-Box system management daemon..."
 
-# Install packages.
+# DEPENDENCIES
+
+# Install Python packages that are available from the Ubuntu
+# apt repository:
 # flask, yaml, dnspython, and dateutil are all for our Python 3 management daemon itself.
 # duplicity does backups. python-pip is so we can 'pip install boto' for Python 2, for duplicity, so it can do backups to AWS S3.
 apt_install python3-flask links duplicity libyaml-dev python3-dnspython python3-dateutil python-pip
@@ -12,17 +15,45 @@ apt_install python3-flask links duplicity libyaml-dev python3-dnspython python3-
 # These are required to pip install cryptography.
 apt_install build-essential libssl-dev libffi-dev python3-dev
 
+# pip<6.1 + setuptools>=34 have a problem with packages that
+# try to update setuptools during installation, like cryptography.
+# See https://github.com/pypa/pip/issues/4253. The Ubuntu 14.04
+# package versions are pip 1.5.4 and setuptools 3.3. When we
+# install cryptography under those versions, it tries to update
+# setuptools to version 34, which now creates the conflict, and
+# then pip gets permanently broken with errors like
+# "ImportError: No module named 'packaging'".
+#
+# Let's test for the error:
+if ! python3 -c "from pkg_resources import load_entry_point" 2&> /dev/null; then
+	# This system seems to be broken already.
+	echo "Fixing broken pip and setuptools..."
+	rm -rf /usr/local/lib/python3.4/dist-packages/{pkg_resources,setuptools}*
+	apt-get install --reinstall python3-setuptools python3-pip python3-pkg-resources
+fi
+#
+# The easiest work-around on systems that aren't already broken is
+# to upgrade pip (to >=9.0.1) and setuptools (to >=34.1) individually
+# before we install any package that tries to update setuptools.
+hide_output pip3 install --upgrade pip
+hide_output pip3 install --upgrade setuptools
+
 # Install other Python 3 packages used by the management daemon.
 # The first line is the packages that Josh maintains himself!
 # NOTE: email_validator is repeated in setup/questions.sh, so please keep the versions synced.
+# Force acme to be updated because it seems to need it after the
+# pip/setuptools breakage (see above) and the ACME protocol may
+# have changed (I got an error on one of my systems).
 hide_output pip3 install --upgrade \
-	rtyaml "email_validator>=1.0.0" "free_tls_certificates>=0.1.3" \
+	rtyaml "email_validator>=1.0.0" "free_tls_certificates>=0.1.3" "exclusiveprocess" \
-	"idna>=2.0.0" "cryptography>=1.0.2" boto psutil
+	"idna>=2.0.0" "cryptography>=1.0.2" acme boto psutil
 
 # duplicity uses python 2 so we need to get the python 2 package of boto to have backups to S3.
 # boto from the Ubuntu package manager is too out-of-date -- it doesn't support the newer
 # S3 api used in some regions, which breaks backups to those regions.  See #627, #653.
-hide_output pip install --upgrade boto
+hide_output pip2 install --upgrade boto
+
+# CONFIGURATION
 
 # Create a backup directory and a random key for encrypting backups.
 mkdir -p $STORAGE_ROOT/backup

setup/owncloud.sh

@@ -12,7 +12,7 @@ echo "Installing ownCloud (contacts/calendar)..."
 apt_install \
 	dbconfig-common \
 	php5-cli php5-sqlite php5-gd php5-imap php5-curl php-pear php-apc curl libapr1 libtool libcurl4-openssl-dev php-xml-parser \
-	php5 php5-dev php5-gd php5-fpm memcached php5-memcached unzip
+	php5 php5-dev php5-gd php5-fpm memcached php5-memcached
 
 apt-get purge -qq -y owncloud*
 
@@ -29,12 +29,13 @@ if [ ! -f $STORAGE_ROOT/owncloud/config.php ] \
 fi
 
 InstallOwncloud() {
-	echo
-	echo "Upgrading to ownCloud version $1"
-	echo
 
-        version=$1
+	version=$1
-        hash=$2
+	hash=$2
+
+	echo
+	echo "Upgrading to ownCloud version $version"
+	echo
 
 	# Remove the current owncloud
 	rm -rf /usr/local/lib/owncloud
@@ -85,7 +86,8 @@ InstallOwncloud() {
 	fi
 }
 
-owncloud_ver=9.1.1
+owncloud_ver=9.1.4
+owncloud_hash=e637cab7b2ca3346164f3506b1a0eb812b4e841a
 
 # Check if ownCloud dir exist, and check if version matches owncloud_ver (if either doesn't - install/upgrade)
 if [ ! -d /usr/local/lib/owncloud/ ] \
@@ -111,13 +113,13 @@ if [ ! -d /usr/local/lib/owncloud/ ] \
 
 	# We only need to check if we do upgrades when owncloud was previously installed
 	if [ -e /usr/local/lib/owncloud/version.php ]; then
-		if grep -q "8.1.[0-9]" /usr/local/lib/owncloud/version.php; then
+		if grep -q "8\.1\.[0-9]" /usr/local/lib/owncloud/version.php; then
 			echo "We are running 8.1.x, upgrading to 8.2.3 first"
 			InstallOwncloud 8.2.3 bfdf6166fbf6fc5438dc358600e7239d1c970613
 		fi
 
 		# If we are upgrading from 8.2.x we should go to 9.0 first. Owncloud doesn't support skipping minor versions
-		if grep -q "8.2.[0-9]" /usr/local/lib/owncloud/version.php; then
+		if grep -q "8\.2\.[0-9]" /usr/local/lib/owncloud/version.php; then
 			echo "We are running version 8.2.x, upgrading to 9.0.2 first"
 
 			# We need to disable memcached. The upgrade and install fails
@@ -152,7 +154,7 @@ EOF
 		fi
 	fi
 
-	InstallOwncloud $owncloud_ver 72ed9812432f01b3a459c4afc33f5c76b71eec09
+	InstallOwncloud $owncloud_ver $owncloud_hash
 fi
 
 # ### Configuring ownCloud

setup/questions.sh

@@ -180,9 +180,6 @@ if [ "$PUBLIC_IPV6" = "auto" ]; then
 fi
 if [ "$PRIMARY_HOSTNAME" = "auto" ]; then
 	PRIMARY_HOSTNAME=$(get_default_hostname)
-elif [ "$PRIMARY_HOSTNAME" = "auto-easy" ]; then
-	# Generate a probably-unique subdomain under our justtesting.email domain.
-	PRIMARY_HOSTNAME=`echo $PUBLIC_IP | sha1sum | cut -c1-5`.justtesting.email
 fi
 
 # Set STORAGE_USER and STORAGE_ROOT to default values (user-data and /home/user-data), unless

setup/spamassassin.sh

@@ -48,7 +48,7 @@ echo "public.pyzor.org:24441" > /etc/spamassassin/pyzor/servers
 # * Disable localmode so Pyzor, DKIM and DNS checks can be used.
 tools/editconf.py /etc/default/spampd \
 	DESTPORT=10026 \
-	ADDOPTS="\"--maxsize=500\"" \
+	ADDOPTS="\"--maxsize=2000\"" \
 	LOCALONLY=0
 
 # Spamassassin normally wraps spam as an attachment inside a fresh
@@ -63,7 +63,8 @@ tools/editconf.py /etc/default/spampd \
 # Tell Spamassassin not to modify the original message except for adding
 # the X-Spam-Status mail header and related headers.
 tools/editconf.py /etc/spamassassin/local.cf -s \
-	report_safe=0
+	report_safe=0 \
+	add_header="all Report _REPORT_"
 
 # Bayesean learning
 # -----------------

setup/start.sh

@@ -147,17 +147,17 @@ if management/status_checks.py --check-primary-hostname; then
 	echo https://$PRIMARY_HOSTNAME/admin
 	echo
 	echo "If you have a DNS problem put the box's IP address in the URL"
-	echo "(https://$PUBLIC_IP/admin) but then check the SSL fingerprint:"
+	echo "(https://$PUBLIC_IP/admin) but then check the TLS fingerprint:"
-	openssl x509 -in $STORAGE_ROOT/ssl/ssl_certificate.pem -noout -fingerprint \
+	openssl x509 -in $STORAGE_ROOT/ssl/ssl_certificate.pem -noout -fingerprint -sha256\
-        	| sed "s/SHA1 Fingerprint=//"
+        	| sed "s/SHA256 Fingerprint=//"
 else
 	echo https://$PUBLIC_IP/admin
 	echo
 	echo You will be alerted that the website has an invalid certificate. Check that
 	echo the certificate fingerprint matches:
 	echo
-	openssl x509 -in $STORAGE_ROOT/ssl/ssl_certificate.pem -noout -fingerprint \
+	openssl x509 -in $STORAGE_ROOT/ssl/ssl_certificate.pem -noout -fingerprint -sha256\
-        	| sed "s/SHA1 Fingerprint=//"
+        	| sed "s/SHA256 Fingerprint=//"
 	echo
 	echo Then you can confirm the security exception and continue.
 	echo

setup/system.sh

@@ -116,7 +116,7 @@ apt_get_quiet upgrade
 echo Installing system packages...
 apt_install python3 python3-dev python3-pip \
 	netcat-openbsd wget curl git sudo coreutils bc \
-	haveged pollinate \
+	haveged pollinate unzip \
 	unattended-upgrades cron ntp fail2ban
 
 # ### Suppress Upgrade Prompts

setup/webmail.sh

@@ -34,12 +34,21 @@ apt-get purge -qq -y roundcube* #NODOC
 # Install Roundcube from source if it is not already present or if it is out of date.
 # Combine the Roundcube version number with the commit hash of vacation_sieve to track
 # whether we have the latest version.
-VERSION=1.2.1
+VERSION=1.2.4
-HASH=81fbfba4683522f6e54006d0300a48e6da3f3bbd
+HASH=e2091ea775b80eda43ab225130d5a2e888c3789a
 VACATION_SIEVE_VERSION=91ea6f52216390073d1f5b70b5f6bea0bfaee7e5
-PERSISTENT_LOGIN_VERSION=1e9d724476a370ce917a2fcd5b3217b0c306c24e
+PERSISTENT_LOGIN_VERSION=c4516c4be37d12ef653de86497304e073a863c2a
 HTML5_NOTIFIER_VERSION=4b370e3cd60dabd2f428a26f45b677ad1b7118d5
-UPDATE_KEY=$VERSION:$VACATION_SIEVE_VERSION:$PERSISTENT_LOGIN_VERSION:$HTML5_NOTIFIER_VERSION:a
+CARDDAV_VERSION=2.0.4
+CARDDAV_HASH=d93f3cfb3038a519e71c7c3212c1d16f5da609a4
+
+UPDATE_KEY=$VERSION:$VACATION_SIEVE_VERSION:$PERSISTENT_LOGIN_VERSION:$HTML5_NOTIFIER_VERSION:$CARDDAV_VERSION:a
+
+# paths that are often reused.
+RCM_DIR=/usr/local/lib/roundcubemail
+RCM_PLUGIN_DIR=${RCM_DIR}/plugins
+RCM_CONFIG=${RCM_DIR}/config/config.inc.php
+
 needs_update=0 #NODOC
 if [ ! -f /usr/local/lib/roundcubemail/version ]; then
 	# not installed yet #NODOC
@@ -56,20 +65,30 @@ if [ $needs_update == 1 ]; then
 		/tmp/roundcube.tgz
 	tar -C /usr/local/lib --no-same-owner -zxf /tmp/roundcube.tgz
 	rm -rf /usr/local/lib/roundcubemail
-	mv /usr/local/lib/roundcubemail-$VERSION/ /usr/local/lib/roundcubemail
+	mv /usr/local/lib/roundcubemail-$VERSION/ $RCM_DIR
 	rm -f /tmp/roundcube.tgz
 
 	# install roundcube autoreply/vacation plugin
-	git_clone https://github.com/arodier/Roundcube-Plugins.git $VACATION_SIEVE_VERSION plugins/vacation_sieve /usr/local/lib/roundcubemail/plugins/vacation_sieve
+	git_clone https://github.com/arodier/Roundcube-Plugins.git $VACATION_SIEVE_VERSION plugins/vacation_sieve ${RCM_PLUGIN_DIR}/vacation_sieve
 
 	# install roundcube persistent_login plugin
-	git_clone https://github.com/mfreiholz/Roundcube-Persistent-Login-Plugin.git $PERSISTENT_LOGIN_VERSION '' /usr/local/lib/roundcubemail/plugins/persistent_login
+	git_clone https://github.com/mfreiholz/Roundcube-Persistent-Login-Plugin.git $PERSISTENT_LOGIN_VERSION '' ${RCM_PLUGIN_DIR}/persistent_login
 
 	# install roundcube html5_notifier plugin
-	git_clone https://github.com/kitist/html5_notifier.git $HTML5_NOTIFIER_VERSION '' /usr/local/lib/roundcubemail/plugins/html5_notifier
+	git_clone https://github.com/kitist/html5_notifier.git $HTML5_NOTIFIER_VERSION '' ${RCM_PLUGIN_DIR}/html5_notifier
+
+	# download and verify the full release of the carddav plugin
+	wget_verify \
+		https://github.com/blind-coder/rcmcarddav/releases/download/v${CARDDAV_VERSION}/carddav-${CARDDAV_VERSION}.zip \
+		$CARDDAV_HASH \
+		/tmp/carddav.zip
+
+	# unzip and cleanup
+	unzip -q /tmp/carddav.zip -d ${RCM_PLUGIN_DIR}
+	rm -f /tmp/carddav.zip
 
 	# record the version we've installed
-	echo $UPDATE_KEY > /usr/local/lib/roundcubemail/version
+	echo $UPDATE_KEY > ${RCM_DIR}/version
 fi
 
 # ### Configuring Roundcube
@@ -82,7 +101,7 @@ SECRET_KEY=$(dd if=/dev/urandom bs=1 count=18 2>/dev/null | base64 | fold -w 24
 # For security, temp and log files are not stored in the default locations
 # which are inside the roundcube sources directory. We put them instead
 # in normal places.
-cat > /usr/local/lib/roundcubemail/config/config.inc.php <<EOF;
+cat > $RCM_CONFIG <<EOF;
 <?php
 /*
  * Do not edit. Written by Mail-in-a-Box. Regenerated on updates.
@@ -101,7 +120,7 @@ cat > /usr/local/lib/roundcubemail/config/config.inc.php <<EOF;
 \$config['support_url'] = 'https://mailinabox.email/';
 \$config['product_name'] = '$PRIMARY_HOSTNAME Webmail';
 \$config['des_key'] = '$SECRET_KEY';
-\$config['plugins'] = array('html5_notifier', 'archive', 'zipdownload', 'password', 'managesieve', 'jqueryui', 'vacation_sieve', 'persistent_login');
+\$config['plugins'] = array('html5_notifier', 'archive', 'zipdownload', 'password', 'managesieve', 'jqueryui', 'vacation_sieve', 'persistent_login', 'carddav');
 \$config['skin'] = 'classic';
 \$config['login_autocomplete'] = 2;
 \$config['password_charset'] = 'UTF-8';
@@ -109,6 +128,26 @@ cat > /usr/local/lib/roundcubemail/config/config.inc.php <<EOF;
 ?>
 EOF
 
+# Configure CardDav
+cat > ${RCM_PLUGIN_DIR}/carddav/config.inc.php <<EOF;
+<?php
+/* Do not edit. Written by Mail-in-a-Box. Regenerated on updates. */
+\$prefs['_GLOBAL']['hide_preferences'] = true;
+\$prefs['_GLOBAL']['suppress_version_warning'] = true;
+\$prefs['ownCloud'] = array(
+	 'name'         =>  'ownCloud',
+	 'username'     =>  '%u', // login username
+	 'password'     =>  '%p', // login password
+	 'url'          =>  'https://${PRIMARY_HOSTNAME}/cloud/remote.php/carddav/addressbooks/%u/contacts',
+	 'active'       =>  true,
+	 'readonly'     =>  false,
+	 'refresh_time' => '02:00:00',
+	 'fixed'        =>  array('username','password'),
+	 'preemptive_auth' => '1',
+	 'hide'        =>  false,
+);
+EOF
+
 # Configure vaction_sieve.
 cat > /usr/local/lib/roundcubemail/plugins/vacation_sieve/config.inc.php <<EOF;
 <?php
@@ -137,13 +176,13 @@ chown -R www-data.www-data /var/log/roundcubemail /tmp/roundcubemail $STORAGE_RO
 sudo -u www-data touch /var/log/roundcubemail/errors
 
 # Password changing plugin settings
-# The config comes empty by default, so we need the settings 
+# The config comes empty by default, so we need the settings
 # we're not planning to change in config.inc.dist...
-cp /usr/local/lib/roundcubemail/plugins/password/config.inc.php.dist \
+cp ${RCM_PLUGIN_DIR}/password/config.inc.php.dist \
-	/usr/local/lib/roundcubemail/plugins/password/config.inc.php
+	${RCM_PLUGIN_DIR}/password/config.inc.php
 
-tools/editconf.py /usr/local/lib/roundcubemail/plugins/password/config.inc.php \
+tools/editconf.py ${RCM_PLUGIN_DIR}/password/config.inc.php \
-	"\$config['password_minimum_length']=6;" \
+	"\$config['password_minimum_length']=8;" \
 	"\$config['password_db_dsn']='sqlite:///$STORAGE_ROOT/mail/users.sqlite';" \
 	"\$config['password_query']='UPDATE users SET password=%D WHERE email=%u';" \
 	"\$config['password_dovecotpw']='/usr/bin/doveadm pw';" \
@@ -157,11 +196,16 @@ usermod -a -G dovecot www-data
 # could use dovecot instead of www-data, but not sure it matters
 chown root.www-data $STORAGE_ROOT/mail
 chmod 775 $STORAGE_ROOT/mail
-chown root.www-data $STORAGE_ROOT/mail/users.sqlite 
+chown root.www-data $STORAGE_ROOT/mail/users.sqlite
-chmod 664 $STORAGE_ROOT/mail/users.sqlite 
+chmod 664 $STORAGE_ROOT/mail/users.sqlite
+
+# Fix Carddav permissions:
+chown -f -R root.www-data ${RCM_PLUGIN_DIR}/carddav
+# root.www-data need all permissions, others only read
+chmod -R 774 ${RCM_PLUGIN_DIR}/carddav
 
 # Run Roundcube database migration script (database is created if it does not exist)
-/usr/local/lib/roundcubemail/bin/updatedb.sh --dir /usr/local/lib/roundcubemail/SQL --package roundcube
+${RCM_DIR}/bin/updatedb.sh --dir ${RCM_DIR}/SQL --package roundcube
 chown www-data:www-data $STORAGE_ROOT/mail/roundcube/roundcube.sqlite
 chmod 664 $STORAGE_ROOT/mail/roundcube/roundcube.sqlite
 

tools/mail.py

@@ -30,8 +30,8 @@ def mgmt(cmd, data=None, is_json=False):
 def read_password():
     while True:
         first = getpass.getpass('password: ')
-        if len(first) < 4:
+        if len(first) < 8:
-            print("Passwords must be at least four characters.")
+            print("Passwords must be at least eight characters.")
             continue
         if re.search(r'[\s]', first):
             print("Passwords cannot contain spaces.")