Merge pull request #2 from mail-in-a-box/master

Update to latest mailinabox master

CHANGELOG.md

@@ -8,12 +8,31 @@ Mail:
 
 * Spam checking is now performed on messages larger than the previous limit of 64KB.
 * POP3S is now enabled (port 995).
-* In order to guard against misconfiguration that can lead to domain control validation hijacking, email addresses that begin with admin, administrator, postmaster, hostmaster, and webmaster can no longer be used for (new) mail user accounts, and aliases for these addresses may direct mail only to the box's administrator(s).
+* Roundcube updated to version 1.1.1.
+* More mail headers with user agent info are anonymized.
 
-System:
+ownCloud:
+
+* Downloading files you uploaded to ownCloud broke because of a change in ownCloud 8.
+
+DNS:
 
 * Internationalized Domain Names (IDNs) should now work in email. If you had custom DNS or custom web settings for internationalized domains, check that they are still working.
+* It is now possible to set multiple TXT and other types of records on the same domain in the control panel.
+* The custom DNS API was completely rewritten to support setting multiple records of the same type on a domain. Any existing client code using the DNS API will have to be rewritten. (Existing code will just get 404s back.)
 
+System / Control Panel:
+
+* In order to guard against misconfiguration that can lead to domain control validation hijacking, email addresses that begin with admin, administrator, postmaster, hostmaster, and webmaster can no longer be used for (new) mail user accounts, and aliases for these addresses may direct mail only to the box's administrator(s).
+* Backups now use duplicity's built-in gpg symmetric AES256 encryption rather than my home-brewed encryption. Old backups will be incorporated inside the first backup after this update but then deleted from disk (i.e. your backups from the previous few days will be backed up).
+* There was a race condition between backups and the new nightly status checks.
+* The control panel would sometimes lock up with an unnecessary loading indicator.
+* You can no longer delete your own account from the control panel.
+
+Setup:
+
+* All Mail-in-a-Box release tags are now signed on github, instructions for verifying the signature are added to the README, and the integrity of some packages downloaded during setup is now verified against a SHA1 hash stored in the tag itself.
+* Bugs in first user account creation were fixed.
 
 v0.08 (April 1, 2015)
 ---------------------

conf/nginx-primaryonly.conf

@@ -28,6 +28,7 @@
 		fastcgi_param SCRIPT_FILENAME /usr/local/lib/owncloud/$2;
 		fastcgi_param SCRIPT_NAME $1$2;
 		fastcgi_param PATH_INFO $3;
+		# TODO: see the dispreferred "method 2" for xaccel at https://doc.owncloud.org/server/8.1/admin_manual/configuration_files/serving_static_files_configuration.html
 		fastcgi_param MOD_X_ACCEL_REDIRECT_ENABLED on;
 		fastcgi_read_timeout 630;
 		fastcgi_pass php-fpm;
@@ -36,10 +37,13 @@
 		client_max_body_size 1G;
 		fastcgi_buffers 64 4K;
 	}
-	location ^~ /cloud/data {
+	location ^~ $STORAGE_ROOT/owncloud {
-		# In order to support MOD_X_ACCEL_REDIRECT_ENABLED, we need to expose
+		# This directory is for MOD_X_ACCEL_REDIRECT_ENABLED. It works a little weird.
-		# the data directory but only allow 'internal' redirects within nginx
+		# The full path on disk of the file is passed as the URL path. ownCloud 8 totally
-		# so that this is not exposed to the world.
+		# busted the sane way this worked in ownCloud 7. There's a pending change using
+		# a new parameter to make this make more sense.
+		# We need to only allow 'internal' redirects within nginx so that the filesystem
+		# is not exposed to the world.
 		internal;
 		alias $STORAGE_ROOT/owncloud;
 	}

conf/postfix_outgoing_mail_header_filters

@@ -8,3 +8,7 @@
 /^\s*X-Enigmail:/        IGNORE
 /^\s*X-Mailer:/          IGNORE
 /^\s*X-Originating-IP:/  IGNORE
+/^\s*X-Pgp-Agent:/       IGNORE
+
+# The Mime-Version header can leak the user agent too, e.g. in Mime-Version: 1.0 (Mac OS X Mail 8.1 \(2010.6\)).
+/^\s*(Mime-Version:\s*[0-9\.]+)\s.+/  REPLACE $1

management/auth.py

@@ -111,12 +111,12 @@ class KeyAuthService:
 				# Login failed.
 				raise ValueError("Invalid password.")
 
-		# Get privileges for authorization.
+		# Get privileges for authorization. This call should never fail on a valid user,
-
+		# but if the caller passed a user-specific API key then the user may no longer
-		# (This call should never fail on a valid user. But if it did fail, it would
+		# exist --- in that case, get_mail_user_privileges will return a tuple of an
-		# return a tuple of an error message and an HTTP status code.)
+		# error message and an HTTP status code.
 		privs = get_mail_user_privileges(email, env)
-		if isinstance(privs, tuple): raise Exception("Error getting privileges.")
+		if isinstance(privs, tuple): raise ValueError(privs[0])
 
 		# Return a list of privileges.
 		return privs

management/backup.py

@@ -2,25 +2,24 @@
 
 # This script performs a backup of all user data:
 # 1) System services are stopped while a copy of user data is made.
-# 2) An incremental backup is made using duplicity into the
+# 2) An incremental encrypted backup is made using duplicity into the
-#    directory STORAGE_ROOT/backup/duplicity.
+#    directory STORAGE_ROOT/backup/encrypted. The password used for
+#    encryption is stored in backup/secret_key.txt.
 # 3) The stopped services are restarted.
-# 4) The backup files are encrypted with a long password (stored in
-#    backup/secret_key.txt) to STORAGE_ROOT/backup/encrypted.
 # 5) STORAGE_ROOT/backup/after-backup is executd if it exists.
 
 import os, os.path, shutil, glob, re, datetime
 import dateutil.parser, dateutil.relativedelta, dateutil.tz
 
-from utils import exclusive_process, load_environment, shell
+from utils import exclusive_process, load_environment, shell, wait_for_service
 
-# destroy backups when the most recent increment in the chain
+# Destroy backups when the most recent increment in the chain
 # that depends on it is this many days old.
 keep_backups_for_days = 3
 
 def backup_status(env):
 	# What is the current status of backups?
-	# Loop through all of the files in STORAGE_ROOT/backup/duplicity to
+	# Loop through all of the files in STORAGE_ROOT/backup/encrypted to
 	# get a list of all of the backups taken and sum up file sizes to
 	# see how large the storage is.
 
@@ -36,10 +35,10 @@ def backup_status(env):
 		return "%d hours, %d minutes" % (rd.hours, rd.minutes)
 
 	backups = { }
-	basedir = os.path.join(env['STORAGE_ROOT'], 'backup/duplicity/')
+	backup_root = os.path.join(env["STORAGE_ROOT"], 'backup')
-	encdir = os.path.join(env['STORAGE_ROOT'], 'backup/encrypted/')
+	backup_dir = os.path.join(backup_root, 'encrypted')
-	os.makedirs(basedir, exist_ok=True) # os.listdir fails if directory does not exist
+	os.makedirs(backup_dir, exist_ok=True) # os.listdir fails if directory does not exist
-	for fn in os.listdir(basedir):
+	for fn in os.listdir(backup_dir):
 		m = re.match(r"duplicity-(full|full-signatures|(inc|new-signatures)\.(?P<incbase>\d+T\d+Z)\.to)\.(?P<date>\d+T\d+Z)\.", fn)
 		if not m: raise ValueError(fn)
 
@@ -53,15 +52,9 @@ def backup_status(env):
 				"full": m.group("incbase") is None,
 				"previous": m.group("incbase"),
 				"size": 0,
-				"encsize": 0,
 			}
 
-		backups[key]["size"] += os.path.getsize(os.path.join(basedir, fn))
+		backups[key]["size"] += os.path.getsize(os.path.join(backup_dir, fn))
-
-		# Also check encrypted size.
-		encfn = os.path.join(encdir, fn + ".enc")
-		if os.path.exists(encfn):
-			backups[key]["encsize"] += os.path.getsize(encfn)
 
 	# Ensure the rows are sorted reverse chronologically.
 	# This is relied on by should_force_full() and the next step.
@@ -78,7 +71,7 @@ def backup_status(env):
 			break
 		incremental_count += 1
 		incremental_size += bak["size"]
-	
+
 	# Predict how many more increments until the next full backup,
 	# and add to that the time we hold onto backups, to predict
 	# how long the most recent full backup+increments will be held
@@ -106,9 +99,8 @@ def backup_status(env):
 			bak["deleted_in"] = deleted_in
 
 	return {
-		"directory": basedir,
+		"directory": backup_dir,
-		"encpwfile": os.path.join(env['STORAGE_ROOT'], 'backup/secret_key.txt'),
+		"encpwfile": os.path.join(backup_root, 'secret_key.txt'),
-		"encdirectory": encdir,
 		"tz": now.tzname(),
 		"backups": backups,
 	}
@@ -137,10 +129,35 @@ def perform_backup(full_backup):
 
 	exclusive_process("backup")
 
-	# Ensure the backup directory exists.
+	backup_root = os.path.join(env["STORAGE_ROOT"], 'backup')
-	backup_dir = os.path.join(env["STORAGE_ROOT"], 'backup')
+	backup_cache_dir = os.path.join(backup_root, 'cache')
-	backup_duplicity_dir = os.path.join(backup_dir, 'duplicity')
+	backup_dir = os.path.join(backup_root, 'encrypted')
-	os.makedirs(backup_duplicity_dir, exist_ok=True)
+
+	# In an older version of this script, duplicity was called
+	# such that it did not encrypt the backups it created (in
+	# backup/duplicity), and instead openssl was called separately
+	# after each backup run, creating AES256 encrypted copies of
+	# each file created by duplicity in backup/encrypted.
+	#
+	# We detect the transition by the presence of backup/duplicity
+	# and handle it by 'dupliception': we move all the old *un*encrypted
+	# duplicity files up out of the backup/duplicity directory (as
+	# backup/ is excluded from duplicity runs) in order that it is
+	# included in the next run, and we delete backup/encrypted (which
+	# duplicity will output files directly to, post-transition).
+	old_backup_dir = os.path.join(backup_root, 'duplicity')
+	migrated_unencrypted_backup_dir = os.path.join(env["STORAGE_ROOT"], "migrated_unencrypted_backup")
+	if os.path.isdir(old_backup_dir):
+		# Move the old unencrpyted files to a new location outside of
+		# the backup root so they get included in the next (new) backup.
+		# Then we'll delete them. Also so that they do not get in the
+		# way of duplicity doing a full backup on the first run after
+		# we take care of this.
+		shutil.move(old_backup_dir, migrated_unencrypted_backup_dir)
+
+		# The backup_dir (backup/encrypted) now has a new purpose.
+		# Clear it out.
+		shutil.rmtree(backup_dir)
 
 	# On the first run, always do a full backup. Incremental
 	# will fail. Otherwise do a full backup when the size of
@@ -152,81 +169,111 @@ def perform_backup(full_backup):
 	shell('check_call', ["/usr/sbin/service", "dovecot", "stop"])
 	shell('check_call', ["/usr/sbin/service", "postfix", "stop"])
 
+	# Get the encryption passphrase. secret_key.txt is 2048 random
+	# bits base64-encoded and with line breaks every 65 characters.
+	# gpg will only take the first line of text, so sanity check that
+	# that line is long enough to be a reasonable passphrase. It
+	# only needs to be 43 base64-characters to match AES256's key
+	# length of 32 bytes.
+	with open(os.path.join(backup_root, 'secret_key.txt')) as f:
+		passphrase = f.readline().strip()
+	if len(passphrase) < 43: raise Exception("secret_key.txt's first line is too short!")
+	env_with_passphrase = { "PASSPHRASE" : passphrase }
+
 	# Update the backup mirror directory which mirrors the current
 	# STORAGE_ROOT (but excluding the backups themselves!).
 	try:
 		shell('check_call', [
 			"/usr/bin/duplicity",
 			"full" if full_backup else "incr",
-			"--no-encryption",
+			"--archive-dir", backup_cache_dir,
-			"--archive-dir", "/tmp/duplicity-archive-dir",
+			"--exclude", backup_root,
-			"--name", "mailinabox",
+			"--volsize", "250",
-			"--exclude", backup_dir,
+			"--gpg-options", "--cipher-algo=AES256",
-			"--volsize", "100",
-			"--verbosity", "warning",
 			env["STORAGE_ROOT"],
-			"file://" + backup_duplicity_dir
+			"file://" + backup_dir
-			])
+			],
+			env_with_passphrase)
 	finally:
 		# Start services again.
 		shell('check_call', ["/usr/sbin/service", "dovecot", "start"])
 		shell('check_call', ["/usr/sbin/service", "postfix", "start"])
 
+	# Once the migrated backup is included in a new backup, it can be deleted.
+	if os.path.isdir(migrated_unencrypted_backup_dir):
+		shutil.rmtree(migrated_unencrypted_backup_dir)
+
 	# Remove old backups. This deletes all backup data no longer needed
-	# from more than 31 days ago. Must do this before destroying the
+	# from more than 3 days ago.
-	# cache directory or else this command will re-create it.
 	shell('check_call', [
 		"/usr/bin/duplicity",
 		"remove-older-than",
 		"%dD" % keep_backups_for_days,
-		"--archive-dir", "/tmp/duplicity-archive-dir",
+		"--archive-dir", backup_cache_dir,
-		"--name", "mailinabox",
 		"--force",
-		"--verbosity", "warning",
+		"file://" + backup_dir
-		"file://" + backup_duplicity_dir
+		],
-		])
+		env_with_passphrase)
 
-	# Remove duplicity's cache directory because it's redundant with our backup directory.
+	# From duplicity's manual:
-	shutil.rmtree("/tmp/duplicity-archive-dir")
+	# "This should only be necessary after a duplicity session fails or is
+	# aborted prematurely."
+	# That may be unlikely here but we may as well ensure we tidy up if
+	# that does happen - it might just have been a poorly timed reboot.
+	shell('check_call', [
+		"/usr/bin/duplicity",
+		"cleanup",
+		"--archive-dir", backup_cache_dir,
+		"--force",
+		"file://" + backup_dir
+		],
+		env_with_passphrase)
 
-	# Encrypt all of the new files.
+	# Change ownership of backups to the user-data user, so that the after-bcakup
-	backup_encrypted_dir = os.path.join(backup_dir, 'encrypted')
+	# script can access them.
-	os.makedirs(backup_encrypted_dir, exist_ok=True)
+	shell('check_call', ["/bin/chown", "-R", env["STORAGE_USER"], backup_dir])
-	for fn in os.listdir(backup_duplicity_dir):
-		fn2 = os.path.join(backup_encrypted_dir, fn) + ".enc"
-		if os.path.exists(fn2): continue
-
-		# Encrypt the backup using the backup private key.
-		shell('check_call', [
-			"/usr/bin/openssl",
-			"enc",
-			"-aes-256-cbc",
-			"-a",
-			"-salt",
-			"-in", os.path.join(backup_duplicity_dir, fn),
-			"-out", fn2,
-			"-pass", "file:%s" % os.path.join(backup_dir, "secret_key.txt"),
-			])
-
-		# The backup can be decrypted with:
-		# openssl enc -d -aes-256-cbc -a -in latest.tgz.enc -out /dev/stdout -pass file:secret_key.txt | tar -z
-
-	# Remove encrypted backups that are no longer needed.
-	for fn in os.listdir(backup_encrypted_dir):
-		fn2 = os.path.join(backup_duplicity_dir, fn.replace(".enc", ""))
-		if os.path.exists(fn2): continue
-		os.unlink(os.path.join(backup_encrypted_dir, fn))
 
 	# Execute a post-backup script that does the copying to a remote server.
 	# Run as the STORAGE_USER user, not as root. Pass our settings in
 	# environment variables so the script has access to STORAGE_ROOT.
-	post_script = os.path.join(backup_dir, 'after-backup')
+	post_script = os.path.join(backup_root, 'after-backup')
 	if os.path.exists(post_script):
 		shell('check_call',
 			['su', env['STORAGE_USER'], '-c', post_script],
 			env=env)
 
+	# Our nightly cron job executes system status checks immediately after this
+	# backup. Since it checks that dovecot and postfix are running, block for a
+	# bit (maximum of 10 seconds each) to give each a chance to finish restarting
+	# before the status checks might catch them down. See #381.
+	wait_for_service(25, True, env, 10)
+	wait_for_service(993, True, env, 10)
+
+def run_duplicity_verification():
+	env = load_environment()
+	backup_root = os.path.join(env["STORAGE_ROOT"], 'backup')
+	backup_cache_dir = os.path.join(backup_root, 'cache')
+	backup_dir = os.path.join(backup_root, 'encrypted')
+	env_with_passphrase = { "PASSPHRASE" : open(os.path.join(backup_root, 'secret_key.txt')).read() }
+	shell('check_call', [
+		"/usr/bin/duplicity",
+		"--verbosity", "info",
+		"verify",
+		"--compare-data",
+		"--archive-dir", backup_cache_dir,
+		"--exclude", backup_root,
+		"file://" + backup_dir,
+		env["STORAGE_ROOT"],
+	], env_with_passphrase)
+
 if __name__ == "__main__":
 	import sys
-	full_backup = "--full" in sys.argv
+	if sys.argv[-1] == "--verify":
-	perform_backup(full_backup)
+		# Run duplicity's verification command to check a) the backup files
+		# are readable, and b) report if they are up to date.
+		run_duplicity_verification()
+	else:
+		# Perform a backup. Add --full to force a full backup rather than
+		# possibly performing an incremental backup.
+		full_backup = "--full" in sys.argv
+		perform_backup(full_backup)

management/daemon.py

@@ -80,7 +80,7 @@ def unauthorized(error):
 	return auth_service.make_unauthorized_response()
 
 def json_response(data):
-	return Response(json.dumps(data), status=200, mimetype='application/json')
+	return Response(json.dumps(data, indent=2, sort_keys=True)+'\n', status=200, mimetype='application/json')
 
 ###################################
 
@@ -90,10 +90,12 @@ def json_response(data):
 def index():
 	# Render the control panel. This route does not require user authentication
 	# so it must be safe!
+	no_users_exist = (len(get_mail_users(env)) == 0)
 	no_admins_exist = (len(get_admins(env)) == 0)
 	return render_template('index.html',
 		hostname=env['PRIMARY_HOSTNAME'],
 		storage_root=env['STORAGE_ROOT'],
+		no_users_exist=no_users_exist,
 		no_admins_exist=no_admins_exist,
 	)
 
@@ -219,8 +221,8 @@ def dns_update():
 @app.route('/dns/secondary-nameserver')
 @authorized_personnel_only
 def dns_get_secondary_nameserver():
-	from dns_update import get_custom_dns_config
+	from dns_update import get_custom_dns_config, get_secondary_dns
-	return json_response({ "hostname": get_custom_dns_config(env).get("_secondary_nameserver") })
+	return json_response({ "hostname": get_secondary_dns(get_custom_dns_config(env)) })
 
 @app.route('/dns/secondary-nameserver', methods=['POST'])
 @authorized_personnel_only
@@ -231,38 +233,70 @@ def dns_set_secondary_nameserver():
 	except ValueError as e:
 		return (str(e), 400)
 
-@app.route('/dns/set')
+@app.route('/dns/custom')
 @authorized_personnel_only
-def dns_get_records():
+def dns_get_records(qname=None, rtype=None):
-	from dns_update import get_custom_dns_config, get_custom_records
+	from dns_update import get_custom_dns_config
-	additional_records = get_custom_dns_config(env)
+	return json_response([
-	records = get_custom_records(None, additional_records, env)
+	{
-	return json_response([{
 		"qname": r[0],
 		"rtype": r[1],
 		"value": r[2],
-		} for r in records])
+	}
+	for r in get_custom_dns_config(env)
+	if r[0] != "_secondary_nameserver"
+		and (not qname or r[0] == qname)
+		and (not rtype or r[1] == rtype) ])
 
-@app.route('/dns/set/<qname>', methods=['POST'])
+@app.route('/dns/custom/<qname>', methods=['GET', 'POST', 'PUT', 'DELETE'])
-@app.route('/dns/set/<qname>/<rtype>', methods=['POST'])
+@app.route('/dns/custom/<qname>/<rtype>', methods=['GET', 'POST', 'PUT', 'DELETE'])
-@app.route('/dns/set/<qname>/<rtype>/<value>', methods=['POST'])
 @authorized_personnel_only
-def dns_set_record(qname, rtype="A", value=None):
+def dns_set_record(qname, rtype="A"):
 	from dns_update import do_dns_update, set_custom_dns_record
 	try:
-		# Get the value from the URL, then the POST parameters, or if it is not set then
+		# Normalize.
-		# use the remote IP address of the request --- makes dynamic DNS easy. To clear a
+		rtype = rtype.upper()
-		# value, '' must be explicitly passed.
+
-		if value is None:
+		# Read the record value from the request BODY, which must be
-			value = request.form.get("value")
+		# ASCII-only. Not used with GET.
-		if value is None:
+		value = request.stream.read().decode("ascii", "ignore").strip()
-			value = request.environ.get("HTTP_X_FORWARDED_FOR") # normally REMOTE_ADDR but we're behind nginx as a reverse proxy
+
-		if value == '' or value == '__delete__':
+		if request.method == "GET":
-			# request deletion
+			# Get the existing records matching the qname and rtype.
-			value = None
+			return dns_get_records(qname, rtype)
-		if set_custom_dns_record(qname, rtype, value, env):
+
-			return do_dns_update(env)
+		elif request.method in ("POST", "PUT"):
+			# There is a default value for A/AAAA records.
+			if rtype in ("A", "AAAA") and value == "":
+				value = request.environ.get("HTTP_X_FORWARDED_FOR") # normally REMOTE_ADDR but we're behind nginx as a reverse proxy
+
+			# Cannot add empty records.
+			if value == '':
+				return ("No value for the record provided.", 400)
+
+			if request.method == "POST":
+				# Add a new record (in addition to any existing records
+				# for this qname-rtype pair).
+				action = "add"
+			elif request.method == "PUT":
+				# In REST, PUT is supposed to be idempotent, so we'll
+				# make this action set (replace all records for this
+				# qname-rtype pair) rather than add (add a new record).
+				action = "set"
+		
+		elif request.method == "DELETE":
+			if value == '':
+				# Delete all records for this qname-type pair.
+				value = None
+			else:
+				# Delete just the qname-rtype-value record exactly.
+				pass
+			action = "remove"
+
+		if set_custom_dns_record(qname, rtype, value, action, env):
+			return do_dns_update(env) or "Something isn't right."
 		return "OK"
+
 	except ValueError as e:
 		return (str(e), 400)
 

management/dns_update.py

@@ -4,7 +4,7 @@
 # and mail aliases and restarts nsd.
 ########################################################################
 
-import os, os.path, urllib.parse, datetime, re, hashlib, base64
+import sys, os, os.path, urllib.parse, datetime, re, hashlib, base64
 import ipaddress
 import rtyaml
 import dns.resolver
@@ -50,24 +50,13 @@ def get_dns_zones(env):
 
 	return zonefiles
 	
-def get_custom_dns_config(env):
-	try:
-		return rtyaml.load(open(os.path.join(env['STORAGE_ROOT'], 'dns/custom.yaml')))
-	except:
-		return { }
-
-def write_custom_dns_config(config, env):
-	config_yaml = rtyaml.dump(config)
-	with open(os.path.join(env['STORAGE_ROOT'], 'dns/custom.yaml'), "w") as f:
-		f.write(config_yaml)
-
 def do_dns_update(env, force=False):
 	# What domains (and their zone filenames) should we build?
 	domains = get_dns_domains(env)
 	zonefiles = get_dns_zones(env)
 
 	# Custom records to add to zones.
-	additional_records = get_custom_dns_config(env)
+	additional_records = list(get_custom_dns_config(env))
 
 	# Write zone files.
 	os.makedirs('/etc/nsd/zones', exist_ok=True)
@@ -153,7 +142,7 @@ def build_zone(domain, all_domains, additional_records, env, is_zone=True):
 		records.append((None,  "NS",  "ns1.%s." % env["PRIMARY_HOSTNAME"], False))
 
 		# Define ns2.PRIMARY_HOSTNAME or whatever the user overrides.
-		secondary_ns = additional_records.get("_secondary_nameserver", "ns2." + env["PRIMARY_HOSTNAME"])
+		secondary_ns = get_secondary_dns(additional_records) or ("ns2." + env["PRIMARY_HOSTNAME"])
 		records.append((None,  "NS", secondary_ns+'.', False))
 
 
@@ -196,20 +185,34 @@ def build_zone(domain, all_domains, additional_records, env, is_zone=True):
 				child_qname += "." + subdomain_qname
 			records.append((child_qname, child_rtype, child_value, child_explanation))
 
+	has_rec_base = list(records) # clone current state
 	def has_rec(qname, rtype, prefix=None):
-		for rec in records:
+		for rec in has_rec_base:
 			if rec[0] == qname and rec[1] == rtype and (prefix is None or rec[2].startswith(prefix)):
 				return True
 		return False
 
 	# The user may set other records that don't conflict with our settings.
 	# Don't put any TXT records above this line, or it'll prevent any custom TXT records.
-	for qname, rtype, value in get_custom_records(domain, additional_records, env):
+	for qname, rtype, value in filter_custom_records(domain, additional_records):
+		# Don't allow custom records for record types that override anything above.
+		# But allow multiple custom records for the same rtype --- see how has_rec_base is used.
 		if has_rec(qname, rtype): continue
+
+		# The "local" keyword on A/AAAA records are short-hand for our own IP.
+		# This also flags for web configuration that the user wants a website here.
+		if rtype == "A" and value == "local":
+			value = env["PUBLIC_IP"]
+		if rtype == "AAAA" and value == "local":
+			if "PUBLIC_IPV6" in env:
+				value = env["PUBLIC_IPV6"]
+			else:
+				continue
 		records.append((qname, rtype, value, "(Set by user.)"))
 
 	# Add defaults if not overridden by the user's custom settings (and not otherwise configured).
 	# Any "CNAME" record on the qname overrides A and AAAA.
+	has_rec_base = records
 	defaults = [
 		(None,  "A",    env["PUBLIC_IP"],       "Required. May have a different value. Sets the IP address that %s resolves to for web hosting and other services besides mail. The A record must be present but its value does not affect mail delivery." % domain),
 		("www", "A",    env["PUBLIC_IP"],       "Optional. Sets the IP address that www.%s resolves to, e.g. for web hosting." % domain),
@@ -263,52 +266,6 @@ def build_zone(domain, all_domains, additional_records, env, is_zone=True):
 
 ########################################################################
 
-def get_custom_records(domain, additional_records, env):
-	for qname, value in additional_records.items():
-		# We don't count the secondary nameserver config (if present) as a record - that would just be
-		# confusing to users. Instead it is accessed/manipulated directly via (get/set)_custom_dns_config.
-		if qname == "_secondary_nameserver": continue
-
-		# Is this record for the domain or one of its subdomains?
-		# If `domain` is None, return records for all domains.
-		if domain is not None and qname != domain and not qname.endswith("." + domain): continue
-
-		# Turn the fully qualified domain name in the YAML file into
-		# our short form (None => domain, or a relative QNAME) if
-		# domain is not None.
-		if domain is not None:
-			if qname == domain:
-				qname = None
-			else:
-				qname = qname[0:len(qname)-len("." + domain)]
-
-		# Short form. Mapping a domain name to a string is short-hand
-		# for creating A records.
-		if isinstance(value, str):
-			values = [("A", value)]
-			if value == "local" and env.get("PUBLIC_IPV6"):
-				values.append( ("AAAA", value) )
-
-		# A mapping creates multiple records.
-		elif isinstance(value, dict):
-			values = value.items()
-
-		# No other type of data is allowed.
-		else:
-			raise ValueError()
-
-		for rtype, value2 in values:
-			# The "local" keyword on A/AAAA records are short-hand for our own IP.
-			# This also flags for web configuration that the user wants a website here.
-			if rtype == "A" and value2 == "local":
-				value2 = env["PUBLIC_IP"]
-			if rtype == "AAAA" and value2 == "local":
-				if "PUBLIC_IPV6" not in env: continue # no IPv6 address is available so don't set anything
-				value2 = env["PUBLIC_IPV6"]
-			yield (qname, rtype, value2)
-
-########################################################################
-
 def build_tlsa_record(env):
 	# A DANE TLSA record in DNS specifies that connections on a port
 	# must use TLS and the certificate must match a particular certificate.
@@ -475,26 +432,10 @@ $TTL 1800           ; default time to live
 ########################################################################
 
 def write_nsd_conf(zonefiles, additional_records, env):
-	# Basic header.
+	# Write the list of zones to a configuration file.
-	nsdconf = """
+	nsd_conf_file = "/etc/nsd/zones.conf"
-server:
+	nsdconf = ""
-  hide-version: yes
-
-  # identify the server (CH TXT ID.SERVER entry).
-  identity: ""
-
-  # The directory for zonefile: files.
-  zonesdir: "/etc/nsd/zones"
-"""
 	
-	# Since we have bind9 listening on localhost for locally-generated
-	# DNS queries that require a recursive nameserver, and the system
-	# might have other network interfaces for e.g. tunnelling, we have
-	# to be specific about the network interfaces that nsd binds to.
-	for ipaddr in (env.get("PRIVATE_IP", "") + " " + env.get("PRIVATE_IPV6", "")).split(" "):
-		if ipaddr == "": continue
-		nsdconf += "  ip-address: %s\n" % ipaddr
-
 	# Append the zones.
 	for domain, zonefile in zonefiles:
 		nsdconf += """
@@ -505,9 +446,9 @@ zone:
 
 		# If a custom secondary nameserver has been set, allow zone transfers
 		# and notifies to that nameserver.
-		if additional_records.get("_secondary_nameserver"):
+		if get_secondary_dns(additional_records):
 			# Get the IP address of the nameserver by resolving it.
-			hostname = additional_records.get("_secondary_nameserver")
+			hostname = get_secondary_dns(additional_records)
 			resolver = dns.resolver.get_default_resolver()
 			response = dns.resolver.query(hostname+'.', "A")
 			ipaddr = str(response[0])
@@ -515,16 +456,17 @@ zone:
 	provide-xfr: %s NOKEY
 """ % (ipaddr, ipaddr)
 
-
+	# Check if the file is changing. If it isn't changing,
-	# Check if the nsd.conf is changing. If it isn't changing,
 	# return False to flag that no change was made.
-	with open("/etc/nsd/nsd.conf") as f:
+	if os.path.exists(nsd_conf_file):
-		if f.read() == nsdconf:
+		with open(nsd_conf_file) as f:
-			return False
+			if f.read() == nsdconf:
+				return False
 
-	with open("/etc/nsd/nsd.conf", "w") as f:
+	# Write out new contents and return True to signal that
+	# configuration changed.
+	with open(nsd_conf_file, "w") as f:
 		f.write(nsdconf)
-
 	return True
 
 ########################################################################
@@ -668,7 +610,94 @@ def write_opendkim_tables(domains, env):
 
 ########################################################################
 
-def set_custom_dns_record(qname, rtype, value, env):
+def get_custom_dns_config(env):
+	try:
+		custom_dns = rtyaml.load(open(os.path.join(env['STORAGE_ROOT'], 'dns/custom.yaml')))
+		if not isinstance(custom_dns, dict): raise ValueError() # caught below
+	except:
+		return [ ]
+
+	for qname, value in custom_dns.items():
+		# Short form. Mapping a domain name to a string is short-hand
+		# for creating A records.
+		if isinstance(value, str):
+			values = [("A", value)]
+
+		# A mapping creates multiple records.
+		elif isinstance(value, dict):
+			values = value.items()
+
+		# No other type of data is allowed.
+		else:
+			raise ValueError()
+
+		for rtype, value2 in values:
+			if isinstance(value2, str):
+				yield (qname, rtype, value2)
+			elif isinstance(value2, list):
+				for value3 in value2:
+					yield (qname, rtype, value3)
+			# No other type of data is allowed.
+			else:
+				raise ValueError()
+
+def filter_custom_records(domain, custom_dns_iter):
+	for qname, rtype, value in custom_dns_iter:
+		# We don't count the secondary nameserver config (if present) as a record - that would just be
+		# confusing to users. Instead it is accessed/manipulated directly via (get/set)_custom_dns_config.
+		if qname == "_secondary_nameserver": continue
+
+		# Is this record for the domain or one of its subdomains?
+		# If `domain` is None, return records for all domains.
+		if domain is not None and qname != domain and not qname.endswith("." + domain): continue
+
+		# Turn the fully qualified domain name in the YAML file into
+		# our short form (None => domain, or a relative QNAME) if
+		# domain is not None.
+		if domain is not None:
+			if qname == domain:
+				qname = None
+			else:
+				qname = qname[0:len(qname)-len("." + domain)]
+
+		yield (qname, rtype, value)
+
+def write_custom_dns_config(config, env):
+	# We get a list of (qname, rtype, value) triples. Convert this into a
+	# nice dictionary format for storage on disk.
+	from collections import OrderedDict
+	config = list(config)
+	dns = OrderedDict()
+	seen_qnames = set()
+
+	# Process the qnames in the order we see them.
+	for qname in [rec[0] for rec in config]:
+		if qname in seen_qnames: continue
+		seen_qnames.add(qname)
+
+		records = [(rec[1], rec[2]) for rec in config if rec[0] == qname]
+		if len(records) == 1 and records[0][0] == "A":
+			dns[qname] = records[0][1]
+		else:
+			dns[qname] = OrderedDict()
+			seen_rtypes = set()
+
+			# Process the rtypes in the order we see them.
+			for rtype in [rec[0] for rec in records]:
+				if rtype in seen_rtypes: continue
+				seen_rtypes.add(rtype)
+
+				values = [rec[1] for rec in records if rec[0] == rtype]
+				if len(values) == 1:
+					values = values[0]
+				dns[qname][rtype] = values
+	
+	# Write.	
+	config_yaml = rtyaml.dump(dns)
+	with open(os.path.join(env['STORAGE_ROOT'], 'dns/custom.yaml'), "w") as f:
+		f.write(config_yaml)
+
+def set_custom_dns_record(qname, rtype, value, action, env):
 	# validate qname
 	for zone, fn in get_dns_zones(env):
 		# It must match a zone apex or be a subdomain of a zone
@@ -677,15 +706,17 @@ def set_custom_dns_record(qname, rtype, value, env):
 			break
 	else:
 		# No match.
-		raise ValueError("%s is not a domain name or a subdomain of a domain name managed by this box." % qname)
+		if qname != "_secondary_nameserver":
+			raise ValueError("%s is not a domain name or a subdomain of a domain name managed by this box." % qname)
 
 	# validate rtype
 	rtype = rtype.upper()
-	if value is not None:
+	if value is not None and qname != "_secondary_nameserver":
 		if rtype in ("A", "AAAA"):
-			v = ipaddress.ip_address(value)
+			if value != "local": # "local" is a special flag for us
-			if rtype == "A" and not isinstance(v, ipaddress.IPv4Address): raise ValueError("That's an IPv6 address.")
+				v = ipaddress.ip_address(value) # raises a ValueError if there's a problem
-			if rtype == "AAAA" and not isinstance(v, ipaddress.IPv6Address): raise ValueError("That's an IPv4 address.")
+				if rtype == "A" and not isinstance(v, ipaddress.IPv4Address): raise ValueError("That's an IPv6 address.")
+				if rtype == "AAAA" and not isinstance(v, ipaddress.IPv6Address): raise ValueError("That's an IPv4 address.")
 		elif rtype in ("CNAME", "TXT", "SRV", "MX"):
 			# anything goes
 			pass
@@ -693,69 +724,65 @@ def set_custom_dns_record(qname, rtype, value, env):
 			raise ValueError("Unknown record type '%s'." % rtype)
 
 	# load existing config
-	config = get_custom_dns_config(env)
+	config = list(get_custom_dns_config(env))
 
 	# update
-	if qname not in config:
+	newconfig = []
-		if value is None:
+	made_change = False
-			# Is asking to delete a record that does not exist.
+	needs_add = True
-			return False
+	for _qname, _rtype, _value in config:
-		elif rtype == "A":
+		if action == "add":
-			# Add this record using the short form 'qname: value'.
+			if (_qname, _rtype, _value) == (qname, rtype, value):
-			config[qname] = value
+				# Record already exists. Bail.
-		else:
-			# Add this record. This is the qname's first record.
-			config[qname] = { rtype: value }
-	else:
-		if isinstance(config[qname], str):
-			# This is a short-form 'qname: value' implicit-A record.
-			if value is None and rtype != "A":
-				# Is asking to delete a record that doesn't exist.
 				return False
-			elif value is None and rtype == "A":
+		elif action == "set":
-				# Delete record.
+			if (_qname, _rtype) == (qname, rtype):
-				del config[qname]
+				if _value == value:
-			elif rtype == "A":
+					# Flag that the record already exists, don't
-				# Update, keeping short form.
+					# need to add it.
-				if config[qname] == "value":
+					needs_add = False
-					# No change.
-					return False
-				config[qname] = value
-			else:
-				# Expand short form so we can add a new record type.
-				config[qname] = { "A": config[qname], rtype: value }
-		else:
-			# This is the qname: { ... } (dict) format.
-			if value is None:
-				if rtype not in config[qname]:
-					# Is asking to delete a record that doesn't exist.
-					return False
 				else:
-					# Delete the record. If it's the last record, delete the domain.
+					# Drop any other values for this (qname, rtype).
-					del config[qname][rtype]
+					made_change = True
-					if len(config[qname]) == 0:
+					continue
-						del config[qname]
+		elif action == "remove":
-			else:
+			if (_qname, _rtype, _value) == (qname, rtype, value):
-				# Update the record.
+				# Drop this record.
-				if config[qname].get(rtype) == "value":
+				made_change = True
-					# No change.
+				continue
-					return False
+			if value == None and (_qname, _rtype) == (qname, rtype):
-				config[qname][rtype] = value
+				# Drop all qname-rtype records.
+				made_change = True
+				continue
+		else:
+			raise ValueError("Invalid action: " + action)
 
-	# serialize & save
+		# Preserve this record.
-	write_custom_dns_config(config, env)
+		newconfig.append((_qname, _rtype, _value))
 
-	return True
+	if action in ("add", "set") and needs_add and value is not None:
+		newconfig.append((qname, rtype, value))
+		made_change = True
+
+	if made_change:
+		# serialize & save
+		write_custom_dns_config(newconfig, env)
+
+	return made_change
 
 ########################################################################
 
+def get_secondary_dns(custom_dns):
+	for qname, rtype, value in custom_dns:
+		if qname == "_secondary_nameserver":
+			return value
+	return None
+
 def set_secondary_dns(hostname, env):
-	config = get_custom_dns_config(env)
 
 	if hostname in (None, ""):
 		# Clear.
-		if "_secondary_nameserver" in config:
+		set_custom_dns_record("_secondary_nameserver", "A", None, "set", env)
-			del config["_secondary_nameserver"]
 	else:
 		# Validate.
 		hostname = hostname.strip().lower()
@@ -766,10 +793,9 @@ def set_secondary_dns(hostname, env):
 			raise ValueError("Could not resolve the IP address of %s." % hostname)
 
 		# Set.
-		config["_secondary_nameserver"] = hostname
+		set_custom_dns_record("_secondary_nameserver", "A", hostname, "set", env)
 
-	# Save and apply.
+	# Apply.
-	write_custom_dns_config(config, env)
 	return do_dns_update(env)
 
 
@@ -820,7 +846,7 @@ def build_recommended_dns(env):
 	ret = []
 	domains = get_dns_domains(env)
 	zonefiles = get_dns_zones(env)
-	additional_records = get_custom_dns_config(env)
+	additional_records = list(get_custom_dns_config(env))
 	for domain, zonefile in zonefiles:
 		records = build_zone(domain, domains, additional_records, env)
 
@@ -851,8 +877,11 @@ def build_recommended_dns(env):
 if __name__ == "__main__":
 	from utils import load_environment
 	env = load_environment()
-	for zone, records in build_recommended_dns(env):
+	if sys.argv[-1] == "--lint":
-		for record in records:
+		write_custom_dns_config(get_custom_dns_config(env), env)
-			print("; " + record['explanation'])
+	else:
-			print(record['qname'], record['rtype'], record['value'], sep="\t")
+		for zone, records in build_recommended_dns(env):
-			print()
+			for record in records:
+				print("; " + record['explanation'])
+				print(record['qname'], record['rtype'], record['value'], sep="\t")
+				print()

management/mailconfig.py

@@ -2,6 +2,7 @@
 
 import subprocess, shutil, os, sqlite3, re
 import utils
+from email_validator import validate_email as validate_email_, EmailNotValidError
 
 def validate_email(email, mode=None):
 	# Checks that an email address is syntactically valid. Returns True/False.
@@ -15,50 +16,25 @@ def validate_email(email, mode=None):
 	# When mode=="alias", we're allowing anything that can be in a Postfix
 	# alias table, i.e. omitting the local part ("@domain.tld") is OK.
 
-	# Check that the address isn't absurdly long.
+	# Check the syntax of the address.
-	if len(email) > 255: return False
+	try:
+		validate_email_(email,
+			allow_smtputf8=False,
+			check_deliverability=False,
+			allow_empty_local=(mode=="alias")
+			)
+	except EmailNotValidError:
+		return False
 
 	if mode == 'user':
 		# There are a lot of characters permitted in email addresses, but
-		# Dovecot's sqlite driver seems to get confused if there are any
+		# Dovecot's sqlite auth driver seems to get confused if there are any
 		# unusual characters in the address. Bah. Also note that since
 		# the mailbox path name is based on the email address, the address
 		# shouldn't be absurdly long and must not have a forward slash.
-		ATEXT = r'[a-zA-Z0-9_\-]'
+		if len(email) > 255: return False
-	elif mode in (None, 'alias'):
+		if re.search(r'[^\@\.a-zA-Z0-9_\-]+', email):
-		# For aliases, we can allow any valid email address.
+			return False
-		# Based on RFC 2822 and https://github.com/SyrusAkbary/validate_email/blob/master/validate_email.py,
-		# these characters are permitted in email addresses.
-		ATEXT = r'[a-zA-Z0-9_!#$%&\'\*\+\-/=\?\^`\{\|\}~]' # see 3.2.4
-	else:
-		raise ValueError(mode)
-
-	# per RFC 2822 3.2.4
-	DOT_ATOM_TEXT_LOCAL = ATEXT + r'+(?:\.' + ATEXT + r'+)*'
-	if mode == 'alias':
-		# For aliases, Postfix accepts '@domain.tld' format for
-		# catch-all addresses on the source side and domain aliases
-		# on the destination side. Make the local part optional.
-		DOT_ATOM_TEXT_LOCAL = '(?:' + DOT_ATOM_TEXT_LOCAL + ')?'
-
-	# We can require that the host part have at least one period in it,
-	# so use a "+" rather than a "*" at the end.
-	DOT_ATOM_TEXT_HOST = ATEXT + r'+(?:\.' + ATEXT + r'+)+'
-
-	# per RFC 2822 3.4.1
-	ADDR_SPEC = '^(%s)@(%s)$' % (DOT_ATOM_TEXT_LOCAL, DOT_ATOM_TEXT_HOST)
-
-	# Check the regular expression.
-	m = re.match(ADDR_SPEC, email)
-	if not m: return False
-
-	# Check that the domain part is valid IDNA.
-	localpart, domainpart = m.groups()
-	try:
-		domainpart.encode('ascii').decode("idna")
-	except:
-		# Domain is not valid IDNA.
-		return False
 
 	# Everything looks good.
 	return True
@@ -278,9 +254,10 @@ def add_mail_user(email, pw, privs, env):
 		return ("Invalid email address.", 400)
 	elif not validate_email(email, mode='user'):
 		return ("User account email addresses may only use the ASCII letters A-Z, the digits 0-9, underscore (_), hyphen (-), and period (.).", 400)
-	elif is_dcv_address(email):
+	elif is_dcv_address(email) and len(get_mail_users(env)) > 0:
 		# Make domain control validation hijacking a little harder to mess up by preventing the usual
-		# addresses used for DCV from being user accounts.
+		# addresses used for DCV from being user accounts. Except let it be the first account because
+		# during box setup the user won't know the rules.
 		return ("You may not make a user account for that address because it is frequently used for domain control validation. Use an alias instead if necessary.", 400)
 
 	# validate password

management/status_checks.py

@@ -11,7 +11,7 @@ import sys, os, os.path, re, subprocess, datetime, multiprocessing.pool
 import dns.reversename, dns.resolver
 import dateutil.parser, dateutil.tz
 
-from dns_update import get_dns_zones, build_tlsa_record, get_custom_dns_config
+from dns_update import get_dns_zones, build_tlsa_record, get_custom_dns_config, get_secondary_dns
 from web_update import get_web_domains, get_domain_ssl_files
 from mailconfig import get_mail_domains, get_mail_aliases
 
@@ -357,11 +357,11 @@ def check_dns_zone(domain, env, output, dns_zonefiles):
 	# the TLD, and so we're not actually checking the TLD. For that we'd need
 	# to do a DNS trace.
 	ip = query_dns(domain, "A")
-	custom_dns = get_custom_dns_config(env)
+	secondary_ns = get_secondary_dns(get_custom_dns_config(env)) or "ns2." + env['PRIMARY_HOSTNAME']
 	existing_ns = query_dns(domain, "NS")
 	correct_ns = "; ".join(sorted([
 		"ns1." + env['PRIMARY_HOSTNAME'],
-		custom_dns.get("_secondary_nameserver", "ns2." + env['PRIMARY_HOSTNAME']),
+		secondary_ns,
 		]))
 	if existing_ns.lower() == correct_ns.lower():
 		output.print_ok("Nameservers are set correctly at registrar. [%s]" % correct_ns)

management/templates/custom-dns.html

@@ -93,44 +93,56 @@
 
 <p>Use your box&rsquo;s DNS API to set custom DNS records on domains hosted here. For instance, you can create your own dynamic DNS service.</p>
 
-<p>Send a POST request like this:</p>
+<p>Usage:</p>
 
-<pre>curl -d "" --user {email}:{password} https://{{hostname}}/admin/dns/set/<b>qname</b>[/<b>rtype</b>[/<b>value</b>]]</pre>
+<pre>curl -X <b>VERB</b> [-d "<b>value</b>"] --user {email}:{password} https://{{hostname}}/admin/dns/custom[/<b>qname</b>[/<b>rtype</b>]]</pre>
 
-<h4>HTTP POST parameters</h4>
+<p>(Brackets denote an optional argument.)</p>
+
+<h4>Verbs</h4>
+
+<table class="table">
+<thead><th>Verb</th> <th>Usage</th></thead>
+<tr><td>GET</td> <td>Returns matching custom DNS records as a JSON array of objects. Each object has the keys <code>qname</code>, <code>rtype</code>, and <code>value</code>. The optional <code>qname</code> and <code>rtype</code> parameters in the request URL filter the records returned in the response. The request body (<code>-d "..."</code>) must be omitted.</td></tr>
+<tr><td>PUT</td> <td>Sets a custom DNS record replacing any existing records with the same <code>qname</code> and <code>rtype</code>. Use PUT (instead of POST) when you only have one value for a <code>qname</code> and <code>rtype</code>, such as typical <code>A</code> records (without round-robin).</td></tr>
+<tr><td>POST</td> <td>Adds a new custom DNS record. Use POST when you have multiple <code>TXT</code> records or round-robin <code>A</code> records. (PUT would delete previously added records.)</td></tr>
+<tr><td>DELETE</td> <td>Deletes custom DNS records. If the request body (<code>-d "..."</code>) is empty or omitted, deletes all records matching the <code>qname</code> and <code>rtype</code>. If the request body is present, deletes only the record matching the <code>qname</code>, <code>rtype</code> and value.</td></tr>
+</table>
+
+<h4>Parameters</h4>
 
 <table class="table">
 <thead><th>Parameter</th> <th>Value</th></thead>
 <tr><td>email</td> <td>The email address of any administrative user here.</td></tr>
 <tr><td>password</td> <td>That user&rsquo;s password.</td></tr>
-<tr><td>qname</td> <td>The fully qualified domain name for the record you are trying to set.</td></tr>
+<tr><td>qname</td> <td>The fully qualified domain name for the record you are trying to set. It must be one of the domain names or a subdomain of one of the domain names hosted on this box. (Add mail users or aliases to add new domains.)</td></tr>
-<tr><td>rtype</td> <td>The resource type. <code>A</code> if omitted. Possible values: <code>A</code> (an IPv4 address), <code>AAAA</code> (an IPv6 address), <code>TXT</code> (a text string), or <code>CNAME</code> (an alias, which is a fully qualified domain name).</td></tr>
+<tr><td>rtype</td> <td>The resource type. Defaults to <code>A</code> if omitted. Possible values: <code>A</code> (an IPv4 address), <code>AAAA</code> (an IPv6 address), <code>TXT</code> (a text string), <code>CNAME</code> (an alias, which is a fully qualified domain name &mdash; don&rsquo;t forget the final period), <code>MX</code>, or <code>SRV</code>.</td></tr>
-<tr><td>value</td> <td>The new record&rsquo;s value. If omitted, the IPv4 address of the remote host is used. This is handy for dynamic DNS! To delete a record, use &ldquo;__delete__&rdquo;.</td></tr>
+<tr><td>value</td> <td>For PUT, POST, and DELETE, the record&rsquo;s value. If the <code>rtype</code> is <code>A</code> or <code>AAAA</code> and <code>value</code> is empty or omitted, the IPv4 or IPv6 address of the remote host is used (be sure to use the <code>-4</code> or <code>-6</code> options to curl). This is handy for dynamic DNS!</td></tr>
 </table>
 
-<p style="margin-top: 1em">Note that <code>-d ""</code> is merely to ensure curl sends a POST request. You do not need to put anything inside the quotes. You can also pass the value using typical form encoding in the POST body.</p>
-
 <p>Strict <a href="http://tools.ietf.org/html/rfc4408">SPF</a> and <a href="https://datatracker.ietf.org/doc/draft-kucherawy-dmarc-base/?include_text=1">DMARC</a> records will be added to all custom domains unless you override them.</p>
 
 <h4>Examples:</h4>
 
+<p>Try these examples. For simplicity the examples omit the <code>--user me@mydomain.com:yourpassword</code> command line argument which you must fill in with your email address and password.</p>
+
 <pre># sets laptop.mydomain.com to point to the IP address of the machine you are executing curl on
-curl -d "" --user me@mydomain.com:###### https://{{hostname}}/admin/dns/set/laptop.mydomain.com
+curl -X PUT https://{{hostname}}/admin/dns/custom/laptop.mydomain.com
 
-# sets an alias
+# deletes that record and all A records for that domain name
-curl -d "" --user me@mydomain.com:###### https://{{hostname}}/admin/dns/set/foo.mydomain.com/cname/bar.mydomain.com
+curl -X DELETE https://{{hostname}}/admin/dns/custom/laptop.mydomain.com
 
-# clears the alias
+# sets a CNAME alias
-curl -d "" --user me@mydomain.com:###### https://{{hostname}}/admin/dns/set/bar.mydomain.com/cname/__delete__
+curl -X PUT -d "bar.mydomain.com." https://{{hostname}}/admin/dns/custom/foo.mydomain.com/cname
 
-# sets a TXT record using the alternate value syntax
+# deletes that CNAME and all CNAME records for that domain name
-curl -d "value=something%20here" --user me@mydomain.com:###### https://{{hostname}}/admin/dns/set/foo.mydomain.com/txt
+curl -X DELETE https://{{hostname}}/admin/dns/custom/foo.mydomain.com/cname
 
-# sets a <a href="http://en.wikipedia.org/wiki/SRV_record">SRV record</a> for the "service" and "protocol" hosted on "target" server
+# adds a TXT record using POST to preserve any previous TXT records
-curl -d "" --user me@mydomain.com:###### https://{{hostname}}/admin/dns/set/_service._protocol.{{hostname}}/srv/"priority weight port target"
+curl -X POST -d "some text here" https://{{hostname}}/admin/dns/custom/foo.mydomain.com/txt
 
-# sets a SRV record using the value syntax
+# deletes that one TXT record while preserving other TXT records
-curl -d "value=priority weight port target" --user me@mydomain.com:###### https://{{hostname}}/admin/dns/set/_service._protocol.host/srv
+curl -X DELETE -d "some text here" https://{{hostname}}/admin/dns/custom/foo.mydomain.com/txt
 </pre>
 
 <script>
@@ -161,7 +173,7 @@ function show_custom_dns() {
 
 function show_current_custom_dns() {
   api(
-    "/dns/set",
+    "/dns/custom",
     "GET",
     { },
     function(data) {
@@ -176,6 +188,7 @@ function show_current_custom_dns() {
         $('#custom-dns-current').find("tbody").append(tr);
         tr.attr('data-qname', data[i].qname);
         tr.attr('data-rtype', data[i].rtype);
+        tr.attr('data-value', data[i].value);
         tr.append($('<td class="long"/>').text(data[i].qname));
         tr.append($('<td/>').text(data[i].rtype));
         tr.append($('<td class="long"/>').text(data[i].value));
@@ -187,7 +200,8 @@ function show_current_custom_dns() {
 function delete_custom_dns_record(elem) {
   var qname = $(elem).parents('tr').attr('data-qname');
   var rtype = $(elem).parents('tr').attr('data-rtype');
-  do_set_custom_dns(qname, rtype, "__delete__");
+  var value = $(elem).parents('tr').attr('data-value');
+  do_set_custom_dns(qname, rtype, value, "DELETE");
   return false;
 }
 
@@ -208,7 +222,7 @@ function do_set_secondary_dns() {
     });
 }
 
-function do_set_custom_dns(qname, rtype, value) {
+function do_set_custom_dns(qname, rtype, value, method) {
   if (!qname) {
     if ($('#customdnsQname').val() != '')
       qname = $('#customdnsQname').val() + '.' + $('#customdnsZone').val();
@@ -216,21 +230,20 @@ function do_set_custom_dns(qname, rtype, value) {
       qname = $('#customdnsZone').val();
     rtype = $('#customdnsType').val();
     value = $('#customdnsValue').val();
+    method = 'POST';
   }
 
   api(
-    "/dns/set/" + qname + "/" + rtype,
+    "/dns/custom/" + qname + "/" + rtype,
-    "POST",
+    method,
-    {
+    value,
-      value: value
-    },
     function(data) {
       if (data == "") return; // nothing updated
       show_modal_error("Custom DNS", $("<pre/>").text(data));
       show_current_custom_dns();
     },
     function(err) {
-      show_modal_error("Custom DNS", $("<pre/>").text(err));
+      show_modal_error("Custom DNS (Error)", $("<pre/>").text(err));
     });
 }
 

management/templates/index.html

@@ -1,18 +1,15 @@
 <!DOCTYPE html>
-<!--[if lt IE 7]>      <html class="no-js lt-ie9 lt-ie8 lt-ie7"> <![endif]-->
+<html lang="en">
-<!--[if IE 7]>         <html class="no-js lt-ie9 lt-ie8"> <![endif]-->
-<!--[if IE 8]>         <html class="no-js lt-ie9"> <![endif]-->
-<!--[if gt IE 8]><!--> <html class="no-js"> <!--<![endif]-->
     <head>
-        <meta charset="utf-8">
         <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
-        <meta name="viewport" content="width=device-width">
+        <meta charset="utf-8">
+        <meta name="viewport" content="width=device-width, initial-scale=1">
 
         <title>{{hostname}} - Mail-in-a-Box Control Panel</title>
 
         <meta name="robots" content="noindex, nofollow">
 
-        <link rel="stylesheet" href="//maxcdn.bootstrapcdn.com/bootstrap/3.2.0/css/bootstrap.min.css">
+        <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.0/css/bootstrap.min.css">
         <style>
             @import url(https://fonts.googleapis.com/css?family=Raleway:400,700);
             @import url(https://fonts.googleapis.com/css?family=Ubuntu:300);
@@ -73,18 +70,19 @@
 	       margin-bottom: 1em;
 	    }
         </style>
-        <link rel="stylesheet" href="//maxcdn.bootstrapcdn.com/bootstrap/3.2.0/css/bootstrap-theme.min.css">
+        <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.0/css/bootstrap-theme.min.css">
-        <style>
+        <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css">
-        </style>
     </head>
     <body>
-        <!--[if lt IE 7]>
+
-            <p class="chromeframe">You are using an <strong>outdated</strong> browser. Please <a href="http://browsehappy.com/">upgrade your browser</a> or <a href="http://www.google.com/chromeframe/?redirect=true">activate Google Chrome Frame</a> to improve your experience.</p>
+    <!--[if lt IE 8]><p>Internet Explorer version 8 or any modern web browser is required to use this website, sorry.<![endif]-->
-        <![endif]-->
+    <!--[if gt IE 7]><!-->
-    <div class="navbar navbar-inverse navbar-fixed-top">
+
+    <div class="navbar navbar-inverse navbar-fixed-top" role="navigation">
       <div class="container">
         <div class="navbar-header">
-          <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
+          <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target=".navbar-collapse">
+            <span class="sr-only">Toggle navigation</span>
             <span class="icon-bar"></span>
             <span class="icon-bar"></span>
             <span class="icon-bar"></span>
@@ -175,9 +173,9 @@
       </footer>
     </div> <!-- /container -->
 
-        <div id="ajax_loading_indicator" style="display: none; position: fixed; left: 0; top: 0; width: 100%; height: 100%; text-align: center; background-color: rgba(255,255,255,.75)">
+        <div id="ajax_loading_indicator" style="display: none; position: fixed; left: 0; top: 0; width: 100%; height: 100%; z-index: 100000; text-align: center; background-color: rgba(255,255,255,.75)">
           <div style="margin: 20% auto">
-            <div><span class="glyphicon glyphicon-time"></span></div>
+            <div><span class="fa fa-spinner fa-pulse"></span></div>
             <div>Loading...</div>
           </div>
         </div>
@@ -200,8 +198,8 @@
           </div>
         </div>
 
-        <script src="//ajax.googleapis.com/ajax/libs/jquery/1.10.1/jquery.min.js"></script>
+        <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.1/jquery.min.js"></script>
-        <script src="//maxcdn.bootstrapcdn.com/bootstrap/3.2.0/js/bootstrap.min.js"></script>
+        <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.0/js/bootstrap.min.js"></script>
 
         <script>
 var global_modal_state = null;
@@ -233,7 +231,7 @@ $(function() {
 function show_modal_error(title, message, callback) {
   $('#global_modal h4').text(title);
   $('#global_modal .modal-body').html("<p/>");
-  if (typeof question == String) {
+  if (typeof question == 'string') {
     $('#global_modal p').text(message);
     $('#global_modal .modal-dialog').addClass("modal-sm");
   } else {
@@ -245,11 +243,12 @@ function show_modal_error(title, message, callback) {
   global_modal_funcs = [callback, callback];
   global_modal_state = null;
   $('#global_modal').modal({});
+  return false; // handy when called from onclick
 }
 
 function show_modal_confirm(title, question, verb, yes_callback, cancel_callback) {
   $('#global_modal h4').text(title);
-  if (typeof question == String) {
+  if (typeof question == 'string') {
     $('#global_modal .modal-dialog').addClass("modal-sm");
     $('#global_modal .modal-body').html("<p/>");
     $('#global_modal p').text(question);
@@ -257,11 +256,17 @@ function show_modal_confirm(title, question, verb, yes_callback, cancel_callback
     $('#global_modal .modal-dialog').removeClass("modal-sm");
     $('#global_modal .modal-body').html("").append(question);
   }
-  $('#global_modal .btn-default').show().text("Cancel");
+  if (typeof verb == 'string') {
-  $('#global_modal .btn-danger').show().text(verb);
+    $('#global_modal .btn-default').show().text("Cancel");
+    $('#global_modal .btn-danger').show().text(verb);
+  } else {
+    $('#global_modal .btn-default').show().text(verb[1]);
+    $('#global_modal .btn-danger').show().text(verb[0]);
+  }
   global_modal_funcs = [yes_callback, cancel_callback];
   global_modal_state = null;
   $('#global_modal').modal({});
+  return false; // handy when called from onclick
 }
 
 var ajax_num_executing_requests = 0;
@@ -270,7 +275,7 @@ function ajax(options) {
   function hide_loading_indicator() {
     ajax_num_executing_requests--;
     if (ajax_num_executing_requests == 0)
-      $('#ajax_loading_indicator').stop().hide(); // stop() prevents an ongoing fade from causing the thing to be shown again after this call
+      $('#ajax_loading_indicator').stop(true).hide(); // stop() prevents an ongoing fade from causing the thing to be shown again after this call
   }
   var old_success = options.success;
   var old_error = options.error;
@@ -290,6 +295,7 @@ function ajax(options) {
   };
   ajax_num_executing_requests++;
   $.ajax(options);
+  return false; // handy when called from onclick
 }
 
 var api_credentials = ["", ""];
@@ -331,6 +337,11 @@ function api(url, method, data, callback, callback_error) {
     method: method,
     cache: false,
     data: data,
+
+    // the custom DNS api sends raw POST/PUT bodies --- prevent URL-encoding
+    processData: typeof data != "string",
+    mimeType: typeof data == "string" ? "text/plain; charset=ascii" : null,
+
     beforeSend: function(xhr) {
       // We don't store user credentials in a cookie to avoid the hassle of CSRF
       // attacks. The Authorization header only gets set in our AJAX calls triggered

management/templates/login.html

@@ -1,12 +1,20 @@
 <h1 style="margin: 1em; text-align: center">{{hostname}}</h1>
 
-{% if no_admins_exist %}
+{% if no_users_exist or no_admins_exist %}
 <div class="row">
 <div class="col-md-offset-2 col-md-8">
+  {% if no_users_exist %}
+  <p class="text-danger">There are no users on this system! To make an administrative user,
+  log into this machine using SSH (like when you first set it up) and run:</p>
+  <pre>cd mailinabox
+sudo tools/mail.py user add me@{{hostname}}
+sudo tools/mail.py user make-admin me@{{hostname}}</pre>
+  {% else %}
   <p class="text-danger">There are no administrative users on this system! To make an administrative user,
   log into this machine using SSH (like when you first set it up) and run:</p>
   <pre>cd mailinabox
-sudo tools/mail.py user make-admin your@emailaddress.com</pre>
+sudo tools/mail.py user make-admin me@{{hostname}}</pre>
+  {% endif %}
   <hr>
 </div>
 </div>  

management/templates/system-backup.html

@@ -28,8 +28,6 @@
   </tbody>
 </table>
 
-<p style="margin-top: 2em"><small>The size column in the table indicates the size of the encrypted backup, but the total size on disk shown above includes storage for unencrypted intermediate files.</small></p>
-
 <script>
 function nice_size(bytes) {
   var powers = ['bytes', 'KB', 'MB', 'GB', 'TB'];
@@ -54,7 +52,7 @@ function show_system_backup() {
     "GET",
     { },
     function(r) {
-      $('#backup-location').text(r.encdirectory);
+      $('#backup-location').text(r.directory);
       $('#backup-encpassword-file').text(r.encpwfile);
 
       $('#backup-status tbody').html("");
@@ -72,7 +70,7 @@ function show_system_backup() {
         tr.append( $('<td/>').text(b.date_str + " " + r.tz) );
         tr.append( $('<td/>').text(b.date_delta + " ago") );
         tr.append( $('<td/>').text(b.full ? "full" : "increment") );
-        tr.append( $('<td style="text-align: right"/>').text( nice_size(b.encsize)) );
+        tr.append( $('<td style="text-align: right"/>').text( nice_size(b.size)) );
         if (b.deleted_in)
           tr.append( $('<td/>').text(b.deleted_in) );
         else
@@ -80,7 +78,6 @@ function show_system_backup() {
         $('#backup-status tbody').append(tr);
 
         total_disk_size += b.size;
-        total_disk_size += b.encsize;
       }
 
       $('#backup-total-size').text(nice_size(total_disk_size));

management/templates/users.html

@@ -188,6 +188,13 @@ function users_set_password(elem) {
 
 function users_remove(elem) {
   var email = $(elem).parents('tr').attr('data-email');
+
+  // can't remove yourself
+  if (api_credentials != null && email == api_credentials[0]) {
+    show_modal_error("Archive User", "You cannot archive your own account.");
+    return;
+  }
+
   show_modal_confirm(
     "Archive User",
     $("<p>Are you sure you want to archive <b>" + email + "</b>?</p> <p>The user's mailboxes will not be deleted (you can do that later), but the user will no longer be able to log into any services on this machine.</p>"),
@@ -223,7 +230,7 @@ function mod_priv(elem, add_remove) {
   var add_remove1 = add_remove.charAt(0).toUpperCase() + add_remove.substring(1);
   show_modal_confirm(
     "Modify Privileges",
-    "Are you sure you want to " + add_remove + " the " + priv + " privilege for <b>" + email + "</b>?",
+    $("<p>Are you sure you want to " + add_remove + " the " + priv + " privilege for <b>" + email + "</b>?</p>"),
     add_remove1,
     function() {
       api(

management/templates/web.html

@@ -82,7 +82,7 @@ function show_change_web_root(elem) {
   var root = $(elem).parents('tr').attr('data-custom-web-root');
   show_modal_confirm(
     'Change Root Directory for ' + domain,
-    '<p>You can change the static directory for <tt>' + domain + '</tt> to:</p> <p><tt>' + root + '</tt></p> <p>First create this directory on the server. Then click Update to scan for the directory and update web settings.',
+    $('<p>You can change the static directory for <tt>' + domain + '</tt> to:</p> <p><tt>' + root + '</tt></p> <p>First create this directory on the server. Then click Update to scan for the directory and update web settings.'),
     'Update',
     function() { do_web_update(); });
 }

management/utils.py

@@ -184,3 +184,19 @@ def du(path):
             seen.add(stat.st_ino)
             total_size += stat.st_size
     return total_size
+
+def wait_for_service(port, public, env, timeout):
+	# Block until a service on a given port (bound privately or publicly)
+	# is taking connections, with a maximum timeout.
+	import socket, time
+	start = time.perf_counter()
+	while True:
+		s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+		s.settimeout(timeout/3)
+		try:
+			s.connect(("127.0.0.1" if not public else env['PUBLIC_IP'], port))
+			return True
+		except OSError:
+			if time.perf_counter() > start+timeout:
+				return False
+		time.sleep(min(timeout/4, 1))

management/web_update.py

@@ -24,12 +24,9 @@ def get_web_domains(env):
 	# ...Unless the domain has an A/AAAA record that maps it to a different
 	# IP address than this box. Remove those domains from our list.
 	dns = get_custom_dns_config(env)
-	for domain, value in dns.items():
+	for domain, rtype, value in dns:
 		if domain not in domains: continue
-		if (isinstance(value, str) and (value != "local")) \
+		if rtype == "CNAME" or (rtype in ("A", "AAAA") and value != "local"):
-		  or (isinstance(value, dict) and ("CNAME" in value)) \
-		  or (isinstance(value, dict) and ("A" in value) and (value["A"] != "local")) \
-		  or (isinstance(value, dict) and ("AAAA" in value) and (value["AAAA"] != "local")):
 			domains.remove(domain)
 
 	# Sort the list. Put PRIMARY_HOSTNAME first so it becomes the

setup/dns.sh

@@ -10,19 +10,7 @@
 source setup/functions.sh # load our functions
 source /etc/mailinabox.conf # load global vars
 
-# Install `nsd`, our DNS server software, and `ldnsutils` which helps
+# Install the packages.
-# us sign zones for DNSSEC.
-
-# ...but first, we have to create the user because the 
-# current Ubuntu forgets to do so in the .deb
-# (see issue #25 and https://bugs.launchpad.net/ubuntu/+source/nsd/+bug/1311886)
-if id nsd > /dev/null 2>&1; then
-	true #echo "nsd user exists... good"; #NODOC
-else
-	useradd nsd;
-fi
-
-# Okay now install the packages.
 #
 # * nsd: The non-recursive nameserver that publishes our DNS records.
 # * ldnsutils: Helper utilities for signing DNSSEC zones.
@@ -34,6 +22,35 @@ apt_install nsd ldnsutils openssh-client
 
 mkdir -p /var/run/nsd
 
+cat > /etc/nsd/nsd.conf << EOF;
+# No not edit. Overwritten by Mail-in-a-Box setup.
+server:
+  hide-version: yes
+
+  # identify the server (CH TXT ID.SERVER entry).
+  identity: ""
+
+  # The directory for zonefile: files.
+  zonesdir: "/etc/nsd/zones"
+
+  # Allows NSD to bind to IP addresses that are not (yet) added to the
+  # network interface. This allows nsd to start even if the network stack
+  # isn't fully ready, which apparently happens in some cases.
+  # See https://www.nlnetlabs.nl/projects/nsd/nsd.conf.5.html.
+  ip-transparent: yes
+
+EOF
+
+# Since we have bind9 listening on localhost for locally-generated
+# DNS queries that require a recursive nameserver, and the system
+# might have other network interfaces for e.g. tunnelling, we have
+# to be specific about the network interfaces that nsd binds to.
+for ip in $PRIVATE_IP $PRIVATE_IPV6; do
+	echo "  ip-address: $ip" >> /etc/nsd/nsd.conf;
+done
+
+echo "include: /etc/nsd/zones.conf" >> /etc/nsd/nsd.conf;
+
 # Create DNSSEC signing keys.
 
 mkdir -p "$STORAGE_ROOT/dns/dnssec";

setup/functions.sh

@@ -9,13 +9,15 @@ function hide_output {
 	$@ &> $OUTPUT
 
 	# If the command failed, show the output that was captured in the temporary file.
-	if [ $? != 0 ]; then
+	E=$?
+	if [ $E != 0 ]; then
 		# Something failed.
 		echo
 		echo FAILED: $@
 		echo -----------------------------------------
 		cat $OUTPUT
 		echo -----------------------------------------
+		exit $E
 	fi
 
 	# Remove temporary file.
@@ -180,6 +182,28 @@ function input_menu {
 	result_code=$?
 }
 
+function wget_verify {
+	# Downloads a file from the web and checks that it matches
+	# a provided hash. If the comparison fails, exit immediately.
+	URL=$1
+	HASH=$2
+	DEST=$3
+	CHECKSUM="$HASH  $DEST"
+	rm -f $DEST
+	wget -q -O $DEST $URL || exit 1
+	if ! echo "$CHECKSUM" | sha1sum --check --strict > /dev/null; then
+		echo "------------------------------------------------------------"
+		echo "Download of $URL did not match expected checksum."
+		echo "Found:"
+		sha1sum $DEST
+		echo
+		echo "Expected:"
+		echo "$CHECKSUM"
+		rm -f $DEST
+		exit 1
+	fi
+}
+
 function git_clone {
 	# Clones a git repository, checks out a particular commit or tag,
 	# and moves the repository (or a subdirectory in it) to some path.

setup/mail-dovecot.sh

@@ -90,7 +90,7 @@ tools/editconf.py /etc/dovecot/conf.d/20-imap.conf \
 # For new POP3 servers, the easiest way to set up UIDLs is to use IMAP's UIDVALIDITY
 # and UID values, the default in Dovecot.
 tools/editconf.py /etc/dovecot/conf.d/20-pop3.conf \
-	pop3_uidl_format = %08Xu%08Xv
+	pop3_uidl_format="%08Xu%08Xv"
 
 # ### LDA (LMTP)
 

setup/management.sh

@@ -3,7 +3,8 @@
 source setup/functions.sh
 
 apt_install python3-flask links duplicity libyaml-dev python3-dnspython python3-dateutil
-hide_output pip3 install rtyaml
+hide_output pip3 install rtyaml "email_validator==0.1.0-rc4"
+	# email_validator is repeated in setup/questions.sh
 
 # Create a backup directory and a random key for encrypting backups.
 mkdir -p $STORAGE_ROOT/backup

setup/owncloud.sh

@@ -16,6 +16,7 @@ apt-get purge -qq -y owncloud*
 
 # Install ownCloud from source of this version:
 owncloud_ver=8.0.2
+owncloud_hash=a4d1fc44bc40af87948458ae8f60ee427ecd9560
 
 # Check if ownCloud dir exist, and check if version matches owncloud_ver (if either doesn't - install/upgrade)
 if [ ! -d /usr/local/lib/owncloud/ ] \
@@ -32,8 +33,7 @@ if [ ! -d /usr/local/lib/owncloud/ ] \
 	fi
 
 	# Download and extract ownCloud.
-	rm -f /tmp/owncloud.zip
+	wget_verify https://download.owncloud.org/community/owncloud-$owncloud_ver.zip $owncloud_hash /tmp/owncloud.zip
-	wget -qO /tmp/owncloud.zip https://download.owncloud.org/community/owncloud-$owncloud_ver.zip
 	unzip -u -o -q /tmp/owncloud.zip -d /usr/local/lib #either extracts new or replaces current files
 	rm -f /tmp/owncloud.zip
 

setup/questions.sh

@@ -4,7 +4,14 @@ if [ -z "$NONINTERACTIVE" ]; then
 	# e.g. if we piped a bootstrapping install script to bash to get started. In that
 	# case, the nifty '[ -t 0 ]' test won't work. But with Vagrant we must suppress so we
 	# use a shell flag instead. Really supress any output from installing dialog.
-	apt_get_quiet install dialog
+	#
+	# Also install depencies needed to validate the email address.
+	echo Installing packages needed for setup...
+	apt_get_quiet install dialog python3 python3-pip  || exit 1
+
+	# email_validator is repeated in setup/management.sh
+	hide_output pip3 install "email_validator==0.1.0-rc4" || exit 1
+
 	message_box "Mail-in-a-Box Installation" \
 		"Hello and thanks for deploying a Mail-in-a-Box!
 		\n\nI'm going to ask you a few questions.

setup/webmail.sh

@@ -32,8 +32,9 @@ apt-get purge -qq -y roundcube* #NODOC
 # Install Roundcube from source if it is not already present or if it is out of date.
 # Combine the Roundcube version number with the commit hash of vacation_sieve to track
 # whether we have the latest version.
-VERSION=1.1.0
+VERSION=1.1.1
-VACATION_SIEVE_VERSION=06a20e9d44db62259ae41fd8451f3c937d3ab4f3
+HASH=08222f382a8dd89bba7dbbad595f48443bec0aa2
+VACATION_SIEVE_VERSION=91ea6f52216390073d1f5b70b5f6bea0bfaee7e5
 needs_update=0 #NODOC
 if [ ! -f /usr/local/lib/roundcubemail/version ]; then
 	# not installed yet #NODOC
@@ -45,8 +46,10 @@ fi
 if [ $needs_update == 1 ]; then
 	# install roundcube
 	echo installing Roundcube webmail $VERSION...
-	rm -f /tmp/roundcube.tgz
+	wget_verify \
-	wget -qO /tmp/roundcube.tgz http://downloads.sourceforge.net/project/roundcubemail/roundcubemail/$VERSION/roundcubemail-$VERSION.tar.gz
+		http://downloads.sourceforge.net/project/roundcubemail/roundcubemail/$VERSION/roundcubemail-$VERSION.tar.gz \
+		$HASH \
+		/tmp/roundcube.tgz
 	tar -C /usr/local/lib -zxf /tmp/roundcube.tgz
 	rm -rf /usr/local/lib/roundcubemail
 	mv /usr/local/lib/roundcubemail-$VERSION/ /usr/local/lib/roundcubemail

tools/editconf.py

@@ -54,6 +54,14 @@ while settings[0][0] == "-" and settings[0] != "--":
 		print("Invalid option.")
 		sys.exit(1)
 
+# sanity check command line
+for setting in settings:
+	try:
+		name, value = setting.split("=", 1)
+	except:
+		import subprocess
+		print("Invalid command line: ", subprocess.list2cmdline(sys.argv))
+
 # create the new config file in memory
 
 found = set()