merge upstream jelly branch

CHANGELOG.md

@@ -6,7 +6,7 @@ Version 60 (date TBD)
 
 This is the first release for Ubuntu 22.04.
 
-**Before upgrading**, you must **first upgrade your existing Ubuntu 18.04 box to Mail-in-a-Box v0.51** (or any later version of Mail-in-a-Box supporting Ubuntu 18.04), if you haven't already done so. That may not be possible after Ubuntu 18.04 reaches its end of life in April 2023, so please compete the upgrade well before then. (If you are not using Nextcloud's contacts or calendar, you can migrate to the latest version of Mail-in-a-Box from any previous version.) 
+**Before upgrading**, you must **first upgrade your existing Ubuntu 18.04 box to Mail-in-a-Box v0.51 or later**, if you haven't already done so. That may not be possible after Ubuntu 18.04 reaches its end of life in April 2023, so please complete the upgrade well before then. (If you are not using Nextcloud's contacts or calendar, you can migrate to the latest version of Mail-in-a-Box from any previous version.)
 
 For complete upgrade instructions, see:
 
@@ -14,14 +14,33 @@ LINK TBD
 
 No features of Mail-in-a-Box have changed in this release, but with the newer version of Ubuntu the following software packages we use are updated:
 
-* dovecot is upgraded to 2.3.16, postfix to 3.6.3, opendmark to 1.4 (which adds ARC-Authentication-Results headers), and spampd to 2.53 (alleviating a mail delivery rate limiting bug).
+* dovecot is upgraded to 2.3.16, postfix to 3.6.4, opendmark to 1.4 (which adds ARC-Authentication-Results headers), and spampd to 2.53 (alleviating a mail delivery rate limiting bug).
 * Nextcloud is upgraded to 23.0.0 with PHP updated from 7.2 to 8.0.
 * certbot is upgraded to 1.21 (via the Ubuntu repository instead of a PPA).
 * fail2ban is upgraded to 0.11.2.
 * nginx is upgraded to 1.18.
 
-In Development
+=======
---------------
+Version 57 (June 12, 2022)
+--------------------------
+
+Setup:
+
+* Fixed issue upgrading from Mail-in-a-Box v0.40-v0.50 because of a changed URL that Nextcloud is downloaded from.
+
+Backups:
+
+* Fixed S3 backups which broke with duplicity 0.8.23.
+* Fixed Backblaze backups which broke with latest b2sdk package by rolling back its version.
+
+Control panel:
+
+* Fixed spurious changes in system status checks messages by sorting DNSSEC DS records.
+* Fixed fail2ban lockout over IPv6 from excessive loads of the system status checks.
+* Fixed an incorrect IPv6 system status check message.
+
+Version 56 (January 19, 2022)
+-----------------------------
 
 Software updates:
 

conf/fail2ban/jails.conf

@@ -5,7 +5,7 @@
 # Whitelist our own IP addresses. 127.0.0.1/8 is the default. But our status checks
 # ping services over the public interface so we should whitelist that address of
 # ours too. The string is substituted during installation.
-ignoreip = 127.0.0.1/8 ::1/128 PUBLIC_IP PUBLIC_IPV6/64
+ignoreip = 127.0.0.1/8 PUBLIC_IP ::1 PUBLIC_IPV6
 
 [dovecot]
 enabled = true

management/backup.py

@@ -12,12 +12,7 @@ import dateutil.parser, dateutil.relativedelta, dateutil.tz
 import rtyaml
 from exclusiveprocess import Lock
 
-from utils import load_environment, shell, wait_for_service, fix_boto, get_php_version
+from utils import load_environment, shell, wait_for_service, get_php_version
-
-rsync_ssh_options = [
-	"--ssh-options= -i /root/.ssh/id_rsa_miab",
-	"--rsync-options= -e \"/usr/bin/ssh -oStrictHostKeyChecking=no -oBatchMode=yes -p 22 -i /root/.ssh/id_rsa_miab\"",
-]
 
 def backup_status(env):
 	# If backups are disabled, return no status.
@@ -64,9 +59,9 @@ def backup_status(env):
 		"--archive-dir", backup_cache_dir,
 		"--gpg-options", "--cipher-algo=AES256",
 		"--log-fd", "1",
-		config["target"],
+		get_duplicity_target_url(config),
-		] + rsync_ssh_options,
+		] + get_duplicity_additional_args(env),
-		get_env(env),
+		get_duplicity_env_vars(env),
 		trap=True)
 	if code != 0:
 		# Command failed. This is likely due to an improperly configured remote
@@ -195,7 +190,43 @@ def get_passphrase(env):
 
 	return passphrase
 
-def get_env(env):
+def get_duplicity_target_url(config):
+	target = config["target"]
+
+	if get_target_type(config) == "s3":
+		from urllib.parse import urlsplit, urlunsplit
+		target = list(urlsplit(target))
+
+		# Although we store the S3 hostname in the target URL,
+		# duplicity no longer accepts it in the target URL. The hostname in
+		# the target URL must be the bucket name. The hostname is passed
+		# via get_duplicity_additional_args. Move the first part of the
+		# path (the bucket name) into the hostname URL component, and leave
+		# the rest for the path.
+		target[1], target[2] = target[2].lstrip('/').split('/', 1)
+
+		target = urlunsplit(target)
+
+	return target
+
+def get_duplicity_additional_args(env):
+	config = get_backup_config(env)
+
+	if get_target_type(config) == 'rsync':
+		return [
+			"--ssh-options= -i /root/.ssh/id_rsa_miab",
+			"--rsync-options= -e \"/usr/bin/ssh -oStrictHostKeyChecking=no -oBatchMode=yes -p 22 -i /root/.ssh/id_rsa_miab\"",
+		]
+	elif get_target_type(config) == 's3':
+		# See note about hostname in get_duplicity_target_url.
+		from urllib.parse import urlsplit, urlunsplit
+		target = urlsplit(config["target"])
+		endpoint_url = urlunsplit(("https", target.netloc, '', '', ''))
+		return ["--s3-endpoint-url",  endpoint_url]
+
+	return []
+
+def get_duplicity_env_vars(env):
 	config = get_backup_config(env)
 
 	env = { "PASSPHRASE" : get_passphrase(env) }
@@ -274,10 +305,10 @@ def perform_backup(full_backup):
 			"--volsize", "250",
 			"--gpg-options", "--cipher-algo=AES256",
 			env["STORAGE_ROOT"],
-			config["target"],
+			get_duplicity_target_url(config),
 			"--allow-source-mismatch"
-			] + rsync_ssh_options,
+			] + get_duplicity_additional_args(env),
-			get_env(env))
+			get_duplicity_env_vars(env))
 	finally:
 		# Start services again.
 		service_command("dovecot", "start", quit=False)
@@ -293,9 +324,9 @@ def perform_backup(full_backup):
 		"--verbosity", "error",
 		"--archive-dir", backup_cache_dir,
 		"--force",
-		config["target"]
+		get_duplicity_target_url(config)
-		] + rsync_ssh_options,
+		] + get_duplicity_additional_args(env),
-		get_env(env))
+		get_duplicity_env_vars(env))
 
 	# From duplicity's manual:
 	# "This should only be necessary after a duplicity session fails or is
@@ -308,9 +339,9 @@ def perform_backup(full_backup):
 		"--verbosity", "error",
 		"--archive-dir", backup_cache_dir,
 		"--force",
-		config["target"]
+		get_duplicity_target_url(config)
-		] + rsync_ssh_options,
+		] + get_duplicity_additional_args(env),
-		get_env(env))
+		get_duplicity_env_vars(env))
 
 	# Change ownership of backups to the user-data user, so that the after-bcakup
 	# script can access them.
@@ -346,9 +377,9 @@ def run_duplicity_verification():
 		"--compare-data",
 		"--archive-dir", backup_cache_dir,
 		"--exclude", backup_root,
-		config["target"],
+		get_duplicity_target_url(config),
 		env["STORAGE_ROOT"],
-	] + rsync_ssh_options, get_env(env))
+	] + get_duplicity_additional_args(env), get_duplicity_env_vars(env))
 
 def run_duplicity_restore(args):
 	env = load_environment()
@@ -358,9 +389,9 @@ def run_duplicity_restore(args):
 		"/usr/bin/duplicity",
 		"restore",
 		"--archive-dir", backup_cache_dir,
-		config["target"],
+		get_duplicity_target_url(config),
-		] + rsync_ssh_options + args,
+		] + get_duplicity_additional_args(env) + args,
-	get_env(env))
+	get_duplicity_env_vars(env))
 
 def list_target_files(config):
 	import urllib.parse
@@ -417,7 +448,6 @@ def list_target_files(config):
 
 	elif target.scheme == "s3":
 		# match to a Region
-		fix_boto() # must call prior to importing boto
 		import boto.s3
 		from boto.exception import BotoServerError
 		custom_region = False

management/daemon.py

@@ -121,7 +121,6 @@ def index():
 	no_users_exist = (len(get_mail_users(env)) == 0)
 	no_admins_exist = (len(get_admins(env)) == 0)
 
-	utils.fix_boto() # must call prior to importing boto
 	import boto.s3
 	backup_s3_hosts = [(r.name, r.endpoint) for r in boto.s3.regions()]
 

management/ssl_certificates.py

@@ -58,36 +58,33 @@ def get_ssl_certificates(env):
 			# Not a valid PEM format for a PEM type we care about.
 			continue
 
-		# Remember where we got this object.
-		pem._filename = fn
-
 		# Is it a private key?
 		if isinstance(pem, RSAPrivateKey):
-			private_keys[pem.public_key().public_numbers()] = pem
+			private_keys[pem.public_key().public_numbers()] = { "filename": fn, "key": pem }
 
 		# Is it a certificate?
 		if isinstance(pem, Certificate):
-			certificates.append(pem)
+			certificates.append({ "filename": fn, "cert": pem })
 
 	# Process the certificates.
 	domains = { }
 	for cert in certificates:
 		# What domains is this certificate good for?
-		cert_domains, primary_domain = get_certificate_domains(cert)
+		cert_domains, primary_domain = get_certificate_domains(cert["cert"])
-		cert._primary_domain = primary_domain
+		cert["primary_domain"] = primary_domain
 
 		# Is there a private key file for this certificate?
-		private_key = private_keys.get(cert.public_key().public_numbers())
+		private_key = private_keys.get(cert["cert"].public_key().public_numbers())
 		if not private_key:
 			continue
-		cert._private_key = private_key
+		cert["private_key"] = private_key
 
 		# Add this cert to the list of certs usable for the domains.
 		for domain in cert_domains:
 			# The primary hostname can only use a certificate mapped
 			# to the system private key.
 			if domain == env['PRIMARY_HOSTNAME']:
-				if cert._private_key._filename != os.path.join(env['STORAGE_ROOT'], 'ssl', 'ssl_private_key.pem'):
+				if cert["private_key"]["filename"] != os.path.join(env['STORAGE_ROOT'], 'ssl', 'ssl_private_key.pem'):
 					continue
 
 			domains.setdefault(domain, []).append(cert)
@@ -100,10 +97,10 @@ def get_ssl_certificates(env):
 		#for c in cert_list: print(domain, c.not_valid_before, c.not_valid_after, "("+str(now)+")", c.issuer, c.subject, c._filename)
 		cert_list.sort(key = lambda cert : (
 			# must be valid NOW
-			cert.not_valid_before <= now <= cert.not_valid_after,
+			cert["cert"].not_valid_before <= now <= cert["cert"].not_valid_after,
 
 			# prefer one that is not self-signed
-			cert.issuer != cert.subject,
+			cert["cert"].issuer != cert["cert"].subject,
 
 			###########################################################
 			# The above lines ensure that valid certificates are chosen
@@ -113,7 +110,7 @@ def get_ssl_certificates(env):
 
 			# prefer one with the expiration furthest into the future so
 			# that we can easily rotate to new certs as we get them
-			cert.not_valid_after,
+			cert["cert"].not_valid_after,
 
 			###########################################################
 			# We always choose the certificate that is good for the
@@ -128,15 +125,15 @@ def get_ssl_certificates(env):
 
 			# in case a certificate is installed in multiple paths,
 			# prefer the... lexicographically last one?
-			cert._filename,
+			cert["filename"],
 
 		), reverse=True)
 		cert = cert_list.pop(0)
 		ret[domain] = {
-			"private-key": cert._private_key._filename,
+			"private-key": cert["private_key"]["filename"],
-			"certificate": cert._filename,
+			"certificate": cert["filename"],
-			"primary-domain": cert._primary_domain,
+			"primary-domain": cert["primary_domain"],
-			"certificate_object": cert,
+			"certificate_object": cert["cert"],
 			}
 
 	return ret

management/status_checks.py

@@ -255,6 +255,18 @@ def check_free_disk_space(rounded_values, env, output):
 		if rounded_values: disk_msg = "The disk has less than 15% free space."
 		output.print_error(disk_msg)
 
+	# Check that there's only one duplicity cache. If there's more than one,
+	# it's probably no longer in use, and we can recommend clearing the cache
+	# to save space. The cache directory may not exist yet, which is OK.
+	backup_cache_path = os.path.join(env['STORAGE_ROOT'], 'backup/cache')
+	try:
+		backup_cache_count = len(os.listdir(backup_cache_path))
+	except:
+		backup_cache_count = 0
+	if backup_cache_count > 1:
+		output.print_warning("The backup cache directory {} has more than one backup target cache. Consider clearing this directory to save disk space."
+			.format(backup_cache_path))
+
 def check_free_memory(rounded_values, env, output):
 	# Check free memory.
 	percent_free = 100 - psutil.virtual_memory().percent
@@ -660,7 +672,7 @@ def check_dnssec(domain, env, output, dns_zonefiles, is_checking_primary=False):
 	if len(ds) > 0:
 		output.print_line("")
 		output.print_line("The DS record is currently set to:")
-		for rr in ds:
+		for rr in sorted(ds):
 			output.print_line("Key Tag: {0}, Algorithm: {1}, Digest Type: {2}, Digest: {3}".format(*rr))
 
 def check_mail_domain(domain, env, output):

management/utils.py

@@ -175,13 +175,6 @@ def wait_for_service(port, public, env, timeout):
 				return False
 		time.sleep(min(timeout/4, 1))
 
-def fix_boto():
-	# Google Compute Engine instances install some Python-2-only boto plugins that
-	# conflict with boto running under Python 3. Disable boto's default configuration
-	# file prior to importing boto so that GCE's plugin is not loaded:
-	import os
-	os.environ["BOTO_CONFIG"] = "/etc/boto3.cfg"
-
 def get_php_version():
 	# Gets the version of PHP installed in the system.
 	return shell("check_output", ["/usr/bin/php", "-v"])[4:7]

setup/bootstrap.sh

@@ -28,10 +28,10 @@ if [ -z "$TAG" ]; then
 		# This machine is running Ubuntu 18.04, which is supported by
 		# Mail-in-a-Box versions 0.40 through 5x.
 		echo "Support is ending for Ubuntu 18.04."
-		echo "Please immediately begin to migrate your information to"
+		echo "Please immediately begin to migrate your data to"
 		echo "a new machine running Ubuntu 22.04. See:"
 		echo "https://mailinabox.email/maintenance.html#upgrade"
-		TAG=v56
+		TAG=v57
 	elif [ "$UBUNTU_VERSION" == "Ubuntu 14.04 LTS" ]; then
 		# This machine is running Ubuntu 14.04, which is supported by
 		# Mail-in-a-Box versions 1 through v0.30.

setup/functions.sh

@@ -4,6 +4,8 @@
 # -o pipefail: don't ignore errors in the non-last command in a pipeline
 set -euo pipefail
 
+PHP_VER=8.0
+
 function hide_output {
 	# This function hides the output of a command unless the command fails
 	# and returns a non-zero exit code.

setup/management.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 source setup/functions.sh
+source /etc/mailinabox.conf # load global vars
 
 echo "Installing Mail-in-a-Box system management daemon..."
 
@@ -51,7 +52,8 @@ hide_output $venv/bin/pip install --upgrade \
 	rtyaml "email_validator>=1.0.0" "exclusiveprocess" \
 	flask dnspython python-dateutil expiringdict \
 	qrcode[pil] pyotp \
-	"idna>=2.0.0" "cryptography==2.2.2" boto psutil postfix-mta-sts-resolver b2sdk
+	"idna>=2.0.0" "cryptography==37.0.2" psutil postfix-mta-sts-resolver \
+	b2sdk boto
 
 # CONFIGURATION
 

setup/nextcloud.sh

@@ -402,7 +402,6 @@ tools/editconf.py /etc/php/$(php_version)/cli/conf.d/10-opcache.ini -c ';' \
 	opcache.save_comments=1 \
 	opcache.revalidate_freq=1
 
-
 # Set up a cron job for Nextcloud.
 cat > /etc/cron.d/mailinabox-nextcloud << EOF;
 #!/bin/bash

setup/system.sh

@@ -99,6 +99,9 @@ fi
 # come from there and minimal Ubuntu installs may have it turned off.
 hide_output add-apt-repository -y universe
 
+# Install the duplicity PPA.
+hide_output add-apt-repository -y ppa:duplicity-team/duplicity-release-git
+
 # ### Update Packages
 
 # Update system packages to make sure we have the latest upstream versions
@@ -356,6 +359,7 @@ systemctl restart systemd-resolved
 rm -f /etc/fail2ban/jail.local # we used to use this file but don't anymore
 rm -f /etc/fail2ban/jail.d/defaults-debian.conf # removes default config so we can manage all of fail2ban rules in one config
 cat conf/fail2ban/jails.conf \
+    | sed "s/PUBLIC_IPV6/$PUBLIC_IPV6/g" \
 	| sed "s/PUBLIC_IP/$PUBLIC_IP/g" \
 	| sed "s#STORAGE_ROOT#$STORAGE_ROOT#" \
 	> /etc/fail2ban/jail.d/mailinabox.conf

setup/webmail.sh

@@ -202,10 +202,10 @@ chown -f -R root.www-data ${RCM_PLUGIN_DIR}/carddav
 chmod -R 774 ${RCM_PLUGIN_DIR}/carddav
 
 # Run Roundcube database migration script (database is created if it does not exist)
-${RCM_DIR}/bin/updatedb.sh --dir ${RCM_DIR}/SQL --package roundcube
+php ${RCM_DIR}/bin/updatedb.sh --dir ${RCM_DIR}/SQL --package roundcube
 chown www-data:www-data $STORAGE_ROOT/mail/roundcube/roundcube.sqlite
 chmod 664 $STORAGE_ROOT/mail/roundcube/roundcube.sqlite
 
 # Enable PHP modules.
-phpenmod -v php mcrypt imap
+phpenmod -v php imap
 restart_service php$(php_version)-fpm

setup/zpush.sh

@@ -42,8 +42,6 @@ if [ $needs_update == 1 ]; then
 	rm -rf /tmp/z-push.zip /tmp/z-push
 
 	rm -f /usr/sbin/z-push-{admin,top}
-	ln -s /usr/local/lib/z-push/z-push-admin.php /usr/sbin/z-push-admin
-	ln -s /usr/local/lib/z-push/z-push-top.php /usr/sbin/z-push-top
 	echo $VERSION > /usr/local/lib/z-push/version
 fi
 
@@ -106,4 +104,4 @@ restart_service php$(php_version)-fpm
 
 # Fix states after upgrade
 
-hide_output z-push-admin -a fixstates
+hide_output php /usr/local/lib/z-push/z-push-admin.php -a fixstates