Merge branch 'jammyjellyfish2204' of https://github.com/mail-in-a-box/mailinabox into jammyjellyfish2204

# Conflicts:
#	CHANGELOG.md
#	README.md
#	conf/nginx-top.conf
#	management/backup.py
#	setup/bootstrap.sh
#	setup/management.sh
#	setup/nextcloud.sh
#	setup/system.sh
#	setup/web.sh
#	setup/webmail.sh
#	setup/zpush.sh
#	tests/test_mail.py

CHANGELOG.md

@@ -6,7 +6,7 @@ Version 60 (date TBD)
 
 This is the first release for Ubuntu 22.04.
 
-**Before upgrading**, you must **first upgrade your existing Ubuntu 18.04 box to Mail-in-a-Box v0.51** (or any later version of Mail-in-a-Box supporting Ubuntu 18.04), if you haven't already done so. That may not be possible after Ubuntu 18.04 reaches its end of life in April 2023, so please compete the upgrade well before then. (If you are not using Nextcloud's contacts or calendar, you can migrate to the latest version of Mail-in-a-Box from any previous version.) 
+**Before upgrading**, you must **first upgrade your existing Ubuntu 18.04 box to Mail-in-a-Box v0.51 or later**, if you haven't already done so. That may not be possible after Ubuntu 18.04 reaches its end of life in April 2023, so please compete the upgrade well before then. (If you are not using Nextcloud's contacts or calendar, you can migrate to the latest version of Mail-in-a-Box from any previous version.)
 
 For complete upgrade instructions, see:
 
@@ -14,14 +14,32 @@ LINK TBD
 
 No features of Mail-in-a-Box have changed in this release, but with the newer version of Ubuntu the following software packages we use are updated:
 
-* dovecot is upgraded to 2.3.16, postfix to 3.6.3, opendmark to 1.4 (which adds ARC-Authentication-Results headers), and spampd to 2.53 (alleviating a mail delivery rate limiting bug).
+* dovecot is upgraded to 2.3.16, postfix to 3.6.4, opendmark to 1.4 (which adds ARC-Authentication-Results headers), and spampd to 2.53 (alleviating a mail delivery rate limiting bug).
 * Nextcloud is upgraded to 23.0.0 with PHP updated from 7.2 to 8.0.
 * certbot is upgraded to 1.21 (via the Ubuntu repository instead of a PPA).
 * fail2ban is upgraded to 0.11.2.
 * nginx is upgraded to 1.18.
 
-In Development
+Version 57 (June 12, 2022)
---------------
+--------------------------
+
+Setup:
+
+* Fixed issue upgrading from Mail-in-a-Box v0.40-v0.50 because of a changed URL that Nextcloud is downloaded from.
+
+Backups:
+
+* Fixed S3 backups which broke with duplicity 0.8.23.
+* Fixed Backblaze backups which broke with latest b2sdk package by rolling back its version.
+
+Control panel:
+
+* Fixed spurious changes in system status checks messages by sorting DNSSEC DS records.
+* Fixed fail2ban lockout over IPv6 from excessive loads of the system status checks.
+* Fixed an incorrect IPv6 system status check message.
+
+Version 56 (January 19, 2022)
+-----------------------------
 
 Software updates:
 

conf/fail2ban/jails.conf

@@ -5,7 +5,7 @@
 # Whitelist our own IP addresses. 127.0.0.1/8 is the default. But our status checks
 # ping services over the public interface so we should whitelist that address of
 # ours too. The string is substituted during installation.
-ignoreip = 127.0.0.1/8 PUBLIC_IP
+ignoreip = 127.0.0.1/8 PUBLIC_IP ::1 PUBLIC_IPV6
 
 [dovecot]
 enabled = true

conf/nginx-top.conf

@@ -7,6 +7,6 @@
 ##       your own --- please do not ask for help from us.
 
 upstream php-fpm {
-	server unix:/var/run/php/php8.1-fpm.sock;
+	server unix:/var/run/php/php8.0-fpm.sock;
 }
 

management/backup.py

@@ -13,12 +13,7 @@ import dateutil.parser, dateutil.relativedelta, dateutil.tz
 import rtyaml
 from exclusiveprocess import Lock
 
-from utils import load_environment, shell, wait_for_service, fix_boto
+from utils import load_environment, shell, wait_for_service
-
-rsync_ssh_options = [
-	"--ssh-options= -i /root/.ssh/id_rsa_miab",
-	"--rsync-options= -e \"/usr/bin/ssh -oStrictHostKeyChecking=no -oBatchMode=yes -p 22 -i /root/.ssh/id_rsa_miab\"",
-]
 
 def backup_status(env):
 	# If backups are dissbled, return no status.
@@ -65,9 +60,9 @@ def backup_status(env):
 		"--archive-dir", backup_cache_dir,
 		"--gpg-options", "--cipher-algo=AES256",
 		"--log-fd", "1",
-		config["target"],
+		get_duplicity_target_url(config),
-		] + rsync_ssh_options,
+		] + get_duplicity_additional_args(env),
-		get_env(env),
+		get_duplicity_env_vars(env),
 		trap=True)
 	if code != 0:
 		# Command failed. This is likely due to an improperly configured remote
@@ -196,7 +191,43 @@ def get_passphrase(env):
 
 	return passphrase
 
-def get_env(env):
+def get_duplicity_target_url(config):
+	target = config["target"]
+
+	if get_target_type(config) == "s3":
+		from urllib.parse import urlsplit, urlunsplit
+		target = list(urlsplit(target))
+
+		# Although we store the S3 hostname in the target URL,
+		# duplicity no longer accepts it in the target URL. The hostname in
+		# the target URL must be the bucket name. The hostname is passed
+		# via get_duplicity_additional_args. Move the first part of the
+		# path (the bucket name) into the hostname URL component, and leave
+		# the rest for the path.
+		target[1], target[2] = target[2].lstrip('/').split('/', 1)
+
+		target = urlunsplit(target)
+
+	return target
+
+def get_duplicity_additional_args(env):
+	config = get_backup_config(env)
+
+	if get_target_type(config) == 'rsync':
+		return [
+			"--ssh-options= -i /root/.ssh/id_rsa_miab",
+			"--rsync-options= -e \"/usr/bin/ssh -oStrictHostKeyChecking=no -oBatchMode=yes -p 22 -i /root/.ssh/id_rsa_miab\"",
+		]
+	elif get_target_type(config) == 's3':
+		# See note about hostname in get_duplicity_target_url.
+		from urllib.parse import urlsplit, urlunsplit
+		target = urlsplit(config["target"])
+		endpoint_url = urlunsplit(("https", target.netloc, '', '', ''))
+		return ["--s3-endpoint-url",  endpoint_url]
+
+	return []
+
+def get_duplicity_env_vars(env):
 	config = get_backup_config(env)
 
 	env = { "PASSPHRASE" : get_passphrase(env) }
@@ -248,7 +279,7 @@ def perform_backup(full_backup):
 			if quit:
 				sys.exit(code)
 
-	service_command("php8.1-fpm", "stop", quit=True)
+	service_command("php8.0-fpm", "stop", quit=True)
 	service_command("postfix", "stop", quit=True)
 	service_command("dovecot", "stop", quit=True)
 	service_command("slapd", "stop", quit=True)
@@ -276,17 +307,17 @@ def perform_backup(full_backup):
 			"--volsize", "250",
 			"--gpg-options", "--cipher-algo=AES256",
 			env["STORAGE_ROOT"],
-			config["target"],
+			get_duplicity_target_url(config),
 			"--allow-source-mismatch"
-			] + rsync_ssh_options,
+			] + get_duplicity_additional_args(env),
-			get_env(env))
+			get_duplicity_env_vars(env))
 	finally:
 		# Start services again.
 		service_command("miabldap-capture", "start", quit=False)
 		service_command("slapd", "start", quit=False)
 		service_command("dovecot", "start", quit=False)
 		service_command("postfix", "start", quit=False)
-		service_command("php8.1-fpm", "start", quit=False)
+		service_command("php8.0-fpm", "start", quit=False)
 
 	# Remove old backups. This deletes all backup data no longer needed
 	# from more than 3 days ago.
@@ -297,9 +328,9 @@ def perform_backup(full_backup):
 		"--verbosity", "error",
 		"--archive-dir", backup_cache_dir,
 		"--force",
-		config["target"]
+		get_duplicity_target_url(config)
-		] + rsync_ssh_options,
+		] + get_duplicity_additional_args(env),
-		get_env(env))
+		get_duplicity_env_vars(env))
 
 	# From duplicity's manual:
 	# "This should only be necessary after a duplicity session fails or is
@@ -312,9 +343,9 @@ def perform_backup(full_backup):
 		"--verbosity", "error",
 		"--archive-dir", backup_cache_dir,
 		"--force",
-		config["target"]
+		get_duplicity_target_url(config)
-		] + rsync_ssh_options,
+		] + get_duplicity_additional_args(env),
-		get_env(env))
+		get_duplicity_env_vars(env))
 
 	# Change ownership of backups to the user-data user, so that the after-bcakup
 	# script can access them.
@@ -350,9 +381,9 @@ def run_duplicity_verification():
 		"--compare-data",
 		"--archive-dir", backup_cache_dir,
 		"--exclude", backup_root,
-		config["target"],
+		get_duplicity_target_url(config),
 		env["STORAGE_ROOT"],
-	] + rsync_ssh_options, get_env(env))
+	] + get_duplicity_additional_args(env), get_duplicity_env_vars(env))
 
 def run_duplicity_restore(args):
 	env = load_environment()
@@ -362,9 +393,9 @@ def run_duplicity_restore(args):
 		"/usr/bin/duplicity",
 		"restore",
 		"--archive-dir", backup_cache_dir,
-		config["target"],
+		get_duplicity_target_url(config),
-		] + rsync_ssh_options + args,
+		] + get_duplicity_additional_args(env) + args,
-	get_env(env))
+	get_duplicity_env_vars(env))
 
 def list_target_files(config):
 	import urllib.parse
@@ -421,7 +452,6 @@ def list_target_files(config):
 
 	elif target.scheme == "s3":
 		# match to a Region
-		fix_boto() # must call prior to importing boto
 		import boto.s3
 		from boto.exception import BotoServerError
 		custom_region = False

management/daemon.py

@@ -122,7 +122,6 @@ def index():
 	no_users_exist = (len(get_mail_users(env)) == 0)
 	no_admins_exist = (len(get_admins(env)) == 0)
 
-	utils.fix_boto() # must call prior to importing boto
 	import boto.s3
 	backup_s3_hosts = [(r.name, r.endpoint) for r in boto.s3.regions()]
 

management/ssl_certificates.py

@@ -58,36 +58,33 @@ def get_ssl_certificates(env):
 			# Not a valid PEM format for a PEM type we care about.
 			continue
 
-		# Remember where we got this object.
-		pem._filename = fn
-
 		# Is it a private key?
 		if isinstance(pem, RSAPrivateKey):
-			private_keys[pem.public_key().public_numbers()] = pem
+			private_keys[pem.public_key().public_numbers()] = { "filename": fn, "key": pem }
 
 		# Is it a certificate?
 		if isinstance(pem, Certificate):
-			certificates.append(pem)
+			certificates.append({ "filename": fn, "cert": pem })
 
 	# Process the certificates.
 	domains = { }
 	for cert in certificates:
 		# What domains is this certificate good for?
-		cert_domains, primary_domain = get_certificate_domains(cert)
+		cert_domains, primary_domain = get_certificate_domains(cert["cert"])
-		cert._primary_domain = primary_domain
+		cert["primary_domain"] = primary_domain
 
 		# Is there a private key file for this certificate?
-		private_key = private_keys.get(cert.public_key().public_numbers())
+		private_key = private_keys.get(cert["cert"].public_key().public_numbers())
 		if not private_key:
 			continue
-		cert._private_key = private_key
+		cert["private_key"] = private_key
 
 		# Add this cert to the list of certs usable for the domains.
 		for domain in cert_domains:
 			# The primary hostname can only use a certificate mapped
 			# to the system private key.
 			if domain == env['PRIMARY_HOSTNAME']:
-				if cert._private_key._filename != os.path.join(env['STORAGE_ROOT'], 'ssl', 'ssl_private_key.pem'):
+				if cert["private_key"]["filename"] != os.path.join(env['STORAGE_ROOT'], 'ssl', 'ssl_private_key.pem'):
 					continue
 
 			domains.setdefault(domain, []).append(cert)
@@ -97,16 +94,16 @@ def get_ssl_certificates(env):
 	now = datetime.datetime.utcnow()
 	ret = { }
 	for domain, cert_list in domains.items():
-		#for c in cert_list: print(domain, c.not_valid_before, c.not_valid_after, "("+str(now)+")", c.issuer, c.subject, c._filename)
+		#for c in cert_list: print(domain, c["cert"].not_valid_before, c["cert"].not_valid_after, "("+str(now)+")", c["cert"].issuer, c["cert"].subject, c._filename if hasattr(c,"_filename") else "")
 		cert_list.sort(key = lambda cert : (
 			# must be valid NOW
-			cert.not_valid_before <= now <= cert.not_valid_after,
+			cert["cert"].not_valid_before <= now <= cert["cert"].not_valid_after,
 
 			# prefer one that is not self-signed
-			cert.issuer != cert.subject,
+			cert["cert"].issuer != cert["cert"].subject,
 
                         # prefer one that is not our temporary ca
-			"Temporary-Mail-In-A-Box-CA" not in "%s" % cert.issuer.rdns,
+			"Temporary-Mail-In-A-Box-CA" not in "%s" % cert["cert"].issuer.rdns,
 
 			###########################################################
 			# The above lines ensure that valid certificates are chosen
@@ -116,7 +113,7 @@ def get_ssl_certificates(env):
 
 			# prefer one with the expiration furthest into the future so
 			# that we can easily rotate to new certs as we get them
-			cert.not_valid_after,
+			cert["cert"].not_valid_after,
 
 			###########################################################
 			# We always choose the certificate that is good for the
@@ -131,15 +128,15 @@ def get_ssl_certificates(env):
 
 			# in case a certificate is installed in multiple paths,
 			# prefer the... lexicographically last one?
-			cert._filename,
+			cert["filename"],
 
 		), reverse=True)
 		cert = cert_list.pop(0)
 		ret[domain] = {
-			"private-key": cert._private_key._filename,
+			"private-key": cert["private_key"]["filename"],
-			"certificate": cert._filename,
+			"certificate": cert["filename"],
-			"primary-domain": cert._primary_domain,
+			"primary-domain": cert["primary_domain"],
-			"certificate_object": cert,
+			"certificate_object": cert["cert"],
 			}
 
 	return ret

management/status_checks.py

@@ -135,7 +135,7 @@ def check_service(i, service, env):
 
 			# IPv4 ok but IPv6 failed. Try the PRIVATE_IPV6 address to see if the service is bound to the interface.
 			elif service["port"] != 53 and try_connect(env["PRIVATE_IPV6"]):
-				output.print_error("%s is running (and available over IPv4 and the local IPv6 address), but it is not publicly accessible at %s:%d." % (service['name'], env['PUBLIC_IP'], service['port']))
+				output.print_error("%s is running (and available over IPv4 and the local IPv6 address), but it is not publicly accessible at %s:%d." % (service['name'], env['PUBLIC_IPV6'], service['port']))
 			else:
 				output.print_error("%s is running and available over IPv4 but is not accessible over IPv6 at %s port %d." % (service['name'], env['PUBLIC_IPV6'], service['port']))
 
@@ -253,6 +253,18 @@ def check_free_disk_space(rounded_values, env, output):
 		if rounded_values: disk_msg = "The disk has less than 15% free space."
 		output.print_error(disk_msg)
 
+	# Check that there's only one duplicity cache. If there's more than one,
+	# it's probably no longer in use, and we can recommend clearing the cache
+	# to save space. The cache directory may not exist yet, which is OK.
+	backup_cache_path = os.path.join(env['STORAGE_ROOT'], 'backup/cache')
+	try:
+		backup_cache_count = len(os.listdir(backup_cache_path))
+	except:
+		backup_cache_count = 0
+	if backup_cache_count > 1:
+		output.print_warning("The backup cache directory {} has more than one backup target cache. Consider clearing this directory to save disk space."
+			.format(backup_cache_path))
+
 def check_free_memory(rounded_values, env, output):
 	# Check free memory.
 	percent_free = 100 - psutil.virtual_memory().percent
@@ -658,7 +670,7 @@ def check_dnssec(domain, env, output, dns_zonefiles, is_checking_primary=False):
 	if len(ds) > 0:
 		output.print_line("")
 		output.print_line("The DS record is currently set to:")
-		for rr in ds:
+		for rr in sorted(ds):
 			output.print_line("Key Tag: {0}, Algorithm: {1}, Digest Type: {2}, Digest: {3}".format(*rr))
 
 def check_mail_domain(domain, env, output):

management/utils.py

@@ -193,14 +193,6 @@ def wait_for_service(port, public, env, timeout):
 				return False
 		time.sleep(min(timeout/4, 1))
 
-def fix_boto():
-	# Google Compute Engine instances install some Python-2-only boto plugins that
-	# conflict with boto running under Python 3. Disable boto's default configuration
-	# file prior to importing boto so that GCE's plugin is not loaded:
-	import os
-	os.environ["BOTO_CONFIG"] = "/etc/boto3.cfg"
-
-
 if __name__ == "__main__":
 	from web_update import get_web_domains
 	env = load_environment()

setup/bootstrap.sh

@@ -28,10 +28,10 @@ if [ -z "$TAG" ]; then
 		# This machine is running Ubuntu 18.04, which is supported by
 		# Mail-in-a-Box versions 0.40 through 5x.
 		echo "Support is ending for Ubuntu 18.04."
-		echo "Please immediately begin to migrate your information to"
+		echo "Please immediately begin to migrate your data to"
 		echo "a new machine running Ubuntu 22.04. See:"
 		echo "https://mailinabox.email/maintenance.html#upgrade"
-		TAG=v56
+		TAG=v57
 	elif [ "$UBUNTU_VERSION" == "Ubuntu 14.04 LTS" ]; then
 		# This machine is running Ubuntu 14.04, which is supported by
 		# Mail-in-a-Box versions 1 through v0.30.

setup/functions.sh

@@ -5,6 +5,8 @@
 # -o pipefail: don't ignore errors in the non-last command in a pipeline
 set -euo pipefail
 
+PHP_VER=8.0
+
 function hide_output {
 	# This function hides the output of a command unless the command fails
 	# and returns a non-zero exit code.

setup/management.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 source setup/functions.sh
+source /etc/mailinabox.conf # load global vars
 
 echo "Installing Mail-in-a-Box system management daemon..."
 
@@ -51,7 +52,8 @@ hide_output $venv/bin/python3 -m pip install --upgrade \
 	rtyaml "email_validator>=1.0.0" "exclusiveprocess" \
 	flask dnspython python-dateutil expiringdict \
 	qrcode[pil] pyotp \
-	"idna>=2.0.0" "cryptography==2.2.2" boto psutil postfix-mta-sts-resolver b2sdk ldap3
+	"idna>=2.0.0" "cryptography==37.0.2" psutil postfix-mta-sts-resolver \
+	b2sdk boto ldap3
 
 # CONFIGURATION
 

setup/nextcloud.sh

@@ -29,8 +29,8 @@ echo "Installing Nextcloud (contacts/calendar)..."
 #   we automatically install intermediate versions as needed.
 # * The hash is the SHA1 hash of the ZIP package, which you can find by just running this script and
 #   copying it from the error message when it doesn't match what is below.
-nextcloud_ver=24.0.0
+nextcloud_ver=23.0.0
-nextcloud_hash=f072f5863a15cefe577b47f72bb3e41d2a339335
+nextcloud_hash=0d496eb0808c292502479e93cd37fe2daf95786a
 
 # Nextcloud apps
 # --------------
@@ -42,20 +42,20 @@ nextcloud_hash=f072f5863a15cefe577b47f72bb3e41d2a339335
 # * The hash is the SHA1 hash of the ZIP package, which you can find by just running this script and
 #   copying it from the error message when it doesn't match what is below.
 contacts_ver=4.0.7
-contacts_hash=8ab31d205408e4f12067d8a4daa3595d46b513e3
+contacts_hash=45e7cf4bfe99cd8d03625cf9e5a1bb2e90549136
 calendar_ver=3.0.4
-calendar_hash=6fb1e998d307c53245faf1c37a96eb982bbee8ba
+calendar_hash=d0284b68135777ec9ca713c307216165b294d0fe
-user_external_ver=1.0.0
+user_external_ver=2.1.0
-user_external_hash=3bf2609061d7214e7f0f69dd8883e55c4ec8f50a
+user_external_hash=6e5afe7f36f398f864bfdce9cad72200e70322aa
 
 # Clear prior packages and install dependencies from apt.
 
 apt-get purge -qq -y owncloud* # we used to use the package manager
 
-apt_install php php-fpm \
+apt_install curl php${PHP_VER} php${PHP_VER}-fpm \
-	php-cli php-sqlite3 php-gd php-imap php-curl php-pear curl \
+	php${PHP_VER}-cli php${PHP_VER}-sqlite3 php${PHP_VER}-gd php${PHP_VER}-imap php${PHP_VER}-curl \
-	php-dev php-gd php-xml php-mbstring php-zip php-apcu php-json \
+	php${PHP_VER}-dev php${PHP_VER}-gd php${PHP_VER}-xml php${PHP_VER}-mbstring php${PHP_VER}-zip php${PHP_VER}-apcu \
-	php-intl php-imagick php-gmp php-bcmath
+	php${PHP_VER}-intl php${PHP_VER}-imagick php${PHP_VER}-gmp php${PHP_VER}-bcmath
 
 InstallNextcloud() {
 
@@ -90,11 +90,11 @@ InstallNextcloud() {
 	# their github repositories.
 	mkdir -p /usr/local/lib/owncloud/apps
 
-	wget_verify https://github.com/nextcloud-releases/contacts/releases/download/v$version_contacts/contacts-v$version_contacts.tar.gz $hash_contacts /tmp/contacts.tgz
+	wget_verify https://github.com/nextcloud-releases/contacts/archive/refs/tags/v$version_contacts.tar.gz $hash_contacts /tmp/contacts.tgz
 	tar xf /tmp/contacts.tgz -C /usr/local/lib/owncloud/apps/
 	rm /tmp/contacts.tgz
 
-	wget_verify https://github.com/nextcloud-releases/calendar/releases/download/v$version_calendar/calendar-v$version_calendar.tar.gz $hash_calendar /tmp/calendar.tgz
+	wget_verify https://github.com/nextcloud-releases/calendar/archive/refs/tags/v$version_calendar.tar.gz $hash_calendar /tmp/calendar.tgz
 	tar xf /tmp/calendar.tgz -C /usr/local/lib/owncloud/apps/
 	rm /tmp/calendar.tgz
 
@@ -123,20 +123,20 @@ InstallNextcloud() {
 	if [ -e $STORAGE_ROOT/owncloud/owncloud.db ]; then
 		# ownCloud 8.1.1 broke upgrades. It may fail on the first attempt, but
 		# that can be OK.
-		sudo -u www-data php /usr/local/lib/owncloud/occ upgrade
+		sudo -u www-data php$PHP_VER /usr/local/lib/owncloud/occ upgrade
 		if [ \( $? -ne 0 \) -a \( $? -ne 3 \) ]; then
 			echo "Trying ownCloud upgrade again to work around ownCloud upgrade bug..."
-			sudo -u www-data php /usr/local/lib/owncloud/occ upgrade
+			sudo -u www-data php$PHP_VER /usr/local/lib/owncloud/occ upgrade
 			if [ \( $? -ne 0 \) -a \( $? -ne 3 \) ]; then exit 1; fi
-			sudo -u www-data php /usr/local/lib/owncloud/occ maintenance:mode --off
+			sudo -u www-data php$PHP_VER /usr/local/lib/owncloud/occ maintenance:mode --off
 			echo "...which seemed to work."
 		fi
 
 		# Add missing indices. NextCloud didn't include this in the normal upgrade because it might take some time.
-		sudo -u www-data php /usr/local/lib/owncloud/occ db:add-missing-indices
+		sudo -u www-data php$PHP_VER /usr/local/lib/owncloud/occ db:add-missing-indices
 
 		# Run conversion to BigInt identifiers, this process may take some time on large tables.
-		sudo -u www-data php /usr/local/lib/owncloud/occ db:convert-filecache-bigint --no-interaction
+		sudo -u www-data php$PHP_VER /usr/local/lib/owncloud/occ db:convert-filecache-bigint --no-interaction
 	fi
 }
 
@@ -152,7 +152,7 @@ if [ -d "/usr/local/lib/owncloud" ]; then
 fi
 # If config.php exists, get version number, otherwise CURRENT_NEXTCLOUD_VER is empty.
 if [ -f "$STORAGE_ROOT/owncloud/config.php" ]; then
-	CURRENT_NEXTCLOUD_VER=$(php -r "include(\"$STORAGE_ROOT/owncloud/config.php\"); echo(\$CONFIG['version']);")
+	CURRENT_NEXTCLOUD_VER=$(php$PHP_VER -r "include(\"$STORAGE_ROOT/owncloud/config.php\"); echo(\$CONFIG['version']);")
 else
 	CURRENT_NEXTCLOUD_VER=""
 fi
@@ -162,7 +162,7 @@ fi
 if [ ! -d /usr/local/lib/owncloud/ ] || [[ ! ${CURRENT_NEXTCLOUD_VER} =~ ^$nextcloud_ver ]]; then
 
 	# Stop php-fpm if running. If they are not running (which happens on a previously failed install), dont bail.
-	service php8.1-fpm stop &> /dev/null || /bin/true
+	service php$PHP_VER-fpm stop &> /dev/null || /bin/true
 
 	# Backup the existing ownCloud/Nextcloud.
 	# Create a backup directory to store the current installation and database to
@@ -192,20 +192,20 @@ if [ ! -d /usr/local/lib/owncloud/ ] || [[ ! ${CURRENT_NEXTCLOUD_VER} =~ ^$nextc
 		elif [[ ${CURRENT_NEXTCLOUD_VER} =~ ^1[3456789] ]]; then
 			echo "Upgrades from Mail-in-a-Box prior to v60 with Nextcloud 19 or earlier are not supported. Upgrade to the latest Mail-in-a-Box version supported on your machine first. Setup will continue, but skip the Nextcloud migration."
 			return 0
-    elif [[ ${CURRENT_NEXTCLOUD_VER} =~ ^20 ]]; then
+		elif [[ ${CURRENT_NEXTCLOUD_VER} =~ ^20 ]]; then
-      InstallNextcloud 21.0.7 f5c7079c5b56ce1e301c6a27c0d975d608bb01c9 4.0.7 8ab31d205408e4f12067d8a4daa3595d46b513e3 3.0.4 6fb1e998d307c53245faf1c37a96eb982bbee8ba 1.0.0 3bf2609061d7214e7f0f69dd8883e55c4ec8f50a
+			InstallNextcloud 21.0.7 f5c7079c5b56ce1e301c6a27c0d975d608bb01c9 4.0.7 8ab31d205408e4f12067d8a4daa3595d46b513e3 3.0.4 6fb1e998d307c53245faf1c37a96eb982bbee8ba 1.0.0 3bf2609061d7214e7f0f69dd8883e55c4ec8f50a
-      CURRENT_NEXTCLOUD_VER="21.0.7"
+			CURRENT_NEXTCLOUD_VER="21.0.7"
-    elif [[ ${CURRENT_NEXTCLOUD_VER} =~ ^21 ]]; then
+		elif [[ ${CURRENT_NEXTCLOUD_VER} =~ ^21 ]]; then
-      InstallNextcloud 22.2.2 489eaf4147ad1b59385847b7d7db293712cced88 4.0.7 8ab31d205408e4f12067d8a4daa3595d46b513e3 3.0.4 6fb1e998d307c53245faf1c37a96eb982bbee8ba 1.0.0 3bf2609061d7214e7f0f69dd8883e55c4ec8f50a
+			InstallNextcloud 22.2.2 489eaf4147ad1b59385847b7d7db293712cced88 4.0.7 8ab31d205408e4f12067d8a4daa3595d46b513e3 3.0.4 6fb1e998d307c53245faf1c37a96eb982bbee8ba 1.0.0 3bf2609061d7214e7f0f69dd8883e55c4ec8f50a
-      CURRENT_NEXTCLOUD_VER="22.2.2"
+			CURRENT_NEXTCLOUD_VER="22.2.2"
-    fi
+		fi
 	fi
 
 	InstallNextcloud $nextcloud_ver $nextcloud_hash $contacts_ver $contacts_hash $calendar_ver $calendar_hash $user_external_ver $user_external_hash
 fi
 
 # ### Configure apcu
-cat > /etc/php/8.1/cli/conf.d/20-miab.ini <<EOF
+cat > /etc/php/$PHP_VER/cli/conf.d/20-miab.ini <<EOF
 apc.enable_cli=1
 EOF
 
@@ -277,7 +277,7 @@ EOF
 	# Execute Nextcloud's setup step, which creates the Nextcloud sqlite database.
 	# It also wipes it if it exists. And it updates config.php with database
 	# settings and deletes the autoconfig.php file.
-	(cd /usr/local/lib/owncloud; sudo -u www-data php /usr/local/lib/owncloud/index.php;)
+	(cd /usr/local/lib/owncloud; sudo -u www-data php$PHP_VER /usr/local/lib/owncloud/index.php;)
 fi
 
 # Update config.php.
@@ -293,7 +293,7 @@ fi
 # Use PHP to read the settings file, modify it, and write out the new settings array.
 TIMEZONE=$(cat /etc/timezone)
 CONFIG_TEMP=$(/bin/mktemp)
-php <<EOF > $CONFIG_TEMP && mv $CONFIG_TEMP $STORAGE_ROOT/owncloud/config.php;
+php$PHP_VER <<EOF > $CONFIG_TEMP && mv $CONFIG_TEMP $STORAGE_ROOT/owncloud/config.php;
 <?php
 include("$STORAGE_ROOT/owncloud/config.php");
 
@@ -323,25 +323,25 @@ chown www-data.www-data $STORAGE_ROOT/owncloud/config.php
 # The firstrunwizard gave Josh all sorts of problems, so disabling that.
 # user_external is what allows Nextcloud to use IMAP for login. The contacts
 # and calendar apps are the extensions we really care about here.
-hide_output sudo -u www-data php /usr/local/lib/owncloud/console.php app:disable firstrunwizard
+hide_output sudo -u www-data php$PHP_VER /usr/local/lib/owncloud/console.php app:disable firstrunwizard
-hide_output sudo -u www-data php /usr/local/lib/owncloud/console.php app:enable user_external
+hide_output sudo -u www-data php$PHP_VER /usr/local/lib/owncloud/console.php app:enable user_external
-hide_output sudo -u www-data php /usr/local/lib/owncloud/console.php app:enable contacts
+hide_output sudo -u www-data php$PHP_VER /usr/local/lib/owncloud/console.php app:enable contacts
-hide_output sudo -u www-data php /usr/local/lib/owncloud/console.php app:enable calendar
+hide_output sudo -u www-data php$PHP_VER /usr/local/lib/owncloud/console.php app:enable calendar
 
 # When upgrading, run the upgrade script again now that apps are enabled. It seems like
 # the first upgrade at the top won't work because apps may be disabled during upgrade?
 # Check for success (0=ok, 3=no upgrade needed).
-sudo -u www-data php /usr/local/lib/owncloud/occ upgrade
+sudo -u www-data php$PHP_VER /usr/local/lib/owncloud/occ upgrade
 if [ \( $? -ne 0 \) -a \( $? -ne 3 \) ]; then exit 1; fi
 
 # Disable default apps that we don't support
 sudo -u www-data \
-	php /usr/local/lib/owncloud/occ app:disable photos dashboard activity \
+	php$PHP_VER /usr/local/lib/owncloud/occ app:disable photos dashboard activity \
 	| (grep -v "No such app enabled" || /bin/true)
 
 # Set PHP FPM values to support large file uploads
 # (semicolon is the comment character in this file, hashes produce deprecation warnings)
-tools/editconf.py /etc/php/8.1/fpm/php.ini -c ';' \
+tools/editconf.py /etc/php/$PHP_VER/fpm/php.ini -c ';' \
 	upload_max_filesize=16G \
 	post_max_size=16G \
 	output_buffering=16384 \
@@ -350,7 +350,7 @@ tools/editconf.py /etc/php/8.1/fpm/php.ini -c ';' \
 	short_open_tag=On
 
 # Set Nextcloud recommended opcache settings
-tools/editconf.py /etc/php/8.1/cli/conf.d/10-opcache.ini -c ';' \
+tools/editconf.py /etc/php/$PHP_VER/cli/conf.d/10-opcache.ini -c ';' \
 	opcache.enable=1 \
 	opcache.enable_cli=1 \
 	opcache.interned_strings_buffer=8 \
@@ -360,8 +360,8 @@ tools/editconf.py /etc/php/8.1/cli/conf.d/10-opcache.ini -c ';' \
 	opcache.revalidate_freq=1
 
 # If apc is explicitly disabled we need to enable it
-if grep -q apc.enabled=0 /etc/php/8.1/mods-available/apcu.ini; then
+if grep -q apc.enabled=0 /etc/php/$PHP_VER/mods-available/apcu.ini; then
-	tools/editconf.py /etc/php/8.1/mods-available/apcu.ini -c ';' \
+	tools/editconf.py /etc/php/$PHP_VER/mods-available/apcu.ini -c ';' \
 		apc.enabled=1
 fi
 
@@ -369,7 +369,7 @@ fi
 cat > /etc/cron.d/mailinabox-nextcloud << EOF;
 #!/bin/bash
 # Mail-in-a-Box
-*/5 * * * *	root	sudo -u www-data php -f /usr/local/lib/owncloud/cron.php
+*/5 * * * *	root	sudo -u www-data php$PHP_VER -f /usr/local/lib/owncloud/cron.php
 EOF
 chmod +x /etc/cron.d/mailinabox-nextcloud
 
@@ -386,4 +386,4 @@ rm -f /etc/cron.hourly/mailinabox-owncloud
 # ```
 
 # Enable PHP modules and restart PHP.
-restart_service php8.1-fpm
+restart_service php$PHP_VER-fpm

setup/system.sh

@@ -97,6 +97,13 @@ fi
 # come from there and minimal Ubuntu installs may have it turned off.
 hide_output add-apt-repository -y universe
 
+# Install the duplicity PPA.
+hide_output add-apt-repository -y ppa:duplicity-team/duplicity-release-git
+
+# Stock PHP is now 8.1, but we're transitioning through 8.0 because
+# of Nextcloud.
+hide_output add-apt-repository --y ppa:ondrej/php
+
 # ### Update Packages
 
 # Update system packages to make sure we have the latest upstream versions
@@ -356,6 +363,7 @@ systemctl restart systemd-resolved
 rm -f /etc/fail2ban/jail.local # we used to use this file but don't anymore
 rm -f /etc/fail2ban/jail.d/defaults-debian.conf # removes default config so we can manage all of fail2ban rules in one config
 cat conf/fail2ban/jails.conf \
+    | sed "s/PUBLIC_IPV6/$PUBLIC_IPV6/g" \
 	| sed "s/PUBLIC_IP/$PUBLIC_IP/g" \
 	| sed "s#STORAGE_ROOT#$STORAGE_ROOT#" \
 	> /etc/fail2ban/jail.d/mailinabox.conf

setup/web.sh

@@ -19,7 +19,7 @@ fi
 
 echo "Installing Nginx (web server)..."
 
-apt_install nginx php-cli php-fpm idn2
+apt_install nginx php${PHP_VER}-cli php${PHP_VER}-fpm idn2
 
 rm -f /etc/nginx/sites-enabled/default
 
@@ -46,15 +46,15 @@ tools/editconf.py /etc/nginx/nginx.conf -s \
 	ssl_protocols="TLSv1.2 TLSv1.3;"
 
 # Tell PHP not to expose its version number in the X-Powered-By header.
-tools/editconf.py /etc/php/8.1/fpm/php.ini -c ';' \
+tools/editconf.py /etc/php/$PHP_VER/fpm/php.ini -c ';' \
 	expose_php=Off
 
 # Set PHPs default charset to UTF-8, since we use it. See #367.
-tools/editconf.py /etc/php/8.1/fpm/php.ini -c ';' \
+tools/editconf.py /etc/php/$PHP_VER/fpm/php.ini -c ';' \
         default_charset="UTF-8"
 
 # Configure the path environment for php-fpm
-tools/editconf.py /etc/php/8.1/fpm/pool.d/www.conf -c ';' \
+tools/editconf.py /etc/php/$PHP_VER/fpm/pool.d/www.conf -c ';' \
 	env[PATH]=/usr/local/bin:/usr/bin:/bin \
 
 # Configure php-fpm based on the amount of memory the machine has
@@ -64,7 +64,7 @@ tools/editconf.py /etc/php/8.1/fpm/pool.d/www.conf -c ';' \
 TOTAL_PHYSICAL_MEM=$(head -n 1 /proc/meminfo | awk '{print $2}' || /bin/true)
 if [ $TOTAL_PHYSICAL_MEM -lt 1000000 ]
 then
-        tools/editconf.py /etc/php/8.1/fpm/pool.d/www.conf -c ';' \
+        tools/editconf.py /etc/php/$PHP_VER/fpm/pool.d/www.conf -c ';' \
                 pm=ondemand \
                 pm.max_children=8 \
                 pm.start_servers=2 \
@@ -72,7 +72,7 @@ then
                 pm.max_spare_servers=3
 elif [ $TOTAL_PHYSICAL_MEM -lt 2000000 ]
 then
-        tools/editconf.py /etc/php/8.1/fpm/pool.d/www.conf -c ';' \
+        tools/editconf.py /etc/php/$PHP_VER/fpm/pool.d/www.conf -c ';' \
                 pm=ondemand \
                 pm.max_children=16 \
                 pm.start_servers=4 \
@@ -80,14 +80,14 @@ then
                 pm.max_spare_servers=6
 elif [ $TOTAL_PHYSICAL_MEM -lt 3000000 ]
 then
-        tools/editconf.py /etc/php/8.1/fpm/pool.d/www.conf -c ';' \
+        tools/editconf.py /etc/php/$PHP_VER/fpm/pool.d/www.conf -c ';' \
                 pm=dynamic \
                 pm.max_children=60 \
                 pm.start_servers=6 \
                 pm.min_spare_servers=3 \
                 pm.max_spare_servers=9
 else
-        tools/editconf.py /etc/php/8.1/fpm/pool.d/www.conf -c ';' \
+        tools/editconf.py /etc/php/$PHP_VER/fpm/pool.d/www.conf -c ';' \
                 pm=dynamic \
                 pm.max_children=120 \
                 pm.start_servers=12 \
@@ -147,7 +147,7 @@ chown -R $STORAGE_USER $STORAGE_ROOT/www
 
 # Start services.
 restart_service nginx
-restart_service php8.1-fpm
+restart_service php$PHP_VER-fpm
 
 # Open ports.
 ufw_allow http

setup/webmail.sh

@@ -23,8 +23,10 @@ source ${STORAGE_ROOT}/ldap/miab_ldap.conf
 echo "Installing Roundcube (webmail)..."
 apt_install \
 	dbconfig-common \
-	php-cli php-sqlite3 php-intl php-json php-common php-curl php-ldap \
+	php${PHP_VER}-cli php${PHP_VER}-sqlite3 php${PHP_VER}-intl php${PHP_VER}-common php${PHP_VER}-curl php${PHP_VER}-imap \
-	php-gd php-pspell libjs-jquery libjs-jquery-mousewheel libmagic1 php-mbstring
+	php${PHP_VER}-gd php${PHP_VER}-pspell php${PHP_VER}-mbstring libjs-jquery libjs-jquery-mousewheel libmagic1
+
+apt_install php${PHP_VER}-ldap
 
 # Install Roundcube from source if it is not already present or if it is out of date.
 # Combine the Roundcube version number with the commit hash of plugins to track
@@ -231,10 +233,10 @@ chown -f -R root.www-data ${RCM_PLUGIN_DIR}/carddav
 chmod -R 774 ${RCM_PLUGIN_DIR}/carddav
 
 # Run Roundcube database migration script (database is created if it does not exist)
-${RCM_DIR}/bin/updatedb.sh --dir ${RCM_DIR}/SQL --package roundcube
+php$PHP_VER ${RCM_DIR}/bin/updatedb.sh --dir ${RCM_DIR}/SQL --package roundcube
 chown www-data:www-data $STORAGE_ROOT/mail/roundcube/roundcube.sqlite
 chmod 664 $STORAGE_ROOT/mail/roundcube/roundcube.sqlite
 
 # Enable PHP modules.
-phpenmod -v php mcrypt imap ldap
+phpenmod -v $PHP_VER imap ldap
-restart_service php8.1-fpm
+restart_service php$PHP_VER-fpm

setup/zpush.sh

@@ -17,9 +17,9 @@ source /etc/mailinabox.conf # load global vars
 
 echo "Installing Z-Push (Exchange/ActiveSync server)..."
 apt_install \
-	php-soap php-imap libawl-php php8.1-xml
+       php${PHP_VER}-soap php${PHP_VER}-imap libawl-php php$PHP_VER-xml
 
-phpenmod -v php imap
+phpenmod -v $PHP_VER imap
 
 # Copy Z-Push into place.
 VERSION=2.6.2
@@ -42,8 +42,6 @@ if [ $needs_update == 1 ]; then
 	rm -rf /tmp/z-push.zip /tmp/z-push
 
 	rm -f /usr/sbin/z-push-{admin,top}
-	ln -s /usr/local/lib/z-push/z-push-admin.php /usr/sbin/z-push-admin
-	ln -s /usr/local/lib/z-push/z-push-top.php /usr/sbin/z-push-top
 	echo $VERSION > /usr/local/lib/z-push/version
 fi
 
@@ -102,8 +100,8 @@ EOF
 
 # Restart service.
 
-restart_service php8.1-fpm
+restart_service php$PHP_VER-fpm
 
 # Fix states after upgrade
 
-hide_output z-push-admin -a fixstates
+hide_output php$PHP_VER /usr/local/lib/z-push/z-push-admin.php -a fixstates

tests/test_mail.py

@@ -159,7 +159,7 @@ def smtp_login(host, login, pw, port):
 	ipaddr = socket.gethostbyname(host) # IPv4 only!
 	reverse_ip = dns.reversename.from_address(ipaddr) # e.g. "1.0.0.127.in-addr.arpa."
 	try:
-		reverse_dns = dns.resolver.query(reverse_ip, 'PTR')[0].target.to_text(omit_final_dot=True) # => hostname
+		reverse_dns = dns.resolver.resolve(reverse_ip, 'PTR')[0].target.to_text(omit_final_dot=True) # => hostname
 	except dns.resolver.NXDOMAIN:
 		print("Reverse DNS lookup failed for %s. SMTP EHLO name check skipped." % ipaddr)
 		reverse_dns = None

tests/vagrant/preloaded/prepvm.sh

@@ -22,7 +22,6 @@ fi
 source tests/lib/system.sh
 source tests/lib/color-output.sh
 
-
 dry_run=true
 
 if [ "$1" == "--no-dry-run" ]; then
@@ -77,6 +76,11 @@ install_packages() {
         # don't install slapd - it requires user input
         pkgs="$(sed 's/slapd//g' <<< "$pkgs")"
 
+        # manually set PHP_VER if necessary
+        if grep "PHP_VER" <<<"$pkgs" >/dev/null; then
+            pkgs="$(sed "s/\${*PHP_VER}*/$PHP_VER/g" <<< "$pkgs")"
+        fi
+
         if [ $OS_MAJOR -ge 22 ]; then
             # don't install opendmarc on ubuntu 22 and higher - it requires
             # interactive user input
@@ -93,6 +97,26 @@ install_packages() {
     done
 }
 
+install_ppas() {
+    H1 "Add apt repositories"
+    grep 'hide_output add-apt-repository' setup/system.sh |
+        while read line; do
+            line=$(sed 's/^hide_output //' <<< "$line")
+            H2 "$line"
+            if ! $dry_run; then
+                exec_no_output $line
+            fi
+        done 
+}
+
+
+# install PPAs from sources
+install_ppas
+
+# obtain PHP_VER variable from sources
+PHP_VER=$(grep "^PHP_VER=" setup/functions.sh  | awk -F= '{ print $2 }')
+
+
 if ! $dry_run; then
     H1 "Upgrade system"
     H2 "apt update"

tools/owncloud-restore.sh

@@ -43,7 +43,7 @@ ln -sf $STORAGE_ROOT/owncloud/config.php /usr/local/lib/owncloud/config/config.p
 chown -f -R www-data.www-data $STORAGE_ROOT/owncloud /usr/local/lib/owncloud
 chown www-data.www-data $STORAGE_ROOT/owncloud/config.php
 
-sudo -u www-data php /usr/local/lib/owncloud/occ maintenance:mode --off
+sudo -u www-data php$PHP_VER /usr/local/lib/owncloud/occ maintenance:mode --off
 
 service php8.0-fpm start
 echo "Done"

tools/owncloud-unlockadmin.sh

@@ -20,4 +20,4 @@ echo
 echo Press enter to continue.
 read
 
-sudo -u www-data php /usr/local/lib/owncloud/occ group:adduser admin $ADMIN && echo Done.
+sudo -u www-data php$PHP_VER /usr/local/lib/owncloud/occ group:adduser admin $ADMIN && echo Done.