merge upstream v57

CHANGELOG.md

@@ -24,6 +24,24 @@ No features of Mail-in-a-Box have changed in this release, but with the newer ve
 In Development
 --------------
 
+Version 57 (June 12, 2022)
+--------------------------
+
+Setup:
+
+* Fixed issue upgrading from Mail-in-a-Box v0.40-v0.50 because of a changed URL that Nextcloud is downloaded from.
+
+Backups:
+
+* Fixed S3 backups which broke with duplicity 0.8.23.
+* Fixed Backblaze backups which broke with latest b2sdk package by rolling back its version.
+
+Control panel:
+
+* Fixed spurious changes in system status checks messages by sorting DNSSEC DS records.
+* Fixed fail2ban lockout over IPv6 from excessive loads of the system status checks.
+* Fixed an incorrect IPv6 system status check message.
+
 Version 56 (January 19, 2022)
 -----------------------------
 

management/backup.py

@@ -17,11 +17,6 @@ from exclusiveprocess import Lock
 
 from utils import load_environment, shell, wait_for_service, fix_boto, get_php_version
 
-rsync_ssh_options = [
-	"--ssh-options= -i /root/.ssh/id_rsa_miab",
-	"--rsync-options= -e \"/usr/bin/ssh -oStrictHostKeyChecking=no -oBatchMode=yes -p 22 -i /root/.ssh/id_rsa_miab\"",
-]
-
 def backup_status(env):
 	# If backups are disabled, return no status.
 	config = get_backup_config(env)
@@ -67,9 +62,9 @@ def backup_status(env):
 		"--archive-dir", backup_cache_dir,
 		"--gpg-options", "--cipher-algo=AES256",
 		"--log-fd", "1",
-		config["target"],
+		get_duplicity_target_url(config),
-		] + rsync_ssh_options,
+		] + get_duplicity_additional_args(env),
-		get_env(env),
+		get_duplicity_env_vars(env),
 		trap=True)
 	if code != 0:
 		# Command failed. This is likely due to an improperly configured remote
@@ -198,7 +193,48 @@ def get_passphrase(env):
 
 	return passphrase
 
-def get_env(env):
+def get_duplicity_target_url(config):
+	target = config["target"]
+
+	if get_target_type(config) == "s3":
+		from urllib.parse import urlsplit, urlunsplit
+		target = list(urlsplit(target))
+
+		# Duplicity now defaults to boto3 as the backend for S3, but we have
+		# legacy boto installed (boto3 doesn't support Ubuntu 18.04) so
+		# we retarget for classic boto.
+		target[0] = "boto+" + target[0]
+
+		# In addition, although we store the S3 hostname in the target URL,
+		# duplicity no longer accepts it in the target URL. The hostname in
+		# the target URL must be the bucket name. The hostname is passed
+		# via get_duplicity_additional_args. Move the first part of the
+		# path (the bucket name) into the hostname URL component, and leave
+		# the rest for the path.
+		target[1], target[2] = target[2].lstrip('/').split('/', 1)
+
+		target = urlunsplit(target)
+
+	return target
+
+def get_duplicity_additional_args(env):
+	config = get_backup_config(env)
+
+	if get_target_type(config) == 'rsync':
+		return [
+			"--ssh-options= -i /root/.ssh/id_rsa_miab",
+			"--rsync-options= -e \"/usr/bin/ssh -oStrictHostKeyChecking=no -oBatchMode=yes -p 22 -i /root/.ssh/id_rsa_miab\"",
+		]
+	elif get_target_type(config) == 's3':
+		# See note about hostname in get_duplicity_target_url.
+		from urllib.parse import urlsplit, urlunsplit
+		target = urlsplit(config["target"])
+		endpoint_url = urlunsplit(("https", target.netloc, '', '', ''))
+		return ["--s3-endpoint-url",  endpoint_url]
+
+	return []
+
+def get_duplicity_env_vars(env):
 	config = get_backup_config(env)
 
 	env = { "PASSPHRASE" : get_passphrase(env) }
@@ -277,10 +313,10 @@ def perform_backup(full_backup):
 			"--volsize", "250",
 			"--gpg-options", "--cipher-algo=AES256",
 			env["STORAGE_ROOT"],
-			config["target"],
+			get_duplicity_target_url(config),
 			"--allow-source-mismatch"
-			] + rsync_ssh_options,
+			] + get_duplicity_additional_args(env),
-			get_env(env))
+			get_duplicity_env_vars(env))
 	finally:
 		# Start services again.
 		service_command("dovecot", "start", quit=False)
@@ -296,9 +332,9 @@ def perform_backup(full_backup):
 		"--verbosity", "error",
 		"--archive-dir", backup_cache_dir,
 		"--force",
-		config["target"]
+		get_duplicity_target_url(config)
-		] + rsync_ssh_options,
+		] + get_duplicity_additional_args(env),
-		get_env(env))
+		get_duplicity_env_vars(env))
 
 	# From duplicity's manual:
 	# "This should only be necessary after a duplicity session fails or is
@@ -311,9 +347,9 @@ def perform_backup(full_backup):
 		"--verbosity", "error",
 		"--archive-dir", backup_cache_dir,
 		"--force",
-		config["target"]
+		get_duplicity_target_url(config)
-		] + rsync_ssh_options,
+		] + get_duplicity_additional_args(env),
-		get_env(env))
+		get_duplicity_env_vars(env))
 
 	# Change ownership of backups to the user-data user, so that the after-backup
 	# script can access them.
@@ -349,9 +385,9 @@ def run_duplicity_verification():
 		"--compare-data",
 		"--archive-dir", backup_cache_dir,
 		"--exclude", backup_root,
-		config["target"],
+		get_duplicity_target_url(config),
 		env["STORAGE_ROOT"],
-	] + rsync_ssh_options, get_env(env))
+	] + get_duplicity_additional_args(env), get_duplicity_env_vars(env))
 
 def run_duplicity_restore(args):
 	env = load_environment()
@@ -362,9 +398,9 @@ def run_duplicity_restore(args):
 		"/usr/bin/duplicity",
 		"restore",
 		"--archive-dir", backup_cache_dir,
-		config["target"],
+		get_duplicity_target_url(config),
-		] + rsync_ssh_options + args,
+		] + get_duplicity_additional_args(env) + args,
-	get_env(env))
+	get_duplicity_env_vars(env))
 
 def list_target_files(config):
 	import urllib.parse

management/status_checks.py

@@ -258,6 +258,18 @@ def check_free_disk_space(rounded_values, env, output):
 		if rounded_values: disk_msg = "The disk has less than 15% free space."
 		output.print_error(disk_msg)
 
+	# Check that there's only one duplicity cache. If there's more than one,
+	# it's probably no longer in use, and we can recommend clearing the cache
+	# to save space. The cache directory may not exist yet, which is OK.
+	backup_cache_path = os.path.join(env['STORAGE_ROOT'], 'backup/cache')
+	try:
+		backup_cache_count = len(os.listdir(backup_cache_path))
+	except:
+		backup_cache_count = 0
+	if backup_cache_count > 1:
+		output.print_warning("The backup cache directory {} has more than one backup target cache. Consider clearing this directory to save disk space."
+			.format(backup_cache_path))
+
 def check_free_memory(rounded_values, env, output):
 	# Check free memory.
 	percent_free = 100 - psutil.virtual_memory().percent

setup/management.sh

@@ -30,7 +30,7 @@ apt_install duplicity python3-pip virtualenv certbot rsync
 # b2sdk is used for backblaze backups.
 # boto is used for amazon aws backups.
 # Both are installed outside the pipenv, so they can be used by duplicity
-hide_output pip3 install --upgrade b2sdk boto
+hide_output pip3 install --upgrade b2sdk==1.14.1 boto
 
 # Create a virtualenv for the installation of Python 3 packages
 # used by the management daemon.