diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a57eac5..cd264718 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,24 @@ No features of Mail-in-a-Box have changed in this release, but with the newer ve In Development -------------- +Version 57 (June 12, 2022) +-------------------------- + +Setup: + +* Fixed issue upgrading from Mail-in-a-Box v0.40-v0.50 because of a changed URL that Nextcloud is downloaded from. + +Backups: + +* Fixed S3 backups which broke with duplicity 0.8.23. +* Fixed Backblaze backups which broke with latest b2sdk package by rolling back its version. + +Control panel: + +* Fixed spurious changes in system status checks messages by sorting DNSSEC DS records. +* Fixed fail2ban lockout over IPv6 from excessive loads of the system status checks. +* Fixed an incorrect IPv6 system status check message. + Version 56 (January 19, 2022) ----------------------------- diff --git a/management/backup.py b/management/backup.py index cd25b197..0f180664 100755 --- a/management/backup.py +++ b/management/backup.py @@ -17,11 +17,6 @@ from exclusiveprocess import Lock from utils import load_environment, shell, wait_for_service, fix_boto, get_php_version -rsync_ssh_options = [ - "--ssh-options= -i /root/.ssh/id_rsa_miab", - "--rsync-options= -e \"/usr/bin/ssh -oStrictHostKeyChecking=no -oBatchMode=yes -p 22 -i /root/.ssh/id_rsa_miab\"", -] - def backup_status(env): # If backups are disabled, return no status. config = get_backup_config(env) @@ -67,9 +62,9 @@ def backup_status(env): "--archive-dir", backup_cache_dir, "--gpg-options", "--cipher-algo=AES256", "--log-fd", "1", - config["target"], - ] + rsync_ssh_options, - get_env(env), + get_duplicity_target_url(config), + ] + get_duplicity_additional_args(env), + get_duplicity_env_vars(env), trap=True) if code != 0: # Command failed. This is likely due to an improperly configured remote @@ -198,7 +193,48 @@ def get_passphrase(env): return passphrase -def get_env(env): +def get_duplicity_target_url(config): + target = config["target"] + + if get_target_type(config) == "s3": + from urllib.parse import urlsplit, urlunsplit + target = list(urlsplit(target)) + + # Duplicity now defaults to boto3 as the backend for S3, but we have + # legacy boto installed (boto3 doesn't support Ubuntu 18.04) so + # we retarget for classic boto. + target[0] = "boto+" + target[0] + + # In addition, although we store the S3 hostname in the target URL, + # duplicity no longer accepts it in the target URL. The hostname in + # the target URL must be the bucket name. The hostname is passed + # via get_duplicity_additional_args. Move the first part of the + # path (the bucket name) into the hostname URL component, and leave + # the rest for the path. + target[1], target[2] = target[2].lstrip('/').split('/', 1) + + target = urlunsplit(target) + + return target + +def get_duplicity_additional_args(env): + config = get_backup_config(env) + + if get_target_type(config) == 'rsync': + return [ + "--ssh-options= -i /root/.ssh/id_rsa_miab", + "--rsync-options= -e \"/usr/bin/ssh -oStrictHostKeyChecking=no -oBatchMode=yes -p 22 -i /root/.ssh/id_rsa_miab\"", + ] + elif get_target_type(config) == 's3': + # See note about hostname in get_duplicity_target_url. + from urllib.parse import urlsplit, urlunsplit + target = urlsplit(config["target"]) + endpoint_url = urlunsplit(("https", target.netloc, '', '', '')) + return ["--s3-endpoint-url", endpoint_url] + + return [] + +def get_duplicity_env_vars(env): config = get_backup_config(env) env = { "PASSPHRASE" : get_passphrase(env) } @@ -277,10 +313,10 @@ def perform_backup(full_backup): "--volsize", "250", "--gpg-options", "--cipher-algo=AES256", env["STORAGE_ROOT"], - config["target"], + get_duplicity_target_url(config), "--allow-source-mismatch" - ] + rsync_ssh_options, - get_env(env)) + ] + get_duplicity_additional_args(env), + get_duplicity_env_vars(env)) finally: # Start services again. service_command("dovecot", "start", quit=False) @@ -296,9 +332,9 @@ def perform_backup(full_backup): "--verbosity", "error", "--archive-dir", backup_cache_dir, "--force", - config["target"] - ] + rsync_ssh_options, - get_env(env)) + get_duplicity_target_url(config) + ] + get_duplicity_additional_args(env), + get_duplicity_env_vars(env)) # From duplicity's manual: # "This should only be necessary after a duplicity session fails or is @@ -311,9 +347,9 @@ def perform_backup(full_backup): "--verbosity", "error", "--archive-dir", backup_cache_dir, "--force", - config["target"] - ] + rsync_ssh_options, - get_env(env)) + get_duplicity_target_url(config) + ] + get_duplicity_additional_args(env), + get_duplicity_env_vars(env)) # Change ownership of backups to the user-data user, so that the after-backup # script can access them. @@ -349,9 +385,9 @@ def run_duplicity_verification(): "--compare-data", "--archive-dir", backup_cache_dir, "--exclude", backup_root, - config["target"], + get_duplicity_target_url(config), env["STORAGE_ROOT"], - ] + rsync_ssh_options, get_env(env)) + ] + get_duplicity_additional_args(env), get_duplicity_env_vars(env)) def run_duplicity_restore(args): env = load_environment() @@ -362,9 +398,9 @@ def run_duplicity_restore(args): "/usr/bin/duplicity", "restore", "--archive-dir", backup_cache_dir, - config["target"], - ] + rsync_ssh_options + args, - get_env(env)) + get_duplicity_target_url(config), + ] + get_duplicity_additional_args(env) + args, + get_duplicity_env_vars(env)) def list_target_files(config): import urllib.parse diff --git a/management/status_checks.py b/management/status_checks.py index 5ae8cd05..2742dfbc 100755 --- a/management/status_checks.py +++ b/management/status_checks.py @@ -258,6 +258,18 @@ def check_free_disk_space(rounded_values, env, output): if rounded_values: disk_msg = "The disk has less than 15% free space." output.print_error(disk_msg) + # Check that there's only one duplicity cache. If there's more than one, + # it's probably no longer in use, and we can recommend clearing the cache + # to save space. The cache directory may not exist yet, which is OK. + backup_cache_path = os.path.join(env['STORAGE_ROOT'], 'backup/cache') + try: + backup_cache_count = len(os.listdir(backup_cache_path)) + except: + backup_cache_count = 0 + if backup_cache_count > 1: + output.print_warning("The backup cache directory {} has more than one backup target cache. Consider clearing this directory to save disk space." + .format(backup_cache_path)) + def check_free_memory(rounded_values, env, output): # Check free memory. percent_free = 100 - psutil.virtual_memory().percent diff --git a/setup/management.sh b/setup/management.sh index 7961aecb..73fac5aa 100755 --- a/setup/management.sh +++ b/setup/management.sh @@ -30,7 +30,7 @@ apt_install duplicity python3-pip virtualenv certbot rsync # b2sdk is used for backblaze backups. # boto is used for amazon aws backups. # Both are installed outside the pipenv, so they can be used by duplicity -hide_output pip3 install --upgrade b2sdk boto +hide_output pip3 install --upgrade b2sdk==1.14.1 boto # Create a virtualenv for the installation of Python 3 packages # used by the management daemon.