From 69d8fdef9915127f016eb6424322a149cdff25d7 Mon Sep 17 00:00:00 2001 From: m-picc <105937591+m-picc@users.noreply.github.com> Date: Sun, 5 Jun 2022 09:24:32 -0400 Subject: [PATCH 1/5] Specify b2sdk version 1.14.1 (#2125) pin b2sdk version to 1.14.1 to resolve exception that occurs when attempting to use backblaze backups. See https://github.com/mail-in-a-box/mailinabox/issues/2124 for details. --- setup/management.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/management.sh b/setup/management.sh index 8dc64f3b..40d0a7eb 100755 --- a/setup/management.sh +++ b/setup/management.sh @@ -30,7 +30,7 @@ apt_install duplicity python-pip virtualenv certbot rsync # b2sdk is used for backblaze backups. # boto is used for amazon aws backups. # Both are installed outside the pipenv, so they can be used by duplicity -hide_output pip3 install --upgrade b2sdk boto +hide_output pip3 install --upgrade b2sdk==1.14.1 boto # Create a virtualenv for the installation of Python 3 packages # used by the management daemon. From 9004bb6e8ecf4ef8062859693919215149c68c47 Mon Sep 17 00:00:00 2001 From: jbandholz <20779634+jbandholz@users.noreply.github.com> Date: Sun, 5 Jun 2022 09:40:54 -0400 Subject: [PATCH 2/5] Add IPV6 addresses to fail2ban ignoreip (#2069) Update jails.conf to include IPV6 localhost and external ip to ignoreip line. Update system.sh to include IPV6 address in replacement. See mail-in-a-box#2066 for details. --- conf/fail2ban/jails.conf | 2 +- setup/system.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/conf/fail2ban/jails.conf b/conf/fail2ban/jails.conf index ce957f41..c1514b45 100644 --- a/conf/fail2ban/jails.conf +++ b/conf/fail2ban/jails.conf @@ -5,7 +5,7 @@ # Whitelist our own IP addresses. 127.0.0.1/8 is the default. But our status checks # ping services over the public interface so we should whitelist that address of # ours too. The string is substituted during installation. -ignoreip = 127.0.0.1/8 PUBLIC_IP +ignoreip = 127.0.0.1/8 PUBLIC_IP ::1 PUBLIC_IPV6 [dovecot] enabled = true diff --git a/setup/system.sh b/setup/system.sh index 036fe3f9..9898cbcd 100755 --- a/setup/system.sh +++ b/setup/system.sh @@ -363,6 +363,7 @@ systemctl restart systemd-resolved rm -f /etc/fail2ban/jail.local # we used to use this file but don't anymore rm -f /etc/fail2ban/jail.d/defaults-debian.conf # removes default config so we can manage all of fail2ban rules in one config cat conf/fail2ban/jails.conf \ + | sed "s/PUBLIC_IPV6/$PUBLIC_IPV6/g" \ | sed "s/PUBLIC_IP/$PUBLIC_IP/g" \ | sed "s#STORAGE_ROOT#$STORAGE_ROOT#" \ > /etc/fail2ban/jail.d/mailinabox.conf From 8bebaf6a484a38aca199bdbe68e937abc6a1394d Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sat, 11 Jun 2022 09:23:58 -0400 Subject: [PATCH 3/5] Simplify duplicity command line by omitting rsync options if the backup target type is not rsync --- management/backup.py | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/management/backup.py b/management/backup.py index 0a8a021e..bc3ac5bd 100755 --- a/management/backup.py +++ b/management/backup.py @@ -14,11 +14,6 @@ from exclusiveprocess import Lock from utils import load_environment, shell, wait_for_service, fix_boto -rsync_ssh_options = [ - "--ssh-options= -i /root/.ssh/id_rsa_miab", - "--rsync-options= -e \"/usr/bin/ssh -oStrictHostKeyChecking=no -oBatchMode=yes -p 22 -i /root/.ssh/id_rsa_miab\"", -] - def backup_status(env): # If backups are dissbled, return no status. config = get_backup_config(env) @@ -65,8 +60,8 @@ def backup_status(env): "--gpg-options", "--cipher-algo=AES256", "--log-fd", "1", config["target"], - ] + rsync_ssh_options, - get_env(env), + ] + get_duplicity_additional_args(env), + get_duplicity_env_vars(env), trap=True) if code != 0: # Command failed. This is likely due to an improperly configured remote @@ -195,7 +190,16 @@ def get_passphrase(env): return passphrase -def get_env(env): +def get_duplicity_additional_args(env): + config = get_backup_config(env) + if get_target_type(config) == 'rsync': + return [ + "--ssh-options= -i /root/.ssh/id_rsa_miab", + "--rsync-options= -e \"/usr/bin/ssh -oStrictHostKeyChecking=no -oBatchMode=yes -p 22 -i /root/.ssh/id_rsa_miab\"", + ] + return [] + +def get_duplicity_env_vars(env): config = get_backup_config(env) env = { "PASSPHRASE" : get_passphrase(env) } @@ -275,8 +279,8 @@ def perform_backup(full_backup): env["STORAGE_ROOT"], config["target"], "--allow-source-mismatch" - ] + rsync_ssh_options, - get_env(env)) + ] + get_duplicity_additional_args(env), + get_duplicity_env_vars(env)) finally: # Start services again. service_command("dovecot", "start", quit=False) @@ -293,8 +297,8 @@ def perform_backup(full_backup): "--archive-dir", backup_cache_dir, "--force", config["target"] - ] + rsync_ssh_options, - get_env(env)) + ] + get_duplicity_additional_args(env), + get_duplicity_env_vars(env)) # From duplicity's manual: # "This should only be necessary after a duplicity session fails or is @@ -308,8 +312,8 @@ def perform_backup(full_backup): "--archive-dir", backup_cache_dir, "--force", config["target"] - ] + rsync_ssh_options, - get_env(env)) + ] + get_duplicity_additional_args(env), + get_duplicity_env_vars(env)) # Change ownership of backups to the user-data user, so that the after-bcakup # script can access them. @@ -347,7 +351,7 @@ def run_duplicity_verification(): "--exclude", backup_root, config["target"], env["STORAGE_ROOT"], - ] + rsync_ssh_options, get_env(env)) + ] + get_duplicity_additional_args(env), get_duplicity_env_vars(env)) def run_duplicity_restore(args): env = load_environment() @@ -358,8 +362,8 @@ def run_duplicity_restore(args): "restore", "--archive-dir", backup_cache_dir, config["target"], - ] + rsync_ssh_options + args, - get_env(env)) + ] + get_duplicity_additional_args(env) + args, + get_duplicity_env_vars(env)) def list_target_files(config): import urllib.parse From 99474b348f0dd2632057c18c6a8c4e6464962878 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sat, 11 Jun 2022 09:24:45 -0400 Subject: [PATCH 4/5] Update backup to be compatible with duplicity 0.8.23 We were using duplicity 0.8.21-ppa202111091602~ubuntu1 from the duplicity PPA probably until June 5, which is when my box automatically updated to 0.8.23-ppa202205151528~ubuntu18.04.1. Starting with that version, two changes broke backups: * The default s3 backend was changed to boto3. But boto3 depends on the AWS SDK which does not support Ubuntu 18.04, so we can't install it. Instead, we map s3: backup target URLs to the boto+s3 scheme which tells duplicity to use legacy boto. This should be reverted when we can switch to boto3. * Contrary to the documentation, the s3 target no longer accepts a S3 hostname in the URL. It now reads the bucket from the hostname part of the URL. So we now drop the hostname from our target URL before passing it to duplicity and we pass the endpoint URL in a separate command-line argument. (The boto backend was dropped from duplicity's "uses_netloc" in https://gitlab.com/duplicity/duplicity/-/commit/74d4cf44b1410652ae5f457f3eba7aaebf3b9d31#f5a07610d36bd242c3e5b98f8348879a468b866a_37_34, but other changes may be related.) The change of target URL (due to both changes) seems to also cause duplicity to store cached data in a different directory within $STORAGE_ROOT/backup/cache, so on the next backup it will re-download cached manifest/signature files. Since the cache directory will still hold the prior data which is no longer needed, it might be a good idea to clear out the cache directory to save space. A system status checks message is added about that. Fixes #2123 --- CHANGELOG.md | 5 +++++ management/backup.py | 44 ++++++++++++++++++++++++++++++++----- management/status_checks.py | 12 ++++++++++ 3 files changed, 55 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d796970e..520c05aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ CHANGELOG ========= +In Development +-------------- + +* Fixed S3 backups which broke with duplicity 0.8.23. + Version 56 (January 19, 2022) ----------------------------- diff --git a/management/backup.py b/management/backup.py index bc3ac5bd..26a61a0e 100755 --- a/management/backup.py +++ b/management/backup.py @@ -59,7 +59,7 @@ def backup_status(env): "--archive-dir", backup_cache_dir, "--gpg-options", "--cipher-algo=AES256", "--log-fd", "1", - config["target"], + get_duplicity_target_url(config), ] + get_duplicity_additional_args(env), get_duplicity_env_vars(env), trap=True) @@ -190,13 +190,45 @@ def get_passphrase(env): return passphrase +def get_duplicity_target_url(config): + target = config["target"] + + if get_target_type(config) == "s3": + from urllib.parse import urlsplit, urlunsplit + target = list(urlsplit(target)) + + # Duplicity now defaults to boto3 as the backend for S3, but we have + # legacy boto installed (boto3 doesn't support Ubuntu 18.04) so + # we retarget for classic boto. + target[0] = "boto+" + target[0] + + # In addition, although we store the S3 hostname in the target URL, + # duplicity no longer accepts it in the target URL. The hostname in + # the target URL must be the bucket name. The hostname is passed + # via get_duplicity_additional_args. Move the first part of the + # path (the bucket name) into the hostname URL component, and leave + # the rest for the path. + target[1], target[2] = target[2].lstrip('/').split('/', 1) + + target = urlunsplit(target) + + return target + def get_duplicity_additional_args(env): config = get_backup_config(env) + if get_target_type(config) == 'rsync': return [ "--ssh-options= -i /root/.ssh/id_rsa_miab", "--rsync-options= -e \"/usr/bin/ssh -oStrictHostKeyChecking=no -oBatchMode=yes -p 22 -i /root/.ssh/id_rsa_miab\"", ] + elif get_target_type(config) == 's3': + # See note about hostname in get_duplicity_target_url. + from urllib.parse import urlsplit, urlunsplit + target = urlsplit(config["target"]) + endpoint_url = urlunsplit(("https", target.netloc, '', '', '')) + return ["--s3-endpoint-url", endpoint_url] + return [] def get_duplicity_env_vars(env): @@ -277,7 +309,7 @@ def perform_backup(full_backup): "--volsize", "250", "--gpg-options", "--cipher-algo=AES256", env["STORAGE_ROOT"], - config["target"], + get_duplicity_target_url(config), "--allow-source-mismatch" ] + get_duplicity_additional_args(env), get_duplicity_env_vars(env)) @@ -296,7 +328,7 @@ def perform_backup(full_backup): "--verbosity", "error", "--archive-dir", backup_cache_dir, "--force", - config["target"] + get_duplicity_target_url(config) ] + get_duplicity_additional_args(env), get_duplicity_env_vars(env)) @@ -311,7 +343,7 @@ def perform_backup(full_backup): "--verbosity", "error", "--archive-dir", backup_cache_dir, "--force", - config["target"] + get_duplicity_target_url(config) ] + get_duplicity_additional_args(env), get_duplicity_env_vars(env)) @@ -349,7 +381,7 @@ def run_duplicity_verification(): "--compare-data", "--archive-dir", backup_cache_dir, "--exclude", backup_root, - config["target"], + get_duplicity_target_url(config), env["STORAGE_ROOT"], ] + get_duplicity_additional_args(env), get_duplicity_env_vars(env)) @@ -361,7 +393,7 @@ def run_duplicity_restore(args): "/usr/bin/duplicity", "restore", "--archive-dir", backup_cache_dir, - config["target"], + get_duplicity_target_url(config), ] + get_duplicity_additional_args(env) + args, get_duplicity_env_vars(env)) diff --git a/management/status_checks.py b/management/status_checks.py index c66bf9b3..220e23e4 100755 --- a/management/status_checks.py +++ b/management/status_checks.py @@ -253,6 +253,18 @@ def check_free_disk_space(rounded_values, env, output): if rounded_values: disk_msg = "The disk has less than 15% free space." output.print_error(disk_msg) + # Check that there's only one duplicity cache. If there's more than one, + # it's probably no longer in use, and we can recommend clearing the cache + # to save space. The cache directory may not exist yet, which is OK. + backup_cache_path = os.path.join(env['STORAGE_ROOT'], 'backup/cache') + try: + backup_cache_count = len(os.listdir(backup_cache_path)) + except: + backup_cache_count = 0 + if backup_cache_count > 1: + output.print_warning("The backup cache directory {} has more than one backup target cache. Consider clearing this directory to save disk space." + .format(backup_cache_path)) + def check_free_memory(rounded_values, env, output): # Check free memory. percent_free = 100 - psutil.virtual_memory().percent From 2aca421415f9f498c21a49588d1cff6ce3d8f2de Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sat, 29 Jan 2022 07:50:12 -0500 Subject: [PATCH 5/5] Version 57 --- CHANGELOG.md | 17 +++++++++++++++-- README.md | 2 +- setup/bootstrap.sh | 2 +- setup/nextcloud.sh | 2 +- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 520c05aa..fd8b65a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,23 @@ CHANGELOG ========= -In Development --------------- +Version 57 (June 12, 2022) +-------------------------- + +Setup: + +* Fixed issue upgrading from Mail-in-a-Box v0.40-v0.50 because of a changed URL that Nextcloud is downloaded from. + +Backups: * Fixed S3 backups which broke with duplicity 0.8.23. +* Fixed Backblaze backups which broke with latest b2sdk package by rolling back its version. + +Control panel: + +* Fixed spurious changes in system status checks messages by sorting DNSSEC DS records. +* Fixed fail2ban lockout over IPv6 from excessive loads of the system status checks. +* Fixed an incorrect IPv6 system status check message. Version 56 (January 19, 2022) ----------------------------- diff --git a/README.md b/README.md index d2271d17..02a274af 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ Clone this repository and checkout the tag corresponding to the most recent rele $ git clone https://github.com/mail-in-a-box/mailinabox $ cd mailinabox - $ git checkout v56 + $ git checkout v57 Begin the installation. diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index b90b1ac6..66294441 100644 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -20,7 +20,7 @@ if [ -z "$TAG" ]; then # want to display in status checks. if [ "$(lsb_release -d | sed 's/.*:\s*//' | sed 's/18\.04\.[0-9]/18.04/' )" == "Ubuntu 18.04 LTS" ]; then # This machine is running Ubuntu 18.04. - TAG=v56 + TAG=v57 elif [ "$(lsb_release -d | sed 's/.*:\s*//' | sed 's/14\.04\.[0-9]/14.04/' )" == "Ubuntu 14.04 LTS" ]; then # This machine is running Ubuntu 14.04. diff --git a/setup/nextcloud.sh b/setup/nextcloud.sh index 7116c172..5525a37a 100755 --- a/setup/nextcloud.sh +++ b/setup/nextcloud.sh @@ -28,7 +28,7 @@ nextcloud_hash=92cac708915f51ee2afc1787fd845476fd090c81 # -------------- # * Find the most recent tag that is compatible with the Nextcloud version above by # consulting the ... node at: -# https://github.com/nextcloud-releases/contacts/blob/maaster/appinfo/info.xml +# https://github.com/nextcloud-releases/contacts/blob/master/appinfo/info.xml # https://github.com/nextcloud-releases/calendar/blob/master/appinfo/info.xml # https://github.com/nextcloud/user_external/blob/master/appinfo/info.xml # * The hash is the SHA1 hash of the ZIP package, which you can find by just running this script and