diff --git a/CHANGELOG.md b/CHANGELOG.md index d796970e..5c20eb7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,29 @@ CHANGELOG ========= +Version 57a (June 19, 2022) +--------------------------- + +* The Backblaze backups fix posted in Version 57 was incomplete. It's now fixed. + +Version 57 (June 12, 2022) +-------------------------- + +Setup: + +* Fixed issue upgrading from Mail-in-a-Box v0.40-v0.50 because of a changed URL that Nextcloud is downloaded from. + +Backups: + +* Fixed S3 backups which broke with duplicity 0.8.23. +* Fixed Backblaze backups which broke with latest b2sdk package by rolling back its version. + +Control panel: + +* Fixed spurious changes in system status checks messages by sorting DNSSEC DS records. +* Fixed fail2ban lockout over IPv6 from excessive loads of the system status checks. +* Fixed an incorrect IPv6 system status check message. + Version 56 (January 19, 2022) ----------------------------- diff --git a/README.md b/README.md index aee77cce..b7a76cab 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Mailbox size recalculation by Dovecot can be forced using the command: Please report any bugs on github. -Installing v0.4x-quota +Installing v5x-quota ----------------------- To install the latest version, log into your box and execute the following commands: @@ -34,7 +34,7 @@ Follow the standard directions for setting up an MiaB installation. There are n The default quota is set to `0` which means unlimited. If you want to set a different default quota, follow the directions above. -Upgrading v0.4x to v.0.4x-quota +Upgrading v5x to v5x-quota -------------------------------- This is experimental software. You have been warned. @@ -71,6 +71,10 @@ Issues Changes ------- +### v57a-quota-0.22-beta + +* Update to v57a of Mail-in-a-Box + ### v56-quota * Update to v56 of Mail-in-a-Box @@ -266,7 +270,7 @@ Clone this repository and checkout the tag corresponding to the most recent rele $ git clone https://github.com/mail-in-a-box/mailinabox $ cd mailinabox - $ git checkout v56 + $ git checkout v57a Begin the installation. diff --git a/conf/fail2ban/jails.conf b/conf/fail2ban/jails.conf index ce957f41..c1514b45 100644 --- a/conf/fail2ban/jails.conf +++ b/conf/fail2ban/jails.conf @@ -5,7 +5,7 @@ # Whitelist our own IP addresses. 127.0.0.1/8 is the default. But our status checks # ping services over the public interface so we should whitelist that address of # ours too. The string is substituted during installation. -ignoreip = 127.0.0.1/8 PUBLIC_IP +ignoreip = 127.0.0.1/8 PUBLIC_IP ::1 PUBLIC_IPV6 [dovecot] enabled = true diff --git a/management/backup.py b/management/backup.py index 0a8a021e..26a61a0e 100755 --- a/management/backup.py +++ b/management/backup.py @@ -14,11 +14,6 @@ from exclusiveprocess import Lock from utils import load_environment, shell, wait_for_service, fix_boto -rsync_ssh_options = [ - "--ssh-options= -i /root/.ssh/id_rsa_miab", - "--rsync-options= -e \"/usr/bin/ssh -oStrictHostKeyChecking=no -oBatchMode=yes -p 22 -i /root/.ssh/id_rsa_miab\"", -] - def backup_status(env): # If backups are dissbled, return no status. config = get_backup_config(env) @@ -64,9 +59,9 @@ def backup_status(env): "--archive-dir", backup_cache_dir, "--gpg-options", "--cipher-algo=AES256", "--log-fd", "1", - config["target"], - ] + rsync_ssh_options, - get_env(env), + get_duplicity_target_url(config), + ] + get_duplicity_additional_args(env), + get_duplicity_env_vars(env), trap=True) if code != 0: # Command failed. This is likely due to an improperly configured remote @@ -195,7 +190,48 @@ def get_passphrase(env): return passphrase -def get_env(env): +def get_duplicity_target_url(config): + target = config["target"] + + if get_target_type(config) == "s3": + from urllib.parse import urlsplit, urlunsplit + target = list(urlsplit(target)) + + # Duplicity now defaults to boto3 as the backend for S3, but we have + # legacy boto installed (boto3 doesn't support Ubuntu 18.04) so + # we retarget for classic boto. + target[0] = "boto+" + target[0] + + # In addition, although we store the S3 hostname in the target URL, + # duplicity no longer accepts it in the target URL. The hostname in + # the target URL must be the bucket name. The hostname is passed + # via get_duplicity_additional_args. Move the first part of the + # path (the bucket name) into the hostname URL component, and leave + # the rest for the path. + target[1], target[2] = target[2].lstrip('/').split('/', 1) + + target = urlunsplit(target) + + return target + +def get_duplicity_additional_args(env): + config = get_backup_config(env) + + if get_target_type(config) == 'rsync': + return [ + "--ssh-options= -i /root/.ssh/id_rsa_miab", + "--rsync-options= -e \"/usr/bin/ssh -oStrictHostKeyChecking=no -oBatchMode=yes -p 22 -i /root/.ssh/id_rsa_miab\"", + ] + elif get_target_type(config) == 's3': + # See note about hostname in get_duplicity_target_url. + from urllib.parse import urlsplit, urlunsplit + target = urlsplit(config["target"]) + endpoint_url = urlunsplit(("https", target.netloc, '', '', '')) + return ["--s3-endpoint-url", endpoint_url] + + return [] + +def get_duplicity_env_vars(env): config = get_backup_config(env) env = { "PASSPHRASE" : get_passphrase(env) } @@ -273,10 +309,10 @@ def perform_backup(full_backup): "--volsize", "250", "--gpg-options", "--cipher-algo=AES256", env["STORAGE_ROOT"], - config["target"], + get_duplicity_target_url(config), "--allow-source-mismatch" - ] + rsync_ssh_options, - get_env(env)) + ] + get_duplicity_additional_args(env), + get_duplicity_env_vars(env)) finally: # Start services again. service_command("dovecot", "start", quit=False) @@ -292,9 +328,9 @@ def perform_backup(full_backup): "--verbosity", "error", "--archive-dir", backup_cache_dir, "--force", - config["target"] - ] + rsync_ssh_options, - get_env(env)) + get_duplicity_target_url(config) + ] + get_duplicity_additional_args(env), + get_duplicity_env_vars(env)) # From duplicity's manual: # "This should only be necessary after a duplicity session fails or is @@ -307,9 +343,9 @@ def perform_backup(full_backup): "--verbosity", "error", "--archive-dir", backup_cache_dir, "--force", - config["target"] - ] + rsync_ssh_options, - get_env(env)) + get_duplicity_target_url(config) + ] + get_duplicity_additional_args(env), + get_duplicity_env_vars(env)) # Change ownership of backups to the user-data user, so that the after-bcakup # script can access them. @@ -345,9 +381,9 @@ def run_duplicity_verification(): "--compare-data", "--archive-dir", backup_cache_dir, "--exclude", backup_root, - config["target"], + get_duplicity_target_url(config), env["STORAGE_ROOT"], - ] + rsync_ssh_options, get_env(env)) + ] + get_duplicity_additional_args(env), get_duplicity_env_vars(env)) def run_duplicity_restore(args): env = load_environment() @@ -357,9 +393,9 @@ def run_duplicity_restore(args): "/usr/bin/duplicity", "restore", "--archive-dir", backup_cache_dir, - config["target"], - ] + rsync_ssh_options + args, - get_env(env)) + get_duplicity_target_url(config), + ] + get_duplicity_additional_args(env) + args, + get_duplicity_env_vars(env)) def list_target_files(config): import urllib.parse diff --git a/management/status_checks.py b/management/status_checks.py index e0598c35..b4ee4f66 100755 --- a/management/status_checks.py +++ b/management/status_checks.py @@ -135,7 +135,7 @@ def check_service(i, service, env): # IPv4 ok but IPv6 failed. Try the PRIVATE_IPV6 address to see if the service is bound to the interface. elif service["port"] != 53 and try_connect(env["PRIVATE_IPV6"]): - output.print_error("%s is running (and available over IPv4 and the local IPv6 address), but it is not publicly accessible at %s:%d." % (service['name'], env['PUBLIC_IP'], service['port'])) + output.print_error("%s is running (and available over IPv4 and the local IPv6 address), but it is not publicly accessible at %s:%d." % (service['name'], env['PUBLIC_IPV6'], service['port'])) else: output.print_error("%s is running and available over IPv4 but is not accessible over IPv6 at %s port %d." % (service['name'], env['PUBLIC_IPV6'], service['port'])) @@ -253,6 +253,18 @@ def check_free_disk_space(rounded_values, env, output): if rounded_values: disk_msg = "The disk has less than 15% free space." output.print_error(disk_msg) + # Check that there's only one duplicity cache. If there's more than one, + # it's probably no longer in use, and we can recommend clearing the cache + # to save space. The cache directory may not exist yet, which is OK. + backup_cache_path = os.path.join(env['STORAGE_ROOT'], 'backup/cache') + try: + backup_cache_count = len(os.listdir(backup_cache_path)) + except: + backup_cache_count = 0 + if backup_cache_count > 1: + output.print_warning("The backup cache directory {} has more than one backup target cache. Consider clearing this directory to save disk space." + .format(backup_cache_path)) + def check_free_memory(rounded_values, env, output): # Check free memory. percent_free = 100 - psutil.virtual_memory().percent @@ -658,7 +670,7 @@ def check_dnssec(domain, env, output, dns_zonefiles, is_checking_primary=False): if len(ds) > 0: output.print_line("") output.print_line("The DS record is currently set to:") - for rr in ds: + for rr in sorted(ds): output.print_line("Key Tag: {0}, Algorithm: {1}, Digest Type: {2}, Digest: {3}".format(*rr)) def check_mail_domain(domain, env, output): diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index d0da1b25..b323ca88 100644 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -20,7 +20,7 @@ if [ -z "$TAG" ]; then # want to display in status checks. if [ "$(lsb_release -d | sed 's/.*:\s*//' | sed 's/18\.04\.[0-9]/18.04/' )" == "Ubuntu 18.04 LTS" ]; then # This machine is running Ubuntu 18.04. - TAG=v56-quota + TAG=v57a-quota-0.22-beta elif [ "$(lsb_release -d | sed 's/.*:\s*//' | sed 's/14\.04\.[0-9]/14.04/' )" == "Ubuntu 14.04 LTS" ]; then # This machine is running Ubuntu 14.04. diff --git a/setup/management.sh b/setup/management.sh index 8dc64f3b..046c5bac 100755 --- a/setup/management.sh +++ b/setup/management.sh @@ -30,7 +30,7 @@ apt_install duplicity python-pip virtualenv certbot rsync # b2sdk is used for backblaze backups. # boto is used for amazon aws backups. # Both are installed outside the pipenv, so they can be used by duplicity -hide_output pip3 install --upgrade b2sdk boto +hide_output pip3 install --upgrade b2sdk==1.14.1 boto # Create a virtualenv for the installation of Python 3 packages # used by the management daemon. @@ -51,7 +51,8 @@ hide_output $venv/bin/pip install --upgrade \ rtyaml "email_validator>=1.0.0" "exclusiveprocess" \ flask dnspython python-dateutil expiringdict \ qrcode[pil] pyotp \ - "idna>=2.0.0" "cryptography==2.2.2" boto psutil postfix-mta-sts-resolver b2sdk + "idna>=2.0.0" "cryptography==2.2.2" psutil postfix-mta-sts-resolver \ + b2sdk==1.14.1 boto # CONFIGURATION diff --git a/setup/nextcloud.sh b/setup/nextcloud.sh index d8ce7635..5525a37a 100755 --- a/setup/nextcloud.sh +++ b/setup/nextcloud.sh @@ -28,15 +28,15 @@ nextcloud_hash=92cac708915f51ee2afc1787fd845476fd090c81 # -------------- # * Find the most recent tag that is compatible with the Nextcloud version above by # consulting the ... node at: -# https://github.com/nextcloud-releases/contacts/blob/maaster/appinfo/info.xml +# https://github.com/nextcloud-releases/contacts/blob/master/appinfo/info.xml # https://github.com/nextcloud-releases/calendar/blob/master/appinfo/info.xml # https://github.com/nextcloud/user_external/blob/master/appinfo/info.xml # * The hash is the SHA1 hash of the ZIP package, which you can find by just running this script and # copying it from the error message when it doesn't match what is below. contacts_ver=4.0.7 -contacts_hash=8ab31d205408e4f12067d8a4daa3595d46b513e3 +contacts_hash=45e7cf4bfe99cd8d03625cf9e5a1bb2e90549136 calendar_ver=3.0.4 -calendar_hash=6fb1e998d307c53245faf1c37a96eb982bbee8ba +calendar_hash=d0284b68135777ec9ca713c307216165b294d0fe user_external_ver=1.0.0 user_external_hash=3bf2609061d7214e7f0f69dd8883e55c4ec8f50a @@ -79,11 +79,11 @@ InstallNextcloud() { # their github repositories. mkdir -p /usr/local/lib/owncloud/apps - wget_verify https://github.com/nextcloud-releases/contacts/releases/download/v$version_contacts/contacts-v$version_contacts.tar.gz $hash_contacts /tmp/contacts.tgz + wget_verify https://github.com/nextcloud-releases/contacts/archive/refs/tags/v$version_contacts.tar.gz $hash_contacts /tmp/contacts.tgz tar xf /tmp/contacts.tgz -C /usr/local/lib/owncloud/apps/ rm /tmp/contacts.tgz - wget_verify https://github.com/nextcloud-releases/calendar/releases/download/v$version_calendar/calendar-v$version_calendar.tar.gz $hash_calendar /tmp/calendar.tgz + wget_verify https://github.com/nextcloud-releases/calendar/archive/refs/tags/v$version_calendar.tar.gz $hash_calendar /tmp/calendar.tgz tar xf /tmp/calendar.tgz -C /usr/local/lib/owncloud/apps/ rm /tmp/calendar.tgz @@ -183,25 +183,25 @@ if [ ! -d /usr/local/lib/owncloud/ ] || [[ ! ${CURRENT_NEXTCLOUD_VER} =~ ^$nextc # During the upgrade from Nextcloud 14 to 15, user_external may cause the upgrade to fail. # We will disable it here before the upgrade and install it again after the upgrade. hide_output sudo -u www-data php /usr/local/lib/owncloud/console.php app:disable user_external - InstallNextcloud 15.0.8 4129d8d4021c435f2e86876225fb7f15adf764a3 3.3.0 e55d0357c6785d3b1f3b5f21780cb6d41d32443a 2.0.3 9d9717b29337613b72c74e9914c69b74b346c466 0.7.0 555a94811daaf5bdd336c5e48a78aa8567b86437 + InstallNextcloud 15.0.8 4129d8d4021c435f2e86876225fb7f15adf764a3 3.3.0 e55d0357c6785d3b1f3b5f21780cb6d41d32443a 2.0.3 a1f3835c752929e3598eb94f22300516867ac6ab 0.7.0 555a94811daaf5bdd336c5e48a78aa8567b86437 CURRENT_NEXTCLOUD_VER="15.0.8" fi if [[ ${CURRENT_NEXTCLOUD_VER} =~ ^15 ]]; then - InstallNextcloud 16.0.6 0bb3098455ec89f5af77a652aad553ad40a88819 3.3.0 e55d0357c6785d3b1f3b5f21780cb6d41d32443a 2.0.3 9d9717b29337613b72c74e9914c69b74b346c466 0.7.0 555a94811daaf5bdd336c5e48a78aa8567b86437 + InstallNextcloud 16.0.6 0bb3098455ec89f5af77a652aad553ad40a88819 3.3.0 e55d0357c6785d3b1f3b5f21780cb6d41d32443a 2.0.3 a1f3835c752929e3598eb94f22300516867ac6ab 0.7.0 555a94811daaf5bdd336c5e48a78aa8567b86437 CURRENT_NEXTCLOUD_VER="16.0.6" fi if [[ ${CURRENT_NEXTCLOUD_VER} =~ ^16 ]]; then - InstallNextcloud 17.0.6 50b98d2c2f18510b9530e558ced9ab51eb4f11b0 3.3.0 e55d0357c6785d3b1f3b5f21780cb6d41d32443a 2.0.3 9d9717b29337613b72c74e9914c69b74b346c466 0.7.0 555a94811daaf5bdd336c5e48a78aa8567b86437 + InstallNextcloud 17.0.6 50b98d2c2f18510b9530e558ced9ab51eb4f11b0 3.3.0 e55d0357c6785d3b1f3b5f21780cb6d41d32443a 2.0.3 a1f3835c752929e3598eb94f22300516867ac6ab 0.7.0 555a94811daaf5bdd336c5e48a78aa8567b86437 CURRENT_NEXTCLOUD_VER="17.0.6" fi if [[ ${CURRENT_NEXTCLOUD_VER} =~ ^17 ]]; then # Don't exit the install if this column already exists (see #2076) (echo "ALTER TABLE oc_flow_operations ADD COLUMN entity VARCHAR;" | sqlite3 $STORAGE_ROOT/owncloud/owncloud.db 2>/dev/null) || true - InstallNextcloud 18.0.10 39c0021a8b8477c3f1733fddefacfa5ebf921c68 3.4.1 aee680a75e95f26d9285efd3c1e25cf7f3bfd27e 2.0.3 9d9717b29337613b72c74e9914c69b74b346c466 1.0.0 3bf2609061d7214e7f0f69dd8883e55c4ec8f50a + InstallNextcloud 18.0.10 39c0021a8b8477c3f1733fddefacfa5ebf921c68 3.4.1 8f685e7dc99758636d660d595e389c324e51e9d1 2.0.3 a1f3835c752929e3598eb94f22300516867ac6ab 1.0.0 3bf2609061d7214e7f0f69dd8883e55c4ec8f50a CURRENT_NEXTCLOUD_VER="18.0.10" fi if [[ ${CURRENT_NEXTCLOUD_VER} =~ ^18 ]]; then - InstallNextcloud 19.0.4 01e98791ba12f4860d3d4047b9803f97a1b55c60 3.4.1 aee680a75e95f26d9285efd3c1e25cf7f3bfd27e 2.0.3 9d9717b29337613b72c74e9914c69b74b346c466 1.0.0 3bf2609061d7214e7f0f69dd8883e55c4ec8f50a + InstallNextcloud 19.0.4 01e98791ba12f4860d3d4047b9803f97a1b55c60 3.4.1 8f685e7dc99758636d660d595e389c324e51e9d1 2.0.3 a1f3835c752929e3598eb94f22300516867ac6ab 1.0.0 3bf2609061d7214e7f0f69dd8883e55c4ec8f50a CURRENT_NEXTCLOUD_VER="19.0.4" fi fi diff --git a/setup/system.sh b/setup/system.sh index 036fe3f9..9898cbcd 100755 --- a/setup/system.sh +++ b/setup/system.sh @@ -363,6 +363,7 @@ systemctl restart systemd-resolved rm -f /etc/fail2ban/jail.local # we used to use this file but don't anymore rm -f /etc/fail2ban/jail.d/defaults-debian.conf # removes default config so we can manage all of fail2ban rules in one config cat conf/fail2ban/jails.conf \ + | sed "s/PUBLIC_IPV6/$PUBLIC_IPV6/g" \ | sed "s/PUBLIC_IP/$PUBLIC_IP/g" \ | sed "s#STORAGE_ROOT#$STORAGE_ROOT#" \ > /etc/fail2ban/jail.d/mailinabox.conf