diff --git a/CHANGELOG.md b/CHANGELOG.md index e96261bf..b9119892 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ Version 60 (date TBD) This is the first release for Ubuntu 22.04. -**Before upgrading**, you must **first upgrade your existing Ubuntu 18.04 box to Mail-in-a-Box v0.51** (or any later version of Mail-in-a-Box supporting Ubuntu 18.04), if you haven't already done so. That may not be possible after Ubuntu 18.04 reaches its end of life in April 2023, so please compete the upgrade well before then. (If you are not using Nextcloud's contacts or calendar, you can migrate to the latest version of Mail-in-a-Box from any previous version.) +**Before upgrading**, you must **first upgrade your existing Ubuntu 18.04 box to Mail-in-a-Box v0.51 or later**, if you haven't already done so. That may not be possible after Ubuntu 18.04 reaches its end of life in April 2023, so please complete the upgrade well before then. (If you are not using Nextcloud's contacts or calendar, you can migrate to the latest version of Mail-in-a-Box from any previous version.) For complete upgrade instructions, see: @@ -14,14 +14,33 @@ LINK TBD No features of Mail-in-a-Box have changed in this release, but with the newer version of Ubuntu the following software packages we use are updated: -* dovecot is upgraded to 2.3.16, postfix to 3.6.3, opendmark to 1.4 (which adds ARC-Authentication-Results headers), and spampd to 2.53 (alleviating a mail delivery rate limiting bug). +* dovecot is upgraded to 2.3.16, postfix to 3.6.4, opendmark to 1.4 (which adds ARC-Authentication-Results headers), and spampd to 2.53 (alleviating a mail delivery rate limiting bug). * Nextcloud is upgraded to 23.0.0 with PHP updated from 7.2 to 8.0. * certbot is upgraded to 1.21 (via the Ubuntu repository instead of a PPA). * fail2ban is upgraded to 0.11.2. * nginx is upgraded to 1.18. -In Development --------------- +======= +Version 57 (June 12, 2022) +-------------------------- + +Setup: + +* Fixed issue upgrading from Mail-in-a-Box v0.40-v0.50 because of a changed URL that Nextcloud is downloaded from. + +Backups: + +* Fixed S3 backups which broke with duplicity 0.8.23. +* Fixed Backblaze backups which broke with latest b2sdk package by rolling back its version. + +Control panel: + +* Fixed spurious changes in system status checks messages by sorting DNSSEC DS records. +* Fixed fail2ban lockout over IPv6 from excessive loads of the system status checks. +* Fixed an incorrect IPv6 system status check message. + +Version 56 (January 19, 2022) +----------------------------- Software updates: diff --git a/conf/fail2ban/jails.conf b/conf/fail2ban/jails.conf index dccfcf5d..c1514b45 100644 --- a/conf/fail2ban/jails.conf +++ b/conf/fail2ban/jails.conf @@ -5,7 +5,7 @@ # Whitelist our own IP addresses. 127.0.0.1/8 is the default. But our status checks # ping services over the public interface so we should whitelist that address of # ours too. The string is substituted during installation. -ignoreip = 127.0.0.1/8 ::1/128 PUBLIC_IP PUBLIC_IPV6/64 +ignoreip = 127.0.0.1/8 PUBLIC_IP ::1 PUBLIC_IPV6 [dovecot] enabled = true diff --git a/management/backup.py b/management/backup.py index a211210d..760ab6c4 100755 --- a/management/backup.py +++ b/management/backup.py @@ -12,12 +12,7 @@ import dateutil.parser, dateutil.relativedelta, dateutil.tz import rtyaml from exclusiveprocess import Lock -from utils import load_environment, shell, wait_for_service, fix_boto, get_php_version - -rsync_ssh_options = [ - "--ssh-options= -i /root/.ssh/id_rsa_miab", - "--rsync-options= -e \"/usr/bin/ssh -oStrictHostKeyChecking=no -oBatchMode=yes -p 22 -i /root/.ssh/id_rsa_miab\"", -] +from utils import load_environment, shell, wait_for_service, get_php_version def backup_status(env): # If backups are disabled, return no status. @@ -64,9 +59,9 @@ def backup_status(env): "--archive-dir", backup_cache_dir, "--gpg-options", "--cipher-algo=AES256", "--log-fd", "1", - config["target"], - ] + rsync_ssh_options, - get_env(env), + get_duplicity_target_url(config), + ] + get_duplicity_additional_args(env), + get_duplicity_env_vars(env), trap=True) if code != 0: # Command failed. This is likely due to an improperly configured remote @@ -195,7 +190,43 @@ def get_passphrase(env): return passphrase -def get_env(env): +def get_duplicity_target_url(config): + target = config["target"] + + if get_target_type(config) == "s3": + from urllib.parse import urlsplit, urlunsplit + target = list(urlsplit(target)) + + # Although we store the S3 hostname in the target URL, + # duplicity no longer accepts it in the target URL. The hostname in + # the target URL must be the bucket name. The hostname is passed + # via get_duplicity_additional_args. Move the first part of the + # path (the bucket name) into the hostname URL component, and leave + # the rest for the path. + target[1], target[2] = target[2].lstrip('/').split('/', 1) + + target = urlunsplit(target) + + return target + +def get_duplicity_additional_args(env): + config = get_backup_config(env) + + if get_target_type(config) == 'rsync': + return [ + "--ssh-options= -i /root/.ssh/id_rsa_miab", + "--rsync-options= -e \"/usr/bin/ssh -oStrictHostKeyChecking=no -oBatchMode=yes -p 22 -i /root/.ssh/id_rsa_miab\"", + ] + elif get_target_type(config) == 's3': + # See note about hostname in get_duplicity_target_url. + from urllib.parse import urlsplit, urlunsplit + target = urlsplit(config["target"]) + endpoint_url = urlunsplit(("https", target.netloc, '', '', '')) + return ["--s3-endpoint-url", endpoint_url] + + return [] + +def get_duplicity_env_vars(env): config = get_backup_config(env) env = { "PASSPHRASE" : get_passphrase(env) } @@ -274,10 +305,10 @@ def perform_backup(full_backup): "--volsize", "250", "--gpg-options", "--cipher-algo=AES256", env["STORAGE_ROOT"], - config["target"], + get_duplicity_target_url(config), "--allow-source-mismatch" - ] + rsync_ssh_options, - get_env(env)) + ] + get_duplicity_additional_args(env), + get_duplicity_env_vars(env)) finally: # Start services again. service_command("dovecot", "start", quit=False) @@ -293,9 +324,9 @@ def perform_backup(full_backup): "--verbosity", "error", "--archive-dir", backup_cache_dir, "--force", - config["target"] - ] + rsync_ssh_options, - get_env(env)) + get_duplicity_target_url(config) + ] + get_duplicity_additional_args(env), + get_duplicity_env_vars(env)) # From duplicity's manual: # "This should only be necessary after a duplicity session fails or is @@ -308,9 +339,9 @@ def perform_backup(full_backup): "--verbosity", "error", "--archive-dir", backup_cache_dir, "--force", - config["target"] - ] + rsync_ssh_options, - get_env(env)) + get_duplicity_target_url(config) + ] + get_duplicity_additional_args(env), + get_duplicity_env_vars(env)) # Change ownership of backups to the user-data user, so that the after-bcakup # script can access them. @@ -346,9 +377,9 @@ def run_duplicity_verification(): "--compare-data", "--archive-dir", backup_cache_dir, "--exclude", backup_root, - config["target"], + get_duplicity_target_url(config), env["STORAGE_ROOT"], - ] + rsync_ssh_options, get_env(env)) + ] + get_duplicity_additional_args(env), get_duplicity_env_vars(env)) def run_duplicity_restore(args): env = load_environment() @@ -358,9 +389,9 @@ def run_duplicity_restore(args): "/usr/bin/duplicity", "restore", "--archive-dir", backup_cache_dir, - config["target"], - ] + rsync_ssh_options + args, - get_env(env)) + get_duplicity_target_url(config), + ] + get_duplicity_additional_args(env) + args, + get_duplicity_env_vars(env)) def list_target_files(config): import urllib.parse @@ -417,7 +448,6 @@ def list_target_files(config): elif target.scheme == "s3": # match to a Region - fix_boto() # must call prior to importing boto import boto.s3 from boto.exception import BotoServerError custom_region = False diff --git a/management/daemon.py b/management/daemon.py index 0bbb1ad5..98c6689c 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -121,7 +121,6 @@ def index(): no_users_exist = (len(get_mail_users(env)) == 0) no_admins_exist = (len(get_admins(env)) == 0) - utils.fix_boto() # must call prior to importing boto import boto.s3 backup_s3_hosts = [(r.name, r.endpoint) for r in boto.s3.regions()] diff --git a/management/ssl_certificates.py b/management/ssl_certificates.py index 96959425..203c9f7b 100755 --- a/management/ssl_certificates.py +++ b/management/ssl_certificates.py @@ -58,36 +58,33 @@ def get_ssl_certificates(env): # Not a valid PEM format for a PEM type we care about. continue - # Remember where we got this object. - pem._filename = fn - # Is it a private key? if isinstance(pem, RSAPrivateKey): - private_keys[pem.public_key().public_numbers()] = pem + private_keys[pem.public_key().public_numbers()] = { "filename": fn, "key": pem } # Is it a certificate? if isinstance(pem, Certificate): - certificates.append(pem) + certificates.append({ "filename": fn, "cert": pem }) # Process the certificates. domains = { } for cert in certificates: # What domains is this certificate good for? - cert_domains, primary_domain = get_certificate_domains(cert) - cert._primary_domain = primary_domain + cert_domains, primary_domain = get_certificate_domains(cert["cert"]) + cert["primary_domain"] = primary_domain # Is there a private key file for this certificate? - private_key = private_keys.get(cert.public_key().public_numbers()) + private_key = private_keys.get(cert["cert"].public_key().public_numbers()) if not private_key: continue - cert._private_key = private_key + cert["private_key"] = private_key # Add this cert to the list of certs usable for the domains. for domain in cert_domains: # The primary hostname can only use a certificate mapped # to the system private key. if domain == env['PRIMARY_HOSTNAME']: - if cert._private_key._filename != os.path.join(env['STORAGE_ROOT'], 'ssl', 'ssl_private_key.pem'): + if cert["private_key"]["filename"] != os.path.join(env['STORAGE_ROOT'], 'ssl', 'ssl_private_key.pem'): continue domains.setdefault(domain, []).append(cert) @@ -100,10 +97,10 @@ def get_ssl_certificates(env): #for c in cert_list: print(domain, c.not_valid_before, c.not_valid_after, "("+str(now)+")", c.issuer, c.subject, c._filename) cert_list.sort(key = lambda cert : ( # must be valid NOW - cert.not_valid_before <= now <= cert.not_valid_after, + cert["cert"].not_valid_before <= now <= cert["cert"].not_valid_after, # prefer one that is not self-signed - cert.issuer != cert.subject, + cert["cert"].issuer != cert["cert"].subject, ########################################################### # The above lines ensure that valid certificates are chosen @@ -113,7 +110,7 @@ def get_ssl_certificates(env): # prefer one with the expiration furthest into the future so # that we can easily rotate to new certs as we get them - cert.not_valid_after, + cert["cert"].not_valid_after, ########################################################### # We always choose the certificate that is good for the @@ -128,15 +125,15 @@ def get_ssl_certificates(env): # in case a certificate is installed in multiple paths, # prefer the... lexicographically last one? - cert._filename, + cert["filename"], ), reverse=True) cert = cert_list.pop(0) ret[domain] = { - "private-key": cert._private_key._filename, - "certificate": cert._filename, - "primary-domain": cert._primary_domain, - "certificate_object": cert, + "private-key": cert["private_key"]["filename"], + "certificate": cert["filename"], + "primary-domain": cert["primary_domain"], + "certificate_object": cert["cert"], } return ret diff --git a/management/status_checks.py b/management/status_checks.py index 3fc9c994..3f78f6fa 100755 --- a/management/status_checks.py +++ b/management/status_checks.py @@ -255,6 +255,18 @@ def check_free_disk_space(rounded_values, env, output): if rounded_values: disk_msg = "The disk has less than 15% free space." output.print_error(disk_msg) + # Check that there's only one duplicity cache. If there's more than one, + # it's probably no longer in use, and we can recommend clearing the cache + # to save space. The cache directory may not exist yet, which is OK. + backup_cache_path = os.path.join(env['STORAGE_ROOT'], 'backup/cache') + try: + backup_cache_count = len(os.listdir(backup_cache_path)) + except: + backup_cache_count = 0 + if backup_cache_count > 1: + output.print_warning("The backup cache directory {} has more than one backup target cache. Consider clearing this directory to save disk space." + .format(backup_cache_path)) + def check_free_memory(rounded_values, env, output): # Check free memory. percent_free = 100 - psutil.virtual_memory().percent @@ -660,7 +672,7 @@ def check_dnssec(domain, env, output, dns_zonefiles, is_checking_primary=False): if len(ds) > 0: output.print_line("") output.print_line("The DS record is currently set to:") - for rr in ds: + for rr in sorted(ds): output.print_line("Key Tag: {0}, Algorithm: {1}, Digest Type: {2}, Digest: {3}".format(*rr)) def check_mail_domain(domain, env, output): diff --git a/management/utils.py b/management/utils.py index bc357040..3aaece2e 100644 --- a/management/utils.py +++ b/management/utils.py @@ -175,13 +175,6 @@ def wait_for_service(port, public, env, timeout): return False time.sleep(min(timeout/4, 1)) -def fix_boto(): - # Google Compute Engine instances install some Python-2-only boto plugins that - # conflict with boto running under Python 3. Disable boto's default configuration - # file prior to importing boto so that GCE's plugin is not loaded: - import os - os.environ["BOTO_CONFIG"] = "/etc/boto3.cfg" - def get_php_version(): # Gets the version of PHP installed in the system. return shell("check_output", ["/usr/bin/php", "-v"])[4:7] diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index af6f4040..bb3a5052 100644 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -28,10 +28,10 @@ if [ -z "$TAG" ]; then # This machine is running Ubuntu 18.04, which is supported by # Mail-in-a-Box versions 0.40 through 5x. echo "Support is ending for Ubuntu 18.04." - echo "Please immediately begin to migrate your information to" + echo "Please immediately begin to migrate your data to" echo "a new machine running Ubuntu 22.04. See:" echo "https://mailinabox.email/maintenance.html#upgrade" - TAG=v56 + TAG=v57 elif [ "$UBUNTU_VERSION" == "Ubuntu 14.04 LTS" ]; then # This machine is running Ubuntu 14.04, which is supported by # Mail-in-a-Box versions 1 through v0.30. diff --git a/setup/functions.sh b/setup/functions.sh index f026e13c..4c29f3c8 100644 --- a/setup/functions.sh +++ b/setup/functions.sh @@ -4,6 +4,8 @@ # -o pipefail: don't ignore errors in the non-last command in a pipeline set -euo pipefail +PHP_VER=8.0 + function hide_output { # This function hides the output of a command unless the command fails # and returns a non-zero exit code. diff --git a/setup/management.sh b/setup/management.sh index 7961aecb..52d38498 100755 --- a/setup/management.sh +++ b/setup/management.sh @@ -1,6 +1,7 @@ #!/bin/bash source setup/functions.sh +source /etc/mailinabox.conf # load global vars echo "Installing Mail-in-a-Box system management daemon..." @@ -51,7 +52,8 @@ hide_output $venv/bin/pip install --upgrade \ rtyaml "email_validator>=1.0.0" "exclusiveprocess" \ flask dnspython python-dateutil expiringdict \ qrcode[pil] pyotp \ - "idna>=2.0.0" "cryptography==2.2.2" boto psutil postfix-mta-sts-resolver b2sdk + "idna>=2.0.0" "cryptography==37.0.2" psutil postfix-mta-sts-resolver \ + b2sdk boto # CONFIGURATION diff --git a/setup/nextcloud.sh b/setup/nextcloud.sh index 260ae3ed..54cd022b 100755 --- a/setup/nextcloud.sh +++ b/setup/nextcloud.sh @@ -402,7 +402,6 @@ tools/editconf.py /etc/php/$(php_version)/cli/conf.d/10-opcache.ini -c ';' \ opcache.save_comments=1 \ opcache.revalidate_freq=1 - # Set up a cron job for Nextcloud. cat > /etc/cron.d/mailinabox-nextcloud << EOF; #!/bin/bash diff --git a/setup/system.sh b/setup/system.sh index 3f7ba3aa..c6bd86fa 100755 --- a/setup/system.sh +++ b/setup/system.sh @@ -99,6 +99,9 @@ fi # come from there and minimal Ubuntu installs may have it turned off. hide_output add-apt-repository -y universe +# Install the duplicity PPA. +hide_output add-apt-repository -y ppa:duplicity-team/duplicity-release-git + # ### Update Packages # Update system packages to make sure we have the latest upstream versions @@ -356,6 +359,7 @@ systemctl restart systemd-resolved rm -f /etc/fail2ban/jail.local # we used to use this file but don't anymore rm -f /etc/fail2ban/jail.d/defaults-debian.conf # removes default config so we can manage all of fail2ban rules in one config cat conf/fail2ban/jails.conf \ + | sed "s/PUBLIC_IPV6/$PUBLIC_IPV6/g" \ | sed "s/PUBLIC_IP/$PUBLIC_IP/g" \ | sed "s#STORAGE_ROOT#$STORAGE_ROOT#" \ > /etc/fail2ban/jail.d/mailinabox.conf diff --git a/setup/webmail.sh b/setup/webmail.sh index f9ae4168..1ef12771 100755 --- a/setup/webmail.sh +++ b/setup/webmail.sh @@ -202,10 +202,10 @@ chown -f -R root.www-data ${RCM_PLUGIN_DIR}/carddav chmod -R 774 ${RCM_PLUGIN_DIR}/carddav # Run Roundcube database migration script (database is created if it does not exist) -${RCM_DIR}/bin/updatedb.sh --dir ${RCM_DIR}/SQL --package roundcube +php ${RCM_DIR}/bin/updatedb.sh --dir ${RCM_DIR}/SQL --package roundcube chown www-data:www-data $STORAGE_ROOT/mail/roundcube/roundcube.sqlite chmod 664 $STORAGE_ROOT/mail/roundcube/roundcube.sqlite # Enable PHP modules. -phpenmod -v php mcrypt imap +phpenmod -v php imap restart_service php$(php_version)-fpm diff --git a/setup/zpush.sh b/setup/zpush.sh index 91aeb02c..16ae7b05 100755 --- a/setup/zpush.sh +++ b/setup/zpush.sh @@ -42,8 +42,6 @@ if [ $needs_update == 1 ]; then rm -rf /tmp/z-push.zip /tmp/z-push rm -f /usr/sbin/z-push-{admin,top} - ln -s /usr/local/lib/z-push/z-push-admin.php /usr/sbin/z-push-admin - ln -s /usr/local/lib/z-push/z-push-top.php /usr/sbin/z-push-top echo $VERSION > /usr/local/lib/z-push/version fi @@ -106,4 +104,4 @@ restart_service php$(php_version)-fpm # Fix states after upgrade -hide_output z-push-admin -a fixstates +hide_output php /usr/local/lib/z-push/z-push-admin.php -a fixstates