From 2abcafd670ee9d0e69b4abe424b6edb4aeea0924 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sat, 8 Jan 2022 19:09:11 -0500 Subject: [PATCH 01/35] Update Ubuntu version checks from 18.04 to 22.04 --- setup/bootstrap.sh | 41 +++++++++++++++++++++++------------------ setup/preflight.sh | 8 ++++---- 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index 4ba0cee7..cc4c060e 100644 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -9,32 +9,37 @@ if [ -z "$TAG" ]; then # If a version to install isn't explicitly given as an environment # variable, then install the latest version. But the latest version - # depends on the operating system. Existing Ubuntu 14.04 users need - # to be able to upgrade to the latest version supporting Ubuntu 14.04, - # in part because an upgrade is required before jumping to Ubuntu 18.04. - # New users on Ubuntu 18.04 need to get the latest version number too. + # depends on the machine's version of Ubuntu. Existing users need to + # be able to upgrade to the latest version available for that version + # of Ubuntu to satisfy the migration requirements. # # Also, the system status checks read this script for TAG = (without the # space, but if we put it in a comment it would confuse the status checks!) # to get the latest version, so the first such line must be the one that we # want to display in status checks. - if [ "$(lsb_release -d | sed 's/.*:\s*//' | sed 's/18\.04\.[0-9]/18.04/' )" == "Ubuntu 18.04 LTS" ]; then - # This machine is running Ubuntu 18.04. + # + # Allow point-release versions of the major releases, e.g. 22.04.1 is OK. + UBUNTU_VERSION=$( lsb_release -d | sed 's/.*:\s*//' | sed 's/\([0-9]*\.[0-9]*\)\.[0-9]/\1/' )" + if [ "$UBUNTU_VERSION" == "Ubuntu 22.04 LTS" ]; then + # This machine is running Ubuntu 22.04, which is supported by + # Mail-in-a-Box versions 60 and later. + TAG=v60 + elif [ "$UBUNTU_VERSION" == "Ubuntu 18.04 LTS" ]; then + # This machine is running Ubuntu 18.04, which is supported by + # Mail-in-a-Box versions 0.40 through 5x. + echo "Support is ending for Ubuntu 18.04." + echo "Please immediately begin to migrate your data to" + echo "a new machine running Ubuntu 22.04. See:" + echo "https://mailinabox.email/maintenance.html#upgrade" TAG=v57a - - elif [ "$(lsb_release -d | sed 's/.*:\s*//' | sed 's/14\.04\.[0-9]/14.04/' )" == "Ubuntu 14.04 LTS" ]; then - # This machine is running Ubuntu 14.04. - echo "You are installing the last version of Mail-in-a-Box that will" - echo "support Ubuntu 14.04. If this is a new installation of Mail-in-a-Box," - echo "stop now and switch to a machine running Ubuntu 18.04. If you are" - echo "upgrading an existing Mail-in-a-Box --- great. After upgrading this" - echo "box, please visit https://mailinabox.email for notes on how to upgrade" - echo "to Ubuntu 18.04." - echo "" + elif [ "$UBUNTU_VERSION" == "Ubuntu 14.04 LTS" ]; then + # This machine is running Ubuntu 14.04, which is supported by + # Mail-in-a-Box versions 1 through v0.30. + echo "Ubuntu 14.04 is no longer supported." + echo "The last version of Mail-in-a-Box supporting Ubuntu 14.04 will be installed." TAG=v0.30 - else - echo "This script must be run on a system running Ubuntu 18.04 or Ubuntu 14.04." + echo "This script may be used only on a machine running Ubuntu 14.04, 18.04, or 22.04." exit 1 fi fi diff --git a/setup/preflight.sh b/setup/preflight.sh index 9d2715c5..bd6d65b7 100644 --- a/setup/preflight.sh +++ b/setup/preflight.sh @@ -7,11 +7,11 @@ if [[ $EUID -ne 0 ]]; then exit 1 fi -# Check that we are running on Ubuntu 18.04 LTS (or 18.04.xx). -if [ "$(lsb_release -d | sed 's/.*:\s*//' | sed 's/18\.04\.[0-9]/18.04/' )" != "Ubuntu 18.04 LTS" ]; then - echo "Mail-in-a-Box only supports being installed on Ubuntu 18.04, sorry. You are running:" +# Check that we are running on Ubuntu 20.04 LTS (or 20.04.xx). +if [ "$( lsb_release --id --short )" != "Ubuntu" ] || [ "$( lsb_release --release --short )" != "22.04" ]; then + echo "Mail-in-a-Box only supports being installed on Ubuntu 22.04, sorry. You are running:" echo - lsb_release -d | sed 's/.*:\s*//' + lsb_release --description --short echo echo "We can't write scripts that run on every possible setup, sorry." exit 1 From f534a530d49b52a105afc9a4bfbc40774c9a1b1b Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sat, 8 Jan 2022 19:24:41 -0500 Subject: [PATCH 02/35] Update and drop some package and file names for Ubuntu 22.04 * Fix path to bind9 startup options file in Ubuntu 22.04. * tinymce has not been a Roundcube requirement recently and is no longer a package in Ubuntu 22.04 * Upgrade Vagrant box to Ubuntu 22.04 --- Vagrantfile | 2 +- setup/system.sh | 2 +- setup/webmail.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 04788292..757c2ec9 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -2,7 +2,7 @@ # vi: set ft=ruby : Vagrant.configure("2") do |config| - config.vm.box = "ubuntu/bionic64" + config.vm.box = "ubuntu/jammy64" # Network config: Since it's a mail server, the machine must be connected # to the public web. However, we currently don't want to expose SSH since diff --git a/setup/system.sh b/setup/system.sh index 9898cbcd..4e438087 100755 --- a/setup/system.sh +++ b/setup/system.sh @@ -331,7 +331,7 @@ fi #NODOC # If more queries than specified are sent, bind9 returns SERVFAIL. After flushing the cache during system checks, # we ran into the limit thus we are increasing it from 75 (default value) to 100. apt_install bind9 -tools/editconf.py /etc/default/bind9 \ +tools/editconf.py /etc/default/named \ "OPTIONS=\"-u bind -4\"" if ! grep -q "listen-on " /etc/bind/named.conf.options; then # Add a listen-on directive if it doesn't exist inside the options block. diff --git a/setup/webmail.sh b/setup/webmail.sh index 4855d0dc..938bcdd3 100755 --- a/setup/webmail.sh +++ b/setup/webmail.sh @@ -23,7 +23,7 @@ echo "Installing Roundcube (webmail)..." apt_install \ dbconfig-common \ php-cli php-sqlite3 php-intl php-json php-common php-curl php-ldap \ - php-gd php-pspell tinymce libjs-jquery libjs-jquery-mousewheel libmagic1 php-mbstring + php-gd php-pspell libjs-jquery libjs-jquery-mousewheel libmagic1 php-mbstring # Install Roundcube from source if it is not already present or if it is out of date. # Combine the Roundcube version number with the commit hash of plugins to track From 8cb360fe36a10e7a9d85f0d96c960da708115d49 Mon Sep 17 00:00:00 2001 From: Daniel Mabbett Date: Fri, 24 Apr 2020 20:37:59 +1000 Subject: [PATCH 03/35] Configure nsd listening interfaces before installing nsd so that it does not interfere with bind9 --- setup/dns.sh | 46 +++++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/setup/dns.sh b/setup/dns.sh index c8a73a73..9b9b1b0a 100755 --- a/setup/dns.sh +++ b/setup/dns.sh @@ -10,17 +10,13 @@ source setup/functions.sh # load our functions source /etc/mailinabox.conf # load global vars -# Install the packages. -# -# * nsd: The non-recursive nameserver that publishes our DNS records. -# * ldnsutils: Helper utilities for signing DNSSEC zones. -# * openssh-client: Provides ssh-keyscan which we use to create SSHFP records. -echo "Installing nsd (DNS server)..." -apt_install nsd ldnsutils openssh-client - # Prepare nsd's configuration. - +# We configure nsd before installation as we only want it to bind to some addresses +# and it otherwise will have port / bind conflicts with bind9 used as the local resolver mkdir -p /var/run/nsd +mkdir -p /etc/nsd +mkdir -p /etc/nsd/zones +touch /etc/nsd/zones.conf cat > /etc/nsd/nsd.conf << EOF; # Do not edit. Overwritten by Mail-in-a-Box setup. @@ -42,18 +38,6 @@ server: EOF -# Add log rotation -cat > /etc/logrotate.d/nsd <> /etc/nsd/nsd.conf; # now be stored in /etc/nsd/nsd.conf.d. rm -f /etc/nsd/zones.conf +# Add log rotation +cat > /etc/logrotate.d/nsd < Date: Thu, 29 Jul 2021 09:49:03 +0300 Subject: [PATCH 04/35] Fix DeprecationWarning in dnspython query vs resolve method The resolve method disables resolving relative names by default. This change probably makes a7710e90 unnecessary. @JoshData added some additional changes from query to resolve. --- management/dns_update.py | 8 ++++---- management/status_checks.py | 2 +- tests/test_dns.py | 2 +- tests/test_mail.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/management/dns_update.py b/management/dns_update.py index fde9b146..45ea94fa 100755 --- a/management/dns_update.py +++ b/management/dns_update.py @@ -1000,9 +1000,9 @@ def get_secondary_dns(custom_dns, mode=None): # doesn't. if not hostname.startswith("xfr:"): if mode == "xfr": - response = dns.resolver.query(hostname+'.', "A", raise_on_no_answer=False) + response = dns.resolver.resolve(hostname+'.', "A", raise_on_no_answer=False) values.extend(map(str, response)) - response = dns.resolver.query(hostname+'.', "AAAA", raise_on_no_answer=False) + response = dns.resolver.resolve(hostname+'.', "AAAA", raise_on_no_answer=False) values.extend(map(str, response)) continue values.append(hostname) @@ -1025,10 +1025,10 @@ def set_secondary_dns(hostnames, env): if not item.startswith("xfr:"): # Resolve hostname. try: - response = resolver.query(item, "A") + response = resolver.resolve(item, "A") except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer): try: - response = resolver.query(item, "AAAA") + response = resolver.resolve(item, "AAAA") except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer): raise ValueError("Could not resolve the IP address of %s." % item) else: diff --git a/management/status_checks.py b/management/status_checks.py index 220e23e4..12b4440d 100755 --- a/management/status_checks.py +++ b/management/status_checks.py @@ -797,7 +797,7 @@ def query_dns(qname, rtype, nxdomain='[Not Set]', at=None, as_list=False): # Do the query. try: - response = resolver.query(qname, rtype) + response = resolver.resolve(qname, rtype) except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer): # Host did not have an answer for this query; not sure what the # difference is between the two exceptions. diff --git a/tests/test_dns.py b/tests/test_dns.py index c5fe8051..ce51c9d9 100755 --- a/tests/test_dns.py +++ b/tests/test_dns.py @@ -48,7 +48,7 @@ def test2(tests, server, description): for qname, rtype, expected_answer in tests: # do the query and format the result as a string try: - response = dns.resolver.query(qname, rtype) + response = dns.resolver.resolve(qname, rtype) except dns.resolver.NoNameservers: # host did not have an answer for this query print("Could not connect to %s for DNS query." % server) diff --git a/tests/test_mail.py b/tests/test_mail.py index 8c8838a5..312f3332 100755 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -48,7 +48,7 @@ server = smtplib.SMTP_SSL(host) ipaddr = socket.gethostbyname(host) # IPv4 only! reverse_ip = dns.reversename.from_address(ipaddr) # e.g. "1.0.0.127.in-addr.arpa." try: - reverse_dns = dns.resolver.query(reverse_ip, 'PTR')[0].target.to_text(omit_final_dot=True) # => hostname + reverse_dns = dns.resolver.resolve(reverse_ip, 'PTR')[0].target.to_text(omit_final_dot=True) # => hostname except dns.resolver.NXDOMAIN: print("Reverse DNS lookup failed for %s. SMTP EHLO name check skipped." % ipaddr) reverse_dns = None From b41a0ad80e3916377b5de060afe66db69100916e Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sat, 8 Jan 2022 19:09:30 -0500 Subject: [PATCH 05/35] Drop some hacks that we needed for Ubuntu 18.04 * certbot's PPA is no longer needed because a recent version is now included in the Ubuntu respository. * Un-pin b2sdk (reverts 69d8fdef9915127f016eb6424322a149cdff25d7 and d829d74048ce9cc3cfda51f45428cfe482a1ce02). * Revert boto+s3 workaround for duplicity (partial revert of 99474b348f0dd2632057c18c6a8c4e6464962878). * Revert old "fix boto 2 conflict on Google Compute Engine instances" (cf33be4596bf3fb784fbeaaea58a934b139ead0e) which is probably no longer needed. --- management/backup.py | 10 ++-------- management/daemon.py | 1 - management/utils.py | 8 -------- setup/management.sh | 4 ++-- setup/system.sh | 3 --- 5 files changed, 4 insertions(+), 22 deletions(-) diff --git a/management/backup.py b/management/backup.py index 26a61a0e..ce358e06 100755 --- a/management/backup.py +++ b/management/backup.py @@ -12,7 +12,7 @@ import dateutil.parser, dateutil.relativedelta, dateutil.tz import rtyaml from exclusiveprocess import Lock -from utils import load_environment, shell, wait_for_service, fix_boto +from utils import load_environment, shell, wait_for_service def backup_status(env): # If backups are dissbled, return no status. @@ -197,12 +197,7 @@ def get_duplicity_target_url(config): from urllib.parse import urlsplit, urlunsplit target = list(urlsplit(target)) - # Duplicity now defaults to boto3 as the backend for S3, but we have - # legacy boto installed (boto3 doesn't support Ubuntu 18.04) so - # we retarget for classic boto. - target[0] = "boto+" + target[0] - - # In addition, although we store the S3 hostname in the target URL, + # Although we store the S3 hostname in the target URL, # duplicity no longer accepts it in the target URL. The hostname in # the target URL must be the bucket name. The hostname is passed # via get_duplicity_additional_args. Move the first part of the @@ -452,7 +447,6 @@ def list_target_files(config): elif target.scheme == "s3": # match to a Region - fix_boto() # must call prior to importing boto import boto.s3 from boto.exception import BotoServerError custom_region = False diff --git a/management/daemon.py b/management/daemon.py index 0bbb1ad5..98c6689c 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -121,7 +121,6 @@ def index(): no_users_exist = (len(get_mail_users(env)) == 0) no_admins_exist = (len(get_admins(env)) == 0) - utils.fix_boto() # must call prior to importing boto import boto.s3 backup_s3_hosts = [(r.name, r.endpoint) for r in boto.s3.regions()] diff --git a/management/utils.py b/management/utils.py index 652b48f6..fc04ed4d 100644 --- a/management/utils.py +++ b/management/utils.py @@ -175,14 +175,6 @@ def wait_for_service(port, public, env, timeout): return False time.sleep(min(timeout/4, 1)) -def fix_boto(): - # Google Compute Engine instances install some Python-2-only boto plugins that - # conflict with boto running under Python 3. Disable boto's default configuration - # file prior to importing boto so that GCE's plugin is not loaded: - import os - os.environ["BOTO_CONFIG"] = "/etc/boto3.cfg" - - if __name__ == "__main__": from web_update import get_web_domains env = load_environment() diff --git a/setup/management.sh b/setup/management.sh index 046c5bac..483a9c5d 100755 --- a/setup/management.sh +++ b/setup/management.sh @@ -30,7 +30,7 @@ apt_install duplicity python-pip virtualenv certbot rsync # b2sdk is used for backblaze backups. # boto is used for amazon aws backups. # Both are installed outside the pipenv, so they can be used by duplicity -hide_output pip3 install --upgrade b2sdk==1.14.1 boto +hide_output pip3 install --upgrade b2sdk boto # Create a virtualenv for the installation of Python 3 packages # used by the management daemon. @@ -52,7 +52,7 @@ hide_output $venv/bin/pip install --upgrade \ flask dnspython python-dateutil expiringdict \ qrcode[pil] pyotp \ "idna>=2.0.0" "cryptography==2.2.2" psutil postfix-mta-sts-resolver \ - b2sdk==1.14.1 boto + b2sdk boto # CONFIGURATION diff --git a/setup/system.sh b/setup/system.sh index 4e438087..b128a6bf 100755 --- a/setup/system.sh +++ b/setup/system.sh @@ -97,9 +97,6 @@ fi # come from there and minimal Ubuntu installs may have it turned off. hide_output add-apt-repository -y universe -# Install the certbot PPA. -hide_output add-apt-repository -y ppa:certbot/certbot - # Install the duplicity PPA. hide_output add-apt-repository -y ppa:duplicity-team/duplicity-release-git From 78d71498faaf93374273b4423fc0c4bd85883df7 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sun, 19 Jun 2022 07:12:02 -0400 Subject: [PATCH 06/35] Upgrade from PHP 7.2 to 8.0 for Ubuntu 22.04 * Add the PHP PPA. * Specify the version when invoking the php CLI. * Specify the version in package names. * Update paths to 8.0 (using a variable in the setup scripts). * Update z-push's php-xsl dependency to php8.0-xml. * php-json is now built-into PHP. Although PHP 8.1 is the stock version in Ubuntu 22.04, it's not supported by Nextcloud yet, and it likely will never be supported by the the version of Nextcloud that succeeds the last version of Nextcloud that supports PHP 7.2, and we have to install the next version so that an upgrade is permitted, so skipping to PHP 8.1 may not be easily possible. --- conf/nginx-top.conf | 2 +- management/backup.py | 4 +-- setup/functions.sh | 2 ++ setup/management.sh | 2 +- setup/nextcloud.sh | 52 +++++++++++++++++------------------ setup/system.sh | 4 +++ setup/web.sh | 18 ++++++------ setup/webmail.sh | 10 +++---- setup/zpush.sh | 10 +++---- tools/owncloud-restore.sh | 6 ++-- tools/owncloud-unlockadmin.sh | 2 +- 11 files changed, 58 insertions(+), 54 deletions(-) diff --git a/conf/nginx-top.conf b/conf/nginx-top.conf index 4d888366..c3f4c0d6 100644 --- a/conf/nginx-top.conf +++ b/conf/nginx-top.conf @@ -7,6 +7,6 @@ ## your own --- please do not ask for help from us. upstream php-fpm { - server unix:/var/run/php/php7.2-fpm.sock; + server unix:/var/run/php/php8.0-fpm.sock; } diff --git a/management/backup.py b/management/backup.py index ce358e06..2e88c8d0 100755 --- a/management/backup.py +++ b/management/backup.py @@ -278,7 +278,7 @@ def perform_backup(full_backup): if quit: sys.exit(code) - service_command("php7.2-fpm", "stop", quit=True) + service_command("php8.0-fpm", "stop", quit=True) service_command("postfix", "stop", quit=True) service_command("dovecot", "stop", quit=True) @@ -312,7 +312,7 @@ def perform_backup(full_backup): # Start services again. service_command("dovecot", "start", quit=False) service_command("postfix", "start", quit=False) - service_command("php7.2-fpm", "start", quit=False) + service_command("php8.0-fpm", "start", quit=False) # Remove old backups. This deletes all backup data no longer needed # from more than 3 days ago. diff --git a/setup/functions.sh b/setup/functions.sh index 718a2283..151c5f40 100644 --- a/setup/functions.sh +++ b/setup/functions.sh @@ -4,6 +4,8 @@ # -o pipefail: don't ignore errors in the non-last command in a pipeline set -euo pipefail +PHP_VER=8.0 + function hide_output { # This function hides the output of a command unless the command fails # and returns a non-zero exit code. diff --git a/setup/management.sh b/setup/management.sh index 483a9c5d..a4e9c8d2 100755 --- a/setup/management.sh +++ b/setup/management.sh @@ -25,7 +25,7 @@ done # # certbot installs EFF's certbot which we use to # provision free TLS certificates. -apt_install duplicity python-pip virtualenv certbot rsync +apt_install duplicity python3-pip virtualenv certbot rsync # b2sdk is used for backblaze backups. # boto is used for amazon aws backups. diff --git a/setup/nextcloud.sh b/setup/nextcloud.sh index 5525a37a..4cd5a86f 100755 --- a/setup/nextcloud.sh +++ b/setup/nextcloud.sh @@ -44,10 +44,10 @@ user_external_hash=3bf2609061d7214e7f0f69dd8883e55c4ec8f50a apt-get purge -qq -y owncloud* # we used to use the package manager -apt_install php php-fpm \ - php-cli php-sqlite3 php-gd php-imap php-curl php-pear curl \ - php-dev php-gd php-xml php-mbstring php-zip php-apcu php-json \ - php-intl php-imagick php-gmp php-bcmath +apt_install curl php${PHP_VER} php${PHP_VER}-fpm \ + php${PHP_VER}-cli php${PHP_VER}-sqlite3 php${PHP_VER}-gd php${PHP_VER}-imap php${PHP_VER}-curl \ + php${PHP_VER}-dev php${PHP_VER}-gd php${PHP_VER}-xml php${PHP_VER}-mbstring php${PHP_VER}-zip php${PHP_VER}-apcu \ + php${PHP_VER}-intl php${PHP_VER}-imagick php${PHP_VER}-gmp php${PHP_VER}-bcmath InstallNextcloud() { @@ -112,20 +112,20 @@ InstallNextcloud() { if [ -e $STORAGE_ROOT/owncloud/owncloud.db ]; then # ownCloud 8.1.1 broke upgrades. It may fail on the first attempt, but # that can be OK. - sudo -u www-data php /usr/local/lib/owncloud/occ upgrade + sudo -u www-data php$PHP_VER /usr/local/lib/owncloud/occ upgrade if [ \( $? -ne 0 \) -a \( $? -ne 3 \) ]; then echo "Trying ownCloud upgrade again to work around ownCloud upgrade bug..." - sudo -u www-data php /usr/local/lib/owncloud/occ upgrade + sudo -u www-data php$PHP_VER /usr/local/lib/owncloud/occ upgrade if [ \( $? -ne 0 \) -a \( $? -ne 3 \) ]; then exit 1; fi - sudo -u www-data php /usr/local/lib/owncloud/occ maintenance:mode --off + sudo -u www-data php$PHP_VER /usr/local/lib/owncloud/occ maintenance:mode --off echo "...which seemed to work." fi # Add missing indices. NextCloud didn't include this in the normal upgrade because it might take some time. - sudo -u www-data php /usr/local/lib/owncloud/occ db:add-missing-indices + sudo -u www-data php$PHP_VER /usr/local/lib/owncloud/occ db:add-missing-indices # Run conversion to BigInt identifiers, this process may take some time on large tables. - sudo -u www-data php /usr/local/lib/owncloud/occ db:convert-filecache-bigint --no-interaction + sudo -u www-data php$PHP_VER /usr/local/lib/owncloud/occ db:convert-filecache-bigint --no-interaction fi } @@ -137,7 +137,7 @@ InstallNextcloud() { # If config.php exists, get version number, otherwise CURRENT_NEXTCLOUD_VER is empty. if [ -f "$STORAGE_ROOT/owncloud/config.php" ]; then - CURRENT_NEXTCLOUD_VER=$(php -r "include(\"$STORAGE_ROOT/owncloud/config.php\"); echo(\$CONFIG['version']);") + CURRENT_NEXTCLOUD_VER=$(php$PHP_VER -r "include(\"$STORAGE_ROOT/owncloud/config.php\"); echo(\$CONFIG['version']);") else CURRENT_NEXTCLOUD_VER="" fi @@ -146,8 +146,8 @@ fi # from the version currently installed, do the install/upgrade if [ ! -d /usr/local/lib/owncloud/ ] || [[ ! ${CURRENT_NEXTCLOUD_VER} =~ ^$nextcloud_ver ]]; then - # Stop php-fpm if running. If theyre not running (which happens on a previously failed install), dont bail. - service php7.2-fpm stop &> /dev/null || /bin/true + # Stop php-fpm if running. If they are not running (which happens on a previously failed install), dont bail. + service php$PHP_VER-fpm stop &> /dev/null || /bin/true # Backup the existing ownCloud/Nextcloud. # Create a backup directory to store the current installation and database to @@ -280,7 +280,7 @@ EOF # Execute Nextcloud's setup step, which creates the Nextcloud sqlite database. # It also wipes it if it exists. And it updates config.php with database # settings and deletes the autoconfig.php file. - (cd /usr/local/lib/owncloud; sudo -u www-data php /usr/local/lib/owncloud/index.php;) + (cd /usr/local/lib/owncloud; sudo -u www-data php$PHP_VER /usr/local/lib/owncloud/index.php;) fi # Update config.php. @@ -296,7 +296,7 @@ fi # Use PHP to read the settings file, modify it, and write out the new settings array. TIMEZONE=$(cat /etc/timezone) CONFIG_TEMP=$(/bin/mktemp) -php < $CONFIG_TEMP && mv $CONFIG_TEMP $STORAGE_ROOT/owncloud/config.php; +php$PHP_VER < $CONFIG_TEMP && mv $CONFIG_TEMP $STORAGE_ROOT/owncloud/config.php; /etc/cron.d/mailinabox-nextcloud << EOF; #!/bin/bash # Mail-in-a-Box -*/5 * * * * root sudo -u www-data php -f /usr/local/lib/owncloud/cron.php +*/5 * * * * root sudo -u www-data php$PHP_VER -f /usr/local/lib/owncloud/cron.php EOF chmod +x /etc/cron.d/mailinabox-nextcloud @@ -387,4 +387,4 @@ rm -f /etc/cron.hourly/mailinabox-owncloud # ``` # Enable PHP modules and restart PHP. -restart_service php7.2-fpm +restart_service php$PHP_VER-fpm diff --git a/setup/system.sh b/setup/system.sh index b128a6bf..8c7ff137 100755 --- a/setup/system.sh +++ b/setup/system.sh @@ -100,6 +100,10 @@ hide_output add-apt-repository -y universe # Install the duplicity PPA. hide_output add-apt-repository -y ppa:duplicity-team/duplicity-release-git +# Stock PHP is now 8.1, but we're transitioning through 8.0 because +# of Nextcloud. +hide_output add-apt-repository --y ppa:ondrej/php + # ### Update Packages # Update system packages to make sure we have the latest upstream versions diff --git a/setup/web.sh b/setup/web.sh index 4433ff0d..392b6d92 100755 --- a/setup/web.sh +++ b/setup/web.sh @@ -19,7 +19,7 @@ fi echo "Installing Nginx (web server)..." -apt_install nginx php-cli php-fpm idn2 +apt_install nginx php${PHP_VER}-cli php${PHP_VER}-fpm idn2 rm -f /etc/nginx/sites-enabled/default @@ -46,15 +46,15 @@ tools/editconf.py /etc/nginx/nginx.conf -s \ ssl_protocols="TLSv1.2 TLSv1.3;" # Tell PHP not to expose its version number in the X-Powered-By header. -tools/editconf.py /etc/php/7.2/fpm/php.ini -c ';' \ +tools/editconf.py /etc/php/$PHP_VER/fpm/php.ini -c ';' \ expose_php=Off # Set PHPs default charset to UTF-8, since we use it. See #367. -tools/editconf.py /etc/php/7.2/fpm/php.ini -c ';' \ +tools/editconf.py /etc/php/$PHP_VER/fpm/php.ini -c ';' \ default_charset="UTF-8" # Configure the path environment for php-fpm -tools/editconf.py /etc/php/7.2/fpm/pool.d/www.conf -c ';' \ +tools/editconf.py /etc/php/$PHP_VER/fpm/pool.d/www.conf -c ';' \ env[PATH]=/usr/local/bin:/usr/bin:/bin \ # Configure php-fpm based on the amount of memory the machine has @@ -64,7 +64,7 @@ tools/editconf.py /etc/php/7.2/fpm/pool.d/www.conf -c ';' \ TOTAL_PHYSICAL_MEM=$(head -n 1 /proc/meminfo | awk '{print $2}' || /bin/true) if [ $TOTAL_PHYSICAL_MEM -lt 1000000 ] then - tools/editconf.py /etc/php/7.2/fpm/pool.d/www.conf -c ';' \ + tools/editconf.py /etc/php/$PHP_VER/fpm/pool.d/www.conf -c ';' \ pm=ondemand \ pm.max_children=8 \ pm.start_servers=2 \ @@ -72,7 +72,7 @@ then pm.max_spare_servers=3 elif [ $TOTAL_PHYSICAL_MEM -lt 2000000 ] then - tools/editconf.py /etc/php/7.2/fpm/pool.d/www.conf -c ';' \ + tools/editconf.py /etc/php/$PHP_VER/fpm/pool.d/www.conf -c ';' \ pm=ondemand \ pm.max_children=16 \ pm.start_servers=4 \ @@ -80,14 +80,14 @@ then pm.max_spare_servers=6 elif [ $TOTAL_PHYSICAL_MEM -lt 3000000 ] then - tools/editconf.py /etc/php/7.2/fpm/pool.d/www.conf -c ';' \ + tools/editconf.py /etc/php/$PHP_VER/fpm/pool.d/www.conf -c ';' \ pm=dynamic \ pm.max_children=60 \ pm.start_servers=6 \ pm.min_spare_servers=3 \ pm.max_spare_servers=9 else - tools/editconf.py /etc/php/7.2/fpm/pool.d/www.conf -c ';' \ + tools/editconf.py /etc/php/$PHP_VER/fpm/pool.d/www.conf -c ';' \ pm=dynamic \ pm.max_children=120 \ pm.start_servers=12 \ @@ -147,7 +147,7 @@ chown -R $STORAGE_USER $STORAGE_ROOT/www # Start services. restart_service nginx -restart_service php7.2-fpm +restart_service php$PHP_VER-fpm # Open ports. ufw_allow http diff --git a/setup/webmail.sh b/setup/webmail.sh index 938bcdd3..839b6536 100755 --- a/setup/webmail.sh +++ b/setup/webmail.sh @@ -22,8 +22,8 @@ source /etc/mailinabox.conf # load global vars echo "Installing Roundcube (webmail)..." apt_install \ dbconfig-common \ - php-cli php-sqlite3 php-intl php-json php-common php-curl php-ldap \ - php-gd php-pspell libjs-jquery libjs-jquery-mousewheel libmagic1 php-mbstring + php${PHP_VER}-cli php${PHP_VER}-sqlite3 php${PHP_VER}-intl php${PHP_VER}-common php${PHP_VER}-curl php${PHP_VER}-ldap \ + php${PHP_VER}-gd php${PHP_VER}-pspell php${PHP_VER}-mbstring libjs-jquery libjs-jquery-mousewheel libmagic1 # Install Roundcube from source if it is not already present or if it is out of date. # Combine the Roundcube version number with the commit hash of plugins to track @@ -202,10 +202,10 @@ chown -f -R root.www-data ${RCM_PLUGIN_DIR}/carddav chmod -R 774 ${RCM_PLUGIN_DIR}/carddav # Run Roundcube database migration script (database is created if it does not exist) -${RCM_DIR}/bin/updatedb.sh --dir ${RCM_DIR}/SQL --package roundcube +php$PHP_VER ${RCM_DIR}/bin/updatedb.sh --dir ${RCM_DIR}/SQL --package roundcube chown www-data:www-data $STORAGE_ROOT/mail/roundcube/roundcube.sqlite chmod 664 $STORAGE_ROOT/mail/roundcube/roundcube.sqlite # Enable PHP modules. -phpenmod -v php mcrypt imap -restart_service php7.2-fpm +phpenmod -v $PHP_VER imap +restart_service php$PHP_VER-fpm diff --git a/setup/zpush.sh b/setup/zpush.sh index c1c00f2a..4fdfadc4 100755 --- a/setup/zpush.sh +++ b/setup/zpush.sh @@ -17,9 +17,9 @@ source /etc/mailinabox.conf # load global vars echo "Installing Z-Push (Exchange/ActiveSync server)..." apt_install \ - php-soap php-imap libawl-php php-xsl + php${PHP_VER}-soap php${PHP_VER}-imap libawl-php php$PHP_VER-xml -phpenmod -v php imap +phpenmod -v $PHP_VER imap # Copy Z-Push into place. VERSION=2.6.2 @@ -42,8 +42,6 @@ if [ $needs_update == 1 ]; then rm -rf /tmp/z-push.zip /tmp/z-push rm -f /usr/sbin/z-push-{admin,top} - ln -s /usr/local/lib/z-push/z-push-admin.php /usr/sbin/z-push-admin - ln -s /usr/local/lib/z-push/z-push-top.php /usr/sbin/z-push-top echo $VERSION > /usr/local/lib/z-push/version fi @@ -102,8 +100,8 @@ EOF # Restart service. -restart_service php7.2-fpm +restart_service php$PHP_VER-fpm # Fix states after upgrade -hide_output z-push-admin -a fixstates +hide_output php$PHP_VER /usr/local/lib/z-push/z-push-admin.php -a fixstates diff --git a/tools/owncloud-restore.sh b/tools/owncloud-restore.sh index 4b0ba4de..108c8b77 100755 --- a/tools/owncloud-restore.sh +++ b/tools/owncloud-restore.sh @@ -26,7 +26,7 @@ if [ ! -f $1/config.php ]; then fi echo "Restoring backup from $1" -service php7.2-fpm stop +service php8.0-fpm stop # remove the current ownCloud/Nextcloud installation rm -rf /usr/local/lib/owncloud/ @@ -43,7 +43,7 @@ ln -sf $STORAGE_ROOT/owncloud/config.php /usr/local/lib/owncloud/config/config.p chown -f -R www-data.www-data $STORAGE_ROOT/owncloud /usr/local/lib/owncloud chown www-data.www-data $STORAGE_ROOT/owncloud/config.php -sudo -u www-data php /usr/local/lib/owncloud/occ maintenance:mode --off +sudo -u www-data php$PHP_VER /usr/local/lib/owncloud/occ maintenance:mode --off -service php7.2-fpm start +service php8.0-fpm start echo "Done" diff --git a/tools/owncloud-unlockadmin.sh b/tools/owncloud-unlockadmin.sh index 50e3b010..0b7ffc9c 100755 --- a/tools/owncloud-unlockadmin.sh +++ b/tools/owncloud-unlockadmin.sh @@ -20,4 +20,4 @@ echo echo Press enter to continue. read -sudo -u www-data php /usr/local/lib/owncloud/occ group:adduser admin $ADMIN && echo Done. +sudo -u www-data php$PHP_VER /usr/local/lib/owncloud/occ group:adduser admin $ADMIN && echo Done. From 1eddf9a22030edb000c5fd75c584da397f12d5dd Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sat, 8 Jan 2022 19:29:25 -0500 Subject: [PATCH 07/35] Upgrade to Nextcloud 23.0.4 The first version supporting PHP 8.0 is Nextcloud 21. Therefore we can add migrations only to Nextcloud 21 forward, and so we only support migrating from Nextcloud 20 (Mail-in-a-Box versions v0.51+). Migration steps through Nextcloud 21 and 22 are added. Also: * Fix PHP APUc settings to be before Nextcloud tools are run. --- setup/nextcloud.sh | 66 ++++++++++++++++------------------------------ 1 file changed, 22 insertions(+), 44 deletions(-) diff --git a/setup/nextcloud.sh b/setup/nextcloud.sh index 4cd5a86f..6614c134 100755 --- a/setup/nextcloud.sh +++ b/setup/nextcloud.sh @@ -21,8 +21,8 @@ echo "Installing Nextcloud (contacts/calendar)..." # we automatically install intermediate versions as needed. # * The hash is the SHA1 hash of the ZIP package, which you can find by just running this script and # copying it from the error message when it doesn't match what is below. -nextcloud_ver=20.0.14 -nextcloud_hash=92cac708915f51ee2afc1787fd845476fd090c81 +nextcloud_ver=23.0.4 +nextcloud_hash=87afec0bf90b3c66289e6fedd851867bc5a58f01 # Nextcloud apps # -------------- @@ -33,10 +33,10 @@ nextcloud_hash=92cac708915f51ee2afc1787fd845476fd090c81 # https://github.com/nextcloud/user_external/blob/master/appinfo/info.xml # * The hash is the SHA1 hash of the ZIP package, which you can find by just running this script and # copying it from the error message when it doesn't match what is below. -contacts_ver=4.0.7 -contacts_hash=45e7cf4bfe99cd8d03625cf9e5a1bb2e90549136 -calendar_ver=3.0.4 -calendar_hash=d0284b68135777ec9ca713c307216165b294d0fe +contacts_ver=4.1.0 +contacts_hash=697f6b4a664e928d72414ea2731cb2c9d1dc3077 +calendar_ver=3.2.2 +calendar_hash=ce4030ab57f523f33d5396c6a81396d440756f5f user_external_ver=1.0.0 user_external_hash=3bf2609061d7214e7f0f69dd8883e55c4ec8f50a @@ -49,6 +49,11 @@ apt_install curl php${PHP_VER} php${PHP_VER}-fpm \ php${PHP_VER}-dev php${PHP_VER}-gd php${PHP_VER}-xml php${PHP_VER}-mbstring php${PHP_VER}-zip php${PHP_VER}-apcu \ php${PHP_VER}-intl php${PHP_VER}-imagick php${PHP_VER}-gmp php${PHP_VER}-bcmath +# Enable APC before Nextcloud tools are run. +tools/editconf.py /etc/php/$PHP_VER/mods-available/apcu.ini -c ';' \ + apc.enabled=1 \ + apc.enable_cli=1 + InstallNextcloud() { version=$1 @@ -174,42 +179,19 @@ if [ ! -d /usr/local/lib/owncloud/ ] || [[ ! ${CURRENT_NEXTCLOUD_VER} =~ ^$nextc elif [[ ${CURRENT_NEXTCLOUD_VER} =~ ^1[012] ]]; then echo "Upgrades from Mail-in-a-Box prior to v0.28 (dated July 30, 2018) with Nextcloud < 13.0.6 (you have ownCloud 10, 11 or 12) are not supported. Upgrade to Mail-in-a-Box version v0.30 first. Setup will continue, but skip the Nextcloud migration." return 0 - elif [[ ${CURRENT_NEXTCLOUD_VER} =~ ^13 ]]; then - # If we are running Nextcloud 13, upgrade to Nextcloud 14 - InstallNextcloud 14.0.6 4e43a57340f04c2da306c8eea98e30040399ae5a 3.3.0 e55d0357c6785d3b1f3b5f21780cb6d41d32443a 2.0.3 9d9717b29337613b72c74e9914c69b74b346c466 - CURRENT_NEXTCLOUD_VER="14.0.6" + elif [[ ${CURRENT_NEXTCLOUD_VER} =~ ^1[3456789] ]]; then + echo "Upgrades from Mail-in-a-Box prior to v60 with Nextcloud 19 or earlier are not supported. Upgrade to the latest Mail-in-a-Box version supported on your machine first. Setup will continue, but skip the Nextcloud migration." + return 0 + elif [[ ${CURRENT_NEXTCLOUD_VER} =~ ^20 ]]; then + InstallNextcloud 21.0.7 f5c7079c5b56ce1e301c6a27c0d975d608bb01c9 4.0.7 8ab31d205408e4f12067d8a4daa3595d46b513e3 3.0.4 6fb1e998d307c53245faf1c37a96eb982bbee8ba 1.0.0 3bf2609061d7214e7f0f69dd8883e55c4ec8f50a + CURRENT_NEXTCLOUD_VER="21.0.7" + elif [[ ${CURRENT_NEXTCLOUD_VER} =~ ^21 ]]; then + InstallNextcloud 22.2.6 9d39741f051a8da42ff7df46ceef2653a1dc70d9 4.1.0 38653b507bd7d953816bbc5e8bea7855867eb1cd 3.2.2 54e9a836adc739be4a2a9301b8d6d2e9d88e02f4 3.0.0 0df781b261f55bbde73d8c92da3f99397000972f + CURRENT_NEXTCLOUD_VER="22.2.6" fi - if [[ ${CURRENT_NEXTCLOUD_VER} =~ ^14 ]]; then - # During the upgrade from Nextcloud 14 to 15, user_external may cause the upgrade to fail. - # We will disable it here before the upgrade and install it again after the upgrade. - hide_output sudo -u www-data php /usr/local/lib/owncloud/console.php app:disable user_external - InstallNextcloud 15.0.8 4129d8d4021c435f2e86876225fb7f15adf764a3 3.3.0 e55d0357c6785d3b1f3b5f21780cb6d41d32443a 2.0.3 a1f3835c752929e3598eb94f22300516867ac6ab 0.7.0 555a94811daaf5bdd336c5e48a78aa8567b86437 - CURRENT_NEXTCLOUD_VER="15.0.8" - fi - if [[ ${CURRENT_NEXTCLOUD_VER} =~ ^15 ]]; then - InstallNextcloud 16.0.6 0bb3098455ec89f5af77a652aad553ad40a88819 3.3.0 e55d0357c6785d3b1f3b5f21780cb6d41d32443a 2.0.3 a1f3835c752929e3598eb94f22300516867ac6ab 0.7.0 555a94811daaf5bdd336c5e48a78aa8567b86437 - CURRENT_NEXTCLOUD_VER="16.0.6" - fi - if [[ ${CURRENT_NEXTCLOUD_VER} =~ ^16 ]]; then - InstallNextcloud 17.0.6 50b98d2c2f18510b9530e558ced9ab51eb4f11b0 3.3.0 e55d0357c6785d3b1f3b5f21780cb6d41d32443a 2.0.3 a1f3835c752929e3598eb94f22300516867ac6ab 0.7.0 555a94811daaf5bdd336c5e48a78aa8567b86437 - CURRENT_NEXTCLOUD_VER="17.0.6" - fi - if [[ ${CURRENT_NEXTCLOUD_VER} =~ ^17 ]]; then - # Don't exit the install if this column already exists (see #2076) - (echo "ALTER TABLE oc_flow_operations ADD COLUMN entity VARCHAR;" | sqlite3 $STORAGE_ROOT/owncloud/owncloud.db 2>/dev/null) || true - InstallNextcloud 18.0.10 39c0021a8b8477c3f1733fddefacfa5ebf921c68 3.4.1 8f685e7dc99758636d660d595e389c324e51e9d1 2.0.3 a1f3835c752929e3598eb94f22300516867ac6ab 1.0.0 3bf2609061d7214e7f0f69dd8883e55c4ec8f50a - CURRENT_NEXTCLOUD_VER="18.0.10" - fi - if [[ ${CURRENT_NEXTCLOUD_VER} =~ ^18 ]]; then - InstallNextcloud 19.0.4 01e98791ba12f4860d3d4047b9803f97a1b55c60 3.4.1 8f685e7dc99758636d660d595e389c324e51e9d1 2.0.3 a1f3835c752929e3598eb94f22300516867ac6ab 1.0.0 3bf2609061d7214e7f0f69dd8883e55c4ec8f50a - CURRENT_NEXTCLOUD_VER="19.0.4" - fi fi InstallNextcloud $nextcloud_ver $nextcloud_hash $contacts_ver $contacts_hash $calendar_ver $calendar_hash $user_external_ver $user_external_hash - - # Nextcloud 20 needs to have some optional columns added - sudo -u www-data php /usr/local/lib/owncloud/occ db:add-missing-columns fi # ### Configuring Nextcloud @@ -300,6 +282,8 @@ php$PHP_VER < $CONFIG_TEMP && mv $CONFIG_TEMP $STORAGE_ROOT/owncloud/confi /etc/cron.d/mailinabox-nextcloud << EOF; #!/bin/bash From 0a7b9d5089a5183f81fda2b28beae472980a9f7f Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sun, 9 Jan 2022 10:32:36 -0500 Subject: [PATCH 08/35] Update dovecot, spampd settings for Ubuntu 22.04 * dovecot's ssl_protocols became ssl_min_protocol in 2.3 * spampd fixed a bug so we can remove lmtp_destination_recipient_limit=1 in postfix --- setup/mail-dovecot.sh | 2 +- setup/mail-postfix.sh | 15 ++++++++------- tools/editconf.py | 26 +++++++++++++++++++------- 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/setup/mail-dovecot.sh b/setup/mail-dovecot.sh index 26d32895..394ede8b 100755 --- a/setup/mail-dovecot.sh +++ b/setup/mail-dovecot.sh @@ -84,7 +84,7 @@ tools/editconf.py /etc/dovecot/conf.d/10-ssl.conf \ ssl=required \ "ssl_cert=<$STORAGE_ROOT/ssl/ssl_certificate.pem" \ "ssl_key=<$STORAGE_ROOT/ssl/ssl_private_key.pem" \ - "ssl_protocols=TLSv1.2" \ + "ssl_min_protocol=TLSv1.2" \ "ssl_cipher_list=ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384" \ "ssl_prefer_server_ciphers=no" \ "ssl_dh_parameters_length=2048" diff --git a/setup/mail-postfix.sh b/setup/mail-postfix.sh index dc1fff85..196d371d 100755 --- a/setup/mail-postfix.sh +++ b/setup/mail-postfix.sh @@ -13,8 +13,8 @@ # destinations according to aliases, and passses email on to # another service for local mail delivery. # -# The first hop in local mail delivery is to Spamassassin via -# LMTP. Spamassassin then passes mail over to Dovecot for +# The first hop in local mail delivery is to spampd via +# LMTP. spampd then passes mail over to Dovecot for # storage in the user's mailbox. # # Postfix also listens on ports 465/587 (SMTPS, SMTP+STARTLS) for @@ -193,16 +193,17 @@ tools/editconf.py /etc/postfix/main.cf \ # ### Incoming Mail -# Pass any incoming mail over to a local delivery agent. Spamassassin -# will act as the LDA agent at first. It is listening on port 10025 -# with LMTP. Spamassassin will pass the mail over to Dovecot after. +# Pass mail to spampd, which acts as the local delivery agent (LDA), +# which then passes the mail over to the Dovecot LMTP server after. +# spampd runs on port 10025 by default. # # In a basic setup we would pass mail directly to Dovecot by setting # virtual_transport to `lmtp:unix:private/dovecot-lmtp`. tools/editconf.py /etc/postfix/main.cf "virtual_transport=lmtp:[127.0.0.1]:10025" -# Because of a spampd bug, limit the number of recipients in each connection. +# Clear the lmtp_destination_recipient_limit setting which in previous +# versions of Mail-in-a-Box was set to 1 because of a spampd bug. # See https://github.com/mail-in-a-box/mailinabox/issues/1523. -tools/editconf.py /etc/postfix/main.cf lmtp_destination_recipient_limit=1 +tools/editconf.py /etc/postfix/main.cf -e lmtp_destination_recipient_limit= # Who can send mail to us? Some basic filters. diff --git a/tools/editconf.py b/tools/editconf.py index d665f861..e80742e4 100755 --- a/tools/editconf.py +++ b/tools/editconf.py @@ -14,6 +14,10 @@ # # NAME VALUE # +# If the -e option is given and VALUE is empty, the setting is removed +# from the configuration file if it is set (i.e. existing occurrences +# are commented out and no new setting is added). +# # If the -c option is given, then the supplied character becomes the comment character # # If the -w option is given, then setting lines continue onto following @@ -35,6 +39,7 @@ settings = sys.argv[2:] delimiter = "=" delimiter_re = r"\s*=\s*" +erase_setting = False comment_char = "#" folded_lines = False testing = False @@ -44,6 +49,9 @@ while settings[0][0] == "-" and settings[0] != "--": # Space is the delimiter delimiter = " " delimiter_re = r"\s+" + elif opt == "-e": + # Erase settings that have empty values. + erase_setting = True elif opt == "-w": # Line folding is possible in this file. folded_lines = True @@ -81,7 +89,7 @@ while len(input_lines) > 0: # See if this line is for any settings passed on the command line. for i in range(len(settings)): - # Check that this line contain this setting from the command-line arguments. + # Check if this line contain this setting from the command-line arguments. name, val = settings[i].split("=", 1) m = re.match( "(\s*)" @@ -91,8 +99,10 @@ while len(input_lines) > 0: if not m: continue indent, is_comment, existing_val = m.groups() - # If this is already the setting, do nothing. - if is_comment is None and existing_val == val: + # If this is already the setting, keep it in the file, except: + # * If we've already seen it before, then remove this duplicate line. + # * If val is empty and erase_setting is on, then comment it out. + if is_comment is None and existing_val == val and not (not val and erase_setting): # It may be that we've already inserted this setting higher # in the file so check for that first. if i in found: break @@ -107,8 +117,9 @@ while len(input_lines) > 0: # the line is already commented, pass it through buf += line - # if this option oddly appears more than once, don't add the setting again - if i in found: + # if this option already is set don't add the setting again, + # or if we're clearing the setting with -e, don't add it + if (i in found) or (not val and erase_setting): break # add the new setting @@ -122,9 +133,10 @@ while len(input_lines) > 0: # If did not match any setting names, pass this line through. buf += line -# Put any settings we didn't see at the end of the file. +# Put any settings we didn't see at the end of the file, +# except settings being cleared. for i in range(len(settings)): - if i not in found: + if (i not in found) and not (not val and erase_setting): name, val = settings[i].split("=", 1) buf += name + delimiter + val + "\n" From c23dd701f07d3d79a0f1ed88f7288ed819e012a4 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sun, 9 Jan 2022 10:39:29 -0500 Subject: [PATCH 09/35] Start changelog and instructions updates for version 60 supporting Ubuntu 22.04 To scan for updated apt packages in Ubuntu 22.04, I ran on Ubuntu 18.04 and 22.04 and compared the output: ``` for package in openssl openssh-client haveged pollinate fail2ban ufw bind9 nsd ldnsutils nginx dovecot-core postfix opendkim opendkim-tools opendmarc postgrey spampd razor pyzor dovecot-antispam sqlite3 duplicity certbot munin munin-node php python3; do echo -n "$package "; dpkg-query --showformat='${Version}' --show $package; echo done ``` --- CHANGELOG.md | 19 +++++++++++++++++++ README.md | 6 +++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c20eb7d..1234a898 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,25 @@ CHANGELOG ========= +Version 60 (date TBD) +--------------------- + +This is the first release for Ubuntu 22.04. + +**Before upgrading**, you must **first upgrade your existing Ubuntu 18.04 box to Mail-in-a-Box v0.51 or later**, if you haven't already done so. That may not be possible after Ubuntu 18.04 reaches its end of life in April 2023, so please complete the upgrade well before then. (If you are not using Nextcloud's contacts or calendar, you can migrate to the latest version of Mail-in-a-Box from any previous version.) + +For complete upgrade instructions, see: + +LINK TBD + +No features of Mail-in-a-Box have changed in this release, but with the newer version of Ubuntu the following software packages we use are updated: + +* dovecot is upgraded to 2.3.16, postfix to 3.6.4, opendmark to 1.4 (which adds ARC-Authentication-Results headers), and spampd to 2.53 (alleviating a mail delivery rate limiting bug). +* Nextcloud is upgraded to 23.0.4 with PHP updated from 7.2 to 8.0. +* certbot is upgraded to 1.21 (via the Ubuntu repository instead of a PPA). +* fail2ban is upgraded to 0.11.2. +* nginx is upgraded to 1.18. + Version 57a (June 19, 2022) --------------------------- diff --git a/README.md b/README.md index 8fd2405c..ab9a8039 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Additionally, this project has a [Code of Conduct](CODE_OF_CONDUCT.md), which su In The Box ---------- -Mail-in-a-Box turns a fresh Ubuntu 18.04 LTS 64-bit machine into a working mail server by installing and configuring various components. +Mail-in-a-Box turns a fresh Ubuntu 22.04 LTS 64-bit machine into a working mail server by installing and configuring various components. It is a one-click email appliance. There are no user-configurable setup options. It "just works." @@ -54,13 +54,13 @@ Installation See the [setup guide](https://mailinabox.email/guide.html) for detailed, user-friendly instructions. -For experts, start with a completely fresh (really, I mean it) Ubuntu 18.04 LTS 64-bit machine. On the machine... +For experts, start with a completely fresh (really, I mean it) Ubuntu 22.04 LTS 64-bit machine. On the machine... Clone this repository and checkout the tag corresponding to the most recent release: $ git clone https://github.com/mail-in-a-box/mailinabox $ cd mailinabox - $ git checkout v57a + $ git checkout v60 Begin the installation. From 558f2db31f7d43511788e14c1b5d052baa64a4b8 Mon Sep 17 00:00:00 2001 From: Felix Matouschek Date: Sun, 5 Jun 2022 15:38:56 +0200 Subject: [PATCH 10/35] system.sh: Remove no longer needed haveged (#2090) Starting from kernels 5.6 haveged is obsolete. Therefore remove it in Ubuntu 22.04. See https://github.com/jirka-h/haveged/issues/57 --- setup/system.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/setup/system.sh b/setup/system.sh index 8c7ff137..d9ee1f3f 100755 --- a/setup/system.sh +++ b/setup/system.sh @@ -124,9 +124,6 @@ apt_get_quiet autoremove # Install basic utilities. # -# * haveged: Provides extra entropy to /dev/random so it doesn't stall -# when generating random numbers for private keys (e.g. during -# ldns-keygen). # * unattended-upgrades: Apt tool to install security updates automatically. # * cron: Runs background processes periodically. # * ntp: keeps the system time correct @@ -141,7 +138,7 @@ apt_get_quiet autoremove echo Installing system packages... apt_install python3 python3-dev python3-pip python3-setuptools \ netcat-openbsd wget curl git sudo coreutils bc \ - haveged pollinate openssh-client unzip \ + pollinate openssh-client unzip \ unattended-upgrades cron ntp fail2ban rsyslog # ### Suppress Upgrade Prompts From 87e6df9e2853d3a38db20a4ce3ea80d58fef4e6a Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sun, 19 Jun 2022 06:47:23 -0400 Subject: [PATCH 11/35] Fix roundcube dependency missing imap and unneeded ldap --- setup/webmail.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/webmail.sh b/setup/webmail.sh index 839b6536..e064a201 100755 --- a/setup/webmail.sh +++ b/setup/webmail.sh @@ -22,7 +22,7 @@ source /etc/mailinabox.conf # load global vars echo "Installing Roundcube (webmail)..." apt_install \ dbconfig-common \ - php${PHP_VER}-cli php${PHP_VER}-sqlite3 php${PHP_VER}-intl php${PHP_VER}-common php${PHP_VER}-curl php${PHP_VER}-ldap \ + php${PHP_VER}-cli php${PHP_VER}-sqlite3 php${PHP_VER}-intl php${PHP_VER}-common php${PHP_VER}-curl php${PHP_VER}-imap \ php${PHP_VER}-gd php${PHP_VER}-pspell php${PHP_VER}-mbstring libjs-jquery libjs-jquery-mousewheel libmagic1 # Install Roundcube from source if it is not already present or if it is out of date. From ab71abbc7cb5f1fae724c307c196fa53ddd5a7b1 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sun, 19 Jun 2022 06:48:55 -0400 Subject: [PATCH 12/35] Update to latest cryptography Python package, add missing source at top of management.sh so it can run standalone (needs STORAGE_ROOT) --- management/ssl_certificates.py | 33 +++++++++++++++------------------ setup/management.sh | 3 ++- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/management/ssl_certificates.py b/management/ssl_certificates.py index 3e1b5856..ab4f2dc8 100755 --- a/management/ssl_certificates.py +++ b/management/ssl_certificates.py @@ -58,36 +58,33 @@ def get_ssl_certificates(env): # Not a valid PEM format for a PEM type we care about. continue - # Remember where we got this object. - pem._filename = fn - # Is it a private key? if isinstance(pem, RSAPrivateKey): - private_keys[pem.public_key().public_numbers()] = pem + private_keys[pem.public_key().public_numbers()] = { "filename": fn, "key": pem } # Is it a certificate? if isinstance(pem, Certificate): - certificates.append(pem) + certificates.append({ "filename": fn, "cert": pem }) # Process the certificates. domains = { } for cert in certificates: # What domains is this certificate good for? - cert_domains, primary_domain = get_certificate_domains(cert) - cert._primary_domain = primary_domain + cert_domains, primary_domain = get_certificate_domains(cert["cert"]) + cert["primary_domain"] = primary_domain # Is there a private key file for this certificate? - private_key = private_keys.get(cert.public_key().public_numbers()) + private_key = private_keys.get(cert["cert"].public_key().public_numbers()) if not private_key: continue - cert._private_key = private_key + cert["private_key"] = private_key # Add this cert to the list of certs usable for the domains. for domain in cert_domains: # The primary hostname can only use a certificate mapped # to the system private key. if domain == env['PRIMARY_HOSTNAME']: - if cert._private_key._filename != os.path.join(env['STORAGE_ROOT'], 'ssl', 'ssl_private_key.pem'): + if cert["private_key"]["filename"] != os.path.join(env['STORAGE_ROOT'], 'ssl', 'ssl_private_key.pem'): continue domains.setdefault(domain, []).append(cert) @@ -100,10 +97,10 @@ def get_ssl_certificates(env): #for c in cert_list: print(domain, c.not_valid_before, c.not_valid_after, "("+str(now)+")", c.issuer, c.subject, c._filename) cert_list.sort(key = lambda cert : ( # must be valid NOW - cert.not_valid_before <= now <= cert.not_valid_after, + cert["cert"].not_valid_before <= now <= cert["cert"].not_valid_after, # prefer one that is not self-signed - cert.issuer != cert.subject, + cert["cert"].issuer != cert["cert"].subject, ########################################################### # The above lines ensure that valid certificates are chosen @@ -113,7 +110,7 @@ def get_ssl_certificates(env): # prefer one with the expiration furthest into the future so # that we can easily rotate to new certs as we get them - cert.not_valid_after, + cert["cert"].not_valid_after, ########################################################### # We always choose the certificate that is good for the @@ -128,15 +125,15 @@ def get_ssl_certificates(env): # in case a certificate is installed in multiple paths, # prefer the... lexicographically last one? - cert._filename, + cert["filename"], ), reverse=True) cert = cert_list.pop(0) ret[domain] = { - "private-key": cert._private_key._filename, - "certificate": cert._filename, - "primary-domain": cert._primary_domain, - "certificate_object": cert, + "private-key": cert["private_key"]["filename"], + "certificate": cert["filename"], + "primary-domain": cert["primary_domain"], + "certificate_object": cert["cert"], } return ret diff --git a/setup/management.sh b/setup/management.sh index a4e9c8d2..52d38498 100755 --- a/setup/management.sh +++ b/setup/management.sh @@ -1,6 +1,7 @@ #!/bin/bash source setup/functions.sh +source /etc/mailinabox.conf # load global vars echo "Installing Mail-in-a-Box system management daemon..." @@ -51,7 +52,7 @@ hide_output $venv/bin/pip install --upgrade \ rtyaml "email_validator>=1.0.0" "exclusiveprocess" \ flask dnspython python-dateutil expiringdict \ qrcode[pil] pyotp \ - "idna>=2.0.0" "cryptography==2.2.2" psutil postfix-mta-sts-resolver \ + "idna>=2.0.0" "cryptography==37.0.2" psutil postfix-mta-sts-resolver \ b2sdk boto # CONFIGURATION From 268b31685da65b665fbd159c949e6b3291813bd1 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Thu, 28 Jul 2022 13:20:49 -0400 Subject: [PATCH 13/35] Ensure STORAGE_ROOT has a+rx permission since processes run by different system users need to access files within it --- setup/start.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/setup/start.sh b/setup/start.sh index bd743ac5..0626ab01 100755 --- a/setup/start.sh +++ b/setup/start.sh @@ -67,6 +67,10 @@ fi fi # Create the STORAGE_USER and STORAGE_ROOT directory if they don't already exist. +# +# Set the directory and all of its parent directories' permissions to world +# readable since it holds files owned by different processes. +# # If the STORAGE_ROOT is missing the mailinabox.version file that lists a # migration (schema) number for the files stored there, assume this is a fresh # installation to that directory and write the file to contain the current @@ -77,6 +81,8 @@ fi if [ ! -d $STORAGE_ROOT ]; then mkdir -p $STORAGE_ROOT fi +f=$STORAGE_ROOT +while [[ $f != / ]]; do chmod a+rx "$f"; f=$(dirname "$f"); done; if [ ! -f $STORAGE_ROOT/mailinabox.version ]; then setup/migrate.py --current > $STORAGE_ROOT/mailinabox.version chown $STORAGE_USER.$STORAGE_USER $STORAGE_ROOT/mailinabox.version From e0c0b5053c2c8e3092a2b82894e499a67669a253 Mon Sep 17 00:00:00 2001 From: David Duque Date: Wed, 27 Apr 2022 23:22:55 +0100 Subject: [PATCH 14/35] Upgrade Nextcloud External User Backend to v3.0.0 Co-Authored-By: Joshua Tauberer --- setup/nextcloud.sh | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/setup/nextcloud.sh b/setup/nextcloud.sh index 6614c134..b9c3a6d4 100755 --- a/setup/nextcloud.sh +++ b/setup/nextcloud.sh @@ -37,8 +37,8 @@ contacts_ver=4.1.0 contacts_hash=697f6b4a664e928d72414ea2731cb2c9d1dc3077 calendar_ver=3.2.2 calendar_hash=ce4030ab57f523f33d5396c6a81396d440756f5f -user_external_ver=1.0.0 -user_external_hash=3bf2609061d7214e7f0f69dd8883e55c4ec8f50a +user_external_ver=3.0.0 +user_external_hash=0df781b261f55bbde73d8c92da3f99397000972f # Clear prior packages and install dependencies from apt. @@ -95,7 +95,7 @@ InstallNextcloud() { # Starting with Nextcloud 15, the app user_external is no longer included in Nextcloud core, # we will install from their github repository. if [ -n "$version_user_external" ]; then - wget_verify https://github.com/nextcloud/user_external/releases/download/v$version_user_external/user_external-$version_user_external.tar.gz $hash_user_external /tmp/user_external.tgz + wget_verify https://github.com/nextcloud-releases/user_external/releases/download/v$version_user_external/user_external-v$version_user_external.tar.gz $hash_user_external /tmp/user_external.tgz tar -xf /tmp/user_external.tgz -C /usr/local/lib/owncloud/apps/ rm /tmp/user_external.tgz fi @@ -217,10 +217,10 @@ if [ ! -f $STORAGE_ROOT/owncloud/owncloud.db ]; then 'overwrite.cli.url' => '/cloud', 'user_backends' => array( array( - 'class' => 'OC_User_IMAP', - 'arguments' => array( - '127.0.0.1', 143, null - ), + 'class' => '\OCA\UserExternal\IMAP', + 'arguments' => array( + '127.0.0.1', 143, null, null, false, false + ), ), ), 'memcache.local' => '\OC\Memcache\APCu', @@ -295,7 +295,14 @@ include("$STORAGE_ROOT/owncloud/config.php"); \$CONFIG['mail_domain'] = '$PRIMARY_HOSTNAME'; -\$CONFIG['user_backends'] = array(array('class' => 'OC_User_IMAP','arguments' => array('127.0.0.1', 143, null),),); +\$CONFIG['user_backends'] = array( + array( + 'class' => '\OCA\UserExternal\IMAP', + 'arguments' => array( + '127.0.0.1', 143, null, null, false, false + ), + ), +); echo " /etc/cron.d/mailinabox-nextcloud << EOF; #!/bin/bash From d7244ed920a4807970794dc902a1b5cf9d00c351 Mon Sep 17 00:00:00 2001 From: Sudheesh Singanamalla Date: Fri, 19 Aug 2022 18:23:42 +0100 Subject: [PATCH 15/35] Fixes #2149 Append ; in policy strings for DMARC settings (#2151) Signed-off-by: Sudheesh Singanamalla --- management/dns_update.py | 4 ++-- tests/test_dns.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/management/dns_update.py b/management/dns_update.py index fde9b146..a3f376c8 100755 --- a/management/dns_update.py +++ b/management/dns_update.py @@ -298,7 +298,7 @@ def build_zone(domain, domain_properties, additional_records, env, is_zone=True) # Append a DMARC record. # Skip if the user has set a DMARC record already. if not has_rec("_dmarc", "TXT", prefix="v=DMARC1; "): - records.append(("_dmarc", "TXT", 'v=DMARC1; p=quarantine', "Recommended. Specifies that mail that does not originate from the box but claims to be from @%s or which does not have a valid DKIM signature is suspect and should be quarantined by the recipient's mail system." % domain)) + records.append(("_dmarc", "TXT", 'v=DMARC1; p=quarantine;', "Recommended. Specifies that mail that does not originate from the box but claims to be from @%s or which does not have a valid DKIM signature is suspect and should be quarantined by the recipient's mail system." % domain)) if domain_properties[domain]["user"]: # Add CardDAV/CalDAV SRV records on the non-primary hostname that points to the primary hostname @@ -363,7 +363,7 @@ def build_zone(domain, domain_properties, additional_records, env, is_zone=True) if not has_rec(qname, "TXT", prefix="v=spf1 "): records.append((qname, "TXT", 'v=spf1 -all', "Recommended. Prevents use of this domain name for outbound mail by specifying that no servers are valid sources for mail from @%s. If you do send email from this domain name you should either override this record such that the SPF rule does allow the originating server, or, take the recommended approach and have the box handle mail for this domain (simply add any receiving alias at this domain name to make this machine treat the domain name as one of its mail domains)." % d)) if not has_rec("_dmarc" + ("."+qname if qname else ""), "TXT", prefix="v=DMARC1; "): - records.append(("_dmarc" + ("."+qname if qname else ""), "TXT", 'v=DMARC1; p=reject', "Recommended. Prevents use of this domain name for outbound mail by specifying that the SPF rule should be honoured for mail from @%s." % d)) + records.append(("_dmarc" + ("."+qname if qname else ""), "TXT", 'v=DMARC1; p=reject;', "Recommended. Prevents use of this domain name for outbound mail by specifying that the SPF rule should be honoured for mail from @%s." % d)) # And with a null MX record (https://explained-from-first-principles.com/email/#null-mx-record) if not has_rec(qname, "MX"): diff --git a/tests/test_dns.py b/tests/test_dns.py index c5fe8051..e734e3ce 100755 --- a/tests/test_dns.py +++ b/tests/test_dns.py @@ -30,7 +30,7 @@ def test(server, description): (hostname, "TXT", "\"v=spf1 mx -all\""), ("mail._domainkey." + hostname, "TXT", "\"v=DKIM1; k=rsa; s=email; \" \"p=__KEY__\""), #("_adsp._domainkey." + hostname, "TXT", "\"dkim=all\""), - ("_dmarc." + hostname, "TXT", "\"v=DMARC1; p=quarantine\""), + ("_dmarc." + hostname, "TXT", "\"v=DMARC1; p=quarantine;\""), ] return test2(tests, server, description) From 91fc74b40899eaccf7433ed05967041baa28d9ff Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sat, 3 Sep 2022 07:50:36 -0400 Subject: [PATCH 16/35] Setup fixes for Ubuntu 22.04 Nextcloud: * The Nextcloud user_external 1.0.0 package for Nextcloud 21.0.7 isn't available from Nextcloud's releases page, but it's not needed in an intermediate upgrade step (hopefully), so we can skip it. * Nextcloud updgrade steps should not be elifs because multiple intermediate upgrades may be needed. * Continue if the user_external backend migration fails. Maybe it's not necessary. It gives a scary error message though. * Remove a line that removes an old file that hasn't been in use since 2019 and the expectation is that Ubuntu 22.04 installations are on fresh machines. Backups: * For duplicity, we now need boto3 for AWS. --- setup/management.sh | 6 +++--- setup/nextcloud.sh | 18 +++++++++--------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/setup/management.sh b/setup/management.sh index 52d38498..cfac5db9 100755 --- a/setup/management.sh +++ b/setup/management.sh @@ -29,9 +29,9 @@ done apt_install duplicity python3-pip virtualenv certbot rsync # b2sdk is used for backblaze backups. -# boto is used for amazon aws backups. +# boto3 is used for amazon aws backups. # Both are installed outside the pipenv, so they can be used by duplicity -hide_output pip3 install --upgrade b2sdk boto +hide_output pip3 install --upgrade b2sdk boto3 # Create a virtualenv for the installation of Python 3 packages # used by the management daemon. @@ -53,7 +53,7 @@ hide_output $venv/bin/pip install --upgrade \ flask dnspython python-dateutil expiringdict \ qrcode[pil] pyotp \ "idna>=2.0.0" "cryptography==37.0.2" psutil postfix-mta-sts-resolver \ - b2sdk boto + b2sdk boto3 # CONFIGURATION diff --git a/setup/nextcloud.sh b/setup/nextcloud.sh index b9c3a6d4..37181a83 100755 --- a/setup/nextcloud.sh +++ b/setup/nextcloud.sh @@ -182,11 +182,13 @@ if [ ! -d /usr/local/lib/owncloud/ ] || [[ ! ${CURRENT_NEXTCLOUD_VER} =~ ^$nextc elif [[ ${CURRENT_NEXTCLOUD_VER} =~ ^1[3456789] ]]; then echo "Upgrades from Mail-in-a-Box prior to v60 with Nextcloud 19 or earlier are not supported. Upgrade to the latest Mail-in-a-Box version supported on your machine first. Setup will continue, but skip the Nextcloud migration." return 0 - elif [[ ${CURRENT_NEXTCLOUD_VER} =~ ^20 ]]; then - InstallNextcloud 21.0.7 f5c7079c5b56ce1e301c6a27c0d975d608bb01c9 4.0.7 8ab31d205408e4f12067d8a4daa3595d46b513e3 3.0.4 6fb1e998d307c53245faf1c37a96eb982bbee8ba 1.0.0 3bf2609061d7214e7f0f69dd8883e55c4ec8f50a + fi + if [[ ${CURRENT_NEXTCLOUD_VER} =~ ^20 ]]; then + InstallNextcloud 21.0.7 f5c7079c5b56ce1e301c6a27c0d975d608bb01c9 4.0.7 45e7cf4bfe99cd8d03625cf9e5a1bb2e90549136 3.0.4 d0284b68135777ec9ca713c307216165b294d0fe CURRENT_NEXTCLOUD_VER="21.0.7" - elif [[ ${CURRENT_NEXTCLOUD_VER} =~ ^21 ]]; then - InstallNextcloud 22.2.6 9d39741f051a8da42ff7df46ceef2653a1dc70d9 4.1.0 38653b507bd7d953816bbc5e8bea7855867eb1cd 3.2.2 54e9a836adc739be4a2a9301b8d6d2e9d88e02f4 3.0.0 0df781b261f55bbde73d8c92da3f99397000972f + fi + if [[ ${CURRENT_NEXTCLOUD_VER} =~ ^21 ]]; then + InstallNextcloud 22.2.6 9d39741f051a8da42ff7df46ceef2653a1dc70d9 4.1.0 697f6b4a664e928d72414ea2731cb2c9d1dc3077 3.2.2 ce4030ab57f523f33d5396c6a81396d440756f5f 3.0.0 0df781b261f55bbde73d8c92da3f99397000972f CURRENT_NEXTCLOUD_VER="22.2.6" fi fi @@ -353,8 +355,9 @@ tools/editconf.py /etc/php/$PHP_VER/cli/conf.d/10-opcache.ini -c ';' \ # Migrate users_external data from <0.6.0 to version 3.0.0 (see https://github.com/nextcloud/user_external). # This version was probably in use in Mail-in-a-Box v0.41 (February 26, 2019) and earlier. -# We moved to v0.6.3 in 193763f8. -sqlite3 $STORAGE_ROOT/owncloud/owncloud.db "UPDATE oc_users_external SET backend='127.0.0.1';" +# We moved to v0.6.3 in 193763f8. Ignore errors - maybe there are duplicated users with the +# correct backend already. +sqlite3 $STORAGE_ROOT/owncloud/owncloud.db "UPDATE oc_users_external SET backend='127.0.0.1';" || /bin/true # Set up a cron job for Nextcloud. cat > /etc/cron.d/mailinabox-nextcloud << EOF; @@ -364,9 +367,6 @@ cat > /etc/cron.d/mailinabox-nextcloud << EOF; EOF chmod +x /etc/cron.d/mailinabox-nextcloud -# Remove previous hourly cronjob -rm -f /etc/cron.hourly/mailinabox-owncloud - # There's nothing much of interest that a user could do as an admin for Nextcloud, # and there's a lot they could mess up, so we don't make any users admins of Nextcloud. # But if we wanted to, we would do this: From 7cda439c8055d36f12d79efb167a12d8f594be0c Mon Sep 17 00:00:00 2001 From: Steve Hay Date: Sat, 17 Sep 2022 07:57:12 -0400 Subject: [PATCH 17/35] Port boto to boto3 and fix asyncio issue in the management daemon (#2156) Co-authored-by: Steve Hay --- management/backup.py | 41 ++++++++++++------------------------- management/daemon.py | 6 ++++-- management/status_checks.py | 2 +- 3 files changed, 18 insertions(+), 31 deletions(-) diff --git a/management/backup.py b/management/backup.py index 2e88c8d0..012ccccb 100755 --- a/management/backup.py +++ b/management/backup.py @@ -446,25 +446,13 @@ def list_target_files(config): raise ValueError("Connection to rsync host failed: {}".format(reason)) elif target.scheme == "s3": - # match to a Region - import boto.s3 - from boto.exception import BotoServerError - custom_region = False - for region in boto.s3.regions(): - if region.endpoint == target.hostname: - break - else: - # If region is not found this is a custom region - custom_region = True - + import boto3.s3 + from botocore.exceptions import ClientError + + # separate bucket from path in target bucket = target.path[1:].split('/')[0] path = '/'.join(target.path[1:].split('/')[1:]) + '/' - # Create a custom region with custom endpoint - if custom_region: - from boto.s3.connection import S3Connection - region = boto.s3.S3RegionInfo(name=bucket, endpoint=target.hostname, connection_cls=S3Connection) - # If no prefix is specified, set the path to '', otherwise boto won't list the files if path == '/': path = '' @@ -474,18 +462,15 @@ def list_target_files(config): # connect to the region & bucket try: - conn = region.connect(aws_access_key_id=config["target_user"], aws_secret_access_key=config["target_pass"]) - bucket = conn.get_bucket(bucket) - except BotoServerError as e: - if e.status == 403: - raise ValueError("Invalid S3 access key or secret access key.") - elif e.status == 404: - raise ValueError("Invalid S3 bucket name.") - elif e.status == 301: - raise ValueError("Incorrect region for this bucket.") - raise ValueError(e.reason) - - return [(key.name[len(path):], key.size) for key in bucket.list(prefix=path)] + s3 = boto3.client('s3', \ + endpoint_url=f'https://{target.hostname}', \ + aws_access_key_id=config['target_user'], \ + aws_secret_access_key=config['target_pass']) + bucket_objects = s3.list_objects_v2(Bucket=bucket, Prefix=path)['Contents'] + backup_list = [(key['Key'][len(path):], key['Size']) for key in bucket_objects] + except ClientError as e: + raise ValueError(e) + return backup_list elif target.scheme == 'b2': from b2sdk.v1 import InMemoryAccountInfo, B2Api from b2sdk.v1.exception import NonExistentBucket diff --git a/management/daemon.py b/management/daemon.py index 98c6689c..2be32504 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -121,8 +121,10 @@ def index(): no_users_exist = (len(get_mail_users(env)) == 0) no_admins_exist = (len(get_admins(env)) == 0) - import boto.s3 - backup_s3_hosts = [(r.name, r.endpoint) for r in boto.s3.regions()] + import boto3.s3 + from urllib.parse import urlparse + backup_s3_hosts = [(r, f"s3.{r}.amazonaws.com") for r in boto3.session.Session().get_available_regions('s3')] + return render_template('index.html', hostname=env['PRIMARY_HOSTNAME'], diff --git a/management/status_checks.py b/management/status_checks.py index 12b4440d..0d555441 100755 --- a/management/status_checks.py +++ b/management/status_checks.py @@ -715,7 +715,7 @@ def check_mail_domain(domain, env, output): output.print_ok(good_news) # Check MTA-STS policy. - loop = asyncio.get_event_loop() + loop = asyncio.new_event_loop() sts_resolver = postfix_mta_sts_resolver.resolver.STSResolver(loop=loop) valid, policy = loop.run_until_complete(sts_resolver.resolve(domain)) if valid == postfix_mta_sts_resolver.resolver.STSFetchResult.VALID: From 3fd2e3efa9078593ff19a209fcd72743d4d91490 Mon Sep 17 00:00:00 2001 From: Steve Hay Date: Sat, 17 Sep 2022 08:03:16 -0400 Subject: [PATCH 18/35] Replace Flask built-in WSGI server with gunicorn (#2158) --- conf/mailinabox.service | 1 + management/auth.py | 16 ++-------------- management/wsgi.py | 7 +++++++ setup/management.sh | 10 ++++++++-- 4 files changed, 18 insertions(+), 16 deletions(-) create mode 100644 management/wsgi.py diff --git a/conf/mailinabox.service b/conf/mailinabox.service index b4cfa6cf..c1d98a03 100644 --- a/conf/mailinabox.service +++ b/conf/mailinabox.service @@ -4,6 +4,7 @@ After=multi-user.target [Service] Type=idle +IgnoreSIGPIPE=False ExecStart=/usr/local/lib/mailinabox/start [Install] diff --git a/management/auth.py b/management/auth.py index 0a88c457..c576d01c 100644 --- a/management/auth.py +++ b/management/auth.py @@ -22,20 +22,8 @@ class AuthService: def init_system_api_key(self): """Write an API key to a local file so local processes can use the API""" - def create_file_with_mode(path, mode): - # Based on answer by A-B-B: http://stackoverflow.com/a/15015748 - old_umask = os.umask(0) - try: - return os.fdopen(os.open(path, os.O_WRONLY | os.O_CREAT, mode), 'w') - finally: - os.umask(old_umask) - - self.key = secrets.token_hex(32) - - os.makedirs(os.path.dirname(self.key_path), exist_ok=True) - - with create_file_with_mode(self.key_path, 0o640) as key_file: - key_file.write(self.key + '\n') + with open(self.key_path, 'r') as file: + self.key = file.read() def authenticate(self, request, env, login_only=False, logout=False): """Test if the HTTP Authorization header's username matches the system key, a session key, diff --git a/management/wsgi.py b/management/wsgi.py new file mode 100644 index 00000000..86cf3af4 --- /dev/null +++ b/management/wsgi.py @@ -0,0 +1,7 @@ +from daemon import app +import auth, utils + +app.logger.addHandler(utils.create_syslog_handler()) + +if __name__ == "__main__": + app.run(port=10222) \ No newline at end of file diff --git a/setup/management.sh b/setup/management.sh index cfac5db9..cebed8d5 100755 --- a/setup/management.sh +++ b/setup/management.sh @@ -50,7 +50,7 @@ hide_output $venv/bin/pip install --upgrade pip # NOTE: email_validator is repeated in setup/questions.sh, so please keep the versions synced. hide_output $venv/bin/pip install --upgrade \ rtyaml "email_validator>=1.0.0" "exclusiveprocess" \ - flask dnspython python-dateutil expiringdict \ + flask dnspython python-dateutil expiringdict gunicorn \ qrcode[pil] pyotp \ "idna>=2.0.0" "cryptography==37.0.2" psutil postfix-mta-sts-resolver \ b2sdk boto3 @@ -90,6 +90,7 @@ rm -f /tmp/bootstrap.zip # Create an init script to start the management daemon and keep it # running after a reboot. +# Note: Authentication currently breaks with more than 1 gunicorn worker. cat > $inst_dir/start < /var/lib/mailinabox/api.key +chmod 640 /var/lib/mailinabox/api.key + source $venv/bin/activate -exec python $(pwd)/management/daemon.py +export PYTHONPATH=$(pwd)/management +exec gunicorn -b localhost:10222 -w 1 wsgi:app EOF chmod +x $inst_dir/start cp --remove-destination conf/mailinabox.service /lib/systemd/system/mailinabox.service # target was previously a symlink so remove it first From 58ded74181b750ae654ef8ca19f10527471b5e82 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Mon, 12 Sep 2022 18:13:31 -0400 Subject: [PATCH 19/35] Restore the backup S3 host select box if an S3 target has been set Also remove unnecessary import added in 7cda439c. Was a mistake from edits during PR review. --- management/daemon.py | 1 - management/templates/system-backup.html | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/management/daemon.py b/management/daemon.py index 2be32504..dc59c19b 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -122,7 +122,6 @@ def index(): no_admins_exist = (len(get_admins(env)) == 0) import boto3.s3 - from urllib.parse import urlparse backup_s3_hosts = [(r, f"s3.{r}.amazonaws.com") for r in boto3.session.Session().get_available_regions('s3')] diff --git a/management/templates/system-backup.html b/management/templates/system-backup.html index 3075b912..5450b6e5 100644 --- a/management/templates/system-backup.html +++ b/management/templates/system-backup.html @@ -269,6 +269,7 @@ function show_custom_backup() { $("#backup-target-type").val("s3"); var hostpath = r.target.substring(5).split('/'); var host = hostpath.shift(); + $("#backup-target-s3-host-select").val(host); $("#backup-target-s3-host").val(host); $("#backup-target-s3-path").val(hostpath.join('/')); } else if (r.target.substring(0, 5) == "b2://") { From 84da4e600013e0ce7ee67a8676f442362973ef6a Mon Sep 17 00:00:00 2001 From: Steve Hay Date: Mon, 5 Sep 2022 19:25:20 -0400 Subject: [PATCH 20/35] Update dovecot to use same DH parameters file as the other services Originally from #2157. --- setup/mail-dovecot.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup/mail-dovecot.sh b/setup/mail-dovecot.sh index 394ede8b..a4bb563b 100755 --- a/setup/mail-dovecot.sh +++ b/setup/mail-dovecot.sh @@ -87,7 +87,8 @@ tools/editconf.py /etc/dovecot/conf.d/10-ssl.conf \ "ssl_min_protocol=TLSv1.2" \ "ssl_cipher_list=ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384" \ "ssl_prefer_server_ciphers=no" \ - "ssl_dh_parameters_length=2048" + "ssl_dh_parameters_length=2048" \ + "ssl_dh=<$STORAGE_ROOT/ssl/dh2048.pem" # Disable in-the-clear IMAP/POP because there is no reason for a user to transmit # login credentials outside of an encrypted connection. Only the over-TLS versions From 30631b0fc535d2d83944e2175379cd41d015e397 Mon Sep 17 00:00:00 2001 From: downtownallday Date: Sat, 25 Jun 2022 12:35:03 -0400 Subject: [PATCH 21/35] Fix undefined variable 'val' in tools/editconf.py (#2137) Merges #2137. --- tools/editconf.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/editconf.py b/tools/editconf.py index e80742e4..dc184966 100755 --- a/tools/editconf.py +++ b/tools/editconf.py @@ -136,9 +136,10 @@ while len(input_lines) > 0: # Put any settings we didn't see at the end of the file, # except settings being cleared. for i in range(len(settings)): - if (i not in found) and not (not val and erase_setting): + if i not in found: name, val = settings[i].split("=", 1) - buf += name + delimiter + val + "\n" + if not (not val and erase_setting): + buf += name + delimiter + val + "\n" if not testing: # Write out the new file. From 56074ae03592e9a4b590409b2e756aa12998ef86 Mon Sep 17 00:00:00 2001 From: downtownallday Date: Tue, 28 Jun 2022 07:46:24 -0400 Subject: [PATCH 22/35] Tighten roundcube session config (#2138) Merges #2138. --- CHANGELOG.md | 4 ++++ setup/webmail.sh | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1234a898..4188f5cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,10 @@ No features of Mail-in-a-Box have changed in this release, but with the newer ve * fail2ban is upgraded to 0.11.2. * nginx is upgraded to 1.18. +Also: + +* Roundcube's login session cookie was tightened. Existing sessions may require a manual logout. + Version 57a (June 19, 2022) --------------------------- diff --git a/setup/webmail.sh b/setup/webmail.sh index e064a201..131f7aa5 100755 --- a/setup/webmail.sh +++ b/setup/webmail.sh @@ -141,6 +141,10 @@ cat > $RCM_CONFIG < EOF From d584a41e604e1a1393f9ceb0b118c4ce97c1cd33 Mon Sep 17 00:00:00 2001 From: kiekerjan Date: Sat, 17 Sep 2022 15:20:20 +0200 Subject: [PATCH 23/35] Update Roundcube to 1.6.0 (#2153) --- CHANGELOG.md | 4 +++- setup/webmail.sh | 19 +++++++++---------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4188f5cc..72a2608a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,10 +15,12 @@ LINK TBD No features of Mail-in-a-Box have changed in this release, but with the newer version of Ubuntu the following software packages we use are updated: * dovecot is upgraded to 2.3.16, postfix to 3.6.4, opendmark to 1.4 (which adds ARC-Authentication-Results headers), and spampd to 2.53 (alleviating a mail delivery rate limiting bug). -* Nextcloud is upgraded to 23.0.4 with PHP updated from 7.2 to 8.0. +* Nextcloud is upgraded to 23.0.4. +* Roundcube is upgraded to 1.6.0. * certbot is upgraded to 1.21 (via the Ubuntu repository instead of a PPA). * fail2ban is upgraded to 0.11.2. * nginx is upgraded to 1.18. +* PHP is upgraded from 7.2 to 8.0. Also: diff --git a/setup/webmail.sh b/setup/webmail.sh index 131f7aa5..791bda57 100755 --- a/setup/webmail.sh +++ b/setup/webmail.sh @@ -35,12 +35,12 @@ apt_install \ # https://github.com/mstilkerich/rcmcarddav/releases # The easiest way to get the package hashes is to run this script and get the hash from # the error message. -VERSION=1.5.2 -HASH=208ce4ca0be423cc0f7070ff59bd03588b4439bf -PERSISTENT_LOGIN_VERSION=59ca1b0d3a02cff5fa621c1ad581d15f9d642fe8 +VERSION=1.6.0 +HASH=fd84b4fac74419bb73e7a3bcae1978d5589c52de +PERSISTENT_LOGIN_VERSION=bde7b6840c7d91de627ea14e81cf4133cbb3c07a # version 5.2 HTML5_NOTIFIER_VERSION=68d9ca194212e15b3c7225eb6085dbcf02fd13d7 # version 0.6.4+ -CARDDAV_VERSION=4.3.0 -CARDDAV_HASH=4ad7df8843951062878b1375f77c614f68bc5c61 +CARDDAV_VERSION=4.4.3 +CARDDAV_HASH=74f8ba7aee33e78beb9de07f7f44b81f6071b644 UPDATE_KEY=$VERSION:$PERSISTENT_LOGIN_VERSION:$HTML5_NOTIFIER_VERSION:$CARDDAV_VERSION @@ -83,7 +83,7 @@ if [ $needs_update == 1 ]; then # download and verify the full release of the carddav plugin wget_verify \ - https://github.com/blind-coder/rcmcarddav/releases/download/v${CARDDAV_VERSION}/carddav-v${CARDDAV_VERSION}.tar.gz \ + https://github.com/mstilkerich/rcmcarddav/releases/download/v${CARDDAV_VERSION}/carddav-v${CARDDAV_VERSION}.tar.gz \ $CARDDAV_HASH \ /tmp/carddav.tar.gz @@ -115,8 +115,7 @@ cat > $RCM_CONFIG < array( 'verify_peer' => false, @@ -124,7 +123,7 @@ cat > $RCM_CONFIG < array( 'verify_peer' => false, @@ -158,7 +157,7 @@ cat > ${RCM_PLUGIN_DIR}/carddav/config.inc.php < 'ownCloud', 'username' => '%u', // login username 'password' => '%p', // login password - 'url' => 'https://${PRIMARY_HOSTNAME}/cloud/remote.php/carddav/addressbooks/%u/contacts', + 'url' => 'https://${PRIMARY_HOSTNAME}/cloud/remote.php/dav/addressbooks/users/%u/contacts/', 'active' => true, 'readonly' => false, 'refresh_time' => '02:00:00', From 1e1a0546868cb062ba4057cc7587f7d6c175c18b Mon Sep 17 00:00:00 2001 From: Steve Hay Date: Sat, 24 Sep 2022 09:56:27 -0400 Subject: [PATCH 24/35] BUGFIX: Correctly handle the multiprocessing for run_checks in the management daemon (#2163) See discussion here: #2083 Co-authored-by: Steve Hay --- management/daemon.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/management/daemon.py b/management/daemon.py index dc59c19b..cbbfd6bf 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -571,6 +571,8 @@ def system_status(): # Create a temporary pool of processes for the status checks with multiprocessing.pool.Pool(processes=5) as pool: run_checks(False, env, output, pool) + pool.close() + pool.join() return json_response(output.items) @app.route('/system/updates') From 3c4460431686142fbc9d8da98a829e58a4e277ac Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sat, 24 Sep 2022 10:08:57 -0400 Subject: [PATCH 25/35] Install 'file' package The command is used in mailinabox-postgrey-whitelist. Reported missing (on systems that don't install it by default) in #2083. --- setup/system.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/system.sh b/setup/system.sh index d9ee1f3f..d0003f9c 100755 --- a/setup/system.sh +++ b/setup/system.sh @@ -137,7 +137,7 @@ apt_get_quiet autoremove echo Installing system packages... apt_install python3 python3-dev python3-pip python3-setuptools \ - netcat-openbsd wget curl git sudo coreutils bc \ + netcat-openbsd wget curl git sudo coreutils bc file \ pollinate openssh-client unzip \ unattended-upgrades cron ntp fail2ban rsyslog From b8feb77ef4d504f9cefb04bb5c57b821012b8194 Mon Sep 17 00:00:00 2001 From: jvolkenant Date: Sat, 24 Sep 2022 10:17:55 -0700 Subject: [PATCH 26/35] Move postgrey database under $STORAGE_ROOT (#2077) --- CHANGELOG.md | 1 + management/backup.py | 2 ++ setup/mail-postfix.sh | 25 +++++++++++++++++++++++-- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72a2608a..cad3dd21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ No features of Mail-in-a-Box have changed in this release, but with the newer ve Also: * Roundcube's login session cookie was tightened. Existing sessions may require a manual logout. +* Move Postgrey's database under $STORAGE_ROOT Version 57a (June 19, 2022) --------------------------- diff --git a/management/backup.py b/management/backup.py index 012ccccb..8a82c4ad 100755 --- a/management/backup.py +++ b/management/backup.py @@ -281,6 +281,7 @@ def perform_backup(full_backup): service_command("php8.0-fpm", "stop", quit=True) service_command("postfix", "stop", quit=True) service_command("dovecot", "stop", quit=True) + service_command("postgrey", "stop", quit=True) # Execute a pre-backup script that copies files outside the homedir. # Run as the STORAGE_USER user, not as root. Pass our settings in @@ -310,6 +311,7 @@ def perform_backup(full_backup): get_duplicity_env_vars(env)) finally: # Start services again. + service_command("postgrey", "start", quit=False) service_command("dovecot", "start", quit=False) service_command("postfix", "start", quit=False) service_command("php8.0-fpm", "start", quit=False) diff --git a/setup/mail-postfix.sh b/setup/mail-postfix.sh index 196d371d..5787e8c9 100755 --- a/setup/mail-postfix.sh +++ b/setup/mail-postfix.sh @@ -233,11 +233,32 @@ tools/editconf.py /etc/postfix/main.cf \ # As a matter of fact RFC is not strict about retry timer so postfix and # other MTA have their own intervals. To fix the problem of receiving # e-mails really latter, delay of greylisting has been set to -# 180 seconds (default is 300 seconds). +# 180 seconds (default is 300 seconds). We will move the postgrey database +# under $STORAGE_ROOT. This prevents a "warming up" that would have occured +# previously with a migrated or reinstalled OS. We will specify this new path +# with the --dbdir=... option. Arguments within POSTGREY_OPTS can not have spaces, +# including dbdir. This is due to the way the init script sources the +# /etc/default/postgrey file. --dbdir=... either needs to be a path without spaces +# (luckily $STORAGE_ROOT does not currently work with spaces), or it needs to be a +# symlink without spaces that can point to a folder with spaces). We'll just assume +# $STORAGE_ROOT won't have spaces to simplify things. tools/editconf.py /etc/default/postgrey \ - POSTGREY_OPTS=\"'--inet=127.0.0.1:10023 --delay=180'\" + POSTGREY_OPTS=\""--inet=127.0.0.1:10023 --delay=180 --dbdir=$STORAGE_ROOT/mail/postgrey/db"\" +# If the $STORAGE_ROOT/mail/postgrey is empty, copy the postgrey database over from the old location +if [ ! -d $STORAGE_ROOT/mail/postgrey/db ]; then + # Stop the service + service postgrey stop + # Ensure the new paths for postgrey db exists + mkdir -p $STORAGE_ROOT/mail/postgrey/db + # Move over database files + mv /var/lib/postgrey/* $STORAGE_ROOT/mail/postgrey/db/ || true +fi +# Ensure permissions are set +chown -R postgrey:postgrey $STORAGE_ROOT/mail/postgrey/ +chmod 700 $STORAGE_ROOT/mail/postgrey/{,db} + # We are going to setup a newer whitelist for postgrey, the version included in the distribution is old cat > /etc/cron.daily/mailinabox-postgrey-whitelist << EOF; #!/bin/bash From 9b111e24937df6f65e770cfb50007b43c381bbee Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sat, 3 Sep 2022 07:50:42 -0400 Subject: [PATCH 27/35] Update to Nextcloud 23.0.8 (contacts 4.2.0, calendar 3.5.0) --- setup/nextcloud.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/setup/nextcloud.sh b/setup/nextcloud.sh index 37181a83..5c42c655 100755 --- a/setup/nextcloud.sh +++ b/setup/nextcloud.sh @@ -21,8 +21,8 @@ echo "Installing Nextcloud (contacts/calendar)..." # we automatically install intermediate versions as needed. # * The hash is the SHA1 hash of the ZIP package, which you can find by just running this script and # copying it from the error message when it doesn't match what is below. -nextcloud_ver=23.0.4 -nextcloud_hash=87afec0bf90b3c66289e6fedd851867bc5a58f01 +nextcloud_ver=23.0.8 +nextcloud_hash=9d63416a0697eeecf243d09461f5881f8997f50a # Nextcloud apps # -------------- @@ -33,10 +33,10 @@ nextcloud_hash=87afec0bf90b3c66289e6fedd851867bc5a58f01 # https://github.com/nextcloud/user_external/blob/master/appinfo/info.xml # * The hash is the SHA1 hash of the ZIP package, which you can find by just running this script and # copying it from the error message when it doesn't match what is below. -contacts_ver=4.1.0 -contacts_hash=697f6b4a664e928d72414ea2731cb2c9d1dc3077 -calendar_ver=3.2.2 -calendar_hash=ce4030ab57f523f33d5396c6a81396d440756f5f +contacts_ver=4.2.0 +contacts_hash=79b506574834db5e1b6ab47aadd4041e12ad9a9c +calendar_ver=3.5.0 +calendar_hash=941381536287a015081669513f8f79f6f262508a user_external_ver=3.0.0 user_external_hash=0df781b261f55bbde73d8c92da3f99397000972f From 0a970f4bb2dba052ce1c321dd8b4e9f016a56550 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sun, 2 Oct 2022 21:20:21 -0400 Subject: [PATCH 28/35] Use nsd-control to refresh nsd after zone files are rewritten rather than 'service nsd restart' I am not sure if this was the problem but nsd didn't serve updated zonefiles on my box and 'service nsd restart' must have been used, so maybe it doesn't reload zones. --- management/dns_update.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/management/dns_update.py b/management/dns_update.py index 45ea94fa..a9c44f01 100755 --- a/management/dns_update.py +++ b/management/dns_update.py @@ -96,9 +96,9 @@ def do_dns_update(env, force=False): if len(updated_domains) == 0: updated_domains.append("DNS configuration") - # Kick nsd if anything changed. + # Tell nsd to reload changed zone files. if len(updated_domains) > 0: - shell('check_call', ["/usr/sbin/service", "nsd", "restart"]) + shell('check_call', ["/usr/sbin/nsd-control", "reload"]) # Write the OpenDKIM configuration tables for all of the mail domains. from mailconfig import get_mail_domains From 22a6270657ed6906a5572c2779a381a0f55f8eab Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sat, 8 Oct 2022 08:16:51 -0400 Subject: [PATCH 29/35] Remove old setup step to uninstall acme library --- setup/management.sh | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/setup/management.sh b/setup/management.sh index cebed8d5..1d1f4cc5 100755 --- a/setup/management.sh +++ b/setup/management.sh @@ -7,18 +7,6 @@ echo "Installing Mail-in-a-Box system management daemon..." # DEPENDENCIES -# We used to install management daemon-related Python packages -# directly to /usr/local/lib. We moved to a virtualenv because -# these packages might conflict with apt-installed packages. -# We may have a lingering version of acme that conflcits with -# certbot, which we're about to install below, so remove it -# first. Once acme is installed by an apt package, this might -# break the package version and `apt-get install --reinstall python3-acme` -# might be needed in that case. -while [ -d /usr/local/lib/python3.4/dist-packages/acme ]; do - pip3 uninstall -y acme; -done - # duplicity is used to make backups of user data. # # virtualenv is used to isolate the Python 3 packages we From 89cd9fb6113770cdda3c458e80fa601eb1569173 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sat, 8 Oct 2022 08:17:24 -0400 Subject: [PATCH 30/35] Increase gunicorn's worker timeout since some /admin commands take a long time --- setup/management.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup/management.sh b/setup/management.sh index 1d1f4cc5..7fe4da25 100755 --- a/setup/management.sh +++ b/setup/management.sh @@ -78,6 +78,8 @@ rm -f /tmp/bootstrap.zip # Create an init script to start the management daemon and keep it # running after a reboot. +# Set a long timeout since some commands take a while to run, matching +# the timeout we set for PHP (fastcgi_read_timeout in the nginx confs). # Note: Authentication currently breaks with more than 1 gunicorn worker. cat > $inst_dir/start < Date: Tue, 11 Oct 2022 21:14:31 -0400 Subject: [PATCH 31/35] Version 60 --- CHANGELOG.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cad3dd21..4f454bbd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ CHANGELOG ========= -Version 60 (date TBD) ---------------------- +Version 60 (October 11, 2022) +----------------------------- This is the first release for Ubuntu 22.04. @@ -10,9 +10,11 @@ This is the first release for Ubuntu 22.04. For complete upgrade instructions, see: -LINK TBD +https://discourse.mailinabox.email/t/version-60-for-ubuntu-22-04-is-about-to-be-released/9558 -No features of Mail-in-a-Box have changed in this release, but with the newer version of Ubuntu the following software packages we use are updated: +No major features of Mail-in-a-Box have changed in this release, although some minor fixes were made. + +With the newer version of Ubuntu the following software packages we use are updated: * dovecot is upgraded to 2.3.16, postfix to 3.6.4, opendmark to 1.4 (which adds ARC-Authentication-Results headers), and spampd to 2.53 (alleviating a mail delivery rate limiting bug). * Nextcloud is upgraded to 23.0.4. @@ -25,7 +27,7 @@ No features of Mail-in-a-Box have changed in this release, but with the newer ve Also: * Roundcube's login session cookie was tightened. Existing sessions may require a manual logout. -* Move Postgrey's database under $STORAGE_ROOT +* Moved Postgrey's database under $STORAGE_ROOT. Version 57a (June 19, 2022) --------------------------- From 286a4bd9e744cba67fd5762ca1d92b229f29bc80 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Wed, 12 Oct 2022 06:11:02 -0400 Subject: [PATCH 32/35] Remove stray quote in bootstrap.sh Reported at https://discourse.mailinabox.email/t/version-60-for-ubuntu-22-04-is-released/9558/4. --- setup/bootstrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index cc4c060e..b700ee52 100644 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -19,7 +19,7 @@ if [ -z "$TAG" ]; then # want to display in status checks. # # Allow point-release versions of the major releases, e.g. 22.04.1 is OK. - UBUNTU_VERSION=$( lsb_release -d | sed 's/.*:\s*//' | sed 's/\([0-9]*\.[0-9]*\)\.[0-9]/\1/' )" + UBUNTU_VERSION=$( lsb_release -d | sed 's/.*:\s*//' | sed 's/\([0-9]*\.[0-9]*\)\.[0-9]/\1/' ) if [ "$UBUNTU_VERSION" == "Ubuntu 22.04 LTS" ]; then # This machine is running Ubuntu 22.04, which is supported by # Mail-in-a-Box versions 60 and later. From 32c68874c5f3b82aa968f27c05e4f0462de73d57 Mon Sep 17 00:00:00 2001 From: alento-group <42585319+alento-group@users.noreply.github.com> Date: Sun, 30 Oct 2022 13:16:03 +0100 Subject: [PATCH 33/35] Fix NSD not restarting (#2182) A previous commit (0a970f4bb2dba052ce1c321dd8b4e9f016a56550) broke nsd restarting. This fixes that change by reverting it. Josh added: Use nsd-control with reconfig and reload if they succeed and only fall back to restarting nsd if they fail Co-authored-by: Joshua Tauberer --- management/dns_update.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/management/dns_update.py b/management/dns_update.py index 1567371a..2bfc104f 100755 --- a/management/dns_update.py +++ b/management/dns_update.py @@ -98,7 +98,16 @@ def do_dns_update(env, force=False): # Tell nsd to reload changed zone files. if len(updated_domains) > 0: - shell('check_call', ["/usr/sbin/nsd-control", "reload"]) + # 'reconfig' is needed if there are added or removed zones, but + # it may not reload existing zones, so we call 'reload' too. If + # nsd isn't running, nsd-control fails, so in that case revert + # to restarting nsd to make sure it is running. Restarting nsd + # should also refresh everything. + try: + shell('check_call', ["/usr/sbin/nsd-control", "reconfig"]) + shell('check_call', ["/usr/sbin/nsd-control", "reload"]) + except: + shell('check_call', ["/usr/sbin/service", "nsd", "restart"]) # Write the OpenDKIM configuration tables for all of the mail domains. from mailconfig import get_mail_domains From 1f60236985332b89b1add380ab95509ea2f19dc4 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Thu, 27 Oct 2022 10:46:09 -0400 Subject: [PATCH 34/35] Upgrade Nextcloud to 23.0.4 (contacts to 4.2.0, calendar to 3.5.0) This fixes the monthly view calendar items being in random order. --- CHANGELOG.md | 7 ++++++- setup/nextcloud.sh | 16 ++++++++-------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f454bbd..b51c1364 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ CHANGELOG ========= +In Development +-------------- + +* Nextcloud is updated to 23.0.10 (contacts to 4.2.2, calendar to 3.5.1). + Version 60 (October 11, 2022) ----------------------------- @@ -17,7 +22,7 @@ No major features of Mail-in-a-Box have changed in this release, although some m With the newer version of Ubuntu the following software packages we use are updated: * dovecot is upgraded to 2.3.16, postfix to 3.6.4, opendmark to 1.4 (which adds ARC-Authentication-Results headers), and spampd to 2.53 (alleviating a mail delivery rate limiting bug). -* Nextcloud is upgraded to 23.0.4. +* Nextcloud is upgraded to 23.0.4 (contacts to 4.2.0, calendar to 3.5.0). * Roundcube is upgraded to 1.6.0. * certbot is upgraded to 1.21 (via the Ubuntu repository instead of a PPA). * fail2ban is upgraded to 0.11.2. diff --git a/setup/nextcloud.sh b/setup/nextcloud.sh index 5c42c655..13afc6b7 100755 --- a/setup/nextcloud.sh +++ b/setup/nextcloud.sh @@ -21,22 +21,22 @@ echo "Installing Nextcloud (contacts/calendar)..." # we automatically install intermediate versions as needed. # * The hash is the SHA1 hash of the ZIP package, which you can find by just running this script and # copying it from the error message when it doesn't match what is below. -nextcloud_ver=23.0.8 -nextcloud_hash=9d63416a0697eeecf243d09461f5881f8997f50a +nextcloud_ver=23.0.10 +nextcloud_hash=8831c7862e39460fbb789bacac8729fab0ba02dd # Nextcloud apps # -------------- # * Find the most recent tag that is compatible with the Nextcloud version above by # consulting the ... node at: -# https://github.com/nextcloud-releases/contacts/blob/master/appinfo/info.xml -# https://github.com/nextcloud-releases/calendar/blob/master/appinfo/info.xml +# https://github.com/nextcloud-releases/contacts/blob/main/appinfo/info.xml +# https://github.com/nextcloud-releases/calendar/blob/main/appinfo/info.xml # https://github.com/nextcloud/user_external/blob/master/appinfo/info.xml # * The hash is the SHA1 hash of the ZIP package, which you can find by just running this script and # copying it from the error message when it doesn't match what is below. -contacts_ver=4.2.0 -contacts_hash=79b506574834db5e1b6ab47aadd4041e12ad9a9c -calendar_ver=3.5.0 -calendar_hash=941381536287a015081669513f8f79f6f262508a +contacts_ver=4.2.2 +contacts_hash=ca13d608ed8955aa374cb4f31b6026b57ef88887 +calendar_ver=3.5.1 +calendar_hash=c8136a3deb872a3ef73ce1155b58f3ab27ec7110 user_external_ver=3.0.0 user_external_hash=0df781b261f55bbde73d8c92da3f99397000972f From 3314c4f7de75bdf4ef7141e1042a5c3ae5cad56e Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Thu, 27 Oct 2022 10:48:45 -0400 Subject: [PATCH 35/35] v60.1 --- CHANGELOG.md | 5 +++-- README.md | 2 +- setup/bootstrap.sh | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b51c1364..c4033df1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,10 @@ CHANGELOG ========= -In Development --------------- +Version 60.1 (October 30, 2022) +------------------------------- +* A setup issue where the DNS server nsd isn't running at the end of setup is (hopefully) fixed. * Nextcloud is updated to 23.0.10 (contacts to 4.2.2, calendar to 3.5.1). Version 60 (October 11, 2022) diff --git a/README.md b/README.md index ab9a8039..42792025 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ Clone this repository and checkout the tag corresponding to the most recent rele $ git clone https://github.com/mail-in-a-box/mailinabox $ cd mailinabox - $ git checkout v60 + $ git checkout v60.1 Begin the installation. diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index b700ee52..1c6d475a 100644 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -23,7 +23,7 @@ if [ -z "$TAG" ]; then if [ "$UBUNTU_VERSION" == "Ubuntu 22.04 LTS" ]; then # This machine is running Ubuntu 22.04, which is supported by # Mail-in-a-Box versions 60 and later. - TAG=v60 + TAG=v60.1 elif [ "$UBUNTU_VERSION" == "Ubuntu 18.04 LTS" ]; then # This machine is running Ubuntu 18.04, which is supported by # Mail-in-a-Box versions 0.40 through 5x.