From 2e23e4458244b7a9af8b7348fda94e19f73178cb Mon Sep 17 00:00:00 2001 From: "github@kiekerjan.isdronken.nl" Date: Wed, 16 Feb 2022 23:32:30 +0100 Subject: [PATCH] merge prelim 22.04 changes from upstream --- CHANGELOG.md | 22 ++++++++++++ README.md | 6 ++-- Vagrantfile | 2 +- management/dns_update.py | 2 +- setup/bootstrap.sh | 72 ++++++++++++++++++++++++--------------- setup/dns.sh | 42 +++++++++++------------ setup/mail-dovecot.sh | 5 +-- setup/mail-postfix.sh | 19 ++++++----- setup/nextcloud.sh | 10 +++--- setup/preflight.sh | 9 +++-- setup/system.sh | 19 +++++++---- tests/test_dns.py | 2 +- tests/test_mail.py | 2 +- tools/editconf.py | 26 ++++++++++---- tools/owncloud-restore.sh | 5 +-- 15 files changed, 151 insertions(+), 92 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d796970e..70ac8cc6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,28 @@ CHANGELOG ========= +Version 60 (date TBD) +--------------------- + +This is the first release for Ubuntu 22.04. + +**Before upgrading**, you must **first upgrade your existing Ubuntu 18.04 box to Mail-in-a-Box v0.51** (or any later version of Mail-in-a-Box supporting Ubuntu 18.04), if you haven't already done so. That may not be possible after Ubuntu 18.04 reaches its end of life in April 2023, so please compete the upgrade well before then. (If you are not using Nextcloud's contacts or calendar, you can migrate to the latest version of Mail-in-a-Box from any previous version.) + +For complete upgrade instructions, see: + +LINK TBD + +No features of Mail-in-a-Box have changed in this release, but with the newer version of Ubuntu the following software packages we use are updated: + +* dovecot is upgraded to 2.3.16, postfix to 3.6.3, opendmark to 1.4 (which adds ARC-Authentication-Results headers), and spampd to 2.53 (alleviating a mail delivery rate limiting bug). +* Nextcloud is upgraded to 23.0.0 with PHP updated from 7.2 to 8.0. +* certbot is upgraded to 1.21 (via the Ubuntu repository instead of a PPA). +* fail2ban is upgraded to 0.11.2. +* nginx is upgraded to 1.18. + +In Development +-------------- + Version 56 (January 19, 2022) ----------------------------- diff --git a/README.md b/README.md index b4cffa55..72159006 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ Additionally, this project has a [Code of Conduct](CODE_OF_CONDUCT.md), which su In The Box ---------- -Mail-in-a-Box turns a fresh Ubuntu 20.04 or 18.04 LTS 64-bit machine into a working mail server by installing and configuring various components. +Mail-in-a-Box turns a fresh Ubuntu 22.04 or 20.04 LTS 64-bit machine into a working mail server by installing and configuring various components. It is a one-click email appliance. There are no user-configurable setup options. It "just works." @@ -113,13 +113,13 @@ Installation See the [setup guide](https://mailinabox.email/guide.html) for detailed, user-friendly instructions. -For experts, start with a completely fresh (really, I mean it) Ubuntu 18.04 LTS 64-bit machine. On the machine... +For experts, start with a completely fresh (really, I mean it) Ubuntu 22.04 LTS 64-bit machine. On the machine... Clone this repository and checkout the tag corresponding to the most recent release: $ git clone https://github.com/mail-in-a-box/mailinabox $ cd mailinabox - $ git checkout v56 + $ git checkout v60 Begin the installation. diff --git a/Vagrantfile b/Vagrantfile index da235aca..373f3659 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -2,7 +2,7 @@ # vi: set ft=ruby : Vagrant.configure("2") do |config| - config.vm.box = "ubuntu/focal64" + config.vm.box = "ubuntu/jammy64" # Network config: Since it's a mail server, the machine must be connected # to the public web. However, we currently don't want to expose SSH since diff --git a/management/dns_update.py b/management/dns_update.py index edeaa9c2..62a053f5 100755 --- a/management/dns_update.py +++ b/management/dns_update.py @@ -1109,7 +1109,7 @@ def set_secondary_dns(hostnames, env): response = resolver.resolve(item, "A") except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer): try: - response = resolver.query(item, "AAAA") + response = resolver.resolve(item, "AAAA") except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer): raise ValueError("Could not resolve the IP address of %s." % item) else: diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index 0fe4fc44..edee7096 100644 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -6,39 +6,47 @@ # ######################################################### +GITSRC=kj + if [ -z "$TAG" ]; then # If a version to install isn't explicitly given as an environment # variable, then install the latest version. But the latest version - # depends on the operating system. Existing Ubuntu 14.04 users need - # to be able to upgrade to the latest version supporting Ubuntu 14.04, - # in part because an upgrade is required before jumping to Ubuntu 18.04. - # New users on Ubuntu 18.04 need to get the latest version number too. + # depends on the machine's version of Ubuntu. Existing users need to + # be able to upgrade to the latest version available for that version + # of Ubuntu to satisfy the migration requirements. # # Also, the system status checks read this script for TAG = (without the # space, but if we put it in a comment it would confuse the status checks!) # to get the latest version, so the first such line must be the one that we # want to display in status checks. - if [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/20\.04\.[0-9]/20.04/' `" == "Ubuntu 20.04 LTS" ]; then - # This machine is running Ubuntu 20.04. - TAG=v55 - - elif [ "$(lsb_release -d | sed 's/.*:\s*//' | sed 's/18\.04\.[0-9]/18.04/' )" == "Ubuntu 18.04 LTS" ]; then - # This machine is running Ubuntu 18.04. + # + # Allow point-release versions of the major releases, e.g. 22.04.1 is OK. + UBUNTU_VERSION=$( lsb_release -d | sed 's/.*:\s*//' | sed 's/\([0-9]*\.[0-9]*\)\.[0-9]/\1/' )" + if [ "$UBUNTU_VERSION" == "Ubuntu 22.04 LTS" ]; then + # This machine is running Ubuntu 22.04, which is supported by + # Mail-in-a-Box versions 60 and later. + TAG=v60 + elif [ "$UBUNTU_VERSION" == "Ubuntu 20.04 LTS" ]; then + # This machine is running Ubuntu 20.04, which is supported by + # Mail-in-a-Box versions 56 and later. TAG=v56 - - elif [ "$(lsb_release -d | sed 's/.*:\s*//' | sed 's/14\.04\.[0-9]/14.04/' )" == "Ubuntu 14.04 LTS" ]; then - # This machine is running Ubuntu 14.04. - echo "You are installing the last version of Mail-in-a-Box that will" - echo "support Ubuntu 14.04. If this is a new installation of Mail-in-a-Box," - echo "stop now and switch to a machine running Ubuntu 18.04. If you are" - echo "upgrading an existing Mail-in-a-Box --- great. After upgrading this" - echo "box, please visit https://mailinabox.email for notes on how to upgrade" - echo "to Ubuntu 18.04." - echo "" + elif [ "$UBUNTU_VERSION" == "Ubuntu 18.04 LTS" ]; then + # This machine is running Ubuntu 18.04, which is supported by + # Mail-in-a-Box versions 0.40 through 5x. + echo "Support is ending for Ubuntu 18.04." + echo "Please immediately begin to migrate your information to" + echo "a new machine running Ubuntu 22.04. See:" + echo "https://mailinabox.email/maintenance.html#upgrade" + TAG=v56 + GITSRC=miab + elif [ "$UBUNTU_VERSION" == "Ubuntu 14.04 LTS" ]; then + # This machine is running Ubuntu 14.04, which is supported by + # Mail-in-a-Box versions 1 through v0.30. + echo "Ubuntu 14.04 is no longer supported." + echo "The last version of Mail-in-a-Box supporting Ubuntu 14.04 will be installed." TAG=v0.30 - else - echo "This script must be run on a system running Ubuntu 20.04, 18.04 or 14.04." + echo "This script may be used only on a machine running Ubuntu 14.04, 18.04, 20.04 or 22.04." exit 1 fi fi @@ -59,12 +67,20 @@ if [ ! -d $HOME/mailinabox ]; then fi echo Downloading Mail-in-a-Box $TAG. . . - git clone \ - -b $TAG --depth 1 \ - https://github.com/mail-in-a-box/mailinabox \ - $HOME/mailinabox \ - < /dev/null 2> /dev/null - + if [ "$GITSRC" == "miab" ]; then + git clone \ + -b $TAG --depth 1 \ + https://github.com/mail-in-a-box/mailinabox \ + $HOME/mailinabox \ + < /dev/null 2> /dev/null + else + git clone \ + -b $TAG --depth 1 \ + https://github.com/kiekerjan/mailinabox \ + $HOME/mailinabox \ + < /dev/null 2> /dev/null + fi + echo fi diff --git a/setup/dns.sh b/setup/dns.sh index 808d9b57..fd0dec90 100755 --- a/setup/dns.sh +++ b/setup/dns.sh @@ -10,21 +10,15 @@ source setup/functions.sh # load our functions source /etc/mailinabox.conf # load global vars -# Install the packages. -# -# * nsd: The non-recursive nameserver that publishes our DNS records. -# * ldnsutils: Helper utilities for signing DNSSEC zones. -# * openssh-client: Provides ssh-keyscan which we use to create SSHFP records. echo "Installing nsd (DNS server)..." -apt_install ldnsutils openssh-client # Prepare nsd's configuration. - +# We configure nsd before installation as we only want it to bind to some addresses +# and it otherwise will have port / bind conflicts with bind9 used as the local resolver mkdir -p /var/run/nsd mkdir -p /etc/nsd mkdir -p /etc/nsd/zones touch /etc/nsd/zones.conf -touch /etc/nsd/nsd.conf cat > /etc/nsd/nsd.conf << EOF; # Do not edit. Overwritten by Mail-in-a-Box setup. @@ -46,18 +40,6 @@ server: EOF -# Add log rotation -cat > /etc/logrotate.d/nsd <> /etc/nsd/nsd.conf; # now be stored in /etc/nsd/nsd.conf.d. rm -f /etc/nsd/zones.conf +# Add log rotation +cat > /etc/logrotate.d/nsd < /etc/systemd/system/nsd.service.d/nsd-permissions.conf << EOF @@ -82,8 +76,12 @@ ReadWritePaths=/var/lib/nsd /etc/nsd /run /var/log /run/nsd CapabilityBoundingSet=CAP_CHOWN CAP_IPC_LOCK CAP_NET_BIND_SERVICE CAP_SETGID CAP_SETUID CAP_SYS_CHROOT CAP_NET_ADMIN EOF -# Attempting a late install of nsd (after configuration) -apt_install nsd +# Install the packages. +# +# * nsd: The non-recursive nameserver that publishes our DNS records. +# * ldnsutils: Helper utilities for signing DNSSEC zones. +# * openssh-client: Provides ssh-keyscan which we use to create SSHFP records. +apt_install nsd ldnsutils openssh-client # Create DNSSEC signing keys. diff --git a/setup/mail-dovecot.sh b/setup/mail-dovecot.sh index 5482d04f..588f2b12 100755 --- a/setup/mail-dovecot.sh +++ b/setup/mail-dovecot.sh @@ -78,13 +78,14 @@ tools/editconf.py /etc/dovecot/conf.d/10-auth.conf \ "auth_mechanisms=plain login" # Enable SSL, specify the location of the SSL certificate and private key files. -# Use Mozilla's "Intermediate" recommendations at https://ssl-config.mozilla.org/#server=dovecot&server-version=2.2.33&config=intermediate&openssl-version=1.1.1, -# specify a minimum of TLSv1.2. +# Use Mozilla's "Intermediate" recommendations at https://ssl-config.mozilla.org/#server=dovecot&server-version=2.3.7.2&config=intermediate&openssl-version=1.1.1, +# except that the current version of Dovecot does not have a TLSv1.3 setting, so we only use TLSv1.2. tools/editconf.py /etc/dovecot/conf.d/10-ssl.conf \ ssl=required \ "ssl_cert=<$STORAGE_ROOT/ssl/ssl_certificate.pem" \ "ssl_key=<$STORAGE_ROOT/ssl/ssl_private_key.pem" \ "ssl_min_protocol=TLSv1.2" \ + "ssl_cipher_list=ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384" \ "ssl_prefer_server_ciphers=yes" \ "ssl_dh=<$STORAGE_ROOT/ssl/dh4096.pem" diff --git a/setup/mail-postfix.sh b/setup/mail-postfix.sh index 96f86c45..347a2fc5 100755 --- a/setup/mail-postfix.sh +++ b/setup/mail-postfix.sh @@ -13,8 +13,8 @@ # destinations according to aliases, and passses email on to # another service for local mail delivery. # -# The first hop in local mail delivery is to Spamassassin via -# LMTP. Spamassassin then passes mail over to Dovecot for +# The first hop in local mail delivery is to spampd via +# LMTP. spampd then passes mail over to Dovecot for # storage in the user's mailbox. # # Postfix also listens on ports 465/587 (SMTPS, SMTP+STARTLS) for @@ -124,7 +124,7 @@ sed -i "s/PUBLIC_IP/$PUBLIC_IP/" /etc/postfix/outgoing_mail_header_filters # the world are very far behind and if we disable too much, they may not be able to use TLS and # won't fall back to cleartext. So we don't disable too much. smtpd_tls_exclude_ciphers applies to # both port 25 and port 587, but because we override the cipher list for both, it probably isn't used. -# Use Mozilla's "Old" recommendations at https://ssl-config.mozilla.org/#server=postfix&server-version=3.3.0&config=old&openssl-version=1.1.1 +# Use Mozilla's "Old" recommendations at https://ssl-config.mozilla.org/#server=postfix&server-version=3.4.13&config=old&openssl-version=1.1.1 tools/editconf.py /etc/postfix/main.cf \ smtpd_tls_security_level=may\ smtpd_tls_auth_only=yes \ @@ -133,7 +133,7 @@ tools/editconf.py /etc/postfix/main.cf \ smtpd_tls_dh1024_param_file=$STORAGE_ROOT/ssl/dh4096.pem \ smtpd_tls_protocols="!SSLv2,!SSLv3,!TLSv1,!TLSv1.1" \ smtpd_tls_ciphers=medium \ - tls_medium_cipherlist=ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA256 \ + tls_medium_cipherlist=ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384 \ smtpd_tls_exclude_ciphers="MD5, DES, ADH, RC4, PSD, SRP, 3DES, eNULL, aNULL" \ tls_preempt_cipherlist=yes \ smtpd_tls_received_header=yes @@ -205,16 +205,17 @@ tools/editconf.py /etc/postfix/main.cf \ # ### Incoming Mail -# Pass any incoming mail over to a local delivery agent. Spamassassin -# will act as the LDA agent at first. It is listening on port 10025 -# with LMTP. Spamassassin will pass the mail over to Dovecot after. +# Pass mail to spampd, which acts as the local delivery agent (LDA), +# which then passes the mail over to the Dovecot LMTP server after. +# spampd runs on port 10025 by default. # # In a basic setup we would pass mail directly to Dovecot by setting # virtual_transport to `lmtp:unix:private/dovecot-lmtp`. tools/editconf.py /etc/postfix/main.cf "virtual_transport=lmtp:[127.0.0.1]:10025" -# Because of a spampd bug, limit the number of recipients in each connection. +# Clear the lmtp_destination_recipient_limit setting which in previous +# versions of Mail-in-a-Box was set to 1 because of a spampd bug. # See https://github.com/mail-in-a-box/mailinabox/issues/1523. -tools/editconf.py /etc/postfix/main.cf lmtp_destination_recipient_limit=1 +tools/editconf.py /etc/postfix/main.cf -e lmtp_destination_recipient_limit= # Who can send mail to us? Some basic filters. diff --git a/setup/nextcloud.sh b/setup/nextcloud.sh index 29ff1214..ccbdc7c5 100755 --- a/setup/nextcloud.sh +++ b/setup/nextcloud.sh @@ -28,7 +28,7 @@ nextcloud_hash=58d2d897ba22a057aa03d29c762c5306211fefd2 # -------------- # * Find the most recent tag that is compatible with the Nextcloud version above by # consulting the ... node at: -# https://github.com/nextcloud-releases/contacts/blob/maaster/appinfo/info.xml +# https://github.com/nextcloud-releases/contacts/blob/master/appinfo/info.xml # https://github.com/nextcloud-releases/calendar/blob/master/appinfo/info.xml # https://github.com/nextcloud/user_external/blob/master/appinfo/info.xml # * The hash is the SHA1 hash of the ZIP package, which you can find by just running this script and @@ -49,10 +49,10 @@ apt_install php php-fpm \ php-dev php-gd php-xml php-mbstring php-zip php-apcu php-json \ php-intl php-imagick php-gmp php-bcmath -# Enable apc is required before installing nextcloud 21 +# Enable apc is required before installing nextcloud tools/editconf.py /etc/php/$(php_version)/mods-available/apcu.ini -c ';' \ apc.enabled=1 \ - apc.enable_cli=1 + apc.enable_cli=0 restart_service php$(php_version)-fpm @@ -156,7 +156,7 @@ fi # from the version currently installed, do the install/upgrade if [ ! -d /usr/local/lib/owncloud/ ] || [[ ! ${CURRENT_NEXTCLOUD_VER} =~ ^$nextcloud_ver ]]; then - # Stop php-fpm if running. If theyre not running (which happens on a previously failed install), dont bail. + # Stop php-fpm if running. If they are not running (which happens on a previously failed install), dont bail. service php$(php_version)-fpm stop &> /dev/null || /bin/true # Backup the existing ownCloud/Nextcloud. @@ -318,6 +318,8 @@ php < $CONFIG_TEMP && mv $CONFIG_TEMP $STORAGE_ROOT/owncloud/config.php; hostname + reverse_dns = dns.resolver.resolve(reverse_ip, 'PTR')[0].target.to_text(omit_final_dot=True) # => hostname except dns.resolver.NXDOMAIN: print("Reverse DNS lookup failed for %s. SMTP EHLO name check skipped." % ipaddr) reverse_dns = None diff --git a/tools/editconf.py b/tools/editconf.py index d665f861..e80742e4 100755 --- a/tools/editconf.py +++ b/tools/editconf.py @@ -14,6 +14,10 @@ # # NAME VALUE # +# If the -e option is given and VALUE is empty, the setting is removed +# from the configuration file if it is set (i.e. existing occurrences +# are commented out and no new setting is added). +# # If the -c option is given, then the supplied character becomes the comment character # # If the -w option is given, then setting lines continue onto following @@ -35,6 +39,7 @@ settings = sys.argv[2:] delimiter = "=" delimiter_re = r"\s*=\s*" +erase_setting = False comment_char = "#" folded_lines = False testing = False @@ -44,6 +49,9 @@ while settings[0][0] == "-" and settings[0] != "--": # Space is the delimiter delimiter = " " delimiter_re = r"\s+" + elif opt == "-e": + # Erase settings that have empty values. + erase_setting = True elif opt == "-w": # Line folding is possible in this file. folded_lines = True @@ -81,7 +89,7 @@ while len(input_lines) > 0: # See if this line is for any settings passed on the command line. for i in range(len(settings)): - # Check that this line contain this setting from the command-line arguments. + # Check if this line contain this setting from the command-line arguments. name, val = settings[i].split("=", 1) m = re.match( "(\s*)" @@ -91,8 +99,10 @@ while len(input_lines) > 0: if not m: continue indent, is_comment, existing_val = m.groups() - # If this is already the setting, do nothing. - if is_comment is None and existing_val == val: + # If this is already the setting, keep it in the file, except: + # * If we've already seen it before, then remove this duplicate line. + # * If val is empty and erase_setting is on, then comment it out. + if is_comment is None and existing_val == val and not (not val and erase_setting): # It may be that we've already inserted this setting higher # in the file so check for that first. if i in found: break @@ -107,8 +117,9 @@ while len(input_lines) > 0: # the line is already commented, pass it through buf += line - # if this option oddly appears more than once, don't add the setting again - if i in found: + # if this option already is set don't add the setting again, + # or if we're clearing the setting with -e, don't add it + if (i in found) or (not val and erase_setting): break # add the new setting @@ -122,9 +133,10 @@ while len(input_lines) > 0: # If did not match any setting names, pass this line through. buf += line -# Put any settings we didn't see at the end of the file. +# Put any settings we didn't see at the end of the file, +# except settings being cleared. for i in range(len(settings)): - if i not in found: + if (i not in found) and not (not val and erase_setting): name, val = settings[i].split("=", 1) buf += name + delimiter + val + "\n" diff --git a/tools/owncloud-restore.sh b/tools/owncloud-restore.sh index 27fe1f38..99bd6648 100755 --- a/tools/owncloud-restore.sh +++ b/tools/owncloud-restore.sh @@ -1,6 +1,7 @@ #!/bin/bash # # This script will restore the backup made during an installation +source setup/functions.sh # load our functions source /etc/mailinabox.conf # load global vars if [ -z "$1" ]; then @@ -26,7 +27,7 @@ if [ ! -f $1/config.php ]; then fi echo "Restoring backup from $1" -service php7.3-fpm stop +service php$(php_version)-fpm stop # remove the current ownCloud/Nextcloud installation rm -rf /usr/local/lib/owncloud/ @@ -45,5 +46,5 @@ chown www-data.www-data $STORAGE_ROOT/owncloud/config.php sudo -u www-data php /usr/local/lib/owncloud/occ maintenance:mode --off -service php7.3-fpm start +service php$(php_version)-fpm start echo "Done"