Merge branch 'mergeupstream2204'

setup/bootstrap.sh

@@ -26,15 +26,11 @@ if [ -z "$TAG" ]; then
 		# This machine is running Ubuntu 22.04, which is supported by
 		# Mail-in-a-Box versions 60 and later.
 		TAG=v60
-	elif [ "$UBUNTU_VERSION" == "Ubuntu 20.04 LTS" ]; then
-		# This machine is running Ubuntu 20.04, which is supported by
-		# Mail-in-a-Box versions 56 and later.
-		TAG=v57a
 	elif [ "$UBUNTU_VERSION" == "Ubuntu 18.04 LTS" ]; then
 		# This machine is running Ubuntu 18.04, which is supported by
 		# Mail-in-a-Box versions 0.40 through 5x.
 		echo "Support is ending for Ubuntu 18.04."
-		echo "Please immediately begin to migrate your information to"
+		echo "Please immediately begin to migrate your data to"
 		echo "a new machine running Ubuntu 22.04. See:"
 		echo "https://mailinabox.email/maintenance.html#upgrade"
 		TAG=v57a
@@ -46,7 +42,7 @@ if [ -z "$TAG" ]; then
 		echo "The last version of Mail-in-a-Box supporting Ubuntu 14.04 will be installed."
 		TAG=v0.30
 	else
-		echo "This script may be used only on a machine running Ubuntu 14.04, 18.04, 20.04 or 22.04."
+		echo "This script may be used only on a machine running Ubuntu 14.04, 18.04, or 22.04."
 		exit 1
 	fi
 fi

setup/dns.sh

@@ -10,8 +10,6 @@
 source setup/functions.sh # load our functions
 source /etc/mailinabox.conf # load global vars
 
-echo "Installing nsd (DNS server)..."
-
 # Prepare nsd's configuration.
 # We configure nsd before installation as we only want it to bind to some addresses
 # and it otherwise will have port / bind conflicts with unbound used as the local resolver
@@ -68,19 +66,12 @@ cat > /etc/logrotate.d/nsd <<EOF;
 }
 EOF
 
-# Add systemd override file to fix some permissions
-mkdir -p /etc/systemd/system/nsd.service.d/
-cat > /etc/systemd/system/nsd.service.d/nsd-permissions.conf << EOF
-[Service]
-ReadWritePaths=/var/lib/nsd /etc/nsd /run /var/log /run/nsd
-CapabilityBoundingSet=CAP_CHOWN CAP_IPC_LOCK CAP_NET_BIND_SERVICE CAP_SETGID CAP_SETUID CAP_SYS_CHROOT CAP_NET_ADMIN
-EOF
-
 # Install the packages.
 #
 # * nsd: The non-recursive nameserver that publishes our DNS records.
 # * ldnsutils: Helper utilities for signing DNSSEC zones.
 # * openssh-client: Provides ssh-keyscan which we use to create SSHFP records.
+echo "Installing nsd (DNS server)..."
 apt_install nsd ldnsutils openssh-client
 
 # Create DNSSEC signing keys.

setup/dovecot-fts-xapian.sh

@@ -35,8 +35,6 @@ if [ ! -f /usr/lib/dovecot/decode2text.sh ]; then
 	cp -f /usr/share/doc/dovecot-core/examples/decode2text.sh /usr/lib/dovecot
 fi
 
-#cp -f lib/lib21_fts_xapian_plugin.so /usr/lib/dovecot/modules/
-
 # Create configuration file
 cat > /etc/dovecot/conf.d/90-plugin-fts.conf << EOF;
 plugin {

setup/functions.sh

@@ -4,6 +4,8 @@
 # -o pipefail: don't ignore errors in the non-last command in a pipeline
 set -euo pipefail
 
+PHP_VER=php_version
+
 function hide_output {
 	# This function hides the output of a command unless the command fails
 	# and returns a non-zero exit code.

setup/mail-postfix.sh

@@ -245,12 +245,33 @@ tools/editconf.py /etc/postfix/main.cf \
 # As a matter of fact RFC is not strict about retry timer so postfix and
 # other MTA have their own intervals. To fix the problem of receiving
 # e-mails really later, delay of greylisting has been set to
-# 180 seconds (default is 300 seconds).
+# 180 seconds (default is 300 seconds). We will move the postgrey database
+# under $STORAGE_ROOT. This prevents a "warming up" that would have occured
+# previously with a migrated or reinstalled OS.  We will specify this new path
+# with the --dbdir=... option. Arguments within POSTGREY_OPTS can not have spaces,
+# including dbdir. This is due to the way the init script sources the
+# /etc/default/postgrey file. --dbdir=... either needs to be a path without spaces
+# (luckily $STORAGE_ROOT does not currently work with spaces), or it needs to be a
+# symlink without spaces that can point to a folder with spaces).  We'll just assume
+# $STORAGE_ROOT won't have spaces to simplify things.
 # Postgrey removes entries after 185 days of not being used.
 tools/editconf.py /etc/default/postgrey \
-	POSTGREY_OPTS=\"'--inet=127.0.0.1:10023 --delay=180  --max-age=185'\"
+	POSTGREY_OPTS=\""--inet=127.0.0.1:10023 --delay=180 --max-age=185 --dbdir=$STORAGE_ROOT/mail/postgrey/db"\"
 
 
+# If the $STORAGE_ROOT/mail/postgrey is empty, copy the postgrey database over from the old location
+if [ ! -d $STORAGE_ROOT/mail/postgrey/db ]; then
+	# Stop the service
+	service postgrey stop
+	# Ensure the new paths for postgrey db exists
+	mkdir -p $STORAGE_ROOT/mail/postgrey/db
+	# Move over database files
+	mv /var/lib/postgrey/* $STORAGE_ROOT/mail/postgrey/db/ || true
+fi
+# Ensure permissions are set
+chown -R postgrey:postgrey $STORAGE_ROOT/mail/postgrey/
+chmod 700 $STORAGE_ROOT/mail/postgrey/{,db}
+
 # We are going to setup a newer whitelist for postgrey, the version included in the distribution is old
 cat > /etc/cron.daily/mailinabox-postgrey-whitelist << EOF;
 #!/bin/bash

setup/management.sh

@@ -1,23 +1,12 @@
 #!/bin/bash
 
 source setup/functions.sh
+source /etc/mailinabox.conf # load global vars
 
 echo "Installing Mail-in-a-Box system management daemon..."
 
 # DEPENDENCIES
 
-# We used to install management daemon-related Python packages
-# directly to /usr/local/lib. We moved to a virtualenv because
-# these packages might conflict with apt-installed packages.
-# We may have a lingering version of acme that conflcits with
-# certbot, which we're about to install below, so remove it
-# first. Once acme is installed by an apt package, this might
-# break the package version and `apt-get install --reinstall python3-acme`
-# might be needed in that case.
-while [ -d /usr/local/lib/python3.4/dist-packages/acme ]; do
-	pip3 uninstall -y acme;
-done
-
 # duplicity is used to make backups of user data.
 #
 # virtualenv is used to isolate the Python 3 packages we
@@ -28,9 +17,9 @@ done
 apt_install duplicity python3-pip virtualenv certbot rsync
 
 # b2sdk is used for backblaze backups.
-# boto is used for amazon aws backups.
+# boto3 is used for amazon aws backups.
 # Both are installed outside the pipenv, so they can be used by duplicity
-hide_output pip3 install --upgrade b2sdk==1.14.1 boto
+hide_output pip3 install --upgrade b2sdk boto3
 
 # Create a virtualenv for the installation of Python 3 packages
 # used by the management daemon.
@@ -49,10 +38,10 @@ hide_output $venv/bin/pip install --upgrade pip
 # NOTE: email_validator is repeated in setup/questions.sh, so please keep the versions synced.
 hide_output $venv/bin/pip install --upgrade \
 	rtyaml "email_validator>=1.0.0" "exclusiveprocess" \
-	flask dnspython python-dateutil expiringdict \
+	flask dnspython python-dateutil expiringdict gunicorn \
 	qrcode[pil] pyotp \
-	"idna>=2.0.0" "cryptography==2.2.2" psutil postfix-mta-sts-resolver \
-	b2sdk==1.14.1 boto
+	"idna>=2.0.0" "cryptography==37.0.2" psutil postfix-mta-sts-resolver \
+	b2sdk boto3
 
 # CONFIGURATION
 
@@ -89,6 +78,9 @@ rm -f /tmp/bootstrap.zip
 
 # Create an init script to start the management daemon and keep it
 # running after a reboot.
+# Set a long timeout since some commands take a while to run, matching
+# the timeout we set for PHP (fastcgi_read_timeout in the nginx confs).
+# Note: Authentication currently breaks with more than 1 gunicorn worker.
 cat > $inst_dir/start <<EOF;
 #!/bin/bash
 # Set character encoding flags to ensure that any non-ASCII don't cause problems.
@@ -97,8 +89,13 @@ export LC_ALL=en_US.UTF-8
 export LANG=en_US.UTF-8
 export LC_TYPE=en_US.UTF-8
 
+mkdir -p /var/lib/mailinabox
+tr -cd '[:xdigit:]' < /dev/urandom | head -c 32 > /var/lib/mailinabox/api.key
+chmod 640 /var/lib/mailinabox/api.key
+
 source $venv/bin/activate
-exec python $(pwd)/management/daemon.py
+export PYTHONPATH=$(pwd)/management
+exec gunicorn -b localhost:10222 -w 1 --timeout 630 wsgi:app
 EOF
 chmod +x $inst_dir/start
 cp --remove-destination conf/mailinabox.service /lib/systemd/system/mailinabox.service # target was previously a symlink so remove it first

setup/nextcloud.sh

@@ -49,8 +49,8 @@ apt_install php php-fpm \
 	php-dev php-xml php-mbstring php-zip php-apcu php-json \
 	php-intl php-imagick php-gmp php-bcmath
 
-# Enable apc is required before installing nextcloud
-tools/editconf.py /etc/php/$(php_version)/mods-available/apcu.ini -c ';' \
+# Enable APC before Nextcloud tools are run.
+tools/editconf.py /etc/php/$PHP_VER/mods-available/apcu.ini -c ';' \
     apc.enabled=1 \
     apc.enable_cli=1
 
@@ -155,7 +155,7 @@ fi
 if [ ! -d /usr/local/lib/owncloud/ ] || [[ ! ${CURRENT_NEXTCLOUD_VER} =~ ^$nextcloud_ver ]]; then
 
 	# Stop php-fpm if running. If they are not running (which happens on a previously failed install), dont bail.
-	service php$(php_version)-fpm stop &> /dev/null || /bin/true
+	service php$PHP_VER-fpm stop &> /dev/null || /bin/true
 
 	# Backup the existing ownCloud/Nextcloud.
 	# Create a backup directory to store the current installation and database to
@@ -280,7 +280,7 @@ if [ ! -f $STORAGE_ROOT/owncloud/owncloud.db ]; then
     array(
       'class' => '\OCA\UserExternal\IMAP',
           'arguments' => array(
-          '127.0.0.1', 143, null
+        '127.0.0.1', 143, null, null, false, false
          ),
     ),
   ),
@@ -343,7 +343,7 @@ php <<EOF > $CONFIG_TEMP && mv $CONFIG_TEMP $STORAGE_ROOT/owncloud/config.php;
 <?php
 include("$STORAGE_ROOT/owncloud/config.php");
 
-\$CONFIG['config_is_read_only'] = false; # should prevent warnings from occ tool but doesn't
+\$CONFIG['config_is_read_only'] = true;
 
 \$CONFIG['trusted_domains'] = array('$PRIMARY_HOSTNAME');
 
@@ -358,7 +358,14 @@ include("$STORAGE_ROOT/owncloud/config.php");
 
 \$CONFIG['mail_domain'] = '$PRIMARY_HOSTNAME';
 
-\$CONFIG['user_backends'] = array(array('class' => '\OCA\UserExternal\IMAP','arguments' => array('127.0.0.1', 143, null),),);
+\$CONFIG['user_backends'] = array(
+  array(
+    'class' => '\OCA\UserExternal\IMAP',
+    'arguments' => array(
+      '127.0.0.1', 143, null, null, false, false
+    ),
+  ),
+);
 
 echo "<?php\n\\\$CONFIG = ";
 var_export(\$CONFIG);
@@ -366,7 +373,7 @@ echo ";";
 ?>
 EOF
 chown www-data.www-data $STORAGE_ROOT/owncloud/config.php
-chmod 640 $STORAGE_ROOT/owncloud/config.php
+#chmod 640 $STORAGE_ROOT/owncloud/config.php
 
 # Enable/disable apps. Note that this must be done after the Nextcloud setup.
 # The firstrunwizard gave Josh all sorts of problems, so disabling that.
@@ -402,7 +409,7 @@ sudo -u www-data php /usr/local/lib/owncloud/occ app:update --all
 
 # Set PHP FPM values to support large file uploads
 # (semicolon is the comment character in this file, hashes produce deprecation warnings)
-tools/editconf.py /etc/php/$(php_version)/fpm/php.ini -c ';' \
+tools/editconf.py /etc/php/$PHP_VER/fpm/php.ini -c ';' \
 	upload_max_filesize=16G \
 	post_max_size=16G \
 	output_buffering=16384 \
@@ -411,7 +418,7 @@ tools/editconf.py /etc/php/$(php_version)/fpm/php.ini -c ';' \
 	short_open_tag=On
 
 # Set Nextcloud recommended opcache settings
-tools/editconf.py /etc/php/$(php_version)/cli/conf.d/10-opcache.ini -c ';' \
+tools/editconf.py /etc/php/$PHP_VER/cli/conf.d/10-opcache.ini -c ';' \
 	opcache.enable=1 \
 	opcache.enable_cli=1 \
 	opcache.interned_strings_buffer=8 \
@@ -420,6 +427,12 @@ tools/editconf.py /etc/php/$(php_version)/cli/conf.d/10-opcache.ini -c ';' \
 	opcache.save_comments=1 \
 	opcache.revalidate_freq=1
 
+# Migrate users_external data from <0.6.0 to version 3.0.0 (see https://github.com/nextcloud/user_external).
+# This version was probably in use in Mail-in-a-Box v0.41 (February 26, 2019) and earlier.
+# We moved to v0.6.3 in 193763f8. Ignore errors - maybe there are duplicated users with the
+# correct backend already.
+sqlite3 $STORAGE_ROOT/owncloud/owncloud.db "UPDATE oc_users_external SET backend='127.0.0.1';" || /bin/true
+
 # Set up a cron job for Nextcloud.
 cat > /etc/cron.d/mailinabox-nextcloud << EOF;
 #!/bin/bash
@@ -428,9 +441,6 @@ cat > /etc/cron.d/mailinabox-nextcloud << EOF;
 EOF
 chmod +x /etc/cron.d/mailinabox-nextcloud
 
-# Remove previous hourly cronjob
-rm -f /etc/cron.hourly/mailinabox-owncloud
-
 # There's nothing much of interest that a user could do as an admin for Nextcloud,
 # and there's a lot they could mess up, so we don't make any users admins of Nextcloud.
 # But if we wanted to, we would do this:
@@ -441,4 +451,4 @@ rm -f /etc/cron.hourly/mailinabox-owncloud
 # ```
 
 # Enable PHP modules and restart PHP.
-restart_service php$(php_version)-fpm
+restart_service php$PHP_VER-fpm

setup/preflight.sh

@@ -7,9 +7,9 @@ if [[ $EUID -ne 0 ]]; then
 	exit 1
 fi
 
-# Check that we are running on Ubuntu 20.04 LTS or Ubuntu 22.04 LTS
-if [ "$( lsb_release --id --short )" != "Ubuntu" ] || [ "$( lsb_release --release --short )" != "22.04" -a "$( lsb_release --release --short )" != "20.04" ]; then
-	echo "Mail-in-a-Box only supports being installed on Ubuntu 20.04 or 22.04, sorry. You are running:"
+# Check that we are running on Ubuntu 22.04 LTS (or 22.04.xx).
+if [ "$( lsb_release --id --short )" != "Ubuntu" ] || [ "$( lsb_release --release --short )" != "22.04" ]; then
+	echo "Mail-in-a-Box only supports being installed on Ubuntu 22.04, sorry. You are running:"
 	echo
 	lsb_release --description --short
 	echo

setup/start.sh

@@ -72,6 +72,10 @@ fi
 fi
 
 # Create the STORAGE_USER and STORAGE_ROOT directory if they don't already exist.
+#
+# Set the directory and all of its parent directories' permissions to world
+# readable since it holds files owned by different processes.
+#
 # If the STORAGE_ROOT is missing the mailinabox.version file that lists a
 # migration (schema) number for the files stored there, assume this is a fresh
 # installation to that directory and write the file to contain the current
@@ -82,6 +86,8 @@ fi
 if [ ! -d $STORAGE_ROOT ]; then
 	mkdir -p $STORAGE_ROOT
 fi
+f=$STORAGE_ROOT
+while [[ $f != / ]]; do chmod a+rx "$f"; f=$(dirname "$f"); done;
 if [ ! -f $STORAGE_ROOT/mailinabox.version ]; then
 	setup/migrate.py --current > $STORAGE_ROOT/mailinabox.version
 	chown $STORAGE_USER.$STORAGE_USER $STORAGE_ROOT/mailinabox.version

setup/system.sh

@@ -83,6 +83,9 @@ fi
 tools/editconf.py /etc/systemd/journald.conf MaxRetentionSec=10day
 
 hide_output systemctl restart systemd-journald.service
+
+# ### Add PPAs.
+
 # We install some non-standard Ubuntu packages maintained by other
 # third-party providers. First ensure add-apt-repository is installed.
 
@@ -96,6 +99,8 @@ fi
 # come from there and minimal Ubuntu installs may have it turned off.
 hide_output add-apt-repository -y universe
 
+# Install the duplicity PPA.
+hide_output add-apt-repository -y ppa:duplicity-team/duplicity-release-git
 # ### Update Packages
 
 # Update system packages to make sure we have the latest upstream versions

setup/web.sh

@@ -46,11 +46,11 @@ tools/editconf.py /etc/nginx/nginx.conf -s \
 	ssl_protocols="TLSv1.2 TLSv1.3;"
 
 # Tell PHP not to expose its version number in the X-Powered-By header.
-tools/editconf.py /etc/php/$(php_version)/fpm/php.ini -c ';' \
+tools/editconf.py /etc/php/$PHP_VER/fpm/php.ini -c ';' \
 	expose_php=Off
 
 # Set PHPs default charset to UTF-8, since we use it. See #367.
-tools/editconf.py /etc/php/$(php_version)/fpm/php.ini -c ';' \
+tools/editconf.py /etc/php/$PHP_VER/fpm/php.ini -c ';' \
         default_charset="UTF-8"
 
 # Set higher timeout since fts searches with Roundcube may take longer
@@ -60,7 +60,7 @@ tools/editconf.py /etc/php/$(php_version)/fpm/php.ini -c ';' \
         default_socket_timeout=180
 
 # Configure the path environment for php-fpm
-tools/editconf.py /etc/php/$(php_version)/fpm/pool.d/www.conf -c ';' \
+tools/editconf.py /etc/php/$PHP_VER/fpm/pool.d/www.conf -c ';' \
 	env[PATH]=/usr/local/bin:/usr/bin:/bin \
 
 # Configure php-fpm based on the amount of memory the machine has
@@ -70,7 +70,7 @@ tools/editconf.py /etc/php/$(php_version)/fpm/pool.d/www.conf -c ';' \
 TOTAL_PHYSICAL_MEM=$(head -n 1 /proc/meminfo | awk '{print $2}' || /bin/true)
 if [ $TOTAL_PHYSICAL_MEM -lt 1000000 ]
 then
-        tools/editconf.py /etc/php/$(php_version)/fpm/pool.d/www.conf -c ';' \
+        tools/editconf.py /etc/php/$PHP_VER/fpm/pool.d/www.conf -c ';' \
                 pm=ondemand \
                 pm.max_children=8 \
                 pm.start_servers=2 \
@@ -78,7 +78,7 @@ then
                 pm.max_spare_servers=3
 elif [ $TOTAL_PHYSICAL_MEM -lt 2000000 ]
 then
-        tools/editconf.py /etc/php/$(php_version)/fpm/pool.d/www.conf -c ';' \
+        tools/editconf.py /etc/php/$PHP_VER/fpm/pool.d/www.conf -c ';' \
                 pm=ondemand \
                 pm.max_children=16 \
                 pm.start_servers=4 \
@@ -86,14 +86,14 @@ then
                 pm.max_spare_servers=6
 elif [ $TOTAL_PHYSICAL_MEM -lt 3000000 ]
 then
-        tools/editconf.py /etc/php/$(php_version)/fpm/pool.d/www.conf -c ';' \
+        tools/editconf.py /etc/php/$PHP_VER/fpm/pool.d/www.conf -c ';' \
                 pm=dynamic \
                 pm.max_children=60 \
                 pm.start_servers=6 \
                 pm.min_spare_servers=3 \
                 pm.max_spare_servers=9
 else
-        tools/editconf.py /etc/php/$(php_version)/fpm/pool.d/www.conf -c ';' \
+        tools/editconf.py /etc/php/$PHP_VER/fpm/pool.d/www.conf -c ';' \
                 pm=dynamic \
                 pm.max_children=120 \
                 pm.start_servers=12 \
@@ -162,7 +162,7 @@ chown www-data /var/log/nginx/geoipblock.log
 
 # Start services.
 restart_service nginx
-restart_service php$(php_version)-fpm
+restart_service php$PHP_VER-fpm
 
 # Open ports.
 ufw_allow http

setup/webmail.sh

@@ -22,7 +22,7 @@ source /etc/mailinabox.conf # load global vars
 echo "Installing Roundcube (webmail)..."
 apt_install \
 	dbconfig-common \
-	php-cli php-sqlite3 php-intl php-json php-common php-curl php-ldap \
+	php-cli php-sqlite3 php-intl php-json php-common php-curl php-imap \
 	php-gd php-pspell libjs-jquery libjs-jquery-mousewheel libmagic1 php-mbstring
 
 # Install Roundcube from source if it is not already present or if it is out of date.
@@ -124,8 +124,7 @@ cat > $RCM_CONFIG <<EOF;
 \$config['log_dir'] = '/var/log/roundcubemail/';
 \$config['temp_dir'] = '/var/tmp/roundcubemail/';
 \$config['db_dsnw'] = 'sqlite:///$STORAGE_ROOT/mail/roundcube/roundcube.sqlite?mode=0640';
-\$config['default_host'] = 'ssl://localhost';
-\$config['default_port'] = 993;
+\$config['imap_host'] = 'ssl://localhost:993';
 \$config['imap_conn_options'] = array(
   'ssl'         => array(
      'verify_peer'  => false,
@@ -133,7 +132,7 @@ cat > $RCM_CONFIG <<EOF;
    ),
  );
 \$config['imap_timeout'] = 180;
-\$config['smtp_server'] = 'tls://127.0.0.1';
+\$config['smtp_host'] = 'tls://127.0.0.1';
 \$config['smtp_conn_options'] = array(
   'ssl'         => array(
      'verify_peer'  => false,
@@ -150,6 +149,10 @@ cat > $RCM_CONFIG <<EOF;
 \$config['login_username_filter'] = 'email';
 \$config['password_charset'] = 'UTF-8';
 \$config['junk_mbox'] = 'Spam';
+/* ensure roudcube session id's aren't leaked to other parts of the server */
+\$config['session_path'] = '/mail/';
+/* prevent CSRF, requires php 7.3+ */
+\$config['session_samesite'] = 'Strict';
 ?>
 EOF
 
@@ -216,5 +219,5 @@ chown www-data:www-data $STORAGE_ROOT/mail/roundcube/roundcube.sqlite
 chmod 664 $STORAGE_ROOT/mail/roundcube/roundcube.sqlite
 
 # Enable PHP modules.
-phpenmod -v php mcrypt imap
-restart_service php$(php_version)-fpm
+phpenmod -v php imap
+restart_service php$PHP_VER-fpm