Merge tag 'v0.52' of https://github.com/mail-in-a-box/mailinabox

v0.52 (January 31, 2021)
------------------------

Software updates:

* Upgraded Roundcube to version 1.4.10.
* Upgraded zpush to 2.6.1.

Mail:

* Incoming emails with SPF/DKIM/DMARC failures now get a higher spam score, and these messages are more likely to appear in the junk folder, since they are often spam/phishing.
* Fixed the MTA-STS policy file's line endings.

Control panel:

* A new Download button in the control panel's External DNS page can be used to download the required DNS records in zonefile format.
* Fixed the problem when the control panel would report DNS entries as Not Set by increasing a bind query limit.
* Fixed a control panel startup bug on some systems.
* Improved an error message on a DNS lookup timeout.
* A typo was fixed.

DNS:

* The TTL for NS records has been increased to 1 day to comply with some registrar requirements.

System:

* Nextcloud's photos, dashboard, and activity apps are disabled since we only support contacts and calendar.

# Conflicts:
#	setup/bootstrap.sh

setup/bootstrap.sh

@@ -20,7 +20,7 @@ if [ -z "$TAG" ]; then
 	# want to display in status checks.
 	if [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/18\.04\.[0-9]/18.04/' `" == "Ubuntu 18.04 LTS" ]; then
 		# This machine is running Ubuntu 18.04.
-		TAG=v0.51-quota-0.22-beta
+		TAG=v0.52-quota-0.22-beta
 
 	elif [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/14\.04\.[0-9]/14.04/' `" == "Ubuntu 14.04 LTS" ]; then
 		# This machine is running Ubuntu 14.04.

setup/dkim.sh

@@ -64,6 +64,32 @@ tools/editconf.py /etc/opendmarc.conf -s \
 	"Syslog=true" \
 	"Socket=inet:8893@[127.0.0.1]"
 
+# SPFIgnoreResults causes the filter to ignore any SPF results in the header
+# of the message. This is useful if you want the filter to perfrom SPF checks
+# itself, or because you don't trust the arriving header. This added header is
+# used by spamassassin to evaluate the mail for spamminess.
+
+tools/editconf.py /etc/opendmarc.conf -s \
+        "SPFIgnoreResults=true"
+
+# SPFSelfValidate causes the filter to perform a fallback SPF check itself
+# when it can find no SPF results in the message header. If SPFIgnoreResults
+# is also set, it never looks for SPF results in headers and always performs
+# the SPF check itself when this is set. This added header is used by
+# spamassassin to evaluate the mail for spamminess.
+
+tools/editconf.py /etc/opendmarc.conf -s \
+        "SPFSelfValidate=true"
+
+# AlwaysAddARHeader Adds an "Authentication-Results:" header field even to
+# unsigned messages from domains with no "signs all" policy. The reported DKIM
+# result will be  "none" in such cases. Normally unsigned mail from non-strict
+# domains does not cause the results header field to be added. This added header
+# is used by spamassassin to evaluate the mail for spamminess.
+
+tools/editconf.py /etc/opendkim.conf -s \
+        "AlwaysAddARHeader=true"
+
 # Add OpenDKIM and OpenDMARC as milters to postfix, which is how OpenDKIM
 # intercepts outgoing mail to perform the signing (by adding a mail header)
 # and how they both intercept incoming mail to add Authentication-Results

setup/management.sh

@@ -18,11 +18,7 @@ while [ -d /usr/local/lib/python3.4/dist-packages/acme ]; do
 	pip3 uninstall -y acme;
 done
 
-# duplicity is used to make backups of user data. It uses boto
-# (via Python 2) to do backups to AWS S3. boto from the Ubuntu
-# package manager is too out-of-date -- it doesn't support the newer
-# S3 api used in some regions, which breaks backups to those regions.
-# See #627, #653.
+# duplicity is used to make backups of user data.
 #
 # virtualenv is used to isolate the Python 3 packages we
 # install via pip from the system-installed packages.
@@ -30,7 +26,10 @@ done
 # certbot installs EFF's certbot which we use to
 # provision free TLS certificates.
 apt_install duplicity python-pip virtualenv certbot
-hide_output pip2 install --upgrade boto
+
+# boto is used for amazon aws backups.
+# Both are installed outside the pipenv, so they can be used by duplicity
+hide_output pip3 install --upgrade boto
 
 # Create a virtualenv for the installation of Python 3 packages
 # used by the management daemon.
@@ -50,7 +49,7 @@ hide_output $venv/bin/pip install --upgrade pip
 hide_output $venv/bin/pip install --upgrade \
 	rtyaml "email_validator>=1.0.0" "exclusiveprocess" \
 	flask dnspython python-dateutil \
-    qrcode[pil] pyotp \
+  qrcode[pil] pyotp \
 	"idna>=2.0.0" "cryptography==2.2.2" boto psutil postfix-mta-sts-resolver
 
 # CONFIGURATION
@@ -90,6 +89,12 @@ rm -f /tmp/bootstrap.zip
 # running after a reboot.
 cat > $inst_dir/start <<EOF;
 #!/bin/bash
+# Set character encoding flags to ensure that any non-ASCII don't cause problems.
+export LANGUAGE=en_US.UTF-8
+export LC_ALL=en_US.UTF-8
+export LANG=en_US.UTF-8
+export LC_TYPE=en_US.UTF-8
+
 source $venv/bin/activate
 exec python `pwd`/management/daemon.py
 EOF

setup/nextcloud.sh

@@ -311,6 +311,9 @@ hide_output sudo -u www-data php /usr/local/lib/owncloud/console.php app:enable
 sudo -u www-data php /usr/local/lib/owncloud/occ upgrade
 if [ \( $? -ne 0 \) -a \( $? -ne 3 \) ]; then exit 1; fi
 
+# Disable default apps that we don't support
+sudo -u www-data php /usr/local/lib/owncloud/occ app:disable photos dashboard activity
+
 # Set PHP FPM values to support large file uploads
 # (semicolon is the comment character in this file, hashes produce deprecation warnings)
 tools/editconf.py /etc/php/7.2/fpm/php.ini -c ';' \

setup/spamassassin.sh

@@ -67,6 +67,56 @@ tools/editconf.py /etc/spamassassin/local.cf -s \
 	"add_header all Report"=_REPORT_ \
 	"add_header all Score"=_SCORE_
 
+
+# Authentication-Results SPF/Dmarc checks
+# ---------------------------------------
+# OpenDKIM and OpenDMARC are configured to validate and add "Authentication-Results: ..."
+# headers by checking the sender's SPF & DMARC policies. Instead of blocking mail that fails
+# these checks, we can use these headers to evaluate the mail as spam.
+#
+# Our custom rules are added to their own file so that an update to the deb package config
+# does not remove our changes.
+#
+# We need to escape period's in $PRIMARY_HOSTNAME since spamassassin config uses regex.
+
+escapedprimaryhostname="${PRIMARY_HOSTNAME//./\\.}"
+
+cat > /etc/spamassassin/miab_spf_dmarc.cf << EOF
+# Evaluate DMARC Authentication-Results
+header DMARC_PASS Authentication-Results =~ /$escapedprimaryhostname; dmarc=pass/
+describe DMARC_PASS DMARC check passed
+score DMARC_PASS -0.1
+
+header DMARC_NONE Authentication-Results =~ /$escapedprimaryhostname; dmarc=none/
+describe DMARC_NONE DMARC record not found
+score DMARC_NONE 0.1
+
+header DMARC_FAIL_NONE Authentication-Results =~ /$escapedprimaryhostname; dmarc=fail \(p=none/
+describe DMARC_FAIL_NONE DMARC check failed (p=none)
+score DMARC_FAIL_NONE 2.0
+
+header DMARC_FAIL_QUARANTINE Authentication-Results =~ /$escapedprimaryhostname; dmarc=fail \(p=quarantine/
+describe DMARC_FAIL_QUARANTINE DMARC check failed (p=quarantine)
+score DMARC_FAIL_QUARANTINE 5.0
+
+header DMARC_FAIL_REJECT Authentication-Results =~ /$escapedprimaryhostname; dmarc=fail \(p=reject/
+describe DMARC_FAIL_REJECT DMARC check failed (p=reject)
+score DMARC_FAIL_REJECT 10.0
+
+# Evaluate SPF Authentication-Results
+header SPF_PASS Authentication-Results =~ /$escapedprimaryhostname; spf=pass/
+describe SPF_PASS SPF check passed
+score SPF_PASS -0.1
+
+header SPF_NONE Authentication-Results =~ /$escapedprimaryhostname; spf=none/
+describe SPF_NONE SPF record not found
+score SPF_NONE 2.0
+
+header SPF_FAIL Authentication-Results =~ /$escapedprimaryhostname; spf=fail/
+describe SPF_FAIL SPF check failed
+score SPF_FAIL 5.0
+EOF
+
 # Bayesean learning
 # -----------------
 #

setup/system.sh

@@ -93,6 +93,9 @@ hide_output add-apt-repository -y universe
 # Install the certbot PPA.
 hide_output add-apt-repository -y ppa:certbot/certbot
 
+# Install the duplicity PPA.
+hide_output add-apt-repository -y ppa:duplicity-team/duplicity-release-git
+
 # ### Update Packages
 
 # Update system packages to make sure we have the latest upstream versions
@@ -317,6 +320,9 @@ fi #NODOC
 #   name server, on IPV6.
 # * The listen-on directive in named.conf.options restricts `bind9` to
 #   binding to the loopback interface instead of all interfaces.
+# * The max-recursion-queries directive increases the maximum number of iterative queries.
+#  	If more queries than specified are sent, bind9 returns SERVFAIL. After flushing the cache during system checks,
+#	we ran into the limit thus we are increasing it from 75 (default value) to 100.
 apt_install bind9
 tools/editconf.py /etc/default/bind9 \
 	"OPTIONS=\"-u bind -4\""
@@ -324,6 +330,10 @@ if ! grep -q "listen-on " /etc/bind/named.conf.options; then
 	# Add a listen-on directive if it doesn't exist inside the options block.
 	sed -i "s/^}/\n\tlisten-on { 127.0.0.1; };\n}/" /etc/bind/named.conf.options
 fi
+if ! grep -q "max-recursion-queries " /etc/bind/named.conf.options; then
+	# Add a max-recursion-queries directive if it doesn't exist inside the options block.
+	sed -i "s/^}/\n\tmax-recursion-queries 100;\n}/" /etc/bind/named.conf.options
+fi
 
 # First we'll disable systemd-resolved's management of resolv.conf and its stub server.
 # Breaking the symlink to /run/systemd/resolve/stub-resolv.conf means

setup/webmail.sh

@@ -28,8 +28,8 @@ apt_install \
 # Install Roundcube from source if it is not already present or if it is out of date.
 # Combine the Roundcube version number with the commit hash of plugins to track
 # whether we have the latest version of everything.
-VERSION=1.4.9
-HASH=df650f4d3eae9eaae2d5a5f06d68665691daf57d
+VERSION=1.4.10
+HASH=36b2351030e1ebddb8e39190d7b0ba82b1bbec1b
 PERSISTENT_LOGIN_VERSION=6b3fc450cae23ccb2f393d0ef67aa319e877e435
 HTML5_NOTIFIER_VERSION=4b370e3cd60dabd2f428a26f45b677ad1b7118d5
 CARDDAV_VERSION=3.0.3

setup/zpush.sh

@@ -22,8 +22,8 @@ apt_install \
 phpenmod -v php imap
 
 # Copy Z-Push into place.
-VERSION=2.5.2
-TARGETHASH=2dc3dbd791b96b0ba2638df0d3d1e03c7e1cbab2
+VERSION=2.6.1
+TARGETHASH=a4415f0dc0ed884acc8ad5c506944fc7e6d68eeb
 needs_update=0 #NODOC
 if [ ! -f /usr/local/lib/z-push/version ]; then
 	needs_update=1 #NODOC