Merge remote-tracking branch 'upstream/master' into solr-jetty

2026-04-16 23:57:24 +02:00 · 2019-09-03 08:51:14 -07:00
parent 2f4bc0876d 9e29564f48
commit 44b3258146
28 changed files with 302 additions and 72 deletions
--- a/setup/bootstrap.sh
+++ b/setup/bootstrap.sh
@@ -20,7 +20,7 @@ if [ -z "$TAG" ]; then
 	# want to display in status checks.
 	if [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/18\.04\.[0-9]/18.04/' `" == "Ubuntu 18.04 LTS" ]; then
 		# This machine is running Ubuntu 18.04.
-		TAG=v0.40
+		TAG=v0.43

 	elif [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/14\.04\.[0-9]/14.04/' `" == "Ubuntu 14.04 LTS" ]; then
 		# This machine is running Ubuntu 14.04.
--- a/setup/functions.sh
+++ b/setup/functions.sh
@@ -1,7 +1,7 @@
 # Turn on "strict mode." See http://redsymbol.net/articles/unofficial-bash-strict-mode/.
 # -e: exit if any command unexpectedly fails.
 # -u: exit if we have a variable typo.
-# -o pipefail: don't ignore errors in the non-last command in a pipeline 
+# -o pipefail: don't ignore errors in the non-last command in a pipeline
 set -euo pipefail

 function hide_output {
@@ -127,7 +127,7 @@ function get_default_privateip {
 	if [ "$1" == "6" ]; then target=2001:4860:4860::8888; fi

 	# Get the route information.
-	route=$(ip -$1 -o route get $target | grep -v unreachable)
+	route=$(ip -$1 -o route get $target 2>/dev/null | grep -v unreachable)

 	# Parse the address out of the route information.
 	address=$(echo $route | sed "s/.* src \([^ ]*\).*/\1/")
--- a/setup/mail-dovecot.sh
+++ b/setup/mail-dovecot.sh
@@ -80,11 +80,12 @@ tools/editconf.py /etc/dovecot/conf.d/10-auth.conf \
 # Enable SSL, specify the location of the SSL certificate and private key files.
 # Disable obsolete SSL protocols and allow only good ciphers per http://baldric.net/2013/12/07/tls-ciphers-in-postfix-and-dovecot/.
 # Enable strong ssl dh parameters
+
 tools/editconf.py /etc/dovecot/conf.d/10-ssl.conf \
 	ssl=required \
 	"ssl_cert=<$STORAGE_ROOT/ssl/ssl_certificate.pem" \
 	"ssl_key=<$STORAGE_ROOT/ssl/ssl_private_key.pem" \
-	"ssl_protocols=!SSLv3 !SSLv2" \
+	"ssl_protocols=!SSLv3" \
 	"ssl_cipher_list=ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS" \
 	"ssl_prefer_server_ciphers = yes" \
 	"ssl_dh_parameters_length = 2048"
@@ -135,6 +136,14 @@ service lmtp {
  }
 }

+# Enable imap-login on localhost to allow the user_external plugin
+# for Nextcloud to do imap authentication. (See #1577)
+service imap-login {
+  inet_listener imap {
+    address = 127.0.0.1
+    port = 143
+  }
+}
 protocol imap {
  mail_max_userip_connections = 20
 }
--- a/setup/mail-postfix.sh
+++ b/setup/mail-postfix.sh
@@ -173,8 +173,11 @@ tools/editconf.py /etc/postfix/main.cf \
 #
 # In a basic setup we would pass mail directly to Dovecot by setting
 # virtual_transport to `lmtp:unix:private/dovecot-lmtp`.
-#
 tools/editconf.py /etc/postfix/main.cf virtual_transport=lmtp:[127.0.0.1]:10025
+# Because of a spampd bug, limit the number of recipients in each connection.
+# See https://github.com/mail-in-a-box/mailinabox/issues/1523.
+tools/editconf.py /etc/postfix/main.cf lmtp_destination_recipient_limit=1
+

 # Who can send mail to us? Some basic filters.
 #
@@ -205,7 +208,34 @@ tools/editconf.py /etc/postfix/main.cf \
 # e-mails really latter, delay of greylisting has been set to
 # 180 seconds (default is 300 seconds).
 tools/editconf.py /etc/default/postgrey \
-	POSTGREY_OPTS=\"'--inet=127.0.0.1:10023 --delay=180'\"
+	POSTGREY_OPTS=\"'--inet=127.0.0.1:10023 --delay=180 --whitelist-recipients=/etc/postgrey/whitelist_clients'\"
+
+
+# We are going to setup a newer whitelist for postgrey, the version included in the distribution is old
+cat > /etc/cron.daily/mailinabox-postgrey-whitelist << EOF;
+#!/bin/bash
+
+# Mail-in-a-Box
+
+# check we have a postgrey_whitelist_clients file and that it is not older than 28 days
+if [ ! -f /etc/postgrey/whitelist_clients ] || find /etc/postgrey/whitelist_clients -mtime +28 > /dev/null ; then
+    # ok we need to update the file, so lets try to fetch it
+    if curl https://postgrey.schweikert.ch/pub/postgrey_whitelist_clients --output /tmp/postgrey_whitelist_clients -sS --fail > /dev/null 2>&1 ; then
+        # if fetching hasn't failed yet then check it is a plain text file
+        # curl manual states that --fail sometimes still produces output
+        # this final check will at least check the output is not html
+        # before moving it into place
+        if [ "\$(file -b --mime-type /tmp/postgrey_whitelist_clients)" == "text/plain" ]; then
+            mv /tmp/postgrey_whitelist_clients /etc/postgrey/whitelist_clients
+            service postgrey restart
+	else
+            rm /tmp/postgrey_whitelist_clients
+        fi
+    fi
+fi
+EOF
+chmod +x /etc/cron.daily/mailinabox-postgrey-whitelist
+/etc/cron.daily/mailinabox-postgrey-whitelist

 # Increase the message size limit from 10MB to 128MB.
 # The same limit is specified in nginx.conf for mail submitted via webmail and Z-Push.
--- a/setup/management.sh
+++ b/setup/management.sh
@@ -38,7 +38,7 @@ inst_dir=/usr/local/lib/mailinabox
 mkdir -p $inst_dir
 venv=$inst_dir/env
 if [ ! -d $venv ]; then
-	virtualenv -ppython3 $venv
+	hide_output virtualenv -ppython3 $venv
 fi

 # Upgrade pip because the Ubuntu-packaged version is out of date.
@@ -93,7 +93,7 @@ source $venv/bin/activate
 exec python `pwd`/management/daemon.py
 EOF
 chmod +x $inst_dir/start
-hide_output systemctl link conf/mailinabox.service
+hide_output systemctl link -f conf/mailinabox.service
 hide_output systemctl daemon-reload
 hide_output systemctl enable mailinabox.service

--- a/setup/munin.sh
+++ b/setup/munin.sh
@@ -64,7 +64,7 @@ mkdir -p /var/lib/munin-node/plugin-state/
 # Create a systemd service for munin.
 ln -sf $(pwd)/management/munin_start.sh /usr/local/lib/mailinabox/munin_start.sh
 chmod 0744 /usr/local/lib/mailinabox/munin_start.sh
-hide_output systemctl link conf/munin.service
+hide_output systemctl link -f conf/munin.service
 hide_output systemctl daemon-reload
 hide_output systemctl unmask munin.service
 hide_output systemctl enable munin.service
@@ -76,4 +76,8 @@ restart_service munin-node
 # generate initial statistics so the directory isn't empty
 # (We get "Pango-WARNING **: error opening config file '/root/.config/pango/pangorc': Permission denied"
 # if we don't explicitly set the HOME directory when sudo'ing.)
-sudo -H -u munin munin-cron
+# We check to see if munin-cron is already running, if it is, there is no need to run it simultaneously
+# generating an error.
+if [ ! -f /var/run/munin/munin-update.lock ]; then
+	sudo -H -u munin munin-cron
+fi
--- a/setup/nextcloud.sh
+++ b/setup/nextcloud.sh
@@ -13,7 +13,8 @@ apt-get purge -qq -y owncloud* # we used to use the package manager

 apt_install php php-fpm \
 	php-cli php-sqlite3 php-gd php-imap php-curl php-pear curl \
-	php-dev php-gd php-xml php-mbstring php-zip php-apcu php-json php-intl
+	php-dev php-gd php-xml php-mbstring php-zip php-apcu php-json \
+	php-intl php-imagick

 InstallNextcloud() {

@@ -24,12 +25,12 @@ InstallNextcloud() {
 	echo "Upgrading to Nextcloud version $version"
 	echo

+        # Download and verify
+        wget_verify https://download.nextcloud.com/server/releases/nextcloud-$version.zip $hash /tmp/nextcloud.zip
+
 	# Remove the current owncloud/Nextcloud
 	rm -rf /usr/local/lib/owncloud

-	# Download and verify
-	wget_verify https://download.nextcloud.com/server/releases/nextcloud-$version.zip $hash /tmp/nextcloud.zip
-
 	# Extract ownCloud/Nextcloud
 	unzip -q /tmp/nextcloud.zip -d /usr/local/lib
 	mv /usr/local/lib/nextcloud /usr/local/lib/owncloud
@@ -39,14 +40,22 @@ InstallNextcloud() {
 	# their github repositories.
 	mkdir -p /usr/local/lib/owncloud/apps

-	wget_verify https://github.com/nextcloud/contacts/releases/download/v2.1.5/contacts.tar.gz b7460d15f1b78d492ed502d778c0c458d503ba17 /tmp/contacts.tgz
+	wget_verify https://github.com/nextcloud/contacts/releases/download/v3.1.1/contacts.tar.gz a06bd967197dcb03c94ec1dbd698c037018669e5 /tmp/contacts.tgz
 	tar xf /tmp/contacts.tgz -C /usr/local/lib/owncloud/apps/
 	rm /tmp/contacts.tgz

-	wget_verify https://github.com/nextcloud/calendar/releases/download/v1.6.1/calendar.tar.gz f93a247cbd18bc624f427ba2a967d93ebb941f21 /tmp/calendar.tgz
+	wget_verify https://github.com/nextcloud/calendar/releases/download/v1.6.5/calendar.tar.gz 79941255521a5172f7e4ce42dc7773838b5ede2f /tmp/calendar.tgz
 	tar xf /tmp/calendar.tgz -C /usr/local/lib/owncloud/apps/
 	rm /tmp/calendar.tgz

+	# Starting with Nextcloud 15, the app user_external is no longer included in Nextcloud core,
+	# we will install from their github repository.
+	if [[ $version =~ ^15 ]]; then
+		wget_verify https://github.com/nextcloud/user_external/releases/download/v0.6.3/user_external-0.6.3.tar.gz 0f756d35fef6b64a177d6a16020486b76ea5799c /tmp/user_external.tgz
+		tar -xf /tmp/user_external.tgz -C /usr/local/lib/owncloud/apps/
+		rm /tmp/user_external.tgz
+	fi
+
 	# Fix weird permissions.
 	chmod 750 /usr/local/lib/owncloud/{apps,config}

@@ -72,15 +81,35 @@ InstallNextcloud() {
 			sudo -u www-data php /usr/local/lib/owncloud/occ maintenance:mode --off
 			echo "...which seemed to work."
 		fi
+
+		# Add missing indices. NextCloud didn't include this in the normal upgrade because it might take some time.
+		sudo -u www-data php /usr/local/lib/owncloud/occ db:add-missing-indices
+
+		# Run conversion to BigInt identifiers, this process may take some time on large tables.
+		sudo -u www-data php /usr/local/lib/owncloud/occ db:convert-filecache-bigint --no-interaction
 	fi
 }

-nextcloud_ver=13.0.6
-nextcloud_hash=33e41f476f0e2be5dc7cdb9d496673d9647aa3d6
+# Nextcloud Version to install. Checks are done down below to step through intermediate versions.
+nextcloud_ver=15.0.8
+nextcloud_hash=4129d8d4021c435f2e86876225fb7f15adf764a3

-# Check if Nextcloud dir exist, and check if version matches nextcloud_ver (if either doesn't - install/upgrade)
-if [ ! -d /usr/local/lib/owncloud/ ] \
-		|| ! grep -q $nextcloud_ver /usr/local/lib/owncloud/version.php; then
+# Current Nextcloud Version, #1623
+# Checking /usr/local/lib/owncloud/version.php shows version of the Nextcloud application, not the DB
+# $STORAGE_ROOT/owncloud is kept together even during a backup.  It is better to rely on config.php than
+# version.php since the restore procedure can leave the system in a state where you have a newer Nextcloud
+# application version than the database.
+
+# If config.php exists, get version number, otherwise CURRENT_NEXTCLOUD_VER is empty.
+if [ -f "$STORAGE_ROOT/owncloud/config.php" ]; then
+	CURRENT_NEXTCLOUD_VER=$(php -r "include(\"$STORAGE_ROOT/owncloud/config.php\"); echo(\$CONFIG['version']);")
+else
+	CURRENT_NEXTCLOUD_VER=""
+fi
+
+# If the Nextcloud directory is missing (never been installed before, or the nextcloud version to be installed is different
+# from the version currently installed, do the install/upgrade
+if [ ! -d /usr/local/lib/owncloud/ ] || [[ ! ${CURRENT_NEXTCLOUD_VER} =~ ^$nextcloud_ver ]]; then

 	# Stop php-fpm if running. If theyre not running (which happens on a previously failed install), dont bail.
 	service php7.2-fpm stop &> /dev/null || /bin/true
@@ -93,30 +122,30 @@ if [ ! -d /usr/local/lib/owncloud/ ] \
 		echo "Upgrading Nextcloud --- backing up existing installation, configuration, and database to directory to $BACKUP_DIRECTORY..."
 		cp -r /usr/local/lib/owncloud "$BACKUP_DIRECTORY/owncloud-install"
 	fi
-	if [ -e /home/user-data/owncloud/owncloud.db ]; then
-		cp /home/user-data/owncloud/owncloud.db $BACKUP_DIRECTORY
+	if [ -e $STORAGE_ROOT/owncloud/owncloud.db ]; then
+		cp $STORAGE_ROOT/owncloud/owncloud.db $BACKUP_DIRECTORY
 	fi
-	if [ -e /home/user-data/owncloud/config.php ]; then
-		cp /home/user-data/owncloud/config.php $BACKUP_DIRECTORY
+	if [ -e $STORAGE_ROOT/owncloud/config.php ]; then
+		cp $STORAGE_ROOT/owncloud/config.php $BACKUP_DIRECTORY
 	fi

 	# If ownCloud or Nextcloud was previously installed....
-	if [ -e /usr/local/lib/owncloud/version.php ]; then
+	if [ ! -z ${CURRENT_NEXTCLOUD_VER} ]; then
 		# Database migrations from ownCloud are no longer possible because ownCloud cannot be run under
 		# PHP 7.
-		if grep -q "OC_VersionString = '[89]\." /usr/local/lib/owncloud/version.php; then
-			echo "Upgrades from Mail-in-a-Box prior to v0.26c (dated February 13, 2018) with Nextcloud < 12.0.5 (you have ownCloud 8 or 9) are not supported. Upgrade to Mail-in-a-Box version v0.28 first. Setup aborting."
+		if [[ ${CURRENT_NEXTCLOUD_VER} =~ ^[89] ]]; then
+			echo "Upgrades from Mail-in-a-Box prior to v0.28 (dated July 30, 2018) with Nextcloud < 13.0.6 (you have ownCloud 8 or 9) are not supported. Upgrade to Mail-in-a-Box version v0.30 first. Setup aborting."
 			exit 1
-		fi
-		if grep -q "OC_VersionString = '10\." /usr/local/lib/owncloud/version.php; then
-			echo "Upgrades from Mail-in-a-Box prior to v0.26c (dated February 13, 2018) with Nextcloud < 12.0.5 (you have ownCloud 10) are not supported. Upgrade to Mail-in-a-Box version v0.28 first. Setup aborting."
+		elif [[ ${CURRENT_NEXTCLOUD_VER} =~ ^1[012] ]]; then
+			echo "Upgrades from Mail-in-a-Box prior to v0.28 (dated July 30, 2018) with Nextcloud < 13.0.6 (you have ownCloud 10, 11 or 12) are not supported. Upgrade to Mail-in-a-Box version v0.30 first. Setup aborting."
 			exit 1
-		fi
-
-		# If we are upgrading from Nextcloud 11 we should go to Nextcloud 12 first.
-		if grep -q "OC_VersionString = '11\." /usr/local/lib/owncloud/version.php; then
-			echo "We are running Nextcloud 11, upgrading to Nextcloud 12.0.5 first"
-			InstallNextcloud 12.0.5 d25afbac977a4e331f5e38df50aed0844498ca86
+		elif [[ ${CURRENT_NEXTCLOUD_VER} =~ ^13 ]]; then
+			# If we are running Nextcloud 13, upgrade to Nextcloud 14
+			InstallNextcloud 14.0.6 4e43a57340f04c2da306c8eea98e30040399ae5a
+		elif [[ ${CURRENT_NEXTCLOUD_VER} =~ ^14 ]]; then
+			# During the upgrade from Nextcloud 14 to 15, user_external may cause the upgrade to fail.
+			# We will disable it here before the upgrade and install it again after the upgrade.
+			hide_output sudo -u www-data php /usr/local/lib/owncloud/console.php app:disable user_external
 		fi
 	fi

@@ -145,10 +174,12 @@ if [ ! -f $STORAGE_ROOT/owncloud/owncloud.db ]; then
  'overwritewebroot' => '/cloud',
  'overwrite.cli.url' => '/cloud',
  'user_backends' => array(
-	array(
-	  'class'=>'OC_User_IMAP',
-	  'arguments'=>array('{127.0.0.1:993/imap/ssl/novalidate-cert}')
-	)
+    array(
+      'class' => 'OC_User_IMAP',
+        'arguments' => array(
+          '127.0.0.1', 143, null
+         ),
+    ),
  ),
  'memcache.local' => '\OC\Memcache\APCu',
  'mail_smtpmode' => 'sendmail',
@@ -220,6 +251,8 @@ include("$STORAGE_ROOT/owncloud/config.php");

 \$CONFIG['mail_domain'] = '$PRIMARY_HOSTNAME';

+\$CONFIG['user_backends'] = array(array('class' => 'OC_User_IMAP','arguments' => array('127.0.0.1', 143, null),),);
+
 echo "<?php\n\\\$CONFIG = ";
 var_export(\$CONFIG);
 echo ";";
--- a/setup/preflight.sh
+++ b/setup/preflight.sh
@@ -26,7 +26,7 @@ fi
 #
 # Skip the check if we appear to be running inside of Vagrant, because that's really just for testing.
 TOTAL_PHYSICAL_MEM=$(head -n 1 /proc/meminfo | awk '{print $2}')
-if [ $TOTAL_PHYSICAL_MEM -lt 500000 ]; then
+if [ $TOTAL_PHYSICAL_MEM -lt 490000 ]; then
 if [ ! -d /vagrant ]; then
 	TOTAL_PHYSICAL_MEM=$(expr \( \( $TOTAL_PHYSICAL_MEM \* 1024 \) / 1000 \) / 1000)
 	echo "Your Mail-in-a-Box needs more memory (RAM) to function properly."
--- a/setup/system.sh
+++ b/setup/system.sh
@@ -339,6 +339,7 @@ systemctl restart systemd-resolved

 # Configure the Fail2Ban installation to prevent dumb bruce-force attacks against dovecot, postfix, ssh, etc.
 rm -f /etc/fail2ban/jail.local # we used to use this file but don't anymore
+rm -f /etc/fail2ban/jail.d/defaults-debian.conf # removes default config so we can manage all of fail2ban rules in one config
 cat conf/fail2ban/jails.conf \
 	| sed "s/PUBLIC_IP/$PUBLIC_IP/g" \
 	| sed "s#STORAGE_ROOT#$STORAGE_ROOT#" \
--- a/setup/webmail.sh
+++ b/setup/webmail.sh
@@ -28,8 +28,8 @@ apt_install \
 # Install Roundcube from source if it is not already present or if it is out of date.
 # Combine the Roundcube version number with the commit hash of plugins to track
 # whether we have the latest version of everything.
-VERSION=1.3.8
-HASH=90c7900ccf7b2f46fe49c650d5adb9b85ee9cc22
+VERSION=1.3.10
+HASH=431625fc737e301f9b7e502cccc61e50a24786b8
 PERSISTENT_LOGIN_VERSION=dc5ca3d3f4415cc41edb2fde533c8a8628a94c76
 HTML5_NOTIFIER_VERSION=4b370e3cd60dabd2f428a26f45b677ad1b7118d5
 CARDDAV_VERSION=3.0.3
--- a/setup/zpush.sh
+++ b/setup/zpush.sh
@@ -22,8 +22,8 @@ apt_install \
 phpenmod -v php imap

 # Copy Z-Push into place.
-VERSION=2.4.4
-TARGETHASH=104d44426852429dac8ec2783a4e9ad7752d4682
+VERSION=2.5.0
+TARGETHASH=30ce5c1af3f10939036361b6032d1187651b621e
 needs_update=0 #NODOC
 if [ ! -f /usr/local/lib/z-push/version ]; then
 	needs_update=1 #NODOC