Merge remote-tracking branch 'upstream/main' into merge-upstream

# Conflicts: # .gitignore # management/auth.py # management/daemon.py # management/mail_log.py # management/mailconfig.py # management/mfa.py # management/ssl_certificates.py # management/status_checks.py # management/utils.py # management/web_update.py # setup/mail-postfix.sh # setup/migrate.py # setup/preflight.sh # setup/webmail.sh # tests/test_mail.py # tools/editconf.py
2026-03-12 17:07:23 +01:00 · 2024-03-12 07:41:14 -04:00
parent c884d63dda 18b8f9ab4b
commit d349150dd0
33 changed files with 582 additions and 571 deletions
--- a/setup/mail-postfix.sh
+++ b/setup/mail-postfix.sh
@@ -70,7 +70,7 @@ tools/editconf.py /etc/postfix/main.cf \
 	myhostname=$PRIMARY_HOSTNAME\
 	smtpd_banner="\$myhostname ESMTP Hi, I'm a Mail-in-a-Box (Ubuntu/Postfix; see https://mailinabox.email/)" \
 	mydestination=localhost
-    
+
 # Tweak some queue settings:
 # * Inform users when their e-mail delivery is delayed more than 3 hours (default is not to warn).
 # * Stop trying to send an undeliverable e-mail after 2 days (instead of 5), and for bounce messages just try for 1 day.
@@ -245,14 +245,15 @@ tools/editconf.py /etc/postfix/main.cf  -e lmtp_destination_recipient_limit=
 # * `reject_unlisted_recipient`: Although Postfix will reject mail to unknown recipients, it's nicer to reject such mail ahead of greylisting rather than after.
 # * `check_policy_service`: Apply greylisting using postgrey.
 #
+# Note the spamhaus rbl return codes are taken into account as adviced here: https://docs.spamhaus.com/datasets/docs/source/40-real-world-usage/PublicMirrors/MTAs/020-Postfix.html
 # Notes: #NODOC
 # permit_dnswl_client can pass through mail from whitelisted IP addresses, which would be good to put before greylisting #NODOC
 # so these IPs get mail delivered quickly. But when an IP is not listed in the permit_dnswl_client list (i.e. it is not #NODOC
 # whitelisted) then postfix does a DEFER_IF_REJECT, which results in all "unknown user" sorts of messages turning into #NODOC
 # "450 4.7.1 Client host rejected: Service unavailable". This is a retry code, so the mail doesn't properly bounce. #NODOC
 tools/editconf.py /etc/postfix/main.cf \
-	smtpd_sender_restrictions="reject_non_fqdn_sender,reject_unknown_sender_domain,reject_authenticated_sender_login_mismatch,reject_rhsbl_sender dbl.spamhaus.org" \
-	smtpd_recipient_restrictions=permit_sasl_authenticated,permit_mynetworks,"reject_rbl_client zen.spamhaus.org",reject_unlisted_recipient,"check_policy_service unix:private/policy-spf","check_policy_service inet:127.0.0.1:10023"
+	smtpd_sender_restrictions="reject_non_fqdn_sender,reject_unknown_sender_domain,reject_authenticated_sender_login_mismatch,reject_rhsbl_sender dbl.spamhaus.org=127.0.1.[2..99]" \
+	smtpd_recipient_restrictions="permit_sasl_authenticated,permit_mynetworks,reject_rbl_client zen.spamhaus.org=127.0.0.[2..11],reject_unlisted_recipient,check_policy_service unix:private/policy-spf,check_policy_service inet:127.0.0.1:10023"

 # Postfix connects to Postgrey on the 127.0.0.1 interface specifically. Ensure that
 # Postgrey listens on the same interface (and not IPv6, for instance).
--- a/setup/management.sh
+++ b/setup/management.sh
@@ -75,18 +75,18 @@ rm -rf $assets_dir
 mkdir -p $assets_dir

 # jQuery CDN URL
-jquery_version=2.1.4
+jquery_version=2.2.4
 jquery_url=https://code.jquery.com

 # Get jQuery
-wget_verify $jquery_url/jquery-$jquery_version.min.js 43dc554608df885a59ddeece1598c6ace434d747 $assets_dir/jquery.min.js
+wget_verify $jquery_url/jquery-$jquery_version.min.js 69bb69e25ca7d5ef0935317584e6153f3fd9a88c $assets_dir/jquery.min.js

 # Bootstrap CDN URL
-bootstrap_version=3.3.7
+bootstrap_version=3.4.1
 bootstrap_url=https://github.com/twbs/bootstrap/releases/download/v$bootstrap_version/bootstrap-$bootstrap_version-dist.zip

 # Get Bootstrap
-wget_verify $bootstrap_url e6b1000b94e835ffd37f4c6dcbdad43f4b48a02a /tmp/bootstrap.zip
+wget_verify $bootstrap_url 0bb64c67c2552014d48ab4db81c2e8c01781f580 /tmp/bootstrap.zip
 unzip -q /tmp/bootstrap.zip -d $assets_dir
 mv $assets_dir/bootstrap-$bootstrap_version-dist $assets_dir/bootstrap
 rm -f /tmp/bootstrap.zip
--- a/setup/migrate.py
+++ b/setup/migrate.py
@@ -19,6 +19,7 @@ import sys, os, os.path, glob, re, shutil

 sys.path.insert(0, 'management')
 from utils import load_environment, load_env_vars_from_file, save_environment, shell
+import contextlib

 def migration_1(env):
 	# Re-arrange where we store SSL certificates. There was a typo also.
@@ -41,10 +42,8 @@ def migration_1(env):
 			move_file(sslfn, domain_name, file_type)

 	# Move the old domains directory if it is now empty.
-	try:
+	with contextlib.suppress(Exception):
 		os.rmdir(os.path.join( env["STORAGE_ROOT"], 'ssl/domains'))
-	except:
-		pass

 def migration_2(env):
 	# Delete the .dovecot_sieve script everywhere. This was formerly a copy of our spam -> Spam
@@ -178,7 +177,7 @@ def migration_12(env):
                        dropcmd = "DROP TABLE %s" % table
                        c.execute(dropcmd)
                    except:
-                        print("Failed to drop table", table, e)
+                        print("Failed to drop table", table)
            # Save.
            conn.commit()
            conn.close()
@@ -215,7 +214,7 @@ def migration_miabldap_1(env):
 	#   maildrop: [email]
 	#   userPassword: [password]
 	#   mailaccess: [privilege]   # multi-valued
-	# 
+	#
 	# aliases table:
 	# for each row create an ldap entry of the form:
 	#   dn: cn=[uuid],ou=aliases,ou=Users,dc=mailinabox
@@ -229,15 +228,15 @@ def migration_miabldap_1(env):
 	#   objectClass: mailGroup
 	#   mail: [source]
 	#   member: [user-dn]         # multi-valued
-	
+
 	print("Migrating users and aliases from sqlite to ldap")
-	
+
 	# Get the ldap server up and running
 	shell("check_call", ["setup/ldap.sh", "-v"])
-	
+
 	import sqlite3, ldap3
 	import migration_13 as m13
-				
+
 	# 2. get ldap site details (miab_ldap.conf was created by ldap.sh)
 	ldapvars = load_env_vars_from_file(os.path.join(env["STORAGE_ROOT"], "ldap/miab_ldap.conf"), strip_quotes=True)
 	ldap_base = ldapvars.LDAP_BASE
@@ -253,7 +252,7 @@ def migration_miabldap_1(env):
 	conn = sqlite3.connect(os.path.join(env["STORAGE_ROOT"], "mail/users.sqlite"))
 	ldap = ldap3.Connection('127.0.0.1', ldap_admin_dn, ldap_admin_pass, raise_exceptions=True)
 	ldap.bind()
-	
+
 	# 4. perform the migration
 	users=m13.create_users(env, conn, ldap, ldap_base, ldap_users_base, ldap_domains_base)
 	aliases=m13.create_aliases(env, conn, ldap, ldap_aliases_base)
@@ -301,7 +300,7 @@ def migration_miabldap_2(env):
 		"-LLL",
 		"olcObjectClasses"
 	])
-	
+
 	if "rfc822MailMember" in ret:
 		def ldif_change_fn(ldif):
 			return ldif.replace("rfc822MailMember: ", "mailMember: ")
@@ -327,10 +326,10 @@ def migration_miabldap_2(env):

 	print("Ensure all required aliases are created")
 	m14.ensure_required_aliases(env, ldapvars, ldap)
-	
+
 	ldap.unbind()

-	
+
 def get_current_migration():
 	ver = 0
 	while True:
@@ -350,8 +349,8 @@ def run_migrations():
 	migration_id_file = os.path.join(env['STORAGE_ROOT'], 'mailinabox.version')
 	migration_id = None
 	if os.path.exists(migration_id_file):
-		with open(migration_id_file) as f:
-			migration_id = f.read().strip();
+		with open(migration_id_file, encoding='utf-8') as f:
+			migration_id = f.read().strip()

 	if migration_id is None:
 		# Load the legacy location of the migration ID. We'll drop support
@@ -360,7 +359,7 @@ def run_migrations():

 	if migration_id is None:
 		print()
-		print("%s file doesn't exists. Skipping migration..." % (migration_id_file,))
+		print(f"{migration_id_file} file doesn't exists. Skipping migration...")
 		return

 	ourver = int(migration_id)
@@ -391,7 +390,7 @@ def run_migrations():

 		# Write out our current version now. Do this sooner rather than later
 		# in case of any problems.
-		with open(migration_id_file, "w") as f:
+		with open(migration_id_file, "w", encoding='utf-8') as f:
 			f.write(str(ourver) + "\n")

 		# Delete the legacy location of this field.
@@ -422,7 +421,7 @@ def run_miabldap_migrations():
 			print()
 			print("%s file doesn't exists. Skipping migration..." % (migration_id_file,))
 			return
-	
+
 	ourver = int(migration_id)

 	while True:
@@ -464,13 +463,12 @@ if __name__ == "__main__":
 	elif sys.argv[-1] == "--migrate":
 		# Perform migrations.
 		env = load_environment()
-		
+
 		# if miab-ldap already installed, only run miab-ldap migrations
 		if 'LDAP_USERS_BASE' in env:
 			run_miabldap_migrations()
-			
+
 		# otherwise, run both
 		else:
 			run_migrations()
 			run_miabldap_migrations()
-
--- a/setup/nextcloud.sh
+++ b/setup/nextcloud.sh
@@ -38,31 +38,48 @@ echo "Installing Nextcloud (contacts/calendar)..."
 #   we automatically install intermediate versions as needed.
 # * The hash is the SHA1 hash of the ZIP package, which you can find by just running this script and
 #   copying it from the error message when it doesn't match what is below.
-nextcloud_ver=25.0.7
-nextcloud_hash=a5a565c916355005c7b408dd41a1e53505e1a080
+nextcloud_ver=26.0.12
+nextcloud_hash=b55e9f51171c0a9b9ab3686cf5c8ad1a4292ca15

 # Nextcloud apps
 # --------------
-# * Find the most recent tag that is compatible with the Nextcloud version above by
-#   consulting the <dependencies>...<nextcloud> node at:
-#   https://github.com/nextcloud-releases/contacts/blob/main/appinfo/info.xml
-#   https://github.com/nextcloud-releases/calendar/blob/main/appinfo/info.xml
-#   https://github.com/nextcloud/user_external/blob/master/appinfo/info.xml
-# * The hash is the SHA1 hash of the ZIP package, which you can find by just running this script and
-#   copying it from the error message when it doesn't match what is below.
-contacts_ver=5.3.0
-contacts_hash=4b0a6666374e3b55cfd2ae9b72e1d458b87d4c8c
+# * Find the most recent tag that is compatible with the Nextcloud version above by:
+#   https://github.com/nextcloud-releases/contacts/tags
+#   https://github.com/nextcloud-releases/calendar/tags
+#   https://github.com/nextcloud/user_external/tags
+#
+# * For these three packages, contact, calendar and user_external, the hash is the SHA1 hash of
+# the ZIP package, which you can find by just running this script and copying it from
+# the error message when it doesn't match what is below:
+
+# Always ensure the versions are supported, see https://apps.nextcloud.com/apps/contacts
+contacts_ver=5.5.3
+contacts_hash=799550f38e46764d90fa32ca1a6535dccd8316e5

 # Always ensure the versions are supported, see https://apps.nextcloud.com/apps/calendar
-calendar_ver=4.4.2
-calendar_hash=21a42e15806adc9b2618760ef94f1797ef399e2f
+calendar_ver=4.6.6
+calendar_hash=e34a71669a52d997e319d64a984dcd041389eb22

-# And https://apps.nextcloud.com/apps/user_external
+# Always ensure the versions are supported, see https://apps.nextcloud.com/apps/user_external
 user_external_ver=3.2.0
 user_external_hash=a494073dcdecbbbc79a9c77f72524ac9994d2eec

-# Clear prior packages and install dependencies from apt.
+# Developer advice (test plan)
+# ----------------------------
+# When upgrading above versions, how to test?
+#
+# 1. Enter your server instance (or on the Vagrant image)
+# 1. Git clone <your fork>
+# 2. Git checkout <your fork>
+# 3. Run `sudo ./setup/nextcloud.sh`
+# 4. Ensure the installation completes. If any hashes mismatch, correct them.
+# 5. Enter nextcloud web, run following tests:
+# 5.1 You still can create, edit and delete contacts
+# 5.2 You still can create, edit and delete calendar events
+# 5.3 You still can create, edit and delete users
+# 5.4 Go to Administration > Logs and ensure no new errors are shown

+# Clear prior packages and install dependencies from apt.
 apt-get purge -qq -y owncloud* # we used to use the package manager

 apt_install curl php${PHP_VER} php${PHP_VER}-fpm \
@@ -167,7 +184,7 @@ InstallNextcloud() {

 # Current Nextcloud Version, #1623
 # Checking /usr/local/lib/owncloud/version.php shows version of the Nextcloud application, not the DB
-# $STORAGE_ROOT/owncloud is kept together even during a backup.  It is better to rely on config.php than
+# $STORAGE_ROOT/owncloud is kept together even during a backup. It is better to rely on config.php than
 # version.php since the restore procedure can leave the system in a state where you have a newer Nextcloud
 # application version than the database.

@@ -225,6 +242,11 @@ if [ ! -d /usr/local/lib/owncloud/ ] || [[ ! ${CURRENT_NEXTCLOUD_VER} =~ ^$nextc
 			return 0
 		fi

+		# Hint: whenever you bump, remember this:
+		# - Run a server with the previous version
+		# - On a new if-else block, copy the versions/hashes from the previous version
+		# - Run sudo ./setup/start.sh on the new machine. Upon completion, test its basic functionalities.
+
 		if [[ ${CURRENT_NEXTCLOUD_VER} =~ ^20 ]]; then
 			InstallNextcloud 21.0.7 f5c7079c5b56ce1e301c6a27c0d975d608bb01c9 4.0.7 45e7cf4bfe99cd8d03625cf9e5a1bb2e90549136 3.0.4 d0284b68135777ec9ca713c307216165b294d0fe
 			CURRENT_NEXTCLOUD_VER="21.0.7"
@@ -241,6 +263,10 @@ if [ ! -d /usr/local/lib/owncloud/ ] || [[ ! ${CURRENT_NEXTCLOUD_VER} =~ ^$nextc
 			InstallNextcloud 24.0.12 7aa5d61632c1ccf4ca3ff00fb6b295d318c05599 4.1.0 697f6b4a664e928d72414ea2731cb2c9d1dc3077 3.2.2 ce4030ab57f523f33d5396c6a81396d440756f5f 3.0.0 0df781b261f55bbde73d8c92da3f99397000972f
 			CURRENT_NEXTCLOUD_VER="24.0.12"
 		fi
+        if [[ ${CURRENT_NEXTCLOUD_VER} =~ ^24 ]]; then
+			InstallNextcloud 25.0.7 a5a565c916355005c7b408dd41a1e53505e1a080 5.3.0 4b0a6666374e3b55cfd2ae9b72e1d458b87d4c8c 4.4.2 21a42e15806adc9b2618760ef94f1797ef399e2f 3.2.0 a494073dcdecbbbc79a9c77f72524ac9994d2eec
+			CURRENT_NEXTCLOUD_VER="25.0.7"
+		fi
 	fi

 	InstallNextcloud $nextcloud_ver $nextcloud_hash $contacts_ver $contacts_hash $calendar_ver $calendar_hash $user_external_ver $user_external_hash
--- a/setup/preflight.sh
+++ b/setup/preflight.sh
@@ -1,3 +1,4 @@
+#!/bin/bash
 #####
 ##### This file is part of Mail-in-a-Box-LDAP which is released under the
 ##### terms of the GNU Affero General Public License as published by the
@@ -35,16 +36,16 @@ fi
 #
 # Skip the check if we appear to be running inside of Vagrant, because that's really just for testing.
 TOTAL_PHYSICAL_MEM=$(head -n 1 /proc/meminfo | awk '{print $2}')
-if [ $TOTAL_PHYSICAL_MEM -lt 490000 ]; then
+if [ "$TOTAL_PHYSICAL_MEM" -lt 490000 ]; then
 if [ ! -d /vagrant ]; then
-	TOTAL_PHYSICAL_MEM=$(expr \( \( $TOTAL_PHYSICAL_MEM \* 1024 \) / 1000 \) / 1000)
+	TOTAL_PHYSICAL_MEM=$(( TOTAL_PHYSICAL_MEM * 1024 / 1000 / 1000 ))
 	echo "Your Mail-in-a-Box needs more memory (RAM) to function properly."
 	echo "Please provision a machine with at least 512 MB, 1 GB recommended."
 	echo "This machine has $TOTAL_PHYSICAL_MEM MB memory."
 	exit
 fi
 fi
-if [ $TOTAL_PHYSICAL_MEM -lt 750000 ]; then
+if [ "$TOTAL_PHYSICAL_MEM" -lt 750000 ]; then
 	echo "WARNING: Your Mail-in-a-Box has less than 768 MB of memory."
 	echo "         It might run unreliably when under heavy load."
 fi
--- a/setup/webmail.sh
+++ b/setup/webmail.sh
@@ -49,8 +49,8 @@ apt_install php${PHP_VER}-ldap
 #   https://github.com/mstilkerich/rcmcarddav/releases
 # The easiest way to get the package hashes is to run this script and get the hash from
 # the error message.
-VERSION=1.6.5
-HASH=326fcc206cddc00355e98d1e40fd0bcd9baec69f
+VERSION=1.6.6
+HASH=7705d2736890c49e7ae3ac75e3ae00ba56187056
 PERSISTENT_LOGIN_VERSION=version-5.3.0
 HTML5_NOTIFIER_VERSION=68d9ca194212e15b3c7225eb6085dbcf02fd13d7 # version 0.6.4+
 CARDDAV_VERSION=4.4.3
@@ -302,4 +302,3 @@ cat > /etc/cron.daily/mailinabox-roundcubemail << EOF
 cd $RCM_DIR && bin/cleandb.sh >/dev/null
 EOF
 chmod +x /etc/cron.daily/mailinabox-roundcubemail
-