merge upstream

CHANGELOG.md

@@ -1,8 +1,8 @@
 CHANGELOG
 =========
 
-Version 60 (date TBD)
----------------------
+Version 60 (October 11, 2022)
+-----------------------------
 
 This is the first release for Ubuntu 22.04.
 
@@ -10,9 +10,11 @@ This is the first release for Ubuntu 22.04.
 
 For complete upgrade instructions, see:
 
-LINK TBD
+https://discourse.mailinabox.email/t/version-60-for-ubuntu-22-04-is-about-to-be-released/9558
 
-No features of Mail-in-a-Box have changed in this release, but with the newer version of Ubuntu the following software packages we use are updated:
+No major features of Mail-in-a-Box have changed in this release, although some minor fixes were made.
+
+With the newer version of Ubuntu the following software packages we use are updated:
 
 * dovecot is upgraded to 2.3.16, postfix to 3.6.4, opendmark to 1.4 (which adds ARC-Authentication-Results headers), and spampd to 2.53 (alleviating a mail delivery rate limiting bug).
 * Nextcloud is upgraded to 24.0.0
@@ -26,6 +28,7 @@ No features of Mail-in-a-Box have changed in this release, but with the newer ve
 Also:
 
 * Roundcube's login session cookie was tightened. Existing sessions may require a manual logout.
+* Moved Postgrey's database under $STORAGE_ROOT.
 
 Version 57a (June 19, 2022)
 ---------------------------

management/backup.py

@@ -285,6 +285,7 @@ def perform_backup(full_backup):
 	service_command(php_fpm, "stop", quit=True)
 	service_command("postfix", "stop", quit=True)
 	service_command("dovecot", "stop", quit=True)
+	service_command("postgrey", "stop", quit=True)
 
 	# Execute a pre-backup script that copies files outside the homedir.
 	# Run as the STORAGE_USER user, not as root. Pass our settings in
@@ -314,6 +315,7 @@ def perform_backup(full_backup):
 			get_duplicity_env_vars(env))
 	finally:
 		# Start services again.
+		service_command("postgrey", "start", quit=False)
 		service_command("dovecot", "start", quit=False)
 		service_command("postfix", "start", quit=False)
 		service_command(php_fpm, "start", quit=False)

management/daemon.py

@@ -573,6 +573,8 @@ def system_status():
 	# Create a temporary pool of processes for the status checks
 	with multiprocessing.pool.Pool(processes=5) as pool:
 		run_checks(False, env, output, pool)
+		pool.close()
+		pool.join()
 	return json_response(output.items)
 
 @app.route('/system/updates')

management/dns_update.py

@@ -102,9 +102,9 @@ def do_dns_update(env, force=False):
 		if len(updated_domains) == 0:
 			updated_domains.append("DNS configuration")
 
-	# Kick nsd if anything changed.
+	# Tell nsd to reload changed zone files.
 	if len(updated_domains) > 0:
-		shell('check_call', ["/usr/sbin/service", "nsd", "restart"])
+		shell('check_call', ["/usr/sbin/nsd-control", "reload"])
 
 	# Write the DKIM configuration tables for all of the mail domains.
 	from mailconfig import get_mail_domains
@@ -325,7 +325,7 @@ def build_zone(domain, domain_properties, additional_records, env, is_zone=True)
 		# Append a DMARC record.
 		# Skip if the user has set a DMARC record already.
 		if not has_rec("_dmarc", "TXT", prefix="v=DMARC1; "):
-			records.append(("_dmarc", "TXT", 'v=DMARC1; p=quarantine', "Recommended. Specifies that mail that does not originate from the box but claims to be from @%s or which does not have a valid DKIM signature is suspect and should be quarantined by the recipient's mail system." % domain))
+			records.append(("_dmarc", "TXT", 'v=DMARC1; p=quarantine;', "Recommended. Specifies that mail that does not originate from the box but claims to be from @%s or which does not have a valid DKIM signature is suspect and should be quarantined by the recipient's mail system." % domain))
 
 	if domain_properties[domain]["user"]:
 		# Add CardDAV/CalDAV SRV records on the non-primary hostname that points to the primary hostname
@@ -390,7 +390,7 @@ def build_zone(domain, domain_properties, additional_records, env, is_zone=True)
 			if not has_rec(qname, "TXT", prefix="v=spf1 "):
 				records.append((qname,  "TXT", 'v=spf1 -all', "Recommended. Prevents use of this domain name for outbound mail by specifying that no servers are valid sources for mail from @%s. If you do send email from this domain name you should either override this record such that the SPF rule does allow the originating server, or, take the recommended approach and have the box handle mail for this domain (simply add any receiving alias at this domain name to make this machine treat the domain name as one of its mail domains)." % d))
 			if not has_rec("_dmarc" + ("."+qname if qname else ""), "TXT", prefix="v=DMARC1; "):
-				records.append(("_dmarc" + ("."+qname if qname else ""), "TXT", 'v=DMARC1; p=reject', "Recommended. Prevents use of this domain name for outbound mail by specifying that the SPF rule should be honoured for mail from @%s." % d))
+				records.append(("_dmarc" + ("."+qname if qname else ""), "TXT", 'v=DMARC1; p=reject;', "Recommended. Prevents use of this domain name for outbound mail by specifying that the SPF rule should be honoured for mail from @%s." % d))
 
 			# And with a null MX record (https://explained-from-first-principles.com/email/#null-mx-record)
 			if not has_rec(qname, "MX"):

setup/dns.sh

@@ -66,14 +66,6 @@ cat > /etc/logrotate.d/nsd <<EOF;
 }
 EOF
 
-# Add systemd override file to fix some permissions
-#mkdir -p /etc/systemd/system/nsd.service.d/
-#cat > /etc/systemd/system/nsd.service.d/nsd-permissions.conf << EOF
-#[Service]
-#ReadWritePaths=/var/lib/nsd /etc/nsd /run /var/log /run/nsd
-#CapabilityBoundingSet=CAP_CHOWN CAP_IPC_LOCK CAP_NET_BIND_SERVICE CAP_SETGID CAP_SETUID CAP_SYS_CHROOT CAP_NET_ADMIN
-#EOF
-
 # Install the packages.
 #
 # * nsd: The non-recursive nameserver that publishes our DNS records.

setup/mail-postfix.sh

@@ -245,12 +245,33 @@ tools/editconf.py /etc/postfix/main.cf \
 # As a matter of fact RFC is not strict about retry timer so postfix and
 # other MTA have their own intervals. To fix the problem of receiving
 # e-mails really later, delay of greylisting has been set to
-# 180 seconds (default is 300 seconds).
+# 180 seconds (default is 300 seconds). We will move the postgrey database
+# under $STORAGE_ROOT. This prevents a "warming up" that would have occured
+# previously with a migrated or reinstalled OS.  We will specify this new path
+# with the --dbdir=... option. Arguments within POSTGREY_OPTS can not have spaces,
+# including dbdir. This is due to the way the init script sources the
+# /etc/default/postgrey file. --dbdir=... either needs to be a path without spaces
+# (luckily $STORAGE_ROOT does not currently work with spaces), or it needs to be a
+# symlink without spaces that can point to a folder with spaces).  We'll just assume
+# $STORAGE_ROOT won't have spaces to simplify things.
 # Postgrey removes entries after 185 days of not being used.
 tools/editconf.py /etc/default/postgrey \
-	POSTGREY_OPTS=\"'--inet=127.0.0.1:10023 --delay=180  --max-age=185'\"
+	POSTGREY_OPTS=\""--inet=127.0.0.1:10023 --delay=180 --max-age=185 --dbdir=$STORAGE_ROOT/mail/postgrey/db"\"
 
 
+# If the $STORAGE_ROOT/mail/postgrey is empty, copy the postgrey database over from the old location
+if [ ! -d $STORAGE_ROOT/mail/postgrey/db ]; then
+	# Stop the service
+	service postgrey stop
+	# Ensure the new paths for postgrey db exists
+	mkdir -p $STORAGE_ROOT/mail/postgrey/db
+	# Move over database files
+	mv /var/lib/postgrey/* $STORAGE_ROOT/mail/postgrey/db/ || true
+fi
+# Ensure permissions are set
+chown -R postgrey:postgrey $STORAGE_ROOT/mail/postgrey/
+chmod 700 $STORAGE_ROOT/mail/postgrey/{,db}
+
 # We are going to setup a newer whitelist for postgrey, the version included in the distribution is old
 cat > /etc/cron.daily/mailinabox-postgrey-whitelist << EOF;
 #!/bin/bash

setup/management.sh

@@ -7,18 +7,6 @@ echo "Installing Mail-in-a-Box system management daemon..."
 
 # DEPENDENCIES
 
-# We used to install management daemon-related Python packages
-# directly to /usr/local/lib. We moved to a virtualenv because
-# these packages might conflict with apt-installed packages.
-# We may have a lingering version of acme that conflcits with
-# certbot, which we're about to install below, so remove it
-# first. Once acme is installed by an apt package, this might
-# break the package version and `apt-get install --reinstall python3-acme`
-# might be needed in that case.
-while [ -d /usr/local/lib/python3.4/dist-packages/acme ]; do
-	pip3 uninstall -y acme;
-done
-
 # duplicity is used to make backups of user data.
 #
 # virtualenv is used to isolate the Python 3 packages we
@@ -90,6 +78,8 @@ rm -f /tmp/bootstrap.zip
 
 # Create an init script to start the management daemon and keep it
 # running after a reboot.
+# Set a long timeout since some commands take a while to run, matching
+# the timeout we set for PHP (fastcgi_read_timeout in the nginx confs).
 # Note: Authentication currently breaks with more than 1 gunicorn worker.
 cat > $inst_dir/start <<EOF;
 #!/bin/bash
@@ -105,7 +95,7 @@ chmod 640 /var/lib/mailinabox/api.key
 
 source $venv/bin/activate
 export PYTHONPATH=$(pwd)/management
-exec gunicorn -b localhost:10222 -w 1 wsgi:app
+exec gunicorn -b localhost:10222 -w 1 --timeout 630 wsgi:app
 EOF
 chmod +x $inst_dir/start
 cp --remove-destination conf/mailinabox.service /lib/systemd/system/mailinabox.service # target was previously a symlink so remove it first

setup/nextcloud.sh

@@ -343,7 +343,7 @@ php <<EOF > $CONFIG_TEMP && mv $CONFIG_TEMP $STORAGE_ROOT/owncloud/config.php;
 <?php
 include("$STORAGE_ROOT/owncloud/config.php");
 
-\$CONFIG['config_is_read_only'] = false;
+\$CONFIG['config_is_read_only'] = true;
 
 \$CONFIG['trusted_domains'] = array('$PRIMARY_HOSTNAME');
 
@@ -373,7 +373,7 @@ echo ";";
 ?>
 EOF
 chown www-data.www-data $STORAGE_ROOT/owncloud/config.php
-chmod 640 $STORAGE_ROOT/owncloud/config.php
+#chmod 640 $STORAGE_ROOT/owncloud/config.php
 
 # Enable/disable apps. Note that this must be done after the Nextcloud setup.
 # The firstrunwizard gave Josh all sorts of problems, so disabling that.

tests/test_dns.py

@@ -30,7 +30,7 @@ def test(server, description):
 		(hostname, "TXT", "\"v=spf1 mx -all\""),
 		("mail._domainkey." + hostname, "TXT", "\"v=DKIM1; k=rsa; s=email; \" \"p=__KEY__\""),
 		#("_adsp._domainkey." + hostname, "TXT", "\"dkim=all\""),
-		("_dmarc." + hostname, "TXT", "\"v=DMARC1; p=quarantine\""),
+		("_dmarc." + hostname, "TXT", "\"v=DMARC1; p=quarantine;\""),
 	]
 	return test2(tests, server, description)