v0.46

Fixes #1776

CHANGELOG.md

@@ -1,6 +1,40 @@
 CHANGELOG
 =========
 
+v0.46 (June 11, 2020)
+---------------------
+
+Security fixes:
+
+* Roundcube is updated to version 1.4.6 (https://roundcube.net/news/2020/06/02/security-updates-1.4.5-and-1.3.12).
+
+v0.45 (May 16, 2020)
+--------------------
+
+Security fixes:
+
+* Fix missing brute force login protection for Roundcube logins.
+
+Software updates:
+
+* Upgraded Roundcube from 1.4.2 to 1.4.4.
+* Upgraded Nextcloud from 17.0.2 to 17.0.6 (with Contacts from 3.1.6 to 3.3.0 and Calendar from 1.7.1 to v2.0.3)
+* Upgraded Z-Push to 2.5.2.
+
+System:
+
+* Nightly backups now occur on a random minute in the 3am hour (in the system time zone). The minute is chosen during Mail-in-a-Box installation/upgrade and remains the same until the next upgrade.
+* Fix for mail log statistics report on leap days.
+* Fix Mozilla autoconfig useGlobalPreferredServer setting.
+
+Web:
+
+* Add a new hidden feature to set nginx alias in www/custom.yaml.
+
+Setup:
+
+* Improved error handling.
+
 v0.44 (February 15, 2020)
 -------------------------
 

README.md

@@ -58,7 +58,7 @@ by him:
 	$ curl -s https://keybase.io/joshdata/key.asc | gpg --import
 	gpg: key C10BDD81: public key "Joshua Tauberer <jt@occams.info>" imported
 
-	$ git verify-tag v0.44
+	$ git verify-tag v0.46
 	gpg: Signature made ..... using RSA key ID C10BDD81
 	gpg: Good signature from "Joshua Tauberer <jt@occams.info>"
 	gpg: WARNING: This key is not certified with a trusted signature!
@@ -71,7 +71,7 @@ and on his [personal homepage](https://razor.occams.info/). (Of course, if this
 
 Checkout the tag corresponding to the most recent release:
 
-	$ git checkout v0.44
+	$ git checkout v0.46
 
 Begin the installation.
 

conf/fail2ban/jails.conf

@@ -50,7 +50,7 @@ findtime = 30
 enabled  = true
 port     = http,https
 filter   = miab-roundcube
-logpath  = /var/log/roundcubemail/errors
+logpath  = /var/log/roundcubemail/errors.log
 maxretry = 20
 findtime = 30
 

conf/mozilla-autoconfig.xml

@@ -21,7 +21,7 @@
          <username>%EMAILADDRESS%</username>
          <authentication>password-cleartext</authentication>
          <addThisServer>true</addThisServer>
-         <useGlobalPreferredServer>true</useGlobalPreferredServer>
+         <useGlobalPreferredServer>false</useGlobalPreferredServer>
       </outgoingServer>
 
       <documentation url="https://PRIMARY_HOSTNAME/">

management/mail_log.py

@@ -18,13 +18,13 @@ import utils
 
 
 LOG_FILES = (
-    '/var/log/mail.log',
-    '/var/log/mail.log.1',
-    '/var/log/mail.log.2.gz',
-    '/var/log/mail.log.3.gz',
-    '/var/log/mail.log.4.gz',
-    '/var/log/mail.log.5.gz',
     '/var/log/mail.log.6.gz',
+    '/var/log/mail.log.5.gz',
+    '/var/log/mail.log.4.gz',
+    '/var/log/mail.log.3.gz',
+    '/var/log/mail.log.2.gz',
+    '/var/log/mail.log.1',
+    '/var/log/mail.log',
 )
 
 TIME_DELTAS = OrderedDict([
@@ -80,7 +80,7 @@ def scan_files(collector):
             print("Processing file", fn, "...")
         fn = tmp_file.name if tmp_file else fn
 
-        for line in reverse_readline(fn):
+        for line in readline(fn):
             if scan_mail_log_line(line.strip(), collector) is False:
                 if stop_scan:
                     return
@@ -344,16 +344,22 @@ def scan_mail_log_line(line, collector):
 
     # Replaced the dateutil parser for a less clever way of parser that is roughly 4 times faster.
     # date = dateutil.parser.parse(date)
-    date = datetime.datetime.strptime(date, '%b %d %H:%M:%S')
-    date = date.replace(START_DATE.year)
+
+    # date = datetime.datetime.strptime(date, '%b %d %H:%M:%S')
+    # date = date.replace(START_DATE.year)
+
+    # strptime fails on Feb 29 if correct year is not provided. See https://bugs.python.org/issue26460
+    date = datetime.datetime.strptime(str(START_DATE.year) + ' ' + date, '%Y %b %d %H:%M:%S')
+    # print("date:", date)
 
     # Check if the found date is within the time span we are scanning
+    # END_DATE < START_DATE
     if date > START_DATE:
-        # Don't process, but continue
-        return True
-    elif date < END_DATE:
         # Don't process, and halt
         return False
+    elif date < END_DATE:
+        # Don't process, but continue
+        return True
 
     if service == "postfix/submission/smtpd":
         if SCAN_OUT:
@@ -453,9 +459,9 @@ def scan_postfix_smtpd_line(date, log, collector):
                     if m:
                         message = "domain blocked: " + m.group(2)
 
-                if data["latest"] is None:
-                    data["latest"] = date
-                data["earliest"] = date
+                if data["earliest"] is None:
+                    data["earliest"] = date
+                data["latest"] = date
                 data["blocked"].append((date, sender, message))
 
                 collector["rejected"][user] = data
@@ -487,9 +493,9 @@ def add_login(user, date, protocol_name, host, collector):
                 }
             )
 
-            if data["latest"] is None:
-                data["latest"] = date
-            data["earliest"] = date
+            if data["earliest"] is None:
+                data["earliest"] = date
+            data["latest"] = date
 
             data["totals_by_protocol"][protocol_name] += 1
             data["totals_by_protocol_and_host"][(protocol_name, host)] += 1
@@ -528,9 +534,9 @@ def scan_postfix_lmtp_line(date, log, collector):
             data["received_count"] += 1
             data["activity-by-hour"][date.hour] += 1
 
-            if data["latest"] is None:
-                data["latest"] = date
-            data["earliest"] = date
+            if data["earliest"] is None:
+                data["earliest"] = date
+            data["latest"] = date
 
             collector["received_mail"][user] = data
 
@@ -567,9 +573,9 @@ def scan_postfix_submission_line(date, log, collector):
             data["hosts"].add(client)
             data["activity-by-hour"][date.hour] += 1
 
-            if data["latest"] is None:
-                data["latest"] = date
-            data["earliest"] = date
+            if data["earliest"] is None:
+                data["earliest"] = date
+            data["latest"] = date
 
             collector["sent_mail"][user] = data
 
@@ -578,42 +584,15 @@ def scan_postfix_submission_line(date, log, collector):
 
 # Utility functions
 
-def reverse_readline(filename, buf_size=8192):
-    """ A generator that returns the lines of a file in reverse order
-
-    http://stackoverflow.com/a/23646049/801870
-
+def readline(filename):
+    """ A generator that returns the lines of a file
     """
-
-    with open(filename) as fh:
-        segment = None
-        offset = 0
-        fh.seek(0, os.SEEK_END)
-        file_size = remaining_size = fh.tell()
-        while remaining_size > 0:
-            offset = min(file_size, offset + buf_size)
-            fh.seek(file_size - offset)
-            buff = fh.read(min(remaining_size, buf_size))
-            remaining_size -= buf_size
-            lines = buff.split('\n')
-            # the first line of the buffer is probably not a complete line so
-            # we'll save it and append it to the last line of the next buffer
-            # we read
-            if segment is not None:
-                # if the previous chunk starts right from the beginning of line
-                # do not concat the segment to the last line of new chunk
-                # instead, yield the segment first
-                if buff[-1] is not '\n':
-                    lines[-1] += segment
-                else:
-                    yield segment
-            segment = lines[0]
-            for index in range(len(lines) - 1, 0, -1):
-                if len(lines[index]):
-                    yield lines[index]
-        # Don't yield None if the file was empty
-        if segment is not None:
-            yield segment
+    with open(filename) as file:
+        while True:
+          line = file.readline()
+          if not line:
+              break
+          yield line
 
 
 def user_match(user):

management/templates/users.html

@@ -99,7 +99,7 @@
 <thead><th>Verb</th> <th>Action</th><th></th></thead>
 <tr><td>GET</td><td><i>(none)</i></td> <td>Returns a list of existing mail users. Adding <code>?format=json</code> to the URL will give JSON-encoded results.</td></tr>
 <tr><td>POST</td><td>/add</td> <td>Adds a new mail user. Required POST-body parameters are <code>email</code> and <code>password</code>.</td></tr>
-<tr><td>POST</td><td>/remove</td> <td>Removes a mail user. Required POST-by parameter is <code>email</code>.</td></tr>
+<tr><td>POST</td><td>/remove</td> <td>Removes a mail user. Required POST-body parameter is <code>email</code>.</td></tr>
 <tr><td>POST</td><td>/privileges/add</td> <td>Used to make a mail user an admin. Required POST-body parameters are <code>email</code> and <code>privilege=admin</code>.</td></tr>
 <tr><td>POST</td><td>/privileges/remove</td> <td>Used to remove the admin privilege from a mail user. Required POST-body parameter is <code>email</code>.</td></tr>
 </table>

management/web_update.py

@@ -159,6 +159,10 @@ def make_domain_config(domain, templates, ssl_certificates, env):
 				nginx_conf_extra += "\n\t\tproxy_pass %s;" % url
 				nginx_conf_extra += "\n\t\tproxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;"
 				nginx_conf_extra += "\n\t}\n"
+			for path, alias in yaml.get("aliases", {}).items():
+				nginx_conf_extra += "\tlocation %s {" % path
+				nginx_conf_extra += "\n\t\talias %s;" % alias
+				nginx_conf_extra += "\n\t}\n"
 			for path, url in yaml.get("redirects", {}).items():
 				nginx_conf_extra += "\trewrite %s %s permanent;\n" % (path, url)
 

setup/bootstrap.sh

@@ -20,7 +20,7 @@ if [ -z "$TAG" ]; then
 	# want to display in status checks.
 	if [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/18\.04\.[0-9]/18.04/' `" == "Ubuntu 18.04 LTS" ]; then
 		# This machine is running Ubuntu 18.04.
-		TAG=v0.44
+		TAG=v0.46
 
 	elif [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/14\.04\.[0-9]/14.04/' `" == "Ubuntu 14.04 LTS" ]; then
 		# This machine is running Ubuntu 14.04.
@@ -35,14 +35,14 @@ if [ -z "$TAG" ]; then
 
 	else
 		echo "This script must be run on a system running Ubuntu 18.04 or Ubuntu 14.04."
-		exit
+		exit 1
 	fi
 fi
 
 # Are we running as root?
 if [[ $EUID -ne 0 ]]; then
 	echo "This script must be run as root. Did you leave out sudo?"
-	exit
+	exit 1
 fi
 
 # Clone the Mail-in-a-Box repository if it doesn't exist.
@@ -73,7 +73,7 @@ if [ "$TAG" != `git describe` ]; then
 	git fetch --depth 1 --force --prune origin tag $TAG
 	if ! git checkout -q $TAG; then
 		echo "Update failed. Did you modify something in `pwd`?"
-		exit
+		exit 1
 	fi
 	echo
 fi

setup/functions.sh

@@ -57,15 +57,6 @@ function apt_install {
 	apt_get_quiet install $PACKAGES
 }
 
-function apt_add_repository_to_unattended_upgrades {
-	if [ -f /etc/apt/apt.conf.d/50unattended-upgrades ]; then
-		if ! grep -q "$1" /etc/apt/apt.conf.d/50unattended-upgrades; then
-			sed -i "/Allowed-Origins/a \
-	    \"$1\";" /etc/apt/apt.conf.d/50unattended-upgrades
-		fi
-	fi
-}
-
 function get_default_hostname {
 	# Guess the machine's hostname. It should be a fully qualified
 	# domain name suitable for DNS. None of these calls may provide

setup/management.sh

@@ -101,10 +101,11 @@ hide_output systemctl enable mailinabox.service
 # Perform nightly tasks at 3am in system time: take a backup, run
 # status checks and email the administrator any changes.
 
+minute=$((RANDOM % 60))  # avoid overloading mailinabox.email
 cat > /etc/cron.d/mailinabox-nightly << EOF;
 # Mail-in-a-Box --- Do not edit / will be overwritten on update.
 # Run nightly tasks: backup, status checks.
-0 3 * * *	root	(cd `pwd` && management/daily_tasks.sh)
+$minute 3 * * *	root	(cd `pwd` && management/daily_tasks.sh)
 EOF
 
 # Start the management server.

setup/nextcloud.sh

@@ -40,11 +40,11 @@ InstallNextcloud() {
 	# their github repositories.
 	mkdir -p /usr/local/lib/owncloud/apps
 
-	wget_verify https://github.com/nextcloud/contacts/releases/download/v3.1.6/contacts.tar.gz d331dc6db2ecf7c8e6166926a055dfa3b59722c3 /tmp/contacts.tgz
+	wget_verify https://github.com/nextcloud/contacts/releases/download/v3.3.0/contacts.tar.gz e55d0357c6785d3b1f3b5f21780cb6d41d32443a /tmp/contacts.tgz
 	tar xf /tmp/contacts.tgz -C /usr/local/lib/owncloud/apps/
 	rm /tmp/contacts.tgz
 
-	wget_verify https://github.com/nextcloud/calendar/releases/download/v1.7.1/calendar.tar.gz bd7c846bad06da6d6ba04280f6fbf37ef846c2ad /tmp/calendar.tgz
+	wget_verify https://github.com/nextcloud/calendar/releases/download/v2.0.3/calendar.tar.gz 9d9717b29337613b72c74e9914c69b74b346c466 /tmp/calendar.tgz
 	tar xf /tmp/calendar.tgz -C /usr/local/lib/owncloud/apps/
 	rm /tmp/calendar.tgz
 
@@ -91,8 +91,8 @@ InstallNextcloud() {
 }
 
 # Nextcloud Version to install. Checks are done down below to step through intermediate versions.
-nextcloud_ver=17.0.2
-nextcloud_hash=8095fb46e9e0c536163708aee3d17fab8b498ad6
+nextcloud_ver=17.0.6
+nextcloud_hash=50b98d2c2f18510b9530e558ced9ab51eb4f11b0
 
 # Current Nextcloud Version, #1623
 # Checking /usr/local/lib/owncloud/version.php shows version of the Nextcloud application, not the DB

setup/preflight.sh

@@ -4,7 +4,7 @@ if [[ $EUID -ne 0 ]]; then
 	echo
 	echo "sudo $0"
 	echo
-	exit
+	exit 1
 fi
 
 # Check that we are running on Ubuntu 18.04 LTS (or 18.04.xx).
@@ -14,7 +14,7 @@ if [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/18\.04\.[0-9]/18.04/' `" != "U
 	lsb_release -d | sed 's/.*:\s*//'
 	echo
 	echo "We can't write scripts that run on every possible setup, sorry."
-	exit
+	exit 1
 fi
 
 # Check that we have enough memory.

setup/webmail.sh

@@ -28,8 +28,8 @@ apt_install \
 # Install Roundcube from source if it is not already present or if it is out of date.
 # Combine the Roundcube version number with the commit hash of plugins to track
 # whether we have the latest version of everything.
-VERSION=1.4.2
-HASH=d53fcd7f1109a63364d5d4a43f879c6f47d34a89
+VERSION=1.4.6
+HASH=44961ef62bb9c9875141ca34704bbc7d6f36373d
 PERSISTENT_LOGIN_VERSION=6b3fc450cae23ccb2f393d0ef67aa319e877e435
 HTML5_NOTIFIER_VERSION=4b370e3cd60dabd2f428a26f45b677ad1b7118d5
 CARDDAV_VERSION=3.0.3
@@ -160,7 +160,7 @@ mkdir -p /var/log/roundcubemail /var/tmp/roundcubemail $STORAGE_ROOT/mail/roundc
 chown -R www-data.www-data /var/log/roundcubemail /var/tmp/roundcubemail $STORAGE_ROOT/mail/roundcube
 
 # Ensure the log file monitored by fail2ban exists, or else fail2ban can't start.
-sudo -u www-data touch /var/log/roundcubemail/errors
+sudo -u www-data touch /var/log/roundcubemail/errors.log
 
 # Password changing plugin settings
 # The config comes empty by default, so we need the settings

setup/zpush.sh

@@ -22,8 +22,8 @@ apt_install \
 phpenmod -v php imap
 
 # Copy Z-Push into place.
-VERSION=2.5.1
-TARGETHASH=4fa55863a429b0033497ae477aca4c8699b8f332
+VERSION=2.5.2
+TARGETHASH=2dc3dbd791b96b0ba2638df0d3d1e03c7e1cbab2
 needs_update=0 #NODOC
 if [ ! -f /usr/local/lib/z-push/version ]; then
 	needs_update=1 #NODOC

tools/owncloud-restore.sh

@@ -22,7 +22,7 @@ fi
 
 if [ ! -f $1/config.php ]; then
 	echo "This isn't a valid backup location"
-	exit
+	exit 1
 fi
 
 echo "Restoring backup from $1"